From 201b58add4fb592efd0e889e0eeb2faa14c51d0a Mon Sep 17 00:00:00 2001 From: Daz DeBoer Date: Wed, 10 Nov 2021 20:11:55 -0700 Subject: [PATCH] Store all cache metadata in a single JSON file Instead of storing cache metadata in a separate .cache file per artifact bundle, all of the metadata is now stored in a single `.json` file. This will make it easier to implement more flexible artifact-caching strategies, such as caching each wrapper zip separately. --- src/cache-gradle-user-home.ts | 108 +++++++++++++++++++--------------- 1 file changed, 61 insertions(+), 47 deletions(-) diff --git a/src/cache-gradle-user-home.ts b/src/cache-gradle-user-home.ts index c5d4bcf..953c32b 100644 --- a/src/cache-gradle-user-home.ts +++ b/src/cache-gradle-user-home.ts @@ -14,6 +14,16 @@ const INCLUDE_PATHS_PARAMETER = 'gradle-home-cache-includes' const EXCLUDE_PATHS_PARAMETER = 'gradle-home-cache-excludes' const ARTIFACT_BUNDLES_PARAMETER = 'gradle-home-cache-artifact-bundles' +class CacheResult { + readonly bundle: string + readonly cacheKey: string | undefined + + constructor(bundle: string, cacheKey: string | undefined) { + this.bundle = bundle + this.cacheKey = cacheKey + } +} + export class GradleUserHomeCache extends AbstractCache { private gradleUserHome: string @@ -34,24 +44,21 @@ export class GradleUserHomeCache extends AbstractCache { } private async restoreArtifactBundles(listener: CacheListener): Promise { - const processes: Promise[] = [] - - const bundleMetaFiles = await this.getBundleMetaFiles() + const bundleMetadata = this.loadBundleMetadata() const bundlePatterns = this.getArtifactBundles() - // Iterate over all bundle meta files and try to restore - for (const bundleMetaFile of bundleMetaFiles) { - const bundle = path.basename(bundleMetaFile, '.cache') + const processes: Promise[] = [] + + for (const [bundle, cacheKey] of bundleMetadata) { const entryListener = listener.entry(bundle) const bundlePattern = bundlePatterns.get(bundle) // Handle case where the 'artifactBundlePatterns' have been changed if (bundlePattern === undefined) { - core.info(`Found bundle metafile for ${bundle} but no such bundle defined`) + core.info(`Found bundle metadata for ${bundle} but no such bundle defined`) entryListener.markRequested('BUNDLE_NOT_CONFIGURED') - tryDelete(bundleMetaFile) } else { - const p = this.restoreArtifactBundle(bundle, bundlePattern, bundleMetaFile, entryListener) + const p = this.restoreArtifactBundle(bundle, cacheKey, bundlePattern, entryListener) // Run sequentially when debugging enabled if (this.cacheDebuggingEnabled) { await p @@ -60,39 +67,30 @@ export class GradleUserHomeCache extends AbstractCache { } } - await Promise.all(processes) + const results = await Promise.all(processes) + + this.saveMetadataForCacheResults(results) } private async restoreArtifactBundle( bundle: string, + cacheKey: string, bundlePattern: string, - bundleMetaFile: string, listener: CacheEntryListener - ): Promise { - const cacheKey = fs.readFileSync(bundleMetaFile, 'utf-8').trim() + ): Promise { listener.markRequested(cacheKey) const restoredEntry = await this.restoreCache([bundlePattern], cacheKey) if (restoredEntry) { core.info(`Restored ${bundle} with key ${cacheKey} to ${bundlePattern}`) listener.markRestored(restoredEntry.key, restoredEntry.size) + return new CacheResult(bundle, cacheKey) } else { core.info(`Did not restore ${bundle} with key ${cacheKey} to ${bundlePattern}`) - tryDelete(bundleMetaFile) + return new CacheResult(bundle, undefined) } } - private getBundleMetaFile(name: string): string { - return path.resolve(this.gradleUserHome, META_FILE_DIR, `${name}.cache`) - } - - private async getBundleMetaFiles(): Promise { - const metaFiles = path.resolve(this.gradleUserHome, META_FILE_DIR, '*.cache') - const globber = await glob.create(metaFiles) - const bundleFiles = await globber.glob() - return bundleFiles - } - async beforeSave(listener: CacheListener): Promise { await this.reportGradleUserHomeSize('before saving common artifacts') this.removeExcludedPaths() @@ -113,11 +111,13 @@ export class GradleUserHomeCache extends AbstractCache { } private async saveArtifactBundles(listener: CacheListener): Promise { - const processes: Promise[] = [] + const bundleMetadata = this.loadBundleMetadata() + + const processes: Promise[] = [] for (const [bundle, pattern] of this.getArtifactBundles()) { const entryListener = listener.entry(bundle) - - const p = this.saveArtifactBundle(bundle, pattern, entryListener) + const previouslyRestoredKey = bundleMetadata.get(bundle) + const p = this.saveArtifactBundle(bundle, pattern, previouslyRestoredKey, entryListener) // Run sequentially when debugging enabled if (this.cacheDebuggingEnabled) { await p @@ -125,16 +125,17 @@ export class GradleUserHomeCache extends AbstractCache { processes.push(p) } - await Promise.all(processes) + const results = await Promise.all(processes) + + this.saveMetadataForCacheResults(results) } private async saveArtifactBundle( bundle: string, artifactPath: string, + previouslyRestoredKey: string | undefined, listener: CacheEntryListener - ): Promise { - const bundleMetaFile = this.getBundleMetaFile(bundle) - + ): Promise { const globber = await glob.create(artifactPath, { implicitDescendants: false, followSymbolicLinks: false @@ -144,15 +145,9 @@ export class GradleUserHomeCache extends AbstractCache { // Handle no matching files if (bundleFiles.length === 0) { this.debug(`No files found to cache for ${bundle}`) - if (fs.existsSync(bundleMetaFile)) { - tryDelete(bundleMetaFile) - } - return + return new CacheResult(bundle, undefined) } - const previouslyRestoredKey = fs.existsSync(bundleMetaFile) - ? fs.readFileSync(bundleMetaFile, 'utf-8').trim() - : '' const cacheKey = this.createCacheKey(bundle, bundleFiles) if (previouslyRestoredKey === cacheKey) { @@ -161,7 +156,6 @@ export class GradleUserHomeCache extends AbstractCache { core.info(`Caching ${bundle} with cache key: ${cacheKey}`) const savedEntry = await this.saveCache([artifactPath], cacheKey) if (savedEntry !== undefined) { - this.writeBundleMetaFile(bundleMetaFile, cacheKey) listener.markSaved(savedEntry.key, savedEntry.size) } } @@ -169,6 +163,8 @@ export class GradleUserHomeCache extends AbstractCache { for (const file of bundleFiles) { tryDelete(file) } + + return new CacheResult(bundle, cacheKey) } protected createCacheKey(bundle: string, files: string[]): string { @@ -181,15 +177,33 @@ export class GradleUserHomeCache extends AbstractCache { return `${cacheKeyPrefix}${bundle}-${key}` } - private writeBundleMetaFile(metaFile: string, cacheKey: string): void { - this.debug(`Writing bundle metafile: ${metaFile}`) - - const dirName = path.dirname(metaFile) - if (!fs.existsSync(dirName)) { - fs.mkdirSync(dirName) + private loadBundleMetadata(): Map { + const bundleMetaFile = path.resolve(this.gradleUserHome, META_FILE_DIR, 'bundles.json') + if (!fs.existsSync(bundleMetaFile)) { + return new Map() } + const filedata = fs.readFileSync(bundleMetaFile, 'utf-8') + core.debug(`Loaded bundle metadata: ${filedata}`) + return new Map(JSON.parse(filedata)) + } - fs.writeFileSync(metaFile, cacheKey) + private saveMetadataForCacheResults(results: CacheResult[]): void { + const metadata = new Map() + for (const result of results) { + if (result.cacheKey !== undefined) { + metadata.set(result.bundle, result.cacheKey) + } + } + const filedata = JSON.stringify(Array.from(metadata)) + core.debug(`Saving bundle metadata: ${filedata}`) + + const bundleMetaDir = path.resolve(this.gradleUserHome, META_FILE_DIR) + const bundleMetaFile = path.resolve(bundleMetaDir, 'bundles.json') + + if (!fs.existsSync(bundleMetaDir)) { + fs.mkdirSync(bundleMetaDir, {recursive: true}) + } + fs.writeFileSync(bundleMetaFile, filedata, 'utf-8') } protected determineGradleUserHome(rootDir: string): string {