From 37f2880a8a39b1d327343b74a4fa4857fe839d13 Mon Sep 17 00:00:00 2001 From: Daz DeBoer Date: Thu, 2 Dec 2021 14:38:23 -0700 Subject: [PATCH] Cache wrapper zips and generated jars individually Using a single cache entry for all files of a type is necessary to avoid overloading the cache service. However, this mechanism is not very efficient for certain artifacts like wrapper zips and generated-gradle-jars, where the same individual files are often shared between different jobs. With this change, any configured file patterns that do not end in '*' will be cached as individual files. At this time this includes downloaded wrapper zips and generated-gradle-jars. Fixes #78 --- .../workflows/integTest-caching-config.yml | 6 +- .../integTest-caching-configuration-cache.yml | 8 +- .../integTest-caching-gradle-home.yml | 8 +- action.yml | 4 +- src/cache-gradle-user-home.ts | 283 ++++++++++-------- 5 files changed, 176 insertions(+), 133 deletions(-) diff --git a/.github/workflows/integTest-caching-config.yml b/.github/workflows/integTest-caching-config.yml index dfec98b..f5294b5 100644 --- a/.github/workflows/integTest-caching-config.yml +++ b/.github/workflows/integTest-caching-config.yml @@ -25,7 +25,7 @@ jobs: with: build-root-directory: __tests__/samples/groovy-dsl arguments: test - # Add "wrapper" to main cache entry and remove 'wrapper-zips' bundle + # Add "wrapper" to main cache entry and remove 'wrapper-zips' cache entry # Exclude build-cache from main cache entry gradle-home-cache-includes: | caches @@ -33,7 +33,7 @@ jobs: wrapper gradle-home-cache-excludes: | caches/build-cache-1 - gradle-home-cache-artifact-bundles: | + gradle-home-extracted-cache-entries: | [ ["generated-gradle-jars", "caches/*/generated-gradle-jars/*.jar"], ["dependencies", "caches/modules-*/files-*/*/*/*/*/"], @@ -64,7 +64,7 @@ jobs: wrapper gradle-home-cache-excludes: | caches/build-cache-1 - gradle-home-cache-artifact-bundles: | + gradle-home-extracted-cache-entries: | [ ["generated-gradle-jars", "caches/*/generated-gradle-jars/*.jar"], ["dependencies", "caches/modules-*/files-*/*/*/*/*/"], diff --git a/.github/workflows/integTest-caching-configuration-cache.yml b/.github/workflows/integTest-caching-configuration-cache.yml index 0f3607c..7ba7f27 100644 --- a/.github/workflows/integTest-caching-configuration-cache.yml +++ b/.github/workflows/integTest-caching-configuration-cache.yml @@ -45,8 +45,8 @@ jobs: arguments: test --configuration-cache cache-read-only: true - # Check that the build can run when no bundles are restored - no-bundles-restored: + # Check that the build can run when no extracted cache entries are restored + no-extracted-cache-entries-restored: needs: seed-build strategy: matrix: @@ -55,11 +55,11 @@ jobs: steps: - name: Checkout sources uses: actions/checkout@v2 - - name: Execute Gradle build with no cache artifact bundles restored + - name: Execute Gradle build with no cache extracted cache entries restored uses: ./ with: build-root-directory: __tests__/samples/groovy-dsl arguments: test --configuration-cache cache-read-only: true - gradle-home-cache-artifact-bundles: '[]' + gradle-home-extracted-cache-entries: '[]' diff --git a/.github/workflows/integTest-caching-gradle-home.yml b/.github/workflows/integTest-caching-gradle-home.yml index 2a66a1a..1ee5c67 100644 --- a/.github/workflows/integTest-caching-gradle-home.yml +++ b/.github/workflows/integTest-caching-gradle-home.yml @@ -59,8 +59,8 @@ jobs: arguments: test -DverifyCachedBuild=true cache-read-only: true - # Check that the build can run when no bundles are restored - no-bundles-restored: + # Check that the build can run when Gradle User Home is not fully restored + no-extracted-cache-entries-restored: needs: seed-build strategy: matrix: @@ -69,11 +69,11 @@ jobs: steps: - name: Checkout sources uses: actions/checkout@v2 - - name: Execute Gradle build with no cache artifact bundles restored + - name: Execute Gradle build with no extracted cache entries restored uses: ./ with: build-root-directory: __tests__/samples/groovy-dsl arguments: test cache-read-only: true - gradle-home-cache-artifact-bundles: '[]' + gradle-home-extracted-cache-entries: '[]' diff --git a/action.yml b/action.yml index 1b577a6..5a1ad52 100644 --- a/action.yml +++ b/action.yml @@ -55,8 +55,8 @@ inputs: description: Used to uniquely identify the current job invocation. Defaults to the matrix values for this job; this should not be overridden by users (INTERNAL). required: false default: ${{ toJSON(matrix) }} - gradle-home-cache-artifact-bundles: - description: Names and patterns of artifact bundles to cache separately. (EXPERIMENTAL - may be changed/removed without notice) + gradle-home-extracted-cache-entries: + description: Names and patterns of artifacts in Gradle User Home to cache separately. (EXPERIMENTAL - may be changed/removed without notice) required: false default: | [ diff --git a/src/cache-gradle-user-home.ts b/src/cache-gradle-user-home.ts index f6fb0a2..dbff622 100644 --- a/src/cache-gradle-user-home.ts +++ b/src/cache-gradle-user-home.ts @@ -13,25 +13,36 @@ const META_FILE = 'cache-metadata.json' const INCLUDE_PATHS_PARAMETER = 'gradle-home-cache-includes' const EXCLUDE_PATHS_PARAMETER = 'gradle-home-cache-excludes' -const ARTIFACT_BUNDLES_PARAMETER = 'gradle-home-cache-artifact-bundles' +const EXTRACTED_CACHE_ENTRIES_PARAMETER = 'gradle-home-extracted-cache-entries' /** - * Represents the result of attempting to load or store a cache bundle entry. + * Represents the result of attempting to load or store an extracted cache entry. * An undefined cacheKey indicates that the operation did not succeed. * The collected results are then used to populate the `cache-metadata.json` file for later use. */ -class CacheBundleResult { - readonly bundle: string - readonly cacheKey: string | undefined +class ExtractedCacheEntry { + artifactType: string + pattern: string + cacheKey: string | undefined - constructor(bundle: string, cacheKey: string | undefined) { - this.bundle = bundle + constructor(artifactType: string, pattern: string, cacheKey: string | undefined) { + this.artifactType = artifactType + this.pattern = pattern this.cacheKey = cacheKey } } /** - * Caches and restores the entire Gradle User Home directory, extracting bundles of common artifacts + * Representation of all of the extracted cache entries for this Gradle User Home. + * This object is persisted to JSON file in the Gradle User Home directory for storing, + * and subsequently used to restore the Gradle User Home. + */ +class ExtractedCacheEntryMetadata { + entries: ExtractedCacheEntry[] = [] +} + +/** + * Caches and restores the entire Gradle User Home directory, extracting entries containing common artifacts * for more efficient storage. */ export class GradleUserHomeCache extends AbstractCache { @@ -48,73 +59,73 @@ export class GradleUserHomeCache extends AbstractCache { } /** - * Restore any artifact bundles after the main Gradle User Home entry is restored. + * Restore any extracted cache entries after the main Gradle User Home entry is restored. */ async afterRestore(listener: CacheListener): Promise { await this.debugReportGradleUserHomeSize('as restored from cache') - await this.restoreArtifactBundles(listener) + await this.restoreExtractedCacheEntries(listener) await this.debugReportGradleUserHomeSize('after restoring common artifacts') } /** * Restores any artifacts that were cached separately, based on the information in the `cache-metadata.json` file. - * Each artifact bundle is restored in parallel, except when debugging is enabled. + * Each extracted cache entry is restored in parallel, except when debugging is enabled. */ - private async restoreArtifactBundles(listener: CacheListener): Promise { - const bundleMetadata = this.loadBundleMetadata() - const bundlePatterns = this.getArtifactBundleDefinitions() + private async restoreExtractedCacheEntries(listener: CacheListener): Promise { + const extractedCacheEntryDefinitions = this.getExtractedCacheEntryDefinitions() + const previouslyExtractedCacheEntries = this.loadExtractedCacheEntries() - const processes: Promise[] = [] + const processes: Promise[] = [] - for (const [bundle, cacheKey] of bundleMetadata) { - const entryListener = listener.entry(bundle) - const bundlePattern = bundlePatterns.get(bundle) + for (const cacheEntry of previouslyExtractedCacheEntries) { + const artifactType = cacheEntry.artifactType + const entryListener = listener.entry(cacheEntry.pattern) - // Handle case where the 'artifactBundlePatterns' have been changed - if (bundlePattern === undefined) { - core.info(`Found bundle metadata for ${bundle} but no such bundle defined`) - entryListener.markRequested('BUNDLE_NOT_CONFIGURED') + // Handle case where the extracted-cache-entry definitions have been changed + if (extractedCacheEntryDefinitions.get(artifactType) === undefined) { + core.info(`Found extracted cache entry for ${artifactType} but no such entry defined`) + entryListener.markRequested('EXTRACTED_ENTRY_NOT_DEFINED') } else { - const p = this.restoreArtifactBundle(bundle, cacheKey, bundlePattern, entryListener) - // Run sequentially when debugging enabled - if (this.cacheDebuggingEnabled) { - await p - } - processes.push(p) + processes.push( + this.restoreExtractedCacheEntry( + artifactType, + cacheEntry.cacheKey!, + cacheEntry.pattern, + entryListener + ) + ) } } - const results = await Promise.all(processes) - - this.saveMetadataForCacheResults(results) + this.saveMetadataForCacheResults(await this.collectCacheResults(processes)) } - private async restoreArtifactBundle( - bundle: string, + private async restoreExtractedCacheEntry( + artifactType: string, cacheKey: string, - bundlePattern: string, + pattern: string, listener: CacheEntryListener - ): Promise { + ): Promise { listener.markRequested(cacheKey) - const restoredEntry = await this.restoreCache([bundlePattern], cacheKey) + const restoredEntry = await this.restoreCache([pattern], cacheKey) if (restoredEntry) { - core.info(`Restored ${bundle} with key ${cacheKey} to ${bundlePattern}`) + core.info(`Restored ${artifactType} with key ${cacheKey} to ${pattern}`) listener.markRestored(restoredEntry.key, restoredEntry.size) - return new CacheBundleResult(bundle, cacheKey) + return new ExtractedCacheEntry(artifactType, pattern, cacheKey) } else { - core.info(`Did not restore ${bundle} with key ${cacheKey} to ${bundlePattern}`) - return new CacheBundleResult(bundle, undefined) + core.info(`Did not restore ${artifactType} with key ${cacheKey} to ${pattern}`) + return new ExtractedCacheEntry(artifactType, pattern, undefined) } } /** - * Save and delete any artifact bundles prior to the main Gradle User Home entry being saved. + * Extract and save any defined extracted cache entries prior to the main Gradle User Home entry being saved. */ async beforeSave(listener: CacheListener): Promise { await this.debugReportGradleUserHomeSize('before saving common artifacts') this.removeExcludedPaths() - await this.saveArtifactBundles(listener) + await this.saveExtractedCacheEntries(listener) await this.debugReportGradleUserHomeSize( "after saving common artifacts (only 'caches' and 'notifications' will be stored)" ) @@ -134,110 +145,143 @@ export class GradleUserHomeCache extends AbstractCache { } /** - * Saves any artifacts that are configured to be cached separately, based on the artifact bundle definitions. - * These definitions are normally fixed, but can be overridden by the `gradle-home-cache-artifact-bundles` parameter. - * Each artifact bundle is saved in parallel, except when debugging is enabled. + * Saves any artifacts that are configured to be cached separately, based on the extracted cache entry definitions. + * These definitions are normally fixed, but can be overridden by the `gradle-home-extracted-cache-entries` parameter. + * Each entry is extracted and saved in parallel, except when debugging is enabled. */ - private async saveArtifactBundles(listener: CacheListener): Promise { - const bundleMetadata = this.loadBundleMetadata() + private async saveExtractedCacheEntries(listener: CacheListener): Promise { + // Load the cache entry definitions (from config) and the previously restored entries (from filesystem) + const cacheEntryDefinitions = this.getExtractedCacheEntryDefinitions() + const previouslyRestoredEntries = this.loadExtractedCacheEntries() + const cacheActions: Promise[] = [] - const processes: Promise[] = [] - for (const [bundle, pattern] of this.getArtifactBundleDefinitions()) { - const entryListener = listener.entry(bundle) - const previouslyRestoredKey = bundleMetadata.get(bundle) - const p = this.saveArtifactBundle(bundle, pattern, previouslyRestoredKey, entryListener) - // Run sequentially when debugging enabled - if (this.cacheDebuggingEnabled) { - await p + for (const [artifactType, pattern] of cacheEntryDefinitions) { + // Find all matching files for this cache entry definition + const globber = await glob.create(pattern, { + implicitDescendants: false, + followSymbolicLinks: false + }) + const matchingFiles = await globber.glob() + + if (matchingFiles.length === 0) { + this.debug(`No files found to cache for ${artifactType}`) + continue + } + + if (this.isBundlePattern(pattern)) { + // For an extracted "bundle", use the defined pattern and cache all matching files in a single entry. + cacheActions.push( + this.saveExtractedCacheEntry( + matchingFiles, + artifactType, + pattern, + previouslyRestoredEntries, + listener.entry(pattern) + ) + ) + } else { + // Otherwise cache each matching file in a separate entry, using the complete file path as the cache pattern. + for (const cacheFile of matchingFiles) { + cacheActions.push( + this.saveExtractedCacheEntry( + [cacheFile], + artifactType, + cacheFile, + previouslyRestoredEntries, + listener.entry(cacheFile) + ) + ) + } } - processes.push(p) } - const results = await Promise.all(processes) - - this.saveMetadataForCacheResults(results) + this.saveMetadataForCacheResults(await this.collectCacheResults(cacheActions)) } - private async saveArtifactBundle( - bundle: string, - artifactPath: string, - previouslyRestoredKey: string | undefined, - listener: CacheEntryListener - ): Promise { - const globber = await glob.create(artifactPath, { - implicitDescendants: false, - followSymbolicLinks: false - }) - const bundleFiles = await globber.glob() - - // Handle no matching files - if (bundleFiles.length === 0) { - this.debug(`No files found to cache for ${bundle}`) - return new CacheBundleResult(bundle, undefined) - } - - const cacheKey = this.createCacheKeyForArtifacts(bundle, bundleFiles) + private async saveExtractedCacheEntry( + matchingFiles: string[], + artifactType: string, + pattern: string, + previouslyRestoredEntries: ExtractedCacheEntry[], + entryListener: CacheEntryListener + ): Promise { + const cacheKey = this.createCacheKeyForArtifacts(artifactType, matchingFiles) + const previouslyRestoredKey = previouslyRestoredEntries.find( + x => x.artifactType === artifactType && x.pattern === pattern + )?.cacheKey if (previouslyRestoredKey === cacheKey) { - this.debug(`No change to previously restored ${bundle}. Not caching.`) + this.debug(`No change to previously restored ${artifactType}. Not saving.`) } else { - core.info(`Caching ${bundle} with cache key: ${cacheKey}`) - const savedEntry = await this.saveCache([artifactPath], cacheKey) + core.info(`Caching ${artifactType} with path '${pattern}' and cache key: ${cacheKey}`) + const savedEntry = await this.saveCache([pattern], cacheKey) if (savedEntry !== undefined) { - listener.markSaved(savedEntry.key, savedEntry.size) + entryListener.markSaved(savedEntry.key, savedEntry.size) } } - for (const file of bundleFiles) { + for (const file of matchingFiles) { tryDelete(file) } - return new CacheBundleResult(bundle, cacheKey) + return new ExtractedCacheEntry(artifactType, pattern, cacheKey) } - protected createCacheKeyForArtifacts(bundle: string, files: string[]): string { + protected createCacheKeyForArtifacts(artifactType: string, files: string[]): string { const cacheKeyPrefix = getCacheKeyPrefix() const relativeFiles = files.map(x => path.relative(this.gradleUserHome, x)) const key = hashFileNames(relativeFiles) - this.debug(`Generating cache key for ${bundle} from files: ${relativeFiles}`) + this.debug(`Generating cache key for ${artifactType} from files: ${relativeFiles}`) - return `${cacheKeyPrefix}${bundle}-${key}` + return `${cacheKeyPrefix}${artifactType}-${key}` } - /** - * Load information about the previously restored/saved artifact bundles from the 'cache-metadata.json' file. - */ - private loadBundleMetadata(): Map { - const bundleMetaFile = path.resolve(this.gradleUserHome, META_FILE_DIR, META_FILE) - if (!fs.existsSync(bundleMetaFile)) { - return new Map() - } - const filedata = fs.readFileSync(bundleMetaFile, 'utf-8') - core.debug(`Loaded bundle metadata: ${filedata}`) - return new Map(JSON.parse(filedata)) + private isBundlePattern(pattern: string): boolean { + return pattern.endsWith('*') } - /** - * Saves information about the artifact bundle restore/save into the 'cache-metadata.json' file. - */ - private saveMetadataForCacheResults(results: CacheBundleResult[]): void { - const metadata = new Map() - for (const result of results) { - if (result.cacheKey !== undefined) { - metadata.set(result.bundle, result.cacheKey) + private async collectCacheResults(processes: Promise[]): Promise { + // Run cache actions sequentially when debugging enabled + if (this.cacheDebuggingEnabled) { + for (const p of processes) { + await p } } - const filedata = JSON.stringify(Array.from(metadata)) - core.debug(`Saving bundle metadata: ${filedata}`) - const bundleMetaDir = path.resolve(this.gradleUserHome, META_FILE_DIR) - const bundleMetaFile = path.resolve(bundleMetaDir, META_FILE) + return await Promise.all(processes) + } - if (!fs.existsSync(bundleMetaDir)) { - fs.mkdirSync(bundleMetaDir, {recursive: true}) + /** + * Load information about the extracted cache entries previously restored/saved. This is loaded from the 'cache-metadata.json' file. + */ + private loadExtractedCacheEntries(): ExtractedCacheEntry[] { + const cacheMetadataFile = path.resolve(this.gradleUserHome, META_FILE_DIR, META_FILE) + if (!fs.existsSync(cacheMetadataFile)) { + return [] } - fs.writeFileSync(bundleMetaFile, filedata, 'utf-8') + + const filedata = fs.readFileSync(cacheMetadataFile, 'utf-8') + core.debug(`Loaded cache metadata: ${filedata}`) + const extractedCacheEntryMetadata = JSON.parse(filedata) as ExtractedCacheEntryMetadata + return extractedCacheEntryMetadata.entries + } + + /** + * Saves information about the extracted cache entries into the 'cache-metadata.json' file. + */ + private saveMetadataForCacheResults(results: ExtractedCacheEntry[]): void { + const extractedCacheEntryMetadata = new ExtractedCacheEntryMetadata() + extractedCacheEntryMetadata.entries = results.filter(x => x.cacheKey !== undefined) + + const filedata = JSON.stringify(extractedCacheEntryMetadata) + core.debug(`Saving cache metadata: ${filedata}`) + + const actionMetadataDirectory = path.resolve(this.gradleUserHome, META_FILE_DIR) + const cacheMetadataFile = path.resolve(actionMetadataDirectory, META_FILE) + + fs.mkdirSync(actionMetadataDirectory, {recursive: true}) + fs.writeFileSync(cacheMetadataFile, filedata, 'utf-8') } protected determineGradleUserHome(rootDir: string): string { @@ -277,15 +321,14 @@ export class GradleUserHomeCache extends AbstractCache { } /** - * Return the artifact bundle definitions, which determine which artifacts will be cached + * Return the extracted cache entry definitions, which determine which artifacts will be cached * separately from the rest of the Gradle User Home cache entry. - * This is normally a fixed set, but can be overridden by the `gradle-home-cache-artifact-bundles` parameter. + * This is normally a fixed set, but can be overridden by the `gradle-home-extracted-cache-entries` parameter. */ - private getArtifactBundleDefinitions(): Map { - const artifactBundleDefinition = core.getInput(ARTIFACT_BUNDLES_PARAMETER) - this.debug(`Using artifact bundle definition: ${artifactBundleDefinition}`) - const artifactBundles = JSON.parse(artifactBundleDefinition) - return new Map(Array.from(artifactBundles, ([key, value]) => [key, path.resolve(this.gradleUserHome, value)])) + private getExtractedCacheEntryDefinitions(): Map { + const rawDefinitions = core.getInput(EXTRACTED_CACHE_ENTRIES_PARAMETER) + const parsedDefinitions = JSON.parse(rawDefinitions) + return new Map(Array.from(parsedDefinitions, ([key, value]) => [key, path.resolve(this.gradleUserHome, value)])) } /**