Cache wrapper zips and generated jars individually

Using a single cache entry for all files of a type is necessary to avoid
overloading the cache service. However, this mechanism is not very efficient
for certain artifacts like wrapper zips and generated-gradle-jars, where the
same individual files are often shared between different jobs.

With this change, any configured file patterns that do not end in '*' will
be cached as individual files. At this time this includes downloaded wrapper
zips and generated-gradle-jars.

Fixes #78
This commit is contained in:
Daz DeBoer 2021-12-02 14:38:23 -07:00
parent b88c4086b9
commit 37f2880a8a
5 changed files with 176 additions and 133 deletions

View file

@ -25,7 +25,7 @@ jobs:
with:
build-root-directory: __tests__/samples/groovy-dsl
arguments: test
# Add "wrapper" to main cache entry and remove 'wrapper-zips' bundle
# Add "wrapper" to main cache entry and remove 'wrapper-zips' cache entry
# Exclude build-cache from main cache entry
gradle-home-cache-includes: |
caches
@ -33,7 +33,7 @@ jobs:
wrapper
gradle-home-cache-excludes: |
caches/build-cache-1
gradle-home-cache-artifact-bundles: |
gradle-home-extracted-cache-entries: |
[
["generated-gradle-jars", "caches/*/generated-gradle-jars/*.jar"],
["dependencies", "caches/modules-*/files-*/*/*/*/*/"],
@ -64,7 +64,7 @@ jobs:
wrapper
gradle-home-cache-excludes: |
caches/build-cache-1
gradle-home-cache-artifact-bundles: |
gradle-home-extracted-cache-entries: |
[
["generated-gradle-jars", "caches/*/generated-gradle-jars/*.jar"],
["dependencies", "caches/modules-*/files-*/*/*/*/*/"],

View file

@ -45,8 +45,8 @@ jobs:
arguments: test --configuration-cache
cache-read-only: true
# Check that the build can run when no bundles are restored
no-bundles-restored:
# Check that the build can run when no extracted cache entries are restored
no-extracted-cache-entries-restored:
needs: seed-build
strategy:
matrix:
@ -55,11 +55,11 @@ jobs:
steps:
- name: Checkout sources
uses: actions/checkout@v2
- name: Execute Gradle build with no cache artifact bundles restored
- name: Execute Gradle build with no cache extracted cache entries restored
uses: ./
with:
build-root-directory: __tests__/samples/groovy-dsl
arguments: test --configuration-cache
cache-read-only: true
gradle-home-cache-artifact-bundles: '[]'
gradle-home-extracted-cache-entries: '[]'

View file

@ -59,8 +59,8 @@ jobs:
arguments: test -DverifyCachedBuild=true
cache-read-only: true
# Check that the build can run when no bundles are restored
no-bundles-restored:
# Check that the build can run when Gradle User Home is not fully restored
no-extracted-cache-entries-restored:
needs: seed-build
strategy:
matrix:
@ -69,11 +69,11 @@ jobs:
steps:
- name: Checkout sources
uses: actions/checkout@v2
- name: Execute Gradle build with no cache artifact bundles restored
- name: Execute Gradle build with no extracted cache entries restored
uses: ./
with:
build-root-directory: __tests__/samples/groovy-dsl
arguments: test
cache-read-only: true
gradle-home-cache-artifact-bundles: '[]'
gradle-home-extracted-cache-entries: '[]'

View file

@ -55,8 +55,8 @@ inputs:
description: Used to uniquely identify the current job invocation. Defaults to the matrix values for this job; this should not be overridden by users (INTERNAL).
required: false
default: ${{ toJSON(matrix) }}
gradle-home-cache-artifact-bundles:
description: Names and patterns of artifact bundles to cache separately. (EXPERIMENTAL - may be changed/removed without notice)
gradle-home-extracted-cache-entries:
description: Names and patterns of artifacts in Gradle User Home to cache separately. (EXPERIMENTAL - may be changed/removed without notice)
required: false
default: |
[

View file

@ -13,25 +13,36 @@ const META_FILE = 'cache-metadata.json'
const INCLUDE_PATHS_PARAMETER = 'gradle-home-cache-includes'
const EXCLUDE_PATHS_PARAMETER = 'gradle-home-cache-excludes'
const ARTIFACT_BUNDLES_PARAMETER = 'gradle-home-cache-artifact-bundles'
const EXTRACTED_CACHE_ENTRIES_PARAMETER = 'gradle-home-extracted-cache-entries'
/**
* Represents the result of attempting to load or store a cache bundle entry.
* Represents the result of attempting to load or store an extracted cache entry.
* An undefined cacheKey indicates that the operation did not succeed.
* The collected results are then used to populate the `cache-metadata.json` file for later use.
*/
class CacheBundleResult {
readonly bundle: string
readonly cacheKey: string | undefined
class ExtractedCacheEntry {
artifactType: string
pattern: string
cacheKey: string | undefined
constructor(bundle: string, cacheKey: string | undefined) {
this.bundle = bundle
constructor(artifactType: string, pattern: string, cacheKey: string | undefined) {
this.artifactType = artifactType
this.pattern = pattern
this.cacheKey = cacheKey
}
}
/**
* Caches and restores the entire Gradle User Home directory, extracting bundles of common artifacts
* Representation of all of the extracted cache entries for this Gradle User Home.
* This object is persisted to JSON file in the Gradle User Home directory for storing,
* and subsequently used to restore the Gradle User Home.
*/
class ExtractedCacheEntryMetadata {
entries: ExtractedCacheEntry[] = []
}
/**
* Caches and restores the entire Gradle User Home directory, extracting entries containing common artifacts
* for more efficient storage.
*/
export class GradleUserHomeCache extends AbstractCache {
@ -48,73 +59,73 @@ export class GradleUserHomeCache extends AbstractCache {
}
/**
* Restore any artifact bundles after the main Gradle User Home entry is restored.
* Restore any extracted cache entries after the main Gradle User Home entry is restored.
*/
async afterRestore(listener: CacheListener): Promise<void> {
await this.debugReportGradleUserHomeSize('as restored from cache')
await this.restoreArtifactBundles(listener)
await this.restoreExtractedCacheEntries(listener)
await this.debugReportGradleUserHomeSize('after restoring common artifacts')
}
/**
* Restores any artifacts that were cached separately, based on the information in the `cache-metadata.json` file.
* Each artifact bundle is restored in parallel, except when debugging is enabled.
* Each extracted cache entry is restored in parallel, except when debugging is enabled.
*/
private async restoreArtifactBundles(listener: CacheListener): Promise<void> {
const bundleMetadata = this.loadBundleMetadata()
const bundlePatterns = this.getArtifactBundleDefinitions()
private async restoreExtractedCacheEntries(listener: CacheListener): Promise<void> {
const extractedCacheEntryDefinitions = this.getExtractedCacheEntryDefinitions()
const previouslyExtractedCacheEntries = this.loadExtractedCacheEntries()
const processes: Promise<CacheBundleResult>[] = []
const processes: Promise<ExtractedCacheEntry>[] = []
for (const [bundle, cacheKey] of bundleMetadata) {
const entryListener = listener.entry(bundle)
const bundlePattern = bundlePatterns.get(bundle)
for (const cacheEntry of previouslyExtractedCacheEntries) {
const artifactType = cacheEntry.artifactType
const entryListener = listener.entry(cacheEntry.pattern)
// Handle case where the 'artifactBundlePatterns' have been changed
if (bundlePattern === undefined) {
core.info(`Found bundle metadata for ${bundle} but no such bundle defined`)
entryListener.markRequested('BUNDLE_NOT_CONFIGURED')
// Handle case where the extracted-cache-entry definitions have been changed
if (extractedCacheEntryDefinitions.get(artifactType) === undefined) {
core.info(`Found extracted cache entry for ${artifactType} but no such entry defined`)
entryListener.markRequested('EXTRACTED_ENTRY_NOT_DEFINED')
} else {
const p = this.restoreArtifactBundle(bundle, cacheKey, bundlePattern, entryListener)
// Run sequentially when debugging enabled
if (this.cacheDebuggingEnabled) {
await p
}
processes.push(p)
processes.push(
this.restoreExtractedCacheEntry(
artifactType,
cacheEntry.cacheKey!,
cacheEntry.pattern,
entryListener
)
)
}
}
const results = await Promise.all(processes)
this.saveMetadataForCacheResults(results)
this.saveMetadataForCacheResults(await this.collectCacheResults(processes))
}
private async restoreArtifactBundle(
bundle: string,
private async restoreExtractedCacheEntry(
artifactType: string,
cacheKey: string,
bundlePattern: string,
pattern: string,
listener: CacheEntryListener
): Promise<CacheBundleResult> {
): Promise<ExtractedCacheEntry> {
listener.markRequested(cacheKey)
const restoredEntry = await this.restoreCache([bundlePattern], cacheKey)
const restoredEntry = await this.restoreCache([pattern], cacheKey)
if (restoredEntry) {
core.info(`Restored ${bundle} with key ${cacheKey} to ${bundlePattern}`)
core.info(`Restored ${artifactType} with key ${cacheKey} to ${pattern}`)
listener.markRestored(restoredEntry.key, restoredEntry.size)
return new CacheBundleResult(bundle, cacheKey)
return new ExtractedCacheEntry(artifactType, pattern, cacheKey)
} else {
core.info(`Did not restore ${bundle} with key ${cacheKey} to ${bundlePattern}`)
return new CacheBundleResult(bundle, undefined)
core.info(`Did not restore ${artifactType} with key ${cacheKey} to ${pattern}`)
return new ExtractedCacheEntry(artifactType, pattern, undefined)
}
}
/**
* Save and delete any artifact bundles prior to the main Gradle User Home entry being saved.
* Extract and save any defined extracted cache entries prior to the main Gradle User Home entry being saved.
*/
async beforeSave(listener: CacheListener): Promise<void> {
await this.debugReportGradleUserHomeSize('before saving common artifacts')
this.removeExcludedPaths()
await this.saveArtifactBundles(listener)
await this.saveExtractedCacheEntries(listener)
await this.debugReportGradleUserHomeSize(
"after saving common artifacts (only 'caches' and 'notifications' will be stored)"
)
@ -134,110 +145,143 @@ export class GradleUserHomeCache extends AbstractCache {
}
/**
* Saves any artifacts that are configured to be cached separately, based on the artifact bundle definitions.
* These definitions are normally fixed, but can be overridden by the `gradle-home-cache-artifact-bundles` parameter.
* Each artifact bundle is saved in parallel, except when debugging is enabled.
* Saves any artifacts that are configured to be cached separately, based on the extracted cache entry definitions.
* These definitions are normally fixed, but can be overridden by the `gradle-home-extracted-cache-entries` parameter.
* Each entry is extracted and saved in parallel, except when debugging is enabled.
*/
private async saveArtifactBundles(listener: CacheListener): Promise<void> {
const bundleMetadata = this.loadBundleMetadata()
private async saveExtractedCacheEntries(listener: CacheListener): Promise<void> {
// Load the cache entry definitions (from config) and the previously restored entries (from filesystem)
const cacheEntryDefinitions = this.getExtractedCacheEntryDefinitions()
const previouslyRestoredEntries = this.loadExtractedCacheEntries()
const cacheActions: Promise<ExtractedCacheEntry>[] = []
const processes: Promise<CacheBundleResult>[] = []
for (const [bundle, pattern] of this.getArtifactBundleDefinitions()) {
const entryListener = listener.entry(bundle)
const previouslyRestoredKey = bundleMetadata.get(bundle)
const p = this.saveArtifactBundle(bundle, pattern, previouslyRestoredKey, entryListener)
// Run sequentially when debugging enabled
if (this.cacheDebuggingEnabled) {
await p
for (const [artifactType, pattern] of cacheEntryDefinitions) {
// Find all matching files for this cache entry definition
const globber = await glob.create(pattern, {
implicitDescendants: false,
followSymbolicLinks: false
})
const matchingFiles = await globber.glob()
if (matchingFiles.length === 0) {
this.debug(`No files found to cache for ${artifactType}`)
continue
}
if (this.isBundlePattern(pattern)) {
// For an extracted "bundle", use the defined pattern and cache all matching files in a single entry.
cacheActions.push(
this.saveExtractedCacheEntry(
matchingFiles,
artifactType,
pattern,
previouslyRestoredEntries,
listener.entry(pattern)
)
)
} else {
// Otherwise cache each matching file in a separate entry, using the complete file path as the cache pattern.
for (const cacheFile of matchingFiles) {
cacheActions.push(
this.saveExtractedCacheEntry(
[cacheFile],
artifactType,
cacheFile,
previouslyRestoredEntries,
listener.entry(cacheFile)
)
)
}
}
processes.push(p)
}
const results = await Promise.all(processes)
this.saveMetadataForCacheResults(results)
this.saveMetadataForCacheResults(await this.collectCacheResults(cacheActions))
}
private async saveArtifactBundle(
bundle: string,
artifactPath: string,
previouslyRestoredKey: string | undefined,
listener: CacheEntryListener
): Promise<CacheBundleResult> {
const globber = await glob.create(artifactPath, {
implicitDescendants: false,
followSymbolicLinks: false
})
const bundleFiles = await globber.glob()
// Handle no matching files
if (bundleFiles.length === 0) {
this.debug(`No files found to cache for ${bundle}`)
return new CacheBundleResult(bundle, undefined)
}
const cacheKey = this.createCacheKeyForArtifacts(bundle, bundleFiles)
private async saveExtractedCacheEntry(
matchingFiles: string[],
artifactType: string,
pattern: string,
previouslyRestoredEntries: ExtractedCacheEntry[],
entryListener: CacheEntryListener
): Promise<ExtractedCacheEntry> {
const cacheKey = this.createCacheKeyForArtifacts(artifactType, matchingFiles)
const previouslyRestoredKey = previouslyRestoredEntries.find(
x => x.artifactType === artifactType && x.pattern === pattern
)?.cacheKey
if (previouslyRestoredKey === cacheKey) {
this.debug(`No change to previously restored ${bundle}. Not caching.`)
this.debug(`No change to previously restored ${artifactType}. Not saving.`)
} else {
core.info(`Caching ${bundle} with cache key: ${cacheKey}`)
const savedEntry = await this.saveCache([artifactPath], cacheKey)
core.info(`Caching ${artifactType} with path '${pattern}' and cache key: ${cacheKey}`)
const savedEntry = await this.saveCache([pattern], cacheKey)
if (savedEntry !== undefined) {
listener.markSaved(savedEntry.key, savedEntry.size)
entryListener.markSaved(savedEntry.key, savedEntry.size)
}
}
for (const file of bundleFiles) {
for (const file of matchingFiles) {
tryDelete(file)
}
return new CacheBundleResult(bundle, cacheKey)
return new ExtractedCacheEntry(artifactType, pattern, cacheKey)
}
protected createCacheKeyForArtifacts(bundle: string, files: string[]): string {
protected createCacheKeyForArtifacts(artifactType: string, files: string[]): string {
const cacheKeyPrefix = getCacheKeyPrefix()
const relativeFiles = files.map(x => path.relative(this.gradleUserHome, x))
const key = hashFileNames(relativeFiles)
this.debug(`Generating cache key for ${bundle} from files: ${relativeFiles}`)
this.debug(`Generating cache key for ${artifactType} from files: ${relativeFiles}`)
return `${cacheKeyPrefix}${bundle}-${key}`
return `${cacheKeyPrefix}${artifactType}-${key}`
}
/**
* Load information about the previously restored/saved artifact bundles from the 'cache-metadata.json' file.
*/
private loadBundleMetadata(): Map<string, string> {
const bundleMetaFile = path.resolve(this.gradleUserHome, META_FILE_DIR, META_FILE)
if (!fs.existsSync(bundleMetaFile)) {
return new Map<string, string>()
}
const filedata = fs.readFileSync(bundleMetaFile, 'utf-8')
core.debug(`Loaded bundle metadata: ${filedata}`)
return new Map(JSON.parse(filedata))
private isBundlePattern(pattern: string): boolean {
return pattern.endsWith('*')
}
/**
* Saves information about the artifact bundle restore/save into the 'cache-metadata.json' file.
*/
private saveMetadataForCacheResults(results: CacheBundleResult[]): void {
const metadata = new Map<string, string>()
for (const result of results) {
if (result.cacheKey !== undefined) {
metadata.set(result.bundle, result.cacheKey)
private async collectCacheResults(processes: Promise<ExtractedCacheEntry>[]): Promise<ExtractedCacheEntry[]> {
// Run cache actions sequentially when debugging enabled
if (this.cacheDebuggingEnabled) {
for (const p of processes) {
await p
}
}
const filedata = JSON.stringify(Array.from(metadata))
core.debug(`Saving bundle metadata: ${filedata}`)
const bundleMetaDir = path.resolve(this.gradleUserHome, META_FILE_DIR)
const bundleMetaFile = path.resolve(bundleMetaDir, META_FILE)
return await Promise.all(processes)
}
if (!fs.existsSync(bundleMetaDir)) {
fs.mkdirSync(bundleMetaDir, {recursive: true})
/**
* Load information about the extracted cache entries previously restored/saved. This is loaded from the 'cache-metadata.json' file.
*/
private loadExtractedCacheEntries(): ExtractedCacheEntry[] {
const cacheMetadataFile = path.resolve(this.gradleUserHome, META_FILE_DIR, META_FILE)
if (!fs.existsSync(cacheMetadataFile)) {
return []
}
fs.writeFileSync(bundleMetaFile, filedata, 'utf-8')
const filedata = fs.readFileSync(cacheMetadataFile, 'utf-8')
core.debug(`Loaded cache metadata: ${filedata}`)
const extractedCacheEntryMetadata = JSON.parse(filedata) as ExtractedCacheEntryMetadata
return extractedCacheEntryMetadata.entries
}
/**
* Saves information about the extracted cache entries into the 'cache-metadata.json' file.
*/
private saveMetadataForCacheResults(results: ExtractedCacheEntry[]): void {
const extractedCacheEntryMetadata = new ExtractedCacheEntryMetadata()
extractedCacheEntryMetadata.entries = results.filter(x => x.cacheKey !== undefined)
const filedata = JSON.stringify(extractedCacheEntryMetadata)
core.debug(`Saving cache metadata: ${filedata}`)
const actionMetadataDirectory = path.resolve(this.gradleUserHome, META_FILE_DIR)
const cacheMetadataFile = path.resolve(actionMetadataDirectory, META_FILE)
fs.mkdirSync(actionMetadataDirectory, {recursive: true})
fs.writeFileSync(cacheMetadataFile, filedata, 'utf-8')
}
protected determineGradleUserHome(rootDir: string): string {
@ -277,15 +321,14 @@ export class GradleUserHomeCache extends AbstractCache {
}
/**
* Return the artifact bundle definitions, which determine which artifacts will be cached
* Return the extracted cache entry definitions, which determine which artifacts will be cached
* separately from the rest of the Gradle User Home cache entry.
* This is normally a fixed set, but can be overridden by the `gradle-home-cache-artifact-bundles` parameter.
* This is normally a fixed set, but can be overridden by the `gradle-home-extracted-cache-entries` parameter.
*/
private getArtifactBundleDefinitions(): Map<string, string> {
const artifactBundleDefinition = core.getInput(ARTIFACT_BUNDLES_PARAMETER)
this.debug(`Using artifact bundle definition: ${artifactBundleDefinition}`)
const artifactBundles = JSON.parse(artifactBundleDefinition)
return new Map(Array.from(artifactBundles, ([key, value]) => [key, path.resolve(this.gradleUserHome, value)]))
private getExtractedCacheEntryDefinitions(): Map<string, string> {
const rawDefinitions = core.getInput(EXTRACTED_CACHE_ENTRIES_PARAMETER)
const parsedDefinitions = JSON.parse(rawDefinitions)
return new Map(Array.from(parsedDefinitions, ([key, value]) => [key, path.resolve(this.gradleUserHome, value)]))
}
/**