import path from 'path' import fs from 'fs' import * as core from '@actions/core' import * as glob from '@actions/glob' import {META_FILE_DIR, PROJECT_ROOTS_FILE} from './cache-base' import {CacheEntryListener, CacheListener} from './cache-reporting' import { cacheDebug, getCacheKeyPrefix, hashFileNames, isCacheDebuggingEnabled, restoreCache, saveCache, tryDelete } from './cache-utils' const SKIP_RESTORE_VAR = 'GRADLE_BUILD_ACTION_SKIP_RESTORE' /** * Represents the result of attempting to load or store an extracted cache entry. * An undefined cacheKey indicates that the operation did not succeed. * The collected results are then used to populate the `cache-metadata.json` file for later use. */ class ExtractedCacheEntry { artifactType: string pattern: string cacheKey: string | undefined constructor(artifactType: string, pattern: string, cacheKey: string | undefined) { this.artifactType = artifactType this.pattern = pattern this.cacheKey = cacheKey } } /** * Representation of all of the extracted cache entries for this Gradle User Home. * This object is persisted to JSON file in the Gradle User Home directory for storing, * and subsequently used to restore the Gradle User Home. */ class ExtractedCacheEntryMetadata { entries: ExtractedCacheEntry[] = [] } /** * The specification for a type of extracted cache entry. */ class ExtractedCacheEntryDefinition { artifactType: string pattern: string bundle: boolean uniqueFileNames = true constructor(artifactType: string, pattern: string, bundle: boolean) { this.artifactType = artifactType this.pattern = pattern this.bundle = bundle } withNonUniqueFileNames(): ExtractedCacheEntryDefinition { this.uniqueFileNames = false return this } } /** * Caches and restores the entire Gradle User Home directory, extracting entries containing common artifacts * for more efficient storage. */ abstract class AbstractEntryExtractor { protected readonly gradleUserHome: string private extractorName: string constructor(gradleUserHome: string, extractorName: string) { this.gradleUserHome = gradleUserHome this.extractorName = extractorName } /** * Restores any artifacts that were cached separately, based on the information in the `cache-metadata.json` file. * Each extracted cache entry is restored in parallel, except when debugging is enabled. */ async restore(listener: CacheListener): Promise { const previouslyExtractedCacheEntries = this.loadExtractedCacheEntries() const processes: Promise[] = [] for (const cacheEntry of previouslyExtractedCacheEntries) { const artifactType = cacheEntry.artifactType const entryListener = listener.entry(cacheEntry.pattern) // Handle case where the extracted-cache-entry definitions have been changed const skipRestore = process.env[SKIP_RESTORE_VAR] || '' if (skipRestore.includes(artifactType)) { core.info(`Not restoring extracted cache entry for ${artifactType}`) entryListener.markRequested('SKIP_RESTORE') } else { processes.push( this.awaitForDebugging( this.restoreExtractedCacheEntry( artifactType, cacheEntry.cacheKey!, cacheEntry.pattern, entryListener ) ) ) } } this.saveMetadataForCacheResults(await Promise.all(processes)) } private async restoreExtractedCacheEntry( artifactType: string, cacheKey: string, pattern: string, listener: CacheEntryListener ): Promise { const restoredEntry = await restoreCache([pattern], cacheKey, [], listener) if (restoredEntry) { core.info(`Restored ${artifactType} with key ${cacheKey} to ${pattern}`) return new ExtractedCacheEntry(artifactType, pattern, cacheKey) } else { core.info(`Did not restore ${artifactType} with key ${cacheKey} to ${pattern}`) return new ExtractedCacheEntry(artifactType, pattern, undefined) } } /** * Saves any artifacts that are configured to be cached separately, based on the extracted cache entry definitions. * Each entry is extracted and saved in parallel, except when debugging is enabled. */ async extract(listener: CacheListener): Promise { // Load the cache entry definitions (from config) and the previously restored entries (from persisted metadata file) const cacheEntryDefinitions = this.getExtractedCacheEntryDefinitions() cacheDebug( `Extracting cache entries for ${this.extractorName}: ${JSON.stringify(cacheEntryDefinitions, null, 2)}` ) const previouslyRestoredEntries = this.loadExtractedCacheEntries() const cacheActions: Promise[] = [] // For each cache entry definition, determine if it has already been restored, and if not, extract it for (const cacheEntryDefinition of cacheEntryDefinitions) { const artifactType = cacheEntryDefinition.artifactType const pattern = cacheEntryDefinition.pattern // Find all matching files for this cache entry definition const globber = await glob.create(pattern, { implicitDescendants: false, followSymbolicLinks: false }) const matchingFiles = await globber.glob() if (matchingFiles.length === 0) { cacheDebug(`No files found to cache for ${artifactType}`) continue } if (cacheEntryDefinition.bundle) { // For an extracted "bundle", use the defined pattern and cache all matching files in a single entry. cacheActions.push( this.awaitForDebugging( this.saveExtractedCacheEntry( matchingFiles, artifactType, pattern, cacheEntryDefinition.uniqueFileNames, previouslyRestoredEntries, listener.entry(pattern) ) ) ) } else { // Otherwise cache each matching file in a separate entry, using the complete file path as the cache pattern. for (const cacheFile of matchingFiles) { cacheActions.push( this.awaitForDebugging( this.saveExtractedCacheEntry( [cacheFile], artifactType, cacheFile, cacheEntryDefinition.uniqueFileNames, previouslyRestoredEntries, listener.entry(cacheFile) ) ) ) } } } this.saveMetadataForCacheResults(await Promise.all(cacheActions)) } private async saveExtractedCacheEntry( matchingFiles: string[], artifactType: string, pattern: string, uniqueFileNames: boolean, previouslyRestoredEntries: ExtractedCacheEntry[], entryListener: CacheEntryListener ): Promise { const cacheKey = uniqueFileNames ? this.createCacheKeyFromFileNames(artifactType, matchingFiles) : await this.createCacheKeyFromFileContents(artifactType, pattern) const previouslyRestoredKey = previouslyRestoredEntries.find( x => x.artifactType === artifactType && x.pattern === pattern )?.cacheKey if (previouslyRestoredKey === cacheKey) { cacheDebug(`No change to previously restored ${artifactType}. Not saving.`) entryListener.markUnchanged('entry contents unchanged') } else { core.info(`Caching ${artifactType} with path '${pattern}' and cache key: ${cacheKey}`) await saveCache([pattern], cacheKey, entryListener) } for (const file of matchingFiles) { tryDelete(file) } return new ExtractedCacheEntry(artifactType, pattern, cacheKey) } protected createCacheKeyFromFileNames(artifactType: string, files: string[]): string { const cacheKeyPrefix = getCacheKeyPrefix() const relativeFiles = files.map(x => path.relative(this.gradleUserHome, x)) const key = hashFileNames(relativeFiles) cacheDebug(`Generating cache key for ${artifactType} from file names: ${relativeFiles}`) return `${cacheKeyPrefix}${artifactType}-${key}` } protected async createCacheKeyFromFileContents(artifactType: string, pattern: string): Promise { const cacheKeyPrefix = getCacheKeyPrefix() const key = await glob.hashFiles(pattern) cacheDebug(`Generating cache key for ${artifactType} from files matching: ${pattern}`) return `${cacheKeyPrefix}${artifactType}-${key}` } // Run actions sequentially if debugging is enabled private async awaitForDebugging(p: Promise): Promise { if (isCacheDebuggingEnabled()) { await p } return p } /** * Load information about the extracted cache entries previously restored/saved. This is loaded from the 'cache-metadata.json' file. */ protected loadExtractedCacheEntries(): ExtractedCacheEntry[] { const cacheMetadataFile = this.getCacheMetadataFile() if (!fs.existsSync(cacheMetadataFile)) { return [] } const filedata = fs.readFileSync(cacheMetadataFile, 'utf-8') cacheDebug(`Loaded cache metadata: ${filedata}`) const extractedCacheEntryMetadata = JSON.parse(filedata) as ExtractedCacheEntryMetadata return extractedCacheEntryMetadata.entries } /** * Saves information about the extracted cache entries into the 'cache-metadata.json' file. */ private saveMetadataForCacheResults(results: ExtractedCacheEntry[]): void { const extractedCacheEntryMetadata = new ExtractedCacheEntryMetadata() extractedCacheEntryMetadata.entries = results.filter(x => x.cacheKey !== undefined) const filedata = JSON.stringify(extractedCacheEntryMetadata) cacheDebug(`Saving cache metadata: ${filedata}`) fs.writeFileSync(this.getCacheMetadataFile(), filedata, 'utf-8') } private getCacheMetadataFile(): string { const actionMetadataDirectory = path.resolve(this.gradleUserHome, META_FILE_DIR) fs.mkdirSync(actionMetadataDirectory, {recursive: true}) return path.resolve(actionMetadataDirectory, `${this.extractorName}-entry-metadata.json`) } protected abstract getExtractedCacheEntryDefinitions(): ExtractedCacheEntryDefinition[] } export class GradleHomeEntryExtractor extends AbstractEntryExtractor { constructor(gradleUserHome: string) { super(gradleUserHome, 'gradle-home') } async extract(listener: CacheListener): Promise { await this.deleteWrapperZips() return super.extract(listener) } /** * Delete any downloaded wrapper zip files that are not needed after extraction. * These files are cleaned up by Gradle >= 7.5, but for older versions we remove them manually. */ private async deleteWrapperZips(): Promise { const wrapperZips = path.resolve(this.gradleUserHome, 'wrapper/dists/*/*/*.zip') const globber = await glob.create(wrapperZips, { implicitDescendants: false, followSymbolicLinks: false }) for (const p of await globber.glob()) { cacheDebug(`Deleting wrapper zip: ${p}`) tryDelete(p) } } /** * Return the extracted cache entry definitions, which determine which artifacts will be cached * separately from the rest of the Gradle User Home cache entry. */ protected getExtractedCacheEntryDefinitions(): ExtractedCacheEntryDefinition[] { const entryDefinition = ( artifactType: string, patterns: string[], bundle: boolean ): ExtractedCacheEntryDefinition => { const resolvedPatterns = patterns .map(x => { const isDir = x.endsWith('/') const resolved = path.resolve(this.gradleUserHome, x) return isDir ? `${resolved}/` : resolved // Restore trailing '/' removed by path.resolve() }) .join('\n') return new ExtractedCacheEntryDefinition(artifactType, resolvedPatterns, bundle) } return [ entryDefinition('generated-gradle-jars', ['caches/*/generated-gradle-jars/*.jar'], false), entryDefinition('wrapper-zips', ['wrapper/dists/*/*/'], false), // Entire wrapper directory cached together entryDefinition('java-toolchains', ['jdks/*.zip', 'jdks/*.tar.gz'], false), entryDefinition('dependencies', ['caches/modules-*/files-*/*/*/*/*'], true), entryDefinition('instrumented-jars', ['caches/jars-*/*'], true), entryDefinition('kotlin-dsl', ['caches/*/kotlin-dsl/*/*'], true) ] } } export class ConfigurationCacheEntryExtractor extends AbstractEntryExtractor { constructor(gradleUserHome: string) { super(gradleUserHome, 'configuration-cache') } /** * Handle the case where Gradle User Home has not been fully restored, so that the configuration-cache * entry is not reusable. */ async restore(listener: CacheListener): Promise { if (listener.fullyRestored) { return super.restore(listener) } core.info('Not restoring configuration-cache state, as Gradle User Home was not fully restored') for (const cacheEntry of this.loadExtractedCacheEntries()) { listener.entry(cacheEntry.pattern).markRequested('NOT_RESTORED') } } /** * Extract cache entries for the configuration cache in each project. */ protected getExtractedCacheEntryDefinitions(): ExtractedCacheEntryDefinition[] { return this.getProjectRoots().map(projectRoot => { const configCachePath = path.resolve(projectRoot, '.gradle/configuration-cache') return new ExtractedCacheEntryDefinition( 'configuration-cache', configCachePath, true ).withNonUniqueFileNames() }) } /** * For every Gradle invocation, we record the project root directory. This method returns the entire * set of project roots, to allow saving of configuration-cache entries for each. */ private getProjectRoots(): string[] { const projectList = path.resolve(this.gradleUserHome, PROJECT_ROOTS_FILE) if (!fs.existsSync(projectList)) { core.info(`Missing project list file ${projectList}`) return [] } const projectRoots = fs.readFileSync(projectList, 'utf-8') core.info(`Found project roots '${projectRoots}' in ${projectList}`) return projectRoots.trim().split('\n') } }