From e1f84aa44dfc9126a97daa5bbc9e5cface729962 Mon Sep 17 00:00:00 2001 From: Daz DeBoer Date: Tue, 7 Dec 2021 16:52:53 -0700 Subject: [PATCH] Restore/save configuration-cache data in first action Previously, the action was restoring/saving the configuration-cache data for each step that applied the action. In order to support Gradle invocations that are _not_ managed by the action, the configuration-cache restore is now performed in the initial action step, and save is performed in the final post-action step. The build root directories are recorded for each invocation via an init script. --- src/cache-base.ts | 30 ++++++++++++------- src/cache-gradle-user-home.ts | 51 +++++++++---------------------- src/cache-project-dot-gradle.ts | 53 ++++++++++++++++++++++++--------- src/cache-utils.ts | 16 +++++++--- src/caches.ts | 19 ++++++------ src/main.ts | 13 +++++++- 6 files changed, 107 insertions(+), 75 deletions(-) diff --git a/src/cache-base.ts b/src/cache-base.ts index e7a6868..47f58dd 100644 --- a/src/cache-base.ts +++ b/src/cache-base.ts @@ -1,11 +1,15 @@ import * as core from '@actions/core' import * as cache from '@actions/cache' import * as github from '@actions/github' +import path from 'path' +import fs from 'fs' import {CacheListener} from './cache-reporting' -import {isCacheDebuggingEnabled, getCacheKeyPrefix, hashStrings, handleCacheFailure} from './cache-utils' +import {isCacheDebuggingEnabled, getCacheKeyPrefix, determineJobContext, handleCacheFailure} from './cache-utils' const CACHE_PROTOCOL_VERSION = 'v5-' -const JOB_CONTEXT_PARAMETER = 'workflow-job-context' + +export const META_FILE_DIR = '.gradle-build-action' +export const PROJECT_ROOTS_FILE = 'project-roots.txt' /** * Represents a key used to restore a cache entry. @@ -58,22 +62,17 @@ function generateCacheKey(cacheName: string): CacheKey { return new CacheKey(cacheKey, [cacheKeyForJobContext, cacheKeyForJob, cacheKeyForOs]) } -function determineJobContext(): string { - // By default, we hash the full `matrix` data for the run, to uniquely identify this job invocation - // The only way we can obtain the `matrix` data is via the `workflow-job-context` parameter in action.yml. - const workflowJobContext = core.getInput(JOB_CONTEXT_PARAMETER) - return hashStrings([workflowJobContext]) -} - export abstract class AbstractCache { private cacheName: string private cacheDescription: string private cacheKeyStateKey: string private cacheResultStateKey: string + protected readonly gradleUserHome: string protected readonly cacheDebuggingEnabled: boolean - constructor(cacheName: string, cacheDescription: string) { + constructor(gradleUserHome: string, cacheName: string, cacheDescription: string) { + this.gradleUserHome = gradleUserHome this.cacheName = cacheName this.cacheDescription = cacheDescription this.cacheKeyStateKey = `CACHE_KEY_${cacheName}` @@ -81,6 +80,16 @@ export abstract class AbstractCache { this.cacheDebuggingEnabled = isCacheDebuggingEnabled() } + init(): void { + const actionCacheDir = path.resolve(this.gradleUserHome, '.gradle-build-action') + fs.mkdirSync(actionCacheDir, {recursive: true}) + + const initScriptsDir = path.resolve(this.gradleUserHome, 'init.d') + fs.mkdirSync(initScriptsDir, {recursive: true}) + + this.initializeGradleUserHome(this.gradleUserHome, initScriptsDir) + } + /** * Restores the cache entry, finding the closest match to the currently running job. */ @@ -191,4 +200,5 @@ export abstract class AbstractCache { } protected abstract getCachePath(): string[] + protected abstract initializeGradleUserHome(gradleUserHome: string, initScriptsDir: string): void } diff --git a/src/cache-gradle-user-home.ts b/src/cache-gradle-user-home.ts index 709343e..c59ceea 100644 --- a/src/cache-gradle-user-home.ts +++ b/src/cache-gradle-user-home.ts @@ -1,15 +1,13 @@ import path from 'path' import fs from 'fs' -import os from 'os' import * as core from '@actions/core' import * as glob from '@actions/glob' import * as exec from '@actions/exec' -import {AbstractCache} from './cache-base' +import {AbstractCache, META_FILE_DIR} from './cache-base' import {CacheEntryListener, CacheListener} from './cache-reporting' import {getCacheKeyPrefix, hashFileNames, tryDelete} from './cache-utils' -const META_FILE_DIR = '.gradle-build-action' const META_FILE = 'cache-metadata.json' const INCLUDE_PATHS_PARAMETER = 'gradle-home-cache-includes' @@ -47,16 +45,8 @@ class ExtractedCacheEntryMetadata { * for more efficient storage. */ export class GradleUserHomeCache extends AbstractCache { - private gradleUserHome: string - - constructor(rootDir: string) { - super('gradle', 'Gradle User Home') - this.gradleUserHome = this.determineGradleUserHome(rootDir) - } - - init(): void { - this.debug(`Initializing Gradle User Home with properties and init script: ${this.gradleUserHome}`) - initializeGradleUserHome(this.gradleUserHome) + constructor(gradleUserHome: string) { + super(gradleUserHome, 'gradle', 'Gradle User Home') } /** @@ -285,15 +275,6 @@ export class GradleUserHomeCache extends AbstractCache { fs.writeFileSync(cacheMetadataFile, filedata, 'utf-8') } - protected determineGradleUserHome(rootDir: string): string { - const customGradleUserHome = process.env['GRADLE_USER_HOME'] - if (customGradleUserHome) { - return path.resolve(rootDir, customGradleUserHome) - } - - return path.resolve(os.homedir(), '.gradle') - } - /** * Determines the paths within Gradle User Home to cache. * By default, this is the 'caches' and 'notifications' directories, @@ -358,21 +339,17 @@ export class GradleUserHomeCache extends AbstractCache { core.info('-----------------------') } -} -function initializeGradleUserHome(gradleUserHome: string): void { - fs.mkdirSync(gradleUserHome, {recursive: true}) + protected initializeGradleUserHome(gradleUserHome: string, initScriptsDir: string): void { + const propertiesFile = path.resolve(gradleUserHome, 'gradle.properties') + fs.writeFileSync(propertiesFile, 'org.gradle.daemon=false') - const propertiesFile = path.resolve(gradleUserHome, 'gradle.properties') - fs.writeFileSync(propertiesFile, 'org.gradle.daemon=false') + const buildScanCapture = path.resolve(initScriptsDir, 'build-scan-capture.init.gradle') + fs.writeFileSync( + buildScanCapture, + `import org.gradle.util.GradleVersion - const initScript = path.resolve(gradleUserHome, 'init.gradle') - fs.writeFileSync( - initScript, - ` -import org.gradle.util.GradleVersion - -// Don't run against the included builds (if the main build has any). +// Only run again root build. Do not run against included builds. def isTopLevelBuild = gradle.getParent() == null if (isTopLevelBuild) { def version = GradleVersion.current().baseVersion @@ -412,7 +389,7 @@ def registerCallbacks(buildScanExtension, rootProjectName) { println("::set-output name=build-scan-url::\${buildScan.buildScanUri}") } } -} -` - ) +}` + ) + } } diff --git a/src/cache-project-dot-gradle.ts b/src/cache-project-dot-gradle.ts index 97380d2..3bb55fa 100644 --- a/src/cache-project-dot-gradle.ts +++ b/src/cache-project-dot-gradle.ts @@ -1,27 +1,52 @@ +import * as core from '@actions/core' import path from 'path' -import {AbstractCache} from './cache-base' - -// TODO: Maybe allow the user to override / tweak this set -const PATHS_TO_CACHE = [ - 'configuration-cache' // Only configuration-cache is stored at present -] +import fs from 'fs' +import {AbstractCache, META_FILE_DIR, PROJECT_ROOTS_FILE} from './cache-base' /** * A simple cache that saves and restores the '.gradle/configuration-cache' directory in the project root. */ export class ProjectDotGradleCache extends AbstractCache { - private rootDir: string - constructor(rootDir: string) { - super('project', 'Project configuration cache') - this.rootDir = rootDir + constructor(gradleUserHome: string) { + super(gradleUserHome, 'project', 'Project configuration cache') } protected getCachePath(): string[] { - const dir = this.getProjectDotGradleDir() - return PATHS_TO_CACHE.map(x => path.resolve(dir, x)) + return this.getProjectRoots().map(x => path.resolve(x, '.gradle/configuration-cache')) } - private getProjectDotGradleDir(): string { - return path.resolve(this.rootDir, '.gradle') + protected initializeGradleUserHome(gradleUserHome: string, initScriptsDir: string): void { + const projectRootCapture = path.resolve(initScriptsDir, 'project-root-capture.init.gradle') + fs.writeFileSync( + projectRootCapture, + ` + // Only run again root build. Do not run against included builds. + def isTopLevelBuild = gradle.getParent() == null + if (isTopLevelBuild) { + settingsEvaluated { settings -> + def projectRootEntry = settings.rootDir.absolutePath + "\\n" + def projectRootList = new File(settings.gradle.gradleUserHomeDir, "${META_FILE_DIR}/${PROJECT_ROOTS_FILE}") + println "Adding " + projectRootEntry + " to " + projectRootList + if (!projectRootList.exists() || !projectRootList.text.contains(projectRootEntry)) { + projectRootList << projectRootEntry + } + } + }` + ) + } + + /** + * For every Gradle invocation, we record the project root directory. This method returns the entire + * set of project roots, to allow saving of configuration-cache entries for each. + */ + private getProjectRoots(): string[] { + const projectList = path.resolve(this.gradleUserHome, META_FILE_DIR, PROJECT_ROOTS_FILE) + if (!fs.existsSync(projectList)) { + core.info(`Missing project list file ${projectList}`) + return [] + } + const projectRoots = fs.readFileSync(projectList, 'utf-8') + core.info(`Found project roots '${projectRoots}' in ${projectList}`) + return projectRoots.trim().split('\n') } } diff --git a/src/cache-utils.ts b/src/cache-utils.ts index c596570..8ab749e 100644 --- a/src/cache-utils.ts +++ b/src/cache-utils.ts @@ -4,6 +4,7 @@ import * as crypto from 'crypto' import * as path from 'path' import * as fs from 'fs' +const JOB_CONTEXT_PARAMETER = 'workflow-job-context' const CACHE_DISABLED_PARAMETER = 'cache-disabled' const CACHE_READONLY_PARAMETER = 'cache-read-only' const CACHE_DEBUG_VAR = 'GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED' @@ -26,6 +27,17 @@ export function getCacheKeyPrefix(): string { return process.env[CACHE_PREFIX_VAR] || '' } +export function determineJobContext(): string { + // By default, we hash the full `matrix` data for the run, to uniquely identify this job invocation + // The only way we can obtain the `matrix` data is via the `workflow-job-context` parameter in action.yml. + const workflowJobContext = core.getInput(JOB_CONTEXT_PARAMETER) + return hashStrings([workflowJobContext]) +} + +export function hashFileNames(fileNames: string[]): string { + return hashStrings(fileNames.map(x => x.replace(new RegExp(`\\${path.sep}`, 'g'), '/'))) +} + export function hashStrings(values: string[]): string { const hash = crypto.createHash('md5') for (const value of values) { @@ -34,10 +46,6 @@ export function hashStrings(values: string[]): string { return hash.digest('hex') } -export function hashFileNames(fileNames: string[]): string { - return hashStrings(fileNames.map(x => x.replace(new RegExp(`\\${path.sep}`, 'g'), '/'))) -} - export function handleCacheFailure(error: unknown, message: string): void { if (error instanceof cache.ValidationError) { // Fail on cache validation errors diff --git a/src/caches.ts b/src/caches.ts index 062fec9..d78e061 100644 --- a/src/caches.ts +++ b/src/caches.ts @@ -5,21 +5,22 @@ import {isCacheDisabled, isCacheReadOnly} from './cache-utils' import {logCachingReport, CacheListener} from './cache-reporting' const CACHE_RESTORED_VAR = 'GRADLE_BUILD_ACTION_CACHE_RESTORED' -const BUILD_ROOT_DIR = 'BUILD_ROOT_DIR' +const GRADLE_USER_HOME = 'GRADLE_USER_HOME' const CACHE_LISTENER = 'CACHE_LISTENER' -export async function restore(buildRootDirectory: string): Promise { +export async function restore(gradleUserHome: string): Promise { if (!shouldRestoreCaches()) { return } - const gradleUserHomeCache = new GradleUserHomeCache(buildRootDirectory) - const projectDotGradleCache = new ProjectDotGradleCache(buildRootDirectory) - + const gradleUserHomeCache = new GradleUserHomeCache(gradleUserHome) gradleUserHomeCache.init() + const projectDotGradleCache = new ProjectDotGradleCache(gradleUserHome) + projectDotGradleCache.init() + await core.group('Restore Gradle state from cache', async () => { - core.saveState(BUILD_ROOT_DIR, buildRootDirectory) + core.saveState(GRADLE_USER_HOME, gradleUserHome) const cacheListener = new CacheListener() await gradleUserHomeCache.restore(cacheListener) @@ -51,10 +52,10 @@ export async function save(): Promise { } await core.group('Caching Gradle state', async () => { - const buildRootDirectory = core.getState(BUILD_ROOT_DIR) + const gradleUserHome = core.getState(GRADLE_USER_HOME) return Promise.all([ - new GradleUserHomeCache(buildRootDirectory).save(cacheListener), - new ProjectDotGradleCache(buildRootDirectory).save(cacheListener) + new GradleUserHomeCache(gradleUserHome).save(cacheListener), + new ProjectDotGradleCache(gradleUserHome).save(cacheListener) ]) }) diff --git a/src/main.ts b/src/main.ts index 18e7641..f6edcc8 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,5 +1,6 @@ import * as core from '@actions/core' import * as path from 'path' +import * as os from 'os' import {parseArgsStringToArgv} from 'string-argv' import * as caches from './caches' @@ -14,8 +15,9 @@ export async function run(): Promise { try { const workspaceDirectory = process.env[`GITHUB_WORKSPACE`] || '' const buildRootDirectory = resolveBuildRootDirectory(workspaceDirectory) + const gradleUserHome = determineGradleUserHome(buildRootDirectory) - await caches.restore(buildRootDirectory) + await caches.restore(gradleUserHome) const args: string[] = parseCommandLineArguments() @@ -63,6 +65,15 @@ function resolveBuildRootDirectory(baseDirectory: string): string { return resolvedBuildRootDirectory } +function determineGradleUserHome(rootDir: string): string { + const customGradleUserHome = process.env['GRADLE_USER_HOME'] + if (customGradleUserHome) { + return path.resolve(rootDir, customGradleUserHome) + } + + return path.resolve(os.homedir(), '.gradle') +} + function parseCommandLineArguments(): string[] { const input = core.getInput('arguments') return parseArgsStringToArgv(input)