Merge pull request #142 from gradle/strict-cache-match

Improves logging for caching, and adds 2 experimental cache options:

- `gradle-home-cache-strict-match`: Avoid re-using a Gradle Home cache entry written by a different job. Only cache entries written by the current Job will be used.
- `cache-write-only`: Do not restore existing cache entries, but still save cache entries in post-action.  Fixes #141.
This commit is contained in:
Daz DeBoer 2022-01-24 12:56:25 -07:00 committed by GitHub
commit d2799e6f98
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 125 additions and 56 deletions

View file

@ -101,3 +101,42 @@ jobs:
with: with:
script: | script: |
core.setFailed('No build scan detected') core.setFailed('No build scan detected')
# Test seed the cache with cache-write-only and verify with cache-read-only
seed-build-write-only:
env:
GRADLE_BUILD_ACTION_CACHE_KEY_PREFIX: ${{github.workflow}}#${{github.run_number}}:${{github.run_attempt}}-write-only-
strategy:
matrix:
os: [ubuntu-latest, windows-latest]
runs-on: ${{ matrix.os }}
steps:
- name: Checkout sources
uses: actions/checkout@v2
- name: Setup Gradle
uses: ./
with:
cache-write-only: true
- name: Build using Gradle wrapper
working-directory: __tests__/samples/groovy-dsl
run: ./gradlew test
verify-write-only-build:
env:
GRADLE_BUILD_ACTION_CACHE_KEY_PREFIX: ${{github.workflow}}#${{github.run_number}}:${{github.run_attempt}}-write-only-
needs: seed-build-write-only
strategy:
matrix:
os: [ubuntu-latest, windows-latest]
runs-on: ${{ matrix.os }}
steps:
- name: Checkout sources
uses: actions/checkout@v2
- name: Setup Gradle
uses: ./
with:
cache-read-only: true
- name: Execute Gradle build with --offline
working-directory: __tests__/samples/groovy-dsl
run: ./gradlew test --offline

View file

@ -48,8 +48,16 @@ inputs:
# EXPERIMENTAL & INTERNAL ACTION INPUTS # EXPERIMENTAL & INTERNAL ACTION INPUTS
# The following action properties allow fine-grained tweaking of the action caching behaviour. # The following action properties allow fine-grained tweaking of the action caching behaviour.
# These properties are not designed for production use, and may change without notice in a subsequent release of `gradle-build-action`. # These properties are experimental and not (yet) designed for production use, and may change without notice in a subsequent release of `gradle-build-action`.
# Use at your own risk! # Use at your own risk!
cache-write-only:
description: When 'true', entries will not be restored from the cache but will be saved at the end of the Job. This allows a 'clean' cache entry to be written.
required: false
default: false
gradle-home-cache-strict-match:
description: When 'true', the action will not attempt to restore the Gradle User Home entries from other Jobs.
required: false
default: false
workflow-job-context: workflow-job-context:
description: Used to uniquely identify the current job invocation. Defaults to the matrix values for this job; this should not be overridden by users (INTERNAL). description: Used to uniquely identify the current job invocation. Defaults to the matrix values for this job; this should not be overridden by users (INTERNAL).
required: false required: false

2
dist/main/index.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

2
dist/post/index.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -15,12 +15,14 @@ import {
} from './cache-utils' } from './cache-utils'
import {ConfigurationCacheEntryExtractor, GradleHomeEntryExtractor} from './cache-extract-entries' import {ConfigurationCacheEntryExtractor, GradleHomeEntryExtractor} from './cache-extract-entries'
const CACHE_PROTOCOL_VERSION = 'v5-' const CACHE_PROTOCOL_VERSION = 'v6-'
const RESTORED_CACHE_KEY_KEY = 'restored-cache-key'
export const META_FILE_DIR = '.gradle-build-action' export const META_FILE_DIR = '.gradle-build-action'
export const PROJECT_ROOTS_FILE = 'project-roots.txt' export const PROJECT_ROOTS_FILE = 'project-roots.txt'
const INCLUDE_PATHS_PARAMETER = 'gradle-home-cache-includes' const INCLUDE_PATHS_PARAMETER = 'gradle-home-cache-includes'
const EXCLUDE_PATHS_PARAMETER = 'gradle-home-cache-excludes' const EXCLUDE_PATHS_PARAMETER = 'gradle-home-cache-excludes'
const STRICT_CACHE_MATCH_PARAMETER = 'gradle-home-cache-strict-match'
/** /**
* Represents a key used to restore a cache entry. * Represents a key used to restore a cache entry.
@ -70,14 +72,16 @@ function generateCacheKey(cacheName: string): CacheKey {
// Exact match on Git SHA // Exact match on Git SHA
const cacheKey = `${cacheKeyForJobContext}-${github.context.sha}` const cacheKey = `${cacheKeyForJobContext}-${github.context.sha}`
if (core.getBooleanInput(STRICT_CACHE_MATCH_PARAMETER)) {
return new CacheKey(cacheKey, [cacheKeyForJobContext])
}
return new CacheKey(cacheKey, [cacheKeyForJobContext, cacheKeyForJob, cacheKeyForOs]) return new CacheKey(cacheKey, [cacheKeyForJobContext, cacheKeyForJob, cacheKeyForOs])
} }
export class GradleStateCache { export class GradleStateCache {
private cacheName: string private cacheName: string
private cacheDescription: string private cacheDescription: string
private cacheKeyStateKey: string
private cacheResultStateKey: string
protected readonly gradleUserHome: string protected readonly gradleUserHome: string
@ -85,8 +89,6 @@ export class GradleStateCache {
this.gradleUserHome = gradleUserHome this.gradleUserHome = gradleUserHome
this.cacheName = 'gradle' this.cacheName = 'gradle'
this.cacheDescription = 'Gradle User Home' this.cacheDescription = 'Gradle User Home'
this.cacheKeyStateKey = `CACHE_KEY_gradle`
this.cacheResultStateKey = `CACHE_RESULT_gradle`
} }
init(): void { init(): void {
@ -117,7 +119,6 @@ export class GradleStateCache {
const entryListener = listener.entry(this.cacheDescription) const entryListener = listener.entry(this.cacheDescription)
const cacheKey = generateCacheKey(this.cacheName) const cacheKey = generateCacheKey(this.cacheName)
core.saveState(this.cacheKeyStateKey, cacheKey.key)
cacheDebug( cacheDebug(
`Requesting ${this.cacheDescription} with `Requesting ${this.cacheDescription} with
@ -125,16 +126,13 @@ export class GradleStateCache {
restoreKeys:[${cacheKey.restoreKeys}]` restoreKeys:[${cacheKey.restoreKeys}]`
) )
const cacheResult = await restoreCache(this.getCachePath(), cacheKey.key, cacheKey.restoreKeys) const cacheResult = await restoreCache(this.getCachePath(), cacheKey.key, cacheKey.restoreKeys, entryListener)
entryListener.markRequested(cacheKey.key, cacheKey.restoreKeys)
if (!cacheResult) { if (!cacheResult) {
core.info(`${this.cacheDescription} cache not found. Will initialize empty.`) core.info(`${this.cacheDescription} cache not found. Will initialize empty.`)
return return
} }
core.saveState(this.cacheResultStateKey, cacheResult.key) core.saveState(RESTORED_CACHE_KEY_KEY, cacheResult.key)
entryListener.markRestored(cacheResult.key, cacheResult.size)
core.info(`Restored ${this.cacheDescription} from cache key: ${cacheResult.key}`) core.info(`Restored ${this.cacheDescription} from cache key: ${cacheResult.key}`)
@ -163,12 +161,11 @@ export class GradleStateCache {
* it is saved with the exact key. * it is saved with the exact key.
*/ */
async save(listener: CacheListener): Promise<void> { async save(listener: CacheListener): Promise<void> {
// Retrieve the state set in the previous 'restore' step. const cacheKey = generateCacheKey(this.cacheName).key
const cacheKeyFromRestore = core.getState(this.cacheKeyStateKey) const restoredCacheKey = core.getState(RESTORED_CACHE_KEY_KEY)
const cacheResultFromRestore = core.getState(this.cacheResultStateKey)
if (cacheResultFromRestore && cacheKeyFromRestore === cacheResultFromRestore) { if (restoredCacheKey && cacheKey === restoredCacheKey) {
core.info(`Cache hit occurred on the cache key ${cacheKeyFromRestore}, not saving cache.`) core.info(`Cache hit occurred on the cache key ${cacheKey}, not saving cache.`)
return return
} }
@ -179,13 +176,10 @@ export class GradleStateCache {
return return
} }
core.info(`Caching ${this.cacheDescription} with cache key: ${cacheKeyFromRestore}`) core.info(`Caching ${this.cacheDescription} with cache key: ${cacheKey}`)
const cachePath = this.getCachePath() const cachePath = this.getCachePath()
const savedEntry = await saveCache(cachePath, cacheKeyFromRestore) const entryListener = listener.entry(this.cacheDescription)
await saveCache(cachePath, cacheKey, entryListener)
if (savedEntry) {
listener.entry(this.cacheDescription).markSaved(savedEntry.key, savedEntry.size)
}
return return
} }
@ -272,11 +266,11 @@ if (isTopLevelBuild) {
def registerCallbacks(buildScanExtension, rootProjectName) { def registerCallbacks(buildScanExtension, rootProjectName) {
buildScanExtension.with { buildScanExtension.with {
def buildOutcome = ""
def scanFile = new File("gradle-build-scan.txt") def scanFile = new File("gradle-build-scan.txt")
def buildFailed = false
buildFinished { result -> buildFinished { result ->
buildOutcome = result.failure == null ? " succeeded" : " failed" buildFailed = (result.failure != null)
} }
buildScanPublished { buildScan -> buildScanPublished { buildScan ->
@ -284,8 +278,11 @@ def registerCallbacks(buildScanExtension, rootProjectName) {
// Send commands directly to GitHub Actions via STDOUT. // Send commands directly to GitHub Actions via STDOUT.
def gradleCommand = rootProjectName + " " + gradle.startParameter.taskNames.join(" ") def gradleCommand = rootProjectName + " " + gradle.startParameter.taskNames.join(" ")
def message = "Gradle build '\${gradleCommand}'\${buildOutcome} - \${buildScan.buildScanUri}" if (buildFailed) {
println("::notice ::\${message}") println("::warning ::Gradle build '\${gradleCommand}' FAILED - \${buildScan.buildScanUri}")
} else {
println("::notice ::Gradle build '\${gradleCommand}' - \${buildScan.buildScanUri}")
}
println("::set-output name=build-scan-url::\${buildScan.buildScanUri}") println("::set-output name=build-scan-url::\${buildScan.buildScanUri}")
} }
} }

View file

@ -118,12 +118,9 @@ abstract class AbstractEntryExtractor {
pattern: string, pattern: string,
listener: CacheEntryListener listener: CacheEntryListener
): Promise<ExtractedCacheEntry> { ): Promise<ExtractedCacheEntry> {
listener.markRequested(cacheKey) const restoredEntry = await restoreCache([pattern], cacheKey, [], listener)
const restoredEntry = await restoreCache([pattern], cacheKey)
if (restoredEntry) { if (restoredEntry) {
core.info(`Restored ${artifactType} with key ${cacheKey} to ${pattern}`) core.info(`Restored ${artifactType} with key ${cacheKey} to ${pattern}`)
listener.markRestored(restoredEntry.key, restoredEntry.size)
return new ExtractedCacheEntry(artifactType, pattern, cacheKey) return new ExtractedCacheEntry(artifactType, pattern, cacheKey)
} else { } else {
core.info(`Did not restore ${artifactType} with key ${cacheKey} to ${pattern}`) core.info(`Did not restore ${artifactType} with key ${cacheKey} to ${pattern}`)
@ -217,10 +214,7 @@ abstract class AbstractEntryExtractor {
cacheDebug(`No change to previously restored ${artifactType}. Not saving.`) cacheDebug(`No change to previously restored ${artifactType}. Not saving.`)
} else { } else {
core.info(`Caching ${artifactType} with path '${pattern}' and cache key: ${cacheKey}`) core.info(`Caching ${artifactType} with path '${pattern}' and cache key: ${cacheKey}`)
const savedEntry = await saveCache([pattern], cacheKey) await saveCache([pattern], cacheKey, entryListener)
if (savedEntry !== undefined) {
entryListener.markSaved(savedEntry.key, savedEntry.size)
}
} }
for (const file of matchingFiles) { for (const file of matchingFiles) {

View file

@ -28,6 +28,9 @@ export class CacheListener {
} }
static rehydrate(stringRep: string): CacheListener { static rehydrate(stringRep: string): CacheListener {
if (stringRep === '') {
return new CacheListener()
}
const rehydrated: CacheListener = Object.assign(new CacheListener(), JSON.parse(stringRep)) const rehydrated: CacheListener = Object.assign(new CacheListener(), JSON.parse(stringRep))
const entries = rehydrated.cacheEntries const entries = rehydrated.cacheEntries
for (let index = 0; index < entries.length; index++) { for (let index = 0; index < entries.length; index++) {
@ -76,6 +79,12 @@ export class CacheEntryListener {
this.savedSize = size this.savedSize = size
return this return this
} }
markAlreadyExists(key: string): CacheEntryListener {
this.savedKey = key
this.savedSize = 0
return this
}
} }
export function logCachingReport(listener: CacheListener): void { export function logCachingReport(listener: CacheListener): void {
@ -112,12 +121,18 @@ function getSum(
cacheEntries: CacheEntryListener[], cacheEntries: CacheEntryListener[],
predicate: (value: CacheEntryListener) => number | undefined predicate: (value: CacheEntryListener) => number | undefined
): string { ): string {
if (cacheEntries.length === 0) {
return '0'
}
return formatSize(cacheEntries.map(e => predicate(e) ?? 0).reduce((p, v) => p + v, 0)) return formatSize(cacheEntries.map(e => predicate(e) ?? 0).reduce((p, v) => p + v, 0))
} }
function formatSize(bytes: number | undefined): string { function formatSize(bytes: number | undefined): string {
if (bytes === undefined || bytes === 0) { if (bytes === undefined) {
return '' return ''
} }
if (bytes === 0) {
return '0 (Entry already exists)'
}
return `${Math.round(bytes / (1024 * 1024))} MB (${bytes} B)` return `${Math.round(bytes / (1024 * 1024))} MB (${bytes} B)`
} }

View file

@ -4,9 +4,12 @@ import * as crypto from 'crypto'
import * as path from 'path' import * as path from 'path'
import * as fs from 'fs' import * as fs from 'fs'
import {CacheEntryListener} from './cache-reporting'
const JOB_CONTEXT_PARAMETER = 'workflow-job-context' const JOB_CONTEXT_PARAMETER = 'workflow-job-context'
const CACHE_DISABLED_PARAMETER = 'cache-disabled' const CACHE_DISABLED_PARAMETER = 'cache-disabled'
const CACHE_READONLY_PARAMETER = 'cache-read-only' const CACHE_READONLY_PARAMETER = 'cache-read-only'
const CACHE_WRITEONLY_PARAMETER = 'cache-write-only'
const CACHE_DEBUG_VAR = 'GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED' const CACHE_DEBUG_VAR = 'GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED'
const CACHE_PREFIX_VAR = 'GRADLE_BUILD_ACTION_CACHE_KEY_PREFIX' const CACHE_PREFIX_VAR = 'GRADLE_BUILD_ACTION_CACHE_KEY_PREFIX'
@ -18,6 +21,10 @@ export function isCacheReadOnly(): boolean {
return core.getBooleanInput(CACHE_READONLY_PARAMETER) return core.getBooleanInput(CACHE_READONLY_PARAMETER)
} }
export function isCacheWriteOnly(): boolean {
return core.getBooleanInput(CACHE_WRITEONLY_PARAMETER)
}
export function isCacheDebuggingEnabled(): boolean { export function isCacheDebuggingEnabled(): boolean {
return process.env[CACHE_DEBUG_VAR] ? true : false return process.env[CACHE_DEBUG_VAR] ? true : false
} }
@ -49,23 +56,32 @@ export function hashStrings(values: string[]): string {
export async function restoreCache( export async function restoreCache(
cachePath: string[], cachePath: string[],
cacheKey: string, cacheKey: string,
cacheRestoreKeys: string[] = [] cacheRestoreKeys: string[],
listener: CacheEntryListener
): Promise<cache.CacheEntry | undefined> { ): Promise<cache.CacheEntry | undefined> {
listener.markRequested(cacheKey, cacheRestoreKeys)
try { try {
return await cache.restoreCache(cachePath, cacheKey, cacheRestoreKeys) const restoredEntry = await cache.restoreCache(cachePath, cacheKey, cacheRestoreKeys)
if (restoredEntry !== undefined) {
listener.markRestored(restoredEntry.key, restoredEntry.size)
}
return restoredEntry
} catch (error) { } catch (error) {
handleCacheFailure(error, `Failed to restore ${cacheKey}`) handleCacheFailure(error, `Failed to restore ${cacheKey}`)
return undefined return undefined
} }
} }
export async function saveCache(cachePath: string[], cacheKey: string): Promise<cache.CacheEntry | undefined> { export async function saveCache(cachePath: string[], cacheKey: string, listener: CacheEntryListener): Promise<void> {
try { try {
return await cache.saveCache(cachePath, cacheKey) const savedEntry = await cache.saveCache(cachePath, cacheKey)
listener.markSaved(savedEntry.key, savedEntry.size)
} catch (error) { } catch (error) {
if (error instanceof cache.ReserveCacheError) {
listener.markAlreadyExists(cacheKey)
}
handleCacheFailure(error, `Failed to save cache entry ${cacheKey}`) handleCacheFailure(error, `Failed to save cache entry ${cacheKey}`)
} }
return undefined
} }
export function cacheDebug(message: string): void { export function cacheDebug(message: string): void {
@ -83,11 +99,7 @@ export function handleCacheFailure(error: unknown, message: string): void {
} }
if (error instanceof cache.ReserveCacheError) { if (error instanceof cache.ReserveCacheError) {
// Reserve cache errors are expected if the artifact has been previously cached // Reserve cache errors are expected if the artifact has been previously cached
if (isCacheDebuggingEnabled()) { core.info(`${message}: ${error}`)
core.info(`${message}: ${error}`)
} else {
core.debug(`${message}: ${error}`)
}
} else { } else {
// Warn on all other errors // Warn on all other errors
core.warning(`${message}: ${error}`) core.warning(`${message}: ${error}`)

View file

@ -1,5 +1,5 @@
import * as core from '@actions/core' import * as core from '@actions/core'
import {isCacheDisabled, isCacheReadOnly} from './cache-utils' import {isCacheDisabled, isCacheReadOnly, isCacheWriteOnly} from './cache-utils'
import {logCachingReport, CacheListener} from './cache-reporting' import {logCachingReport, CacheListener} from './cache-reporting'
import {GradleStateCache} from './cache-base' import {GradleStateCache} from './cache-base'
@ -32,18 +32,22 @@ export async function restore(gradleUserHome: string): Promise<void> {
} }
gradleStateCache.init() gradleStateCache.init()
// Mark the state as restored so that post-action will perform save.
core.saveState(CACHE_RESTORED_VAR, true)
// Save the Gradle User Home for the post-action step.
core.saveState(GRADLE_USER_HOME, gradleUserHome)
if (isCacheWriteOnly()) {
core.info('Cache is write-only: will not restore from cache.')
return
}
await core.group('Restore Gradle state from cache', async () => { await core.group('Restore Gradle state from cache', async () => {
core.saveState(GRADLE_USER_HOME, gradleUserHome)
const cacheListener = new CacheListener() const cacheListener = new CacheListener()
await gradleStateCache.restore(cacheListener) await gradleStateCache.restore(cacheListener)
core.saveState(CACHE_LISTENER, cacheListener.stringify()) core.saveState(CACHE_LISTENER, cacheListener.stringify())
}) })
// Export state that is detected in corresponding post-action step
core.saveState(CACHE_RESTORED_VAR, true)
} }
export async function save(): Promise<void> { export async function save(): Promise<void> {