Monitor cache saves and add basic caching report

- Restore `CachingReport` instance in 'post' action
- Record keys for any entries saved
- Report caching activity as JSON in post action
This commit is contained in:
Daz DeBoer 2021-10-29 10:41:30 -06:00
parent 6ff2065a12
commit a74bb0fad6
No known key found for this signature in database
GPG key ID: DD6B9F0B06683D5D
4 changed files with 155 additions and 20 deletions

View file

@ -1,5 +1,4 @@
import * as cacheUtils from '../src/cache-utils' import * as cacheUtils from '../src/cache-utils'
import * as path from 'path'
describe('cacheUtils-utils', () => { describe('cacheUtils-utils', () => {
describe('can hash', () => { describe('can hash', () => {
@ -18,4 +17,97 @@ describe('cacheUtils-utils', () => {
expect(posixHash).toBe(windowsHash) expect(posixHash).toBe(windowsHash)
}) })
}) })
describe('caching report', () => {
describe('reports not fully restored', () => {
it('with one requested entry report', async () => {
const report = new cacheUtils.CachingReport()
report.entryReport('foo').markRequested('1', ['2'])
report.entryReport('bar').markRequested('3').markRestored('4')
expect(report.fullyRestored).toBe(false)
})
})
describe('reports fully restored', () => {
it('when empty', async () => {
const report = new cacheUtils.CachingReport()
expect(report.fullyRestored).toBe(true)
})
it('with empty entry reports', async () => {
const report = new cacheUtils.CachingReport()
report.entryReport('foo')
report.entryReport('bar')
expect(report.fullyRestored).toBe(true)
})
it('with restored entry report', async () => {
const report = new cacheUtils.CachingReport()
report.entryReport('bar').markRequested('3').markRestored('4')
expect(report.fullyRestored).toBe(true)
})
it('with multiple restored entry reportss', async () => {
const report = new cacheUtils.CachingReport()
report.entryReport('foo').markRestored('4')
report.entryReport('bar').markRequested('3').markRestored('4')
expect(report.fullyRestored).toBe(true)
})
})
describe('can be stringified and rehydrated', () => {
it('when empty', async () => {
const report = new cacheUtils.CachingReport()
const stringRep = report.stringify()
const reportClone: cacheUtils.CachingReport = cacheUtils.CachingReport.rehydrate(stringRep)
expect(reportClone.cacheEntryReports).toEqual([])
// Can call methods on rehydrated
expect(reportClone.entryReport('foo')).toBeInstanceOf(cacheUtils.CacheEntryReport)
})
it('with entry reports', async () => {
const report = new cacheUtils.CachingReport()
report.entryReport('foo')
report.entryReport('bar')
report.entryReport('baz')
const stringRep = report.stringify()
const reportClone: cacheUtils.CachingReport = cacheUtils.CachingReport.rehydrate(stringRep)
expect(reportClone.cacheEntryReports.length).toBe(3)
expect(reportClone.cacheEntryReports[0].entryName).toBe('foo')
expect(reportClone.cacheEntryReports[1].entryName).toBe('bar')
expect(reportClone.cacheEntryReports[2].entryName).toBe('baz')
expect(reportClone.entryReport('foo')).toBe(reportClone.cacheEntryReports[0])
})
it('with rehydrated entry report', async () => {
const report = new cacheUtils.CachingReport()
const entryReport = report.entryReport('foo')
entryReport.markRequested('1', ['2', '3'])
entryReport.markSaved('4')
const stringRep = report.stringify()
const reportClone: cacheUtils.CachingReport = cacheUtils.CachingReport.rehydrate(stringRep)
const entryClone = reportClone.entryReport('foo')
expect(entryClone.requestedKey).toBe('1')
expect(entryClone.requestedRestoreKeys).toEqual(['2', '3'])
expect(entryClone.savedKey).toBe('4')
})
it('with live entry report', async () => {
const report = new cacheUtils.CachingReport()
const entryReport = report.entryReport('foo')
entryReport.markRequested('1', ['2', '3'])
const stringRep = report.stringify()
const reportClone: cacheUtils.CachingReport = cacheUtils.CachingReport.rehydrate(stringRep)
const entryClone = reportClone.entryReport('foo')
// Check type and call method on rehydrated entry report
expect(entryClone).toBeInstanceOf(cacheUtils.CacheEntryReport)
entryClone.markSaved('4')
expect(entryClone.requestedKey).toBe('1')
expect(entryClone.requestedRestoreKeys).toEqual(['2', '3'])
expect(entryClone.savedKey).toBe('4')
})
})
})
}) })

View file

@ -43,7 +43,7 @@ export class GradleUserHomeCache extends AbstractCache {
// Iterate over all bundle meta files and try to restore // Iterate over all bundle meta files and try to restore
for (const bundleMetaFile of bundleMetaFiles) { for (const bundleMetaFile of bundleMetaFiles) {
const bundle = path.basename(bundleMetaFile, '.cache') const bundle = path.basename(bundleMetaFile, '.cache')
const bundleEntryReport = report.addEntryReport(bundle) const bundleEntryReport = report.entryReport(bundle)
const bundlePattern = bundlePatterns.get(bundle) const bundlePattern = bundlePatterns.get(bundle)
// Handle case where the 'artifactBundlePatterns' have been changed // Handle case where the 'artifactBundlePatterns' have been changed
@ -94,10 +94,10 @@ export class GradleUserHomeCache extends AbstractCache {
return bundleFiles return bundleFiles
} }
async beforeSave(): Promise<void> { async beforeSave(report: CachingReport): Promise<void> {
await this.reportGradleUserHomeSize('before saving common artifacts') await this.reportGradleUserHomeSize('before saving common artifacts')
this.removeExcludedPaths() this.removeExcludedPaths()
await this.saveArtifactBundles() await this.saveArtifactBundles(report)
await this.reportGradleUserHomeSize( await this.reportGradleUserHomeSize(
"after saving common artifacts (only 'caches' and 'notifications' will be stored)" "after saving common artifacts (only 'caches' and 'notifications' will be stored)"
) )
@ -113,10 +113,12 @@ export class GradleUserHomeCache extends AbstractCache {
} }
} }
private async saveArtifactBundles(): Promise<void> { private async saveArtifactBundles(report: CachingReport): Promise<void> {
const processes: Promise<void>[] = [] const processes: Promise<void>[] = []
for (const [bundle, pattern] of this.getArtifactBundles()) { for (const [bundle, pattern] of this.getArtifactBundles()) {
const p = this.saveArtifactBundle(bundle, pattern) const bundleEntryReport = report.entryReport(bundle)
const p = this.saveArtifactBundle(bundle, pattern, bundleEntryReport)
// Run sequentially when debugging enabled // Run sequentially when debugging enabled
if (this.cacheDebuggingEnabled) { if (this.cacheDebuggingEnabled) {
await p await p
@ -127,7 +129,7 @@ export class GradleUserHomeCache extends AbstractCache {
await Promise.all(processes) await Promise.all(processes)
} }
private async saveArtifactBundle(bundle: string, artifactPath: string): Promise<void> { private async saveArtifactBundle(bundle: string, artifactPath: string, report: CacheEntryReport): Promise<void> {
const bundleMetaFile = this.getBundleMetaFile(bundle) const bundleMetaFile = this.getBundleMetaFile(bundle)
const globber = await glob.create(artifactPath, { const globber = await glob.create(artifactPath, {
@ -156,6 +158,7 @@ export class GradleUserHomeCache extends AbstractCache {
core.info(`Caching ${bundle} with cache key: ${cacheKey}`) core.info(`Caching ${bundle} with cache key: ${cacheKey}`)
await this.saveCache([artifactPath], cacheKey) await this.saveCache([artifactPath], cacheKey)
this.writeBundleMetaFile(bundleMetaFile, cacheKey) this.writeBundleMetaFile(bundleMetaFile, cacheKey)
report.markSaved(cacheKey)
} }
for (const file of bundleFiles) { for (const file of bundleFiles) {

View file

@ -112,13 +112,33 @@ export class CachingReport {
return this.cacheEntryReports.every(x => !x.wasRequestedButNotRestored()) return this.cacheEntryReports.every(x => !x.wasRequestedButNotRestored())
} }
addEntryReport(name: string): CacheEntryReport { entryReport(name: string): CacheEntryReport {
const report = new CacheEntryReport(name) for (const report of this.cacheEntryReports) {
this.cacheEntryReports.push(report) if (report.entryName === name) {
return report return report
} }
} }
const newReport = new CacheEntryReport(name)
this.cacheEntryReports.push(newReport)
return newReport
}
stringify(): string {
return JSON.stringify(this)
}
static rehydrate(stringRep: string): CachingReport {
const rehydrated: CachingReport = Object.assign(new CachingReport(), JSON.parse(stringRep))
const entryReports = rehydrated.cacheEntryReports
for (let index = 0; index < entryReports.length; index++) {
const rawEntryReport = entryReports[index]
entryReports[index] = Object.assign(new CacheEntryReport(rawEntryReport.entryName), rawEntryReport)
}
return rehydrated
}
}
export class CacheEntryReport { export class CacheEntryReport {
entryName: string entryName: string
requestedKey: string | undefined requestedKey: string | undefined
@ -137,13 +157,20 @@ export class CacheEntryReport {
return this.requestedKey !== undefined && this.restoredKey === undefined return this.requestedKey !== undefined && this.restoredKey === undefined
} }
markRequested(key: string, restoreKeys: string[] = []): void { markRequested(key: string, restoreKeys: string[] = []): CacheEntryReport {
this.requestedKey = key this.requestedKey = key
this.requestedRestoreKeys = restoreKeys this.requestedRestoreKeys = restoreKeys
return this
} }
markRestored(key: string): void { markRestored(key: string): CacheEntryReport {
this.restoredKey = key this.restoredKey = key
return this
}
markSaved(key: string): CacheEntryReport {
this.savedKey = key
return this
} }
} }
@ -170,7 +197,7 @@ export abstract class AbstractCache {
} }
const cacheKey = this.prepareCacheKey() const cacheKey = this.prepareCacheKey()
const entryReport = report.addEntryReport(this.cacheName) const entryReport = report.entryReport(this.cacheName)
entryReport.markRequested(cacheKey.key, cacheKey.restoreKeys) entryReport.markRequested(cacheKey.key, cacheKey.restoreKeys)
this.debug( this.debug(
@ -224,7 +251,7 @@ export abstract class AbstractCache {
protected async afterRestore(_report: CachingReport): Promise<void> {} protected async afterRestore(_report: CachingReport): Promise<void> {}
async save(): Promise<void> { async save(report: CachingReport): Promise<void> {
if (!this.cacheOutputExists()) { if (!this.cacheOutputExists()) {
this.debug(`No ${this.cacheDescription} to cache.`) this.debug(`No ${this.cacheDescription} to cache.`)
return return
@ -244,7 +271,7 @@ export abstract class AbstractCache {
} }
try { try {
await this.beforeSave() await this.beforeSave(report)
} catch (error) { } catch (error) {
core.warning(`Save ${this.cacheDescription} failed in 'beforeSave': ${error}`) core.warning(`Save ${this.cacheDescription} failed in 'beforeSave': ${error}`)
return return
@ -254,10 +281,12 @@ export abstract class AbstractCache {
const cachePath = this.getCachePath() const cachePath = this.getCachePath()
await this.saveCache(cachePath, cacheKey) await this.saveCache(cachePath, cacheKey)
report.entryReport(this.cacheName).markSaved(cacheKey)
return return
} }
protected async beforeSave(): Promise<void> {} protected async beforeSave(_report: CachingReport): Promise<void> {}
protected async saveCache(cachePath: string[], cacheKey: string): Promise<void> { protected async saveCache(cachePath: string[], cacheKey: string): Promise<void> {
try { try {

View file

@ -4,6 +4,7 @@ import * as core from '@actions/core'
import {CachingReport, isCacheDisabled, isCacheReadOnly} from './cache-utils' import {CachingReport, isCacheDisabled, isCacheReadOnly} from './cache-utils'
const BUILD_ROOT_DIR = 'BUILD_ROOT_DIR' const BUILD_ROOT_DIR = 'BUILD_ROOT_DIR'
const CACHING_REPORT = 'CACHING_REPORT'
export async function restore(buildRootDirectory: string): Promise<void> { export async function restore(buildRootDirectory: string): Promise<void> {
if (isCacheDisabled()) { if (isCacheDisabled()) {
@ -15,7 +16,6 @@ export async function restore(buildRootDirectory: string): Promise<void> {
core.saveState(BUILD_ROOT_DIR, buildRootDirectory) core.saveState(BUILD_ROOT_DIR, buildRootDirectory)
const cachingReport = new CachingReport() const cachingReport = new CachingReport()
await new GradleUserHomeCache(buildRootDirectory).restore(cachingReport) await new GradleUserHomeCache(buildRootDirectory).restore(cachingReport)
const projectDotGradleCache = new ProjectDotGradleCache(buildRootDirectory) const projectDotGradleCache = new ProjectDotGradleCache(buildRootDirectory)
@ -24,8 +24,11 @@ export async function restore(buildRootDirectory: string): Promise<void> {
await projectDotGradleCache.restore(cachingReport) await projectDotGradleCache.restore(cachingReport)
} else { } else {
// Otherwise, prepare the cache key for later save() // Otherwise, prepare the cache key for later save()
core.info('Gradle Home cache not fully restored: not restoring configuration-cache state')
projectDotGradleCache.prepareCacheKey() projectDotGradleCache.prepareCacheKey()
} }
core.saveState(CACHING_REPORT, cachingReport.stringify())
}) })
} }
@ -35,11 +38,19 @@ export async function save(): Promise<void> {
return return
} }
const cachingReport: CachingReport = CachingReport.rehydrate(core.getState(CACHING_REPORT))
await core.group('Caching Gradle state', async () => { await core.group('Caching Gradle state', async () => {
const buildRootDirectory = core.getState(BUILD_ROOT_DIR) const buildRootDirectory = core.getState(BUILD_ROOT_DIR)
return Promise.all([ return Promise.all([
new GradleUserHomeCache(buildRootDirectory).save(), new GradleUserHomeCache(buildRootDirectory).save(cachingReport),
new ProjectDotGradleCache(buildRootDirectory).save() new ProjectDotGradleCache(buildRootDirectory).save(cachingReport)
]) ])
}) })
logCachingReport(cachingReport)
}
function logCachingReport(report: CachingReport): void {
core.info(JSON.stringify(report, null, 2))
} }