Monitor cache saves and add basic caching report

- Restore `CachingReport` instance in 'post' action
- Record keys for any entries saved
- Report caching activity as JSON in post action
This commit is contained in:
Daz DeBoer 2021-10-29 10:41:30 -06:00
parent 6ff2065a12
commit a74bb0fad6
No known key found for this signature in database
GPG key ID: DD6B9F0B06683D5D
4 changed files with 155 additions and 20 deletions

View file

@ -1,5 +1,4 @@
import * as cacheUtils from '../src/cache-utils'
import * as path from 'path'
describe('cacheUtils-utils', () => {
describe('can hash', () => {
@ -18,4 +17,97 @@ describe('cacheUtils-utils', () => {
expect(posixHash).toBe(windowsHash)
})
})
describe('caching report', () => {
describe('reports not fully restored', () => {
it('with one requested entry report', async () => {
const report = new cacheUtils.CachingReport()
report.entryReport('foo').markRequested('1', ['2'])
report.entryReport('bar').markRequested('3').markRestored('4')
expect(report.fullyRestored).toBe(false)
})
})
describe('reports fully restored', () => {
it('when empty', async () => {
const report = new cacheUtils.CachingReport()
expect(report.fullyRestored).toBe(true)
})
it('with empty entry reports', async () => {
const report = new cacheUtils.CachingReport()
report.entryReport('foo')
report.entryReport('bar')
expect(report.fullyRestored).toBe(true)
})
it('with restored entry report', async () => {
const report = new cacheUtils.CachingReport()
report.entryReport('bar').markRequested('3').markRestored('4')
expect(report.fullyRestored).toBe(true)
})
it('with multiple restored entry reportss', async () => {
const report = new cacheUtils.CachingReport()
report.entryReport('foo').markRestored('4')
report.entryReport('bar').markRequested('3').markRestored('4')
expect(report.fullyRestored).toBe(true)
})
})
describe('can be stringified and rehydrated', () => {
it('when empty', async () => {
const report = new cacheUtils.CachingReport()
const stringRep = report.stringify()
const reportClone: cacheUtils.CachingReport = cacheUtils.CachingReport.rehydrate(stringRep)
expect(reportClone.cacheEntryReports).toEqual([])
// Can call methods on rehydrated
expect(reportClone.entryReport('foo')).toBeInstanceOf(cacheUtils.CacheEntryReport)
})
it('with entry reports', async () => {
const report = new cacheUtils.CachingReport()
report.entryReport('foo')
report.entryReport('bar')
report.entryReport('baz')
const stringRep = report.stringify()
const reportClone: cacheUtils.CachingReport = cacheUtils.CachingReport.rehydrate(stringRep)
expect(reportClone.cacheEntryReports.length).toBe(3)
expect(reportClone.cacheEntryReports[0].entryName).toBe('foo')
expect(reportClone.cacheEntryReports[1].entryName).toBe('bar')
expect(reportClone.cacheEntryReports[2].entryName).toBe('baz')
expect(reportClone.entryReport('foo')).toBe(reportClone.cacheEntryReports[0])
})
it('with rehydrated entry report', async () => {
const report = new cacheUtils.CachingReport()
const entryReport = report.entryReport('foo')
entryReport.markRequested('1', ['2', '3'])
entryReport.markSaved('4')
const stringRep = report.stringify()
const reportClone: cacheUtils.CachingReport = cacheUtils.CachingReport.rehydrate(stringRep)
const entryClone = reportClone.entryReport('foo')
expect(entryClone.requestedKey).toBe('1')
expect(entryClone.requestedRestoreKeys).toEqual(['2', '3'])
expect(entryClone.savedKey).toBe('4')
})
it('with live entry report', async () => {
const report = new cacheUtils.CachingReport()
const entryReport = report.entryReport('foo')
entryReport.markRequested('1', ['2', '3'])
const stringRep = report.stringify()
const reportClone: cacheUtils.CachingReport = cacheUtils.CachingReport.rehydrate(stringRep)
const entryClone = reportClone.entryReport('foo')
// Check type and call method on rehydrated entry report
expect(entryClone).toBeInstanceOf(cacheUtils.CacheEntryReport)
entryClone.markSaved('4')
expect(entryClone.requestedKey).toBe('1')
expect(entryClone.requestedRestoreKeys).toEqual(['2', '3'])
expect(entryClone.savedKey).toBe('4')
})
})
})
})

View file

@ -43,7 +43,7 @@ export class GradleUserHomeCache extends AbstractCache {
// Iterate over all bundle meta files and try to restore
for (const bundleMetaFile of bundleMetaFiles) {
const bundle = path.basename(bundleMetaFile, '.cache')
const bundleEntryReport = report.addEntryReport(bundle)
const bundleEntryReport = report.entryReport(bundle)
const bundlePattern = bundlePatterns.get(bundle)
// Handle case where the 'artifactBundlePatterns' have been changed
@ -94,10 +94,10 @@ export class GradleUserHomeCache extends AbstractCache {
return bundleFiles
}
async beforeSave(): Promise<void> {
async beforeSave(report: CachingReport): Promise<void> {
await this.reportGradleUserHomeSize('before saving common artifacts')
this.removeExcludedPaths()
await this.saveArtifactBundles()
await this.saveArtifactBundles(report)
await this.reportGradleUserHomeSize(
"after saving common artifacts (only 'caches' and 'notifications' will be stored)"
)
@ -113,10 +113,12 @@ export class GradleUserHomeCache extends AbstractCache {
}
}
private async saveArtifactBundles(): Promise<void> {
private async saveArtifactBundles(report: CachingReport): Promise<void> {
const processes: Promise<void>[] = []
for (const [bundle, pattern] of this.getArtifactBundles()) {
const p = this.saveArtifactBundle(bundle, pattern)
const bundleEntryReport = report.entryReport(bundle)
const p = this.saveArtifactBundle(bundle, pattern, bundleEntryReport)
// Run sequentially when debugging enabled
if (this.cacheDebuggingEnabled) {
await p
@ -127,7 +129,7 @@ export class GradleUserHomeCache extends AbstractCache {
await Promise.all(processes)
}
private async saveArtifactBundle(bundle: string, artifactPath: string): Promise<void> {
private async saveArtifactBundle(bundle: string, artifactPath: string, report: CacheEntryReport): Promise<void> {
const bundleMetaFile = this.getBundleMetaFile(bundle)
const globber = await glob.create(artifactPath, {
@ -156,6 +158,7 @@ export class GradleUserHomeCache extends AbstractCache {
core.info(`Caching ${bundle} with cache key: ${cacheKey}`)
await this.saveCache([artifactPath], cacheKey)
this.writeBundleMetaFile(bundleMetaFile, cacheKey)
report.markSaved(cacheKey)
}
for (const file of bundleFiles) {

View file

@ -112,13 +112,33 @@ export class CachingReport {
return this.cacheEntryReports.every(x => !x.wasRequestedButNotRestored())
}
addEntryReport(name: string): CacheEntryReport {
const report = new CacheEntryReport(name)
this.cacheEntryReports.push(report)
entryReport(name: string): CacheEntryReport {
for (const report of this.cacheEntryReports) {
if (report.entryName === name) {
return report
}
}
const newReport = new CacheEntryReport(name)
this.cacheEntryReports.push(newReport)
return newReport
}
stringify(): string {
return JSON.stringify(this)
}
static rehydrate(stringRep: string): CachingReport {
const rehydrated: CachingReport = Object.assign(new CachingReport(), JSON.parse(stringRep))
const entryReports = rehydrated.cacheEntryReports
for (let index = 0; index < entryReports.length; index++) {
const rawEntryReport = entryReports[index]
entryReports[index] = Object.assign(new CacheEntryReport(rawEntryReport.entryName), rawEntryReport)
}
return rehydrated
}
}
export class CacheEntryReport {
entryName: string
requestedKey: string | undefined
@ -137,13 +157,20 @@ export class CacheEntryReport {
return this.requestedKey !== undefined && this.restoredKey === undefined
}
markRequested(key: string, restoreKeys: string[] = []): void {
markRequested(key: string, restoreKeys: string[] = []): CacheEntryReport {
this.requestedKey = key
this.requestedRestoreKeys = restoreKeys
return this
}
markRestored(key: string): void {
markRestored(key: string): CacheEntryReport {
this.restoredKey = key
return this
}
markSaved(key: string): CacheEntryReport {
this.savedKey = key
return this
}
}
@ -170,7 +197,7 @@ export abstract class AbstractCache {
}
const cacheKey = this.prepareCacheKey()
const entryReport = report.addEntryReport(this.cacheName)
const entryReport = report.entryReport(this.cacheName)
entryReport.markRequested(cacheKey.key, cacheKey.restoreKeys)
this.debug(
@ -224,7 +251,7 @@ export abstract class AbstractCache {
protected async afterRestore(_report: CachingReport): Promise<void> {}
async save(): Promise<void> {
async save(report: CachingReport): Promise<void> {
if (!this.cacheOutputExists()) {
this.debug(`No ${this.cacheDescription} to cache.`)
return
@ -244,7 +271,7 @@ export abstract class AbstractCache {
}
try {
await this.beforeSave()
await this.beforeSave(report)
} catch (error) {
core.warning(`Save ${this.cacheDescription} failed in 'beforeSave': ${error}`)
return
@ -254,10 +281,12 @@ export abstract class AbstractCache {
const cachePath = this.getCachePath()
await this.saveCache(cachePath, cacheKey)
report.entryReport(this.cacheName).markSaved(cacheKey)
return
}
protected async beforeSave(): Promise<void> {}
protected async beforeSave(_report: CachingReport): Promise<void> {}
protected async saveCache(cachePath: string[], cacheKey: string): Promise<void> {
try {

View file

@ -4,6 +4,7 @@ import * as core from '@actions/core'
import {CachingReport, isCacheDisabled, isCacheReadOnly} from './cache-utils'
const BUILD_ROOT_DIR = 'BUILD_ROOT_DIR'
const CACHING_REPORT = 'CACHING_REPORT'
export async function restore(buildRootDirectory: string): Promise<void> {
if (isCacheDisabled()) {
@ -15,7 +16,6 @@ export async function restore(buildRootDirectory: string): Promise<void> {
core.saveState(BUILD_ROOT_DIR, buildRootDirectory)
const cachingReport = new CachingReport()
await new GradleUserHomeCache(buildRootDirectory).restore(cachingReport)
const projectDotGradleCache = new ProjectDotGradleCache(buildRootDirectory)
@ -24,8 +24,11 @@ export async function restore(buildRootDirectory: string): Promise<void> {
await projectDotGradleCache.restore(cachingReport)
} else {
// Otherwise, prepare the cache key for later save()
core.info('Gradle Home cache not fully restored: not restoring configuration-cache state')
projectDotGradleCache.prepareCacheKey()
}
core.saveState(CACHING_REPORT, cachingReport.stringify())
})
}
@ -35,11 +38,19 @@ export async function save(): Promise<void> {
return
}
const cachingReport: CachingReport = CachingReport.rehydrate(core.getState(CACHING_REPORT))
await core.group('Caching Gradle state', async () => {
const buildRootDirectory = core.getState(BUILD_ROOT_DIR)
return Promise.all([
new GradleUserHomeCache(buildRootDirectory).save(),
new ProjectDotGradleCache(buildRootDirectory).save()
new GradleUserHomeCache(buildRootDirectory).save(cachingReport),
new ProjectDotGradleCache(buildRootDirectory).save(cachingReport)
])
})
logCachingReport(cachingReport)
}
function logCachingReport(report: CachingReport): void {
core.info(JSON.stringify(report, null, 2))
}