From a74bb0fad6355bfd615340794f787f39fc9c3bf0 Mon Sep 17 00:00:00 2001 From: Daz DeBoer Date: Fri, 29 Oct 2021 10:41:30 -0600 Subject: [PATCH] Monitor cache saves and add basic caching report - Restore `CachingReport` instance in 'post' action - Record keys for any entries saved - Report caching activity as JSON in post action --- __tests__/cache-utils.test.ts | 94 ++++++++++++++++++++++++++++++++++- src/cache-gradle-user-home.ts | 15 +++--- src/cache-utils.ts | 49 ++++++++++++++---- src/caches.ts | 17 +++++-- 4 files changed, 155 insertions(+), 20 deletions(-) diff --git a/__tests__/cache-utils.test.ts b/__tests__/cache-utils.test.ts index df921c5..d78d2ef 100644 --- a/__tests__/cache-utils.test.ts +++ b/__tests__/cache-utils.test.ts @@ -1,5 +1,4 @@ import * as cacheUtils from '../src/cache-utils' -import * as path from 'path' describe('cacheUtils-utils', () => { describe('can hash', () => { @@ -18,4 +17,97 @@ describe('cacheUtils-utils', () => { expect(posixHash).toBe(windowsHash) }) }) + describe('caching report', () => { + describe('reports not fully restored', () => { + it('with one requested entry report', async () => { + const report = new cacheUtils.CachingReport() + report.entryReport('foo').markRequested('1', ['2']) + report.entryReport('bar').markRequested('3').markRestored('4') + expect(report.fullyRestored).toBe(false) + }) + }) + describe('reports fully restored', () => { + it('when empty', async () => { + const report = new cacheUtils.CachingReport() + expect(report.fullyRestored).toBe(true) + }) + it('with empty entry reports', async () => { + const report = new cacheUtils.CachingReport() + report.entryReport('foo') + report.entryReport('bar') + expect(report.fullyRestored).toBe(true) + }) + it('with restored entry report', async () => { + const report = new cacheUtils.CachingReport() + report.entryReport('bar').markRequested('3').markRestored('4') + expect(report.fullyRestored).toBe(true) + }) + it('with multiple restored entry reportss', async () => { + const report = new cacheUtils.CachingReport() + report.entryReport('foo').markRestored('4') + report.entryReport('bar').markRequested('3').markRestored('4') + expect(report.fullyRestored).toBe(true) + }) + }) + describe('can be stringified and rehydrated', () => { + it('when empty', async () => { + const report = new cacheUtils.CachingReport() + + const stringRep = report.stringify() + const reportClone: cacheUtils.CachingReport = cacheUtils.CachingReport.rehydrate(stringRep) + + expect(reportClone.cacheEntryReports).toEqual([]) + + // Can call methods on rehydrated + expect(reportClone.entryReport('foo')).toBeInstanceOf(cacheUtils.CacheEntryReport) + }) + it('with entry reports', async () => { + const report = new cacheUtils.CachingReport() + report.entryReport('foo') + report.entryReport('bar') + report.entryReport('baz') + + const stringRep = report.stringify() + const reportClone: cacheUtils.CachingReport = cacheUtils.CachingReport.rehydrate(stringRep) + + expect(reportClone.cacheEntryReports.length).toBe(3) + expect(reportClone.cacheEntryReports[0].entryName).toBe('foo') + expect(reportClone.cacheEntryReports[1].entryName).toBe('bar') + expect(reportClone.cacheEntryReports[2].entryName).toBe('baz') + + expect(reportClone.entryReport('foo')).toBe(reportClone.cacheEntryReports[0]) + }) + it('with rehydrated entry report', async () => { + const report = new cacheUtils.CachingReport() + const entryReport = report.entryReport('foo') + entryReport.markRequested('1', ['2', '3']) + entryReport.markSaved('4') + + const stringRep = report.stringify() + const reportClone: cacheUtils.CachingReport = cacheUtils.CachingReport.rehydrate(stringRep) + const entryClone = reportClone.entryReport('foo') + + expect(entryClone.requestedKey).toBe('1') + expect(entryClone.requestedRestoreKeys).toEqual(['2', '3']) + expect(entryClone.savedKey).toBe('4') + }) + it('with live entry report', async () => { + const report = new cacheUtils.CachingReport() + const entryReport = report.entryReport('foo') + entryReport.markRequested('1', ['2', '3']) + + const stringRep = report.stringify() + const reportClone: cacheUtils.CachingReport = cacheUtils.CachingReport.rehydrate(stringRep) + const entryClone = reportClone.entryReport('foo') + + // Check type and call method on rehydrated entry report + expect(entryClone).toBeInstanceOf(cacheUtils.CacheEntryReport) + entryClone.markSaved('4') + + expect(entryClone.requestedKey).toBe('1') + expect(entryClone.requestedRestoreKeys).toEqual(['2', '3']) + expect(entryClone.savedKey).toBe('4') + }) + }) + }) }) diff --git a/src/cache-gradle-user-home.ts b/src/cache-gradle-user-home.ts index eae04d2..f8134ac 100644 --- a/src/cache-gradle-user-home.ts +++ b/src/cache-gradle-user-home.ts @@ -43,7 +43,7 @@ export class GradleUserHomeCache extends AbstractCache { // Iterate over all bundle meta files and try to restore for (const bundleMetaFile of bundleMetaFiles) { const bundle = path.basename(bundleMetaFile, '.cache') - const bundleEntryReport = report.addEntryReport(bundle) + const bundleEntryReport = report.entryReport(bundle) const bundlePattern = bundlePatterns.get(bundle) // Handle case where the 'artifactBundlePatterns' have been changed @@ -94,10 +94,10 @@ export class GradleUserHomeCache extends AbstractCache { return bundleFiles } - async beforeSave(): Promise { + async beforeSave(report: CachingReport): Promise { await this.reportGradleUserHomeSize('before saving common artifacts') this.removeExcludedPaths() - await this.saveArtifactBundles() + await this.saveArtifactBundles(report) await this.reportGradleUserHomeSize( "after saving common artifacts (only 'caches' and 'notifications' will be stored)" ) @@ -113,10 +113,12 @@ export class GradleUserHomeCache extends AbstractCache { } } - private async saveArtifactBundles(): Promise { + private async saveArtifactBundles(report: CachingReport): Promise { const processes: Promise[] = [] for (const [bundle, pattern] of this.getArtifactBundles()) { - const p = this.saveArtifactBundle(bundle, pattern) + const bundleEntryReport = report.entryReport(bundle) + + const p = this.saveArtifactBundle(bundle, pattern, bundleEntryReport) // Run sequentially when debugging enabled if (this.cacheDebuggingEnabled) { await p @@ -127,7 +129,7 @@ export class GradleUserHomeCache extends AbstractCache { await Promise.all(processes) } - private async saveArtifactBundle(bundle: string, artifactPath: string): Promise { + private async saveArtifactBundle(bundle: string, artifactPath: string, report: CacheEntryReport): Promise { const bundleMetaFile = this.getBundleMetaFile(bundle) const globber = await glob.create(artifactPath, { @@ -156,6 +158,7 @@ export class GradleUserHomeCache extends AbstractCache { core.info(`Caching ${bundle} with cache key: ${cacheKey}`) await this.saveCache([artifactPath], cacheKey) this.writeBundleMetaFile(bundleMetaFile, cacheKey) + report.markSaved(cacheKey) } for (const file of bundleFiles) { diff --git a/src/cache-utils.ts b/src/cache-utils.ts index bc01836..54f4f38 100644 --- a/src/cache-utils.ts +++ b/src/cache-utils.ts @@ -112,10 +112,30 @@ export class CachingReport { return this.cacheEntryReports.every(x => !x.wasRequestedButNotRestored()) } - addEntryReport(name: string): CacheEntryReport { - const report = new CacheEntryReport(name) - this.cacheEntryReports.push(report) - return report + entryReport(name: string): CacheEntryReport { + for (const report of this.cacheEntryReports) { + if (report.entryName === name) { + return report + } + } + + const newReport = new CacheEntryReport(name) + this.cacheEntryReports.push(newReport) + return newReport + } + + stringify(): string { + return JSON.stringify(this) + } + + static rehydrate(stringRep: string): CachingReport { + const rehydrated: CachingReport = Object.assign(new CachingReport(), JSON.parse(stringRep)) + const entryReports = rehydrated.cacheEntryReports + for (let index = 0; index < entryReports.length; index++) { + const rawEntryReport = entryReports[index] + entryReports[index] = Object.assign(new CacheEntryReport(rawEntryReport.entryName), rawEntryReport) + } + return rehydrated } } @@ -137,13 +157,20 @@ export class CacheEntryReport { return this.requestedKey !== undefined && this.restoredKey === undefined } - markRequested(key: string, restoreKeys: string[] = []): void { + markRequested(key: string, restoreKeys: string[] = []): CacheEntryReport { this.requestedKey = key this.requestedRestoreKeys = restoreKeys + return this } - markRestored(key: string): void { + markRestored(key: string): CacheEntryReport { this.restoredKey = key + return this + } + + markSaved(key: string): CacheEntryReport { + this.savedKey = key + return this } } @@ -170,7 +197,7 @@ export abstract class AbstractCache { } const cacheKey = this.prepareCacheKey() - const entryReport = report.addEntryReport(this.cacheName) + const entryReport = report.entryReport(this.cacheName) entryReport.markRequested(cacheKey.key, cacheKey.restoreKeys) this.debug( @@ -224,7 +251,7 @@ export abstract class AbstractCache { protected async afterRestore(_report: CachingReport): Promise {} - async save(): Promise { + async save(report: CachingReport): Promise { if (!this.cacheOutputExists()) { this.debug(`No ${this.cacheDescription} to cache.`) return @@ -244,7 +271,7 @@ export abstract class AbstractCache { } try { - await this.beforeSave() + await this.beforeSave(report) } catch (error) { core.warning(`Save ${this.cacheDescription} failed in 'beforeSave': ${error}`) return @@ -254,10 +281,12 @@ export abstract class AbstractCache { const cachePath = this.getCachePath() await this.saveCache(cachePath, cacheKey) + report.entryReport(this.cacheName).markSaved(cacheKey) + return } - protected async beforeSave(): Promise {} + protected async beforeSave(_report: CachingReport): Promise {} protected async saveCache(cachePath: string[], cacheKey: string): Promise { try { diff --git a/src/caches.ts b/src/caches.ts index bcebc24..eac1ecc 100644 --- a/src/caches.ts +++ b/src/caches.ts @@ -4,6 +4,7 @@ import * as core from '@actions/core' import {CachingReport, isCacheDisabled, isCacheReadOnly} from './cache-utils' const BUILD_ROOT_DIR = 'BUILD_ROOT_DIR' +const CACHING_REPORT = 'CACHING_REPORT' export async function restore(buildRootDirectory: string): Promise { if (isCacheDisabled()) { @@ -15,7 +16,6 @@ export async function restore(buildRootDirectory: string): Promise { core.saveState(BUILD_ROOT_DIR, buildRootDirectory) const cachingReport = new CachingReport() - await new GradleUserHomeCache(buildRootDirectory).restore(cachingReport) const projectDotGradleCache = new ProjectDotGradleCache(buildRootDirectory) @@ -24,8 +24,11 @@ export async function restore(buildRootDirectory: string): Promise { await projectDotGradleCache.restore(cachingReport) } else { // Otherwise, prepare the cache key for later save() + core.info('Gradle Home cache not fully restored: not restoring configuration-cache state') projectDotGradleCache.prepareCacheKey() } + + core.saveState(CACHING_REPORT, cachingReport.stringify()) }) } @@ -35,11 +38,19 @@ export async function save(): Promise { return } + const cachingReport: CachingReport = CachingReport.rehydrate(core.getState(CACHING_REPORT)) + await core.group('Caching Gradle state', async () => { const buildRootDirectory = core.getState(BUILD_ROOT_DIR) return Promise.all([ - new GradleUserHomeCache(buildRootDirectory).save(), - new ProjectDotGradleCache(buildRootDirectory).save() + new GradleUserHomeCache(buildRootDirectory).save(cachingReport), + new ProjectDotGradleCache(buildRootDirectory).save(cachingReport) ]) }) + + logCachingReport(cachingReport) +} + +function logCachingReport(report: CachingReport): void { + core.info(JSON.stringify(report, null, 2)) }