gradle/gradle-build-action
1
0
Fork 0
mirror of https://github.com/gradle/gradle-build-action.git synced 2024-11-26 11:02:10 +00:00
Code Issues Projects Releases Packages Wiki Activity

Tidy up cache entry de-duplication

Browse source 
- Rename feature to "common artifact caching"
- Cleanup logging
- Refactor/rename for clarity
This commit is contained in:
Daz DeBoer 2021-09-12 10:08:34 -06:00
parent 5a90152b1f
commit cae99bf6d9
No known key found for this signature in database
GPG key ID: DD6B9F0B06683D5D
3 changed files with 70 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
dist/main/index.js vendored
View file

File diff suppressed because one or more lines are too long

2
dist/post/index.js vendored
View file

File diff suppressed because one or more lines are too long

123
src/cache-gradle-user-home.ts
View file

@ -8,18 +8,24 @@ import * as exec from '@actions/exec'
import {AbstractCache} from './cache-utils' import {AbstractCache} from './cache-utils'
const CACHE_PATH = [ // Paths to artifacts that are common to all/many Gradle User Home caches
'~/.gradle/caches/*', // All directories in 'caches' // These artifacts are cached separately to avoid blowing out the size of each GUH cache
'~/.gradle/notifications/*', // Prevent the re-rendering of first-use message for version const COMMON_ARTIFACT_PATHS = [
'~/.gradle/wrapper/dists/*/*/*.zip.txt' // Only wrapper zips are required : We do not want to cache the exploded distributions
]
const DEDUPLCIATED_PATHS = [
'~/.gradle/wrapper/dists/*/*/*.zip', '~/.gradle/wrapper/dists/*/*/*.zip',
'~/.gradle/caches/*/generated-gradle-jars/*.jar', '~/.gradle/caches/*/generated-gradle-jars/*.jar',
'~/.gradle/caches/modules-*/files-*/**/*.jar' '~/.gradle/caches/modules-*/files-*/**/*.jar'
] ]
const MARKER_FILE_EXTENSION = '.txt'
// When a common artifact is cached separately, it is replaced by a marker file to allow for restore.
const MARKER_FILE_EXTENSION = '.cached'
// Which paths under Gradle User Home should be cached
// TODO: This should adapt for the `GRADLE_USER_HOME` environment variable
const CACHE_PATH = [
'~/.gradle/caches', // All directories in 'caches'
'~/.gradle/notifications', // Prevent the re-rendering of first-use message for version
`~/.gradle/wrapper/dists/*/*/*.zip${MARKER_FILE_EXTENSION}` // Only wrapper zips are required : We do not want to cache the exploded distributions
]
export class GradleUserHomeCache extends AbstractCache { export class GradleUserHomeCache extends AbstractCache {
constructor() { constructor() {
@ -28,95 +34,100 @@ export class GradleUserHomeCache extends AbstractCache {
async restore(): Promise<void> { async restore(): Promise<void> {
await super.restore() await super.restore()
await this.reportCacheEntrySize() await this.reportCacheEntrySize('excluding common artifacts')
await this.restoreDeduplicatedPaths() await this.restoreCommonArtifacts()
await this.reportCacheEntrySize() await this.reportCacheEntrySize('including common artifacts')
} }
private async restoreDeduplicatedPaths(): Promise<void> { private async restoreCommonArtifacts(): Promise<void> {
const markerFilePatterns = DEDUPLCIATED_PATHS.map(targetPath => { const markerFilePatterns = COMMON_ARTIFACT_PATHS.map(targetPath => {
return targetPath + MARKER_FILE_EXTENSION return targetPath + MARKER_FILE_EXTENSION
}).join('\n') }).join('\n')
core.info(`Using marker file patterns: ${markerFilePatterns}`)
const globber = await glob.create(markerFilePatterns) const globber = await glob.create(markerFilePatterns)
const markerFiles = await globber.glob() const markerFiles = await globber.glob()
const processes: Promise<void>[] = [] const processes: Promise<void>[] = []
for (const markerFile of markerFiles) { for (const markerFile of markerFiles) {
const p = this.restoreDeduplicatePath(markerFile) const p = this.restoreCommonArtifact(markerFile)
processes.push(p) processes.push(p)
} }
await Promise.all(processes) await Promise.all(processes)
} }
private async restoreDeduplicatePath(markerFile: string): Promise<void> { private async restoreCommonArtifact(markerFile: string): Promise<void> {
const targetFile = markerFile.substring( const artifactFile = markerFile.substring(
0, 0,
markerFile.length - MARKER_FILE_EXTENSION.length markerFile.length - MARKER_FILE_EXTENSION.length
) )
core.info(`Found marker file: ${markerFile}. Looking for ${targetFile}`) core.debug(
`Found marker file: ${markerFile}. Will attempt to restore ${artifactFile}`
)
if (!fs.existsSync(targetFile)) { if (!fs.existsSync(artifactFile)) {
const key = path.relative(this.getGradleUserHome(), targetFile) const key = path.relative(this.getGradleUserHome(), artifactFile)
const cacheKey = `gradle-dedup-${key}` const cacheKey = `gradle-artifact-${key}`
core.info(`Cache key: ${cacheKey}. Cache path: ${targetFile}`)
const restoreKey = await cache.restoreCache([targetFile], cacheKey) const restoreKey = await cache.restoreCache(
[artifactFile],
cacheKey
)
if (restoreKey) { if (restoreKey) {
core.info(`Restored ${cacheKey} from cache to ${targetFile}`) core.info(`Restored ${cacheKey} from cache to ${artifactFile}`)
} else { } else {
core.info(`Did NOT restore from ${cacheKey} to ${targetFile}`) core.warning(
`Failed to restore from ${cacheKey} to ${artifactFile}`
)
} }
} else { } else {
core.info(`Target file already exists: ${targetFile}`) core.debug(
`Artifact file already exists, not restoring: ${artifactFile}`
)
} }
} }
private async reportCacheEntrySize(): Promise<void> { private async reportCacheEntrySize(label: string): Promise<void> {
const gradleUserHome = path.resolve(os.homedir(), '.gradle') const gradleUserHome = path.resolve(os.homedir(), '.gradle')
if (!fs.existsSync(gradleUserHome)) { if (!fs.existsSync(gradleUserHome)) {
return return
} }
core.info('Gradle User Home cache entry size summary') core.info(`Gradle User Home cache entry: ${label}`)
await exec.exec('du', ['-h', '-c', '-t', '5M'], { await exec.exec('du', ['-h', '-c', '-t', '5M'], {
cwd: gradleUserHome, cwd: gradleUserHome,
ignoreReturnCode: true ignoreReturnCode: true
}) })
core.info('-----------')
} }
async save(): Promise<void> { async save(): Promise<void> {
await this.cacheDeduplicatedPaths() await this.saveCommonArtifacts()
await super.save() await super.save()
} }
private async cacheDeduplicatedPaths(): Promise<void> { private async saveCommonArtifacts(): Promise<void> {
const targetFilePatterns = DEDUPLCIATED_PATHS.join('\n') const artifactFilePatterns = COMMON_ARTIFACT_PATHS.join('\n')
core.info(`Using target file patterns: ${targetFilePatterns}`) const globber = await glob.create(artifactFilePatterns)
const globber = await glob.create(targetFilePatterns) const commonArtifactFiles = await globber.glob()
const targetFiles = await globber.glob()
const processes: Promise<void>[] = [] const processes: Promise<void>[] = []
for (const targetFile of targetFiles) { for (const artifactFile of commonArtifactFiles) {
const p = this.cacheDeplucatePath(targetFile) const p = this.saveCommonArtifact(artifactFile)
processes.push(p) processes.push(p)
} }
await Promise.all(processes) await Promise.all(processes)
} }
private async cacheDeplucatePath(targetFile: string): Promise<void> { private async saveCommonArtifact(artifactFile: string): Promise<void> {
core.info(`Deduplicate caching: ${targetFile}`) const markerFile = `${artifactFile}${MARKER_FILE_EXTENSION}`
const markerFile = `${targetFile}${MARKER_FILE_EXTENSION}`
if (!fs.existsSync(markerFile)) { if (!fs.existsSync(markerFile)) {
const key = path.relative(this.getGradleUserHome(), targetFile) const filePath = path.relative(
const cacheKey = `gradle-dedup-${key}` this.getGradleUserHome(),
core.info(`Cache key: ${cacheKey}. Cache path: ${targetFile}`) artifactFile
)
const cacheKey = `gradle-artifact-${filePath}`
core.info(`Caching ${artifactFile} with cache key: ${cacheKey}`)
try { try {
await cache.saveCache([targetFile], cacheKey) await cache.saveCache([artifactFile], cacheKey)
} catch (error) { } catch (error) {
// Fail on validation errors or non-errors (the latter to keep Typescript happy) // Fail on validation errors or non-errors (the latter to keep Typescript happy)
if ( if (
@ -129,15 +140,21 @@ export class GradleUserHomeCache extends AbstractCache {
core.warning(error.message) core.warning(error.message)
} }
// Write the marker file and delete the original // Write the marker file that will stand in place of the original
fs.writeFileSync(markerFile, 'dummy') fs.writeFileSync(markerFile, 'cached')
} else { } else {
core.info(`Marker file already exists: ${markerFile}`) core.debug(
`Marker file already exists: ${markerFile}. Not caching ${artifactFile}`
)
} }
// TODO : Should not need to delete. Just exclude from cache path. // TODO : Should not need to delete. Just exclude from cache path.
// Delete the target file // Delete the original artifact file
fs.unlinkSync(targetFile) fs.unlinkSync(artifactFile)
}
protected getGradleUserHome(): string {
return path.resolve(os.homedir(), '.gradle')
} }
protected cacheOutputExists(): boolean { protected cacheOutputExists(): boolean {
@ -149,8 +166,4 @@ export class GradleUserHomeCache extends AbstractCache {
protected getCachePath(): string[] { protected getCachePath(): string[] {
return CACHE_PATH return CACHE_PATH
} }
protected getGradleUserHome(): string {
return path.resolve(os.homedir(), '.gradle')
}
} }