Merge pull request #97 from gradle/dd/configurable-cache

Add some internal options for fine-tuning the gradle-build-action cache

- Can specify cache-paths to override the default directories cached from Gradle User Home
- Can specify cache-exclude-paths to exclude files from the Gradle User Home cache
- Files under caches/<gradle-version>/kotlin-dsl are now cached as a bundle

.github/workflows/integTest-caching-config.yml

@@ -0,0 +1,71 @@
+name: Test caching configuration
+
+on:
+  pull_request:
+  push:
+  workflow_dispatch:
+
+env:
+  CACHE_KEY_PREFIX: ${{github.workflow}}#${{github.run_number}}-
+  CACHE_DEBUG_ENABLED: true
+
+jobs:
+  # Run initial Gradle builds to push initial cache entries
+  # These builds should start fresh without cache hits, due to the seed injected into the cache key above.
+  seed-build:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Checkout sources
+      uses: actions/checkout@v2
+    - name: Build using Gradle wrapper
+      uses: ./
+      with:
+        build-root-directory: __tests__/samples/groovy-dsl
+        arguments: test
+        # Add "wrapper" to main cache entry and remove 'wrapper-zips' bundle
+        # Exclude build-cache from main cache entry
+        cache-paths: |
+            ["caches", "notifications", "wrapper"]
+        cache-exclude-paths: |
+            ["caches/build-cache-1"]
+        cache-artifact-bundles: |
+            [
+              ["generated-gradle-jars", "caches/*/generated-gradle-jars/*.jar"],
+              ["dependency-jars", "caches/modules-*/files-*/**/*.jar"],
+              ["instrumented-jars", "caches/jars-*/*/"],
+              ["kotlin-dsl", "caches/*/kotlin-dsl/*/*/"]
+            ]
+
+  # Test that the gradle-user-home cache will cache dependencies, by running build with --offline
+  verify-build:
+    needs: seed-build
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Checkout sources
+      uses: actions/checkout@v2
+    - name: Execute Gradle build with --offline
+      uses: ./
+      with:
+        build-root-directory: __tests__/samples/groovy-dsl
+        arguments: test --offline
+        cache-read-only: true
+        # Need the same configuration when restoring state from cache
+        cache-paths: |
+            ["caches", "notifications", "wrapper"]
+        cache-exclude-paths: |
+            ["caches/build-cache-1"]
+        cache-artifact-bundles: |
+            [
+              ["generated-gradle-jars", "caches/*/generated-gradle-jars/*.jar"],
+              ["dependency-jars", "caches/modules-*/files-*/**/*.jar"],
+              ["instrumented-jars", "caches/jars-*/*/"],
+              ["kotlin-dsl", "caches/*/kotlin-dsl/*/*/"]
+            ]
+
+

action.yml

@@ -7,38 +7,63 @@ inputs:
   gradle-executable:
     description: Path to the Gradle executable
     required: false
+
   gradle-version:
     description: Gradle version to use
     required: false
+
   build-root-directory:
     description: Path to the root directory of the build
     required: false
+
   arguments:
     description: Gradle command line arguments (supports multi-line input)
     required: false
+
   cache-disabled:
     description: When 'true', all caching is disabled. No entries will be written to or read from the cache.
     required: false
     default: false
+
   cache-read-only:
     description: When 'true', existing entries will be read from the cache but no entries will be written
     required: false
     # TODO: It might be useful to default to read-only for PRs, or non-main branch.
     default: false 
 
+
+  # EXPERIMENTAL & INTERNAL CONFIGURATION PROPERTIES
+  # The following action properties allow fine-grained tweaking of the action caching behaviour.
+  # These properties are not designed for production use, and may change without notice in a subsequent release of `gradle-build-action`.
+  # Use at your own risk!
   workflow-job-context:
-    description: Used to uniquely identify the current job invocation. Defaults to the matrix values for this job; this should not be overridden by users.
+    description: Used to uniquely identify the current job invocation. Defaults to the matrix values for this job; this should not be overridden by users (INTERNAL).
     required: false
     default: ${{ toJSON(matrix) }}
+
+  cache-paths:
+    description: Paths in Gradle User Home to cache. (EXPERIMENTAL - may be changed/removed without notice)
+    required: false
+    default: |
+        ["caches", "notifications"]
+
+  cache-exclude-paths:
+    description: Paths in Gradle User Home to exclude from cache. (EXPERIMENTAL - may be changed/removed without notice)
+    required: false
+    # eg ["caches/build-cache-1"] will prevent the local build cache from being saved/restored. 
+    default: |
+        [] 
+
   cache-artifact-bundles:
-    description: Names and patterns of artifact bundles to cache separately. For internal use only.
+    description: Names and patterns of artifact bundles to cache separately. (EXPERIMENTAL - may be changed/removed without notice)
     required: false
     default: |
         [
           ["generated-gradle-jars", "caches/*/generated-gradle-jars/*.jar"],
           ["wrapper-zips", "wrapper/dists/*/*/*.zip"],
           ["dependency-jars", "caches/modules-*/files-*/**/*.jar"],
-          ["instrumented-jars", "caches/jars-*/*/"]
+          ["instrumented-jars", "caches/jars-*/*/"],
+          ["kotlin-dsl", "caches/*/kotlin-dsl/*/*/"]
         ]
 
 outputs:

src/cache-gradle-user-home.ts

@@ -12,8 +12,7 @@ import {
     tryDelete
 } from './cache-utils'
 
-// Which paths under Gradle User Home should be cached
-const CACHE_PATH = ['caches', 'notifications']
+const META_FILE_DIR = '.gradle-build-action'
 
 export class GradleUserHomeCache extends AbstractCache {
     private gradleUserHome: string
@@ -68,21 +67,32 @@ export class GradleUserHomeCache extends AbstractCache {
     }
 
     private getBundleMetaFile(name: string): string {
-        return path.resolve(
-            this.gradleUserHome,
-            'caches',
-            `.gradle-build-action.${name}.cache`
-        )
+        return path.resolve(this.gradleUserHome, META_FILE_DIR, `${name}.cache`)
     }
 
     async beforeSave(): Promise<void> {
         await this.reportGradleUserHomeSize('before saving common artifacts')
+        this.removeExcludedPaths()
         await this.saveArtifactBundles()
         await this.reportGradleUserHomeSize(
-            'after saving common artifacts (./wrapper dir is not cached)'
+            "after saving common artifacts (only 'caches' and 'notifications' will be stored)"
         )
     }
 
+    private removeExcludedPaths(): void {
+        const rawPaths: string[] = JSON.parse(
+            core.getInput('cache-exclude-paths')
+        )
+        const resolvedPaths = rawPaths.map(x =>
+            path.resolve(this.gradleUserHome, x)
+        )
+
+        for (const p of resolvedPaths) {
+            this.debug(`Deleting excluded path: ${p}`)
+            tryDelete(p)
+        }
+    }
+
     private async saveArtifactBundles(): Promise<void> {
         const processes: Promise<void>[] = []
         for (const [bundle, pattern] of this.getArtifactBundles()) {
@@ -130,9 +140,7 @@ export class GradleUserHomeCache extends AbstractCache {
         } else {
             core.info(`Caching ${bundle} with cache key: ${cacheKey}`)
             await this.saveCache([artifactPath], cacheKey)
-
-            this.debug(`Writing cache metafile: ${bundleMetaFile}`)
-            fs.writeFileSync(bundleMetaFile, cacheKey)
+            this.writeBundleMetaFile(bundleMetaFile, cacheKey)
         }
 
         for (const file of bundleFiles) {
@@ -154,6 +162,17 @@ export class GradleUserHomeCache extends AbstractCache {
         return `${cacheKeyPrefix}${bundle}-${key}`
     }
 
+    private writeBundleMetaFile(metaFile: string, cacheKey: string): void {
+        this.debug(`Writing bundle metafile: ${metaFile}`)
+
+        const dirName = path.dirname(metaFile)
+        if (!fs.existsSync(dirName)) {
+            fs.mkdirSync(dirName)
+        }
+
+        fs.writeFileSync(metaFile, cacheKey)
+    }
+
     protected determineGradleUserHome(rootDir: string): string {
         const customGradleUserHome = process.env['GRADLE_USER_HOME']
         if (customGradleUserHome) {
@@ -170,7 +189,19 @@ export class GradleUserHomeCache extends AbstractCache {
     }
 
     protected getCachePath(): string[] {
-        return CACHE_PATH.map(x => path.resolve(this.gradleUserHome, x))
+        const rawPaths: string[] = JSON.parse(core.getInput('cache-paths'))
+        rawPaths.push(META_FILE_DIR)
+        const resolvedPaths = rawPaths.map(x => this.resolveCachePath(x))
+        this.debug(`Using cache paths: ${resolvedPaths}`)
+        return resolvedPaths
+    }
+
+    private resolveCachePath(rawPath: string): string {
+        if (rawPath.startsWith('!')) {
+            const resolved = this.resolveCachePath(rawPath.substring(1))
+            return `!${resolved}`
+        }
+        return path.resolve(this.gradleUserHome, rawPath)
     }
 
     private getArtifactBundles(): Map<string, string> {

src/cache-utils.ts

@@ -19,7 +19,7 @@ export function isCacheDebuggingEnabled(): boolean {
 
 export function getCacheKeyPrefix(): string {
     // Prefix can be used to force change all cache keys (defaults to cache protocol version)
-    return process.env['CACHE_KEY_PREFIX'] || 'v2-'
+    return process.env['CACHE_KEY_PREFIX'] || 'v3-'
 }
 
 function generateCacheKey(cacheName: string): CacheKey {