Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MultiCI: Add class wrapping cache-key generation #506

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 67 additions & 62 deletions sources/src/caching/cache-key.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,74 +27,79 @@ export class CacheKey {
}

/**
* Generates a cache key specific to the current job execution.
* The key is constructed from the following inputs (with some user overrides):
* - The cache key prefix: defaults to 'gradle-' but can be overridden by the user
* - The cache protocol version
* - The runner operating system
* - The name of the workflow and Job being executed
* - The matrix values for the Job being executed (job context)
* - The SHA of the commit being executed
*
* Caches are restored by trying to match the these key prefixes in order:
* - The full key with SHA
* - A previous key for this Job + matrix
* - Any previous key for this Job (any matrix)
* - Any previous key for this cache on the current OS
* Provides generation fascilities for [CacheKey] values.
*/
export function generateCacheKey(cacheName: string, config: CacheConfig): CacheKey {
const prefix = process.env[CACHE_KEY_PREFIX_VAR] || ''

const cacheKeyBase = `${prefix}${getCacheKeyBase(cacheName, CACHE_PROTOCOL_VERSION)}`

// At the most general level, share caches for all executions on the same OS
const cacheKeyForEnvironment = `${cacheKeyBase}|${getCacheKeyEnvironment()}`

// Then prefer caches that run job with the same ID
const cacheKeyForJob = `${cacheKeyForEnvironment}|${getCacheKeyJob()}`

// Prefer (even more) jobs that run this job in the same workflow with the same context (matrix)
const cacheKeyForJobContext = `${cacheKeyForJob}[${getCacheKeyJobInstance()}]`

// Exact match on Git SHA
const cacheKey = `${cacheKeyForJobContext}-${getCacheKeyJobExecution()}`

if (config.isCacheStrictMatch()) {
return new CacheKey(cacheKey, [cacheKeyForJobContext])
export class CacheKeyGenerator {
/**
* Generates a cache key specific to the current job execution.
* The key is constructed from the following inputs (with some user overrides):
* - The cache key prefix: defaults to 'gradle-' but can be overridden by the user
* - The cache protocol version
* - The runner operating system
* - The name of the workflow and Job being executed
* - The matrix values for the Job being executed (job context)
* - The SHA of the commit being executed
*
* Caches are restored by trying to match the these key prefixes in order:
* - The full key with SHA
* - A previous key for this Job + matrix
* - Any previous key for this Job (any matrix)
* - Any previous key for this cache on the current OS
*/
generateCacheKey(cacheName: string, config: CacheConfig): CacheKey {
const prefix = process.env[CACHE_KEY_PREFIX_VAR] || ''

const cacheKeyBase = `${prefix}${this.getCacheKeyBase(cacheName, CACHE_PROTOCOL_VERSION)}`

// At the most general level, share caches for all executions on the same OS
const cacheKeyForEnvironment = `${cacheKeyBase}|${this.getCacheKeyEnvironment()}`

// Then prefer caches that run job with the same ID
const cacheKeyForJob = `${cacheKeyForEnvironment}|${this.getCacheKeyJob()}`

// Prefer (even more) jobs that run this job in the same workflow with the same context (matrix)
const cacheKeyForJobContext = `${cacheKeyForJob}[${this.getCacheKeyJobInstance()}]`

// Exact match on Git SHA
const cacheKey = `${cacheKeyForJobContext}-${this.getCacheKeyJobExecution()}`

if (config.isCacheStrictMatch()) {
return new CacheKey(cacheKey, [cacheKeyForJobContext])
}

return new CacheKey(cacheKey, [cacheKeyForJobContext, cacheKeyForJob, cacheKeyForEnvironment])
}

return new CacheKey(cacheKey, [cacheKeyForJobContext, cacheKeyForJob, cacheKeyForEnvironment])
}

export function getCacheKeyBase(cacheName: string, cacheProtocolVersion: string): string {
// Prefix can be used to force change all cache keys (defaults to cache protocol version)
return `gradle-${cacheName}-${cacheProtocolVersion}`
}

function getCacheKeyEnvironment(): string {
const runnerOs = process.env['RUNNER_OS'] || ''
const runnerArch = process.env['RUNNER_ARCH'] || ''
return process.env[CACHE_KEY_OS_VAR] || `${runnerOs}-${runnerArch}`
}
getCacheKeyBase(cacheName: string, cacheProtocolVersion: string): string {
// Prefix can be used to force change all cache keys (defaults to cache protocol version)
return `gradle-${cacheName}-${cacheProtocolVersion}`
}

function getCacheKeyJob(): string {
return process.env[CACHE_KEY_JOB_VAR] || github.context.job
}
private getCacheKeyEnvironment(): string {
const runnerOs = process.env['RUNNER_OS'] || ''
const runnerArch = process.env['RUNNER_ARCH'] || ''
return process.env[CACHE_KEY_OS_VAR] || `${runnerOs}-${runnerArch}`
}

function getCacheKeyJobInstance(): string {
const override = process.env[CACHE_KEY_JOB_INSTANCE_VAR]
if (override) {
return override
private getCacheKeyJob(): string {
return process.env[CACHE_KEY_JOB_VAR] || github.context.job
}

// By default, we hash the workflow name and the full `matrix` data for the run, to uniquely identify this job invocation
// The only way we can obtain the `matrix` data is via the `workflow-job-context` parameter in action.yml.
const workflowName = github.context.workflow
const workflowJobContext = getJobMatrix()
return hashStrings([workflowName, workflowJobContext])
}
private getCacheKeyJobInstance(): string {
const override = process.env[CACHE_KEY_JOB_INSTANCE_VAR]
if (override) {
return override
}

// By default, we hash the workflow name and the full `matrix` data for the run, to uniquely identify this job invocation
// The only way we can obtain the `matrix` data is via the `workflow-job-context` parameter in action.yml.
const workflowName = github.context.workflow
const workflowJobContext = getJobMatrix()
return hashStrings([workflowName, workflowJobContext])
}

function getCacheKeyJobExecution(): string {
// Used to associate a cache key with a particular execution (default is bound to the git commit sha)
return process.env[CACHE_KEY_JOB_EXECUTION_VAR] || github.context.sha
private getCacheKeyJobExecution(): string {
// Used to associate a cache key with a particular execution (default is bound to the git commit sha)
return process.env[CACHE_KEY_JOB_EXECUTION_VAR] || github.context.sha
}
}
7 changes: 5 additions & 2 deletions sources/src/caching/caches.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {CacheCleaner} from './cache-cleaner'
import {DaemonController} from '../daemon-controller'
import {CacheConfig} from '../configuration'
import {BuildResults} from '../build-results'
import {CacheKeyGenerator} from './cache-key'

const CACHE_RESTORED_VAR = 'GRADLE_BUILD_ACTION_CACHE_RESTORED'

Expand All @@ -26,7 +27,8 @@ export async function restore(
}
core.exportVariable(CACHE_RESTORED_VAR, true)

const gradleStateCache = new GradleUserHomeCache(userHome, gradleUserHome, cacheConfig)
// TODO(Nava2): Move `new CacheKeyGenerator()` to a class property.
const gradleStateCache = new GradleUserHomeCache(userHome, gradleUserHome, cacheConfig, new CacheKeyGenerator())

if (cacheConfig.isCacheDisabled()) {
core.info('Cache is disabled: will not restore state from previous builds.')
Expand Down Expand Up @@ -110,7 +112,8 @@ export async function save(
}

await core.group('Caching Gradle state', async () => {
return new GradleUserHomeCache(userHome, gradleUserHome, cacheConfig).save(cacheListener)
const cacheKeyGenerator = new CacheKeyGenerator()
return new GradleUserHomeCache(userHome, gradleUserHome, cacheConfig, cacheKeyGenerator).save(cacheListener)
})
}

Expand Down
32 changes: 24 additions & 8 deletions sources/src/caching/gradle-home-extry-extractor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import {cacheDebug, hashFileNames, isCacheDebuggingEnabled, restoreCache, saveCa

import {BuildResult, loadBuildResults} from '../build-results'
import {CacheConfig, ACTION_METADATA_DIR} from '../configuration'
import {getCacheKeyBase} from './cache-key'
import {CacheKeyGenerator} from './cache-key'
import {versionIsAtLeast} from '../execution/gradle'

const SKIP_RESTORE_VAR = 'GRADLE_BUILD_ACTION_SKIP_RESTORE'
Expand Down Expand Up @@ -85,10 +85,18 @@ abstract class AbstractEntryExtractor {
protected readonly gradleUserHome: string
private extractorName: string

constructor(gradleUserHome: string, extractorName: string, cacheConfig: CacheConfig) {
private readonly cacheKeyGenerator: CacheKeyGenerator

constructor(
gradleUserHome: string,
extractorName: string,
cacheConfig: CacheConfig,
cacheKeyGenerator: CacheKeyGenerator
) {
this.gradleUserHome = gradleUserHome
this.extractorName = extractorName
this.cacheConfig = cacheConfig
this.cacheKeyGenerator = cacheKeyGenerator
}

/**
Expand Down Expand Up @@ -247,15 +255,15 @@ abstract class AbstractEntryExtractor {

cacheDebug(`Generating cache key for ${artifactType} from file names: ${relativeFiles}`)

return `${getCacheKeyBase(artifactType, CACHE_PROTOCOL_VERSION)}-${key}`
return `${this.cacheKeyGenerator.getCacheKeyBase(artifactType, CACHE_PROTOCOL_VERSION)}-${key}`
}

protected async createCacheKeyFromFileContents(artifactType: string, pattern: string): Promise<string> {
const key = await glob.hashFiles(pattern)

cacheDebug(`Generating cache key for ${artifactType} from files matching: ${pattern}`)

return `${getCacheKeyBase(artifactType, CACHE_PROTOCOL_VERSION)}-${key}`
return `${this.cacheKeyGenerator.getCacheKeyBase(artifactType, CACHE_PROTOCOL_VERSION)}-${key}`
}

// Run actions sequentially if debugging is enabled
Expand Down Expand Up @@ -305,8 +313,12 @@ abstract class AbstractEntryExtractor {
}

export class GradleHomeEntryExtractor extends AbstractEntryExtractor {
constructor(gradleUserHome: string, cacheConfig: CacheConfig) {
super(gradleUserHome, 'gradle-home', cacheConfig)
constructor(
gradleUserHome: string,
cacheConfig: CacheConfig,
cacheKeyGenerator: CacheKeyGenerator = new CacheKeyGenerator()
) {
super(gradleUserHome, 'gradle-home', cacheConfig, cacheKeyGenerator)
}

async extract(listener: CacheListener): Promise<void> {
Expand Down Expand Up @@ -364,8 +376,12 @@ export class GradleHomeEntryExtractor extends AbstractEntryExtractor {
}

export class ConfigurationCacheEntryExtractor extends AbstractEntryExtractor {
constructor(gradleUserHome: string, cacheConfig: CacheConfig) {
super(gradleUserHome, 'configuration-cache', cacheConfig)
constructor(
gradleUserHome: string,
cacheConfig: CacheConfig,
cacheKeyGenerator: CacheKeyGenerator = new CacheKeyGenerator()
) {
super(gradleUserHome, 'configuration-cache', cacheConfig, cacheKeyGenerator)
}

/**
Expand Down
16 changes: 12 additions & 4 deletions sources/src/caching/gradle-user-home-cache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import * as glob from '@actions/glob'

import path from 'path'
import fs from 'fs'
import {generateCacheKey} from './cache-key'
import {CacheKeyGenerator} from './cache-key'
import {CacheListener} from './cache-reporting'
import {saveCache, restoreCache, cacheDebug, isCacheDebuggingEnabled, tryDelete} from './cache-utils'
import {CacheConfig, ACTION_METADATA_DIR} from '../configuration'
Expand All @@ -21,10 +21,18 @@ export class GradleUserHomeCache {
private readonly gradleUserHome: string
private readonly cacheConfig: CacheConfig

constructor(userHome: string, gradleUserHome: string, cacheConfig: CacheConfig) {
private readonly cacheKeyGenerator: CacheKeyGenerator

constructor(
userHome: string,
gradleUserHome: string,
cacheConfig: CacheConfig,
cacheKeyGenerator: CacheKeyGenerator
) {
this.userHome = userHome
this.gradleUserHome = gradleUserHome
this.cacheConfig = cacheConfig
this.cacheKeyGenerator = cacheKeyGenerator
}

init(): void {
Expand Down Expand Up @@ -52,7 +60,7 @@ export class GradleUserHomeCache {
async restore(listener: CacheListener): Promise<void> {
const entryListener = listener.entry(this.cacheDescription)

const cacheKey = generateCacheKey(this.cacheName, this.cacheConfig)
const cacheKey = this.cacheKeyGenerator.generateCacheKey(this.cacheName, this.cacheConfig)

cacheDebug(
`Requesting ${this.cacheDescription} with
Expand Down Expand Up @@ -95,7 +103,7 @@ export class GradleUserHomeCache {
* it is saved with the exact key.
*/
async save(listener: CacheListener): Promise<void> {
const cacheKey = generateCacheKey(this.cacheName, this.cacheConfig).key
const cacheKey = this.cacheKeyGenerator.generateCacheKey(this.cacheName, this.cacheConfig).key
const restoredCacheKey = core.getState(RESTORED_CACHE_KEY_KEY)
const gradleHomeEntryListener = listener.entry(this.cacheDescription)

Expand Down
5 changes: 3 additions & 2 deletions sources/test/jest/cache-debug.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import * as path from 'path'
import * as fs from 'fs'
import {GradleUserHomeCache} from "../../src/caching/gradle-user-home-cache"
import {CacheConfig} from "../../src/configuration"
import { CacheKeyGenerator } from '../../src/caching/cache-key'

const testTmp = 'test/jest/tmp'
fs.rmSync(testTmp, {recursive: true, force: true})
Expand All @@ -12,7 +13,7 @@ describe("--info and --stacktrace", () => {
const emptyGradleHome = `${testTmp}/empty-gradle-home`
fs.mkdirSync(emptyGradleHome, {recursive: true})

const stateCache = new GradleUserHomeCache("ignored", emptyGradleHome, new CacheConfig())
const stateCache = new GradleUserHomeCache("ignored", emptyGradleHome, new CacheConfig(), new CacheKeyGenerator())
stateCache.configureInfoLogLevel()

expect(fs.readFileSync(path.resolve(emptyGradleHome, "gradle.properties"), 'utf-8'))
Expand All @@ -25,7 +26,7 @@ describe("--info and --stacktrace", () => {
fs.mkdirSync(existingGradleHome, {recursive: true})
fs.writeFileSync(path.resolve(existingGradleHome, "gradle.properties"), "org.gradle.logging.level=debug\n")

const stateCache = new GradleUserHomeCache("ignored", existingGradleHome, new CacheConfig())
const stateCache = new GradleUserHomeCache("ignored", existingGradleHome, new CacheConfig(), new CacheKeyGenerator())
stateCache.configureInfoLogLevel()

expect(fs.readFileSync(path.resolve(existingGradleHome, "gradle.properties"), 'utf-8'))
Expand Down