Enhance github upload artifact to encrypt before

This commit is contained in:
Ben Jefferies 2023-11-28 10:48:40 +00:00
parent a8a3f3ad30
commit ee74028ec2
10 changed files with 51505 additions and 322 deletions

View file

@ -1,24 +1,8 @@
# Upload-Artifact v3
This uploads artifacts from your workflow allowing you to share data between jobs and store data once a workflow is complete.
This uploads encrypted artifacts from your workflow allowing you to share data between jobs and store data once a workflow is complete.
See also [download-artifact](https://github.com/actions/download-artifact).
# What's new
- Easier upload
- Specify a wildcard pattern
- Specify an individual file
- Specify a directory (previously you were limited to only this option)
- Multi path upload
- Use a combination of individual files, wildcards or directories
- Support for excluding certain files
- Upload an artifact without providing a name
- Fix for artifact uploads sometimes not working with containers
- Proxy support out of the box
- Port entire action to typescript from a runner plugin so it is easier to collaborate and accept contributions
Refer [here](https://github.com/actions/upload-artifact/tree/releases/v1) for the previous version
See also [download-artifact](https://github.com/benjefferies/download-encrypted-artifact).
# Usage
@ -28,46 +12,50 @@ See [action.yml](action.yml)
```yaml
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v1
- run: mkdir -p path/to/artifact
- run: echo hello > path/to/artifact/world.txt
- uses: actions/upload-artifact@v3
- uses: benjefferies/upload-encrypted-artifact@v1
with:
name: my-artifact
path: path/to/artifact/world.txt
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
```
### Upload an Entire Directory
```yaml
- uses: actions/upload-artifact@v3
- uses: benjefferies/upload-encrypted-artifact@v1
with:
name: my-artifact
path: path/to/artifact/ # or path/to/artifact
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
```
### Upload using a Wildcard Pattern
```yaml
- uses: actions/upload-artifact@v3
- uses: benjefferies/upload-encrypted-artifact@v1
with:
name: my-artifact
path: path/**/[abc]rtifac?/*
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
```
### Upload using Multiple Paths and Exclusions
```yaml
- uses: actions/upload-artifact@v3
- uses: benjefferies/upload-encrypted-artifact@v1
with:
name: my-artifact
path: |
path/output/bin/
path/output/test-results
!path/**/*.tmp
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
```
For supported wildcards along with behavior and documentation, see [@actions/glob](https://github.com/actions/toolkit/tree/main/packages/glob) which is used internally to search for files.
@ -97,11 +85,12 @@ The [@actions/artifact](https://github.com/actions/toolkit/tree/main/packages/ar
If a path (or paths), result in no files being found for the artifact, the action will succeed but print out a warning. In certain scenarios it may be desirable to fail the action or suppress the warning. The `if-no-files-found` option allows you to customize the behavior of the action if no files are found:
```yaml
- uses: actions/upload-artifact@v3
- uses: benjefferies/upload-encrypted-artifact@v1
with:
name: my-artifact
path: path/to/artifact/
if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn`
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
```
### Conditional Artifact Upload
@ -109,11 +98,12 @@ If a path (or paths), result in no files being found for the artifact, the actio
To upload artifacts only when the previous step of a job failed, use [`if: failure()`](https://help.github.com/en/articles/contexts-and-expression-syntax-for-github-actions#job-status-check-functions):
```yaml
- uses: actions/upload-artifact@v3
- uses: benjefferies/upload-encrypted-artifact@v1
if: failure()
with:
name: my-artifact
path: path/to/artifact/
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
```
### Uploading without an artifact name
@ -121,9 +111,10 @@ To upload artifacts only when the previous step of a job failed, use [`if: failu
You can upload an artifact without specifying a name
```yaml
- uses: actions/upload-artifact@v3
- uses: benjefferies/upload-encrypted-artifact@v1
with:
path: path/to/artifact/world.txt
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
```
If not provided, `artifact` will be used as the default name which will manifest itself in the UI after upload.
@ -134,19 +125,22 @@ With the following example, the available artifact (named `artifact` by default
```yaml
- run: echo hi > world.txt
- uses: actions/upload-artifact@v3
- uses: benjefferies/upload-encrypted-artifact@v1
with:
path: world.txt
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
- run: echo howdy > extra-file.txt
- uses: actions/upload-artifact@v3
- uses: benjefferies/upload-encrypted-artifact@v1
with:
path: extra-file.txt
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
- run: echo hello > world.txt
- uses: actions/upload-artifact@v3
- uses: benjefferies/upload-encrypted-artifact@v1
with:
path: world.txt
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
```
Each artifact behaves as a file share. Uploading to the same artifact multiple times in the same workflow can overwrite and append already uploaded files:
@ -159,10 +153,11 @@ Each artifact behaves as a file share. Uploading to the same artifact multiple t
- name: Create a file
run: echo ${{ matrix.node-version }} > my_file.txt
- name: Accidentally upload to the same artifact via multiple jobs
uses: actions/upload-artifact@v3
uses: benjefferies/upload-encrypted-artifact@v1
with:
name: my-artifact
path: ${{ github.workspace }}
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
```
> **_Warning:_** Be careful when uploading to the same artifact via multiple jobs as artifacts may become corrupted. When uploading a file with an identical name and path in multiple jobs, uploads may fail with 503 errors due to conflicting uploads happening at the same time. Ensure uploads to identical locations to not interfere with each other.
@ -170,10 +165,11 @@ Each artifact behaves as a file share. Uploading to the same artifact multiple t
In the above example, four jobs will upload four different files to the same artifact but there will only be one file available when `my-artifact` is downloaded. Each job overwrites what was previously uploaded. To ensure that jobs don't overwrite existing artifacts, use a different name per job:
```yaml
uses: actions/upload-artifact@v3
uses: benjefferies/upload-encrypted-artifact@v1
with:
name: my-artifact ${{ matrix.node-version }}
path: ${{ github.workspace }}
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
```
### Environment Variables and Tilde Expansion
@ -184,10 +180,11 @@ You can use `~` in the path input as a substitute for `$HOME`. Basic tilde expan
- run: |
mkdir -p ~/new/artifact
echo hello > ~/new/artifact/world.txt
- uses: actions/upload-artifact@v3
- uses: benjefferies/upload-encrypted-artifact@v1
with:
name: Artifacts-V3
path: ~/new/**/*
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
```
Environment variables along with context expressions can also be used for input. For documentation see [context and expression syntax](https://help.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions):
@ -199,10 +196,11 @@ Environment variables along with context expressions can also be used for input.
- run: |
mkdir -p ${{ github.workspace }}/artifact
echo hello > ${{ github.workspace }}/artifact/world.txt
- uses: actions/upload-artifact@v3
- uses: benjefferies/upload-encrypted-artifact@v1
with:
name: ${{ env.name }}-name
path: ${{ github.workspace }}/artifact/**/*
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
```
For environment variables created in other steps, make sure to use the `env` expression syntax
@ -213,10 +211,11 @@ For environment variables created in other steps, make sure to use the `env` exp
mkdir testing
echo "This is a file to upload" > testing/file.txt
echo "artifactPath=testing/file.txt" >> $GITHUB_ENV
- uses: actions/upload-artifact@v3
- uses: benjefferies/upload-encrypted-artifact@v1
with:
name: artifact
path: ${{ env.artifactPath }} # this will resolve to testing/file.txt at runtime
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
```
### Retention Period
@ -228,11 +227,12 @@ Artifacts are retained for 90 days by default. You can specify a shorter retenti
run: echo "I won't live long" > my_file.txt
- name: Upload Artifact
uses: actions/upload-artifact@v3
uses: benjefferies/upload-encrypted-artifact@v1
with:
name: my-artifact
path: my_file.txt
retention-days: 5
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
```
The retention period must be between 1 and 90 inclusive. For more information see [artifact and log retention policies](https://docs.github.com/en/free-pro-team@latest/actions/reference/usage-limits-billing-and-administration#artifact-and-log-retention-policy).
@ -270,10 +270,11 @@ If file permissions and case sensitivity are required, you can `tar` all of your
run: tar -cvf my_files.tar /path/to/my/directory
- name: Upload Artifact
uses: actions/upload-artifact@v3
uses: benjefferies/upload-encrypted-artifact@v1
with:
name: my-artifact
path: my_files.tar
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
```
### Too many uploads resulting in 429 responses

View file

@ -23,6 +23,10 @@ inputs:
Minimum 1 day.
Maximum 90 days unless changed from the repository settings page.
kms-key-id:
description: >
The ID of the customer-managed AWS Key Management Service (AWS KMS) key that should be used to encrypt the artifact.
required: true
runs:
using: 'node16'
main: 'dist/index.js'

49106
dist/index.js vendored

File diff suppressed because one or more lines are too long

2587
package-lock.json generated

File diff suppressed because it is too large Load diff

View file

@ -32,7 +32,8 @@
"@actions/artifact": "^1.1.2",
"@actions/core": "^1.10.0",
"@actions/glob": "^0.3.0",
"@actions/io": "^1.1.2"
"@actions/io": "^1.1.2",
"@aws-crypto/client-node": "^4.0.0"
},
"devDependencies": {
"@types/jest": "^29.2.5",

View file

@ -3,7 +3,8 @@ export enum Inputs {
Name = 'name',
Path = 'path',
IfNoFilesFound = 'if-no-files-found',
RetentionDays = 'retention-days'
RetentionDays = 'retention-days',
kmsKeyId = 'kms-key-id'
}
export enum NoFileOptions {

29
src/encrypt.ts Normal file
View file

@ -0,0 +1,29 @@
import {
CommitmentPolicy,
KmsKeyringNode,
buildEncrypt
} from '@aws-crypto/client-node'
import {readFileSync, writeFileSync} from 'fs'
export async function encryptFile(
filePath: string,
kmsKeyId: string
): Promise<void> {
const keyring = new KmsKeyringNode({generatorKeyId: kmsKeyId})
const client = buildEncrypt(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT)
// Read the file content
const fileBuffer = readFileSync(filePath)
try {
// Encrypt the data
const {result} = await client.encrypt(keyring, fileBuffer)
// Overwrite file with encrypted data
writeFileSync(filePath, result)
console.log('File encrypted successfully')
} catch (error) {
console.error('Error encrypting file:', error)
throw error
}
}

View file

@ -1,5 +1,7 @@
import * as core from '@actions/core'
import {Inputs, NoFileOptions} from './constants'
import {UploadInputs} from './upload-inputs'
/**
@ -8,6 +10,7 @@ import {UploadInputs} from './upload-inputs'
export function getInputs(): UploadInputs {
const name = core.getInput(Inputs.Name)
const path = core.getInput(Inputs.Path, {required: true})
const kmsKeyId = core.getInput(Inputs.kmsKeyId)
const ifNoFilesFound = core.getInput(Inputs.IfNoFilesFound)
const noFileBehavior: NoFileOptions = NoFileOptions[ifNoFilesFound]
@ -25,7 +28,8 @@ export function getInputs(): UploadInputs {
const inputs = {
artifactName: name,
searchPath: path,
ifNoFilesFound: noFileBehavior
ifNoFilesFound: noFileBehavior,
kmsKeyId
} as UploadInputs
const retentionDaysStr = core.getInput(Inputs.RetentionDays)

View file

@ -1,8 +1,11 @@
import * as core from '@actions/core'
import {create, UploadOptions} from '@actions/artifact'
import {UploadOptions, create} from '@actions/artifact'
import {NoFileOptions} from './constants'
import {encryptFile} from './encrypt'
import {findFilesToUpload} from './search'
import {getInputs} from './input-helper'
import {NoFileOptions} from './constants'
async function run(): Promise<void> {
try {
@ -43,6 +46,10 @@ async function run(): Promise<void> {
)
}
for (const file of searchResult.filesToUpload) {
await encryptFile(file, inputs.kmsKeyId)
}
const artifactClient = create()
const options: UploadOptions = {
continueOnError: false

View file

@ -20,4 +20,9 @@ export interface UploadInputs {
* Duration after which artifact will expire in days
*/
retentionDays: number
/**
* KMS Key Id to use for artifact upload
*/
kmsKeyId: string
}