Enhance github upload artifact to encrypt before

This commit is contained in:
Ben Jefferies 2023-11-28 10:48:40 +00:00
parent a8a3f3ad30
commit ee74028ec2
10 changed files with 51505 additions and 322 deletions

View file

@ -1,24 +1,8 @@
# Upload-Artifact v3 # Upload-Artifact v3
This uploads artifacts from your workflow allowing you to share data between jobs and store data once a workflow is complete. This uploads encrypted artifacts from your workflow allowing you to share data between jobs and store data once a workflow is complete.
See also [download-artifact](https://github.com/actions/download-artifact). See also [download-artifact](https://github.com/benjefferies/download-encrypted-artifact).
# What's new
- Easier upload
- Specify a wildcard pattern
- Specify an individual file
- Specify a directory (previously you were limited to only this option)
- Multi path upload
- Use a combination of individual files, wildcards or directories
- Support for excluding certain files
- Upload an artifact without providing a name
- Fix for artifact uploads sometimes not working with containers
- Proxy support out of the box
- Port entire action to typescript from a runner plugin so it is easier to collaborate and accept contributions
Refer [here](https://github.com/actions/upload-artifact/tree/releases/v1) for the previous version
# Usage # Usage
@ -28,46 +12,50 @@ See [action.yml](action.yml)
```yaml ```yaml
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v1
- run: mkdir -p path/to/artifact - run: mkdir -p path/to/artifact
- run: echo hello > path/to/artifact/world.txt - run: echo hello > path/to/artifact/world.txt
- uses: actions/upload-artifact@v3 - uses: benjefferies/upload-encrypted-artifact@v1
with: with:
name: my-artifact name: my-artifact
path: path/to/artifact/world.txt path: path/to/artifact/world.txt
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
``` ```
### Upload an Entire Directory ### Upload an Entire Directory
```yaml ```yaml
- uses: actions/upload-artifact@v3 - uses: benjefferies/upload-encrypted-artifact@v1
with: with:
name: my-artifact name: my-artifact
path: path/to/artifact/ # or path/to/artifact path: path/to/artifact/ # or path/to/artifact
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
``` ```
### Upload using a Wildcard Pattern ### Upload using a Wildcard Pattern
```yaml ```yaml
- uses: actions/upload-artifact@v3 - uses: benjefferies/upload-encrypted-artifact@v1
with: with:
name: my-artifact name: my-artifact
path: path/**/[abc]rtifac?/* path: path/**/[abc]rtifac?/*
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
``` ```
### Upload using Multiple Paths and Exclusions ### Upload using Multiple Paths and Exclusions
```yaml ```yaml
- uses: actions/upload-artifact@v3 - uses: benjefferies/upload-encrypted-artifact@v1
with: with:
name: my-artifact name: my-artifact
path: | path: |
path/output/bin/ path/output/bin/
path/output/test-results path/output/test-results
!path/**/*.tmp !path/**/*.tmp
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
``` ```
For supported wildcards along with behavior and documentation, see [@actions/glob](https://github.com/actions/toolkit/tree/main/packages/glob) which is used internally to search for files. For supported wildcards along with behavior and documentation, see [@actions/glob](https://github.com/actions/toolkit/tree/main/packages/glob) which is used internally to search for files.
@ -97,11 +85,12 @@ The [@actions/artifact](https://github.com/actions/toolkit/tree/main/packages/ar
If a path (or paths), result in no files being found for the artifact, the action will succeed but print out a warning. In certain scenarios it may be desirable to fail the action or suppress the warning. The `if-no-files-found` option allows you to customize the behavior of the action if no files are found: If a path (or paths), result in no files being found for the artifact, the action will succeed but print out a warning. In certain scenarios it may be desirable to fail the action or suppress the warning. The `if-no-files-found` option allows you to customize the behavior of the action if no files are found:
```yaml ```yaml
- uses: actions/upload-artifact@v3 - uses: benjefferies/upload-encrypted-artifact@v1
with: with:
name: my-artifact name: my-artifact
path: path/to/artifact/ path: path/to/artifact/
if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn` if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn`
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
``` ```
### Conditional Artifact Upload ### Conditional Artifact Upload
@ -109,11 +98,12 @@ If a path (or paths), result in no files being found for the artifact, the actio
To upload artifacts only when the previous step of a job failed, use [`if: failure()`](https://help.github.com/en/articles/contexts-and-expression-syntax-for-github-actions#job-status-check-functions): To upload artifacts only when the previous step of a job failed, use [`if: failure()`](https://help.github.com/en/articles/contexts-and-expression-syntax-for-github-actions#job-status-check-functions):
```yaml ```yaml
- uses: actions/upload-artifact@v3 - uses: benjefferies/upload-encrypted-artifact@v1
if: failure() if: failure()
with: with:
name: my-artifact name: my-artifact
path: path/to/artifact/ path: path/to/artifact/
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
``` ```
### Uploading without an artifact name ### Uploading without an artifact name
@ -121,9 +111,10 @@ To upload artifacts only when the previous step of a job failed, use [`if: failu
You can upload an artifact without specifying a name You can upload an artifact without specifying a name
```yaml ```yaml
- uses: actions/upload-artifact@v3 - uses: benjefferies/upload-encrypted-artifact@v1
with: with:
path: path/to/artifact/world.txt path: path/to/artifact/world.txt
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
``` ```
If not provided, `artifact` will be used as the default name which will manifest itself in the UI after upload. If not provided, `artifact` will be used as the default name which will manifest itself in the UI after upload.
@ -134,19 +125,22 @@ With the following example, the available artifact (named `artifact` by default
```yaml ```yaml
- run: echo hi > world.txt - run: echo hi > world.txt
- uses: actions/upload-artifact@v3 - uses: benjefferies/upload-encrypted-artifact@v1
with: with:
path: world.txt path: world.txt
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
- run: echo howdy > extra-file.txt - run: echo howdy > extra-file.txt
- uses: actions/upload-artifact@v3 - uses: benjefferies/upload-encrypted-artifact@v1
with: with:
path: extra-file.txt path: extra-file.txt
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
- run: echo hello > world.txt - run: echo hello > world.txt
- uses: actions/upload-artifact@v3 - uses: benjefferies/upload-encrypted-artifact@v1
with: with:
path: world.txt path: world.txt
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
``` ```
Each artifact behaves as a file share. Uploading to the same artifact multiple times in the same workflow can overwrite and append already uploaded files: Each artifact behaves as a file share. Uploading to the same artifact multiple times in the same workflow can overwrite and append already uploaded files:
@ -159,10 +153,11 @@ Each artifact behaves as a file share. Uploading to the same artifact multiple t
- name: Create a file - name: Create a file
run: echo ${{ matrix.node-version }} > my_file.txt run: echo ${{ matrix.node-version }} > my_file.txt
- name: Accidentally upload to the same artifact via multiple jobs - name: Accidentally upload to the same artifact via multiple jobs
uses: actions/upload-artifact@v3 uses: benjefferies/upload-encrypted-artifact@v1
with: with:
name: my-artifact name: my-artifact
path: ${{ github.workspace }} path: ${{ github.workspace }}
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
``` ```
> **_Warning:_** Be careful when uploading to the same artifact via multiple jobs as artifacts may become corrupted. When uploading a file with an identical name and path in multiple jobs, uploads may fail with 503 errors due to conflicting uploads happening at the same time. Ensure uploads to identical locations to not interfere with each other. > **_Warning:_** Be careful when uploading to the same artifact via multiple jobs as artifacts may become corrupted. When uploading a file with an identical name and path in multiple jobs, uploads may fail with 503 errors due to conflicting uploads happening at the same time. Ensure uploads to identical locations to not interfere with each other.
@ -170,10 +165,11 @@ Each artifact behaves as a file share. Uploading to the same artifact multiple t
In the above example, four jobs will upload four different files to the same artifact but there will only be one file available when `my-artifact` is downloaded. Each job overwrites what was previously uploaded. To ensure that jobs don't overwrite existing artifacts, use a different name per job: In the above example, four jobs will upload four different files to the same artifact but there will only be one file available when `my-artifact` is downloaded. Each job overwrites what was previously uploaded. To ensure that jobs don't overwrite existing artifacts, use a different name per job:
```yaml ```yaml
uses: actions/upload-artifact@v3 uses: benjefferies/upload-encrypted-artifact@v1
with: with:
name: my-artifact ${{ matrix.node-version }} name: my-artifact ${{ matrix.node-version }}
path: ${{ github.workspace }} path: ${{ github.workspace }}
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
``` ```
### Environment Variables and Tilde Expansion ### Environment Variables and Tilde Expansion
@ -184,10 +180,11 @@ You can use `~` in the path input as a substitute for `$HOME`. Basic tilde expan
- run: | - run: |
mkdir -p ~/new/artifact mkdir -p ~/new/artifact
echo hello > ~/new/artifact/world.txt echo hello > ~/new/artifact/world.txt
- uses: actions/upload-artifact@v3 - uses: benjefferies/upload-encrypted-artifact@v1
with: with:
name: Artifacts-V3 name: Artifacts-V3
path: ~/new/**/* path: ~/new/**/*
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
``` ```
Environment variables along with context expressions can also be used for input. For documentation see [context and expression syntax](https://help.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions): Environment variables along with context expressions can also be used for input. For documentation see [context and expression syntax](https://help.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions):
@ -199,10 +196,11 @@ Environment variables along with context expressions can also be used for input.
- run: | - run: |
mkdir -p ${{ github.workspace }}/artifact mkdir -p ${{ github.workspace }}/artifact
echo hello > ${{ github.workspace }}/artifact/world.txt echo hello > ${{ github.workspace }}/artifact/world.txt
- uses: actions/upload-artifact@v3 - uses: benjefferies/upload-encrypted-artifact@v1
with: with:
name: ${{ env.name }}-name name: ${{ env.name }}-name
path: ${{ github.workspace }}/artifact/**/* path: ${{ github.workspace }}/artifact/**/*
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
``` ```
For environment variables created in other steps, make sure to use the `env` expression syntax For environment variables created in other steps, make sure to use the `env` expression syntax
@ -213,10 +211,11 @@ For environment variables created in other steps, make sure to use the `env` exp
mkdir testing mkdir testing
echo "This is a file to upload" > testing/file.txt echo "This is a file to upload" > testing/file.txt
echo "artifactPath=testing/file.txt" >> $GITHUB_ENV echo "artifactPath=testing/file.txt" >> $GITHUB_ENV
- uses: actions/upload-artifact@v3 - uses: benjefferies/upload-encrypted-artifact@v1
with: with:
name: artifact name: artifact
path: ${{ env.artifactPath }} # this will resolve to testing/file.txt at runtime path: ${{ env.artifactPath }} # this will resolve to testing/file.txt at runtime
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
``` ```
### Retention Period ### Retention Period
@ -228,11 +227,12 @@ Artifacts are retained for 90 days by default. You can specify a shorter retenti
run: echo "I won't live long" > my_file.txt run: echo "I won't live long" > my_file.txt
- name: Upload Artifact - name: Upload Artifact
uses: actions/upload-artifact@v3 uses: benjefferies/upload-encrypted-artifact@v1
with: with:
name: my-artifact name: my-artifact
path: my_file.txt path: my_file.txt
retention-days: 5 retention-days: 5
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
``` ```
The retention period must be between 1 and 90 inclusive. For more information see [artifact and log retention policies](https://docs.github.com/en/free-pro-team@latest/actions/reference/usage-limits-billing-and-administration#artifact-and-log-retention-policy). The retention period must be between 1 and 90 inclusive. For more information see [artifact and log retention policies](https://docs.github.com/en/free-pro-team@latest/actions/reference/usage-limits-billing-and-administration#artifact-and-log-retention-policy).
@ -270,10 +270,11 @@ If file permissions and case sensitivity are required, you can `tar` all of your
run: tar -cvf my_files.tar /path/to/my/directory run: tar -cvf my_files.tar /path/to/my/directory
- name: Upload Artifact - name: Upload Artifact
uses: actions/upload-artifact@v3 uses: benjefferies/upload-encrypted-artifact@v1
with: with:
name: my-artifact name: my-artifact
path: my_files.tar path: my_files.tar
kms-key-id: = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
``` ```
### Too many uploads resulting in 429 responses ### Too many uploads resulting in 429 responses

View file

@ -23,6 +23,10 @@ inputs:
Minimum 1 day. Minimum 1 day.
Maximum 90 days unless changed from the repository settings page. Maximum 90 days unless changed from the repository settings page.
kms-key-id:
description: >
The ID of the customer-managed AWS Key Management Service (AWS KMS) key that should be used to encrypt the artifact.
required: true
runs: runs:
using: 'node16' using: 'node16'
main: 'dist/index.js' main: 'dist/index.js'

49106
dist/index.js vendored

File diff suppressed because one or more lines are too long

2587
package-lock.json generated

File diff suppressed because it is too large Load diff

View file

@ -32,7 +32,8 @@
"@actions/artifact": "^1.1.2", "@actions/artifact": "^1.1.2",
"@actions/core": "^1.10.0", "@actions/core": "^1.10.0",
"@actions/glob": "^0.3.0", "@actions/glob": "^0.3.0",
"@actions/io": "^1.1.2" "@actions/io": "^1.1.2",
"@aws-crypto/client-node": "^4.0.0"
}, },
"devDependencies": { "devDependencies": {
"@types/jest": "^29.2.5", "@types/jest": "^29.2.5",

View file

@ -3,7 +3,8 @@ export enum Inputs {
Name = 'name', Name = 'name',
Path = 'path', Path = 'path',
IfNoFilesFound = 'if-no-files-found', IfNoFilesFound = 'if-no-files-found',
RetentionDays = 'retention-days' RetentionDays = 'retention-days',
kmsKeyId = 'kms-key-id'
} }
export enum NoFileOptions { export enum NoFileOptions {

29
src/encrypt.ts Normal file
View file

@ -0,0 +1,29 @@
import {
CommitmentPolicy,
KmsKeyringNode,
buildEncrypt
} from '@aws-crypto/client-node'
import {readFileSync, writeFileSync} from 'fs'
export async function encryptFile(
filePath: string,
kmsKeyId: string
): Promise<void> {
const keyring = new KmsKeyringNode({generatorKeyId: kmsKeyId})
const client = buildEncrypt(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT)
// Read the file content
const fileBuffer = readFileSync(filePath)
try {
// Encrypt the data
const {result} = await client.encrypt(keyring, fileBuffer)
// Overwrite file with encrypted data
writeFileSync(filePath, result)
console.log('File encrypted successfully')
} catch (error) {
console.error('Error encrypting file:', error)
throw error
}
}

View file

@ -1,5 +1,7 @@
import * as core from '@actions/core' import * as core from '@actions/core'
import {Inputs, NoFileOptions} from './constants' import {Inputs, NoFileOptions} from './constants'
import {UploadInputs} from './upload-inputs' import {UploadInputs} from './upload-inputs'
/** /**
@ -8,6 +10,7 @@ import {UploadInputs} from './upload-inputs'
export function getInputs(): UploadInputs { export function getInputs(): UploadInputs {
const name = core.getInput(Inputs.Name) const name = core.getInput(Inputs.Name)
const path = core.getInput(Inputs.Path, {required: true}) const path = core.getInput(Inputs.Path, {required: true})
const kmsKeyId = core.getInput(Inputs.kmsKeyId)
const ifNoFilesFound = core.getInput(Inputs.IfNoFilesFound) const ifNoFilesFound = core.getInput(Inputs.IfNoFilesFound)
const noFileBehavior: NoFileOptions = NoFileOptions[ifNoFilesFound] const noFileBehavior: NoFileOptions = NoFileOptions[ifNoFilesFound]
@ -25,7 +28,8 @@ export function getInputs(): UploadInputs {
const inputs = { const inputs = {
artifactName: name, artifactName: name,
searchPath: path, searchPath: path,
ifNoFilesFound: noFileBehavior ifNoFilesFound: noFileBehavior,
kmsKeyId
} as UploadInputs } as UploadInputs
const retentionDaysStr = core.getInput(Inputs.RetentionDays) const retentionDaysStr = core.getInput(Inputs.RetentionDays)

View file

@ -1,8 +1,11 @@
import * as core from '@actions/core' import * as core from '@actions/core'
import {create, UploadOptions} from '@actions/artifact'
import {UploadOptions, create} from '@actions/artifact'
import {NoFileOptions} from './constants'
import {encryptFile} from './encrypt'
import {findFilesToUpload} from './search' import {findFilesToUpload} from './search'
import {getInputs} from './input-helper' import {getInputs} from './input-helper'
import {NoFileOptions} from './constants'
async function run(): Promise<void> { async function run(): Promise<void> {
try { try {
@ -43,6 +46,10 @@ async function run(): Promise<void> {
) )
} }
for (const file of searchResult.filesToUpload) {
await encryptFile(file, inputs.kmsKeyId)
}
const artifactClient = create() const artifactClient = create()
const options: UploadOptions = { const options: UploadOptions = {
continueOnError: false continueOnError: false

View file

@ -20,4 +20,9 @@ export interface UploadInputs {
* Duration after which artifact will expire in days * Duration after which artifact will expire in days
*/ */
retentionDays: number retentionDays: number
/**
* KMS Key Id to use for artifact upload
*/
kmsKeyId: string
} }