Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ci: improve caching #2329

Merged
merged 42 commits into from
Nov 18, 2024
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
0bba026
ci: fix docker cache mounts
shumkov Nov 13, 2024
ea76396
Merge branch 'refs/heads/v1.6-dev' into ci/fix-docker-cache-mounts
shumkov Nov 13, 2024
4124bc3
ci: switch more to gha
shumkov Nov 13, 2024
fad022e
Merge branch 'v1.6-dev' into ci/fix-docker-cache-mounts
shumkov Nov 13, 2024
83f6659
ci: fix artifact key
shumkov Nov 13, 2024
6a03bd3
ci: try using download artifact
shumkov Nov 14, 2024
98232ef
ci: use artifacts
shumkov Nov 14, 2024
bf3fb89
ci: remove unnecessary archive step
shumkov Nov 14, 2024
9c79aec
ci: rename artifact
shumkov Nov 14, 2024
a12d16d
ci: fix get modified files
shumkov Nov 14, 2024
30111be
ci: fix get modified files again
shumkov Nov 14, 2024
8aa56d9
ci: fix download certificates
shumkov Nov 14, 2024
5eab16c
docs: add comment
shumkov Nov 14, 2024
71bddba
ci: layer to registry, disable sccache
shumkov Nov 14, 2024
84852e4
ci: fix workflow
shumkov Nov 14, 2024
fe9f565
ci: fix librocksdb cache
shumkov Nov 14, 2024
c9dd655
ci: fix docker layer cache
shumkov Nov 14, 2024
a957dd0
ci: back to ECR
shumkov Nov 14, 2024
e338503
ci: fix js artifacts
shumkov Nov 14, 2024
b4f6d72
build: remove target cache to enable chief
shumkov Nov 14, 2024
b0795d0
build: fix build-js
shumkov Nov 14, 2024
860a04c
build: fix chief
shumkov Nov 14, 2024
55bb2ed
build: fix chief 2
shumkov Nov 14, 2024
c77aed5
build: pass target and features to chef
shumkov Nov 14, 2024
3b1cf97
build: trying to fix drive build
shumkov Nov 14, 2024
959016d
build: remove target
shumkov Nov 14, 2024
febf93e
build: remove target and s3 for layercache
shumkov Nov 14, 2024
2e34367
ci: enable sccache
shumkov Nov 14, 2024
3321e15
ci: remove target in the same layer
shumkov Nov 14, 2024
7cb68cc
ci: fix cache mounts and remove unnecessary steps
shumkov Nov 14, 2024
3b65eeb
ci: fix RUNNER_TEMP
shumkov Nov 14, 2024
163323c
ci: fix region and home
shumkov Nov 14, 2024
9cdea04
ci: fix cache from
shumkov Nov 14, 2024
732b2b4
chore: kick off CI
shumkov Nov 14, 2024
7aa6617
chore: add new lines
shumkov Nov 14, 2024
cdf633e
ci: fix dashmate e2e tests
shumkov Nov 15, 2024
7b7e5aa
ci: fix restore key
shumkov Nov 15, 2024
9869252
ci: remove commented code
shumkov Nov 15, 2024
72539f4
ci: cache to base manifest only on base branch push
shumkov Nov 18, 2024
cd35f65
ci: fix manifestNames is not defined
shumkov Nov 18, 2024
f078ec0
chore: fix security vulnerability
shumkov Nov 18, 2024
143f980
ci: we don't want to remove artifacts
shumkov Nov 18, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/actions/docker/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ inputs:
aws_secret_access_key:
description: AWS secret access key
required: true
cache_to_name:
lklimek marked this conversation as resolved.
Show resolved Hide resolved
description: 'Save cache to name manifest (should be used only on default branch)'
default: 'false'
outputs:
digest:
value: ${{ steps.docker_build.outputs.digest }}
Expand Down Expand Up @@ -79,6 +82,7 @@ runs:
name: ${{ inputs.image_name }}
region: ${{ inputs.region }}
bucket: ${{ inputs.bucket }}
cache_to_name: ${{ inputs.cache_to_name }}

- name: Set HOME variable to github context
shell: bash
Expand Down
36 changes: 26 additions & 10 deletions .github/actions/s3-layer-cache-settings/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ inputs:
mode:
description: Cache mode
default: max
cache_to_name:
description: 'Save cache to name manifest (should be used only on default branch)'
default: 'false'

outputs:
cache_to:
Expand All @@ -41,14 +44,6 @@ runs:
id: script
with:
script: |
const sanitizedHeadRef = '${{ inputs.head_ref }}'.replace(/[^a-zA-Z0-9]/g, '-');

const manifestNames = [
'${{ inputs.name }}_sha_${{ github.sha }}',
`${{ inputs.name }}_tag_${ sanitizedHeadRef }`,
'${{ inputs.name }}' // TODO: must be only on default branch
];

const settings = {
type: 's3',
region: '${{ inputs.region }}',
Expand All @@ -61,9 +56,30 @@ runs:
.map(([key, value]) => `${key}=${value}`)
.join(',');

const cacheFrom = manifestNames
const sanitizedHeadRef = '${{ inputs.head_ref }}'.replace(/[^a-zA-Z0-9]/g, '-');

const shaManifestName = '${{ inputs.name }}_sha_${{ github.sha }}';
const headRefManifestName = '${{ inputs.name }}_tag_${ sanitizedHeadRef }';

const cacheFromManifestNames = [
shaManifestName,
headRefManifestName,
'${{ inputs.name }}',
];

const cacheFrom = cacheFromManifestNames
.map((name) => `${settingsString},name=${name}`)
.join('\n');

core.setOutput('cache_from', cacheFrom);
core.setOutput('cache_to', `${settingsString},mode=${{ inputs.mode }},name=${manifestNames.join(';')}`);

const cacheToManifestNames = [
shaManifestName,
headRefManifestName,
];

if ('${{ inputs.cache_to_name }}' === 'true') {
cacheToManifestNames.push('${{ inputs.name }}');
}

core.setOutput('cache_to', `${settingsString},mode=${{ inputs.mode }},name=${cacheToManifestNames.join(';')}`);
1 change: 1 addition & 0 deletions .github/workflows/tests-build-image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,3 +48,4 @@ jobs:
region: ${{ secrets.AWS_REGION }}
aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
cache_to_name: ${{ github.event_name == 'push' && 'true' || 'false' }}
shumkov marked this conversation as resolved.
Show resolved Hide resolved
71 changes: 14 additions & 57 deletions .pnp.cjs

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,8 @@
"ws": "^8.17.1",
"body-parser": "^1.20.3",
"path-to-regexp": "^1.9.0",
"cookie": "^0.7.0"
"cookie": "^0.7.0",
"cross-spawn": "^7.0.5"
},
"dependencies": {
"node-gyp": "^10.0.1"
Expand Down
53 changes: 5 additions & 48 deletions yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -6273,27 +6273,14 @@ __metadata:
languageName: node
linkType: hard

"cross-spawn@npm:7.0.3, cross-spawn@npm:^7.0.0, cross-spawn@npm:^7.0.1, cross-spawn@npm:^7.0.2, cross-spawn@npm:^7.0.3":
version: 7.0.3
resolution: "cross-spawn@npm:7.0.3"
"cross-spawn@npm:^7.0.5":
version: 7.0.5
resolution: "cross-spawn@npm:7.0.5"
dependencies:
path-key: "npm:^3.1.0"
shebang-command: "npm:^2.0.0"
which: "npm:^2.0.1"
checksum: e1a13869d2f57d974de0d9ef7acbf69dc6937db20b918525a01dacb5032129bd552d290d886d981e99f1b624cb03657084cc87bd40f115c07ecf376821c729ce
languageName: node
linkType: hard

"cross-spawn@npm:^6.0.5":
version: 6.0.5
resolution: "cross-spawn@npm:6.0.5"
dependencies:
nice-try: "npm:^1.0.4"
path-key: "npm:^2.0.1"
semver: "npm:^5.5.0"
shebang-command: "npm:^1.2.0"
which: "npm:^1.2.9"
checksum: f07e643b4875f26adffcd7f13bc68d9dff20cf395f8ed6f43a23f3ee24fc3a80a870a32b246fd074e514c8fd7da5f978ac6a7668346eec57aa87bac89c1ed3a1
checksum: c95062469d4bdbc1f099454d01c0e77177a3733012d41bf907a71eb8d22d2add43b5adf6a0a14ef4e7feaf804082714d6c262ef4557a1c480b86786c120d18e2
languageName: node
linkType: hard

Expand Down Expand Up @@ -11450,13 +11437,6 @@ __metadata:
languageName: node
linkType: hard

"nice-try@npm:^1.0.4":
version: 1.0.5
resolution: "nice-try@npm:1.0.5"
checksum: 0b4af3b5bb5d86c289f7a026303d192a7eb4417231fe47245c460baeabae7277bcd8fd9c728fb6bd62c30b3e15cd6620373e2cf33353b095d8b403d3e8a15aff
languageName: node
linkType: hard

"nise@npm:^5.1.5":
version: 5.1.5
resolution: "nise@npm:5.1.5"
Expand Down Expand Up @@ -12566,13 +12546,6 @@ __metadata:
languageName: node
linkType: hard

"path-key@npm:^2.0.1":
version: 2.0.1
resolution: "path-key@npm:2.0.1"
checksum: 6e654864e34386a2a8e6bf72cf664dcabb76574dd54013add770b374384d438aca95f4357bb26935b514a4e4c2c9b19e191f2200b282422a76ee038b9258c5e7
languageName: node
linkType: hard

"path-key@npm:^3.0.0, path-key@npm:^3.1.0":
version: 3.1.1
resolution: "path-key@npm:3.1.1"
Expand Down Expand Up @@ -13893,15 +13866,6 @@ __metadata:
languageName: node
linkType: hard

"shebang-command@npm:^1.2.0":
version: 1.2.0
resolution: "shebang-command@npm:1.2.0"
dependencies:
shebang-regex: "npm:^1.0.0"
checksum: 9eed1750301e622961ba5d588af2212505e96770ec376a37ab678f965795e995ade7ed44910f5d3d3cb5e10165a1847f52d3348c64e146b8be922f7707958908
languageName: node
linkType: hard

"shebang-command@npm:^2.0.0":
version: 2.0.0
resolution: "shebang-command@npm:2.0.0"
Expand All @@ -13911,13 +13875,6 @@ __metadata:
languageName: node
linkType: hard

"shebang-regex@npm:^1.0.0":
version: 1.0.0
resolution: "shebang-regex@npm:1.0.0"
checksum: 404c5a752cd40f94591dfd9346da40a735a05139dac890ffc229afba610854d8799aaa52f87f7e0c94c5007f2c6af55bdcaeb584b56691926c5eaf41dc8f1372
languageName: node
linkType: hard

"shebang-regex@npm:^3.0.0":
version: 3.0.0
resolution: "shebang-regex@npm:3.0.0"
Expand Down Expand Up @@ -16047,7 +16004,7 @@ __metadata:
languageName: node
linkType: hard

"which@npm:^1.2.1, which@npm:^1.2.9":
"which@npm:^1.2.1":
version: 1.3.1
resolution: "which@npm:1.3.1"
dependencies:
Expand Down
Loading