From 5320ddc712eaa6779357e60b178a6d2ea511b710 Mon Sep 17 00:00:00 2001 From: Tobias Gabriel Date: Tue, 20 Aug 2024 11:39:45 +0200 Subject: [PATCH 1/4] feat: fallback to "raw" endpoint for manifest when rate limit is reached --- __tests__/setup-go.test.ts | 23 +++++++++++++++++++++ dist/setup/index.js | 30 +++++++++++++++++++++++++++- src/installer.ts | 41 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 91 insertions(+), 3 deletions(-) diff --git a/__tests__/setup-go.test.ts b/__tests__/setup-go.test.ts index 70f2166eb..dcf33ecfb 100644 --- a/__tests__/setup-go.test.ts +++ b/__tests__/setup-go.test.ts @@ -7,6 +7,7 @@ import osm, {type} from 'os'; import path from 'path'; import * as main from '../src/main'; import * as im from '../src/installer'; +import * as httpm from '@actions/http-client'; import goJsonData from './data/golang-dl.json'; import matchers from '../matchers.json'; @@ -46,6 +47,7 @@ describe('setup-go', () => { let execSpy: jest.SpyInstance; let getManifestSpy: jest.SpyInstance; let getAllVersionsSpy: jest.SpyInstance; + let httpmGetJsonSpy: jest.SpyInstance; beforeAll(async () => { process.env['GITHUB_ENV'] = ''; // Stub out Environment file functionality so we can verify it writes to standard out (toolkit is backwards compatible) @@ -90,6 +92,9 @@ describe('setup-go', () => { getManifestSpy = jest.spyOn(tc, 'getManifestFromRepo'); getAllVersionsSpy = jest.spyOn(im, 'getManifest'); + // httm + httpmGetJsonSpy = jest.spyOn(httpm.HttpClient.prototype, 'getJson') + // io whichSpy = jest.spyOn(io, 'which'); existsSpy = jest.spyOn(fs, 'existsSync'); @@ -151,6 +156,23 @@ describe('setup-go', () => { ); }); + it('should return manifest from repo', async () => { + const manifest = await im.getManifest(undefined); + expect(manifest).toEqual(goTestManifest); + }); + + it('should return manifest from raw URL if repo fetch fails', async () => { + getManifestSpy.mockRejectedValue( + new Error('Fetch failed') + ); + httpmGetJsonSpy.mockResolvedValue({ + result: goTestManifest + }); + const manifest = await im.getManifest(undefined); + expect(httpmGetJsonSpy).toHaveBeenCalled(); + expect(manifest).toEqual(goTestManifest); + }); + it('can find 1.9 from manifest on linux', async () => { os.platform = 'linux'; os.arch = 'x64'; @@ -790,6 +812,7 @@ describe('setup-go', () => { getManifestSpy.mockImplementation(() => { throw new Error('Unable to download manifest'); }); + httpmGetJsonSpy.mockRejectedValue(new Error('Unable to download manifest from raw URL')); getAllVersionsSpy.mockImplementationOnce(() => undefined); dlSpy.mockImplementation(async () => '/some/temp/path'); diff --git a/dist/setup/index.js b/dist/setup/index.js index 9ad3784ea..1b5e76fb1 100644 --- a/dist/setup/index.js +++ b/dist/setup/index.js @@ -88254,6 +88254,10 @@ const sys = __importStar(__nccwpck_require__(5632)); const fs_1 = __importDefault(__nccwpck_require__(7147)); const os_1 = __importDefault(__nccwpck_require__(2037)); const utils_1 = __nccwpck_require__(1314); +const MANIFEST_REPO_OWNER = 'actions'; +const MANIFEST_REPO_NAME = 'go-versions'; +const MANIFEST_REPO_BRANCH = 'main'; +const MANIFEST_URL = `https://raw.githubusercontent.com/${MANIFEST_REPO_OWNER}/${MANIFEST_REPO_NAME}/${MANIFEST_REPO_BRANCH}/versions-manifest.json`; function getGo(versionSpec_1, checkLatest_1, auth_1) { return __awaiter(this, arguments, void 0, function* (versionSpec, checkLatest, auth, arch = os_1.default.arch()) { var _a; @@ -88428,10 +88432,34 @@ function extractGoArchive(archivePath) { exports.extractGoArchive = extractGoArchive; function getManifest(auth) { return __awaiter(this, void 0, void 0, function* () { - return tc.getManifestFromRepo('actions', 'go-versions', auth, 'main'); + try { + return yield getManifestFromRepo(auth); + } + catch (err) { + core.debug('Fetching the manifest via the API failed.'); + if (err instanceof Error) { + core.debug(err.message); + } + } + return yield getManifestFromURL(); }); } exports.getManifest = getManifest; +function getManifestFromRepo(auth) { + core.debug(`Getting manifest from ${MANIFEST_REPO_OWNER}/${MANIFEST_REPO_NAME}@${MANIFEST_REPO_BRANCH}`); + return tc.getManifestFromRepo(MANIFEST_REPO_OWNER, MANIFEST_REPO_NAME, auth, MANIFEST_REPO_BRANCH); +} +function getManifestFromURL() { + return __awaiter(this, void 0, void 0, function* () { + core.debug('Falling back to fetching the manifest using raw URL.'); + const http = new httpm.HttpClient('tool-cache'); + const response = yield http.getJson(MANIFEST_URL); + if (!response.result) { + throw new Error(`Unable to get manifest from ${MANIFEST_URL}`); + } + return response.result; + }); +} function getInfoFromManifest(versionSpec_1, stable_1, auth_1) { return __awaiter(this, arguments, void 0, function* (versionSpec, stable, auth, arch = os_1.default.arch(), manifest) { let info = null; diff --git a/src/installer.ts b/src/installer.ts index 817c334f6..c8fc028b2 100644 --- a/src/installer.ts +++ b/src/installer.ts @@ -8,6 +8,12 @@ import fs from 'fs'; import os from 'os'; import {StableReleaseAlias} from './utils'; +const MANIFEST_REPO_OWNER = 'actions'; +const MANIFEST_REPO_NAME = 'go-versions'; +const MANIFEST_REPO_BRANCH = 'main'; +const MANIFEST_URL = `https://raw.githubusercontent.com/${MANIFEST_REPO_OWNER}/${MANIFEST_REPO_NAME}/${MANIFEST_REPO_BRANCH}/versions-manifest.json`; + + type InstallationType = 'dist' | 'manifest'; export interface IGoVersionFile { @@ -274,8 +280,39 @@ export async function extractGoArchive(archivePath: string): Promise { return extPath; } -export async function getManifest(auth: string | undefined) { - return tc.getManifestFromRepo('actions', 'go-versions', auth, 'main'); +export async function getManifest(auth: string | undefined): Promise { + try { + return await getManifestFromRepo(auth); + } catch (err) { + core.debug('Fetching the manifest via the API failed.'); + if (err instanceof Error) { + core.debug(err.message); + } + } + return await getManifestFromURL(); +} + +function getManifestFromRepo(auth: string | undefined): Promise { + core.debug( + `Getting manifest from ${MANIFEST_REPO_OWNER}/${MANIFEST_REPO_NAME}@${MANIFEST_REPO_BRANCH}` + ); + return tc.getManifestFromRepo( + MANIFEST_REPO_OWNER, + MANIFEST_REPO_NAME, + auth, + MANIFEST_REPO_BRANCH + ); +} + +async function getManifestFromURL(): Promise { + core.debug('Falling back to fetching the manifest using raw URL.'); + + const http: httpm.HttpClient = new httpm.HttpClient('tool-cache'); + const response = await http.getJson(MANIFEST_URL); + if (!response.result) { + throw new Error(`Unable to get manifest from ${MANIFEST_URL}`); + } + return response.result; } export async function getInfoFromManifest( From db66857bcd8508d9e1db53b6edad944951647045 Mon Sep 17 00:00:00 2001 From: Tobias Gabriel Date: Thu, 14 Nov 2024 08:26:13 +0100 Subject: [PATCH 2/4] add information about raw access to the README --- README.md | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 44726a602..b2e6de425 100644 --- a/README.md +++ b/README.md @@ -242,18 +242,14 @@ documentation. ## Using `setup-go` on GHES -`setup-go` comes pre-installed on the appliance with GHES if Actions is enabled. When dynamically downloading Go -distributions, `setup-go` downloads distributions from [`actions/go-versions`](https://github.com/actions/go-versions) -on github.com (outside of the appliance). These calls to `actions/go-versions` are made via unauthenticated requests, -which are limited -to [60 requests per hour per IP](https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting). If -more requests are made within the time frame, then you will start to see rate-limit errors during downloading that looks -like: `##[error]API rate limit exceeded for...`. After that error the action will try to download versions directly -from https://storage.googleapis.com/golang, but it also can have rate limit so it's better to put token. - -To get a higher rate limit, you -can [generate a personal access token on github.com](https://github.com/settings/tokens/new) and pass it as the `token` -input for the action: +`setup-go` comes pre-installed on the appliance with GHES if Actions is enabled. +When dynamically downloading Go distributions, `setup-go` downloads distributions from [`actions/go-versions`](https://github.com/actions/go-versions) on github.com (outside of the appliance). + +These calls to `actions/go-versions` are made via unauthenticated requests, which are limited to [60 requests per hour per IP](https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting). +If more requests are made within the time frame, then the action leverages the `raw API` to retrieve the version-manifest. This approach does not impose a rate limit and hence facilitates unrestricted consumption. This is particularly beneficial for GHES runners, which often share the same IP, to avoid the quick exhaustion of the unauthenticated rate limit. +If that fails as well the action will try to download versions directly from https://storage.googleapis.com/golang. + +If that fails as well you can get a higher rate limit with [generating a personal access token on github.com](https://github.com/settings/tokens/new) and passing it as the `token` input to the action: ```yaml uses: actions/setup-go@v5 @@ -262,8 +258,7 @@ with: go-version: '1.18' ``` -If the runner is not able to access github.com, any Go versions requested during a workflow run must come from the -runner's tool cache. +If the runner is not able to access github.com, any Go versions requested during a workflow run must come from the runner's tool cache. See "[Setting up the tool cache on self-hosted runners without internet access](https://docs.github.com/en/enterprise-server@3.2/admin/github-actions/managing-access-to-actions-from-githubcom/setting-up-the-tool-cache-on-self-hosted-runners-without-internet-access)" for more information. From 81d8c17112a7c72669661bf357689b02859650c8 Mon Sep 17 00:00:00 2001 From: Tobias Gabriel Date: Fri, 22 Nov 2024 08:02:11 +0100 Subject: [PATCH 3/4] prettier --- __tests__/setup-go.test.ts | 10 +++++----- src/installer.ts | 9 ++++++--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/__tests__/setup-go.test.ts b/__tests__/setup-go.test.ts index dcf33ecfb..f94f9eebe 100644 --- a/__tests__/setup-go.test.ts +++ b/__tests__/setup-go.test.ts @@ -93,7 +93,7 @@ describe('setup-go', () => { getAllVersionsSpy = jest.spyOn(im, 'getManifest'); // httm - httpmGetJsonSpy = jest.spyOn(httpm.HttpClient.prototype, 'getJson') + httpmGetJsonSpy = jest.spyOn(httpm.HttpClient.prototype, 'getJson'); // io whichSpy = jest.spyOn(io, 'which'); @@ -162,9 +162,7 @@ describe('setup-go', () => { }); it('should return manifest from raw URL if repo fetch fails', async () => { - getManifestSpy.mockRejectedValue( - new Error('Fetch failed') - ); + getManifestSpy.mockRejectedValue(new Error('Fetch failed')); httpmGetJsonSpy.mockResolvedValue({ result: goTestManifest }); @@ -812,7 +810,9 @@ describe('setup-go', () => { getManifestSpy.mockImplementation(() => { throw new Error('Unable to download manifest'); }); - httpmGetJsonSpy.mockRejectedValue(new Error('Unable to download manifest from raw URL')); + httpmGetJsonSpy.mockRejectedValue( + new Error('Unable to download manifest from raw URL') + ); getAllVersionsSpy.mockImplementationOnce(() => undefined); dlSpy.mockImplementation(async () => '/some/temp/path'); diff --git a/src/installer.ts b/src/installer.ts index c8fc028b2..cc0f2dc30 100644 --- a/src/installer.ts +++ b/src/installer.ts @@ -13,7 +13,6 @@ const MANIFEST_REPO_NAME = 'go-versions'; const MANIFEST_REPO_BRANCH = 'main'; const MANIFEST_URL = `https://raw.githubusercontent.com/${MANIFEST_REPO_OWNER}/${MANIFEST_REPO_NAME}/${MANIFEST_REPO_BRANCH}/versions-manifest.json`; - type InstallationType = 'dist' | 'manifest'; export interface IGoVersionFile { @@ -280,7 +279,9 @@ export async function extractGoArchive(archivePath: string): Promise { return extPath; } -export async function getManifest(auth: string | undefined): Promise { +export async function getManifest( + auth: string | undefined +): Promise { try { return await getManifestFromRepo(auth); } catch (err) { @@ -292,7 +293,9 @@ export async function getManifest(auth: string | undefined): Promise { +function getManifestFromRepo( + auth: string | undefined +): Promise { core.debug( `Getting manifest from ${MANIFEST_REPO_OWNER}/${MANIFEST_REPO_NAME}@${MANIFEST_REPO_BRANCH}` ); From 3f9525a07543eebc35e5ddd8f10070e7a13d32df Mon Sep 17 00:00:00 2001 From: Tobias Gabriel Date: Fri, 22 Nov 2024 12:03:38 +0100 Subject: [PATCH 4/4] update cross-spawn to 7.0.6 to fix vulnerability --- package-lock.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index c51f97c27..8a06fd920 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2504,10 +2504,11 @@ } }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, + "license": "MIT", "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0",