diff --git a/src/github/client/github-client.types.ts b/src/github/client/github-client.types.ts index 642a2a232d..e8eb583727 100644 --- a/src/github/client/github-client.types.ts +++ b/src/github/client/github-client.types.ts @@ -30,6 +30,13 @@ export type GetPullRequestParams = { page?: number; } +export type GetSecretScanningAlertRequestParams = { + sort?: string; + direction?: string; + per_page?: number; + page?: number; +} + export type GraphQlQueryResponse = { data: ResponseData; errors?: GraphQLError[]; diff --git a/src/github/client/github-installation-client.ts b/src/github/client/github-installation-client.ts index a883f74d02..a4a5b8c231 100644 --- a/src/github/client/github-installation-client.ts +++ b/src/github/client/github-installation-client.ts @@ -29,6 +29,7 @@ import { ActionsListRepoWorkflowRunsResponseEnhanced, CreateReferenceBody, GetPullRequestParams, + GetSecretScanningAlertRequestParams, PaginatedAxiosResponse, ReposGetContentsResponse } from "./github-client.types"; @@ -38,6 +39,7 @@ import { GithubClientError, GithubClientGraphQLError } from "~/src/github/client import { cloneDeep } from "lodash"; import { BooleanFlags, booleanFlag } from "config/feature-flags"; import { logCurlOutputInChunks, runCurl } from "utils/curl/curl-utils"; +import { SecretScanningAlertResponseItem } from "./secret-scanning-alert.types"; // Unfortunately, the type is not exposed in Octokit... // https://docs.github.com/en/rest/pulls/review-requests?apiVersion=2022-11-28#get-all-requested-reviewers-for-a-pull-request @@ -74,6 +76,13 @@ export class GitHubInstallationClient extends GitHubClient { this.gitHubServerAppId = gshaId; } + + public async getSecretScanningAlerts(owner: string, repo: string, secretScanningAlertRequestParams: GetSecretScanningAlertRequestParams): Promise> { + return await this.get(`/repos/{owner}/{repo}/secret-scanning/alerts`, secretScanningAlertRequestParams, { + owner, + repo + }); + } /** * Lists pull requests for the given repository. */ diff --git a/src/github/client/secret-scanning-alert.types.ts b/src/github/client/secret-scanning-alert.types.ts new file mode 100644 index 0000000000..806143449a --- /dev/null +++ b/src/github/client/secret-scanning-alert.types.ts @@ -0,0 +1,13 @@ +export type SecretScanningAlertResponseItem = { + number: number, + created_at: string, + url: string, + html_url: string, + locations_url: string, + state: "open" | "resolved", + resolution?: string, + resolved_at?: string, + resolution_comment?: string, + secret_type: string, + secret_type_display_name: string + } \ No newline at end of file diff --git a/src/github/installation.ts b/src/github/installation.ts index f801c2500c..602a6b0a64 100644 --- a/src/github/installation.ts +++ b/src/github/installation.ts @@ -75,7 +75,7 @@ export const installationWebhookHandler = async ( ); } catch (err) { - logger.warn({ err }, "Failed to submit security workspace to Jira or trigger backfill via backfill"); + logger.warn({ err }, "Failed to submit security workspace to Jira or trigger backfill"); const webhookReceived = context.webhookReceived; webhookReceived && emitWebhookProcessedMetrics( new Date(webhookReceived).getTime(), diff --git a/src/github/secret-scanning-alert.test.ts b/src/github/secret-scanning-alert.test.ts index 701d7ad6c5..3d96d86aaf 100644 --- a/src/github/secret-scanning-alert.test.ts +++ b/src/github/secret-scanning-alert.test.ts @@ -136,7 +136,7 @@ describe("SecretScanningAlertWebhookHandler", () => { vulnerabilities: [ { schemaVersion: "1.0", - id: "d-456-123", + id: "s-456-123", updateSequenceNumber: Date.now(), containerId: "456", displayName: "personal_access_token secret exposed", @@ -152,10 +152,7 @@ describe("SecretScanningAlertWebhookHandler", () => { "displayName": "personal_access_token", "url":SAMPLE_SECURITY_URL }], - status: JIRA_VULNERABILITY_STATUS_ENUM_OPEN, - additionalInfo: { - content: "personal_access_token" - } + status: JIRA_VULNERABILITY_STATUS_ENUM_OPEN } ] }; diff --git a/src/interfaces/jira.ts b/src/interfaces/jira.ts index b6deb57068..42673cb7f7 100644 --- a/src/interfaces/jira.ts +++ b/src/interfaces/jira.ts @@ -210,7 +210,7 @@ export interface JiraVulnerability { severity: JiraVulnerabilitySeverity; identifiers: JiraVulnerabilityIdentifier[]; status: JiraVulnerabilityStatusEnum; - additionalInfo: JiraVulnerabilityAdditionalInfo; + additionalInfo?: JiraVulnerabilityAdditionalInfo; } export interface JiraVulnerabilitySeverity { diff --git a/src/models/reposyncstate.ts b/src/models/reposyncstate.ts index f19a90c924..e5bede09d3 100644 --- a/src/models/reposyncstate.ts +++ b/src/models/reposyncstate.ts @@ -35,6 +35,7 @@ export interface RepoSyncStateProperties { buildStatus?: TaskStatus; deploymentStatus?: TaskStatus; dependabotAlertStatus?: TaskStatus; + secretScanningAlertStatus?: TaskStatus, branchCursor?: string; commitCursor?: string; issueCursor?: string; @@ -42,12 +43,14 @@ export interface RepoSyncStateProperties { buildCursor?: string; deploymentCursor?: string; dependabotAlertCursor?: string; + secretScanningAlertCursor?: string; commitFrom?: Date; branchFrom?: Date; pullFrom?: Date; buildFrom?: Date; deploymentFrom?: Date; dependabotAlertFrom?: Date; + secretScanningAlertFrom?: Date; forked?: boolean; repoPushedAt: Date; repoUpdatedAt: Date; @@ -76,6 +79,7 @@ export class RepoSyncState extends Model implements RepoSyncStateProperties { buildStatus?: TaskStatus; deploymentStatus?: TaskStatus; dependabotAlertStatus?: TaskStatus; + secretScanningAlertStatus?: TaskStatus; branchCursor?: string; commitCursor?: string; issueCursor?: string; @@ -83,12 +87,14 @@ export class RepoSyncState extends Model implements RepoSyncStateProperties { buildCursor?: string; deploymentCursor?: string; dependabotAlertCursor?: string; + secretScanningAlertCursor?: string; commitFrom?: Date; branchFrom?: Date; pullFrom?: Date; buildFrom?: Date; deploymentFrom?: Date; dependabotAlertFrom?: Date; + secretScanningAlertFrom?: Date; forked?: boolean; repoPushedAt: Date; repoUpdatedAt: Date; @@ -135,7 +141,8 @@ export class RepoSyncState extends Model implements RepoSyncStateProperties { commitStatus: "failed", buildStatus: "failed", deploymentStatus: "failed", - dependabotAlertStatus: "failed" + dependabotAlertStatus: "failed", + secretScanningAlertStatus: "failed" } } }); @@ -152,7 +159,8 @@ export class RepoSyncState extends Model implements RepoSyncStateProperties { commitStatus: "failed", buildStatus: "failed", deploymentStatus: "failed", - dependabotAlertStatus: "failed" + dependabotAlertStatus: "failed", + secretScanningAlertStatus: "failed" } } })); @@ -362,7 +370,10 @@ export class RepoSyncState extends Model implements RepoSyncStateProperties { commitFrom: null, dependabotAlertStatus: null, dependabotAlertCursor: null, - dependabotAlertFrom: null + dependabotAlertFrom: null, + secretScanningAlertFrom: null, + secretScanningAlertStatus: null, + secretScanningAlertCursor: null }, { where: { subscriptionId: subscription.id @@ -410,6 +421,7 @@ RepoSyncState.init({ buildStatus: DataTypes.ENUM("pending", "complete", "failed"), deploymentStatus: DataTypes.ENUM("pending", "complete", "failed"), dependabotAlertStatus: DataTypes.ENUM("pending", "complete", "failed"), + secretScanningAlertStatus: DataTypes.ENUM("pending", "complete", "failed"), branchCursor: STRING, commitCursor: STRING, issueCursor: STRING, @@ -417,12 +429,14 @@ RepoSyncState.init({ buildCursor: STRING, deploymentCursor: STRING, dependabotAlertCursor: STRING, + secretScanningAlertCursor: STRING, commitFrom: DATE, branchFrom: DATE, pullFrom: DATE, buildFrom: DATE, deploymentFrom: DATE, dependabotAlertFrom: DATE, + secretScanningAlertFrom: DATE, forked: BOOLEAN, repoPushedAt: DATE, repoUpdatedAt: DATE, diff --git a/src/routes/jira/sync/jira-sync-post.test.ts b/src/routes/jira/sync/jira-sync-post.test.ts index 57d3907814..acd734de5c 100644 --- a/src/routes/jira/sync/jira-sync-post.test.ts +++ b/src/routes/jira/sync/jira-sync-post.test.ts @@ -132,7 +132,7 @@ describe("sync", () => { installationId: installationIdForServer, jiraHost, commitsFromDate: commitsFromDate.toISOString(), - targetTasks: ["pull", "branch", "commit", "build", "deployment", "dependabotAlert"], + targetTasks: ["pull", "branch", "commit", "build", "deployment", "dependabotAlert", "secretScanningAlert"], gitHubAppConfig: expect.objectContaining({ gitHubAppId: gitHubServerApp.id, uuid: gitHubServerApp.uuid }) }), expect.anything(), expect.anything()); }); diff --git a/src/routes/jira/sync/jira-sync-post.ts b/src/routes/jira/sync/jira-sync-post.ts index b6035ef6f3..0c3c24813b 100644 --- a/src/routes/jira/sync/jira-sync-post.ts +++ b/src/routes/jira/sync/jira-sync-post.ts @@ -85,5 +85,5 @@ const determineSyncTypeAndTargetTasks = async (syncTypeFromReq: string, subscrip return { syncType: "full", targetTasks: undefined }; } - return { syncType: "partial", targetTasks: ["pull", "branch", "commit", "build", "deployment", "dependabotAlert"] }; + return { syncType: "partial", targetTasks: ["pull", "branch", "commit", "build", "deployment", "dependabotAlert", "secretScanningAlert"] }; }; diff --git a/src/sync/installation.test.ts b/src/sync/installation.test.ts index 47bcc071da..3b4d360d4c 100644 --- a/src/sync/installation.test.ts +++ b/src/sync/installation.test.ts @@ -329,8 +329,8 @@ describe("sync/installation", () => { describe("getTargetTasks", () => { it("should return all tasks if no target tasks present", async () => { - expect(getTargetTasks()).toEqual(["pull", "branch", "commit", "build", "deployment", "dependabotAlert"]); - expect(getTargetTasks([])).toEqual(["pull", "branch", "commit", "build", "deployment", "dependabotAlert"]); + expect(getTargetTasks()).toEqual(["pull", "branch", "commit", "build", "deployment", "dependabotAlert", "secretScanningAlert"]); + expect(getTargetTasks([])).toEqual(["pull", "branch", "commit", "build", "deployment", "dependabotAlert", "secretScanningAlert"]); }); it("should return single target task", async () => { @@ -566,7 +566,9 @@ describe("sync/installation", () => { expect(repoSync.dependabotAlertFrom).toBeNull(); }); - describe.each(["pull", "commit", "build", "deployment", "dependabotAlert"] as TaskType[])("Update jobs status for each tasks", (taskType: TaskType) => { + describe.each( + ["pull", "commit", "build", "deployment", "dependabotAlert", "secretScanningAlert"] as TaskType[] + )("Update jobs status for each tasks", (taskType: TaskType) => { const colTaskFrom = `${taskType}From`; const task: Task = { task: taskType, diff --git a/src/sync/installation.ts b/src/sync/installation.ts index cbf76b6d71..1237289c83 100644 --- a/src/sync/installation.ts +++ b/src/sync/installation.ts @@ -26,6 +26,7 @@ import { sendAnalytics } from "utils/analytics-client"; import { AnalyticsEventTypes, AnalyticsTrackEventsEnum } from "interfaces/common"; import { getNextTasks } from "~/src/sync/scheduler"; import { getDependabotAlertTask } from "./dependabot-alerts"; +import { getSecretScanningAlertTask } from "./secret-scanning-alerts"; const tasks: TaskProcessors = { repository: getRepositoryTask, @@ -34,10 +35,11 @@ const tasks: TaskProcessors = { commit: getCommitTask, build: getBuildTask, deployment: getDeploymentTask, - dependabotAlert: getDependabotAlertTask + dependabotAlert: getDependabotAlertTask, + secretScanningAlert: getSecretScanningAlertTask }; -const allTaskTypes: TaskType[] = ["pull", "branch", "commit", "build", "deployment", "dependabotAlert"]; +const allTaskTypes: TaskType[] = ["pull", "branch", "commit", "build", "deployment", "dependabotAlert", "secretScanningAlert"]; const allTasksExceptBranch = without(allTaskTypes, "branch"); export const getTargetTasks = (targetTasks?: TaskType[]): TaskType[] => { @@ -235,9 +237,11 @@ const sendPayloadToJira = async (task: TaskType, jiraClient, jiraPayload, reposi }); break; case "dependabotAlert": + case "secretScanningAlert":{ await jiraClient.security.submitVulnerabilities(jiraPayload, { operationType: "BACKFILL" }); + } break; default: await jiraClient.devinfo.repository.update(jiraPayload, { diff --git a/src/sync/scheduler.test.ts b/src/sync/scheduler.test.ts index ad63eeeab6..0dedd5ac7d 100644 --- a/src/sync/scheduler.test.ts +++ b/src/sync/scheduler.test.ts @@ -241,7 +241,7 @@ describe("scheduler", () => { expect(task.task).toEqual("commit"); }); }); - it("should not filter by dependabot alerts task if ENABLE_GITHUB_SECURITY_IN_JIRA FF is off", async () => { + it("should filter dependabot alerts task if ENABLE_GITHUB_SECURITY_IN_JIRA FF is off", async () => { when(booleanFlag).calledWith(BooleanFlags.ENABLE_GITHUB_SECURITY_IN_JIRA, expect.anything()).mockResolvedValue(false); configureRateLimit(10000, 10000); const repoSyncStates = await RepoSyncState.findAllFromSubscription(subscription); @@ -255,7 +255,7 @@ describe("scheduler", () => { expect(tasks.otherTasks.length).toEqual(0); }); - it("should not filter by dependabot alerts task if ENABLE_GITHUB_SECURITY_IN_JIRA FF is on", async () => { + it("should not filter dependabot alerts task if ENABLE_GITHUB_SECURITY_IN_JIRA FF is on", async () => { when(booleanFlag).calledWith(BooleanFlags.ENABLE_GITHUB_SECURITY_IN_JIRA, expect.anything()).mockResolvedValue(true); configureRateLimit(10000, 10000); const repoSyncStates = await RepoSyncState.findAllFromSubscription(subscription); @@ -271,4 +271,33 @@ describe("scheduler", () => { expect(task.task).toEqual("dependabotAlert"); }); }); + it("should filter secret scanning alerts task if ENABLE_GITHUB_SECURITY_IN_JIRA FF is off", async () => { + when(booleanFlag).calledWith(BooleanFlags.ENABLE_GITHUB_SECURITY_IN_JIRA, expect.anything()).mockResolvedValue(false); + configureRateLimit(10000, 10000); + const repoSyncStates = await RepoSyncState.findAllFromSubscription(subscription); + await Promise.all(repoSyncStates.map((record) => { + record.secretScanningAlertStatus = "pending"; + return record.save(); + })); + githubUserTokenNock(DatabaseStateCreator.GITHUB_INSTALLATION_ID); + const tasks = await getNextTasks(subscription, ["secretScanningAlert"], getLogger("test")); + expect(tasks.mainTask).toBeUndefined(); + expect(tasks.otherTasks.length).toEqual(0); + }); + it("should not filter secret scanning alerts task if ENABLE_GITHUB_SECURITY_IN_JIRA FF is on", async () => { + when(booleanFlag).calledWith(BooleanFlags.ENABLE_GITHUB_SECURITY_IN_JIRA, expect.anything()).mockResolvedValue(true); + configureRateLimit(10000, 10000); + const repoSyncStates = await RepoSyncState.findAllFromSubscription(subscription); + await Promise.all(repoSyncStates.map((record) => { + record.secretScanningAlertStatus = "pending"; + return record.save(); + })); + + githubUserTokenNock(DatabaseStateCreator.GITHUB_INSTALLATION_ID); + const tasks = await getNextTasks(subscription, ["secretScanningAlert"], getLogger("test")); + expect(tasks.mainTask!.task).toEqual("secretScanningAlert"); + tasks.otherTasks.forEach(task => { + expect(task.task).toEqual("secretScanningAlert"); + }); + }); }); diff --git a/src/sync/scheduler.ts b/src/sync/scheduler.ts index 97f0151668..15fe89e62e 100644 --- a/src/sync/scheduler.ts +++ b/src/sync/scheduler.ts @@ -172,7 +172,7 @@ export const getNextTasks = async (subscription: Subscription, targetTasks: Task let tasks = getTargetTasks(targetTasks); if (!await booleanFlag(BooleanFlags.ENABLE_GITHUB_SECURITY_IN_JIRA, subscription.jiraHost)) { - tasks = without(tasks, "dependabotAlert"); + tasks = without(tasks, "dependabotAlert", "secretScanningAlert"); } const nSubTasks = await estimateNumberOfSubtasks(subscription, logger); diff --git a/src/sync/secret-scanning-alerts.test.ts b/src/sync/secret-scanning-alerts.test.ts new file mode 100644 index 0000000000..3e01b852dc --- /dev/null +++ b/src/sync/secret-scanning-alerts.test.ts @@ -0,0 +1,222 @@ +import { DatabaseStateCreator } from "~/test/utils/database-state-creator"; +import secretScanningAlerts from "fixtures/api/secret-scanning-alerts.json"; +import { processInstallation } from "./installation"; +import { Hub } from "@sentry/node"; +import { getLogger } from "../config/logger"; +import { BackfillMessagePayload } from "../sqs/sqs.types"; +import { waitUntil } from "~/test/utils/wait-until"; +import { BooleanFlags, booleanFlag } from "../config/feature-flags"; +import { when } from "jest-when"; +import { GitHubServerApp } from "../models/github-server-app"; + + +jest.mock("config/feature-flags"); +describe("sync/secret-scanning-alerts", () => { + + const sentry: Hub = { setUser: jest.fn() } as any; + const MOCK_SYSTEM_TIMESTAMP_SEC = 12345678; + + + describe("cloud", () => { + + const mockBackfillQueueSendMessage = jest.fn(); + + const verifyMessageSent = async (data: BackfillMessagePayload, delaySec?: number) => { + await waitUntil(async () => { + expect(githubNock).toBeDone(); + expect(jiraNock).toBeDone(); + }); + expect(mockBackfillQueueSendMessage.mock.calls).toHaveLength(1); + expect(mockBackfillQueueSendMessage.mock.calls[0][0]).toEqual(data); + expect(mockBackfillQueueSendMessage.mock.calls[0][1]).toEqual(delaySec || 0); + }; + beforeEach(async () => { + + await new DatabaseStateCreator() + .withActiveRepoSyncState() + .repoSyncStatePendingForSecretScanningAlerts() + .create(); + + mockSystemTime(MOCK_SYSTEM_TIMESTAMP_SEC); + + }); + it("should send secret scanning alerts to Jira", async () => { + when(booleanFlag).calledWith(BooleanFlags.ENABLE_GITHUB_SECURITY_IN_JIRA, expect.anything()).mockResolvedValue(true); + const data = { installationId: DatabaseStateCreator.GITHUB_INSTALLATION_ID, jiraHost }; + githubNock + .get("/repos/integrations/test-repo-name/secret-scanning/alerts?per_page=20&page=1&sort=created&direction=desc") + .reply(200, secretScanningAlerts); + githubUserTokenNock(DatabaseStateCreator.GITHUB_INSTALLATION_ID); + jiraNock + .post("/rest/security/1.0/bulk", expectedResponseCloudServer()) + .reply(200); + + await expect(processInstallation(mockBackfillQueueSendMessage)(data, sentry, getLogger("test"))).toResolve(); + await verifyMessageSent(data); + }); + + it("should not send secret scanning alerts to Jira if FF is disabled", async () => { + when(booleanFlag).calledWith(BooleanFlags.ENABLE_GITHUB_SECURITY_IN_JIRA, expect.anything()).mockResolvedValue(false); + const data = { installationId: DatabaseStateCreator.GITHUB_INSTALLATION_ID, jiraHost }; + + await expect(processInstallation(mockBackfillQueueSendMessage)(data, sentry, getLogger("test"))).toResolve(); + expect(mockBackfillQueueSendMessage).not.toBeCalled(); + }); + + it("should not call Jira if no secret scanning alerts are found", async () => { + when(booleanFlag).calledWith(BooleanFlags.ENABLE_GITHUB_SECURITY_IN_JIRA, expect.anything()).mockResolvedValue(true); + const data = { installationId: DatabaseStateCreator.GITHUB_INSTALLATION_ID, jiraHost }; + githubNock + .get("/repos/integrations/test-repo-name/secret-scanning/alerts?per_page=20&page=1&sort=created&direction=desc") + .reply(200, []); + githubUserTokenNock(DatabaseStateCreator.GITHUB_INSTALLATION_ID); + // No Jira Nock + + await expect(processInstallation(mockBackfillQueueSendMessage)(data, sentry, getLogger("test"))).toResolve(); + await verifyMessageSent(data); + }); + }); + + describe("server", () => { + + const verifyMessageSent = async (data: BackfillMessagePayload, delaySec?: number) => { + await waitUntil(async () => { + expect(gheNock).toBeDone(); + expect(jiraNock).toBeDone(); + }); + expect(mockBackfillQueueSendMessage.mock.calls).toHaveLength(1); + expect(mockBackfillQueueSendMessage.mock.calls[0][0]).toEqual(data); + expect(mockBackfillQueueSendMessage.mock.calls[0][1]).toEqual(delaySec || 0); + }; + + const mockBackfillQueueSendMessage = jest.fn(); + let gitHubServerApp: GitHubServerApp; + + beforeEach(async () => { + + const builderResult = await new DatabaseStateCreator() + .forServer() + .withActiveRepoSyncState() + .repoSyncStatePendingForSecretScanningAlerts() + .create(); + + mockSystemTime(MOCK_SYSTEM_TIMESTAMP_SEC); + + gitHubServerApp = builderResult.gitHubServerApp!; + }); + + it("should send secret scanning alerts to Jira", async () => { + when(booleanFlag).calledWith(BooleanFlags.ENABLE_GITHUB_SECURITY_IN_JIRA, expect.anything()).mockResolvedValue(true); + const data = { + installationId: DatabaseStateCreator.GITHUB_INSTALLATION_ID, + jiraHost, + gitHubAppConfig: { + uuid: gitHubServerApp.uuid, + gitHubAppId: gitHubServerApp.id, + appId: gitHubServerApp.appId, + clientId: gitHubServerApp.gitHubClientId, + gitHubBaseUrl: gitHubServerApp.gitHubBaseUrl, + gitHubApiUrl: gitHubServerApp.gitHubBaseUrl + } + }; + gheUserTokenNock(DatabaseStateCreator.GITHUB_INSTALLATION_ID); + gheNock + .get("/api/v3/repos/integrations/test-repo-name/secret-scanning/alerts?per_page=20&page=1&sort=created&direction=desc") + .reply(200, secretScanningAlerts); + jiraNock + .post("/rest/security/1.0/bulk", expectedResponseGHEServer()) + .reply(200); + + await expect(processInstallation(mockBackfillQueueSendMessage)(data, sentry, getLogger("test"))).toResolve(); + await verifyMessageSent(data); + }); + + it("should not call Jira if no secret scanning alerts are found", async () => { + when(booleanFlag).calledWith(BooleanFlags.ENABLE_GITHUB_SECURITY_IN_JIRA, expect.anything()).mockResolvedValue(true); + const data = { + installationId: DatabaseStateCreator.GITHUB_INSTALLATION_ID, + jiraHost, + gitHubAppConfig: { + uuid: gitHubServerApp.uuid, + gitHubAppId: gitHubServerApp.id, + appId: gitHubServerApp.appId, + clientId: gitHubServerApp.gitHubClientId, + gitHubBaseUrl: gitHubServerApp.gitHubBaseUrl, + gitHubApiUrl: gitHubServerApp.gitHubBaseUrl + "/v3/api" + } + }; + gheUserTokenNock(DatabaseStateCreator.GITHUB_INSTALLATION_ID); + gheNock + .get("/api/v3/repos/integrations/test-repo-name/secret-scanning/alerts?per_page=20&page=1&sort=created&direction=desc") + .reply(200, []); + // No Jira Nock + + await expect(processInstallation(mockBackfillQueueSendMessage)(data, sentry, getLogger("test"))).toResolve(); + await verifyMessageSent(data); + }); + }); +}); + + +const expectedResponseCloudServer = () => ({ + "vulnerabilities": [ + { + "schemaVersion": "1.0", + "id": "s-1-12", + "updateSequenceNumber": 12345678, + "containerId": "1", + "displayName": "GitHub Personal Access Token", + "description": "Secret scanning alert", + "url": "https://github.com/test-owner/sample-repo/security/secret-scanning/12", + "type": "sast", + "introducedDate": "2023-08-04T04:33:44Z", + "lastUpdated": "2023-08-04T04:33:44Z", + "severity": { + "level": "critical" + }, + "identifiers": [ + { + "displayName": "github_personal_access_token", + "url": "https://github.com/test-owner/sample-repo/security/secret-scanning/12" + } + ], + "status": "open" + } + + ], + "properties": { + "gitHubInstallationId": DatabaseStateCreator.GITHUB_INSTALLATION_ID + }, + "operationType": "BACKFILL" +}); + +const expectedResponseGHEServer = () => ({ + "vulnerabilities": [ + { + "schemaVersion": "1.0", + "id": "s-6769746875626d79646f6d61696e636f6d-1-12", + "updateSequenceNumber": 12345678, + "containerId": "6769746875626d79646f6d61696e636f6d-1", + "displayName": "GitHub Personal Access Token", + "description": "Secret scanning alert", + "url": "https://github.com/test-owner/sample-repo/security/secret-scanning/12", + "type": "sast", + "introducedDate": "2023-08-04T04:33:44Z", + "lastUpdated": "2023-08-04T04:33:44Z", + "severity": { + "level": "critical" + }, + "identifiers": [ + { + "displayName": "github_personal_access_token", + "url": "https://github.com/test-owner/sample-repo/security/secret-scanning/12" + } + ], + "status": "open" + } + ], + "properties": { + "gitHubInstallationId": DatabaseStateCreator.GITHUB_INSTALLATION_ID + }, + "operationType": "BACKFILL" +}); \ No newline at end of file diff --git a/src/sync/secret-scanning-alerts.ts b/src/sync/secret-scanning-alerts.ts new file mode 100644 index 0000000000..501ce06121 --- /dev/null +++ b/src/sync/secret-scanning-alerts.ts @@ -0,0 +1,110 @@ +import { Repository } from "models/subscription"; +import { GitHubInstallationClient } from "../github/client/github-installation-client"; +import Logger from "bunyan"; +import { BackfillMessagePayload } from "~/src/sqs/sqs.types"; +import { transformRepositoryId } from "../transforms/transform-repository-id"; +import { getGitHubClientConfigFromAppId } from "../util/get-github-client-config"; +import { JiraVulnerabilityBulkSubmitData, JiraVulnerabilitySeverityEnum } from "../interfaces/jira"; +import { PageSizeAwareCounterCursor } from "./page-counter-cursor"; +import { SortDirection } from "../github/client/github-client.types"; +import { SecretScanningAlertResponseItem } from "../github/client/secret-scanning-alert.types"; +import { transformGitHubStateToJiraStatus } from "../transforms/transform-secret-scanning-alert"; + +export const getSecretScanningAlertTask = async ( + parentLogger: Logger, + gitHubClient: GitHubInstallationClient, + jiraHost: string, + repository: Repository, + cursor: string | undefined, + perPage: number, + messagePayload: BackfillMessagePayload) => { + + const logger = parentLogger.child({ backfillTask: "Secret scanning alerts" }); + const startTime = Date.now(); + + logger.info({ startTime }, "Secret scanning alerts task started"); + const fromDate = messagePayload?.commitsFromDate ? new Date(messagePayload.commitsFromDate) : undefined; + const smartCursor = new PageSizeAwareCounterCursor(cursor).scale(perPage); + + const { data: secretScanningAlerts } = await gitHubClient.getSecretScanningAlerts(repository.owner.login, repository.name, { + per_page: smartCursor.perPage, + page: smartCursor.pageNo, + sort: "created", + direction: SortDirection.DES + }); + + if (!secretScanningAlerts?.length) { + logger.info({ processingTime: Date.now() - startTime, jiraPayloadLength: 0 }, "Backfill task complete"); + return { + edges: [], + jiraPayload: undefined + }; + } + + if (areAllBuildsEarlierThanFromDate(secretScanningAlerts, fromDate)) { + logger.info({ processingTime: Date.now() - startTime, jiraPayloadLength: 0 }, "Backfill task complete"); + return { + edges: [], + jiraPayload: undefined + }; + } + logger.info(`Found ${secretScanningAlerts.length} secret scanning alerts`); + const nextPageCursorStr = smartCursor.copyWithPageNo(smartCursor.pageNo + 1).serialise(); + const edgesWithCursor = [{ secretScanningAlerts, cursor: nextPageCursorStr }]; + + const jiraPayload = await transformSecretScanningAlert(secretScanningAlerts, repository, jiraHost, logger, messagePayload.gitHubAppConfig?.gitHubAppId); + + logger.info({ processingTime: Date.now() - startTime, jiraPayloadLength: jiraPayload?.vulnerabilities?.length }, "Backfill task complete"); + return { + edges: edgesWithCursor, + jiraPayload + }; +}; + +const areAllBuildsEarlierThanFromDate = (alerts: SecretScanningAlertResponseItem[], fromDate: Date | undefined): boolean => { + + if (!fromDate) return false; + + return alerts.every(alert => { + const createdAt = new Date(alert.created_at); + return createdAt.getTime() < fromDate.getTime(); + }); + +}; + + +const transformSecretScanningAlert = async ( + alerts: SecretScanningAlertResponseItem[], + repository: Repository, + jiraHost: string, + logger: Logger, + gitHubAppId: number | undefined +): Promise => { + + const gitHubClientConfig = await getGitHubClientConfigFromAppId(gitHubAppId, jiraHost); + + const vulnerabilities = alerts.map((alert) => { + return { + schemaVersion: "1.0", + id: `s-${transformRepositoryId(repository.id, gitHubClientConfig.baseUrl)}-${alert.number}`, + updateSequenceNumber: Date.now(), + containerId: transformRepositoryId(repository.id, gitHubClientConfig.baseUrl), + displayName: alert.secret_type_display_name || `${alert.secret_type} secret exposed`, + description: "Secret scanning alert", + url: alert.html_url, + type: "sast", + introducedDate: alert.created_at, + lastUpdated: alert?.resolved_at || alert.created_at, + severity: { + level: JiraVulnerabilitySeverityEnum.CRITICAL + }, + identifiers: [{ + displayName: alert.secret_type, + url: alert.html_url + }], + status: transformGitHubStateToJiraStatus(alert.state, logger) + }; + }); + return { vulnerabilities }; + +}; diff --git a/src/sync/sync.types.ts b/src/sync/sync.types.ts index 3cd07fcc06..7172e69d62 100644 --- a/src/sync/sync.types.ts +++ b/src/sync/sync.types.ts @@ -4,7 +4,7 @@ import { BackfillMessagePayload } from "~/src/sqs/sqs.types"; import Logger from "bunyan"; /* valid task types */ -export type TaskType = "repository" | "pull" | "commit" | "branch" | "build" | "deployment" | "dependabotAlert"; +export type TaskType = "repository" | "pull" | "commit" | "branch" | "build" | "deployment" | "dependabotAlert" | "secretScanningAlert"; export type SyncType = "full" | "partial"; diff --git a/src/transforms/transform-secret-scanning-alert.test.ts b/src/transforms/transform-secret-scanning-alert.test.ts index a84bd29b0b..8ab42f9c83 100644 --- a/src/transforms/transform-secret-scanning-alert.test.ts +++ b/src/transforms/transform-secret-scanning-alert.test.ts @@ -56,6 +56,7 @@ describe("transformSecretScanningAlert", () => { it.each([ ["created", JiraVulnerabilityStatusEnum.OPEN], ["reopened", JiraVulnerabilityStatusEnum.OPEN], + ["open", JiraVulnerabilityStatusEnum.OPEN], ["resolved", JiraVulnerabilityStatusEnum.CLOSED], ["revoked", JiraVulnerabilityStatusEnum.CLOSED], ["unmapped_state", JiraVulnerabilityStatusEnum.UNKNOWN] @@ -79,7 +80,7 @@ describe("transformSecretScanningAlert", () => { "containerId": "1", "description": "Secret scanning alert", "displayName": "personal_access_token secret exposed", - "id": "d-1-123", + "id": "s-1-123", "identifiers": [{ "displayName": "personal_access_token", "url": "https://sample/123" @@ -93,10 +94,7 @@ describe("transformSecretScanningAlert", () => { "status": "open", "type": "sast", "updateSequenceNumber": Date.now(), - "url": "https://sample/123", - "additionalInfo": { - "content": "personal_access_token" - } + "url": "https://sample/123" } ]); }); diff --git a/src/transforms/transform-secret-scanning-alert.ts b/src/transforms/transform-secret-scanning-alert.ts index d705b70bd4..602dfb0266 100644 --- a/src/transforms/transform-secret-scanning-alert.ts +++ b/src/transforms/transform-secret-scanning-alert.ts @@ -14,7 +14,7 @@ export const transformSecretScanningAlert = async (context: WebhookContext