ISOC-3693 Secret scanning alerts backfill (#2335)

* initial checkin

* inital checkin

* refacored

* update error message

* initial checkin

* inital checkin

* initial sync

* code refacttored

* missed typing file

---------

Co-authored-by: Harminder <[email protected]>
Co-authored-by: Gary Xue <[email protected]>

src/github/client/github-client.types.ts

@@ -30,6 +30,13 @@ export type GetPullRequestParams = {
 	page?: number;
 }
 
+export type GetSecretScanningAlertRequestParams = {
+	sort?: string;
+	direction?: string;
+	per_page?: number;
+	page?: number;
+}
+
 export type GraphQlQueryResponse<ResponseData> = {
 	data: ResponseData;
 	errors?: GraphQLError[];

src/github/client/github-installation-client.ts

@@ -29,6 +29,7 @@ import {
 	ActionsListRepoWorkflowRunsResponseEnhanced,
 	CreateReferenceBody,
 	GetPullRequestParams,
+	GetSecretScanningAlertRequestParams,
 	PaginatedAxiosResponse,
 	ReposGetContentsResponse
 } from "./github-client.types";
@@ -38,6 +39,7 @@ import { GithubClientError, GithubClientGraphQLError } from "~/src/github/client
 import { cloneDeep } from "lodash";
 import { BooleanFlags, booleanFlag } from "config/feature-flags";
 import { logCurlOutputInChunks, runCurl } from "utils/curl/curl-utils";
+import { SecretScanningAlertResponseItem } from "./secret-scanning-alert.types";
 
 // Unfortunately, the type is not exposed in Octokit...
 // https://docs.github.com/en/rest/pulls/review-requests?apiVersion=2022-11-28#get-all-requested-reviewers-for-a-pull-request
@@ -74,6 +76,13 @@ export class GitHubInstallationClient extends GitHubClient {
 		this.gitHubServerAppId = gshaId;
 	}
 
+
+	public async getSecretScanningAlerts(owner: string, repo: string, secretScanningAlertRequestParams: GetSecretScanningAlertRequestParams): Promise<AxiosResponse<SecretScanningAlertResponseItem[]>> {
+		return await this.get<SecretScanningAlertResponseItem[]>(`/repos/{owner}/{repo}/secret-scanning/alerts`, secretScanningAlertRequestParams, {
+			owner,
+			repo
+		});
+	}
 	/**
 	 * Lists pull requests for the given repository.
 	 */

src/github/client/secret-scanning-alert.types.ts

@@ -0,0 +1,13 @@
+export type SecretScanningAlertResponseItem = {
+    number: number,
+    created_at: string,
+    url: string,
+    html_url: string,
+    locations_url: string,
+    state: "open" | "resolved",
+    resolution?: string,
+    resolved_at?: string,
+    resolution_comment?: string,
+    secret_type: string,
+    secret_type_display_name: string
+  }

src/github/installation.ts

@@ -75,7 +75,7 @@ export const installationWebhookHandler = async (
 		);
 
 	} catch (err) {
-		logger.warn({ err }, "Failed to submit security workspace to Jira or trigger backfill via backfill");
+		logger.warn({ err }, "Failed to submit security workspace to Jira or trigger backfill");
 		const webhookReceived = context.webhookReceived;
 		webhookReceived && emitWebhookProcessedMetrics(
 			new Date(webhookReceived).getTime(),

src/github/secret-scanning-alert.test.ts

@@ -136,7 +136,7 @@ describe("SecretScanningAlertWebhookHandler", () => {
 			vulnerabilities: [
 				{
 					schemaVersion: "1.0",
-					id: "d-456-123",
+					id: "s-456-123",
 					updateSequenceNumber: Date.now(),
 					containerId: "456",
 					displayName: "personal_access_token secret exposed",
@@ -152,10 +152,7 @@ describe("SecretScanningAlertWebhookHandler", () => {
 						"displayName": "personal_access_token",
 						"url":SAMPLE_SECURITY_URL
 					}],
-					status: JIRA_VULNERABILITY_STATUS_ENUM_OPEN,
-					additionalInfo: {
-						content: "personal_access_token"
-					}
+					status: JIRA_VULNERABILITY_STATUS_ENUM_OPEN
 				}
 			]
 		};

src/interfaces/jira.ts

@@ -210,7 +210,7 @@ export interface JiraVulnerability {
 	severity: JiraVulnerabilitySeverity;
 	identifiers: JiraVulnerabilityIdentifier[];
 	status: JiraVulnerabilityStatusEnum;
-	additionalInfo: JiraVulnerabilityAdditionalInfo;
+	additionalInfo?: JiraVulnerabilityAdditionalInfo;
 }
 
 export interface JiraVulnerabilitySeverity {

src/models/reposyncstate.ts

@@ -35,19 +35,22 @@ export interface RepoSyncStateProperties {
 	buildStatus?: TaskStatus;
 	deploymentStatus?: TaskStatus;
 	dependabotAlertStatus?: TaskStatus;
+	secretScanningAlertStatus?: TaskStatus,
 	branchCursor?: string;
 	commitCursor?: string;
 	issueCursor?: string;
 	pullCursor?: string;
 	buildCursor?: string;
 	deploymentCursor?: string;
 	dependabotAlertCursor?: string;
+	secretScanningAlertCursor?: string;
 	commitFrom?: Date;
 	branchFrom?: Date;
 	pullFrom?: Date;
 	buildFrom?: Date;
 	deploymentFrom?: Date;
 	dependabotAlertFrom?: Date;
+	secretScanningAlertFrom?: Date;
 	forked?: boolean;
 	repoPushedAt: Date;
 	repoUpdatedAt: Date;
@@ -76,19 +79,22 @@ export class RepoSyncState extends Model implements RepoSyncStateProperties {
 	buildStatus?: TaskStatus;
 	deploymentStatus?: TaskStatus;
 	dependabotAlertStatus?: TaskStatus;
+	secretScanningAlertStatus?: TaskStatus;
 	branchCursor?: string;
 	commitCursor?: string;
 	issueCursor?: string;
 	pullCursor?: string;
 	buildCursor?: string;
 	deploymentCursor?: string;
 	dependabotAlertCursor?: string;
+	secretScanningAlertCursor?: string;
 	commitFrom?: Date;
 	branchFrom?: Date;
 	pullFrom?: Date;
 	buildFrom?: Date;
 	deploymentFrom?: Date;
 	dependabotAlertFrom?: Date;
+	secretScanningAlertFrom?: Date;
 	forked?: boolean;
 	repoPushedAt: Date;
 	repoUpdatedAt: Date;
@@ -135,7 +141,8 @@ export class RepoSyncState extends Model implements RepoSyncStateProperties {
 					commitStatus: "failed",
 					buildStatus: "failed",
 					deploymentStatus: "failed",
-					dependabotAlertStatus: "failed"
+					dependabotAlertStatus: "failed",
+					secretScanningAlertStatus: "failed"
 				}
 			}
 		});
@@ -152,7 +159,8 @@ export class RepoSyncState extends Model implements RepoSyncStateProperties {
 					commitStatus: "failed",
 					buildStatus: "failed",
 					deploymentStatus: "failed",
-					dependabotAlertStatus: "failed"
+					dependabotAlertStatus: "failed",
+					secretScanningAlertStatus: "failed"
 				}
 			}
 		}));
@@ -362,7 +370,10 @@ export class RepoSyncState extends Model implements RepoSyncStateProperties {
 			commitFrom: null,
 			dependabotAlertStatus: null,
 			dependabotAlertCursor: null,
-			dependabotAlertFrom: null
+			dependabotAlertFrom: null,
+			secretScanningAlertFrom: null,
+			secretScanningAlertStatus: null,
+			secretScanningAlertCursor: null
 		}, {
 			where: {
 				subscriptionId: subscription.id
@@ -410,19 +421,22 @@ RepoSyncState.init({
 	buildStatus: DataTypes.ENUM("pending", "complete", "failed"),
 	deploymentStatus: DataTypes.ENUM("pending", "complete", "failed"),
 	dependabotAlertStatus: DataTypes.ENUM("pending", "complete", "failed"),
+	secretScanningAlertStatus: DataTypes.ENUM("pending", "complete", "failed"),
 	branchCursor: STRING,
 	commitCursor: STRING,
 	issueCursor: STRING,
 	pullCursor: STRING,
 	buildCursor: STRING,
 	deploymentCursor: STRING,
 	dependabotAlertCursor: STRING,
+	secretScanningAlertCursor: STRING,
 	commitFrom: DATE,
 	branchFrom: DATE,
 	pullFrom: DATE,
 	buildFrom: DATE,
 	deploymentFrom: DATE,
 	dependabotAlertFrom: DATE,
+	secretScanningAlertFrom: DATE,
 	forked: BOOLEAN,
 	repoPushedAt: DATE,
 	repoUpdatedAt: DATE,

src/routes/jira/sync/jira-sync-post.test.ts

@@ -132,7 +132,7 @@ describe("sync", () => {
 					installationId: installationIdForServer,
 					jiraHost,
 					commitsFromDate: commitsFromDate.toISOString(),
-					targetTasks: ["pull", "branch", "commit", "build", "deployment", "dependabotAlert"],
+					targetTasks: ["pull", "branch", "commit", "build", "deployment", "dependabotAlert", "secretScanningAlert"],
 					gitHubAppConfig: expect.objectContaining({ gitHubAppId: gitHubServerApp.id, uuid: gitHubServerApp.uuid })
 				}), expect.anything(), expect.anything());
 			});

src/routes/jira/sync/jira-sync-post.ts

@@ -85,5 +85,5 @@ const determineSyncTypeAndTargetTasks = async (syncTypeFromReq: string, subscrip
 		return { syncType: "full", targetTasks: undefined };
 	}
 
-	return { syncType: "partial", targetTasks: ["pull", "branch", "commit", "build", "deployment", "dependabotAlert"] };
+	return { syncType: "partial", targetTasks: ["pull", "branch", "commit", "build", "deployment", "dependabotAlert", "secretScanningAlert"] };
 };

src/sync/installation.test.ts

@@ -329,8 +329,8 @@ describe("sync/installation", () => {
 
 	describe("getTargetTasks", () => {
 		it("should return all tasks if no target tasks present", async () => {
-			expect(getTargetTasks()).toEqual(["pull", "branch", "commit", "build", "deployment", "dependabotAlert"]);
-			expect(getTargetTasks([])).toEqual(["pull", "branch", "commit", "build", "deployment", "dependabotAlert"]);
+			expect(getTargetTasks()).toEqual(["pull", "branch", "commit", "build", "deployment", "dependabotAlert", "secretScanningAlert"]);
+			expect(getTargetTasks([])).toEqual(["pull", "branch", "commit", "build", "deployment", "dependabotAlert", "secretScanningAlert"]);
 		});
 
 		it("should return single target task", async () => {
@@ -566,7 +566,9 @@ describe("sync/installation", () => {
 			expect(repoSync.dependabotAlertFrom).toBeNull();
 		});
 
-		describe.each(["pull", "commit", "build", "deployment", "dependabotAlert"] as TaskType[])("Update jobs status for each tasks", (taskType: TaskType) => {
+		describe.each(
+			["pull", "commit", "build", "deployment", "dependabotAlert", "secretScanningAlert"] as TaskType[]
+		)("Update jobs status for each tasks", (taskType: TaskType) => {
 			const colTaskFrom = `${taskType}From`;
 			const task: Task = {
 				task: taskType,

src/sync/installation.ts

@@ -26,6 +26,7 @@ import { sendAnalytics } from "utils/analytics-client";
 import { AnalyticsEventTypes, AnalyticsTrackEventsEnum } from "interfaces/common";
 import { getNextTasks } from "~/src/sync/scheduler";
 import { getDependabotAlertTask } from "./dependabot-alerts";
+import { getSecretScanningAlertTask } from "./secret-scanning-alerts";
 
 const tasks: TaskProcessors = {
 	repository: getRepositoryTask,
@@ -34,10 +35,11 @@ const tasks: TaskProcessors = {
 	commit: getCommitTask,
 	build: getBuildTask,
 	deployment: getDeploymentTask,
-	dependabotAlert: getDependabotAlertTask
+	dependabotAlert: getDependabotAlertTask,
+	secretScanningAlert: getSecretScanningAlertTask
 };
 
-const allTaskTypes: TaskType[] = ["pull", "branch", "commit", "build", "deployment", "dependabotAlert"];
+const allTaskTypes: TaskType[] = ["pull", "branch", "commit", "build", "deployment", "dependabotAlert", "secretScanningAlert"];
 const allTasksExceptBranch = without(allTaskTypes, "branch");
 
 export const getTargetTasks = (targetTasks?: TaskType[]): TaskType[] => {
@@ -235,9 +237,11 @@ const sendPayloadToJira = async (task: TaskType, jiraClient, jiraPayload, reposi
 				});
 				break;
 			case "dependabotAlert":
+			case "secretScanningAlert":{
 				await jiraClient.security.submitVulnerabilities(jiraPayload, {
 					operationType: "BACKFILL"
 				});
+			}
 				break;
 			default:
 				await jiraClient.devinfo.repository.update(jiraPayload, {

src/sync/scheduler.test.ts

@@ -241,7 +241,7 @@ describe("scheduler", () => {
 			expect(task.task).toEqual("commit");
 		});
 	});
-	it("should not filter by dependabot alerts task if ENABLE_GITHUB_SECURITY_IN_JIRA FF is off", async () => {
+	it("should filter dependabot alerts task if ENABLE_GITHUB_SECURITY_IN_JIRA FF is off", async () => {
 		when(booleanFlag).calledWith(BooleanFlags.ENABLE_GITHUB_SECURITY_IN_JIRA, expect.anything()).mockResolvedValue(false);
 		configureRateLimit(10000, 10000);
 		const repoSyncStates = await RepoSyncState.findAllFromSubscription(subscription);
@@ -255,7 +255,7 @@ describe("scheduler", () => {
 		expect(tasks.otherTasks.length).toEqual(0);
 	});
 
-	it("should not filter by dependabot alerts task if ENABLE_GITHUB_SECURITY_IN_JIRA FF is on", async () => {
+	it("should not filter dependabot alerts task if ENABLE_GITHUB_SECURITY_IN_JIRA FF is on", async () => {
 		when(booleanFlag).calledWith(BooleanFlags.ENABLE_GITHUB_SECURITY_IN_JIRA, expect.anything()).mockResolvedValue(true);
 		configureRateLimit(10000, 10000);
 		const repoSyncStates = await RepoSyncState.findAllFromSubscription(subscription);
@@ -271,4 +271,33 @@ describe("scheduler", () => {
 			expect(task.task).toEqual("dependabotAlert");
 		});
 	});
+	it("should filter secret scanning alerts task if ENABLE_GITHUB_SECURITY_IN_JIRA FF is off", async () => {
+		when(booleanFlag).calledWith(BooleanFlags.ENABLE_GITHUB_SECURITY_IN_JIRA, expect.anything()).mockResolvedValue(false);
+		configureRateLimit(10000, 10000);
+		const repoSyncStates = await RepoSyncState.findAllFromSubscription(subscription);
+		await Promise.all(repoSyncStates.map((record) => {
+			record.secretScanningAlertStatus = "pending";
+			return record.save();
+		}));
+		githubUserTokenNock(DatabaseStateCreator.GITHUB_INSTALLATION_ID);
+		const tasks = await getNextTasks(subscription, ["secretScanningAlert"], getLogger("test"));
+		expect(tasks.mainTask).toBeUndefined();
+		expect(tasks.otherTasks.length).toEqual(0);
+	});
+	it("should not filter secret scanning alerts task if ENABLE_GITHUB_SECURITY_IN_JIRA FF is on", async () => {
+		when(booleanFlag).calledWith(BooleanFlags.ENABLE_GITHUB_SECURITY_IN_JIRA, expect.anything()).mockResolvedValue(true);
+		configureRateLimit(10000, 10000);
+		const repoSyncStates = await RepoSyncState.findAllFromSubscription(subscription);
+		await Promise.all(repoSyncStates.map((record) => {
+			record.secretScanningAlertStatus = "pending";
+			return record.save();
+		}));
+
+		githubUserTokenNock(DatabaseStateCreator.GITHUB_INSTALLATION_ID);
+		const tasks = await getNextTasks(subscription, ["secretScanningAlert"], getLogger("test"));
+		expect(tasks.mainTask!.task).toEqual("secretScanningAlert");
+		tasks.otherTasks.forEach(task => {
+			expect(task.task).toEqual("secretScanningAlert");
+		});
+	});
 });

src/sync/scheduler.ts

@@ -172,7 +172,7 @@ export const getNextTasks = async (subscription: Subscription, targetTasks: Task
 
 	let tasks = getTargetTasks(targetTasks);
 	if (!await booleanFlag(BooleanFlags.ENABLE_GITHUB_SECURITY_IN_JIRA, subscription.jiraHost)) {
-		tasks =  without(tasks, "dependabotAlert");
+		tasks =  without(tasks, "dependabotAlert", "secretScanningAlert");
 	}
 
 	const nSubTasks = await estimateNumberOfSubtasks(subscription, logger);