Update redis service and github service (#256)

* Update redis service and github service

* Fix: github testing

* feat: alterations (#257)

* feat: refactor event source location

* fix: refresh after generating

---------

Co-authored-by: Matthew Bystedt <[email protected]>
Co-authored-by: Matthew Bystedt <[email protected]>

docs/dev_account_token.md

@@ -13,11 +13,23 @@ See: [Broker JWT](/operations_jwt.md)
 * Find the 'Broker Account' section and the account you want to generate the token for. The token expiry (if one has been created) will be shown. Click the 'Generate' button to open the generate/renew token dialog.
 * Read the instructions and click 'Generate' button.
 
-Teams are encouraged to document the client_id used by a service. This documentation should clearly state the locations the account is used.
+Teams are encouraged to document the client_id used by a service. This documentation should clearly state the locations the account is stored. The `reason` field in intentions should be descriptive enough your team understands where it is opened from.
+
+Generated tokens are saved in vault 'tools' space for all associated services by default. This can occur even if Vault has not been enabled for a service.
+
+## Update Github Secrets
+
+### Prerequisites
+
+* Have a [Github App](https://docs.github.com/en/apps/creating-github-apps/about-creating-github-apps/about-creating-github-apps) ready
+
+* Install the [Github App](https://docs.github.com/en/apps/using-github-apps/installing-your-own-github-app) in all repositories associated with services. Grant the app read/write access to repository secrets.
+
+* After a token is generated, all secrets in the tools namespace for a service (`apps-kv-mount`/tools/`project`/`service`) in Vault will be synced to the associated services' Github repository as secrets.
 
 ### Renewing a token
 
-Tokens can be regenerated at anytime. The procedure is identical to generating a token. The previous old token will continue working for an hour (if it is not already expired).
+Tokens can be regenerated at anytime. The procedure is identical to generating a token. The previous token will continue working for an hour (if it is not already expired).
 
 ## How to Lookup an Account from a Token
 

docs/development.md

@@ -95,6 +95,10 @@ Once started, you must run the vault setup script to bootstrap it. MongoDB must
 $ ./scripts/vault-setup.sh
 ```
 
+#### Github secret sync
+
+To setup a Github App to test secret syncing, set the values GITHUB_CLIENT_ID and GITHUB_PRIVATE_KEY at the Vault path `apps/prod/vault/vsync`.
+
 ## Running Locally
 
 The following assumes the setup steps have occurred and the databases have been successfully bootstrapped.

package.json

@@ -44,6 +44,7 @@
  "express-session": "^1.18.0",
  "file-stream-rotator": "^1.0.0",
  "helmet": "^7.1.0",
+ "libsodium-wrappers": "^0.7.15",
  "lodash.merge": "^4.6.2",
  "mongodb": "^5.9.2",
  "openid-client": "^5.6.5",
@@ -73,6 +74,7 @@
  "@types/express": "^4.17.21",
  "@types/express-session": "^1.18.0",
  "@types/jest": "^29.5.12",
+ "@types/libsodium-wrappers": "^0.7.14",
  "@types/lodash.merge": "^4.6.9",
  "@types/node": "^22.5.2",
  "@types/passport": "^1.0.16",

scripts/setenv-backend-dev.sh

@@ -18,6 +18,10 @@ if [ $? != 0 ]; then [ $PS1 ] && return || exit; fi
 
 export AUDIT_URL_TEMPLATE="https://audit.example/dashboard?from=<%= intention.transaction.start %>&to=<%= intention.transaction.end %>&hash=<%= intention.transaction.hash %>"
 export VAULT_TOKEN=$(eval $VAULT_TOKEN_CMD)
+if [ -z "$1" ]; then
+ export GITHUB_CLIENT_ID=$(vault kv get -field=GITHUB_CLIENT_ID apps/prod/vault/vsync)
+ export GITHUB_PRIVATE_KEY=$(vault kv get -field=GITHUB_PRIVATE_KEY apps/prod/vault/vsync)
+fi
 BROKER_ROLE_ID=$(vault read -format json auth/$VAULT_APPROLE_PATH/role/$VAULT_BROKER_ROLE/role-id | jq -r '.data.role_id')
 BROKER_SECRET_ID=$(vault write -format json -f auth/$VAULT_APPROLE_PATH/role/$VAULT_BROKER_ROLE/secret-id | jq -r '.data.secret_id')
 export BROKER_TOKEN=$(vault write -format json -f auth/$VAULT_APPROLE_PATH/login role_id=$BROKER_ROLE_ID secret_id=$BROKER_SECRET_ID | jq -r '.auth.client_token')

src/app.module.ts

@@ -22,6 +22,7 @@ import { RedisModule } from './redis/redis.module';
 import { SystemModule } from './system/system.module';
 import { PackageModule } from './package/package.module';
 import { VaultModule } from './vault/vault.module';
+import { GithubModule } from './github/github.module';
 
 /**
  * Convenience function for converting an environment variable to an object
@@ -82,6 +83,7 @@ function envToObj(key: string, envName: string) {
  SystemModule,
  PackageModule,
  VaultModule,
+ GithubModule,
  ],
  controllers: [],
  providers: [],

src/collection/account.service.ts

@@ -1,5 +1,10 @@
 import { createHmac, randomUUID } from 'node:crypto';
-import { Injectable, BadRequestException } from '@nestjs/common';
+import {
+ Injectable,
+ BadRequestException,
+ NotFoundException,
+ ServiceUnavailableException,
+} from '@nestjs/common';
 import { Request } from 'express';
 import { Cron, CronExpression } from '@nestjs/schedule';
 import { plainToInstance } from 'class-transformer';
@@ -27,6 +32,7 @@ import { CollectionNameEnum } from '../persistence/dto/collection-dto-union.type
 import { ProjectDto } from '../persistence/dto/project.dto';
 import { RedisService } from '../redis/redis.service';
 import { VaultService } from '../vault/vault.service';
+import { GithubService } from '../github/github.service';
 
 export class TokenCreateDTO {
  token: string;
@@ -37,6 +43,7 @@ export class AccountService {
  constructor(
  private readonly auditService: AuditService,
  private readonly opensearchService: OpensearchService,
+ private readonly githubService: GithubService,
  private readonly vaultService: VaultService,
  private readonly redisService: RedisService,
  private readonly graphRepository: GraphRepository,
@@ -187,6 +194,9 @@ export class AccountService {
  if (patchVault) {
  await this.addTokenToAccountServices(token, account);
  }
+ if (this.githubService.isEnabled()) {
+ await this.refresh(account.id.toString());
+ }
  this.auditService.recordAccountTokenLifecycle(
  req,
  payload,
@@ -300,6 +310,45 @@ export class AccountService {
  );
  }
 
+ async refresh(id: string): Promise<void> {
+ const account = await this.collectionRepository.getCollectionById(
+ 'brokerAccount',
+ id,
+ );
+
+ if (!account) {
+ throw new NotFoundException(`Account with ID ${id} not found`);
+ }
+ if (!this.githubService.isEnabled()) {
+ throw new ServiceUnavailableException();
+ }
+ const downstreamServices =
+ await this.graphRepository.getDownstreamVertex<ServiceDto>(
+ account.vertex.toString(),
+ CollectionNameEnum.service,
+ 3,
+ );
+ if (downstreamServices) {
+ for (const service of downstreamServices) {
+ const serviceName = service.collection.name;
+ const projectDtoArr =
+ await this.graphRepository.getUpstreamVertex<ProjectDto>(
+ service.collection.vertex.toString(),
+ CollectionNameEnum.project,
+ null,
+ );
+ const projectName = projectDtoArr[0].collection.name;
+ await this.githubService.refresh(
+ projectName,
+ serviceName,
+ service.collection.scmUrl,
+ );
+ }
+ } else {
+ // console.log('No services associated with this broker account');
+ }
+ }
+
  @Cron(CronExpression.EVERY_MINUTE)
  async runJwtLifecycle() {
  const CURRENT_TIME_MS = Date.now();

src/collection/collection.controller.ts

@@ -3,20 +3,27 @@ import {
  Controller,
  Delete,
  Get,
+ MessageEvent,
  NotFoundException,
  Param,
  Post,
  Put,
  Query,
  Request,
+ Sse,
  UseGuards,
  UseInterceptors,
  UsePipes,
  ValidationPipe,
 } from '@nestjs/common';
 import { Request as ExpressRequest } from 'express';
 import { ApiBearerAuth, ApiOAuth2, ApiQuery } from '@nestjs/swagger';
-import { OAUTH2_CLIENT_MAP_GUID } from '../constants';
+import { Observable } from 'rxjs';
+import {
+ OAUTH2_CLIENT_MAP_GUID,
+ REDIS_PUBSUB,
+ DAYS_10_IN_SECONDS,
+} from '../constants';
 import { CollectionService } from './collection.service';
 import { BrokerOidcAuthGuard } from '../auth/broker-oidc-auth.guard';
 import { BrokerJwtAuthGuard } from '../auth/broker-jwt-auth.guard';
@@ -35,7 +42,7 @@ import { PersistenceCacheKey } from '../persistence/persistence-cache-key.decora
 import { PersistenceCacheInterceptor } from '../persistence/persistence-cache.interceptor';
 import { PERSISTENCE_CACHE_KEY_CONFIG } from '../persistence/persistence.constants';
 import { ExpiryQuery } from './dto/expiry-query.dto';
-import { DAYS_10_IN_SECONDS } from '../constants';
+import { RedisService } from '../redis/redis.service';
 
 @Controller({
  path: 'collection',
@@ -45,6 +52,7 @@ export class CollectionController {
  constructor(
  private readonly accountService: AccountService,
  private readonly service: CollectionService,
+ private readonly redis: RedisService,
  private readonly userCollectionService: UserCollectionService,
  ) {}
 
@@ -93,6 +101,19 @@ export class CollectionController {
  return this.accountService.getRegisteryJwts(id);
  }
 
+ @Post('broker-account/:id/refresh')
+ @Roles('admin')
+ @AllowOwner({
+ graphObjectType: 'collection',
+ graphObjectCollection: 'brokerAccount',
+ graphIdFromParamKey: 'id',
+ permission: 'sudo',
+ })
+ @UseGuards(BrokerOidcAuthGuard)
+ async refresh(@Param('id') id: string): Promise<void> {
+ return await this.accountService.refresh(id);
+ }
+
  @Post('broker-account/:id/token')
  @Roles('admin')
  @AllowOwner({
@@ -148,6 +169,13 @@ export class CollectionController {
  return this.accountService.renewToken(request, ttl, true);
  }
 
+ @Sse('broker-account/events')
+ @UseGuards(BrokerCombinedAuthGuard)
+ @ApiBearerAuth()
+ tokenUpdatedEvents(): Observable<MessageEvent> {
+ return this.redis.getEventSource(REDIS_PUBSUB.VAULT_SERVICE_TOKEN);
+ }
+
  @Get('service/:id/details')
  @UseGuards(BrokerCombinedAuthGuard)
  @ApiBearerAuth()

src/collection/collection.module.ts

@@ -6,6 +6,7 @@ import { UserCollectionService } from './user-collection.service';
 import { AuditModule } from '../audit/audit.module';
 import { AuthModule } from '../auth/auth.module';
 import { AwsModule } from '../aws/aws.module';
+import { GithubModule } from '../github/github.module';
 import { GraphModule } from '../graph/graph.module';
 import { IntentionModule } from '../intention/intention.module';
 import { PersistenceModule } from '../persistence/persistence.module';
@@ -24,6 +25,7 @@ import { VaultModule } from '../vault/vault.module';
  AuditModule,
  AuthModule,
  PersistenceModule,
+ GithubModule,
  GraphModule,
  forwardRef(() => IntentionModule),
  RedisModule,

src/constants.ts

@@ -96,3 +96,6 @@ export const REDIS_PUBSUB = {
  GRAPH: 'graph',
  VAULT_SERVICE_TOKEN: 'vault-service-token',
 } as const;
+
+export const GITHUB_CLIENT_ID = process.env.GITHUB_CLIENT_ID ?? '';
+export const GITHUB_PRIVATE_KEY = process.env.GITHUB_PRIVATE_KEY ?? '';

src/github/github.health.ts

@@ -0,0 +1,17 @@
+import { Injectable } from '@nestjs/common';
+import { HealthIndicator, HealthIndicatorResult } from '@nestjs/terminus';
+import { GithubService } from './github.service';
+
+@Injectable()
+export class GithubHealthIndicator extends HealthIndicator {
+ constructor(private readonly githubService: GithubService) {
+ super();
+ }
+ async isHealthy(key: string): Promise<HealthIndicatorResult> {
+ const result = this.getStatus(key, this.githubService.isEnabled(), {
+ enabled: this.githubService.isEnabled(),
+ });
+
+ return result;
+ }
+}

src/github/github.module.ts

@@ -0,0 +1,12 @@
+import { Module } from '@nestjs/common';
+import { GithubService } from './github.service';
+import { VaultModule } from '../vault/vault.module';
+import { RedisModule } from '../redis/redis.module';
+import { GithubHealthIndicator } from './github.health';
+
+@Module({
+ imports: [VaultModule, RedisModule],
+ providers: [GithubService, GithubHealthIndicator],
+ exports: [GithubService, GithubHealthIndicator],
+})
+export class GithubModule {}

src/github/github.service.spec.ts

@@ -0,0 +1,37 @@
+import { createMock } from '@golevelup/ts-jest';
+import { Test, TestingModule } from '@nestjs/testing';
+import { GithubService } from './github.service';
+
+describe('GithubService', () => {
+ let service: GithubService;
+
+ beforeEach(async () => {
+ const module: TestingModule = await Test.createTestingModule({
+ providers: [GithubService],
+ })
+ .useMocker(createMock)
+ .compile();
+
+ service = module.get<GithubService>(GithubService);
+ });
+
+ it('should be defined', () => {
+ expect(service).toBeDefined();
+ });
+
+ describe('getOwnerAndRepoFromUrl', () => {
+ it('should extract owner and repo from URL', () => {
+ const repoUrl = 'https://github.com/myorg/mytestrepo.git';
+ const result = service['getOwnerAndRepoFromUrl'](repoUrl); // Using private method directly for testing
+
+ expect(result).toEqual({ owner: 'myorg', repo: 'mytestrepo' });
+ });
+
+ it('should throw an error for invalid URL', () => {
+ const repoUrl = 'invalid_url';
+ expect(() => service['getOwnerAndRepoFromUrl'](repoUrl)).toThrow(
+ 'Invalid GitHub URL',
+ );
+ });
+ });
+});