From 66dadd929e7316f1cd079663da75cbdfd301a66d Mon Sep 17 00:00:00 2001 From: Grace Ruan <106621189+GraceRuan@users.noreply.github.com> Date: Wed, 11 Sep 2024 13:59:24 -0700 Subject: [PATCH] Update redis service and github service (#256) * Update redis service and github service * Fix: github testing * feat: alterations (#257) * feat: refactor event source location * fix: refresh after generating --------- Co-authored-by: Matthew Bystedt Co-authored-by: Matthew Bystedt --- docs/dev_account_token.md | 16 +- docs/development.md | 4 + package-lock.json | 24 ++ package.json | 2 + scripts/setenv-backend-dev.sh | 4 + src/app.module.ts | 2 + src/collection/account.service.ts | 51 +++- src/collection/collection.controller.ts | 32 ++- src/collection/collection.module.ts | 2 + src/constants.ts | 3 + src/github/github.health.ts | 17 ++ src/github/github.module.ts | 12 + src/github/github.service.spec.ts | 37 +++ src/github/github.service.ts | 220 ++++++++++++++++++ src/graph/graph.service.ts | 2 - src/health/health.controller.ts | 11 +- src/health/health.module.ts | 11 +- src/vault/vault.service.ts | 20 +- .../inspector-account.component.html | 19 +- .../inspector-account.component.scss | 5 + .../inspector-account.component.ts | 109 +++++++-- .../vertex-dialog/vertex-dialog.component.ts | 1 - ui/src/app/service/system-api.service.ts | 44 +++- 23 files changed, 612 insertions(+), 36 deletions(-) create mode 100644 src/github/github.health.ts create mode 100644 src/github/github.module.ts create mode 100644 src/github/github.service.spec.ts create mode 100644 src/github/github.service.ts diff --git a/docs/dev_account_token.md b/docs/dev_account_token.md index af5cd5b9..dc09279e 100644 --- a/docs/dev_account_token.md +++ b/docs/dev_account_token.md @@ -13,11 +13,23 @@ See: [Broker JWT](/operations_jwt.md) * Find the 'Broker Account' section and the account you want to generate the token for. The token expiry (if one has been created) will be shown. Click the 'Generate' button to open the generate/renew token dialog. * Read the instructions and click 'Generate' button. -Teams are encouraged to document the client_id used by a service. This documentation should clearly state the locations the account is used. +Teams are encouraged to document the client_id used by a service. This documentation should clearly state the locations the account is stored. The `reason` field in intentions should be descriptive enough your team understands where it is opened from. + +Generated tokens are saved in vault 'tools' space for all associated services by default. This can occur even if Vault has not been enabled for a service. + +## Update Github Secrets + +### Prerequisites + +* Have a [Github App](https://docs.github.com/en/apps/creating-github-apps/about-creating-github-apps/about-creating-github-apps) ready + +* Install the [Github App](https://docs.github.com/en/apps/using-github-apps/installing-your-own-github-app) in all repositories associated with services. Grant the app read/write access to repository secrets. + +* After a token is generated, all secrets in the tools namespace for a service (`apps-kv-mount`/tools/`project`/`service`) in Vault will be synced to the associated services' Github repository as secrets. ### Renewing a token -Tokens can be regenerated at anytime. The procedure is identical to generating a token. The previous old token will continue working for an hour (if it is not already expired). +Tokens can be regenerated at anytime. The procedure is identical to generating a token. The previous token will continue working for an hour (if it is not already expired). ## How to Lookup an Account from a Token diff --git a/docs/development.md b/docs/development.md index a85b845d..af4f234a 100644 --- a/docs/development.md +++ b/docs/development.md @@ -95,6 +95,10 @@ Once started, you must run the vault setup script to bootstrap it. MongoDB must $ ./scripts/vault-setup.sh ``` +#### Github secret sync + +To setup a Github App to test secret syncing, set the values GITHUB_CLIENT_ID and GITHUB_PRIVATE_KEY at the Vault path `apps/prod/vault/vsync`. + ## Running Locally The following assumes the setup steps have occurred and the databases have been successfully bootstrapped. diff --git a/package-lock.json b/package-lock.json index d73a0d51..ce37a16e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,6 +30,7 @@ "express-session": "^1.18.0", "file-stream-rotator": "^1.0.0", "helmet": "^7.1.0", + "libsodium-wrappers": "^0.7.15", "lodash.merge": "^4.6.2", "mongodb": "^5.9.2", "openid-client": "^5.6.5", @@ -59,6 +60,7 @@ "@types/express": "^4.17.21", "@types/express-session": "^1.18.0", "@types/jest": "^29.5.12", + "@types/libsodium-wrappers": "^0.7.14", "@types/lodash.merge": "^4.6.9", "@types/node": "^22.5.2", "@types/passport": "^1.0.16", @@ -3565,6 +3567,13 @@ "@types/node": "*" } }, + "node_modules/@types/libsodium-wrappers": { + "version": "0.7.14", + "resolved": "https://registry.npmjs.org/@types/libsodium-wrappers/-/libsodium-wrappers-0.7.14.tgz", + "integrity": "sha512-5Kv68fXuXK0iDuUir1WPGw2R9fOZUlYlSAa0ztMcL0s0BfIDTqg9GXz8K30VJpPP3sxWhbolnQma2x+/TfkzDQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/lodash": { "version": "4.14.197", "dev": true, @@ -7988,6 +7997,21 @@ "version": "1.10.53", "license": "MIT" }, + "node_modules/libsodium": { + "version": "0.7.15", + "resolved": "https://registry.npmjs.org/libsodium/-/libsodium-0.7.15.tgz", + "integrity": "sha512-sZwRknt/tUpE2AwzHq3jEyUU5uvIZHtSssktXq7owd++3CSgn8RGrv6UZJJBpP7+iBghBqe7Z06/2M31rI2NKw==", + "license": "ISC" + }, + "node_modules/libsodium-wrappers": { + "version": "0.7.15", + "resolved": "https://registry.npmjs.org/libsodium-wrappers/-/libsodium-wrappers-0.7.15.tgz", + "integrity": "sha512-E4anqJQwcfiC6+Yrl01C1m8p99wEhLmJSs0VQqST66SbQXXBoaJY0pF4BNjRYa/sOQAxx6lXAaAFIlx+15tXJQ==", + "license": "ISC", + "dependencies": { + "libsodium": "^0.7.15" + } + }, "node_modules/lines-and-columns": { "version": "1.2.4", "dev": true, diff --git a/package.json b/package.json index cd717a2c..69af0c45 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "express-session": "^1.18.0", "file-stream-rotator": "^1.0.0", "helmet": "^7.1.0", + "libsodium-wrappers": "^0.7.15", "lodash.merge": "^4.6.2", "mongodb": "^5.9.2", "openid-client": "^5.6.5", @@ -73,6 +74,7 @@ "@types/express": "^4.17.21", "@types/express-session": "^1.18.0", "@types/jest": "^29.5.12", + "@types/libsodium-wrappers": "^0.7.14", "@types/lodash.merge": "^4.6.9", "@types/node": "^22.5.2", "@types/passport": "^1.0.16", diff --git a/scripts/setenv-backend-dev.sh b/scripts/setenv-backend-dev.sh index 06b80a74..fbfa6c98 100755 --- a/scripts/setenv-backend-dev.sh +++ b/scripts/setenv-backend-dev.sh @@ -18,6 +18,10 @@ if [ $? != 0 ]; then [ $PS1 ] && return || exit; fi export AUDIT_URL_TEMPLATE="https://audit.example/dashboard?from=<%= intention.transaction.start %>&to=<%= intention.transaction.end %>&hash=<%= intention.transaction.hash %>" export VAULT_TOKEN=$(eval $VAULT_TOKEN_CMD) +if [ -z "$1" ]; then + export GITHUB_CLIENT_ID=$(vault kv get -field=GITHUB_CLIENT_ID apps/prod/vault/vsync) + export GITHUB_PRIVATE_KEY=$(vault kv get -field=GITHUB_PRIVATE_KEY apps/prod/vault/vsync) +fi BROKER_ROLE_ID=$(vault read -format json auth/$VAULT_APPROLE_PATH/role/$VAULT_BROKER_ROLE/role-id | jq -r '.data.role_id') BROKER_SECRET_ID=$(vault write -format json -f auth/$VAULT_APPROLE_PATH/role/$VAULT_BROKER_ROLE/secret-id | jq -r '.data.secret_id') export BROKER_TOKEN=$(vault write -format json -f auth/$VAULT_APPROLE_PATH/login role_id=$BROKER_ROLE_ID secret_id=$BROKER_SECRET_ID | jq -r '.auth.client_token') diff --git a/src/app.module.ts b/src/app.module.ts index f6570afd..457df192 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -22,6 +22,7 @@ import { RedisModule } from './redis/redis.module'; import { SystemModule } from './system/system.module'; import { PackageModule } from './package/package.module'; import { VaultModule } from './vault/vault.module'; +import { GithubModule } from './github/github.module'; /** * Convenience function for converting an environment variable to an object @@ -82,6 +83,7 @@ function envToObj(key: string, envName: string) { SystemModule, PackageModule, VaultModule, + GithubModule, ], controllers: [], providers: [], diff --git a/src/collection/account.service.ts b/src/collection/account.service.ts index 42b43f8a..f8ef7d18 100644 --- a/src/collection/account.service.ts +++ b/src/collection/account.service.ts @@ -1,5 +1,10 @@ import { createHmac, randomUUID } from 'node:crypto'; -import { Injectable, BadRequestException } from '@nestjs/common'; +import { + Injectable, + BadRequestException, + NotFoundException, + ServiceUnavailableException, +} from '@nestjs/common'; import { Request } from 'express'; import { Cron, CronExpression } from '@nestjs/schedule'; import { plainToInstance } from 'class-transformer'; @@ -27,6 +32,7 @@ import { CollectionNameEnum } from '../persistence/dto/collection-dto-union.type import { ProjectDto } from '../persistence/dto/project.dto'; import { RedisService } from '../redis/redis.service'; import { VaultService } from '../vault/vault.service'; +import { GithubService } from '../github/github.service'; export class TokenCreateDTO { token: string; @@ -37,6 +43,7 @@ export class AccountService { constructor( private readonly auditService: AuditService, private readonly opensearchService: OpensearchService, + private readonly githubService: GithubService, private readonly vaultService: VaultService, private readonly redisService: RedisService, private readonly graphRepository: GraphRepository, @@ -187,6 +194,9 @@ export class AccountService { if (patchVault) { await this.addTokenToAccountServices(token, account); } + if (this.githubService.isEnabled()) { + await this.refresh(account.id.toString()); + } this.auditService.recordAccountTokenLifecycle( req, payload, @@ -300,6 +310,45 @@ export class AccountService { ); } + async refresh(id: string): Promise { + const account = await this.collectionRepository.getCollectionById( + 'brokerAccount', + id, + ); + + if (!account) { + throw new NotFoundException(`Account with ID ${id} not found`); + } + if (!this.githubService.isEnabled()) { + throw new ServiceUnavailableException(); + } + const downstreamServices = + await this.graphRepository.getDownstreamVertex( + account.vertex.toString(), + CollectionNameEnum.service, + 3, + ); + if (downstreamServices) { + for (const service of downstreamServices) { + const serviceName = service.collection.name; + const projectDtoArr = + await this.graphRepository.getUpstreamVertex( + service.collection.vertex.toString(), + CollectionNameEnum.project, + null, + ); + const projectName = projectDtoArr[0].collection.name; + await this.githubService.refresh( + projectName, + serviceName, + service.collection.scmUrl, + ); + } + } else { + // console.log('No services associated with this broker account'); + } + } + @Cron(CronExpression.EVERY_MINUTE) async runJwtLifecycle() { const CURRENT_TIME_MS = Date.now(); diff --git a/src/collection/collection.controller.ts b/src/collection/collection.controller.ts index 101e49fd..67284ab9 100644 --- a/src/collection/collection.controller.ts +++ b/src/collection/collection.controller.ts @@ -3,12 +3,14 @@ import { Controller, Delete, Get, + MessageEvent, NotFoundException, Param, Post, Put, Query, Request, + Sse, UseGuards, UseInterceptors, UsePipes, @@ -16,7 +18,12 @@ import { } from '@nestjs/common'; import { Request as ExpressRequest } from 'express'; import { ApiBearerAuth, ApiOAuth2, ApiQuery } from '@nestjs/swagger'; -import { OAUTH2_CLIENT_MAP_GUID } from '../constants'; +import { Observable } from 'rxjs'; +import { + OAUTH2_CLIENT_MAP_GUID, + REDIS_PUBSUB, + DAYS_10_IN_SECONDS, +} from '../constants'; import { CollectionService } from './collection.service'; import { BrokerOidcAuthGuard } from '../auth/broker-oidc-auth.guard'; import { BrokerJwtAuthGuard } from '../auth/broker-jwt-auth.guard'; @@ -35,7 +42,7 @@ import { PersistenceCacheKey } from '../persistence/persistence-cache-key.decora import { PersistenceCacheInterceptor } from '../persistence/persistence-cache.interceptor'; import { PERSISTENCE_CACHE_KEY_CONFIG } from '../persistence/persistence.constants'; import { ExpiryQuery } from './dto/expiry-query.dto'; -import { DAYS_10_IN_SECONDS } from '../constants'; +import { RedisService } from '../redis/redis.service'; @Controller({ path: 'collection', @@ -45,6 +52,7 @@ export class CollectionController { constructor( private readonly accountService: AccountService, private readonly service: CollectionService, + private readonly redis: RedisService, private readonly userCollectionService: UserCollectionService, ) {} @@ -93,6 +101,19 @@ export class CollectionController { return this.accountService.getRegisteryJwts(id); } + @Post('broker-account/:id/refresh') + @Roles('admin') + @AllowOwner({ + graphObjectType: 'collection', + graphObjectCollection: 'brokerAccount', + graphIdFromParamKey: 'id', + permission: 'sudo', + }) + @UseGuards(BrokerOidcAuthGuard) + async refresh(@Param('id') id: string): Promise { + return await this.accountService.refresh(id); + } + @Post('broker-account/:id/token') @Roles('admin') @AllowOwner({ @@ -148,6 +169,13 @@ export class CollectionController { return this.accountService.renewToken(request, ttl, true); } + @Sse('broker-account/events') + @UseGuards(BrokerCombinedAuthGuard) + @ApiBearerAuth() + tokenUpdatedEvents(): Observable { + return this.redis.getEventSource(REDIS_PUBSUB.VAULT_SERVICE_TOKEN); + } + @Get('service/:id/details') @UseGuards(BrokerCombinedAuthGuard) @ApiBearerAuth() diff --git a/src/collection/collection.module.ts b/src/collection/collection.module.ts index 2b1f50e3..107f3a57 100644 --- a/src/collection/collection.module.ts +++ b/src/collection/collection.module.ts @@ -6,6 +6,7 @@ import { UserCollectionService } from './user-collection.service'; import { AuditModule } from '../audit/audit.module'; import { AuthModule } from '../auth/auth.module'; import { AwsModule } from '../aws/aws.module'; +import { GithubModule } from '../github/github.module'; import { GraphModule } from '../graph/graph.module'; import { IntentionModule } from '../intention/intention.module'; import { PersistenceModule } from '../persistence/persistence.module'; @@ -24,6 +25,7 @@ import { VaultModule } from '../vault/vault.module'; AuditModule, AuthModule, PersistenceModule, + GithubModule, GraphModule, forwardRef(() => IntentionModule), RedisModule, diff --git a/src/constants.ts b/src/constants.ts index 6c7f2eb1..77d33832 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -96,3 +96,6 @@ export const REDIS_PUBSUB = { GRAPH: 'graph', VAULT_SERVICE_TOKEN: 'vault-service-token', } as const; + +export const GITHUB_CLIENT_ID = process.env.GITHUB_CLIENT_ID ?? ''; +export const GITHUB_PRIVATE_KEY = process.env.GITHUB_PRIVATE_KEY ?? ''; diff --git a/src/github/github.health.ts b/src/github/github.health.ts new file mode 100644 index 00000000..6da6907e --- /dev/null +++ b/src/github/github.health.ts @@ -0,0 +1,17 @@ +import { Injectable } from '@nestjs/common'; +import { HealthIndicator, HealthIndicatorResult } from '@nestjs/terminus'; +import { GithubService } from './github.service'; + +@Injectable() +export class GithubHealthIndicator extends HealthIndicator { + constructor(private readonly githubService: GithubService) { + super(); + } + async isHealthy(key: string): Promise { + const result = this.getStatus(key, this.githubService.isEnabled(), { + enabled: this.githubService.isEnabled(), + }); + + return result; + } +} diff --git a/src/github/github.module.ts b/src/github/github.module.ts new file mode 100644 index 00000000..46b8cda5 --- /dev/null +++ b/src/github/github.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; +import { GithubService } from './github.service'; +import { VaultModule } from '../vault/vault.module'; +import { RedisModule } from '../redis/redis.module'; +import { GithubHealthIndicator } from './github.health'; + +@Module({ + imports: [VaultModule, RedisModule], + providers: [GithubService, GithubHealthIndicator], + exports: [GithubService, GithubHealthIndicator], +}) +export class GithubModule {} diff --git a/src/github/github.service.spec.ts b/src/github/github.service.spec.ts new file mode 100644 index 00000000..a8126924 --- /dev/null +++ b/src/github/github.service.spec.ts @@ -0,0 +1,37 @@ +import { createMock } from '@golevelup/ts-jest'; +import { Test, TestingModule } from '@nestjs/testing'; +import { GithubService } from './github.service'; + +describe('GithubService', () => { + let service: GithubService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [GithubService], + }) + .useMocker(createMock) + .compile(); + + service = module.get(GithubService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('getOwnerAndRepoFromUrl', () => { + it('should extract owner and repo from URL', () => { + const repoUrl = 'https://github.com/myorg/mytestrepo.git'; + const result = service['getOwnerAndRepoFromUrl'](repoUrl); // Using private method directly for testing + + expect(result).toEqual({ owner: 'myorg', repo: 'mytestrepo' }); + }); + + it('should throw an error for invalid URL', () => { + const repoUrl = 'invalid_url'; + expect(() => service['getOwnerAndRepoFromUrl'](repoUrl)).toThrow( + 'Invalid GitHub URL', + ); + }); + }); +}); diff --git a/src/github/github.service.ts b/src/github/github.service.ts new file mode 100644 index 00000000..7e5f99b5 --- /dev/null +++ b/src/github/github.service.ts @@ -0,0 +1,220 @@ +import { Injectable } from '@nestjs/common'; +import axios, { AxiosInstance } from 'axios'; +import { lastValueFrom } from 'rxjs'; +import sodium from 'libsodium-wrappers'; +import * as jwt from 'jsonwebtoken'; +import { + GITHUB_CLIENT_ID, + GITHUB_PRIVATE_KEY, + VAULT_KV_APPS_MOUNT, +} from '../constants'; +import { VaultService } from '../vault/vault.service'; + +@Injectable() +export class GithubService { + private readonly axiosInstance: AxiosInstance; + + constructor(private readonly vaultService: VaultService) { + this.axiosInstance = axios.create({ + baseURL: 'https://api.github.com', + headers: { + Accept: 'application/vnd.github.v3+json', + }, + }); + } + + public isEnabled() { + return GITHUB_CLIENT_ID !== '' && GITHUB_PRIVATE_KEY !== ''; + } + + public async refresh(project: string, service: string, scmUrl: string) { + if (!this.isEnabled()) { + throw new Error(); + } + const path = `tools/${project}/${service}`; + const kvData = await lastValueFrom( + this.vaultService.getKv(VAULT_KV_APPS_MOUNT, path), + ); + if (kvData) { + for (const [secretName, secretValue] of Object.entries(kvData)) { + if (scmUrl) { + await this.updateSecret(scmUrl, secretName, secretValue.toString()); + } else { + console.log( + 'Service does not have Github repo URL to update:', + service, + ); + } + } + } + } + + public async updateSecret( + repoUrl: string, + secretName: string, + secretValue: string, + ): Promise { + const { owner, repo } = this.getOwnerAndRepoFromUrl(repoUrl); + const token = await this.getInstallationAccessToken(owner, repo); + const filteredSecretName = secretName.replace(/[^a-zA-Z0-9_]/g, '_'); + + try { + if (token) { + const { key: base64PublicKey, key_id: keyId } = await this.getPublicKey( + owner, + repo, + token, + ); + // Encrypt secret + const encryptedSecret = await this.encryptSecret( + base64PublicKey.toString('utf-8'), + secretValue, + ); + // Update secret + await this.axiosInstance.put( + `/repos/${owner}/${repo}/actions/secrets/${filteredSecretName}`, + { + encrypted_value: encryptedSecret, + key_id: keyId, + }, + { + headers: { + Authorization: `token ${token}`, + }, + }, + ); + console.log( + `Secret ${filteredSecretName} updated successfully on ${owner}/${repo}!`, + ); + } else { + console.log( + `Github access token is null! No updates on ${owner}/${repo}`, + ); + } + } catch (error) { + console.error('Errors on updating broker JWT with API calls', error); + //throw new Error('Failed to update secret in github repo.'); + } + } + + // Generate JWT + private generateJWT(): string { + const payload = { + iat: Math.floor(Date.now() / 1000) - 60, + exp: Math.floor(Date.now() / 1000) + 2 * 60, // JWT expires in 2 minutes + iss: GITHUB_CLIENT_ID, + }; + return jwt.sign(payload, GITHUB_PRIVATE_KEY, { algorithm: 'RS256' }); + } + + private async getInstallationId( + owner: string, + repo: string, + token: string, + ): Promise { + try { + const response = await this.axiosInstance.get( + `/repos/${owner}/${repo}/installation`, + { + headers: { + Authorization: `Bearer ${token}`, + }, + }, + ); + if (response.data.id) return response.data.id; + } catch (error) { + console.error( + `Catch error on make API call on get Installation ID for ${owner}/${repo}`, + ); + //throw new Error('Failed to get installation id.'); + } + } + + private async getInstallationAccessToken( + owner: string, + repo: string, + ): Promise { + const token = this.generateJWT(); + try { + const installationId = await this.getInstallationId(owner, repo, token); + if (installationId) { + const response = await this.axiosInstance.post( + `/app/installations/${installationId}/access_tokens`, + {}, + { + headers: { + Authorization: `Bearer ${token}`, + }, + }, + ); + if (response.data.token) return response.data.token; + } + } catch (error) { + console.error( + `Github App has not been authorized to access ${owner}/${repo}`, + ); + //throw new Error('Failed to get access token.'); + } + } + + private getOwnerAndRepoFromUrl(repoUrl: string): { + owner: string; + repo: string; + } { + const regex = /github\.com[:/](.+?)\/(.+?)(\.git)?$/; + const match = repoUrl.match(regex); + if (match && match.length >= 3) { + return { owner: match[1], repo: match[2] }; + } + throw new Error('Invalid GitHub URL'); + } + + private async getPublicKey( + owner: string, + repo: string, + accessToken: string, + ): Promise { + const url = `https://api.github.com/repos/${owner}/${repo}/actions/secrets/public-key`; + const response = await axios.get(url, { + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/vnd.github.v3+json', + }, + }); + + return response.data; + } + + private async encryptSecret( + publicKey: string, + secretValue: string, + ): Promise { + try { + await sodium.ready; + // Convert the base64 public key to a Uint8Array + const publicKeyUint8Array = sodium.from_base64( + publicKey, + sodium.base64_variants.ORIGINAL, + ); + + // Convert the secret value to a Uint8Array + const secretUint8Array = sodium.from_string(secretValue); + + // Encrypt the secret using the public key + const encryptedUint8Array = sodium.crypto_box_seal( + secretUint8Array, + publicKeyUint8Array, + ); + + // Convert the encrypted Uint8Array to a base64 string + const encryptedBase64 = sodium.to_base64( + encryptedUint8Array, + sodium.base64_variants.ORIGINAL, + ); + return encryptedBase64; + } catch (error) { + console.error('Error encrypting the secret:', error); + //throw new Error('Failed to encrypt the secret.'); + } + } +} diff --git a/src/graph/graph.service.ts b/src/graph/graph.service.ts index 552371f5..c53a4844 100644 --- a/src/graph/graph.service.ts +++ b/src/graph/graph.service.ts @@ -36,8 +36,6 @@ import { UserPermissionNames } from '../persistence/dto/user-permission-rest.dto @Injectable() export class GraphService { - // private readonly eventSource = new Subject(); - constructor( private readonly auditService: AuditService, private readonly collectionRepository: CollectionRepository, diff --git a/src/health/health.controller.ts b/src/health/health.controller.ts index ab76f40b..67fccf9f 100644 --- a/src/health/health.controller.ts +++ b/src/health/health.controller.ts @@ -1,8 +1,13 @@ import { Controller, Get, HttpCode, UseGuards } from '@nestjs/common'; import { ApiBearerAuth } from '@nestjs/swagger'; -import { HealthCheckService, HttpHealthIndicator } from '@nestjs/terminus'; +import { + HealthCheckService, + HttpHealthIndicator, + TypeOrmHealthIndicator, +} from '@nestjs/terminus'; import { BrokerJwtAuthGuard } from '../auth/broker-jwt-auth.guard'; import { HealthService } from './health.service'; +import { GithubHealthIndicator } from '../github/github.health'; @Controller({ path: 'health', @@ -12,7 +17,9 @@ export class HealthController { constructor( private readonly health: HealthCheckService, private readonly http: HttpHealthIndicator, + private readonly github: GithubHealthIndicator, private readonly healthService: HealthService, + private readonly db: TypeOrmHealthIndicator, ) {} @Get() @@ -23,6 +30,8 @@ export class HealthController { 'broker-api', 'http://localhost:3000/v1/health/ping', ), + () => this.db.pingCheck('database'), + () => this.github.isHealthy('github'), ]); } diff --git a/src/health/health.module.ts b/src/health/health.module.ts index fa07a47e..cf2ea56c 100644 --- a/src/health/health.module.ts +++ b/src/health/health.module.ts @@ -1,16 +1,23 @@ import { Module } from '@nestjs/common'; import { TerminusModule } from '@nestjs/terminus'; import { AuditModule } from '../audit/audit.module'; -import { TokenModule } from '../token/token.module'; +import { GithubModule } from '../github/github.module'; import { HealthController } from './health.controller'; import { HealthService } from './health.service'; import { PersistenceModule } from '../persistence/persistence.module'; +import { TokenModule } from '../token/token.module'; /** * The health module reports on the overall status of broker. */ @Module({ - imports: [AuditModule, PersistenceModule, TerminusModule, TokenModule], + imports: [ + AuditModule, + GithubModule, + PersistenceModule, + TerminusModule, + TokenModule, + ], controllers: [HealthController], providers: [HealthService], }) diff --git a/src/vault/vault.service.ts b/src/vault/vault.service.ts index 0d5da139..27f3502e 100644 --- a/src/vault/vault.service.ts +++ b/src/vault/vault.service.ts @@ -1,7 +1,7 @@ import { HttpService } from '@nestjs/axios'; import { Injectable, Logger } from '@nestjs/common'; import { AxiosRequestConfig, AxiosResponse } from 'axios'; -import { catchError, Observable } from 'rxjs'; +import { catchError, Observable, map } from 'rxjs'; import { VAULT_ADDR, VAULT_SERVICE_WRAP_TTL } from '../constants'; @@ -33,6 +33,24 @@ export class VaultService { ); } + public getKv(mount: string, path: string) { + const config = this.prepareConfig(); + config.headers['Content-Type'] = 'application/json'; + return this.httpService + .get(`${this.vaultAddr}/v1/${mount}/data/${path}`, this.prepareConfig()) + .pipe( + map((response) => { + const kvData = response.data?.data?.data; + + if (!kvData) { + throw new Error(`No secrets found at the specified path: ${path}`); + } + + return kvData; + }), + ); + } + public postKv(mount: string, path: string, data: any) { return this.httpService.post( `${this.vaultAddr}/v1/${mount}/data/${path}`, diff --git a/ui/src/app/graph/inspector-account/inspector-account.component.html b/ui/src/app/graph/inspector-account/inspector-account.component.html index df583d5b..9758ffa0 100644 --- a/ui/src/app/graph/inspector-account/inspector-account.component.html +++ b/ui/src/app/graph/inspector-account/inspector-account.component.html @@ -1,8 +1,21 @@ @if (account) {
-

Token

- - +
+

Token

+
+
+ + + + + +
@if (!lastJwtTokenData) {

No token found. Click generate to create a token.

diff --git a/ui/src/app/graph/inspector-account/inspector-account.component.scss b/ui/src/app/graph/inspector-account/inspector-account.component.scss index 9d14ba1e..219cb8e4 100644 --- a/ui/src/app/graph/inspector-account/inspector-account.component.scss +++ b/ui/src/app/graph/inspector-account/inspector-account.component.scss @@ -32,3 +32,8 @@ table { line-height: 1em; vertical-align: -2px; } + +.fields-wrapper { + display: inline-flex; + flex-wrap: wrap; +} diff --git a/ui/src/app/graph/inspector-account/inspector-account.component.ts b/ui/src/app/graph/inspector-account/inspector-account.component.ts index 8a91ed68..26c69a5a 100644 --- a/ui/src/app/graph/inspector-account/inspector-account.component.ts +++ b/ui/src/app/graph/inspector-account/inspector-account.component.ts @@ -2,6 +2,7 @@ import { Component, Input, OnChanges, + OnDestroy, OnInit, SimpleChanges, } from '@angular/core'; @@ -9,8 +10,15 @@ import { CommonModule } from '@angular/common'; import { MatButtonModule } from '@angular/material/button'; import { MatIconModule } from '@angular/material/icon'; import { MatDialog } from '@angular/material/dialog'; +import { MatMenuModule } from '@angular/material/menu'; +import { + MatSnackBar, + MatSnackBarModule, + MatSnackBarConfig, +} from '@angular/material/snack-bar'; import { MatTableModule } from '@angular/material/table'; import { MatTooltipModule } from '@angular/material/tooltip'; +import { Subscription } from 'rxjs'; import { AccountGenerateDialogComponent } from '../account-generate-dialog/account-generate-dialog.component'; import { SystemApiService } from '../../service/system-api.service'; @@ -24,13 +32,15 @@ import { JwtRegistryDto } from '../../service/dto/jwt-registry-rest.dto'; CommonModule, MatButtonModule, MatIconModule, + MatMenuModule, + MatSnackBarModule, MatTableModule, MatTooltipModule, ], templateUrl: './inspector-account.component.html', styleUrls: ['./inspector-account.component.scss'], }) -export class InspectorAccountComponent implements OnChanges, OnInit { +export class InspectorAccountComponent implements OnChanges, OnInit, OnDestroy { @Input() account!: BrokerAccountRestDto; @Input() userIndex!: number | undefined; @Input() hasSudo = false; @@ -48,13 +58,27 @@ export class InspectorAccountComponent implements OnChanges, OnInit { | undefined; propDisplayedColumns: string[] = ['key', 'value']; + private tokenUpdateSubscription: Subscription | undefined; + constructor( private readonly dialog: MatDialog, + private readonly snackBar: MatSnackBar, private readonly systemApi: SystemApiService, ) {} ngOnInit(): void { - this.updateAccount(); + this.tokenUpdateSubscription = this.systemApi + .createAccountTokenEventSource() + .subscribe({ + next: (data: any) => { + console.log(data); + // TODO: only update if necessary + this.updateAccount(); + }, + error: (error: any) => { + console.error('Error receiving token update events:', error); + }, + }); } ngOnChanges(changes: SimpleChanges): void { @@ -63,6 +87,13 @@ export class InspectorAccountComponent implements OnChanges, OnInit { } } + ngOnDestroy(): void { + // Unsubscribe + if (this.tokenUpdateSubscription) { + this.tokenUpdateSubscription.unsubscribe(); + } + } + openGenerateDialog() { this.dialog .open(AccountGenerateDialogComponent, { @@ -78,6 +109,24 @@ export class InspectorAccountComponent implements OnChanges, OnInit { }); } + sync(): void { + if (this.account && this.userIndex) { + this.systemApi.refresh(this.account.id).subscribe({ + next: () => { + this.openSnackBar('Tools secrets synced successfully'); + }, + error: (err: any) => { + console.log(err); + this.openSnackBar( + 'Syncing token failed: ' + (err?.statusText ?? 'unknown'), + ); + }, + }); + } else { + this.openSnackBar('The account does not exist!'); + } + } + private updateAccount(): void { if (this.account && this.userIndex) { if (this.account.id === this.requestedAccountId) { @@ -86,41 +135,61 @@ export class InspectorAccountComponent implements OnChanges, OnInit { this.requestedAccountId = this.account.id; this.jwtTokens = undefined; this.lastJwtTokenData = undefined; - this.systemApi.getAccountTokens(this.account.id).subscribe((data) => { - this.jwtTokens = data; - const lastJwtToken = this.jwtTokens.pop(); - if (lastJwtToken) { - this.lastJwtTokenData = { - JTI: lastJwtToken.claims.jti, - Expiry: new Date(lastJwtToken.claims.exp * 1000), - }; - if (this.hourlyUsage) { - this.lastJwtTokenData.Usage = this.hourlyUsage; - } + this.systemApi.getAccountTokens(this.account.id).subscribe({ + next: (data: JwtRegistryDto[]) => { + this.jwtTokens = data; + const lastJwtToken = this.jwtTokens.pop(); + if (lastJwtToken) { + this.lastJwtTokenData = { + JTI: lastJwtToken.claims.jti, + Expiry: new Date(lastJwtToken.claims.exp * 1000), + }; + if (this.hourlyUsage) { + this.lastJwtTokenData.Usage = this.hourlyUsage; + } - this.expired = - Date.now() > new Date(lastJwtToken.claims.exp * 1000).valueOf(); - } + this.expired = + Date.now() > new Date(lastJwtToken.claims.exp * 1000).valueOf(); + } + }, + error: (err: { status: number }) => { + if (err.status === 503) { + // ignore + } else { + throw err; + } + }, }); if (this.hasSudo) { this.hourlyUsage = undefined; - this.systemApi.getAccountUsage(this.account.id).subscribe( - (data) => { + this.systemApi.getAccountUsage(this.account.id).subscribe({ + next: ( + data: + | { success: number; unknown: number; failure: number } + | undefined, + ) => { this.hourlyUsage = data; if (this.lastJwtTokenData) { this.lastJwtTokenData.Usage = this.hourlyUsage; } }, - (err) => { + error: (err: { status: number }) => { if (err.status === 503) { // ignore } else { throw err; } }, - ); + }); } } } + + private openSnackBar(message: string) { + const config = new MatSnackBarConfig(); + config.duration = 5000; + config.verticalPosition = 'bottom'; + this.snackBar.open(message, 'Dismiss', config); + } } diff --git a/ui/src/app/graph/vertex-dialog/vertex-dialog.component.ts b/ui/src/app/graph/vertex-dialog/vertex-dialog.component.ts index b1211866..471b474d 100644 --- a/ui/src/app/graph/vertex-dialog/vertex-dialog.component.ts +++ b/ui/src/app/graph/vertex-dialog/vertex-dialog.component.ts @@ -74,7 +74,6 @@ export class VertexDialogComponent implements OnInit { } else { this.collectionControl.enable(); } - this.data.data; } ngAfterViewInit() { diff --git a/ui/src/app/service/system-api.service.ts b/ui/src/app/service/system-api.service.ts index 4518c2e6..016cf8a8 100644 --- a/ui/src/app/service/system-api.service.ts +++ b/ui/src/app/service/system-api.service.ts @@ -1,5 +1,8 @@ import { HttpClient } from '@angular/common/http'; import { Injectable } from '@angular/core'; +import { filter, map, Observable } from 'rxjs'; +import { SseClient } from 'ngx-sse-client'; + import { environment } from '../../environments/environment'; import { JwtRegistryDto, TokenCreateDto } from './dto/jwt-registry-rest.dto'; import { ConnectionConfigRestDto } from './dto/connection-config-rest.dto'; @@ -8,7 +11,10 @@ import { ConnectionConfigRestDto } from './dto/connection-config-rest.dto'; providedIn: 'root', }) export class SystemApiService { - constructor(private readonly http: HttpClient) {} + constructor( + private readonly http: HttpClient, + private sseClient: SseClient, + ) {} getAccountTokens(accountId: string) { return this.http.get( @@ -45,6 +51,32 @@ export class SystemApiService { ); } + createAccountTokenEventSource(): Observable { + return this.sseClient + .stream(`${environment.apiUrl}/v1/graph/token-updated`) + .pipe( + filter((event) => { + if (event.type === 'error') { + const errorEvent = event as ErrorEvent; + if (errorEvent.error) { + console.error(errorEvent.error, errorEvent.message); + } + return false; + } + return true; + }), + map((event) => { + if (event.type !== 'error') { + const messageEvent = event as MessageEvent; + //console.info( + // `SSE request with type "${messageEvent.type}" and data "${messageEvent.data}"`, + //); + return JSON.parse(messageEvent.data); + } + }), + ); + } + getConnectionConfig() { return this.http.get( `${environment.apiUrl}/v1/system/preference/connection`, @@ -53,4 +85,14 @@ export class SystemApiService { }, ); } + + refresh(accountId: string) { + return this.http.post( + `${environment.apiUrl}/v1/collection/broker-account/${accountId}/refresh`, + {}, + { + responseType: 'json', + }, + ); + } }