diff --git a/.DS_Store b/.DS_Store index dced286..94caba4 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 14f352b..596321b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,9 +6,6 @@ variables: APP_TITLE: "topcoder x app" NODE_ENV: development -services: - - mongo - cache: paths: - node_modules/ @@ -60,5 +57,4 @@ deploy_stage: url: $HEROKU_STAGING_URL variables: GIT_STRATEGY: fetch - GIT_SUBMODULE_STRATEGY: recursive - MONGODB_URI: mongodb://mongo:27017/topcoderx' + GIT_SUBMODULE_STRATEGY: recursive \ No newline at end of file diff --git a/DynamoDBMigration.md b/DynamoDBMigration.md deleted file mode 100644 index d6e1255..0000000 --- a/DynamoDBMigration.md +++ /dev/null @@ -1,40 +0,0 @@ -## Migrate the data to Dynamo - -Prerequisite is to turn off any throttling on the DynamoDB, temporarily - -1. Download the migration-tools.zip from here: https://github.com/topcoder-platform/topcoder-x-ui/issues/140#issuecomment-497663806 -2. Unzip -3. Modify `migration-tools/config/default.js` to match prod - * MONGODB_URL: Will provide in Slack - * AWS_ACCESS_KEY_ID - * AWS_SECRET_ACCESS_KEY - * AWS_REGION - * IS_LOCAL = 'false' -4. `npm i` -5. `npm run migrate-data` - -#### Tables created - -* `Topcoder_X.CopilotPayment` -* `Topcoder_X.Issue` -* `Topcoder_X.OwnerUserGroup` -* `Topcoder_X.OwnerUserTeam` -* `Topcoder_X.Project` -* `Topcoder_X.User` -* `Topcoder_X.UserMapping` - - -## Deploy latest code - -* Deploy the latest code from `master` to Heroku: - * topcoder-x-receiver - * topcoder-x-processor - * topcoder-x-ui - -Update the AWS Dynamo config values in Heroku: - -* AWS_ACCESS_KEY_ID -* AWS_SECRET_ACCESS_KEY -* IS_LOCAL='false' -* AWS_DYNAMODB_REGION -* AWS_REGION diff --git a/README.md b/README.md index 7158345..7e7c038 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ## Requirements - Nodejs 8 is required -- MongoDB 3.2 +- DynamoDB - kafka - nodemon (for local development) @@ -69,10 +69,9 @@ Server should be started at port 80. Follow the below steps to deploy the app to heroku 1. `heroku login` 2. `heroku create` -3. `heroku addons:create mongolab` -4. `heroku config:set NPM_CONFIG_PRODUCTION=false` so that heroku will install dev dependencies -5. `git push heroku master` or `git push heroku develop:master` to deploy develop branch -6. `heroku open` to load the app on browser +3. `heroku config:set NPM_CONFIG_PRODUCTION=false` so that heroku will install dev dependencies +4. `git push heroku master` or `git push heroku develop:master` to deploy develop branch +5. `heroku open` to load the app on browser Note: heroku domain should match subdomain of topcoder-dev or topcoder depending upon target topcoder environment diff --git a/TopcoderXDeploy.md b/TopcoderXDeploy.md index 92ef449..b900d65 100644 --- a/TopcoderXDeploy.md +++ b/TopcoderXDeploy.md @@ -22,7 +22,7 @@ Topcoder-X comprises 3 pieces: * Topcoder-X Processor that handles the messages created by the receiver. The processor handles the interactions with the Topcoder platform, via the Topcoder challenge API, and it also handles adding comments back to Github / Gitlab when things are updated on the Topcoder challenge. * Topcoder-X UI that allows copilots and others to manage the Topcoder-X integrations with Github and Gitlab projects. The user can add new Github and Gitlab integrations, including setting up default webhooks and labels for issues, all through the UI. -All 3 pieces will be configured to use the same MongoDB and Kafka installations. +All 3 pieces will be configured to use the same DynamoDB and Kafka installations. ## DynamoDB diff --git a/configuration.md b/configuration.md index 69b8ca2..898f242 100644 --- a/configuration.md +++ b/configuration.md @@ -34,6 +34,8 @@ The following config parameters are supported, they are defined in `src/config.j |IS_LOCAL | Use Amazon DynamoDB Local or server. |true | |AWS_CONNECTION_TIMEOUT | The timeout used to check if the app is healthy. |10000 | |TC_LOGIN_URL | TC login url | | +|DYNAMODB_WAIT_TABLE_FOR_ACTIVE_TIMEOUT | Dynamodb wait for active timeout |10 minutes | + ## GitHub OAuth App Setup diff --git a/package.json b/package.json index d2e5431..d9efd45 100644 --- a/package.json +++ b/package.json @@ -80,7 +80,7 @@ "parse-domain": "^2.1.2", "shortid": "^2.2.8", "ssl-express-www": "^3.0.3", - "superagent": "^3.6.0", + "superagent": "^6.1.0", "superagent-promise": "^1.1.0", "typescript": "~2.3.3", "uuid": "^3.3.2", diff --git a/scripts/mongodb-migration/README.MD b/scripts/mongodb-migration/README.MD deleted file mode 100644 index 7a7abed..0000000 --- a/scripts/mongodb-migration/README.MD +++ /dev/null @@ -1,37 +0,0 @@ -# Migrate mongodb to dynamodb -The following config parameters are supported, they are defined in `config/default.js` and can be configured in env variables: - - -| Name | Description | Default | -| :------------------------------------- | :----------------------------------------: | :------------------------------: | -| LOG_LEVEL | The log level | info | -| MONGODB_URL | The MongoDB URL. This needs to be the same MongoDB used by topcoder-x-receiver, topcoder-x-processor, and topcoder-x-site | mongodb://127.0.0.1:27017/topcoderx | -|COLLECTION_COUNTS | The counts of testing data for each model| | -|AWS_ACCESS_KEY_ID | The Amazon certificate key to use when connecting. Use local dynamodb you can set fake value |FAKE_ACCESS_KEY_ID | -|AWS_SECRET_ACCESS_KEY | The Amazon certificate access key to use when connecting. Use local dynamodb you can set fake value |FAKE_SECRET_ACCESS_KEY | -|AWS_REGION | The Amazon certificate region to use when connecting. Use local dynamodb you can set fake value |FAKE_REGION | -|IS_LOCAL | Use Amazon DynamoDB Local or server | 'true' | -| MIGRATION_DELAY_TIME | The delay used to avoid throughput errors from DynamoDB | 5000 | -| MIGRATION_DELAY_INTERVAL | The number of requests made before we delay the next call to DynamoDB | 50 | - -## Scripts - -migrate mongodb data to dynamodb -``` -npm run migrate-data -``` - -create empty tables to dynamodb -``` -npm run create-empty-tables -``` - -## Test Data - -**NOTE** This is a *destructive* call. It will overwrite existing data in mongodb. Do not do this against production - -Create testing data to mongodb, this command will generate bulk fake data to testing migration. -You also can create regular data by opcoder-x-receiver/topcoder-x-processor/topcoder-x-site -``` -npm run create-test-data -``` diff --git a/scripts/mongodb-migration/config/default.js b/scripts/mongodb-migration/config/default.js deleted file mode 100644 index d272ce4..0000000 --- a/scripts/mongodb-migration/config/default.js +++ /dev/null @@ -1,25 +0,0 @@ -'use strict'; - -/* - * Copyright (c) 2018 TopCoder, Inc. All rights reserved. - */ - -/** - * This module contains the configurations of the app. - * Changes in 1.1: - * @author TCSCODER - * @version 1.1 - */ - -module.exports = { - MONGODB_URL: process.env.MONGODB_URL || ' ', - COLLECTION_COUNTS: process.env.COLLECTION_COUNTS || 100, - DYNAMODB: { - AWS_ACCESS_KEY_ID: process.env.AWS_ACCESS_KEY_ID || ' ', - AWS_SECRET_ACCESS_KEY: process.env.AWS_SECRET_ACCESS_KEY || ' ', - AWS_REGION: process.env.AWS_REGION || 'us-east-1', - IS_LOCAL: process.env.IS_LOCAL || 'false', - }, - MIGRATION_DELAY_TIME: 5 * 1000, // 5 sec - MIGRATION_DELAY_INTERVAL: 50 -}; diff --git a/scripts/mongodb-migration/constants.js b/scripts/mongodb-migration/constants.js deleted file mode 100644 index 9b7385b..0000000 --- a/scripts/mongodb-migration/constants.js +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2017 TopCoder, Inc. All rights reserved. - */ - -/** - * Define constants. - * - * @author TCSCODER - * @version 1.0 - */ - - -// The user roles -const USER_ROLES = { - OWNER: 'owner', -}; - -// The user types -const USER_TYPES = { - GITHUB: 'github', - GITLAB: 'gitlab', -}; - - -module.exports = { - USER_ROLES, - USER_TYPES, -}; diff --git a/scripts/mongodb-migration/create-empty-tables.js b/scripts/mongodb-migration/create-empty-tables.js deleted file mode 100644 index bc69a8a..0000000 --- a/scripts/mongodb-migration/create-empty-tables.js +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2018 TopCoder, Inc. All rights reserved. - */ - -/** - * Create empty tables for DynamoDB - * - * @author TCSCODER - * @version 1.0 - */ - -const Path = require('path'); -const fs = require('fs'); -const dynamoose = require('dynamoose'); -const config = require('config'); -const logger = require('./utils/logger'); -const schemaPath = './dynamodb/models'; - -dynamoose.AWS.config.update({ - accessKeyId: config.DYNAMODB.AWS_ACCESS_KEY_ID, - secretAccessKey: config.DYNAMODB.AWS_SECRET_ACCESS_KEY, - region: config.DYNAMODB.AWS_REGION, -}); - -if (config.DYNAMODB.IS_LOCAL === 'true') { - dynamoose.local(); -} - -dynamoose.setDefaults({ - create: true, - update: true, -}); - -const Table = dynamoose.Table; - -fs.readdirSync(schemaPath).forEach((file) => { // eslint-disable-line no-sync - if (file !== 'index.js') { - const filename = file.split('.')[0]; - const fileFullPath = Path.join(__dirname + schemaPath.replace('.', ''), filename); - const schema = require(fileFullPath); // eslint-disable-line global-require - const table_name = 'Topcoder_X.' + filename - const table = new Table(table_name, schema, null, dynamoose); - table.create((err) => { - if(!err) { - logger.info(`*** Table ${table_name} has been created ***`); - } else { - // if table exists, delete and re-create with empty - table.delete((err) => { - if (!err) { - table.create((err) => { - if (!err) { - logger.info(`*** Table ${table_name} has been created ***`); - } else - { - logger.info(`*** Table ${table_name} created failed -- ${err.message} ***`); - } - }); - } else { - logger.info(`*** Delete exist Table ${table_name} failed -- ${err.message} ***`); - } - }) - } - }); - } -}); diff --git a/scripts/mongodb-migration/dynamodb/db-helper.js b/scripts/mongodb-migration/dynamodb/db-helper.js deleted file mode 100644 index 301890f..0000000 --- a/scripts/mongodb-migration/dynamodb/db-helper.js +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2018 TopCoder, Inc. All rights reserved. - */ -/** - * This module contains the database helper methods. - * - * @version 1.0 - */ - -/** - * Get all data collection - * @param {Object} model The dynamoose model to scan - * @returns {Promise} - */ -async function scan(model) { - return await new Promise((resolve, reject) => { - model.scan().exec((err, result) => { - if (err) { - reject(err); - } - return resolve(result.count === 0 ? [] : result); - }); - }); -} - -/** - * Create item in database - * @param {Object} Model The dynamoose model to create - * @param {Object} data The create data object - * @returns {Promise} - */ -async function create(Model, data) { - return await new Promise((resolve, reject) => { - const dbItem = new Model(data); - dbItem.save((err) => { - if (err) { - reject(err); - } - - return resolve(dbItem); - }); - }); -} - -/** - * Delete all data in database - * @param {Object} Model The dynamoose model to delete - * @param {Array} list The data list - */ -async function removeAll(Model, list) { - return await new Promise((resolve, reject) => { - console.log(`Removing: ${list.length}`) - Model.batchDelete(list, (err) => { - if (err) { - reject(err); - } - - resolve(); - }); - }); -} - -module.exports = { - scan, - create, - removeAll, -}; diff --git a/scripts/mongodb-migration/dynamodb/models/CopilotPayment.js b/scripts/mongodb-migration/dynamodb/models/CopilotPayment.js deleted file mode 100644 index d25092f..0000000 --- a/scripts/mongodb-migration/dynamodb/models/CopilotPayment.js +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2018 TopCoder, Inc. All rights reserved. - */ - -/** - * This module contains the schema of copilot payment with TC-X. - * - * @author TCSCODER - * @version 1.0 - */ - -const dynamoose = require('dynamoose'); - -const Schema = dynamoose.Schema; - -const schema = new Schema({ - id: { - type: String, - hashKey: true, - required: true, - }, - project: { - type: String, - index: { - global: true, - rangeKey: 'id', - project: true, - name: 'ProjectIndex', - }, - }, - amount: {type: Number, required: true}, - description: {type: String, required: true}, - challengeId: { - type: Number, - required: false, - index: { - global: true, - rangeKey: 'id', - project: true, - name: 'ChallengeIdIndex', - }, - }, - closed: { - type: String, - required: true, - default: 'false', - index: { - global: true, - rangeKey: 'id', - project: true, - name: 'ClosedIndex', - }, - }, - username: { - type: String, - required: true, - index: { - global: true, - rangeKey: 'id', - project: true, - name: 'UsernameIndex', - }, - }, - status: { - type: String, - index: { - global: true, - rangeKey: 'id', - project: true, - name: 'StatusIndex', - }, - }, -}); - -module.exports = schema; diff --git a/scripts/mongodb-migration/dynamodb/models/Issue.js b/scripts/mongodb-migration/dynamodb/models/Issue.js deleted file mode 100644 index 9da0d5d..0000000 --- a/scripts/mongodb-migration/dynamodb/models/Issue.js +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2017 TopCoder, Inc. All rights reserved. - */ - -/** - * Schema for Issue. - * @author TCSCODER - * @version 1.0 - */ -const dynamoose = require('dynamoose'); - -const Schema = dynamoose.Schema; - -const schema = new Schema({ - id: {type: String, hashKey: true, required: true}, - // From the receiver service - number: { - type: Number, - required: true, - index: { - global: true, - rangeKey: 'id', - project: true, - name: 'NumberIndex', - }, - }, - title: {type: String, required: true}, - body: {type: String}, - prizes: {type: [Number], required: true}, // extracted from title - provider: { - type: String, - required: true, - index: { - global: true, - rangeKey: 'id', - project: true, - name: 'ProviderIndex', - }, - }, // github or gitlab - repositoryId: { - type: Number, - required: true, - index: { - global: true, - rangeKey: 'id', - project: true, - name: 'RepositoryIdIndex', - }, - }, - labels: { - type: Array, - required: true, - }, - assignee: {type: String, required: false}, - updatedAt: { - type: Date, - default: Date.now, - }, - // From topcoder api - challengeId: {type: Number, required: false}, - projectId: {type: String}, - status: {type: String}, - assignedAt: {type: Date, required: false}, -}); - -module.exports = schema; - diff --git a/scripts/mongodb-migration/dynamodb/models/OwnerUserGroup.js b/scripts/mongodb-migration/dynamodb/models/OwnerUserGroup.js deleted file mode 100644 index 4e2e58e..0000000 --- a/scripts/mongodb-migration/dynamodb/models/OwnerUserGroup.js +++ /dev/null @@ -1,60 +0,0 @@ -/** - * This defines owner user group model. An owner user group is a group to add user to. - */ -const _ = require('lodash'); -const dynamoose = require('dynamoose'); -const constants = require('../../constants'); - -const Schema = dynamoose.Schema; - -const schema = new Schema({ - id: { - type: String, - hashKey: true, - required: true, - }, - ownerUsername: { - type: String, - required: true, - index: { - global: true, - rangeKey: 'id', - project: true, - name: 'OwnerUsernameIndex', - }, - }, - type: { - type: String, - required: true, - enum: _.values(constants.USER_TYPES), - index: { - global: true, - rangeKey: 'id', - project: true, - name: 'TypeIndex', - }, - }, - groupId: { - type: String, - required: true, - index: { - global: true, - rangeKey: 'id', - project: true, - name: 'GroupIdIndex', - }, - }, - identifier: { - type: String, - required: true, - index: { - global: true, - rangeKey: 'id', - project: true, - name: 'IdentifierIndex', - }, - }, -}); - - -module.exports = schema; diff --git a/scripts/mongodb-migration/dynamodb/models/OwnerUserTeam.js b/scripts/mongodb-migration/dynamodb/models/OwnerUserTeam.js deleted file mode 100644 index 4d3d8fe..0000000 --- a/scripts/mongodb-migration/dynamodb/models/OwnerUserTeam.js +++ /dev/null @@ -1,60 +0,0 @@ -/** - * This defines owner user team model. An owner user team is a team to add user to. - */ -const _ = require('lodash'); -const dynamoose = require('dynamoose'); -const constants = require('../../constants'); - -const Schema = dynamoose.Schema; - -const schema = new Schema({ - id: { - type: String, - hashKey: true, - required: true, - }, - ownerUsername: { - type: String, - required: true, - index: { - global: true, - rangeKey: 'id', - project: true, - name: 'OwnerUsernameIndex', - }, - }, - type: { - type: String, - required: true, - enum: _.values(constants.USER_TYPES), - index: { - global: true, - rangeKey: 'id', - project: true, - name: 'TypeIndex', - }, - }, - teamId: { - type: String, - required: true, - index: { - global: true, - rangeKey: 'id', - project: true, - name: 'TeamIdIndex', - }, - }, - ownerToken: {type: String, required: true}, - identifier: { - type: String, - required: true, - index: { - global: true, - rangeKey: 'id', - project: true, - name: 'IdentifierIndex', - }, - }, -}); - -module.exports = schema; diff --git a/scripts/mongodb-migration/dynamodb/models/Project.js b/scripts/mongodb-migration/dynamodb/models/Project.js deleted file mode 100644 index 04f3c80..0000000 --- a/scripts/mongodb-migration/dynamodb/models/Project.js +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2018 TopCoder, Inc. All rights reserved. - */ - -/** - * This module contains the schema of a project registered with TC-X. - * - * @author TCSCODER - * @version 1.0 - */ - -const dynamoose = require('dynamoose'); - -const Schema = dynamoose.Schema; - -const schema = new Schema({ - id: { - type: String, - hashKey: true, - required: true, - }, - title: {type: String, required: true}, - tcDirectId: { - type: Number, - required: true, - index: { - global: true, - rangeKey: 'id', - project: true, - name: 'TcDirectIdIndex', - }, - }, - repoUrl: {type: String, required: true}, - rocketChatWebhook: {type: String, required: false}, - rocketChatChannelName: {type: String, required: false}, - archived: {type: String, required: true}, - owner: {type: String, required: true}, - secretWebhookKey: {type: String, required: true}, - copilot: {type: String, required: true}, -}); - -module.exports = schema; diff --git a/scripts/mongodb-migration/dynamodb/models/User.js b/scripts/mongodb-migration/dynamodb/models/User.js deleted file mode 100644 index 49c0123..0000000 --- a/scripts/mongodb-migration/dynamodb/models/User.js +++ /dev/null @@ -1,65 +0,0 @@ -/** - * This defines user model. - */ - -const _ = require('lodash'); -const dynamoose = require('dynamoose'); -const constants = require('../../constants'); - -const Schema = dynamoose.Schema; - -const schema = new Schema({ - id: { - type: String, - hashKey: true, - required: true, - }, - userProviderId: { - type: Number, - required: true, - index: { - global: true, - rangeKey: 'id', - project: true, - name: 'UsesProviderIdIndex', - }, - }, - username: { - type: String, - required: true, - index: { - global: true, - rangeKey: 'id', - project: true, - name: 'UsernameIndex', - }, - }, - role: { - type: String, - required: true, - enum: _.values(constants.USER_ROLES), - index: { - global: true, - project: true, - name: 'RoleIndex', - rangeKey: 'id', - }, - }, - type: { - type: String, - required: true, - enum: _.values(constants.USER_TYPES), - index: { - global: true, - rangeKey: 'id', - name: 'TypeIndex', - project: true, - }, - }, - // gitlab token data - accessToken: {type: String, required: false}, - accessTokenExpiration: {type: Date, required: false}, - refreshToken: {type: String, required: false}, -}); - -module.exports = schema; diff --git a/scripts/mongodb-migration/dynamodb/models/UserMapping.js b/scripts/mongodb-migration/dynamodb/models/UserMapping.js deleted file mode 100644 index bb1e164..0000000 --- a/scripts/mongodb-migration/dynamodb/models/UserMapping.js +++ /dev/null @@ -1,30 +0,0 @@ -/** - * This defines user mapping model. - */ -const dynamoose = require('dynamoose'); - -const Schema = dynamoose.Schema; - -const schema = new Schema({ - id: { - type: String, - required: true, - hashKey: true, - }, - topcoderUsername: { - type: String, - required: true, - index: { - global: true, - project: true, - rangKey: 'id', - name: 'TopcoderUsernameIndex', - }, - }, - githubUsername: String, - gitlabUsername: String, - githubUserId: Number, - gitlabUserId: Number, -}); - -module.exports = schema; diff --git a/scripts/mongodb-migration/dynamodb/models/index.js b/scripts/mongodb-migration/dynamodb/models/index.js deleted file mode 100644 index 278f94c..0000000 --- a/scripts/mongodb-migration/dynamodb/models/index.js +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Initialize and exports all models. - */ -const Path = require('path'); -const fs = require('fs'); -const dynamoose = require('dynamoose'); -const config = require('config'); - -dynamoose.AWS.config.update({ - accessKeyId: config.DYNAMODB.AWS_ACCESS_KEY_ID, - secretAccessKey: config.DYNAMODB.AWS_SECRET_ACCESS_KEY, - region: config.DYNAMODB.AWS_REGION, -}); - -if (config.DYNAMODB.IS_LOCAL === 'true') { - dynamoose.local(); -} - -dynamoose.setDefaults({ - create: true, - update: true, -}); - -const models = {}; - -// Bootstrap models -fs.readdirSync(__dirname).forEach((file) => { // eslint-disable-line no-sync - if (file !== 'index.js') { - const filename = file.split('.')[0]; - const schema = require(Path.join(__dirname, filename)); // eslint-disable-line global-require - const table_name = 'Topcoder_X.' + filename - const model = dynamoose.model(table_name, schema); - models[filename] = model; - } -}); - -module.exports = models; diff --git a/scripts/mongodb-migration/migrate-data.js b/scripts/mongodb-migration/migrate-data.js deleted file mode 100644 index 7f0b6a7..0000000 --- a/scripts/mongodb-migration/migrate-data.js +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Copyright (c) 2018 TopCoder, Inc. All rights reserved. - */ - -/** - * Migrate data to dynamodb - * - * @author TCSCODER - * @version 1.0 - */ -const fs = require('fs'); -const mongoModels = require('./mongodb/models'); -const dynamoModels = require('./dynamodb/models'); -const logger = require('./utils/logger'); -const dynamoDBHelper = require('./dynamodb/db-helper'); -const helper = require('./utils/helper'); -const config = require('config'); - -const projectIdMapping = {}; - -/** - * Wait to avoid throughput errors from DynamoDB - */ -async function waitSomeTime() { - await new Promise((resolve, reject) => { - setTimeout(() => resolve(), config.MIGRATION_DELAY_TIME); - }); -} - -/** - * Save data on a backup JSON file - * @param {String} model the model name - * @param {Object|Array} data The data to save - */ -async function saveBackup(model, data) { - await fs.writeFile(`${__dirname}/backups/${model}.json`, JSON.stringify(data, null, 2)); -} - -let countRequests = 0; - -/** - * Migrate user data - */ -async function migrateUserData() { - logger.info('*** migrate user data ***'); - const list = await mongoModels.User.find({}); - await saveBackup('User', list); - for(const user of list) { - if (countRequests === config.MIGRATION_DELAY_INTERVAL) { - await waitSomeTime(); - countRequests = 0; - } - countRequests++; - const data = user.toObject(); - data.id = helper.generateIdentifier(); - delete data._id; - delete data.__v; - await dynamoDBHelper.create(dynamoModels.User, data); - } -} - -/** - * Migrate user mapping data - */ -async function migrateUserMappingData() { - logger.info('*** migrate user mapping data ***'); - const list = await mongoModels.UserMapping.find({}); - await saveBackup('UserMapping', list); - for(const mapping of list) { - if (countRequests === config.MIGRATION_DELAY_INTERVAL) { - await waitSomeTime(); - countRequests = 0; - } - countRequests++; - const data = mapping.toObject(); - data.id = helper.generateIdentifier(); - delete data._id; - delete data.__v; - await dynamoDBHelper.create(dynamoModels.UserMapping, data); - } -} - -async function migrateProjectData() { - logger.info('*** migrate project data ***'); - const list = await mongoModels.Project.find({}); - await saveBackup('Project', list); - for(const project of list) { - if (countRequests === config.MIGRATION_DELAY_INTERVAL) { - await waitSomeTime(); - countRequests = 0; - } - countRequests++; - const data = project.toObject(); - data.id = helper.generateIdentifier(); - projectIdMapping[data._id.toString()] = data.id; - delete data._id; - delete data.__v; - await dynamoDBHelper.create(dynamoModels.Project, data); - } -} - -/** - * Migrate issue data - */ -async function migrateIssueData() { - logger.info('*** migrate issue data ***'); - const labelsToIgnore = [ - 'Paid', - 'tcx_Paid', - 'tcx_OutOfScope' - ] - const list = await mongoModels.Issue.find({ - labels: { - $nin: labelsToIgnore - } - }); - await saveBackup('Issue', list); - for(const issue of list) { - if (countRequests === config.MIGRATION_DELAY_INTERVAL) { - await waitSomeTime(); - countRequests = 0; - } - countRequests++; - const data = issue.toObject(); - data.id = helper.generateIdentifier(); - data.projectId = projectIdMapping[data.projectId.toString()]; - delete data._id; - delete data.__v; - await dynamoDBHelper.create(dynamoModels.Issue, data); - } -} - -/** - * Migrate copilot payment data - */ -async function migrateCopilotPaymentData() { - logger.info('*** migrate copilot payments data ***'); - const list = await mongoModels.CopilotPayment.find({}); - await saveBackup('CopilotPayment', list); - for(const payment of list) { - if (countRequests === config.MIGRATION_DELAY_INTERVAL) { - await waitSomeTime(); - countRequests = 0; - } - countRequests++; - const data = payment.toObject(); - data.id = helper.generateIdentifier(); - data.project = projectIdMapping[data.project.toString()]; - delete data._id; - delete data.__v; - await dynamoDBHelper.create(dynamoModels.CopilotPayment, data); - } -} - -/** - * Migrate owner user team data - */ -async function migrateOwnerUserTeamData() { - logger.info('*** migrate owner user team data ***'); - const list = await mongoModels.OwnerUserTeam.find({}); - await saveBackup('OwnerUserTeam', list); - for(const team of list) { - if (countRequests === config.MIGRATION_DELAY_INTERVAL) { - await waitSomeTime(); - countRequests = 0; - } - countRequests++; - const data = team.toObject(); - data.id = helper.generateIdentifier(); - delete data._id; - delete data.__v; - await dynamoDBHelper.create(dynamoModels.OwnerUserTeam, data); - } -} - -/** - * Migrate owner user group data - */ -async function migrateOwnerUserGroupData() { - logger.info('*** migrate owner user group data ***'); - const list = await mongoModels.OwnerUserGroup.find({}); - await saveBackup('OwnerUserGroup', list); - for(const group of list) { - if (countRequests === config.MIGRATION_DELAY_INTERVAL) { - await waitSomeTime(); - countRequests = 0; - } - countRequests++; - const data = group.toObject(); - data.id = helper.generateIdentifier(); - delete data._id; - delete data.__v; - await dynamoDBHelper.create(dynamoModels.OwnerUserGroup, data); - } -} - -/** - * Clean dynamoDB database - */ -async function cleanup() { - logger.info('*** Started cleanup ***'); - - logger.info('*** cleaning up data from groups ***'); - const groups = await dynamoDBHelper.scan(dynamoModels.OwnerUserGroup); - await dynamoDBHelper.removeAll(dynamoModels.OwnerUserGroup, groups); - - logger.info('*** cleaning up data from teams ***'); - const teams = await dynamoDBHelper.scan(dynamoModels.OwnerUserTeam); - await dynamoDBHelper.removeAll(dynamoModels.OwnerUserTeam, teams); - - logger.info('*** cleaning up data from payments ***'); - const payments = await dynamoDBHelper.scan(dynamoModels.CopilotPayment); - await dynamoDBHelper.removeAll(dynamoModels.CopilotPayment, payments); - - logger.info('*** cleaning up data from issues ***'); - const issues = await dynamoDBHelper.scan(dynamoModels.Issue); - await dynamoDBHelper.removeAll(dynamoModels.Issue, issues); - - logger.info('*** cleaning up data from projects ***'); - const projects = await dynamoDBHelper.scan(dynamoModels.Project); - await dynamoDBHelper.removeAll(dynamoModels.Project, projects); - - logger.info('*** cleaning up data from mappings ***'); - const mappings = await dynamoDBHelper.scan(dynamoModels.UserMapping); - await dynamoDBHelper.removeAll(dynamoModels.UserMapping, mappings); - - logger.info('*** cleaning up data from users ***'); - const users = await dynamoDBHelper.scan(dynamoModels.User); - await dynamoDBHelper.removeAll(dynamoModels.User, users); -} - -async function start() { - await cleanup(); - await migrateUserData(); - await migrateUserMappingData(); - await migrateProjectData(); - await migrateIssueData(); - await migrateCopilotPaymentData(); - await migrateOwnerUserTeamData(); - await migrateOwnerUserGroupData(); - - logger.info('migrate data finish!'); - process.exit(0); -} - -start(); diff --git a/scripts/mongodb-migration/mongodb/models/CopilotPayment.js b/scripts/mongodb-migration/mongodb/models/CopilotPayment.js deleted file mode 100644 index cd7b80d..0000000 --- a/scripts/mongodb-migration/mongodb/models/CopilotPayment.js +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2018 TopCoder, Inc. All rights reserved. - */ - -/** - * This module contains the schema of a project registered with TC-X. - * - * @author TCSCODER - * @version 1.0 - */ - - -const mongoose = require('mongoose'); - -const schema = new mongoose.Schema({ - project: { type: mongoose.Schema.Types.ObjectId, ref: 'Project' }, - amount: { type: Number, required: true }, - description: { type: String, required: true }, - challengeId: { type: Number, required: false }, - closed: { type: Boolean, required: true, default: false }, - username: { type: String, required: true }, - status: { type: String }, -}, { - toObject: { - transform: (doc, ret) => { - delete ret.__v; - }, - }, - toJSON: { - transform: (doc, ret) => { - delete ret.__v; - }, - }, - }); - -schema.index({ project: 1 }); -schema.index({ username: 1 }); -schema.index({ challengeId: 1 }); -schema.index({ closed: 1 }); - -module.exports = schema; diff --git a/scripts/mongodb-migration/mongodb/models/Issue.js b/scripts/mongodb-migration/mongodb/models/Issue.js deleted file mode 100644 index abfaa53..0000000 --- a/scripts/mongodb-migration/mongodb/models/Issue.js +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2017 TopCoder, Inc. All rights reserved. - */ - -/** - * Schema for Issue. - * @author TCSCODER - * @version 1.0 - */ -const mongoose = require('mongoose'); - -const schema = new mongoose.Schema({ - // From the receiver service - number: {type: Number, required: true}, - title: {type: String, required: true}, - body: String, - prizes: [{type: Number, required: true}], // extracted from title - provider: {type: String, required: true}, // github or gitlab - repositoryId: {type: Number, required: true}, - labels: [{type: String, required: true}], - assignee: {type: String, required: false}, - updatedAt: { - type: Date, - default: Date.now, - }, - // From topcoder api - challengeId: {type: Number, required: false, unique: true, sparse: true}, - projectId: {type: mongoose.Schema.Types.ObjectId, ref: 'Project'}, - assignedAt: {type: Date, required: false}, -}); - -// Issue number, provider, repositoryId must be unique -schema.index({number: 1, provider: 1, repositoryId: 1}, {unique: true}); -schema.index({labels: 1}); -schema.index({projectId: 1}); - -schema.pre('save', function preSave(next) { - this.updatedAt = Date.now(); // eslint-disable-line - return next(); -}); -module.exports = schema; diff --git a/scripts/mongodb-migration/mongodb/models/OwnerUserGroup.js b/scripts/mongodb-migration/mongodb/models/OwnerUserGroup.js deleted file mode 100644 index 79478c1..0000000 --- a/scripts/mongodb-migration/mongodb/models/OwnerUserGroup.js +++ /dev/null @@ -1,22 +0,0 @@ -/** - * This defines owner user group model. An owner user group is a group to add user to. - */ -const _ = require('lodash'); -const mongoose = require('mongoose'); -const constants = require('../../constants'); - -const Schema = mongoose.Schema; - -const schema = new Schema({ - ownerUsername: {type: String, required: true}, - type: {type: String, required: true, enum: _.values(constants.USER_TYPES)}, - groupId: {type: String, required: true}, - identifier: {type: String, required: true, unique: true}, -}); - -schema.index({ownerUsername: 1}); -schema.index({type: 1}); -schema.index({groupId: 1}); -schema.index({identifier: 1}); - -module.exports = schema; diff --git a/scripts/mongodb-migration/mongodb/models/OwnerUserTeam.js b/scripts/mongodb-migration/mongodb/models/OwnerUserTeam.js deleted file mode 100644 index bf80aad..0000000 --- a/scripts/mongodb-migration/mongodb/models/OwnerUserTeam.js +++ /dev/null @@ -1,23 +0,0 @@ -/** - * This defines owner user team model. An owner user team is a team to add user to. - */ -const _ = require('lodash'); -const mongoose = require('mongoose'); -const constants = require('../../constants'); - -const Schema = mongoose.Schema; - -const schema = new Schema({ - ownerUsername: {type: String, required: true}, - type: {type: String, required: true, enum: _.values(constants.USER_TYPES)}, - teamId: {type: String, required: true}, - ownerToken: {type: String, required: true}, - identifier: {type: String, required: true, unique: true}, -}); - -schema.index({ownerUsername: 1}); -schema.index({type: 1}); -schema.index({teamId: 1}); -schema.index({identifier: 1}); - -module.exports = schema; diff --git a/scripts/mongodb-migration/mongodb/models/Project.js b/scripts/mongodb-migration/mongodb/models/Project.js deleted file mode 100644 index 1d815e5..0000000 --- a/scripts/mongodb-migration/mongodb/models/Project.js +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2018 TopCoder, Inc. All rights reserved. - */ - -/** - * This module contains the schema of a project registered with TC-X. - * - * @author TCSCODER - * @version 1.0 - */ - - -const mongoose = require('mongoose'); - -const schema = new mongoose.Schema({ - title: { type: String, required: true }, - tcDirectId: { type: Number, required: true }, - repoUrl: { type: String, required: true }, - rocketChatWebhook: { type: String, required: false }, - rocketChatChannelName: { type: String, required: false }, - archived: { type: String, required: true }, - owner: { type: String, required: true }, - secretWebhookKey: { type: String, required: true }, - copilot: { type: String, required: true }, -}); - -schema.index({tcDirectId: 1}); - -module.exports = schema; diff --git a/scripts/mongodb-migration/mongodb/models/User.js b/scripts/mongodb-migration/mongodb/models/User.js deleted file mode 100644 index ee355a5..0000000 --- a/scripts/mongodb-migration/mongodb/models/User.js +++ /dev/null @@ -1,25 +0,0 @@ -/** - * This defines user model. - */ -const _ = require('lodash'); -const mongoose = require('mongoose'); -const constants = require('../../constants'); - -const Schema = mongoose.Schema; - -const schema = new Schema({ - userProviderId: {type: Number, required: true}, - username: {type: String, required: true}, - role: {type: String, required: true, enum: _.values(constants.USER_ROLES)}, - type: {type: String, required: true, enum: _.values(constants.USER_TYPES)}, - // gitlab token data - accessToken: String, - accessTokenExpiration: Date, - refreshToken: String, -}); -schema.index({userProviderId: 1}); -schema.index({username: 1}); -schema.index({role: 1}); -schema.index({type: 1}); - -module.exports = schema; diff --git a/scripts/mongodb-migration/mongodb/models/UserMapping.js b/scripts/mongodb-migration/mongodb/models/UserMapping.js deleted file mode 100644 index 8988f76..0000000 --- a/scripts/mongodb-migration/mongodb/models/UserMapping.js +++ /dev/null @@ -1,18 +0,0 @@ -/** - * This defines user mapping model. - */ -const mongoose = require('mongoose'); - -const Schema = mongoose.Schema; - -const schema = new Schema({ - topcoderUsername: {type: String, required: true, unique: true}, - githubUsername: String, - gitlabUsername: String, - githubUserId: Number, - gitlabUserId: Number, -}); - -schema.index({topcoderUsername: 1}, {unique: true}); - -module.exports = schema; diff --git a/scripts/mongodb-migration/mongodb/models/index.js b/scripts/mongodb-migration/mongodb/models/index.js deleted file mode 100644 index 614c7ae..0000000 --- a/scripts/mongodb-migration/mongodb/models/index.js +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Initialize and exports all models. - */ -const Path = require('path'); -const fs = require('fs'); -const mongoose = require('mongoose'); -const config = require('config'); - -mongoose.Promise = Promise; -const conn = mongoose.createConnection(config.MONGODB_URL); -const models = {}; - -// Bootstrap models -fs.readdirSync(__dirname).forEach((file) => { // eslint-disable-line no-sync - if (file !== 'index.js') { - const filename = file.split('.')[0]; - const schema = require(Path.join(__dirname, filename)); // eslint-disable-line global-require - const model = conn.model(filename, schema); - models[filename] = model; - - model.schema.options.minimize = false; - model.schema.options.toJSON = { - transform: (doc, ret) => { - if (ret._id) { - ret.id = String(ret._id); - delete ret._id; - } - delete ret.__v; - return ret; - }, - }; - } -}); - -module.exports = models; diff --git a/scripts/mongodb-migration/package-lock.json b/scripts/mongodb-migration/package-lock.json deleted file mode 100644 index 8835327..0000000 --- a/scripts/mongodb-migration/package-lock.json +++ /dev/null @@ -1,456 +0,0 @@ -{ - "name": "migration-tools", - "requires": true, - "lockfileVersion": 1, - "dependencies": { - "aws-sdk": { - "version": "2.358.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.358.0.tgz", - "integrity": "sha512-nS47i+YecWDAy3JE55GrC2dLbWsc5lqIub8y+VgHPoVI11f/wmWpF1kY+8FD20IGbZQHWiqiMdMZjFS86L1w6g==", - "requires": { - "buffer": "4.9.1", - "events": "1.1.1", - "ieee754": "1.1.8", - "jmespath": "0.15.0", - "querystring": "0.2.0", - "sax": "1.2.1", - "url": "0.10.3", - "uuid": "3.1.0", - "xml2js": "0.4.19" - }, - "dependencies": { - "uuid": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz", - "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==" - } - } - }, - "base64-js": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz", - "integrity": "sha512-ccav/yGvoa80BQDljCxsmmQ3Xvx60/UpBIij5QN21W3wBi/hhIC9OoO+KLpu9IJTS9j4DRVJ3aDDF9cMSoa2lw==" - }, - "bl": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.1.tgz", - "integrity": "sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==", - "requires": { - "readable-stream": "^2.3.5", - "safe-buffer": "^5.1.1" - } - }, - "bluebird": { - "version": "3.5.1", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz", - "integrity": "sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA==" - }, - "bson": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.4.tgz", - "integrity": "sha512-S/yKGU1syOMzO86+dGpg2qGoDL0zvzcb262G+gqEy6TgP6rt6z6qxSFX/8X6vLC91P7G7C3nLs0+bvDzmvBA3Q==" - }, - "buffer": { - "version": "4.9.1", - "resolved": "http://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", - "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", - "requires": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4", - "isarray": "^1.0.0" - } - }, - "colors": { - "version": "1.0.3", - "resolved": "http://registry.npmjs.org/colors/-/colors-1.0.3.tgz", - "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=" - }, - "config": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/config/-/config-2.0.2.tgz", - "integrity": "sha512-duIbkKb0gls0bOtGwd1vaD4236MwepQlZcrMheOGrn3/9Px7oYFh8G4LB3ylGOlPr5wGoJRm8Grb2RihJZxuHQ==", - "requires": { - "json5": "^1.0.1" - } - }, - "core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" - }, - "cycle": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz", - "integrity": "sha1-IegLK+hYD5i0aPN5QwZisEbDStI=" - }, - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "deep-equal": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz", - "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=" - }, - "denque": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz", - "integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ==" - }, - "dynamoose": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/dynamoose/-/dynamoose-1.1.0.tgz", - "integrity": "sha512-aob8/1TgRTRww/r7ze2SLwzsAXsViE6CHXMBWYxMbtK1M+oTnAOWNe0SvY62Nq/AP3wh4zR+CtI+uS3hoFC/rA==", - "requires": { - "aws-sdk": "^2.207.0", - "debug": "^2.6.8", - "deep-equal": "^1.0.1", - "hooks": "0.3.2", - "object-path": "^0.11.4", - "q": "^1.5.0" - } - }, - "events": { - "version": "1.1.1", - "resolved": "http://registry.npmjs.org/events/-/events-1.1.1.tgz", - "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" - }, - "eyes": { - "version": "0.1.8", - "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", - "integrity": "sha1-Ys8SAjTGg3hdkCNIqADvPgzCC8A=" - }, - "hooks": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/hooks/-/hooks-0.3.2.tgz", - "integrity": "sha1-ox8GDCAmzqbPHKPrF4Qw5xjhxKM=" - }, - "ieee754": { - "version": "1.1.8", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.8.tgz", - "integrity": "sha1-vjPUCsEO8ZJnAfbwii2G+/0a0+Q=" - }, - "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" - }, - "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "isstream": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" - }, - "jmespath": { - "version": "0.15.0", - "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", - "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" - }, - "json5": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", - "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", - "requires": { - "minimist": "^1.2.0" - } - }, - "kareem": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/kareem/-/kareem-2.3.1.tgz", - "integrity": "sha512-l3hLhffs9zqoDe8zjmb/mAN4B8VT3L56EUvKNqLFVs9YlFA+zx7ke1DO8STAdDyYNkeSo1nKmjuvQeI12So8Xw==" - }, - "lodash": { - "version": "4.17.19", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", - "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==" - }, - "memory-pager": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", - "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", - "optional": true - }, - "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" - }, - "moment": { - "version": "2.22.2", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.22.2.tgz", - "integrity": "sha1-PCV/mDn8DpP/UxSWMiOeuQeD/2Y=" - }, - "mongodb": { - "version": "3.5.5", - "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.5.5.tgz", - "integrity": "sha512-GCjDxR3UOltDq00Zcpzql6dQo1sVry60OXJY3TDmFc2SWFY6c8Gn1Ardidc5jDirvJrx2GC3knGOImKphbSL3A==", - "requires": { - "bl": "^2.2.0", - "bson": "^1.1.1", - "denque": "^1.4.1", - "require_optional": "^1.0.1", - "safe-buffer": "^5.1.2", - "saslprep": "^1.0.0" - } - }, - "mongoose": { - "version": "5.9.6", - "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-5.9.6.tgz", - "integrity": "sha512-EfFGO2QUoenf/4eFeF5y2R8aBLKHtqwrMk1pVGgl3OyNWufP5XLLPIuihP006YqR1+6xM1YsBzGpgBjMZkINGA==", - "requires": { - "bson": "~1.1.1", - "kareem": "2.3.1", - "mongodb": "3.5.5", - "mongoose-legacy-pluralize": "1.0.2", - "mpath": "0.6.0", - "mquery": "3.2.2", - "ms": "2.1.2", - "regexp-clone": "1.0.0", - "safe-buffer": "5.1.2", - "sift": "7.0.1", - "sliced": "1.0.1" - }, - "dependencies": { - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - } - } - }, - "mongoose-legacy-pluralize": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/mongoose-legacy-pluralize/-/mongoose-legacy-pluralize-1.0.2.tgz", - "integrity": "sha512-Yo/7qQU4/EyIS8YDFSeenIvXxZN+ld7YdV9LqFVQJzTLye8unujAWPZ4NWKfFA+RNjh+wvTWKY9Z3E5XM6ZZiQ==" - }, - "mpath": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/mpath/-/mpath-0.6.0.tgz", - "integrity": "sha512-i75qh79MJ5Xo/sbhxrDrPSEG0H/mr1kcZXJ8dH6URU5jD/knFxCVqVC/gVSW7GIXL/9hHWlT9haLbCXWOll3qw==" - }, - "mquery": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/mquery/-/mquery-3.2.2.tgz", - "integrity": "sha512-XB52992COp0KP230I3qloVUbkLUxJIu328HBP2t2EsxSFtf4W1HPSOBWOXf1bqxK4Xbb66lfMJ+Bpfd9/yZE1Q==", - "requires": { - "bluebird": "3.5.1", - "debug": "3.1.0", - "regexp-clone": "^1.0.0", - "safe-buffer": "5.1.2", - "sliced": "1.0.1" - }, - "dependencies": { - "debug": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", - "requires": { - "ms": "2.0.0" - } - } - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "object-path": { - "version": "0.11.5", - "resolved": "https://registry.npmjs.org/object-path/-/object-path-0.11.5.tgz", - "integrity": "sha512-jgSbThcoR/s+XumvGMTMf81QVBmah+/Q7K7YduKeKVWL7N111unR2d6pZZarSk6kY/caeNxUDyxOvMWyzoU2eg==" - }, - "path": { - "version": "0.12.7", - "resolved": "https://registry.npmjs.org/path/-/path-0.12.7.tgz", - "integrity": "sha1-1NwqUGxM4hl+tIHr/NWzbAFAsQ8=", - "requires": { - "process": "^0.11.1", - "util": "^0.10.3" - } - }, - "process": { - "version": "0.11.10", - "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", - "integrity": "sha1-czIwDoQBYb2j5podHZGn1LwW8YI=" - }, - "process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" - }, - "punycode": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" - }, - "q": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", - "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=" - }, - "querystring": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" - }, - "readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "regexp-clone": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/regexp-clone/-/regexp-clone-1.0.0.tgz", - "integrity": "sha512-TuAasHQNamyyJ2hb97IuBEif4qBHGjPHBS64sZwytpLEqtBQ1gPJTnOaQ6qmpET16cK14kkjbazl6+p0RRv0yw==" - }, - "require_optional": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/require_optional/-/require_optional-1.0.1.tgz", - "integrity": "sha512-qhM/y57enGWHAe3v/NcwML6a3/vfESLe/sGM2dII+gEO0BpKRUkWZow/tyloNqJyN6kXSl3RyyM8Ll5D/sJP8g==", - "requires": { - "resolve-from": "^2.0.0", - "semver": "^5.1.0" - } - }, - "resolve-from": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-2.0.0.tgz", - "integrity": "sha1-lICrIOlP+h2egKgEx+oUdhGWa1c=" - }, - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "saslprep": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz", - "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==", - "optional": true, - "requires": { - "sparse-bitfield": "^3.0.3" - } - }, - "sax": { - "version": "1.2.1", - "resolved": "http://registry.npmjs.org/sax/-/sax-1.2.1.tgz", - "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" - }, - "semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" - }, - "sift": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/sift/-/sift-7.0.1.tgz", - "integrity": "sha512-oqD7PMJ+uO6jV9EQCl0LrRw1OwsiPsiFQR5AR30heR+4Dl7jBBbDLnNvWiak20tzZlSE1H7RB30SX/1j/YYT7g==" - }, - "sliced": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/sliced/-/sliced-1.0.1.tgz", - "integrity": "sha1-CzpmK10Ewxd7GSa+qCsD+Dei70E=" - }, - "sparse-bitfield": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", - "integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=", - "optional": true, - "requires": { - "memory-pager": "^1.0.2" - } - }, - "stack-trace": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", - "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=" - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } - }, - "url": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", - "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", - "requires": { - "punycode": "1.3.2", - "querystring": "0.2.0" - } - }, - "util": { - "version": "0.10.4", - "resolved": "https://registry.npmjs.org/util/-/util-0.10.4.tgz", - "integrity": "sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==", - "requires": { - "inherits": "2.0.3" - } - }, - "util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" - }, - "uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" - }, - "winston": { - "version": "2.4.4", - "resolved": "https://registry.npmjs.org/winston/-/winston-2.4.4.tgz", - "integrity": "sha512-NBo2Pepn4hK4V01UfcWcDlmiVTs7VTB1h7bgnB0rgP146bYhMxX0ypCz3lBOfNxCO4Zuek7yeT+y/zM1OfMw4Q==", - "requires": { - "async": "~1.0.0", - "colors": "1.0.x", - "cycle": "1.0.x", - "eyes": "0.1.x", - "isstream": "0.1.x", - "stack-trace": "0.0.x" - }, - "dependencies": { - "async": { - "version": "1.0.0", - "resolved": "http://registry.npmjs.org/async/-/async-1.0.0.tgz", - "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=" - } - } - }, - "xml2js": { - "version": "0.4.19", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", - "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", - "requires": { - "sax": ">=0.6.0", - "xmlbuilder": "~9.0.1" - } - }, - "xmlbuilder": { - "version": "9.0.7", - "resolved": "http://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", - "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" - } - } -} diff --git a/scripts/mongodb-migration/package.json b/scripts/mongodb-migration/package.json deleted file mode 100644 index 7234fb4..0000000 --- a/scripts/mongodb-migration/package.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "migration-tools", - "description": "The tools of migrate mongodb to dynamodb", - "private": true, - "scripts": { - "create-test-data": "node create-test-data", - "migrate-data": "node migrate-data", - "create-empty-tables": "node create-empty-tables" - }, - "dependencies": { - "config": "^2.0.2", - "dynamoose": "^1.1.0", - "lodash": "^4.17.19", - "moment": "^2.22.2", - "mongoose": "^5.9.6", - "path": "^0.12.7", - "uuid": "^3.3.2", - "winston": "^2.3.1" - } -} diff --git a/scripts/mongodb-migration/utils/helper.js b/scripts/mongodb-migration/utils/helper.js deleted file mode 100644 index 53d008d..0000000 --- a/scripts/mongodb-migration/utils/helper.js +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (c) 2018 TopCoder, Inc. All rights reserved. - */ - -/** - * This file defines helper methods. - * - * @author TCSCODER - * @version 1.0 - */ - -const uuid = require('uuid/v4'); - -/** - * Generate an unique identifier - * - * @returns {String} the generated id - */ -function generateIdentifier() { - return `${uuid()}-${new Date().getTime()}`; -} - - -module.exports = { - generateIdentifier, -}; diff --git a/scripts/mongodb-migration/utils/logger.js b/scripts/mongodb-migration/utils/logger.js deleted file mode 100644 index 1a87e7c..0000000 --- a/scripts/mongodb-migration/utils/logger.js +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2017 TopCoder, Inc. All rights reserved. - */ - -/** - * This module contains the winston logger configuration. - * - * @author TCSCODER - * @version 1.0 - */ -const winston = require('winston'); -const config = require('config'); - -const logger = new winston.Logger({ - transports: [ - new winston.transports.Console({ - level: config.LOG_LEVEL, - }), - ], -}); - -/** - * Log error details with signature - * @param {Error} err the error - * @param {String} signature the signature - */ -logger.logFullError = function logFullError(err, signature) { - if (!err || err.logged) { - return; - } - logger.error(`Error happened in ${signature}\n${err.stack}`); - err.logged = true; -}; - - -module.exports = logger; diff --git a/src/common/db-helper.js b/src/common/db-helper.js index 0cffe92..3ae537b 100644 --- a/src/common/db-helper.js +++ b/src/common/db-helper.js @@ -17,13 +17,33 @@ const logger = require('./logger'); */ async function getById(model, id) { return await new Promise((resolve, reject) => { - model.query('id').eq(id).consistent().all().exec((err, result) => { + model.queryOne('id').eq(id).consistent().all().exec((err, result) => { if (err) { logger.error(`DynamoDB getById error ${err}`); reject(err); } - return resolve(result[0]); + return resolve(result); + }); + }); +} + +/** + * Get Data by model id + * @param {Object} model The dynamoose model to query + * @param {String} key The key name + * @param {String} value The value + * @returns {Promise} + */ +async function getByKey(model, key, value) { + return await new Promise((resolve, reject) => { + model.queryOne(key).eq(value).all().exec((err, result) => { + if (err) { + logger.error(`DynamoDB getById error ${err}`); + reject(err); + } + + return resolve(result); }); }); } @@ -60,7 +80,7 @@ async function queryOneIssue(model, repositoryId, number, provider) { return await new Promise((resolve, reject) => { model.query('repositoryId').eq(repositoryId) - .filter('number').eq(number) + .where('number').eq(number) .filter('provider').eq(provider) .all() .exec((err, result) => { @@ -75,19 +95,190 @@ async function queryOneIssue(model, repositoryId, number, provider) { } /** - * Get single data by scan parameters - * @param {Object} model The dynamoose model to scan - * @param {Object} scanParams The scan parameters object + * Get single data by query parameters + * @param {Object} model The dynamoose model to query + * @param {String} username The user username + * @param {String} type The type of user * @returns {Promise} */ -async function scanOne(model, scanParams) { +async function queryOneUserByType(model, username, type) { return await new Promise((resolve, reject) => { - model.scan(scanParams).consistent().all().exec((err, result) => { - if (err) { - logger.error(`DynamoDB scanOne error ${err}`); - reject(err); + model.query('username').eq(username) + .where('type') + .eq(type) + .all() + .exec((err, result) => { + if (err || !result) { + logger.debug(`queryOneUserByType. Error. ${err}`); + return reject(err); } + return resolve(result.count === 0 ? null : result[0]); + }); + }); +} +/** + * Get single data by query parameters + * @param {Object} model The dynamoose model to query + * @param {String} username The user username + * @param {String} type The type of user + * @param {String} role The role of user + * @returns {Promise} + */ +async function queryOneUserByTypeAndRole(model, username, type, role) { + return await new Promise((resolve, reject) => { + model.query('username').eq(username) + .where('type') + .eq(type) + .filter('role') + .eq(role) + .all() + .exec((err, result) => { + if (err || !result) { + logger.debug(`queryOneUserByTypeAndRole. Error. ${err}`); + return reject(err); + } + return resolve(result.count === 0 ? null : result[0]); + }); + }); +} + +/** + * Get single data by query parameters + * @param {Object} model The dynamoose model to query + * @param {String} tcusername The tc username + * @returns {Promise} + */ +async function queryOneUserMappingByTCUsername(model, tcusername) { + return await new Promise((resolve, reject) => { + model.queryOne('topcoderUsername').eq(tcusername) + .all() + .exec((err, result) => { + if (err || !result) { + logger.debug(`queryOneUserMappingByTCUsername. Error. ${err}`); + return reject(err); + } + return resolve(result); + }); + }); +} + +/** + * Get single data by query parameters + * @param {Object} model The dynamoose model to query + * @param {String} repoUrl The repository url + * @returns {Promise} + */ +async function queryOneActiveProject(model, repoUrl) { + return await new Promise((resolve, reject) => { + model.query('repoUrl').eq(repoUrl) + .where('archived') + .eq('false') + .all() + .exec((err, result) => { + if (err || !result) { + logger.debug(`queryOneActiveProject. Error. ${err}`); + return reject(err); + } + return resolve(result.count === 0 ? null : result[0]); + }); + }); +} + +/** + * Get single data by query parameters + * @param {Object} model The dynamoose model to query + * @param {String} project The project + * @param {String} username The username + * @returns {Promise} + */ +async function queryOneActiveCopilotPayment(model, project, username) { + return await new Promise((resolve, reject) => { + model.query('project').eq(project) + .where('username') + .eq(username) + .filter('closed') + .eq('false') + .all() + .exec((err, result) => { + if (err || !result) { + logger.debug(`queryOneActiveProject. Error. ${err}`); + return reject(err); + } + return resolve(result.count === 0 ? null : result[0]); + }); + }); +} + +/** + * Get single data by query parameters + * @param {Object} model The dynamoose model to query + * @param {String} groupId The gitlab group id + * @param {String} gitlabUserId The gitlab user id + * @returns {Promise} + */ +async function queryOneUserGroupMapping(model, groupId, gitlabUserId) { + return await new Promise((resolve, reject) => { + model.query('groupId').eq(groupId) + .where('gitlabUserId') + .eq(gitlabUserId.toString()) + .all() + .exec((err, result) => { + if (err || !result) { + logger.debug(`queryOneActiveProject. Error. ${err}`); + return reject(err); + } + return resolve(result.count === 0 ? null : result[0]); + }); + }); +} + +/** + * Get single data by query parameters + * @param {Object} model The dynamoose model to query + * @param {String} teamId The github team id + * @param {String} githubUserName The github username + * @param {String} githubOrgId The github org id + * @returns {Promise} + */ +async function queryOneUserTeamMapping(model, teamId, githubUserName, githubOrgId) { + return await new Promise((resolve, reject) => { + model.query('teamId').eq(teamId) + .where('githubUserName') + .eq(githubUserName) + .filter('githubOrgId') + .eq(githubOrgId) + .all() + .exec((err, result) => { + if (err || !result) { + logger.debug(`queryOneActiveProject. Error. ${err}`); + return reject(err); + } + return resolve(result.count === 0 ? null : result[0]); + }); + }); +} + +/** + * Get single data by query parameters + * @param {Object} model The dynamoose model to query + * @param {String} repoUrl The repository url + * @param {String} projectIdToFilter The projectId To Filter + * @returns {Promise} + */ +async function queryOneActiveProjectWithFilter(model, repoUrl, projectIdToFilter) { + return await new Promise((resolve, reject) => { + model.query('repoUrl').eq(repoUrl) + .where('archived') + .eq('false') + .filter('id') + .not().eq(projectIdToFilter) + .all() + .exec((err, result) => { + if (err || !result) { + logger.debug(`queryOneActiveProject. Error. ${err}`); + return reject(err); + } return resolve(result.count === 0 ? null : result[0]); }); }); @@ -140,10 +331,30 @@ async function update(Model, id, data) { /** * Delete item in database * @param {Object} Model The dynamoose model to delete - * @param {Object} queryParams The query parameters object + * @param {String} id The id */ -async function remove(Model, queryParams) { - const dbItem = await scanOne(Model, queryParams); +async function removeById(Model, id) { + const dbItem = await getById(Model, id); + await new Promise((resolve, reject) => { + dbItem.delete((err) => { + if (err) { + logger.error(`DynamoDB remove error ${err}`); + reject(err); + } + + resolve(dbItem); + }); + }); +} + +/** + * Delete item in database + * @param {Object} Model The dynamoose model to delete + * @param {String} username The username + * @param {String} type The type + */ +async function removeUser(Model, username, type) { + const dbItem = await queryOneUserByType(Model, username, type); await new Promise((resolve, reject) => { dbItem.delete((err) => { if (err) { @@ -156,12 +367,22 @@ async function remove(Model, queryParams) { }); } + module.exports = { getById, + getByKey, scan, - scanOne, create, update, - remove, - queryOneIssue + removeById, + removeUser, + queryOneActiveCopilotPayment, + queryOneActiveProject, + queryOneActiveProjectWithFilter, + queryOneIssue, + queryOneUserByType, + queryOneUserByTypeAndRole, + queryOneUserGroupMapping, + queryOneUserTeamMapping, + queryOneUserMappingByTCUsername }; diff --git a/src/common/helper.js b/src/common/helper.js index 6aa6c62..5dae1d2 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -16,6 +16,7 @@ const Joi = require('joi'); const getParams = require('get-parameter-names'); const bluebird = require('bluebird'); const bcrypt = require('bcryptjs'); +const moment = require('moment'); const parseDomain = require('parse-domain'); const config = require('../config'); const logger = require('./logger'); @@ -147,9 +148,9 @@ function convertGitHubError(err, message) { */ function convertGitLabError(err, message) { let resMsg = `${message}. ${err.message}.\n`; - const detail = _.get(err, 'response.body.message'); + const detail = _.get(err, 'response.body.message') || _.get(err, 'response.text'); if (detail) { - resMsg += ` Detail: ${detail}`; + resMsg += ` Detail: ${JSON.stringify(detail)}`; } const apiError = new errors.ApiError( err.status || _.get(err, 'response.status', constants.SERVICE_ERROR_STATUS), @@ -161,28 +162,32 @@ function convertGitLabError(err, message) { /** * Ensure entity exists for given criteria. Return error if no result. - * @param {Object} Model the mongoose model to query - * @param {Object|String|Number} criteria the criteria (if object) or id (if string/number) + * @param {Object} Model the dynamoose model to query + * @param {String|Number} criteria the id (if string/number) * @param {String} modelName the name of model * @returns {Object} the found entity */ async function ensureExists(Model, criteria, modelName) { - let query; - let byId = true; - if (_.isObject(criteria)) { - byId = false; - query = dbHelper.scanOne(Model, criteria); - } else { - query = dbHelper.getById(Model, criteria); + const result = await dbHelper.getById(Model, criteria); + if (!result) { + const msg = util.format('%s not found with id: %s', modelName, criteria); + throw new NotFoundError(msg); } - const result = await query; + return result; +} + +/** + * Ensure entity exists for given key and value. Return error if no result. + * @param {Object} Model the dynamoose model to query + * @param {String} key the key name to query + * @param {Any} value the value to query + * @param {String} modelName the name of model + * @returns {Object} the found entity + */ +async function ensureExistsWithKey(Model, key, value, modelName) { + const result = await dbHelper.getByKey(Model, key, value); if (!result) { - let msg; - if (byId) { - msg = util.format('%s not found with id: %s', modelName, criteria); - } else { - msg = util.format('%s not found with criteria: %j', modelName, criteria); - } + const msg = util.format('%s not found with key: %s value: %s', modelName, key, value); throw new NotFoundError(msg); } return result; @@ -211,9 +216,8 @@ async function getProviderType(repoUrl) { * @returns {Object} the owner/copilot for the project */ async function getProjectCopilotOrOwner(models, project, provider, isCopilot) { - const userMapping = await dbHelper.scanOne(models.UserMapping, { - topcoderUsername: isCopilot ? project.copilot : project.owner, - }); + const userMapping = await dbHelper.queryOneUserMappingByTCUsername(models.UserMapping, + isCopilot ? project.copilot : project.owner); if (!userMapping || (provider === 'github' && !userMapping.githubUserId) @@ -221,11 +225,9 @@ async function getProjectCopilotOrOwner(models, project, provider, isCopilot) { throw new Error(`Couldn't find ${isCopilot ? 'copilot' : 'owner'} username for '${provider}' for this repository.`); } - let user = await dbHelper.scanOne(models.User, { - username: provider === 'github' ? userMapping.githubUsername : // eslint-disable-line no-nested-ternary - userMapping.gitlabUsername, - type: provider, - }); + let user = await dbHelper.queryOneUserByType(models.User, + provider === 'github' ? userMapping.githubUsername : // eslint-disable-line no-nested-ternary + userMapping.gitlabUsername, provider); return user; } @@ -252,14 +254,26 @@ function hashCode(s) { }, 0); } +/** + * Check if expires_at is valid + * + * @param {String} expiresAt the date str yyyy-MM-dd + * @returns {Boolean} valid or not + */ +function isValidGitlabExpiresDate(expiresAt) { + return moment(expiresAt, 'YYYY-MM-DD', true).isValid(); +} + module.exports = { buildService, buildController, convertGitHubError, convertGitLabError, ensureExists, + ensureExistsWithKey, generateIdentifier, getProviderType, getProjectCopilotOrOwner, - hashCode + hashCode, + isValidGitlabExpiresDate }; diff --git a/src/config.js b/src/config.js index 4e3e480..3c7a0ff 100644 --- a/src/config.js +++ b/src/config.js @@ -12,7 +12,6 @@ module.exports = { PORT: process.env.PORT || 80, // eslint-disable-line no-magic-numbers API_VERSION: process.env.API_VERSION || 'v1', LOG_LEVEL: process.env.LOG_LEVEL || 'debug', - // MONGODB_URI: process.env.MONGODB_URI || 'mongodb://localhost:27017/topcoderx', SESSION_SECRET: process.env.SESSION_SECRET || 'kjsdfkj34857', // Github and gitlab client id and secret GITHUB_CLIENT_ID: process.env.GITHUB_CLIENT_ID, @@ -65,6 +64,7 @@ module.exports = { TC_LOGIN_URL: process.env.TC_LOGIN_URL || 'https://accounts-auth0.topcoder.com/', }, }, + DYNAMODB_WAIT_TABLE_FOR_ACTIVE_TIMEOUT: process.env.DYNAMODB_WAIT_TABLE_FOR_ACTIVE_TIMEOUT || 1000 * 60 * 10 // eslint-disable-line no-magic-numbers }; module.exports.frontendConfigs = { diff --git a/src/controllers/GithubController.js b/src/controllers/GithubController.js index dc45ce4..d72fc9a 100644 --- a/src/controllers/GithubController.js +++ b/src/controllers/GithubController.js @@ -116,7 +116,7 @@ async function addUserToTeam(req, res) { const identifier = req.params.identifier; console.log(`addUserToTeam called for ${identifier}`); /* eslint-disable-line no-console */ // validate the identifier - await helper.ensureExists(OwnerUserTeam, {identifier}, 'OwnerUserTeam'); + await helper.ensureExistsWithKey(OwnerUserTeam, 'identifier', identifier, 'OwnerUserTeam'); // store identifier to session, to be compared in callback req.session.identifier = identifier; @@ -144,7 +144,7 @@ async function addUserToTeamCallback(req, res) { if (!code) { throw new errors.ValidationError('Missing code.'); } - const team = await helper.ensureExists(OwnerUserTeam, {identifier}, 'OwnerUserTeam'); + const team = await helper.ensureExistsWithKey(OwnerUserTeam, 'identifier', identifier, 'OwnerUserTeam'); // exchange code to get token const result = await request .post('https://github.com/login/oauth/access_token') @@ -160,9 +160,7 @@ async function addUserToTeamCallback(req, res) { console.log(`adding ${token} to ${team.teamId} with ${team.ownerToken}`); /* eslint-disable-line no-console */ const githubUser = await GithubService.addTeamMember(team.teamId, team.ownerToken, token, team.accessLevel); // associate github username with TC username - const mapping = await dbHelper.scanOne(UserMapping, { - topcoderUsername: {eq: req.session.tcUsername}, - }); + const mapping = await dbHelper.queryOneUserMappingByTCUsername(UserMapping, req.session.tcUsername); // get team details const teamDetails = await GithubService.getTeamDetails(team.ownerToken, team.teamId); @@ -182,11 +180,10 @@ async function addUserToTeamCallback(req, res) { } // associate github username and teamId - const githubUserToTeamMapping = await dbHelper.scanOne(UserTeamMapping, { - teamId: {eq: team.teamId}, - githubUserName: {eq: githubUser.username}, - githubOrgId: {eq: team.githubOrgId}, - }); + const githubUserToTeamMapping = await dbHelper.queryOneUserTeamMapping(UserTeamMapping, + team.teamId, + githubUser.username, + team.githubOrgId); if (!githubUserToTeamMapping) { await dbHelper.create(UserTeamMapping, { @@ -221,7 +218,7 @@ async function deleteUsersFromTeam(req, res) { let teamInDB; const teamId = req.params.id; try { - teamInDB = await helper.ensureExists(OwnerUserTeam, {teamId}, 'OwnerUserTeam'); + teamInDB = await helper.ensureExistsWithKey(OwnerUserTeam, 'teamId', teamId, 'OwnerUserTeam'); } catch (err) { if (!(err instanceof errors.NotFoundError)) { throw err; @@ -238,7 +235,7 @@ async function deleteUsersFromTeam(req, res) { // eslint-disable-next-line no-restricted-syntax for (const userTeamMapItem of userTeamMappings) { await GithubService.deleteUserFromGithubTeam(token, teamId, githubOrgId, userTeamMapItem.githubUserName); - await dbHelper.remove(UserTeamMapping, {id: userTeamMapItem.id}); + await dbHelper.removeById(UserTeamMapping, userTeamMapItem.id); } } catch (err) { throw err; diff --git a/src/controllers/GitlabController.js b/src/controllers/GitlabController.js index e93ba1f..7ff6477 100644 --- a/src/controllers/GitlabController.js +++ b/src/controllers/GitlabController.js @@ -130,7 +130,7 @@ async function getGroupRegistrationUrl(req) { async function addUserToGroup(req, res) { const identifier = req.params.identifier; // validate the identifier - await helper.ensureExists(OwnerUserGroup, {identifier}, 'OwnerUserGroup'); + await helper.ensureExistsWithKey(OwnerUserGroup, 'identifier', identifier, 'OwnerUserGroup'); // store identifier to session, to be compared in callback req.session.identifier = identifier; @@ -161,15 +161,15 @@ async function addUserToGroupCallback(req, res) { if (!code) { throw new errors.ValidationError('Missing code.'); } - const group = await helper.ensureExists(OwnerUserGroup, {identifier}, 'OwnerUserGroup'); + const group = await helper.ensureExistsWithKey(OwnerUserGroup, 'identifier', identifier, 'OwnerUserGroup'); if (!group) { throw new errors.NotFoundError('The group is not found or not accessible.'); } // get owner user - const ownerUser = await helper.ensureExists(User, - {username: group.ownerUsername, type: constants.USER_TYPES.GITLAB, role: constants.USER_ROLES.OWNER}, 'User'); + const ownerUser = await dbHelper.queryOneUserByTypeAndRole(User, + group.ownerUsername, constants.USER_TYPES.GITLAB, constants.USER_ROLES.OWNER); if (!ownerUser) { throw new errors.NotFoundError('The owner user is not found or not accessible.'); @@ -208,9 +208,8 @@ async function addUserToGroupCallback(req, res) { group.accessLevel, group.expiredAt); // associate gitlab username with TC username - const mapping = await dbHelper.scanOne(UserMapping, { - topcoderUsername: {eq: req.session.tcUsername}, - }); + + const mapping = await dbHelper.queryOneUserMappingByTCUsername(UserMapping, req.session.tcUsername); if (mapping) { await dbHelper.update(UserMapping, mapping.id, { gitlabUsername: gitlabUser.username, @@ -226,10 +225,9 @@ async function addUserToGroupCallback(req, res) { } // We get gitlabUser.id and group.groupId and // associate github username and teamId - const gitlabUserToGroupMapping = await dbHelper.scanOne(UserGroupMapping, { - groupId: {eq: group.groupId}, - gitlabUserId: {eq: gitlabUser.id}, - }); + const gitlabUserToGroupMapping = await dbHelper.queryOneUserGroupMapping(UserGroupMapping, + group.groupId, + gitlabUser.id); if (!gitlabUserToGroupMapping) { await dbHelper.create(UserGroupMapping, { @@ -253,7 +251,7 @@ async function deleteUsersFromTeam(req, res) { const groupId = req.params.id; let groupInDB; try { - groupInDB = await helper.ensureExists(OwnerUserGroup, {groupId}, 'OwnerUserGroup'); + groupInDB = await helper.ensureExistsWithKey(OwnerUserGroup, 'groupId', groupId, 'OwnerUserGroup'); } catch (err) { if (!(err instanceof errors.NotFoundError)) { throw err; @@ -262,14 +260,17 @@ async function deleteUsersFromTeam(req, res) { // If groupInDB not exists, then just return if (groupInDB) { try { - const ownerUser = await helper.ensureExists(User, - {username: groupInDB.ownerUsername, type: constants.USER_TYPES.GITLAB, role: constants.USER_ROLES.OWNER}, 'User'); + const ownerUser = await helper.queryOneUserByTypeAndRole(User, + groupInDB.ownerUsername, constants.USER_TYPES.GITLAB, constants.USER_ROLES.OWNER); + if (!ownerUser) { + throw new errors.NotFoundError('The owner user is not found or not accessible.'); + } await GitlabService.refreshGitlabUserAccessToken(ownerUser); const userGroupMappings = await dbHelper.scan(UserGroupMapping, {groupId}); // eslint-disable-next-line no-restricted-syntax for (const userGroupMapItem of userGroupMappings) { await GitlabService.deleteUserFromGitlabGroup(ownerUser.accessToken, groupId, userGroupMapItem.gitlabUserId); - await dbHelper.remove(UserGroupMapping, {id: userGroupMapItem.id}); + await dbHelper.removeById(UserGroupMapping, userGroupMapItem.id); } } catch (err) { throw err; diff --git a/src/front/src/app/git-access-control/access-control.html b/src/front/src/app/git-access-control/access-control.html index ce37622..7bf52ff 100644 --- a/src/front/src/app/git-access-control/access-control.html +++ b/src/front/src/app/git-access-control/access-control.html @@ -17,7 +17,7 @@

Git Access Control

-
+
@@ -27,6 +27,7 @@

Git Access Control

+
Don't see your team? Make sure you have 'Maintainer' rights.
@@ -39,8 +40,14 @@

Git Access Control

- - + +
{{item.name}}{{item.organization.login}} + {{item.name}} + + {{item.organization.login}} +