From 5618842f1933ede1d0ecae797dd28ede68303896 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Mon, 8 Feb 2021 12:36:40 +1100 Subject: [PATCH] Updates for Feb 2021 bug fix challenge https://software.topcoder.com/review/actions/ViewProjectDetails?method=viewProjectDetails&pid=30165062 --- README.md | 8 ++ TeamsVsGroups.md | 40 +++++++ package.json | 3 +- scripts/log-repository-collisions.js | 43 +++++++ src/config.js | 108 +++--------------- src/controllers/GithubController.js | 4 + src/controllers/GitlabController.js | 7 +- src/front/src/app/less/custom.less | 5 +- .../src/app/members/member.controller.js | 5 +- src/front/src/app/projects/projects.html | 9 ++ src/services/GithubService.js | 2 +- src/services/ProjectService.js | 14 +++ 12 files changed, 147 insertions(+), 101 deletions(-) create mode 100644 TeamsVsGroups.md create mode 100644 scripts/log-repository-collisions.js diff --git a/README.md b/README.md index 7e1bb89..7158345 100644 --- a/README.md +++ b/README.md @@ -93,4 +93,12 @@ Direct ID migration can be done with script located in **topcoder-x-processor** ```shell npm run direct-connect-migration ``` +By default it takes 15 projects at time, but you can change this by specifying BATCH_SIZE environment variable. + +## Repository url collisions scan + +To scan and log which projects has colliding (the same) repository urls run +```shell +npm run log-repository-collisions +``` By default it takes 15 projects at time, but you can change this by specifying BATCH_SIZE environment variable. \ No newline at end of file diff --git a/TeamsVsGroups.md b/TeamsVsGroups.md new file mode 100644 index 0000000..0fe3f73 --- /dev/null +++ b/TeamsVsGroups.md @@ -0,0 +1,40 @@ + +## Github's Teams and Gitlab's Groups differences + +Github's Teams are groups of organization members whereas Gitlab's Groups are just groups of Gitlab users (Groups are more like Github' organizations). +Each Github organization can have repositories and assign teams to them and each team can have subteams. +Gitlab groups can have repositories and create subgroups (nested). + +## Setup guide + +### Github + +1. Go to 'https://github.com/settings/organizations' and click 'New organization' button +2. Select your plan. +3. Enter organization's name, contact email address, solve the captcha and click Next +4. Click 'Skip this step' +5. If you receive a survey, you can just go to the bottom and click Submit without filling anything. +6. On your new organization page go to 'Teams' tab and click 'New team' button. +7. Fill in your team's name, description (optional) and visibility. Submit by clicking 'Create team'. +Now you have your team created and you should get redirect to its page. +You can assign it to an organization's repository by clicking 'Add Repository' and entering repository's name, in 'Repositories' tab of a team's page. + +### Gitlab + +1. Go to 'https://gitlab.com/dashboard/groups' and click on 'New group' button. +2. Enter group's name and set visibility level. Finish by clicking 'Create group'. +You can now create repositories for this group or subgroups. + +## Roles + +In Topcoder X you can select role which user who joins via specific invitation link receives. + +### Github + +For github team you can set two roles: Member and Maintainer. +You can read about them here: https://docs.github.com/en/github/setting-up-and-managing-organizations-and-teams/permission-levels-for-an-organization and https://docs.github.com/en/github/setting-up-and-managing-organizations-and-teams/giving-team-maintainer-permissions-to-an-organization-member + +### Gitlab + +For gitlab group you can set five roles: Guest, Reporter, Developer, Maintainer, Owner +You can read about them here: https://docs.gitlab.com/ee/user/permissions.html \ No newline at end of file diff --git a/package.json b/package.json index a761392..d2e5431 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,8 @@ "test:fe": "gulp protractor", "lint": "gulp lint", "heroku-postbuild": "gulp build", - "create-tables": "CREATE_DB=true node scripts/create-update-tables.js" + "create-tables": "CREATE_DB=true node scripts/create-update-tables.js", + "log-repository-collisions": "node scripts/log-repository-collisions.js" }, "dependencies": { "angular": "~1.8.0", diff --git a/scripts/log-repository-collisions.js b/scripts/log-repository-collisions.js new file mode 100644 index 0000000..c4c275c --- /dev/null +++ b/scripts/log-repository-collisions.js @@ -0,0 +1,43 @@ +const logger = require('../src/common/logger') +const models = require('../src/models') +const _ = require('lodash') + +async function main() { + const BATCH_SIZE = process.env.BATCH_SIZE || 15; + let previousSize = BATCH_SIZE; + let previousKey = null; + // Array containing project ids already found colliding + const collidingUrls = [] + let batch = 1; + // Run this loop as long as there can be more objects in a database + while (previousSize === BATCH_SIZE) { + logger.debug(`Running batch no. ${batch}`) + // Go over all active projects, limiting to BATCH_SIZE + const projects = await models.Project.scan({ + archived: 'false' + }).consistent().limit(BATCH_SIZE).startAt(previousKey).exec() + for (const project of projects) { + // If url was already found colliding go to a next iteration + if (collidingUrls.includes(project.repoUrl)) continue; + const collisions = await models.Project.scan({ + repoUrl: project.repoUrl, + archived: 'false' + }).exec() + // If scan found only this project go to a next interation + if (collisions.length < 2) continue; + logger.info(`Repository ${project.repoUrl} has ${collisions.length} collisions`); + _.forEach(collisions, collision => { + logger.info(`--- ID: ${collision.id}`) + }) + collidingUrls.push(project.repoUrl) + } + previousKey = projects.lastKey + previousSize = projects.scannedCount + batch++ + } +} +main().then(() => { + logger.info('Collision scan completed') +}).catch(err => { + logger.logFullError(err, 'collision scan') +}) diff --git a/src/config.js b/src/config.js index 343ca06..7733587 100644 --- a/src/config.js +++ b/src/config.js @@ -22,7 +22,6 @@ module.exports = { // used as base to construct various URLs WEBSITE: process.env.WEBSITE || 'http://topcoderx.topcoder-dev.com', - WEBSITE_SECURE: process.env.WEBSITE_SECURE || 'https://topcoderx.topcoder-dev.com', GITLAB_API_BASE_URL: process.env.GITLAB_API_BASE_URL || 'https://gitlab.com', // kafka configuration @@ -50,7 +49,6 @@ module.exports = { OPEN_FOR_PICKUP_ISSUE_LABEL: process.env.OPEN_FOR_PICKUP_ISSUE_LABEL || 'tcx_OpenForPickup', ALLOWED_TOPCODER_ROLES: process.env.ALLOWED_TOPCODER_ROLES || ['administrator', 'admin', 'connect manager', 'connect admin', 'copilot', 'connect copilot'], COPILOT_ROLE: process.env.COPILOT_ROLE || 'copilot', - HELP_LINK: process.env.HELP_LINK || 'https://github.com/topcoder-platform/topcoder-x-ui/wiki', ADMINISTRATOR_ROLES: process.env.ADMINISTRATOR_ROLES || ['administrator', 'admin'], DYNAMODB: { AWS_ACCESS_KEY_ID: process.env.AWS_ACCESS_KEY_ID, @@ -69,100 +67,20 @@ module.exports = { }, }; -const frontendConfigs = { - local: { - JWT_V3_NAME: 'v3jwt', - JWT_V2_NAME: 'tcjwt', - COOKIES_SECURE: false, - TC_LOGIN_URL: 'https://accounts-auth0.topcoder-dev.com/', - API_URL: 'https://127.0.0.1:8443', - ADMIN_TOOL_URL: 'http://localhost:8080/api/v2', - ACCOUNTS_CONNECTOR_URL: 'https://accounts.topcoder-dev.com/connector.html', - CONNECT_URL_BASE: 'https://connect.topcoder-dev/projects/', - OWNER_LOGIN_GITHUB_URL: '/api/v1/github/owneruser/login', - OWNER_LOGIN_GITLAB_URL: '/api/v1/gitlab/owneruser/login', - TOPCODER_URL: 'https://topcoder-dev.com', - GITHUB_TEAM_URL: 'https://github.com/orgs/', - GITLAB_GROUP_URL: 'https://gitlab.com/groups/', - }, - heroku: { - JWT_V3_NAME: 'v3jwt', - JWT_V2_NAME: 'tcjwt', - COOKIES_SECURE: false, - TC_LOGIN_URL: 'https://accounts-auth0.topcoder-dev.com/', - API_URL: 'https://api.topcoder-dev.com', - ADMIN_TOOL_URL: 'https://api.topcoder-dev.com/v2', - ACCOUNTS_CONNECTOR_URL: 'https://accounts.topcoder-dev.com/connector.html', - CONNECT_URL_BASE: 'https://connect.topcoder-dev.com/projects/', - OWNER_LOGIN_GITHUB_URL: '/api/v1/github/owneruser/login', - OWNER_LOGIN_GITLAB_URL: '/api/v1/gitlab/owneruser/login', - TOPCODER_URL: 'https://topcoder-dev.com', - GITHUB_TEAM_URL: 'https://github.com/orgs/', - GITLAB_GROUP_URL: 'https://gitlab.com/groups/', - }, - dev: { - JWT_V3_NAME: 'v3jwt', - JWT_V2_NAME: 'tcjwt', - COOKIES_SECURE: false, - TC_LOGIN_URL: 'https://accounts-auth0.topcoder-dev.com/', - API_URL: 'https://api.topcoder-dev.com', - ADMIN_TOOL_URL: 'https://api.topcoder-dev.com/v2', - ACCOUNTS_CONNECTOR_URL: 'https://accounts.topcoder-dev.com/connector.html', - CONNECT_URL_BASE: 'https://connect.topcoder-dev.com/projects/', - OWNER_LOGIN_GITHUB_URL: '/api/v1/github/owneruser/login', - OWNER_LOGIN_GITLAB_URL: '/api/v1/gitlab/owneruser/login', - TOPCODER_URL: 'https://topcoder-dev.com', - GITHUB_TEAM_URL: 'https://github.com/orgs/', - GITLAB_GROUP_URL: 'https://gitlab.com/groups/', - }, - qa: { - JWT_V3_NAME: 'v3jwt', - JWT_V2_NAME: 'tcjwt', - COOKIES_SECURE: false, - TC_LOGIN_URL: 'https://accounts-auth0.topcoder-dev.com/', - API_URL: 'https://api.topcoder-qa.com', - ADMIN_TOOL_URL: 'https://api.topcoder-qa.com/v2', - ACCOUNTS_CONNECTOR_URL: 'https://accounts.topcoder-qa.com/connector.html', - CONNECT_URL_BASE: 'https://connect.topcoder-dev.com/projects/', - OWNER_LOGIN_GITHUB_URL: '/api/v1/github/owneruser/login', - OWNER_LOGIN_GITLAB_URL: '/api/v1/gitlab/owneruser/login', - TOPCODER_URL: 'https://topcoder-dev.com', - GITHUB_TEAM_URL: 'https://github.com/orgs/', - GITLAB_GROUP_URL: 'https://gitlab.com/groups/', - }, - prod: { - JWT_V3_NAME: 'v3jwt', - JWT_V2_NAME: 'tcjwt', - COOKIES_SECURE: false, - TC_LOGIN_URL: 'https://accounts.topcoder.com/member', - API_URL: 'https://api.topcoder.com', - ADMIN_TOOL_URL: 'https://api.topcoder.com/v2', - ACCOUNTS_CONNECTOR_URL: 'https://accounts.topcoder.com/connector.html', - CONNECT_URL_BASE: 'https://connect.topcoder.com/projects/', - OWNER_LOGIN_GITHUB_URL: '/api/v1/github/owneruser/login', - OWNER_LOGIN_GITLAB_URL: '/api/v1/gitlab/owneruser/login', - TOPCODER_URL: 'https://topcoder-dev.com', - GITHUB_TEAM_URL: 'https://github.com/orgs/', - GITLAB_GROUP_URL: 'https://gitlab.com/groups/', - }, -}; - -const activeEnv = module.exports.TOPCODER_ENV; module.exports.frontendConfigs = { - helpLink: module.exports.HELP_LINK, copilotRole: module.exports.COPILOT_ROLE, administratorRoles: module.exports.ADMINISTRATOR_ROLES, - JWT_V3_NAME: process.env.JWT_V3_NAME || frontendConfigs[activeEnv].JWT_V3_NAME, - JWT_V2_NAME: process.env.JWT_V2_NAME || frontendConfigs[activeEnv].JWT_V2_NAME, - COOKIES_SECURE: process.env.COOKIES_SECURE || frontendConfigs[activeEnv].COOKIES_SECURE, - TC_LOGIN_URL: process.env.TC_LOGIN_URL || frontendConfigs[activeEnv].TC_LOGIN_URL, - API_URL: process.env.API_URL || frontendConfigs[activeEnv].API_URL, - ADMIN_TOOL_URL: process.env.ADMIN_TOOL_URL || frontendConfigs[activeEnv].ADMIN_TOOL_URL, - ACCOUNTS_CONNECTOR_URL: process.env.ACCOUNTS_CONNECTOR_URL || frontendConfigs[activeEnv].ACCOUNTS_CONNECTOR_URL, - CONNECT_URL_BASE: process.env.CONNECT_URL_BASE || frontendConfigs[activeEnv].CONNECT_URL_BASE, - OWNER_LOGIN_GITHUB_URL: process.env.OWNER_LOGIN_GITHUB_URL || frontendConfigs[activeEnv].OWNER_LOGIN_GITHUB_URL, - OWNER_LOGIN_GITLAB_URL: process.env.OWNER_LOGIN_GITLAB_URL || frontendConfigs[activeEnv].OWNER_LOGIN_GITLAB_URL, - TOPCODER_URL: process.env.TOPCODER_URL || frontendConfigs[activeEnv].TOPCODER_URL, - GITHUB_TEAM_URL: process.env.GITHUB_TEAM_URL || frontendConfigs[activeEnv].GITHUB_TEAM_URL, - GITLAB_GROUP_URL: process.env.GITLAB_GROUP_URL || frontendConfigs[activeEnv].GITLAB_GROUP_URL, + helpLink: process.env.HELP_LINK || 'https://github.com/topcoder-platform/topcoder-x-ui/wiki', + JWT_V3_NAME: process.env.JWT_V3_NAME || 'v3jwt', + JWT_V2_NAME: process.env.JWT_V2_NAME || 'tcjwt', + COOKIES_SECURE: process.env.COOKIES_SECURE || false, + TC_LOGIN_URL: process.env.TC_LOGIN_URL || 'https://accounts-auth0.topcoder-dev.com/', + ADMIN_TOOL_URL: process.env.ADMIN_TOOL_URL || 'https://api.topcoder-dev.com/v2', + ACCOUNTS_CONNECTOR_URL: process.env.ACCOUNTS_CONNECTOR_URL || 'https://accounts.topcoder-dev.com/connector.html', + CONNECT_URL_BASE: process.env.CONNECT_URL_BASE || 'https://connect.topcoder-dev.com/projects/', + OWNER_LOGIN_GITHUB_URL: process.env.OWNER_LOGIN_GITHUB_URL || '/api/v1/github/owneruser/login', + OWNER_LOGIN_GITLAB_URL: process.env.OWNER_LOGIN_GITLAB_URL || '/api/v1/gitlab/owneruser/login', + TOPCODER_URL: process.env.TOPCODER_URL || 'https://topcoder-dev.com', + GITHUB_TEAM_URL: process.env.GITHUB_TEAM_URL || 'https://github.com/orgs/', + GITLAB_GROUP_URL: process.env.GITLAB_GROUP_URL || 'https://gitlab.com/groups/', }; diff --git a/src/controllers/GithubController.js b/src/controllers/GithubController.js index 8421eea..dc45ce4 100644 --- a/src/controllers/GithubController.js +++ b/src/controllers/GithubController.js @@ -151,6 +151,10 @@ async function addUserToTeamCallback(req, res) { .query({client_id: config.GITHUB_CLIENT_ID, client_secret: config.GITHUB_CLIENT_SECRET, code}) .set('Accept', 'application/json') .end(); + // Throw error if github access token was not returned (e.g. invalid code) + if (!result.body.access_token) { + throw new errors.UnauthorizedError('Github authorization failed.', result.body.error_description); + } const token = result.body.access_token; // add user to team console.log(`adding ${token} to ${team.teamId} with ${team.ownerToken}`); /* eslint-disable-line no-console */ diff --git a/src/controllers/GitlabController.js b/src/controllers/GitlabController.js index a1ed04a..e93ba1f 100644 --- a/src/controllers/GitlabController.js +++ b/src/controllers/GitlabController.js @@ -188,6 +188,10 @@ async function addUserToGroupCallback(req, res) { redirect_uri: `${config.WEBSITE}/api/${config.API_VERSION}/gitlab/normaluser/callback`, }) .end(); + // Throw error if github access token was not returned (ex. invalid code) + if (!result.body.access_token) { + throw new errors.UnauthorizedError('Gitlab authorization failed.', result.body.error_description); + } const token = result.body.access_token; // get group name @@ -235,7 +239,8 @@ async function addUserToGroupCallback(req, res) { }); } // redirect to success page - res.redirect(`${constants.USER_ADDED_TO_TEAM_SUCCESS_URL}/gitlab/${currentGroup.full_path}`); + // For gitlab subgroups we need to replace / with something different. Default encoding doesn't work as angular route fails to match %2F + res.redirect(`${constants.USER_ADDED_TO_TEAM_SUCCESS_URL}/gitlab/${currentGroup.full_path.replace('/', '@!2F')}`); } diff --git a/src/front/src/app/less/custom.less b/src/front/src/app/less/custom.less index d0766b1..f3e5882 100644 --- a/src/front/src/app/less/custom.less +++ b/src/front/src/app/less/custom.less @@ -20,7 +20,10 @@ color: green; font-size: 16px; } - +.red-times-icon { + color: red; + font-size: 16px; +} .orange-warning-icon { color: orange; font-size: 16px; diff --git a/src/front/src/app/members/member.controller.js b/src/front/src/app/members/member.controller.js index 350ed87..bbc00a0 100644 --- a/src/front/src/app/members/member.controller.js +++ b/src/front/src/app/members/member.controller.js @@ -11,8 +11,9 @@ angular.module('topcoderX') const org = params[0]; const team = url.replace(org, '').substring(1); $scope.link = $rootScope.appConfig.GITHUB_TEAM_URL + org + '/teams/' + team; - } else if (provider === 'github') { - $scope.link = $rootScope.appConfig.GITLAB_GROUP_URL + url; + } else if (provider === 'gitlab') { + // For gitlab subgroups we can't just pass encoded link to this route because anguler doesn't match %2F, so we use @!2F as a workaround + $scope.link = $rootScope.appConfig.GITLAB_GROUP_URL + url.replace('@!2F', '/'); } }; _getUrl($scope.provider, $stateParams.url); diff --git a/src/front/src/app/projects/projects.html b/src/front/src/app/projects/projects.html index 231d661..b960ae2 100644 --- a/src/front/src/app/projects/projects.html +++ b/src/front/src/app/projects/projects.html @@ -61,6 +61,7 @@

You don't have active projects right now. Please Service Provider Owner Copilot + Copilot payments @@ -78,6 +79,14 @@

You don't have active projects right now. Please {{project.owner}} {{project.copilot}} + +
+ +
+
+ +
+