diff --git a/src/constants.ts b/src/constants.ts index 97dce0c1..61c1d920 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -118,5 +118,6 @@ export const REGEXP_AUTHOR = /(?<=author:\s).+(?=\r?\n)/g; export const MIN_CHUNK_SIZE = Number(process.env.MIN_CHUNK_SIZE) || 1000; export const WORKERS_COUNT = Number(process.env.WORKERS_COUNT) || os.cpus().length - 1; +export const PAGE_PROCESS_CONCURRENCY = Number(process.env.PAGE_PROCESS_CONCURRENCY) || 500; export const metadataBorder = '---'; diff --git a/src/steps/processPages.ts b/src/steps/processPages.ts index 59f0d4a8..312fa3b5 100644 --- a/src/steps/processPages.ts +++ b/src/steps/processPages.ts @@ -26,7 +26,13 @@ import { } from '../models'; import {VCSConnector} from '../vcs-connector/connector-models'; import {getVCSConnector} from '../vcs-connector'; -import {Lang, ResourceType, SINGLE_PAGE_DATA_FILENAME, SINGLE_PAGE_FILENAME} from '../constants'; +import { + Lang, + PAGE_PROCESS_CONCURRENCY, + ResourceType, + SINGLE_PAGE_DATA_FILENAME, + SINGLE_PAGE_FILENAME, +} from '../constants'; const singlePageResults: Record = {}; const singlePagePaths: Record> = {}; @@ -46,11 +52,10 @@ export async function processPages(outputBundlePath: string): Promise { PluginService.setPlugins(); const navigationPaths = TocService.getNavigationPaths(); - const concurrency = 500; await mapLimit( navigationPaths, - concurrency, + PAGE_PROCESS_CONCURRENCY, asyncify(async (pathToFile: string) => { const pathData = getPathData( pathToFile, diff --git a/src/vcs-connector/github.ts b/src/vcs-connector/github.ts index f664eb65..96eb5ae6 100644 --- a/src/vcs-connector/github.ts +++ b/src/vcs-connector/github.ts @@ -32,6 +32,7 @@ const authorByGitEmail: Map = new Map(); const authorByPath: Map = new Map(); const contributorsByPath: Map = new Map(); const contributorsData: Map = new Map(); +const loginUserMap: Map = new Map(); async function getGitHubVCSConnector(): Promise { const {contributors} = ArgvService.getConfig(); @@ -277,21 +278,27 @@ async function getFileContributorsByPath(path: string): Promise { - const user = await github.getRepoUser(octokit, userLogin); + let result = loginUserMap.get(userLogin); + if (!result) { + const user = await github.getRepoUser(octokit, userLogin); + if (!user) { + return null; + } - if (!user) { - return null; - } + const {avatar_url: avatar, html_url: url, email, login, name} = user; - const {avatar_url: avatar, html_url: url, email, login, name} = user; + result = { + avatar, + email, + login, + name, + url, + }; - return { - avatar, - email, - login, - name, - url, - }; + loginUserMap.set(userLogin, result); + } + + return result; } function addNestedContributorsForPathFunction(