Fetch list of repositories by topic (#53)

* Added queryRepositoriesByTopic

* Started queryAndDumpRepositories

* Added test for queryAndDumpRepositories

* Removed previous functions for querying repos

* Added param for max num of repos to return

* Sort queried repos by name

* Updated documentation

* Clarify the purpose of topic oss-portal-featured

CHANGELOG.md

@@ -7,6 +7,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 You should update the version number in the [package.json](./package.json) according to the semantic versioning,
 then run `npm install`.
 
+## 1.5.0
+- Fetch list of repositories by topic with GitHub GraphQL API instead of hardcoded list.
+
 ## 1.4.0
 - Added pagination to Repositories Page.
 

RELEASING.md

@@ -1,14 +1,21 @@
-# How to release
+# How repositories are fetched
 
-Any commit to `main` branch will trigger a [GitHub Actions workflow](./.github/workflows) that builds and releases
-the site to GitHub Pages. The workflow is also automatically scheduled on a daily basis.
+The repositories that are shown in the portal are fetched with GitHub GraphQL API by this [build script](./src/scripts/build-repo-data.js) that is run at build time, producing a [static JSON file](./static/repos.json).
+
+In particular, repositories are fetched within the `ExpediaGroup` organization by [topic](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/classifying-your-repository-with-topics):
+
+- if the repo has topic `oss-portal-listed` it will be shown in the Repositories Page
+- if the repo has topic `oss-portal-featured` it will be shown in the Repositories Page and in the Home Page
 
-The list of repositories that are shown in the Home Page (the "featured" repositories) and in the Repositories Page is hardcoded in this [build script](./src/scripts/build-repo-data.js).
-The information for each listed repository is fetched using GitHub GraphQL API at build time
-producing a [static JSON file](./static/repos.json).
+# How blog posts are fetched
 
 The Medium blog posts [are fetched from the RSS feed](./src/scripts/build-posts-data.js) at build time
 producing a [static JSON file](./static/posts.json).
 
+# How to release
+
+Any commit to `main` branch will trigger a [GitHub Actions workflow](./.github/workflows) that builds and releases
+the site to GitHub Pages. The workflow is also automatically scheduled on a daily basis.
+
 The built static files are pushed to branch `gh-pages`, which automatically triggers the GitHub Pages deployment, usually
 in few minutes. You can check the history of GitHub Pages deployments [here](https://github.com/ExpediaGroup/expediagroup.github.io/deployments).

package.json

@@ -1,12 +1,12 @@
 {
   "name": "expediagroup.github.io",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",
     "start": "docusaurus start",
     "build-posts-data": "run-func src/scripts/build-posts-data.js fetchAndDumpPosts",
-    "build-repo-data": "run-func src/scripts/build-repo-data.js fetchAndDumpRepositories",
+    "build-repo-data": "run-func src/scripts/build-repo-data.js queryAndDumpRepositories",
     "build": "docusaurus build",
     "swizzle": "docusaurus swizzle",
     "deploy": "docusaurus deploy",

src/scripts/build-repo-data.js

@@ -14,55 +14,35 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-const {queryRepository} = require('./github/github-queries');
+const {queryRepositoriesByTopic} = require('./github/github-queries');
 const {writeJsonFile} = require('./filesystem/fs-utils');
 
-const REPOSITORIES = [
-    { organization: "ExpediaGroup", name: "beekeeper",                   featured: true },
-    { organization: "ExpediaGroup", name: "bull",                        featured: true },
-    { organization: "ExpediaGroup", name: "flyte",                       featured: true },
-    { organization: "ExpediaGroup", name: "graphql-component",           featured: true },
-    { organization: "ExpediaGroup", name: "graphql-kotlin",              featured: true },
-    { organization: "ExpediaGroup", name: "jarviz",                      featured: true },
-    { organization: "ExpediaGroup", name: "jenkins-spock",               featured: true },
-    { organization: "ExpediaGroup", name: "mittens",                     featured: true },
-    { organization: "ExpediaGroup", name: "stream-registry",             featured: true },
-    { organization: "ExpediaGroup", name: "catalyst-render",             featured: false },
-    { organization: "ExpediaGroup", name: "catalyst-server",             featured: false },
-    { organization: "ExpediaGroup", name: "cypress-codegen",             featured: false },
-    { organization: "ExpediaGroup", name: "determination",               featured: false },
-    { organization: "ExpediaGroup", name: "flyte-client",                featured: false },
-    { organization: "ExpediaGroup", name: "flyte-jira",                  featured: false },
-    { organization: "ExpediaGroup", name: "flyte-slack",                 featured: false },
-    { organization: "ExpediaGroup", name: "github-helpers",              featured: false },
-    { organization: "ExpediaGroup", name: "github-webhook-proxy",        featured: false },
-    { organization: "ExpediaGroup", name: "insights-explorer",           featured: false },
-    { organization: "ExpediaGroup", name: "kubernetes-sidecar-injector", featured: false },
-    { organization: "ExpediaGroup", name: "overwhelm",                   featured: false },
-    { organization: "ExpediaGroup", name: "package-json-validator",      featured: false },
-    { organization: "ExpediaGroup", name: "parsec",                      featured: false },
-    { organization: "ExpediaGroup", name: "pitchfork",                   featured: false },
-    { organization: "ExpediaGroup", name: "spinnaker-pipeline-trigger",  featured: false },
-    { organization: "ExpediaGroup", name: "steerage",                    featured: false },
-    { organization: "ExpediaGroup", name: "styx",                        featured: false },
-    { organization: "ExpediaGroup", name: "waggle-dance",                featured: false }
-]
+const EXPEDIA_ORG = 'ExpediaGroup'
+const REPOS_FILE = 'static/repos.json'
+const TOPIC_FEATURED_REPO = 'oss-portal-featured'
+const TOPIC_LISTED_REPO = 'oss-portal-listed'
+const MAX_FEATURED_REPOS = 9
 
 /**
- * @typedef Repository
- * @property {string} organization The name of the GitHub organization.
- * @property {string} name The name of the GitHub repository.
- * @property {boolean} featured Whether the repository should be shown in the home page.
- */
-
-/**
- * Fetches information about the given GitHub repositories and write it as JSON to the file at the given path.
- * @param {Repository[]} repositories the repositories to be fetched
- * @param {string} filePath the json file that will be written
+ * Queries the repositories in the Expedia Group GitHub organization and writes them as JSON to the file at the given path.
+ * The repositories are searched by topic:
+ * - {@link TOPIC_LISTED_REPO} if the repo should be shown in the portal
+ * - {@link TOPIC_FEATURED_REPO} if the repo should be particularly highlighted in the portal
+ * @param {string} organization the GitHub organization to query into. Defaults to {@link EXPEDIA_ORG}
+ * @param {string} filePath the json file that will be written. Defaults to {@link REPOS_FILE}
+ * @param {number} maxFeaturedRepos the max number of featured repos to be returned. Defaults to {@link MAX_FEATURED_REPOS}
  * @returns {Promise<void | Error>} a promise resolving to <code>undefined</code> in case of success or rejecting with an error
  */
-exports.fetchAndDumpRepositories = async (repositories = REPOSITORIES, filePath = 'static/repos.json') => {
-    const repoData = await Promise.all(repositories.map(repo => queryRepository(repo.organization, repo.name)
-        .then(fetchedRepo => ({...fetchedRepo, featured: repo.featured}))))
-    await writeJsonFile(filePath, repoData)
+exports.queryAndDumpRepositories = async (organization = EXPEDIA_ORG,
+                                          filePath = REPOS_FILE,
+                                          maxFeaturedRepos = MAX_FEATURED_REPOS) => {
+    const featuredRepos = await queryRepositoriesByTopic(organization, TOPIC_FEATURED_REPO, maxFeaturedRepos)
+        .then(flagAsFeatured(true))
+    const listedRepos = await queryRepositoriesByTopic(organization, TOPIC_LISTED_REPO)
+        .then(flagAsFeatured(false))
+    await writeJsonFile(filePath, featuredRepos.concat(listedRepos))
+}
+
+function flagAsFeatured(featured) {
+    return (repositories) => repositories.map(repo => ({...repo, featured: featured}))
 }

src/scripts/build-repo-data.test.js

@@ -14,53 +14,57 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-const {queryRepository} = require('./github/github-queries')
+const {queryRepositoriesByTopic} = require('./github/github-queries')
 const {writeJsonFile} = require('./filesystem/fs-utils')
-const {fetchAndDumpRepositories} = require('./build-repo-data')
+const {queryAndDumpRepositories} = require('./build-repo-data')
 
 jest.mock('./github/github-queries')
 jest.mock('./filesystem/fs-utils')
 
+const ORG = 'test-org'
 const FILE = 'file1.json'
-const REPO1 = {organization: 'org1', name: 'repo1', featured: false}
-const REPO2 = {organization: 'org2', name: 'repo2', featured: true}
 const REPO1_FETCHED = {foo: 'bar'}
 const REPO2_FETCHED = {baz: 'qux'}
 const ERROR_MSG = 'error1'
-
-test('builds successfully one non-featured repository', async () => {
-    queryRepository.mockResolvedValueOnce(REPO1_FETCHED)
-    writeJsonFile.mockResolvedValueOnce(undefined)
-
-    await fetchAndDumpRepositories([REPO1], FILE)
-
-    expect(queryRepository).toHaveBeenCalledWith(REPO1.organization, REPO1.name)
-    expect(writeJsonFile).toHaveBeenCalledWith(FILE, [{...REPO1_FETCHED, featured: false}])
-})
-
-test('builds successfully two repositories with 2nd repo featured', async () => {
-    queryRepository.mockResolvedValueOnce(REPO1_FETCHED).mockResolvedValueOnce(REPO2_FETCHED)
-    writeJsonFile.mockResolvedValueOnce(undefined)
-
-    await fetchAndDumpRepositories([REPO1, REPO2], FILE)
-
-    expect(queryRepository).toHaveBeenCalledTimes(2)
-    expect(queryRepository).toHaveBeenNthCalledWith(1, REPO1.organization, REPO1.name)
-    expect(queryRepository).toHaveBeenNthCalledWith(2, REPO2.organization, REPO2.name)
-    expect(writeJsonFile).toHaveBeenCalledWith(
-        FILE, [{...REPO1_FETCHED, featured: false}, {...REPO2_FETCHED, featured: true}]
-    )
-})
-
-test('rejects with error if at least one query fails', async () => {
-    queryRepository.mockResolvedValueOnce(REPO1_FETCHED).mockRejectedValueOnce(new Error(ERROR_MSG))
-
-    await expect(fetchAndDumpRepositories([REPO1, REPO2], FILE)).rejects.toThrow(ERROR_MSG)
-})
-
-test('rejects with error if write to file fails', async () => {
-    queryRepository.mockResolvedValueOnce(REPO1_FETCHED)
-    writeJsonFile.mockRejectedValueOnce(new Error(ERROR_MSG))
-
-    await expect(fetchAndDumpRepositories([REPO1], FILE)).rejects.toThrow(ERROR_MSG)
+const TOPIC_FEATURED_REPO = 'oss-portal-featured'
+const TOPIC_LISTED_REPO = 'oss-portal-listed'
+const MAX_FEATURED_REPOS = 13
+
+
+describe('queryAndDumpRepositories', () => {
+
+    test('builds successfully one featured repository', async () => {
+        queryRepositoriesByTopic.mockResolvedValueOnce([REPO1_FETCHED]).mockResolvedValueOnce([])
+        writeJsonFile.mockResolvedValueOnce(undefined)
+
+        await queryAndDumpRepositories(ORG, FILE, MAX_FEATURED_REPOS)
+
+        expect(queryRepositoriesByTopic).toHaveBeenNthCalledWith(1, ORG, TOPIC_FEATURED_REPO, MAX_FEATURED_REPOS)
+        expect(queryRepositoriesByTopic).toHaveBeenNthCalledWith(2, ORG, TOPIC_LISTED_REPO)
+        expect(writeJsonFile).toHaveBeenCalledWith(FILE, [{...REPO1_FETCHED, featured: true}])
+    })
+
+    test('builds successfully one featured and one listed repositories', async () => {
+        queryRepositoriesByTopic.mockResolvedValueOnce([REPO1_FETCHED]).mockResolvedValueOnce([REPO2_FETCHED])
+        writeJsonFile.mockResolvedValueOnce(undefined)
+
+        await queryAndDumpRepositories(ORG, FILE)
+
+        expect(writeJsonFile).toHaveBeenCalledWith(
+            FILE, [{...REPO1_FETCHED, featured: true}, {...REPO2_FETCHED, featured: false}]
+        )
+    })
+
+    test('rejects with error if at least one query fails', async () => {
+        queryRepositoriesByTopic.mockResolvedValueOnce([REPO1_FETCHED]).mockRejectedValueOnce(new Error(ERROR_MSG))
+
+        await expect(queryAndDumpRepositories(ORG, FILE)).rejects.toThrow(ERROR_MSG)
+    })
+
+    test('rejects with error if write to file fails', async () => {
+        queryRepositoriesByTopic.mockResolvedValueOnce([]).mockResolvedValueOnce([])
+        writeJsonFile.mockRejectedValueOnce(new Error(ERROR_MSG))
+
+        await expect(queryAndDumpRepositories(ORG, FILE)).rejects.toThrow(ERROR_MSG)
+    })
 })

src/scripts/github/github-queries.js

@@ -28,13 +28,25 @@ const githubClient = new ApolloClient({
     cache: new InMemoryCache()
 })
 
-const QUERY_REPO_INFO = gql`
-  query ($owner: String!, $name: String!) {
-    repository(owner: $owner, name: $name) {
-      name
-      description
-      openGraphImageUrl
-      url
+/**
+ * The default maximum number of fetched repositories.
+ * It is the maximum allowed by GitHub API with a single query.
+ */
+const MAX_REPOSITORIES_DEFAULT = 100
+
+const buildQueryForReposByTopic = (orgName, topic, maxRepos) => gql`
+  query {
+    search (first: ${maxRepos},
+            type: REPOSITORY,
+            query: "org:${orgName} topic:${topic}") {
+      nodes {
+        ... on Repository {
+          name
+          description
+          openGraphImageUrl
+          url
+        }
+      }
     }
   }`
 
@@ -54,22 +66,19 @@ const QUERY_REPO_INFO = gql`
  */
 
 /**
- * Fetches some information of a given GitHub repository using GitHub GraphQL API.
+ * Searches all repositories in the given GitHub organization having the given topic, using GitHub GraphQL API.
  * @param {string} orgName the name of the GitHub organization
- * @param {string} repoName the name of the GitHub repository
- * @returns {Promise<Repository|Error>} a promise resolving to the repo info or rejecting with an error
+ * @param {string} topic the topic that all repos should have
+ * @param {number} maxRepos the maximum number of repositories that will be returned. If not provided defaults to {@link MAX_REPOSITORIES_DEFAULT}
+ * @returns {Promise<Repository[]|Error>} a promise resolving to the found repos or rejecting with an error
  */
-exports.queryRepository = (orgName, repoName) => {
+ exports.queryRepositoriesByTopic = (orgName, topic, maxRepos = MAX_REPOSITORIES_DEFAULT) => {
     return githubClient.query({
-        query: QUERY_REPO_INFO,
-        variables: {
-            owner : orgName,
-            name : repoName
-        }
-    }).then(result => ({
-        name: result.data.repository.name,
-        description: result.data.repository.description || '',
-        imageUrl: result.data.repository.openGraphImageUrl || '',
-        repoUrl: result.data.repository.url
-    }))
-}
+        query: buildQueryForReposByTopic(orgName, topic, maxRepos)
+    }).then(result => result.data.search.nodes.map(repo => ({
+        name: repo.name,
+        description: repo.description || '',
+        imageUrl: repo.openGraphImageUrl || '',
+        repoUrl: repo.url
+    }))).then(repos => repos.sort((repo1, repo2) => repo1.name.localeCompare(repo2.name)))
+ }