From f6a45d25b87aeb2efeb48b174c9892b63f9757a3 Mon Sep 17 00:00:00 2001 From: Hoan Nguyen <83745591+nqhoan-axonivy@users.noreply.github.com> Date: Thu, 26 Dec 2024 11:42:57 +0700 Subject: [PATCH] MARP-1451 include codeowners in GitHub for each connector (#2) * Added Codeowner detecter * Define codeowner for market * Add gitignore * Remove unuse file * Format codes * Enhance log file * Handle feedback * Handle permission --- .github/workflows/missing-file-check.yml | 10 +- .gitignore | 5 + build/public-repo-missing-files/Jenkinsfile | 37 ------ github-repo-manager/pom.xml | 4 +- .../com/axonivy/github/GitHubProvider.java | 2 +- .../github/file/CodeOwnerFilesDetector.java | 48 +++++++ .../axonivy/github/file/FileReference.java | 20 +-- .../com/axonivy/github/file/GitHubFiles.java | 4 +- .../github/file/GitHubMissingFiles.java | 9 +- .../file/GitHubMissingFilesDetector.java | 118 ++++++++++++------ .../com/axonivy/github/file/CodeOwners.json | 49 ++++++++ 11 files changed, 207 insertions(+), 99 deletions(-) create mode 100644 .gitignore delete mode 100644 build/public-repo-missing-files/Jenkinsfile create mode 100644 github-repo-manager/src/main/java/com/axonivy/github/file/CodeOwnerFilesDetector.java create mode 100644 github-repo-manager/src/main/resources/com/axonivy/github/file/CodeOwners.json diff --git a/.github/workflows/missing-file-check.yml b/.github/workflows/missing-file-check.yml index 29c8a9c..1bc2ffc 100644 --- a/.github/workflows/missing-file-check.yml +++ b/.github/workflows/missing-file-check.yml @@ -28,23 +28,19 @@ jobs: with: java-version: 17 distribution: temurin - - - name: Setup Maven - uses: stCarolas/setup-maven@v5 + cache: maven - name: Set default values for dryRun and workingOrgs - id: set-defaults run: | echo "dryRun=${{ github.event.inputs.dryRun || 'true' }}" >> $GITHUB_ENV - echo "workingOrgs=${{ github.event.inputs.workingOrgs || 'axonivy-market' }}" >> $GITHUB_ENV + echo "workingOrgs=${{ github.event.inputs.workingOrgs || 'axonivy-market' }}" >> $GITHUB_ENV - name: Build with Maven working-directory: ./github-repo-manager run: | mvn -B clean compile exec:java \ -DDRY_RUN="${{ env.dryRun }}" \ - -DGITHUB.TOKEN.FILE="${{ secrets.TOKEN }}" \ + -DGITHUB.TOKEN="${{ secrets.TOKEN }}" \ -Dexec.mainClass="com.axonivy.github.file.GitHubMissingFiles" \ -Dexec.args="${{ github.actor }}" \ -DGITHUB.WORKING.ORGANIZATIONS="${{ env.workingOrgs }}" - continue-on-error: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..52b4d53 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +# Default ignored files +/shelf/ +/workspace.xml +./idea/* +*/idea/* \ No newline at end of file diff --git a/build/public-repo-missing-files/Jenkinsfile b/build/public-repo-missing-files/Jenkinsfile deleted file mode 100644 index 386947b..0000000 --- a/build/public-repo-missing-files/Jenkinsfile +++ /dev/null @@ -1,37 +0,0 @@ -pipeline { - agent { - dockerfile { - dir 'build' - } - } - - options { - buildDiscarder(logRotator(numToKeepStr: '10')) - disableConcurrentBuilds() - } - - parameters { - booleanParam name: 'dryRun', defaultValue: true, description: 'Whether the build should really make a change or not' - } - - triggers { - cron '@midnight' - } - - stages { - stage('build') { - steps { - script { - userId = currentBuild.getBuildCauses()[0].userId - echo "BUILD_TRIGGER_BY: ${userId}" - withCredentials([file(credentialsId: 'github-ivyteam-token-repo-manager', variable: 'tokenFile')]) { - maven cmd: "-f github-repo-manager clean compile exec:java " + - "-DDRY_RUN=${params.dryRun} -DGITHUB.TOKEN.FILE=${tokenFile} " + - "-Dexec.mainClass=\"com.axonivy.github.file.GitHubMissingFiles\" " + - "-Dexec.args=\"${userId}\"" - } - } - } - } - } -} diff --git a/github-repo-manager/pom.xml b/github-repo-manager/pom.xml index 0999223..d8bb02c 100644 --- a/github-repo-manager/pom.xml +++ b/github-repo-manager/pom.xml @@ -3,7 +3,9 @@ com.axonivy.github github-repo-manager 0.0.1-SNAPSHOT - + + UTF-8 + diff --git a/github-repo-manager/src/main/java/com/axonivy/github/GitHubProvider.java b/github-repo-manager/src/main/java/com/axonivy/github/GitHubProvider.java index d213812..d641de2 100644 --- a/github-repo-manager/src/main/java/com/axonivy/github/GitHubProvider.java +++ b/github-repo-manager/src/main/java/com/axonivy/github/GitHubProvider.java @@ -23,7 +23,7 @@ public static GitHub get() { } public static GitHub getGithubToken() { - String token = System.getProperty("GITHUB.TOKEN.FILE"); + String token = System.getProperty("GITHUB.TOKEN"); try { return new GitHubBuilder().withOAuthToken(token).build(); } catch (IOException ex) { diff --git a/github-repo-manager/src/main/java/com/axonivy/github/file/CodeOwnerFilesDetector.java b/github-repo-manager/src/main/java/com/axonivy/github/file/CodeOwnerFilesDetector.java new file mode 100644 index 0000000..974d346 --- /dev/null +++ b/github-repo-manager/src/main/java/com/axonivy/github/file/CodeOwnerFilesDetector.java @@ -0,0 +1,48 @@ +package com.axonivy.github.file; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; + +import java.io.IOException; +import java.util.List; + +public class CodeOwnerFilesDetector extends GitHubMissingFilesDetector { + private static final String CODE_OWNER_FILE_NAME = "CodeOwners.json"; + private static final TypeReference> CODE_OWNER_TYPE_REFERENCE = new TypeReference<>() { + }; + private static final String CODE_OWNER_FORMAT = "* %s"; + private static final ObjectMapper objectMapper = new ObjectMapper(); + private List codeOwners; + + public CodeOwnerFilesDetector(GitHubFiles.FileMeta fileMeta, String user) throws IOException { + super(fileMeta, user); + } + + @Override + protected byte[] loadReferenceFileContent(String repoURL) throws IOException { + if (StringUtils.isBlank(repoURL)) { + return super.loadReferenceFileContent(repoURL); + } + + for (var codeOwner : getAllCodeOwners()) { + if (StringUtils.contains(repoURL, codeOwner.product)) { + return String.format(CODE_OWNER_FORMAT, codeOwner.owner).getBytes(); + } + } + return new byte[0]; + } + + private List getAllCodeOwners() throws IOException { + if (ObjectUtils.isEmpty(codeOwners)) { + try (var is = CodeOwnerFilesDetector.class.getResourceAsStream(CODE_OWNER_FILE_NAME)) { + codeOwners = objectMapper.readValue(is, CODE_OWNER_TYPE_REFERENCE); + } + } + return codeOwners; + } + + record CodeOwner(String product, String owner) { + } +} diff --git a/github-repo-manager/src/main/java/com/axonivy/github/file/FileReference.java b/github-repo-manager/src/main/java/com/axonivy/github/file/FileReference.java index dc5b469..4b5c012 100644 --- a/github-repo-manager/src/main/java/com/axonivy/github/file/FileReference.java +++ b/github-repo-manager/src/main/java/com/axonivy/github/file/FileReference.java @@ -1,16 +1,15 @@ package com.axonivy.github.file; -import java.io.IOException; - -import org.apache.commons.io.IOUtils; - import com.axonivy.github.file.GitHubFiles.FileMeta; +import org.apache.commons.io.IOUtils; +import java.io.IOException; -public record FileReference(FileMeta meta, byte[] content) { +public class FileReference { + FileMeta meta; public FileReference(FileMeta meta) throws IOException { - this(meta, load(meta)); + this.meta = meta; } private static byte[] load(FileMeta meta) throws IOException { @@ -22,4 +21,11 @@ private static byte[] load(FileMeta meta) throws IOException { } } -} + public FileMeta meta() { + return meta; + } + + public byte[] content() throws IOException { + return load(meta); + } +} \ No newline at end of file diff --git a/github-repo-manager/src/main/java/com/axonivy/github/file/GitHubFiles.java b/github-repo-manager/src/main/java/com/axonivy/github/file/GitHubFiles.java index dbb4358..bd0622f 100644 --- a/github-repo-manager/src/main/java/com/axonivy/github/file/GitHubFiles.java +++ b/github-repo-manager/src/main/java/com/axonivy/github/file/GitHubFiles.java @@ -8,8 +8,10 @@ public interface GitHubFiles { "Add Security.md file to repo"); FileMeta CODE_OF_CONDUCT = new FileMeta("CODE_OF_CONDUCT.md", "Add code of conduct file", "Add_Code_of_Conduct_v2", "Add CODE_OF_CONDUCT.md file to repo"); + FileMeta CODE_OWNERS = new FileMeta(".github/CODEOWNERS", "Add code owner file", + "Add_CODEOWNERS", "Add CODEOWNERS file to repo"); - public record FileMeta(String filePath, String pullRequestTitle, String branchName, String commitMessage) { + record FileMeta(String filePath, String pullRequestTitle, String branchName, String commitMessage) { public String filePath() { return filePath; diff --git a/github-repo-manager/src/main/java/com/axonivy/github/file/GitHubMissingFiles.java b/github-repo-manager/src/main/java/com/axonivy/github/file/GitHubMissingFiles.java index ef9712c..bb4c6ec 100644 --- a/github-repo-manager/src/main/java/com/axonivy/github/file/GitHubMissingFiles.java +++ b/github-repo-manager/src/main/java/com/axonivy/github/file/GitHubMissingFiles.java @@ -1,15 +1,13 @@ package com.axonivy.github.file; -import static com.axonivy.github.file.GitHubFiles.CODE_OF_CONDUCT; -import static com.axonivy.github.file.GitHubFiles.LICENSE; -import static com.axonivy.github.file.GitHubFiles.SECURITY; - import java.io.IOException; import java.util.Arrays; import java.util.List; import com.axonivy.github.file.GitHubFiles.FileMeta; +import static com.axonivy.github.file.GitHubFiles.*; + public class GitHubMissingFiles { private static final List REQUIRED_FILES = List.of(LICENSE, SECURITY, CODE_OF_CONDUCT); @@ -33,6 +31,9 @@ public static void main(String[] args) throws IOException { var returnedStatus = detector.removeFile(workingOrganizations); status = returnedStatus != 0 ? returnedStatus : status; } + var codeOwnerDetector = new CodeOwnerFilesDetector(CODE_OWNERS, user); + var returnedStatus = codeOwnerDetector.requireFile(workingOrganizations); + status = returnedStatus != 0 ? returnedStatus : status; System.exit(status); } diff --git a/github-repo-manager/src/main/java/com/axonivy/github/file/GitHubMissingFilesDetector.java b/github-repo-manager/src/main/java/com/axonivy/github/file/GitHubMissingFilesDetector.java index e90ee53..192aae2 100644 --- a/github-repo-manager/src/main/java/com/axonivy/github/file/GitHubMissingFilesDetector.java +++ b/github-repo-manager/src/main/java/com/axonivy/github/file/GitHubMissingFilesDetector.java @@ -7,10 +7,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.io.input.CharSequenceReader; -import org.kohsuke.github.GHContent; -import org.kohsuke.github.GHRepository; -import org.kohsuke.github.GHUser; -import org.kohsuke.github.GitHub; +import org.kohsuke.github.*; import com.axonivy.github.DryRun; import com.axonivy.github.GitHubProvider; @@ -19,11 +16,12 @@ public class GitHubMissingFilesDetector { private static final String GITHUB_ORG = ".github"; + private static final String BRANCH_PREFIX = "refs/heads/"; private static final Logger LOG = new Logger(); private boolean isNotSync; private final FileReference reference; private final GitHub github; - private GHUser ghActor; + private final GHUser ghActor; public GitHubMissingFilesDetector(FileMeta fileMeta, String user) throws IOException { Objects.requireNonNull(fileMeta); @@ -45,7 +43,7 @@ public int requireFile(List orgNames) throws IOException { LOG.error("At least one repository has no {0}.", reference.meta().filePath()); LOG.error("Add a {0} manually or run the build without DRYRUN to add {0} to the repository.", reference.meta().filePath()); - return -1; + return 1; } return 0; } @@ -67,7 +65,7 @@ private void missingFile(GHRepository repo) throws IOException { if (hasSimilarContent(foundFile)) { LOG.info("Repo {0} has {1}.", repo.getFullName(), foundFile.getName()); } else { - handleOtherContent(repo, foundFile); + handleOtherContent(repo); } } else { handleMissingFile(repo); @@ -84,8 +82,11 @@ private GHContent getFileContent(String path, GHRepository repo) { } private boolean hasSimilarContent(GHContent existingFile) throws IOException { - Reader targetContent = new CharSequenceReader(new String(reference.content())); - Reader actualContent = new CharSequenceReader(new String(existingFile.read().readAllBytes())); + Reader targetContent = new CharSequenceReader(new String(loadReferenceFileContent(existingFile.getGitUrl()))); + Reader actualContent; + try (var inputStream = existingFile.read()) { + actualContent = new CharSequenceReader(new String(inputStream.readAllBytes())); + } return IOUtils.contentEqualsIgnoreEOL(targetContent, actualContent); } @@ -99,37 +100,78 @@ private void handleMissingFile(GHRepository repo) throws IOException { } LOG.info("Repo {0} {1} synced.", repo.getFullName(), reference.meta().filePath()); } catch (IOException ex) { - LOG.error("Cannot add {0} to repo {1}.", repo.getFullName(), reference.meta().filePath()); + LOG.error("Cannot add {0} to repo {1}.", reference.meta().filePath(), repo.getFullName()); throw ex; } } private void addMissingFile(GHRepository repo) throws IOException { var defaultBranch = repo.getBranch(repo.getDefaultBranch()); - var sha1 = defaultBranch.getSHA1(); - repo.createRef("refs/heads/" + reference.meta().branchName(), sha1); - repo.createContent() - .branch(reference.meta().branchName()) - .path(reference.meta().filePath()) - .content(reference.content()) - .message(reference.meta().commitMessage()) - .commit(); - var pr = repo.createPullRequest(reference.meta().pullRequestTitle(), reference.meta().branchName(), repo.getDefaultBranch(), ""); - if (ghActor != null) { - pr.setAssignees(ghActor); + String refURL = createBranchIfMissing(repo, BRANCH_PREFIX + reference.meta().branchName(), defaultBranch.getSHA1()); + try { + repo.createContent() + .branch(refURL) + .path(reference.meta().filePath()) + .content(loadReferenceFileContent(repo.getUrl().toString())) + .message(reference.meta().commitMessage()) + .commit(); + } catch (GHFileNotFoundException notFoundException) { + LOG.error("Commit new file {0} to repo failed due to lack of permissions", reference.meta().filePath()); + isNotSync = true; + } + createNewPullRequest(repo, refURL); + } + + private void createNewPullRequest(GHRepository repo, String refURL) throws IOException { + try { + var pr = repo.createPullRequest(reference.meta().pullRequestTitle(), refURL, repo.getDefaultBranch(), ""); + if (ghActor != null) { + pr.setAssignees(ghActor); + } + } catch (HttpException e) { + LOG.error("Create new pull request failed: {0}", e.getMessage()); + isNotSync = true; } - pr.merge(reference.meta().commitMessage()); } - private void handleOtherContent(GHRepository repo, GHContent foundFile) throws IOException { + private String createBranchIfMissing(GHRepository repo, String branchName, String sha) throws IOException { + String createdBranch = branchName; + var isBranchExisted = false; + try { + var existedRef = repo.getRef(branchName); + if (existedRef != null && existedRef.getRef().endsWith(branchName)) { + createdBranch = existedRef.getRef(); + isBranchExisted = true; + } + } catch (Exception exception) { + LOG.error("Get branch {0} failed", branchName); + } + if (!isBranchExisted) { + try { + createdBranch = repo.createRef(branchName, sha).getRef(); + } catch (GHFileNotFoundException notFoundException) { + LOG.error("Create new ref {0} failed due to lack of permissions", branchName); + isNotSync = true; + } catch (HttpException e) { + LOG.error("Create new ref {0} failed: {1}", branchName, e.getMessage()); + isNotSync = true; + } catch (Exception e) { + LOG.error("Create new ref {0} failed due to an unexpected error occurred: {1}", branchName, e.getMessage()); + isNotSync = true; + } + } + return createdBranch; + } + + private void handleOtherContent(GHRepository repo) throws IOException { try { if (DryRun.is()) { isNotSync = true; LOG.info("DRYRUN: "); LOG.info("Repo {0} has {1} but the content is different from required file {2}.", - repo.getFullName(), foundFile.getName(), reference.meta().filePath()); + repo.getFullName(), reference.meta().filePath(), reference.meta().filePath()); } else { - updateFile(repo, foundFile); + updateFile(repo); LOG.info("Repo {0} {1} synced.", repo.getFullName(), reference.meta().filePath()); } } catch (IOException ex) { @@ -138,23 +180,17 @@ private void handleOtherContent(GHRepository repo, GHContent foundFile) throws I } } - private void updateFile(GHRepository repo, GHContent foundFile) throws IOException { + private void updateFile(GHRepository repo) throws IOException { var headBranch = repo.getBranch(repo.getDefaultBranch()); - repo.createRef("refs/heads/" + reference.meta().branchName(), headBranch.getSHA1()); - foundFile.update(reference.content(), - reference.meta().commitMessage(), - reference.meta().branchName() - ); - var pr = repo.createPullRequest( - reference.meta().pullRequestTitle(), - reference.meta().branchName(), - repo.getDefaultBranch(), - "" - ); - if (ghActor != null) { - pr.setAssignees(ghActor); - // we open a PR; but auto-merging of it should be avoided - } + String refURL = createBranchIfMissing(repo, BRANCH_PREFIX + reference.meta().branchName(), headBranch.getSHA1()); + repo.getFileContent(reference.meta().filePath(), refURL) + .update(loadReferenceFileContent(repo.getUrl().toString()), + reference.meta().commitMessage(), + refURL); + createNewPullRequest(repo, refURL); } + protected byte[] loadReferenceFileContent(String repoURL) throws IOException { + return reference.content(); + } } diff --git a/github-repo-manager/src/main/resources/com/axonivy/github/file/CodeOwners.json b/github-repo-manager/src/main/resources/com/axonivy/github/file/CodeOwners.json new file mode 100644 index 0000000..dc19b9a --- /dev/null +++ b/github-repo-manager/src/main/resources/com/axonivy/github/file/CodeOwners.json @@ -0,0 +1,49 @@ +[ + { "product": "axonivy-market/marketplace", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/portal", "owner": "@axonivy-market/team-wawa" }, + { "product": "axonivy-market/demo-projects", "owner": "@axonivy-market/ivyteam" }, + { "product": "axonivy-market/mailstore-connector", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/process-analyser", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/docuware-connector", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/ai-assistant", "owner": "@axonivy-market/team-wawa" }, + { "product": "axonivy-market/amazon-comprehend-connector", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/open-weather-connector", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/doc-factory", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/dmn-decision-table", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/jira-connector", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/cms-editor", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/approval-decision-utils", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/graphql-demo", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/master-detail", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/talentlink-connector", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/successfactors-connector", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/cronjob", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/db-demos", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/email-encryption", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/stateful-datatable", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/excel-connector", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/custom-mail-demo", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/adobe-acrobat-sign-connector", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/express-importer", "owner": "@axonivy-market/team-wawa" }, + { "product": "axonivy-market/rtf-factory", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/process-inspector", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/persistence-utils", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/html-dialog-utils", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/kafka-connector", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/excel-importer", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/db-utils", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/jsf-formarchive-util", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/msgraph-connector", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/skribble-connector", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/api-proxy", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/ldap-connector", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/axonivy-express", "owner": "@axonivy-market/team-wawa" }, + { "product": "axonivy-market/sftp-connector", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/vertexai-google", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/x-connector", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/ups-connector", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/ui-path-connector", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/threema-connector", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/tel-search-ch-connector", "owner": "@axonivy-market/team-octopus" }, + { "product": "axonivy-market/srf-weather-connector", "owner": "@axonivy-market/team-octopus" } +]