Merge branch 'master' into use_fg_reader_hive

.github/PULL_REQUEST_TEMPLATE.md

@@ -12,7 +12,7 @@ _If medium or high, explain what verification was done to mitigate the risks._
 
 ### Documentation Update
 
-_Describe any necessary documentation update if there is any new feature, config, or user-facing change_
+_Describe any necessary documentation update if there is any new feature, config, or user-facing change. If not, put "none"._
 
 - _The config description must be updated if new configs are added or the default value of the configs are changed_
 - _Any new feature or user-facing change requires updating the Hudi website. Please create a Jira ticket, attach the

.github/workflows/azure_ci.js

@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+async function checkAzureCiAndCreateCommitStatus({ github, context, prNumber, latestCommitHash }) {
+ console.log(`- Checking Azure CI status of PR: ${prNumber} ${latestCommitHash}`);
+ const botUsername = 'hudi-bot';
+
+ const comments = await github.paginate(github.rest.issues.listComments, {
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: prNumber,
+ sort: 'updated',
+ direction: 'desc',
+ per_page: 100
+ });
+
+ // Find the latest comment from hudi-bot containing the Azure CI report
+ const botComments = comments.filter(comment => comment.user.login === botUsername);
+
+ let status = 'pending';
+ let message = 'In progress';
+ let azureRunLink = '';
+
+ if (botComments.length > 0) {
+ const lastComment = botComments[0];
+ const reportPrefix = `${latestCommitHash} Azure: `
+ const successReportString = `${reportPrefix}[SUCCESS]`
+ const failureReportString = `${reportPrefix}[FAILURE]`
+
+ if (lastComment.body.includes(reportPrefix)) {
+ if (lastComment.body.includes(successReportString)) {
+ message = 'Successful on the latest commit';
+ status = 'success';
+ } else if (lastComment.body.includes(failureReportString)) {
+ message = 'Failed on the latest commit';
+ status = 'failure';
+ }
+ }
+
+ const linkRegex = /\[[a-zA-Z]+\]\((https?:\/\/[^\s]+)\)/;
+ const parts = lastComment.body.split(reportPrefix);
+ const secondPart = parts.length > 1 ? parts[1] : '';
+ const match = secondPart.match(linkRegex);
+
+ if (match) {
+ azureRunLink = match[1];
+ }
+ }
+
+ console.log(`Status: ${status}`);
+ console.log(`Azure Run Link: ${azureRunLink}`);
+ console.log(`${message}`);
+
+ console.log(`- Create commit status of PR based on Azure CI status: ${prNumber} ${latestCommitHash}`);
+ // Create or update the commit status for Azure CI
+ await github.rest.repos.createCommitStatus({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ sha: latestCommitHash,
+ state: status,
+ target_url: azureRunLink,
+ description: message,
+ context: 'Azure CI'
+ });
+
+ return { status, message, azureRunLink };
+}
+
+module.exports = checkAzureCiAndCreateCommitStatus;

.github/workflows/azure_ci_check.yml

@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Azure CI
+
+on:
+ issue_comment:
+ types: [ created, edited, deleted ]
+
+permissions:
+ statuses: write
+ pull-requests: read
+ issues: read
+
+jobs:
+ check-azure-ci-and-add-commit-status:
+ if: |
+ github.event.issue.pull_request != null &&
+ github.event.comment.user.login == 'hudi-bot'
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v2
+
+ - name: Check PR state
+ id: check_pr_state
+ uses: actions/github-script@v7
+ with:
+ github-token: ${{secrets.GITHUB_TOKEN}}
+ script: |
+ const issueNumber = context.issue.number;
+ const { data: pullRequest } = await github.rest.pulls.get({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ pull_number: issueNumber
+ });
+ 
+ // Only check open PRs and a PR that is not a HOTFIX
+ const shouldSkip = (pullRequest.body.includes('HOTFIX: SKIP AZURE CI')
+ || pullRequest.state != 'open');
+ 
+ if (!shouldSkip) {
+ const commitHash = pullRequest.head.sha;
+ console.log(`Latest commit hash: ${commitHash}`);
+ // Set the output variable to be used in subsequent step
+ core.setOutput("latest_commit_hash", commitHash);
+ }
+ console.log(`Should skip Azure CI? ${shouldSkip}`);
+ return shouldSkip;
+
+ - name: Check Azure CI report and create commit status to PR
+ if: steps.check_pr_state.outputs.result != 'true'
+ uses: actions/github-script@v7
+ with:
+ github-token: ${{secrets.GITHUB_TOKEN}}
+ script: |
+ const latestCommitHash = '${{ steps.check_pr_state.outputs.latest_commit_hash }}' 
+ const issueNumber = context.issue.number;
+ const checkAzureCiAndCreateCommitStatus = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/azure_ci.js`);
+ 
+ await checkAzureCiAndCreateCommitStatus({
+ github,
+ context,
+ prNumber: issueNumber,
+ latestCommitHash: latestCommitHash
+ });