-
Notifications
You must be signed in to change notification settings - Fork 28
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
SNOW-1812871: Setup pipeline with warnings (#777)
- Loading branch information
1 parent
bee0e58
commit 2ff27d5
Showing
13 changed files
with
4,412 additions
and
84 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| name: Code quality | ||
|
|
||
| on: | ||
| push: | ||
| branches: | ||
| - master | ||
| tags: | ||
| - v* | ||
| pull_request: | ||
| branches: | ||
| - '**' | ||
|
|
||
| jobs: | ||
| check-warnings: | ||
| name: Extra-Warnings-Linux | ||
| runs-on: ubuntu-latest | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| build_type: [ 'Release' ] | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: '3.13' | ||
| architecture: 'x64' | ||
| - name: Restore cached deps | ||
| id: cache-restore-deps | ||
| uses: actions/cache/restore@v4 | ||
| with: | ||
| path: dep-cache | ||
| key: ${{ matrix.build_type }}-${{ github.event.pull_request.base.sha }}-Linux-dep-cache | ||
| if: github.event_name == 'pull_request' | ||
| - name: Build | ||
| shell: bash | ||
| env: | ||
| USE_EXTRA_WARNINGS: "true" | ||
| BUILD_TYPE: ${{ matrix.build_type }} | ||
| run: ci/build_linux.sh | ||
| - name: Restore cached warnings | ||
| id: cache-restore-warnings | ||
| uses: actions/cache/restore@v4 | ||
| with: | ||
| path: warnings.json | ||
| key: ${{ github.event.pull_request.base.sha }}-compile-warnings | ||
| if: github.event_name == 'pull_request' | ||
| - name: Use cached warnings as a baseline | ||
| shell: bash | ||
| run: cp warnings.json ./ci/scripts/warnings_baseline.json | ||
| if: steps.cache-restore-warnings.outputs.cache-hit == true | ||
| - name: Warning report | ||
| shell: bash | ||
| run: ci/scripts/warning_report.sh | ||
| - name: Upload build log | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: build log | ||
| path: build.log | ||
| - name: Upload warning report | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: report | ||
| path: report.md | ||
| - name: Upload warnings | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: warnings | ||
| path: warnings.json | ||
| - name: Cache warnings | ||
| id: cache-save-warnings | ||
| uses: actions/cache/save@v4 | ||
| with: | ||
| path: warnings.json | ||
| key: ${{ github.event.pull_request.base.sha }}-compile-warnings | ||
| if: github.ref_name == github.event.repository.default_branch | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -33,3 +33,4 @@ venv/ | |
| wss-*-agent.config | ||
| wss-unified-agent.jar | ||
| whitesource/ | ||
| build.log | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,152 @@ | ||
| import re | ||
| import enum | ||
| from dataclasses import dataclass | ||
| from sys import stderr | ||
| from typing import Optional, List | ||
| import dataclasses | ||
| import json | ||
| import argparse | ||
|
|
||
| """ | ||
| Regexes for matching warnings: | ||
| /tmp/libsnowflakeclient/include/snowflake/SF_CRTFunctionSafe.h:223:16: warning: unused parameter 'in_sizeOfBuffer' [-Wunused-parameter] <- message | ||
| 223 | size_t in_sizeOfBuffer, <- snippet | ||
| """ | ||
| message_re = re.compile(r"""(?P<file_path>[^:]*):(?P<line>\d+):(?:(?P<col>\d+):)?\s+warning:(?P<message>.*)\[(?P<flag>.*)\]""") | ||
|
|
||
| class ParserState(enum.Enum): | ||
| MESSAGE = "MESSAGE" | ||
| SNIPPET = "SNIPPET" | ||
|
|
||
| @dataclass | ||
| class CompilerWarning: | ||
| file_path: str | ||
| line: int | ||
| column: Optional[int] | ||
| message: str | ||
| flag: str | ||
| snippet: Optional[str] | ||
| source: str | ||
| def key(self): | ||
| return self.file_path, self.message, self.flag | ||
|
|
||
| @dataclass | ||
| class WarningDiff: | ||
| new: List[CompilerWarning] | ||
| old: List[CompilerWarning] | ||
|
|
||
| """ | ||
| Parses warnings from compiler output | ||
| """ | ||
| def parse_warnings(path: str) -> List[CompilerWarning]: | ||
| warnings = [] | ||
| state = ParserState.MESSAGE | ||
| with open(path, "r") as f: | ||
| lines = f.readlines() | ||
| for line in lines: | ||
| if state == ParserState.MESSAGE: | ||
| m_match = message_re.match(line) | ||
| if not m_match: | ||
| continue | ||
|
|
||
| col = m_match.group("col") | ||
| if col: | ||
| col = int(col) | ||
|
|
||
| warning = CompilerWarning( | ||
| file_path=m_match.group("file_path"), | ||
| line=int(m_match.group("line")), | ||
| column=col, | ||
| message=m_match.group("message"), | ||
| flag=m_match.group("flag"), | ||
| snippet=None, | ||
| source=line | ||
| ) | ||
|
|
||
| warnings.append(warning) | ||
| state = ParserState.SNIPPET | ||
| continue | ||
|
|
||
| if state == ParserState.SNIPPET: | ||
| warning.snippet = line | ||
| warning.source += line | ||
|
|
||
| state = ParserState.MESSAGE | ||
| continue | ||
|
|
||
| result = [] | ||
| for w in warnings: | ||
| if w not in result: | ||
| result.append(w) | ||
| return result | ||
|
|
||
| def dump_warnings(warnings: List[CompilerWarning]) -> str: | ||
| warnings_as_dict = [dataclasses.asdict(w) for w in warnings] | ||
| return json.dumps(warnings_as_dict, indent=2) | ||
|
|
||
| def load_warnings(warnings_json: str) -> List[CompilerWarning]: | ||
| warnings_as_dict = json.loads(warnings_json) | ||
| return [CompilerWarning(**w) for w in warnings_as_dict] | ||
|
|
||
| def write(path: str, data: str): | ||
| with open(path, "w") as f: | ||
| f.write(data) | ||
|
|
||
| def read(path: str) -> str: | ||
| with open(path, "r") as f: | ||
| return f.read() | ||
|
|
||
| def generate_report(path: str, new_warnings: List[CompilerWarning], old_warnings: List[CompilerWarning]): | ||
| with open(path, "w") as f: | ||
| diff = {} | ||
| for nw in new_warnings: | ||
| if nw.key() not in diff: | ||
| diff[nw.key()] = WarningDiff(new=[], old=[]) | ||
|
|
||
| diff[nw.key()].new.append(nw) | ||
|
|
||
| for ow in old_warnings: | ||
| if ow.key() not in diff: | ||
| diff[ow.key()] = WarningDiff(new=[], old=[]) | ||
|
|
||
| diff[ow.key()].old.append(ow) | ||
|
|
||
| failed = False | ||
| for d in diff.values(): | ||
| if len(d.new) > len(d.old): | ||
| failed = True | ||
|
|
||
| if failed: | ||
| f.write("### Failed\n\n") | ||
| else: | ||
| f.write("### Succeeded\n\n") | ||
|
|
||
| for d in diff.values(): | ||
| balance = len(d.new) - len(d.old) | ||
| if balance < 0: | ||
| f.write("- Removed {} compiler warnings from {} [{}].\n".format(-balance, d.old[0].file_path, d.old[0].flag)) | ||
|
|
||
| if balance > 0: | ||
| f.write("- Added {} compiler warnings to {} [{}]. Please remove following warnings:\n".format(balance, d.new[0].file_path, d.new[0].flag)) | ||
| for w in d.new: | ||
| f.write("```\n") | ||
| f.write(w.source) | ||
| f.write("```\n") | ||
|
|
||
| parser = argparse.ArgumentParser( | ||
| prog='generate_warning_report', | ||
| description='Generate compiler warning report', | ||
| epilog='Text at the bottom of help' | ||
| ) | ||
|
|
||
| parser.add_argument('--build-log', required=True) # option that takes a value | ||
| parser.add_argument('--load-warnings', required=True) | ||
| parser.add_argument('--dump-warnings', required=True) | ||
| parser.add_argument('--report', required=True) | ||
| args = parser.parse_args() | ||
|
|
||
| new_warnings = parse_warnings(args.build_log) | ||
| old_warnings = load_warnings(read(args.load_warnings)) | ||
| generate_report(args.report, new_warnings, old_warnings) | ||
| write(args.dump_warnings, dump_warnings(new_warnings)) | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| #!/bin/bash | ||
|
|
||
| python3 ci/scripts/generate_warning_report.py --build-log build.log --load-warnings ci/scripts/warnings_baseline.json --dump-warnings warnings.json --report report.md | ||
|
|
||
| if [[ -n "${GITHUB_STEP_SUMMARY}" ]]; | ||
| then | ||
| cat report.md >> "$GITHUB_STEP_SUMMARY" | ||
| fi | ||
|
|
||
| if [[ "$(head -n 1 report.md)" == "### Failed" ]]; | ||
| then | ||
| exit 1 | ||
| fi |
Oops, something went wrong.