From dcd84523ab380a5c30ed21d40c622c7a2abb8f3c Mon Sep 17 00:00:00 2001 From: krzys-h Date: Thu, 9 Jul 2020 18:42:52 +0200 Subject: [PATCH 1/5] Send colobot-lint results to GitHub using annotations This makes the results visible directly in the pull request UI changes view --- .github/workflows/lint.yml | 204 +++++++++++++++++++++++-------------- 1 file changed, 129 insertions(+), 75 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 77333b98..bd3dde67 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -84,79 +84,133 @@ jobs: with: name: HTML results path: build/html_report - - name: Update stable/unstable build status - shell: bash + - run: pip install requests + - name: Send linter results to GitHub + shell: python + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - set -e +x - # Update stable/unstable build status - ret=0 - WORKSPACE=$PWD - COLOBOT_LINT_REPORT_FILE="$WORKSPACE/build/colobot_lint_report.xml" - COLOBOT_LINT_DIR="/tmp/colobot-lint/archive" - OVERALL_STABLE_RULES=( - "class naming" - "code block placement" - "compile error" - # "compile warning" - # "enum naming" - # "function naming" - "header file not self-contained" - # "implicit bool cast" - # "include style" - # "inconsistent declaration parameter name" - "license header" - # "naked delete" - # "naked new" - # "old style function" - "old-style null pointer" - # "possible forward declaration" - "undefined function" - # "uninitialized field" - # "uninitialized local variable" - # "unused forward declaration" - # "variable naming" - "whitespace" - ) - echo "Checking rule stability (overall)" - for ((i = 0; i < ${#OVERALL_STABLE_RULES[@]}; i++)); do - rule="${OVERALL_STABLE_RULES[$i]}" - count="$("$COLOBOT_LINT_DIR/Tools/count_errors.py" --rule-filter="$rule" --xml-report-file "$COLOBOT_LINT_REPORT_FILE")" - if [ "$count" != "0" ]; then - echo "UNSTABLE RULE: $rule ($count occurences)" - ret=1 - fi - done - STABLE_RULES_WITHOUT_CBOT=( - "class naming" - "code block placement" - "compile error" - "compile warning" - # "enum naming" - # "function naming" - "header file not self-contained" - # "implicit bool cast" - "include style" - "inconsistent declaration parameter name" - "license header" - "naked delete" - "naked new" - # "old style function" - "old-style null pointer" - # "possible forward declaration" - "undefined function" - "uninitialized field" - # "uninitialized local variable" - "unused forward declaration" - # "variable naming" - "whitespace" - ) - echo "Checking rule stability (without CBOT)" - for ((i = 0; i < ${#STABLE_RULES_WITHOUT_CBOT[@]}; i++)); do - rule="${STABLE_RULES_WITHOUT_CBOT[$i]}" - count="$("$COLOBOT_LINT_DIR/Tools/count_errors.py" --rule-filter="$rule" --file-filter="-.*CBot.*" --xml-report-file "$COLOBOT_LINT_REPORT_FILE")" - if [ "$count" != "0" ]; then - echo "UNSTABLE RULE: $rule (without CBOT, $count occurences)" - ret=1 - fi - done - exit $ret \ No newline at end of file + import os + import sys + import requests + import xml.etree.ElementTree as ET + + OVERALL_STABLE_RULES=[ + "class naming", + "code block placement", + "compile error", + # "compile warning", + # "enum naming", + # "function naming", + "header file not self-contained", + # "implicit bool cast", + # "include style", + # "inconsistent declaration parameter name", + "license header", + # "naked delete", + # "naked new", + # "old style function", + "old-style null pointer", + # "possible forward declaration", + "undefined function", + # "uninitialized field", + # "uninitialized local variable", + # "unused forward declaration", + # "variable naming", + "whitespace", + ] + + STABLE_RULES_WITHOUT_CBOT=[ + "class naming", + "code block placement", + "compile error", + "compile warning", + # "enum naming", + # "function naming", + "header file not self-contained", + # "implicit bool cast", + "include style", + "inconsistent declaration parameter name", + "license header", + "naked delete", + "naked new", + # "old style function", + "old-style null pointer", + # "possible forward declaration", + "undefined function", + "uninitialized field", + # "uninitialized local variable", + "unused forward declaration", + # "variable naming", + "whitespace", + ] + + # None of the available actions seem to do what I want, they all do stupid things like adding another check... let's just do it manually + # GitHub also doesn't seem to provide you with the check suite or check run ID, so we have to get it from the action ID via the API + s = requests.Session() + s.headers.update({ + 'Authorization': 'token ' + os.environ['GITHUB_TOKEN'], + 'Accept': 'application/vnd.github.antiope-preview+json' # Annotations are still technically a preview feature of the API + }) + action_run = s.get(os.environ['GITHUB_API_URL'] + "/repos/" + os.environ['GITHUB_REPOSITORY'] + "/actions/runs/" + os.environ['GITHUB_RUN_ID']).json() + check_suite = s.get(action_run['check_suite_url']).json() + check_suite_runs = s.get(check_suite['check_runs_url']).json() + check_run = check_suite_runs['check_runs'][0] # NOTE: This assumes that the 'lint' job is the first one in the workflow. You could find it by name if you really wanted. + + def we_care_about(file_name, type): + if 'CBot' in file_name: + return type in OVERALL_STABLE_RULES + else: + return type in STABLE_RULES_WITHOUT_CBOT + + results = ET.parse('build/colobot_lint_report.xml') + annotations = [] + for error in results.find('errors').findall('error'): + location = error.find('location') + file_name = os.path.relpath(location.get('file'), os.environ['GITHUB_WORKSPACE']) + line_num = int(location.get('line')) + type = error.get('id') + severity = error.get('severity') + msg = error.get('msg') + + gh_severity = 'warning' + if severity == 'error': + gh_severity = 'failure' + elif severity == 'information': + gh_severity = 'notice' + + if not we_care_about(file_name, type): + # don't send the unstable rules to github at all as there are way too many of them and it would overload the API rate limit + continue + + print('{}:{}: [{}] {}'.format(file_name, line_num, type, msg)) + + annotations.append({ + 'path': file_name, + 'start_line': line_num, + 'end_line': line_num, + 'annotation_level': gh_severity, + 'title': type, + 'message': msg + }) + + summary = 'colobot-lint found {} issues'.format(len(annotations)) + all_ok = len(annotations) == 0 + + # Annotations have to be sent in batches of 50 + first = True + while first or len(annotations) > 0: + first = False + to_send = annotations[:50] + annotations = annotations[50:] + data = { + 'output': { + 'title': summary, + 'summary': summary, + 'annotations': to_send + } + } + r = s.patch(check_run['url'], json=data) + r.raise_for_status() + + sys.exit(0 if all_ok else 1) From 03c60202b889fbc9be959819514136a56a5d37c2 Mon Sep 17 00:00:00 2001 From: krzys-h Date: Thu, 9 Jul 2020 18:55:40 +0200 Subject: [PATCH 2/5] Clean up some of the old colobot-lint run scripts --- .github/workflows/lint.yml | 43 ++++++++++++++------------------------ 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index bd3dde67..1646ac69 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -20,8 +20,18 @@ jobs: mkdir -p /tmp/colobot-lint cd /tmp/colobot-lint wget -O colobot-lint.zip "https://compiled.colobot.info/job/colobot/job/colobot-lint/job/dev/lastSuccessfulBuild/artifact/*zip*/archive.zip" - unzip colobot-lint.zip - chmod +x archive/Tools/count_errors.py # TODO: ??? + + # Unzip the archive + unzip ./colobot-lint.zip + # Workaround for Clang not finding system headers + mkdir ./bin + mv ./archive/build/colobot-lint ./bin/ + chmod +x ./bin/colobot-lint + ln -s ${CLANG_PREFIX}/lib ./lib + # Unpack HtmlReport + tar -zxf ./archive/build/html_report.tar.gz + # Clean up + rm -r ./archive - uses: actions/checkout@v1 - name: Create build directory run: cmake -E make_directory build @@ -32,22 +42,12 @@ jobs: shell: bash run: | set -e +x - # Run colobot-lint - WORKSPACE=$PWD + WORKSPACE="$GITHUB_WORKSPACE" COLOBOT_DIR="$WORKSPACE" COLOBOT_BUILD_DIR="$WORKSPACE/build" - COLOBOT_LINT_BUILD_DIR="/tmp/colobot-lint/archive/build" COLOBOT_LINT_REPORT_FILE="$WORKSPACE/build/colobot_lint_report.xml" - # CLANG_PREFIX="/usr/lib/llvm-3.6" # Set in top-level environment block - cd "$COLOBOT_LINT_BUILD_DIR" - chmod +x ./colobot-lint - # Workaround for Clang not finding system headers - rm -rf bin/ - mkdir -p bin - mv ./colobot-lint ./bin/ - rm -f ./lib - ln -s ${CLANG_PREFIX}/lib ./lib - echo "Running colobot-lint" + + cd "/tmp/colobot-lint" find "$WORKSPACE" \( -wholename "$COLOBOT_DIR/src/*.cpp" \ -or -wholename "$COLOBOT_DIR/test/unit/*.cpp" \ -or -wholename "$COLOBOT_BUILD_DIR/fake_header_sources/src/*.cpp" \ @@ -67,18 +67,7 @@ jobs: path: build/colobot_lint_report.xml - name: Generate HTML report shell: bash - run: | - set -e +x - # Generate HTML report - WORKSPACE=$PWD - COLOBOT_LINT_BUILD_DIR="/tmp/colobot-lint/archive/build" - COLBOT_LINT_REPORT_FILE="$WORKSPACE/build/colobot_lint_report.xml" - HTML_REPORT_DIR="$WORKSPACE/build/html_report" - echo "Generating HTML report" - cd "$COLOBOT_LINT_BUILD_DIR" - rm -rf HtmlReport/ - tar -zxf html_report.tar.gz - HtmlReport/generate.py --xml-report "$COLBOT_LINT_REPORT_FILE" --output-dir "$HTML_REPORT_DIR" + run: /tmp/colobot-lint/HtmlReport/generate.py --xml-report "build/colobot_lint_report.xml" --output-dir "build/html_report" - name: Upload results (HTML) uses: actions/upload-artifact@v2 with: From 76bb08779adf6daba1bf74ba1fc0135e6aecc7b2 Mon Sep 17 00:00:00 2001 From: krzys-h Date: Thu, 9 Jul 2020 18:56:08 +0200 Subject: [PATCH 3/5] Switch to actions/checkout@v2 As per GitHub recommendations --- .github/workflows/build.yml | 4 ++-- .github/workflows/lint.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c70b982b..2b569509 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,7 +20,7 @@ jobs: - name: Install Colobot dependencies run: sudo apt-get install -y --no-install-recommends build-essential cmake libsdl2-dev libsdl2-image-dev libsdl2-ttf-dev libsndfile1-dev libvorbis-dev libogg-dev libpng-dev libglew-dev libopenal-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-regex-dev libphysfs-dev gettext git po4a vorbis-tools librsvg2-bin xmlstarlet if: matrix.container == '' - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Create build directory run: cmake -E make_directory build - name: Run CMake (for Windows using MXE) @@ -88,7 +88,7 @@ jobs: steps: - name: Install Colobot dependencies run: sudo apt-get install -y --no-install-recommends build-essential cmake libsdl2-dev libsdl2-image-dev libsdl2-ttf-dev libsndfile1-dev libvorbis-dev libogg-dev libpng-dev libglew-dev libopenal-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-regex-dev libphysfs-dev gettext git po4a vorbis-tools librsvg2-bin xmlstarlet doxygen graphviz - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Create build directory run: cmake -E make_directory build - name: Run CMake diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1646ac69..19bdafca 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -32,7 +32,7 @@ jobs: tar -zxf ./archive/build/html_report.tar.gz # Clean up rm -r ./archive - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Create build directory run: cmake -E make_directory build - name: Run CMake From 4eaa8588d7acb766cb0179239db61db4938ad045 Mon Sep 17 00:00:00 2001 From: krzys-h Date: Thu, 9 Jul 2020 19:35:25 +0200 Subject: [PATCH 4/5] Add missing apt-get update --- .github/workflows/build.yml | 4 ++-- .github/workflows/lint.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2b569509..d0a5542d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ jobs: fail-fast: false steps: - name: Install Colobot dependencies - run: sudo apt-get install -y --no-install-recommends build-essential cmake libsdl2-dev libsdl2-image-dev libsdl2-ttf-dev libsndfile1-dev libvorbis-dev libogg-dev libpng-dev libglew-dev libopenal-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-regex-dev libphysfs-dev gettext git po4a vorbis-tools librsvg2-bin xmlstarlet + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends build-essential cmake libsdl2-dev libsdl2-image-dev libsdl2-ttf-dev libsndfile1-dev libvorbis-dev libogg-dev libpng-dev libglew-dev libopenal-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-regex-dev libphysfs-dev gettext git po4a vorbis-tools librsvg2-bin xmlstarlet if: matrix.container == '' - uses: actions/checkout@v2 - name: Create build directory @@ -87,7 +87,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install Colobot dependencies - run: sudo apt-get install -y --no-install-recommends build-essential cmake libsdl2-dev libsdl2-image-dev libsdl2-ttf-dev libsndfile1-dev libvorbis-dev libogg-dev libpng-dev libglew-dev libopenal-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-regex-dev libphysfs-dev gettext git po4a vorbis-tools librsvg2-bin xmlstarlet doxygen graphviz + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends build-essential cmake libsdl2-dev libsdl2-image-dev libsdl2-ttf-dev libsndfile1-dev libvorbis-dev libogg-dev libpng-dev libglew-dev libopenal-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-regex-dev libphysfs-dev gettext git po4a vorbis-tools librsvg2-bin xmlstarlet doxygen graphviz - uses: actions/checkout@v2 - name: Create build directory run: cmake -E make_directory build diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 19bdafca..7fd67c4a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,7 +12,7 @@ jobs: CLANG_PREFIX: /usr/lib/llvm-3.6 steps: - name: Download Colobot dependencies - run: sudo apt-get install -y --no-install-recommends build-essential cmake libsdl2-dev libsdl2-image-dev libsdl2-ttf-dev libsndfile1-dev libvorbis-dev libogg-dev libpng-dev libglew-dev libopenal-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-regex-dev libphysfs-dev gettext git po4a vorbis-tools librsvg2-bin xmlstarlet + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends build-essential cmake libsdl2-dev libsdl2-image-dev libsdl2-ttf-dev libsndfile1-dev libvorbis-dev libogg-dev libpng-dev libglew-dev libopenal-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-regex-dev libphysfs-dev gettext git po4a vorbis-tools librsvg2-bin xmlstarlet # TODO: migrate colobot-lint to GitHub Actions - name: Download colobot-lint run: | From 97b266182b467a2c393b58a8531265d6276cacf4 Mon Sep 17 00:00:00 2001 From: MrSimbax Date: Thu, 9 Jul 2020 14:28:55 +0200 Subject: [PATCH 5/5] Fix MXE build --- .github/workflows/build.yml | 2 +- Jenkinsfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d0a5542d..145835bc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,7 +26,7 @@ jobs: - name: Run CMake (for Windows using MXE) working-directory: build # FIXME: without -lsetupapi linking sdl2 fails - run: /opt/mxe/usr/bin/i686-w64-mingw32.static-cmake -DCMAKE_CXX_STANDARD_LIBRARIES="-lkernel32 -luser32 -lgdi32 -lwinspool -lshell32 -lole32 -loleaut32 -luuid -lcomdlg32 -ladvapi32 -lsetupapi" -DCMAKE_INSTALL_PREFIX=/install -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDEV_BUILD=1 -DPORTABLE=1 -DTOOLS=1 -DTESTS=0 .. + run: /opt/mxe/usr/bin/i686-w64-mingw32.static-cmake -DCMAKE_CXX_STANDARD_LIBRARIES="-lkernel32 -luser32 -lgdi32 -lwinspool -lshell32 -lole32 -loleaut32 -luuid -lcomdlg32 -ladvapi32 -lsetupapi" -DCMAKE_INSTALL_PREFIX=/install -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDEV_BUILD=1 -DPORTABLE=1 -DTOOLS=1 -DTESTS=0 -DMXE_USE_CCACHE=0 .. if: matrix.target_os == 'windows' - name: Run CMake (for Linux) working-directory: build diff --git a/Jenkinsfile b/Jenkinsfile index 35f66489..7b52c57b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -31,7 +31,7 @@ pipeline { /opt/mxe/usr/bin/i686-w64-mingw32.static-cmake \ -DCMAKE_CXX_STANDARD_LIBRARIES="-lkernel32 -luser32 -lgdi32 -lwinspool -lshell32 -lole32 -loleaut32 -luuid -lcomdlg32 -ladvapi32 -lsetupapi" \ -DCMAKE_INSTALL_PREFIX=/install \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDEV_BUILD=1 -DPORTABLE=1 -DTOOLS=1 -DTESTS=0 ../.. + -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDEV_BUILD=1 -DPORTABLE=1 -DTOOLS=1 -DTESTS=0 -DMXE_USE_CCACHE=0 ../.. make rm -rf install DESTDIR=. make install