diff --git a/.github/actions/install-codeql-packs/action.yml b/.github/actions/install-codeql-packs/action.yml index 2e6d5f1a2e..3a60e47bb1 100644 --- a/.github/actions/install-codeql-packs/action.yml +++ b/.github/actions/install-codeql-packs/action.yml @@ -7,6 +7,11 @@ inputs: The path to the CodeQL CLI directory. required: false + common_caches: + description: | + The path to the CodeQL common caches directory. + required: false + mode: description: | The `--mode` option to `codeql pack install`. @@ -22,4 +27,6 @@ runs: CODEQL_CLI: ${{ inputs.cli_path }} run: | PATH=$PATH:$CODEQL_CLI - python scripts/install-packs.py --mode ${{ inputs.mode }} + python scripts/install-packs.py \ + --mode ${{ inputs.mode }} \ + --common-caches "${{ inputs.common_caches }}" diff --git a/.github/workflows/code-scanning-pack-gen.yml b/.github/workflows/code-scanning-pack-gen.yml index 295ef36f4c..a880157d8f 100644 --- a/.github/workflows/code-scanning-pack-gen.yml +++ b/.github/workflows/code-scanning-pack-gen.yml @@ -20,7 +20,6 @@ env: XARGS_MAX_PROCS: 4 jobs: - prepare-code-scanning-pack-matrix: name: Prepare CodeQL Code Scanning pack matrix runs-on: ubuntu-22.04 @@ -62,10 +61,19 @@ jobs: codeql-home: ${{ github.workspace }}/codeql_home add-to-path: false + - name: Cache queries compilation + id: cache-queries-compilation + uses: actions/cache/restore@v6 + with: + path: ${{ github.workspace }}/codeql_cache + key: ${{ runner.os }}-codeql-compilation-code-scanning-pack-${{ matrix.codeql_cli }}-${{ matrix.codeql_standard_library }}-${{ github.run_id }} + restore-keys: ${{ runner.os }}-codeql-compilation-code-scanning-pack-${{ matrix.codeql_cli }}-${{ matrix.codeql_standard_library }} + - name: Install CodeQL packs uses: ./.github/actions/install-codeql-packs with: cli_path: ${{ github.workspace }}/codeql_home/codeql + common_caches: ${{ github.workspace }}/codeql_cache - name: Determine ref for external help files id: determine-ref @@ -105,11 +113,22 @@ jobs: PATH=$PATH:$CODEQL_HOME/codeql # Precompile all queries, and use a compilation cache larger than default # to ensure we cache all the queries for later steps - codeql query compile --precompile --threads 0 --compilation-cache-size=1024 cpp c + codeql query compile \ + --common-caches=${{ github.workspace }}/codeql_cache \ + --precompile \ + --threads 0 \ + --compilation-cache-size=1024 \ + cpp c cd .. zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/configuration codeql-coding-standards/scripts/reports codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/guideline_recategorization codeql-coding-standards/schemas + - name: Save queries compilation cache + uses: actions/cache/save@v6 + with: + path: ${{ github.workspace }}/codeql_cache + key: ${{ steps.cache-queries-compilation.outputs.cache-primary-key }} + - name: Upload GHAS Query Pack uses: actions/upload-artifact@v7 with: @@ -119,20 +138,21 @@ jobs: - name: Create qlpack bundles env: CODEQL_HOME: ${{ github.workspace }}/codeql_home + CODEQL_CACHE: ${{ github.workspace }}/codeql_cache run: | PATH=$PATH:$CODEQL_HOME/codeql - codeql pack bundle --output=common-cpp-coding-standards.tgz cpp/common/src - codeql pack bundle --output=common-c-coding-standards.tgz c/common/src - codeql pack bundle --output=misra-c-coding-standards.tgz c/misra/src - codeql pack bundle --output=cert-c-coding-standards.tgz c/cert/src - codeql pack bundle --output=cert-cpp-coding-standards.tgz cpp/cert/src - codeql pack bundle --output=autosar-cpp-coding-standards.tgz cpp/autosar/src - codeql pack bundle --output=misra-cpp-coding-standards.tgz cpp/misra/src - codeql pack bundle --output=report-coding-standards.tgz cpp/report/src + codeql pack bundle --common-caches="$CODEQL_CACHE" --output=common-cpp-coding-standards.tgz cpp/common/src + codeql pack bundle --common-caches="$CODEQL_CACHE" --output=common-c-coding-standards.tgz c/common/src + codeql pack bundle --common-caches="$CODEQL_CACHE" --output=misra-c-coding-standards.tgz c/misra/src + codeql pack bundle --common-caches="$CODEQL_CACHE" --output=cert-c-coding-standards.tgz c/cert/src + codeql pack bundle --common-caches="$CODEQL_CACHE" --output=cert-cpp-coding-standards.tgz cpp/cert/src + codeql pack bundle --common-caches="$CODEQL_CACHE" --output=autosar-cpp-coding-standards.tgz cpp/autosar/src + codeql pack bundle --common-caches="$CODEQL_CACHE" --output=misra-cpp-coding-standards.tgz cpp/misra/src + codeql pack bundle --common-caches="$CODEQL_CACHE" --output=report-coding-standards.tgz cpp/report/src - name: Upload qlpack bundles uses: actions/upload-artifact@v7 with: name: coding-standards-codeql-packs - path: '*-coding-standards.tgz' \ No newline at end of file + path: "*-coding-standards.tgz" diff --git a/.github/workflows/codeql_unit_tests.yml b/.github/workflows/codeql_unit_tests.yml index 23d6972a13..7b9966a7e7 100644 --- a/.github/workflows/codeql_unit_tests.yml +++ b/.github/workflows/codeql_unit_tests.yml @@ -17,7 +17,6 @@ on: - "rc/**" jobs: - prepare-unit-test-matrix: name: Prepare CodeQL unit test matrix runs-on: ubuntu-22.04 @@ -59,7 +58,7 @@ jobs: - name: Cache CodeQL id: cache-codeql - uses: actions/cache@v5 + uses: actions/cache@v6 with: # A list of files, directories, and wildcard patterns to cache and restore path: ${{github.workspace}}/codeql_home @@ -75,16 +74,34 @@ jobs: codeql-home: ${{ github.workspace }}/codeql_home add-to-path: false + - name: Cache queries compilation + id: cache-queries-compilation + uses: actions/cache/restore@v6 + with: + path: ${{ github.workspace }}/codeql_cache + key: ${{ runner.os }}-codeql-compilation-${{ matrix.language }}-${{ matrix.codeql_cli }}-${{ matrix.codeql_standard_library_ident }}-${{ github.run_id }} + restore-keys: ${{ runner.os }}-codeql-compilation-${{ matrix.language }}-${{ matrix.codeql_cli }}-${{ matrix.codeql_standard_library_ident }} + - name: Install CodeQL packs uses: ./.github/actions/install-codeql-packs with: cli_path: ${{ github.workspace }}/codeql_home/codeql + common_caches: ${{ github.workspace }}/codeql_cache - name: Pre-Compile Queries id: pre-compile-queries run: | - ${{ github.workspace }}/codeql_home/codeql/codeql query compile --threads 0 ${{ matrix.language }} - + ${{ github.workspace }}/codeql_home/codeql/codeql query compile \ + --common-caches=${{ github.workspace }}/codeql_cache \ + --compilation-cache-size=1024 \ + --threads 0 \ + ${{ matrix.language }} + + - name: Save queries compilation cache + uses: actions/cache/save@v6 + with: + path: ${{ github.workspace }}/codeql_cache + key: ${{ steps.cache-queries-compilation.outputs.cache-primary-key }} - name: Run test suites id: run-test-suites @@ -135,7 +152,7 @@ jobs: os.makedirs(os.path.dirname(test_report_path), exist_ok=True) test_report_file = open(test_report_path, 'w') files_to_close.append(test_report_file) - procs.append(subprocess.Popen([codeql_bin, "test", "run", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", *test_roots], stdout=test_report_file, stderr=subprocess.PIPE)) + procs.append(subprocess.Popen([codeql_bin, "test", "run", "--common-caches=${{ github.workspace }}/codeql_cache", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", *test_roots], stdout=test_report_file, stderr=subprocess.PIPE)) for p in procs: _, err = p.communicate() diff --git a/scripts/install-packs.py b/scripts/install-packs.py index ab45c32818..f9a703e1d3 100644 --- a/scripts/install-packs.py +++ b/scripts/install-packs.py @@ -6,6 +6,7 @@ parser = argparse.ArgumentParser(description="Install CodeQL library pack dependencies.") parser.add_argument('--mode', required=False, choices=['use-lock', 'update', 'verify', 'no-lock'], default="use-lock", help="Installation mode, identical to the `--mode` argument to `codeql pack install`") parser.add_argument('--codeql', required=False, default='codeql', help="Path to the `codeql` executable.") +parser.add_argument('--common-caches', required=False, help="Path to the CodeQL common caches directory.") args = parser.parse_args() # Find the root of the repo @@ -19,5 +20,7 @@ pack_path = os.path.join(root, pack) # Run `codeql pack install` to install dependencies. command = [args.codeql, 'pack', 'install', '--allow-prerelease', '--mode', args.mode, pack_path] + if args.common_caches: + command.insert(3, f'--common-caches={args.common_caches}') print(f'Running `{" ".join(command)}`') subprocess.check_call(command)