CI: Update Dockerfile, Deps, CMake Profiling & SBOM

.github/ci/docker/iccdev-banner.sh

@@ -94,7 +94,9 @@ if command -v bash >/dev/null 2>&1; then
 fi
 
 qa_tools=""
-for tool in clang-tidy cppcheck scan-build hadolint zizmor valgrind gdb lcov gcovr shellcheck afl-fuzz; do
+for tool in clang-tidy cppcheck scan-build hadolint zizmor valgrind gdb \
+            lcov genhtml gcovr llvm-cov llvm-profdata perf gprof strace \
+            shellcheck afl-fuzz; do
   if command -v "$tool" >/dev/null 2>&1; then
     qa_tools="${qa_tools}${qa_tools:+ }$tool"
   fi
@@ -104,6 +106,14 @@ if [ -n "$qa_tools" ]; then
   printf 'Maintainer QA tools on PATH: %s\n' "$qa_tools"
 fi
 
+if [ -n "${ICCDEV_FLAMEGRAPH_DIR:-}" ] &&
+   [ -x "$ICCDEV_FLAMEGRAPH_DIR/stackcollapse-perf.pl" ] &&
+   [ -x "$ICCDEV_FLAMEGRAPH_DIR/flamegraph.pl" ]; then
+  printf '%s\n' ''
+  printf 'FlameGraph scripts: %s (%s)\n' \
+    "$ICCDEV_FLAMEGRAPH_DIR" "${ICCDEV_FLAMEGRAPH_REVISION:-unknown revision}"
+fi
+
 if command -v iccdev-fuzz-env >/dev/null 2>&1; then
   printf '%s\n' ''
   printf '%s\n' 'AFL/CFL helper: iccdev-fuzz-env'

.github/prompts/regression-container-maintainer.prompt.md

@@ -20,6 +20,7 @@ Canonical guide: `docs/regression-container.md`
 - Expected failure signal:
 - Evidence directory:
 - AFL/CFL scope: none / AFL smoke / CFL smoke / both
+- Instrumentation scope: none / coverage / gprof / perf / FlameGraph
 
 ## Required Workflow
 
@@ -41,6 +42,9 @@ Canonical guide: `docs/regression-container.md`
 8. If the changed behavior is in an excluded suite, run its focused CTest in
    addition to the local container envelope.
 9. Scan output for compiler warnings, ASAN, UBSAN, and signal termination.
+   For instrumentation work, use a separate build tree, inventory the shipped
+   coverage/profiling commands and pinned FlameGraph revision, and run
+   `iccdev.profiling-smoke` for `ENABLE_PROFILING=ON` builds.
 10. Classify exit `1-127` as graceful and `128+` as signal termination.
 11. For AFL/CFL work, run `iccdev-fuzz-env`, verify patch-stack counts, and run
     the smallest patched or unpatched smoke that matches the requested scope.

.github/scripts/iccdev-clut-profile.sh

@@ -19,6 +19,7 @@ output_dir="$2"
 runs="${ICCDEV_CLUT_PROFILE_RUNS:-21}"
 affinity="${ICCDEV_CLUT_PROFILE_AFFINITY:-}"
 enable_flamegraph="${ICCDEV_CLUT_PROFILE_FLAMEGRAPH:-0}"
+flamegraph_dir="${ICCDEV_FLAMEGRAPH_DIR:-${FLAMEGRAPH_DIR:-}}"
 
 case "$runs" in
     ''|*[!0-9]*|0)
@@ -273,11 +274,11 @@ if [ "$enable_flamegraph" = "1" ] && [ "$perf_available" -eq 1 ]; then
     perf record -o "$output_dir/perf.data" -g --call-graph dwarf -- \
         "${runner[@]}" "${ctest_command[@]}" > "$output_dir/perf-record.log" 2>&1
     perf script -i "$output_dir/perf.data" > "$output_dir/perf.script"
-    if [ -n "${FLAMEGRAPH_DIR:-}" ] &&
-       [ -x "$FLAMEGRAPH_DIR/stackcollapse-perf.pl" ] &&
-       [ -x "$FLAMEGRAPH_DIR/flamegraph.pl" ]; then
-        "$FLAMEGRAPH_DIR/stackcollapse-perf.pl" "$output_dir/perf.script" |
-            "$FLAMEGRAPH_DIR/flamegraph.pl" > "$output_dir/flamegraph.svg"
+    if [ -n "$flamegraph_dir" ] &&
+       [ -x "$flamegraph_dir/stackcollapse-perf.pl" ] &&
+       [ -x "$flamegraph_dir/flamegraph.pl" ]; then
+        "$flamegraph_dir/stackcollapse-perf.pl" "$output_dir/perf.script" |
+            "$flamegraph_dir/flamegraph.pl" > "$output_dir/flamegraph.svg"
     fi
 fi
 

.github/skills/avx2-clut-diagnostics/SKILL.md

@@ -77,10 +77,13 @@ hardware counters, syscall summary, and optional FlameGraph inputs:
 ICCDEV_CLUT_PROFILE_AFFINITY=0 \
 ICCDEV_CLUT_PROFILE_RUNS=21 \
 ICCDEV_CLUT_PROFILE_FLAMEGRAPH=1 \
-FLAMEGRAPH_DIR=/path/to/FlameGraph \
+ICCDEV_FLAMEGRAPH_DIR=/path/to/FlameGraph \
 ./.github/scripts/iccdev-clut-profile.sh out/clut-avx2 out/clut-profile
 ```
 
+Inside the unified container, use its preconfigured pinned
+`ICCDEV_FLAMEGRAPH_DIR` without overriding it.
+
 The script preserves a telemetry file per measured CTest run. Do not enable
 `ICC_AVX2_CLUT_DEBUG`, coverage, or strace while comparing cycle counts.
 

.github/skills/regression-container-maintainer/SKILL.md

@@ -55,16 +55,21 @@ Use this skill for repeatable maintainer operations in
     `.github/ci/cfl/build.sh --patches`) before broader validation.
 11. Run broader CTest or GCC 15.2 strict parity only after the focused check
     passes.
-12. Scan logs for compiler warnings, ASAN, UBSAN, and signal termination.
-13. Trigger `ci-pr-action.yml` explicitly for a pre-PR branch; a push alone does
+12. For coverage or profiling work, use a separate build tree and inventory
+    `lcov`, `genhtml`, `gcovr`, `llvm-cov`, `llvm-profdata`, `gprof`, `perf`,
+    `strace`, and the pinned `$ICCDEV_FLAMEGRAPH_DIR`. Run
+    `iccdev.profiling-smoke` for `ENABLE_PROFILING=ON` builds. Treat unavailable
+    hardware counters as environment evidence, not a correctness failure.
+13. Scan logs for compiler warnings, ASAN, UBSAN, and signal termination.
+14. Trigger `ci-pr-action.yml` explicitly for a pre-PR branch; a push alone does
     not trigger that workflow.
-14. Use only `latest`, `ci-qa-pr-docker-testing`, `ci-publish-colourbill-ctrl`,
+15. Use only `latest`, `ci-qa-pr-docker-testing`, `ci-publish-colourbill-ctrl`,
     full-SHA, or release tags as selectors. Resolve the selector at run time; never hardcode one SHA tag as
     a long-lived workflow default. Existing legacy tags are continuity-only; do
     not introduce, recommend, or depend on other branch, run, or image-variant
     tags.
-15. Confirm the canonical image digest and hosted validation explicitly.
-16. Report exact image tag, digest, source revision, commands, results, evidence,
+16. Confirm the canonical image digest and hosted validation explicitly.
+17. Report exact image tag, digest, source revision, commands, results, evidence,
     and workflow URLs.
 
 For container/MCP runtime changes, run

.github/workflows/_build-matrix.yml

@@ -129,6 +129,8 @@ jobs:
         include:
           - name: coverage
             options: "-DENABLE_COVERAGE=ON"
+          - name: profiling
+            options: "-DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DENABLE_PROFILING=ON -DENABLE_TESTS=ON"
           - name: asserts
             options: "-DICC_ENABLE_ASSERTS=ON"
           - name: trace-nan
@@ -163,11 +165,15 @@ jobs:
           source .trusted-workflow/.github/scripts/sanitize-sed.sh
           sudo apt-get update -qq
           sudo apt-get install -y --no-install-recommends \
-            cmake clang libjpeg-dev libpng-dev libtiff-dev libxml2-dev nlohmann-json3-dev
+            binutils cmake clang libjpeg-dev libpng-dev libtiff-dev libxml2-dev nlohmann-json3-dev
           read -r -a cmake_options <<< "$MATRIX_OPTIONS"
           cmake -S Build/Cmake -B "out/option-${MATRIX_NAME}" \
             -DCMAKE_BUILD_TYPE=Release -DENABLE_TOOLS=ON "${cmake_options[@]}"
           cmake --build "out/option-${MATRIX_NAME}" --parallel "$(nproc)"
+          if [ "$MATRIX_NAME" = profiling ]; then
+            ctest --test-dir "out/option-${MATRIX_NAME}" \
+              -R '^iccdev\.profiling-smoke$' --output-on-failure --no-tests=error
+          fi
 
   windows-toolchains:
     if: inputs['include-windows']

.github/workflows/ci-docker.yml

@@ -159,6 +159,8 @@ jobs:
         id: test-image
         env:
           BASH_ENV: /dev/null
+          EXPECTED_SHA: ${{ github.sha }}
+          IMAGE_NAME: ${{ steps.image.outputs.name }}
           IMAGE_TAGS: ${{ steps.meta.outputs.tags }}
         run: |
           set -euo pipefail
@@ -171,7 +173,13 @@ jobs:
             echo "No image tag was generated for validation" >&2
             exit 1
           fi
+          staging_image="${IMAGE_NAME}:sha-${EXPECTED_SHA}"
+          if ! grep -Fqx "$staging_image" <<< "$IMAGE_TAGS"; then
+            echo "Immutable SHA tag was not generated for staging" >&2
+            exit 1
+          fi
           printf 'image=%s\n' "$(sanitize_line "$test_image")" >> "$GITHUB_OUTPUT"
+          printf 'staging_image=%s\n' "$(sanitize_line "$staging_image")" >> "$GITHUB_OUTPUT"
 
       - name: Verify unified image
         env:
@@ -194,9 +202,16 @@ jobs:
             set -euo pipefail
             command -v git gh clang clang++ gcc g++ cmake cppcheck clang-tidy scan-build
             command -v hadolint zizmor shellcheck afl-fuzz valgrind ms_print callgrind_annotate llvm-symbolizer
+            command -v lcov genhtml gcovr llvm-cov llvm-profdata perf gprof strace
+            perl -MJSON::XS -e 1
+            test "$LCOV_HOME" = /
+            grep -Fqx "geninfo_unexecuted_blocks = 1" /etc/lcovrc
             command -v iccDumpProfile iccdev-fuzz-env iccdev-mcp iccdev-mcp-rest
             command -v iccdev-valgrind-build iccdev-valgrind-run
             command -v iccdev-valgrind-status iccdev-valgrind-validate iccdev-valgrind-self-test
+            test -x "$ICCDEV_FLAMEGRAPH_DIR/stackcollapse-perf.pl"
+            test -x "$ICCDEV_FLAMEGRAPH_DIR/flamegraph.pl"
+            test "$(git -C "$ICCDEV_FLAMEGRAPH_DIR" rev-parse HEAD)" = "$ICCDEV_FLAMEGRAPH_REVISION"
             test "$PWD" = /workspace/iccDEV
             test -d .git
             test -d "$ICCDEV_BUILD_DIR"
@@ -251,53 +266,22 @@ jobs:
           retention-days: 7
           if-no-files-found: error
 
-      - name: Publish validated image
-        if: ${{ env.PUBLISH_IMAGE == 'true' }}
-        id: publish
-        env:
-          BASH_ENV: /dev/null
-          IMAGE_TAGS: ${{ steps.meta.outputs.tags }}
-          TEST_IMAGE: ${{ steps.test-image.outputs.image }}
-        run: |
-          set -euo pipefail
-          git config --global credential.helper ""
-          unset GITHUB_TOKEN || true
-          source .github/scripts/sanitize-sed.sh
-
-          pushed=0
-          while IFS= read -r image_tag; do
-            [ -n "$image_tag" ] || continue
-            docker push "$image_tag"
-            pushed=$((pushed + 1))
-          done <<< "$IMAGE_TAGS"
-          if [ "$pushed" -eq 0 ]; then
-            echo "No image tags were available to publish" >&2
-            exit 1
-          fi
-
-          image_digest_ref="$(docker image inspect "$TEST_IMAGE" --format '{{index .RepoDigests 0}}')"
-          image_digest="${image_digest_ref##*@}"
-          if [[ ! "$image_digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then
-            echo "Unable to resolve published image digest" >&2
-            exit 1
-          fi
-          printf 'digest=%s\n' "$(sanitize_line "$image_digest")" >> "$GITHUB_OUTPUT"
-
       - name: Generate SBOM
         if: ${{ env.PUBLISH_IMAGE == 'true' }}
         uses: anchore/sbom-action@3ad7283483fc7af8ff2b4ea19663c2d5ca935e26  # v0.24.2
+        env:
+          SYFT_FILE_METADATA_SELECTION: none
         with:
-          image: ${{ steps.image.outputs.name }}@${{ steps.publish.outputs.digest }}
-          format: spdx-json
-          output-file: sbom.spdx.json
+          image: ${{ steps.test-image.outputs.staging_image }}
+          format: cyclonedx-json
+          output-file: sbom.cyclonedx.json
 
-      - name: Measure SBOM attestation size
+      - name: Validate SBOM attestation size
         if: ${{ env.PUBLISH_IMAGE == 'true' }}
-        id: sbom-size
         env:
           BASH_ENV: /dev/null
           MAX_ATTEST_BYTES: "16777216"
-          SBOM_PATH: sbom.spdx.json
+          SBOM_PATH: sbom.cyclonedx.json
         run: |
           set -euo pipefail
           git config --global credential.helper ""
@@ -309,6 +293,15 @@ jobs:
             exit 1
           fi
 
+          if ! jq -e '
+            .bomFormat == "CycloneDX" and
+            (.specVersion | type == "string" and length > 0) and
+            (.components | type == "array" and length > 0)
+          ' "$SBOM_PATH" >/dev/null; then
+            echo "::error::Invalid or empty CycloneDX SBOM: $SBOM_PATH"
+            exit 1
+          fi
+
           sbom_bytes="$(wc -c < "$SBOM_PATH" | tr -d ' ')"
           case "$sbom_bytes" in
             ''|*[!0-9]*)
@@ -317,12 +310,9 @@ jobs:
               ;;
           esac
 
-          echo "bytes=$sbom_bytes" >> "$GITHUB_OUTPUT"  # elements-sanitized
-          if [ "$sbom_bytes" -le "$MAX_ATTEST_BYTES" ]; then
-            echo "attestable=true" >> "$GITHUB_OUTPUT"  # elements-sanitized
-          else
-            echo "attestable=false" >> "$GITHUB_OUTPUT"  # elements-sanitized
-            echo "::notice title=SBOM Attestation Skipped::SBOM is ${sbom_bytes} bytes, above ${MAX_ATTEST_BYTES}"
+          if [ "$sbom_bytes" -gt "$MAX_ATTEST_BYTES" ]; then
+            echo "::error title=SBOM Attestation Too Large::SBOM is ${sbom_bytes} bytes, above ${MAX_ATTEST_BYTES}"
+            exit 1
           fi
 
           {
@@ -331,25 +321,47 @@ jobs:
             echo "- Limit: $(sanitize_line "$MAX_ATTEST_BYTES")"
           } >> "$GITHUB_STEP_SUMMARY"  # elements-sanitized
 
+      - name: Stage validated image by immutable SHA
+        if: ${{ env.PUBLISH_IMAGE == 'true' }}
+        id: stage
+        env:
+          BASH_ENV: /dev/null
+          STAGING_IMAGE: ${{ steps.test-image.outputs.staging_image }}
+        run: |
+          set -euo pipefail
+          git config --global credential.helper ""
+          unset GITHUB_TOKEN || true
+          source .github/scripts/sanitize-sed.sh
+
+          docker push "$STAGING_IMAGE"
+
+          image_digest_ref="$(docker image inspect "$STAGING_IMAGE" --format '{{index .RepoDigests 0}}')"
+          image_digest="${image_digest_ref##*@}"
+          if [[ ! "$image_digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then
+            echo "Unable to resolve staged image digest" >&2
+            exit 1
+          fi
+          printf 'digest=%s\n' "$(sanitize_line "$image_digest")" >> "$GITHUB_OUTPUT"
+
       - name: Attest build provenance
         if: ${{ env.PUBLISH_IMAGE == 'true' }}
         uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8  # v4.2.2
         with:
           subject-name: ${{ steps.image.outputs.name }}
-          subject-digest: ${{ steps.publish.outputs.digest }}
+          subject-digest: ${{ steps.stage.outputs.digest }}
           push-to-registry: true
 
       - name: Attest SBOM
-        if: ${{ env.PUBLISH_IMAGE == 'true' && steps.sbom-size.outputs.attestable == 'true' }}
+        if: ${{ env.PUBLISH_IMAGE == 'true' }}
         id: attest-sbom
-        # Rekor's transparency-log endpoint can time out after image publication.
+        # Rekor's transparency-log endpoint can time out after immutable staging.
         # Retrying preserves a hard failure when the service remains unavailable.
         continue-on-error: true
         uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6  # v4.2.2
         with:
           subject-name: ${{ steps.image.outputs.name }}
-          subject-digest: ${{ steps.publish.outputs.digest }}
-          sbom-path: sbom.spdx.json
+          subject-digest: ${{ steps.stage.outputs.digest }}
+          sbom-path: sbom.cyclonedx.json
           push-to-registry: true
 
       - name: Retry SBOM attestation
@@ -358,8 +370,8 @@ jobs:
         uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6  # v4.2.2
         with:
           subject-name: ${{ steps.image.outputs.name }}
-          subject-digest: ${{ steps.publish.outputs.digest }}
-          sbom-path: sbom.spdx.json
+          subject-digest: ${{ steps.stage.outputs.digest }}
+          sbom-path: sbom.cyclonedx.json
           push-to-registry: true
 
       - name: Require SBOM attestation
@@ -377,11 +389,33 @@ jobs:
             exit 1
           fi
 
+      - name: Publish attested image tags
+        if: ${{ env.PUBLISH_IMAGE == 'true' }}
+        env:
+          BASH_ENV: /dev/null
+          IMAGE_TAGS: ${{ steps.meta.outputs.tags }}
+        run: |
+          set -euo pipefail
+          git config --global credential.helper ""
+          unset GITHUB_TOKEN || true
+          source .github/scripts/sanitize-sed.sh
+
+          pushed=0
+          while IFS= read -r image_tag; do
+            [ -n "$image_tag" ] || continue
+            docker push "$image_tag"
+            pushed=$((pushed + 1))
+          done <<< "$IMAGE_TAGS"
+          if [ "$pushed" -eq 0 ]; then
+            echo "No image tags were available to publish" >&2
+            exit 1
+          fi
+
       - name: Upload SBOM
         if: ${{ env.PUBLISH_IMAGE == 'true' }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
         with:
           name: iccdev-sbom
-          path: sbom.spdx.json
+          path: sbom.cyclonedx.json
           retention-days: 90
           if-no-files-found: error

Build/Cmake/CMakeLists.txt

@@ -1388,10 +1388,14 @@ endif()
 # Apply profiling instrumentation flags
 if(ENABLE_PROFILING)
   message(STATUS ">>> Profiling instrumentation enabled (-pg)")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pg -fno-omit-frame-pointer")
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pg -fno-omit-frame-pointer")
+  add_compile_options(-pg -fno-omit-frame-pointer)
+  # Executables need the profiling startup/runtime objects selected by -pg.
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pg")
-  set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -pg")
+  # GCC accepts -pg for shared links. Clang does not need it there and emits an
+  # unused-command-line-argument warning when it is passed to a shared link.
+  if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -pg")
+  endif()
 endif()
 
 # ----------------------------------------------------------------------
@@ -1414,7 +1418,9 @@ include(CheckCXXCompilerFlag)
 # aarch64 Clang 11) are correctly rejected rather than added to the build.
 # VAR is a unique cache variable name used to memoize the probe result.
 macro(_iccdev_add_cxx_flag_if_supported FLAG VAR)
-  set(CMAKE_REQUIRED_FLAGS "-Werror=unused-command-line-argument")
+  if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+    set(CMAKE_REQUIRED_FLAGS "-Werror=unused-command-line-argument")
+  endif()
   check_cxx_compiler_flag("${FLAG}" ${VAR})
   unset(CMAKE_REQUIRED_FLAGS)
   if(${VAR})