diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3f82880437..aae5fee0e7 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -157,6 +157,8 @@ jobs: script: ci/build_wheel_cuopt.sh package-name: cuopt package-type: python + # Build a wheel for each CUDA x ARCH x minimum supported Python version + matrix_filter: group_by({CUDA_VER, ARCH}) | map(min_by(.PY_VER | split(".") | map(tonumber))) wheel-publish-cuopt: needs: wheel-build-cuopt permissions: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d54afeebd4..3d2ba4b3f5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -48,6 +48,7 @@ jobs: runs-on: ubuntu-latest outputs: libcuopt_filter: ${{ steps.set-filters.outputs.libcuopt_filter }} + cuopt_filter: ${{ steps.set-filters.outputs.cuopt_filter }} cuopt_server_filter: ${{ steps.set-filters.outputs.cuopt_server_filter }} cuopt_server_test_filter: ${{ steps.set-filters.outputs.cuopt_server_test_filter }} cuopt_sh_client_filter: ${{ steps.set-filters.outputs.cuopt_sh_client_filter }} @@ -56,6 +57,7 @@ jobs: id: set-filters run: | echo "libcuopt_filter=group_by([.ARCH, (.CUDA_VER|split(\".\")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(\".\")|map(tonumber)))" >> $GITHUB_OUTPUT + echo "cuopt_filter=group_by({CUDA_VER, ARCH}) | map(min_by(.PY_VER | split(\".\") | map(tonumber)))" >> $GITHUB_OUTPUT echo "cuopt_server_filter=map(select(.ARCH == \"amd64\")) | group_by(.CUDA_VER|split(\".\")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(\".\")|map(tonumber)), (.CUDA_VER|split(\".\")|map(tonumber))]))" >> $GITHUB_OUTPUT echo "cuopt_server_test_filter=map(select(.ARCH == \"amd64\")) | group_by(.CUDA_VER | split(\".\") | map(tonumber) | .[0]) | map(max_by([(.PY_VER | split(\".\") | map(tonumber)), (.CUDA_VER | split(\".\") | map(tonumber))]))" >> $GITHUB_OUTPUT echo "cuopt_sh_client_filter=[map(select(.ARCH == \"amd64\")) | min_by((.PY_VER | split(\".\") | map(tonumber)), (.CUDA_VER | split(\".\") | map(-tonumber)))]" >> $GITHUB_OUTPUT @@ -479,6 +481,8 @@ jobs: script: ci/build_wheel_cuopt.sh package-name: cuopt package-type: python + # Build a wheel for each CUDA x ARCH x minimum supported Python version + matrix_filter: ${{ needs.compute-matrix-filters.outputs.cuopt_filter }} wheel-tests-cuopt: needs: [wheel-build-cuopt, wheel-build-cuopt-sh-client, changed-files] permissions: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 53ff2ce9dc..f23b0cf181 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -215,6 +215,9 @@ To build all libraries and tests, simply run ``` - **Note**: if Cython files (`*.pyx` or `*.pxd`) have changed, the Python build must be rerun. +- **Note**: the `cuopt` wheel is built against the CPython Limited API (abi3), so Cython code must + use only APIs the Limited API exposes. An unsupported API fails to compile under + `-DPy_LIMITED_API` rather than failing at runtime. To run the C++ tests, run diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 8cac0e7714..92ec1cc2f2 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -7,6 +7,22 @@ set -euo pipefail package_name=$1 package_dir=$2 +shift 2 + +# Parse optional flags +stable_abi=false +while [[ $# -gt 0 ]]; do + case "$1" in + --stable) + stable_abi=true + shift + ;; + *) + echo "Unknown option: $1" >&2 + exit 1 + ;; + esac +done source rapids-configure-sccache source rapids-datetime-string @@ -29,6 +45,11 @@ RAPIDS_PIP_WHEEL_ARGS=( --disable-pip-version-check ) +# Add py-api setting for stable ABI builds +if [[ "${stable_abi}" == "true" ]] && [[ -n "${RAPIDS_PY_API:-}" ]]; then + RAPIDS_PIP_WHEEL_ARGS+=(--config-settings="skbuild.wheel.py-api=${RAPIDS_PY_API}") +fi + # Only use --build-constraint when build isolation is enabled. # # Passing '--build-constraint' and '--no-build-isolation` together results in an error from 'pip', diff --git a/ci/build_wheel_cuopt.sh b/ci/build_wheel_cuopt.sh index f624b27705..a41f8439af 100755 --- a/ci/build_wheel_cuopt.sh +++ b/ci/build_wheel_cuopt.sh @@ -45,12 +45,17 @@ EXCLUDE_ARGS=( --exclude "librmm.so" ) -ci/build_wheel.sh cuopt ${package_dir} +# TODO: move this variable into `ci-wheel` +# Format Python limited API version string +RAPIDS_PY_API="cp${RAPIDS_PY_VERSION//./}" +export RAPIDS_PY_API + +ci/build_wheel.sh cuopt ${package_dir} --stable # repair wheels and write to the location that artifact-uploading code expects to find them python -m auditwheel repair "${EXCLUDE_ARGS[@]}" -w ${RAPIDS_WHEEL_BLD_OUTPUT_DIR} ${package_dir}/dist/* ci/validate_wheel.sh "${package_dir}" "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" -RAPIDS_PACKAGE_NAME="$(rapids-artifact-name wheel_python cuopt cuopt --py "$RAPIDS_PY_VERSION" --cuda "$RAPIDS_CUDA_VERSION")" +RAPIDS_PACKAGE_NAME="$(rapids-artifact-name wheel_python cuopt cuopt --stable --cuda "$RAPIDS_CUDA_VERSION")" export RAPIDS_PACKAGE_NAME diff --git a/ci/test_self_hosted_service.sh b/ci/test_self_hosted_service.sh index 63776a4f6c..07774bf200 100755 --- a/ci/test_self_hosted_service.sh +++ b/ci/test_self_hosted_service.sh @@ -9,7 +9,7 @@ source rapids-init-pip # Download the cuopt built in the previous step LIBCUOPT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_cpp libcuopt cuopt --cuda "$RAPIDS_CUDA_VERSION")") -CUOPT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_python cuopt cuopt --py "$RAPIDS_PY_VERSION" --cuda "$RAPIDS_CUDA_VERSION")") +CUOPT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_python cuopt cuopt --stable --cuda "$RAPIDS_CUDA_VERSION")") CUOPT_SERVER_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_python cuopt-server cuopt --pure --arch any --cuda "$RAPIDS_CUDA_VERSION")") # generate constraints (possibly pinning to oldest support versions of dependencies) diff --git a/ci/test_wheel_cuopt.sh b/ci/test_wheel_cuopt.sh index 12c8d2396b..86c1cbc565 100755 --- a/ci/test_wheel_cuopt.sh +++ b/ci/test_wheel_cuopt.sh @@ -17,7 +17,7 @@ bash "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/utils/install_openssl3_runtim # Download the packages built in the previous step RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" LIBCUOPT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_cpp libcuopt cuopt --cuda "$RAPIDS_CUDA_VERSION")") -CUOPT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_python cuopt cuopt --py "$RAPIDS_PY_VERSION" --cuda "$RAPIDS_CUDA_VERSION")") +CUOPT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_python cuopt cuopt --stable --cuda "$RAPIDS_CUDA_VERSION")") CUOPT_SH_CLIENT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_python cuopt-sh-client cuopt --pure --arch any)") # update pip constraints.txt to ensure all future 'pip install' (including those in ci/thirdparty-testing) diff --git a/ci/test_wheel_cuopt_server.sh b/ci/test_wheel_cuopt_server.sh index df1749a3b5..0f6d6fd19e 100755 --- a/ci/test_wheel_cuopt_server.sh +++ b/ci/test_wheel_cuopt_server.sh @@ -14,7 +14,7 @@ bash "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/utils/install_openssl3_runtim # Download the packages built in the previous step LIBCUOPT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_cpp libcuopt cuopt --cuda "$RAPIDS_CUDA_VERSION")") -CUOPT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_python cuopt cuopt --py "$RAPIDS_PY_VERSION" --cuda "$RAPIDS_CUDA_VERSION")") +CUOPT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_python cuopt cuopt --stable --cuda "$RAPIDS_CUDA_VERSION")") CUOPT_SERVER_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_python cuopt-server cuopt --pure --arch any --cuda "$RAPIDS_CUDA_VERSION")") CUOPT_SH_CLIENT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_python cuopt-sh-client cuopt --pure --arch any)") diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh index a603c69098..685d4b920d 100755 --- a/ci/validate_wheel.sh +++ b/ci/validate_wheel.sh @@ -46,3 +46,13 @@ rapids-logger "validate packages with 'twine'" twine check \ --strict \ "$(echo "${wheel_dir_relative_path}"/*.whl)" + +rapids-logger "validate packages with 'abi3audit'" + +# 'abi3audit' fails on wheels with DSOs that lack an ABI tag (e.g. 'lib*' wheels). +# Filtering by '*abi*' avoids those. +find \ + "${wheel_dir_relative_path}" \ + -type f \ + -name '*abi*' \ + -exec abi3audit --strict --summary --verbose '{}' \+ diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index 64a6ff58db..5f0be5141b 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -21,7 +21,7 @@ dependencies: - cudf==26.10.*,>=0.0.0a0 - cupy>=14.0.1,!=14.1.0 - cxx-compiler -- cython>=3.0.3 +- cython>=3.2.2,<3.3.0a0 - docutils>=0.21 - doxygen=1.9.1 - fastapi diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index e2d2a3b77a..cdd4c8d0c1 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -21,7 +21,7 @@ dependencies: - cudf==26.10.*,>=0.0.0a0 - cupy>=14.0.1,!=14.1.0 - cxx-compiler -- cython>=3.0.3 +- cython>=3.2.2,<3.3.0a0 - docutils>=0.21 - doxygen=1.9.1 - fastapi diff --git a/conda/environments/all_cuda-133_arch-aarch64.yaml b/conda/environments/all_cuda-133_arch-aarch64.yaml index 34e60cfb0b..0628e93e65 100644 --- a/conda/environments/all_cuda-133_arch-aarch64.yaml +++ b/conda/environments/all_cuda-133_arch-aarch64.yaml @@ -21,7 +21,7 @@ dependencies: - cudf==26.10.*,>=0.0.0a0 - cupy>=14.0.1,!=14.1.0 - cxx-compiler -- cython>=3.0.3 +- cython>=3.2.2,<3.3.0a0 - docutils>=0.21 - doxygen=1.9.1 - fastapi diff --git a/conda/environments/all_cuda-133_arch-x86_64.yaml b/conda/environments/all_cuda-133_arch-x86_64.yaml index 8f6c59293f..c05ca31248 100644 --- a/conda/environments/all_cuda-133_arch-x86_64.yaml +++ b/conda/environments/all_cuda-133_arch-x86_64.yaml @@ -21,7 +21,7 @@ dependencies: - cudf==26.10.*,>=0.0.0a0 - cupy>=14.0.1,!=14.1.0 - cxx-compiler -- cython>=3.0.3 +- cython>=3.2.2,<3.3.0a0 - docutils>=0.21 - doxygen=1.9.1 - fastapi diff --git a/dependencies.yaml b/dependencies.yaml index c9eb8d3239..ca87d3d3b8 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -303,7 +303,7 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - cython>=3.0.3 + - cython>=3.2.2,<3.3.0a0 test_python_common: common: - output_types: [conda, requirements, pyproject] diff --git a/python/cuopt/pyproject.toml b/python/cuopt/pyproject.toml index 8aa6d18398..929d066d0e 100644 --- a/python/cuopt/pyproject.toml +++ b/python/cuopt/pyproject.toml @@ -89,6 +89,7 @@ minimum-version = "build-system.requires" ninja.make-fallback = false sdist.reproducible = true wheel.packages = ["cuopt"] +wheel.py-api = "cp311" # overridden in CI builds by arguments from `ci/build_wheel_cuopt.sh` [tool.scikit-build.metadata.version] provider = "scikit_build_core.metadata.regex" @@ -102,7 +103,7 @@ matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" requires = [ "cmake>=4.0", "cupy-cuda13x[ctk]>=14.0.1,!=14.1.0", - "cython>=3.0.3", + "cython>=3.2.2,<3.3.0a0", "libcuopt==26.10.*,>=0.0.0a0", "ninja", "pylibraft==26.10.*,>=0.0.0a0", diff --git a/skills/cuopt-developer/references/contributing.md b/skills/cuopt-developer/references/contributing.md index 8ae36c831d..583cc2dc38 100644 --- a/skills/cuopt-developer/references/contributing.md +++ b/skills/cuopt-developer/references/contributing.md @@ -109,6 +109,7 @@ Follow YAGNI strictly here — flags, fallbacks, env-var overrides, and config k A few non-YAGNI points worth keeping in mind: - Prefer extending an existing script over adding a new one. +- For build/CI conventions shared across RAPIDS (wheel packaging, artifact naming, matrix filters), read the reference implementation in `rapidsai/rmm` or `NVIDIA/cudf` on `main` before writing anything. cuOpt's `ci/` scripts and shared-workflow inputs are near-copies of theirs, and gha-tools expects exact conventions — an equivalent-but-different local invention silently breaks the download side, which looks for the name the build side wrote. - Validate inputs at the top, before any expensive work. - One shell command per line over chained `&&`; no comments that restate the next line. - Keep informational CI jobs (reporting, dashboards, comment posting) out of any required-checks list.