4.0.0 cleanup

.github/workflows/test_docs.yml

@@ -0,0 +1,19 @@
+name: tests_docs
+on: [push, pull_request, workflow_dispatch]
+
+permissions:
+    contents: read
+
+jobs:
+    build-docs:
+        name: Build Sphinx documentation
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+              with:
+                persist-credentials: false
+            - uses: prefix-dev/setup-pixi@d3f436a425481402e6a95a1d1fc10331c708cd9e # v0.10.2
+              with:
+                  environments: default
+            - name: build docs
+              run: pixi run docs

.github/workflows/test_pytest.yml

@@ -4,108 +4,58 @@ on: [push, pull_request, workflow_dispatch]
 permissions:
     contents: read
 
-defaults:
-    run:
-        shell: bash -l {0}
-
 jobs:
     check_versions_matches:
-        name: Check deeptools version matches galaxy tools
+        name: Check deeptools/galaxy/Rust versions stay in sync
         runs-on: ubuntu-latest
-        if: github.base_ref == 'master'
         steps:
             - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
               with:
                 persist-credentials: false
-            - name: Check path
-              run: find /home/runner/work/deepTools/deepTools -name "pyproject.toml"
-            - name: Get Version of Deeptools
-              run: echo "deeptools_version=$(grep "version" /home/runner/work/deepTools/deepTools/pyproject.toml | awk '{print substr($NF, 2, length($NF) - 2)}')" >> $GITHUB_ENV
-            - name: Get Version of Galaxy tools
-              run: echo "galaxy_deeptools_version=$(grep "token.*TOOL_VERSION" /home/runner/work/deepTools/deepTools/galaxy/wrapper/deepTools_macros.xml | awk -F '>|<' '{print $3}')" >> $GITHUB_ENV
-            - name: Versions
-              run: |
-                  echo deeptools version=${deeptools_version}
-                  echo deeptools version in galaxy=${galaxy_deeptools_version}
-            - name: Fail PR if versions not matching
-              run: |
-                  if [ -z $galaxy_deeptools_version ]; then
-                    echo "galaxy_deeptools_version not defined change the github action"
-                    exit 1
-                  fi
-                  if [ -z $deeptools_version ]; then
-                    echo "deeptools_version not defined change the github action"
-                    exit 1
-                  fi
-                  if [ "$galaxy_deeptools_version" != "$deeptools_version" ]; then
-                    echo "versions do not match"
-                    exit 1
-                  fi
+            - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+              with:
+                  python-version: "3.x"
+            - name: Check version sync
+              run: python3 scripts/check_version_sync.py
     build-linux:
         name: Test on Linux
         runs-on: ubuntu-latest
         strategy:
             matrix:
-                python-version: ["3.12", "3.13", "3.14"]
+                pixi-env: [py312, py313, py314]
         steps:
             - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
               with:
                 persist-credentials: false
-            - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+            - uses: prefix-dev/setup-pixi@d3f436a425481402e6a95a1d1fc10331c708cd9e # v0.10.2
               with:
-                  python-version: ${{ matrix.python-version }}
-                  cache: "pip"
-            - name: Install Rust
-              uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0
+                  environments: ${{ matrix.pixi-env }}
             - name: build deeptools
-              run: |
-                  python -m venv venv
-                  source venv/bin/activate
-                  pip install maturin
-                  maturin develop --release --extras actions
+              run: pixi run -e ${{ matrix.pixi-env }} build
             - name: test deeptools
-              run: |
-                  source venv/bin/activate
-                  pytest -v
+              run: pixi run -e ${{ matrix.pixi-env }} test -v
             - name: build wheel
-              run: |
-                  source venv/bin/activate
-                  python -m build
+              run: pixi run -e ${{ matrix.pixi-env }} python -m build
             - name: test wheel
-              run: |
-                  source venv/bin/activate
-                  pip install dist/*whl
+              run: pixi run -e ${{ matrix.pixi-env }} bash -c "pip install dist/*.whl"
     build-osx:
         name: Test on OSX
         runs-on: macOS-latest
         strategy:
             matrix:
-                python-version: ["3.12", "3.13", "3.14"]
+                pixi-env: [py312, py313, py314]
         steps:
             - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
               with:
                 persist-credentials: false
-            - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+            - uses: prefix-dev/setup-pixi@d3f436a425481402e6a95a1d1fc10331c708cd9e # v0.10.2
               with:
-                  python-version: ${{ matrix.python-version }}
-                  cache: "pip"
-            - name: Install Rust
-              uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0
+                  environments: ${{ matrix.pixi-env }}
             - name: build deeptools
-              run: |
-                  python -m venv venv
-                  source venv/bin/activate
-                  pip install maturin
-                  maturin develop --release --extras actions
+              run: pixi run -e ${{ matrix.pixi-env }} build
             - name: test deeptools
-              run: |
-                  source venv/bin/activate
-                  pytest -v
+              run: pixi run -e ${{ matrix.pixi-env }} test -v
             - name: build wheel
-              run: |
-                  source venv/bin/activate
-                  python -m build
+              run: pixi run -e ${{ matrix.pixi-env }} python -m build
             - name: test wheel
-              run: |
-                  source venv/bin/activate
-                  pip install dist/*whl
+              run: pixi run -e ${{ matrix.pixi-env }} bash -c "pip install dist/*.whl"

.github/workflows/wheels.yml

@@ -1,6 +1,4 @@
-name: build wheels
-# maturin generate-ci github > .github/workflows/maturin.yml
-# Also, see: https://github.com/jackh726/bigtools/blob/master/.github/workflows/release-python.yml
+name: build_wheels
 
 on:
     push:
@@ -103,41 +101,6 @@ jobs:
                   name: wheels-musllinux-${{ matrix.platform.target }}
                   path: dist
 
-    # windows:
-    #   runs-on: ${{ matrix.platform.runner }}
-    #   strategy:
-    #     matrix:
-    #       platform:
-    #         - runner: windows-latest
-    #           target: x64
-    #         - runner: windows-latest
-    #           target: x86
-    #   steps:
-    #     - uses: actions/checkout@v4
-    #     - uses: actions/setup-python@v5
-    #       with:
-    #         python-version: 3.x
-    #         architecture: ${{ matrix.platform.target }}
-    #     - name: Install And Configure Dependencies
-    #       env:
-    #         VCPKG_DEFAULT_TRIPLET: ${{ matrix.platform.target }}-windows
-    #         OPENSSL_DIR: ${{ github.workspace }}/vcpkg/installed/${{ matrix.platform.target }}-windows
-    #         OPENSSL_ROOT: ${{ github.workspace }}/vcpkg/installed/${{ matrix.platform.target }}-windows
-    #         LZMA_ROOT: ${{ github.workspace }}/vcpkg/installed/${{ matrix.platform.target }}-windows
-    #       run: |
-    #         vcpkg install openssl liblzma
-    #     - name: Build wheels
-    #       uses: PyO3/maturin-action@v1
-    #       with:
-    #         target: ${{ matrix.platform.target }}
-    #         args: --release --out dist --find-interpreter
-    #         # sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
-    #     - name: Upload wheels
-    #       uses: actions/upload-artifact@v4
-    #       with:
-    #         name: wheels-windows-${{ matrix.platform.target }}
-    #         path: dist
-
     macos:
         runs-on: ${{ matrix.platform.runner }}
         strategy:
@@ -159,7 +122,6 @@ jobs:
               with:
                   target: ${{ matrix.platform.target }}
                   args: --release --out dist --find-interpreter
-                  # sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
             - name: Upload wheels
               uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
               with:
@@ -177,7 +139,6 @@ jobs:
               with:
                   command: sdist
                   args: --out dist
-                  # sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
             - name: Upload sdist
               uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
               with:
@@ -188,7 +149,7 @@ jobs:
         name: Release
         runs-on: ubuntu-latest
         if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }}
-        needs: [linux, macos, sdist]
+        needs: [linux, musllinux, macos, sdist]
         permissions:
             id-token: write
             contents: write

.gitignore

@@ -59,7 +59,6 @@ tool_test*
 env.yml
 
 # Rust stuff
-Cargo.lock
 target*
 
 # snakemake
@@ -73,3 +72,5 @@ target*
 
 # pixi
 .pixi*
+
+.venv

.pre-commit-config.yaml

@@ -3,3 +3,11 @@ repos:
     rev: v1.11.0
     hooks:
       - id: zizmor
+  - repo: local
+    hooks:
+      - id: check-version-sync
+        name: check Rust versions (docs - toml) and deeptools versions (toml - galaxy) to stay in sync
+        entry: python3 scripts/check_version_sync.py
+        language: system
+        files: ^(Cargo\.toml|pyproject\.toml|\.readthedocs\.yaml|galaxy/wrapper/deepTools_macros\.xml)$
+        pass_filenames: false

.readthedocs.yaml

@@ -3,8 +3,8 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.12"
-    rust: "1.82"
+    python: "3.14"
+    rust: "1.91"
   apt_packages:
     - clang
 
@@ -15,4 +15,5 @@ python:
   install:
   - method: pip
     path: .
-  - requirements: docs/requirements.txt
+    extra_requirements:
+      - docs

CHANGES.txt

@@ -1,3 +1,30 @@
+4.0.0
+* new Rust-backed core for bamCoverage, bamCompare, computeMatrix, alignmentSieve and multiBamSummary (rayon multithreading, bigtools-based bigWig I/O), replacing the pure-Python implementations for these five tools; the previous Python implementations remain available as bamCoverage_old, bamCompare_old, computeMatrix_old, alignmentSieve_old and multiBamSummary_old during the transition, but will be removed in a future release
+* gzipped GTF/BED region files are now supported in multiBamSummary, computeMatrix and alignmentSieve; gzipped blacklist files are supported in all five Rust-backed tools (multiBamSummary, computeMatrix, alignmentSieve, bamCompare, bamCoverage)
+* --exactScaling is removed as this is the only scaling option available with the new backend
+* --ignoreDuplicates has been removed from the bamCoverage and bamCompare tools. Duplicate reads can be removed via the --samFlagExclude option (assuming the provided BAM file has duplicate reads marked)
+* plotPCA re-implemented with a lightweight scipy/numpy SVD backend. Proper handling of --transpose, --log2/--rowCenter and --ntop options.
+* plotly backend removed entirely; plotCorrelation, plotPCA, plotHeatmap, plotProfile and plotEnrichment now use matplotlib, with a new optional --ggplot theme, and computeGCBias's --plotFileFormat plotly option has also been removed
+* removed the standalone cm.py colormap module; colormap handling now lives in the plotting tools directly
+* plot labels now show sample names only per default (dropped .filtered.bam/.bed suffixes)
+* prebuilt wheels now cover Linux manylinux_2_28 and musllinux_1_2 (x86_64 + aarch64) and macOS (Intel + Apple Silicon), built via maturin
+* CI overhauled: pytest/rust/planemo test workflows split up (test_pytest.yml, test_rust.yml, test_planemo.yml), zizmor hardening for GitHub Actions, dependabot enabled
+* extensive new pytest/cargo test coverage across the Rust-backed tools and plotCorrelation/plotPCA/plotFingerprint data outputs
+* --nanAfterEnd doesn't rescale the inner region anymore in reference-point mode
+* BED output doesn't show wrong blockstart values anymore
+* blacklist filtering is done at bp level instead of genome chunk level for rewritten tools
+* alignmentSieve output order matches input order exactly
+* --missingDataAsZero no longer takes bases exceeding chromosome bounds as 0 values but rather purges the bins
+* large scale values precision slightly altered with new backend (f32 vs f64)
+
+3.5.6
+* minimal supported python version raised to 3.9 (numpy >= 2 support); NaN handling switched to np.nan
+* drop conda usage in pypi/test/planemo CI in favor of pip installs; samtools now pulled from conda-forge/bioconda where still needed in CI
+* include GRCm39 in the effective genome size docs
+* add ARM/osx specific installation instructions
+* computeMatrix: ensure --beforeRegionStartLength/-b and --afterRegionStartLength/-a are always treated as positive values
+* minor --version output fixes in plotCoverage and multiBigwigSummary
+
 3.5.5
 * drop support for python 3.7
 * doc fixes (argparse properly displayed, minor changes in installation instructions)