fix: soilweb links — cokey-based URL mapping + deterministic ordering

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
     -   id: pre-commit-update
 
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.15.17
+    rev: v0.15.22
     hooks:
     -   id: ruff-check
         args: [--fix]

Makefile

@@ -2,6 +2,28 @@ ifeq ($(DC_ENV),ci)
 	UV_FLAGS = "--system"
 endif
 
+PYTHON ?= python3
+
+# The test targets below need pytest + GDAL + the pinned soil-id-db. They run
+# natively only when THIS interpreter ($(PYTHON)) has pytest AND the *pinned*
+# GDAL version (so results match CI and the pinned image); otherwise they
+# transparently re-run themselves in a GDAL-capable container against the pinned
+# soil-id-db image (see scripts/run_in_container.sh), so the same `make test*`
+# command works on macOS and on CI alike.
+#
+# The version gate matters: snapshots are GDAL-version-sensitive, so a native
+# Homebrew GDAL of a *different* version must NOT take the native path — it would
+# produce snapshots that disagree with CI. It also naturally routes macOS to the
+# container, since the interpreter with GDAL (Homebrew) lacks pytest and the
+# venv with pytest lacks GDAL. Set NATIVE=1 to force native (also set inside the
+# runner container, where the versions do match, to prevent infinite recursion).
+GDAL_PIN := $(shell sed -n 's/^gdal==\([0-9.]*\).*/\1/p' requirements.txt)
+ifeq ($(NATIVE),1)
+	NATIVE_OK := 1
+else
+	NATIVE_OK := $(shell $(PYTHON) -c 'import pytest, osgeo.gdal as g; raise SystemExit(g.__version__ != "$(GDAL_PIN)")' >/dev/null 2>&1 && echo 1)
+endif
+
 install:
 	uv pip install -r requirements.txt $(UV_FLAGS)
 
@@ -37,19 +59,29 @@ clean:
 
 # run the standard test suite (unit + integration, no api_snapshots)
 test:
+	@if [ "$(NATIVE_OK)" != "1" ]; then exec ./scripts/run_in_container.sh test; fi; \
 	if [ -z "$(PATTERN)" ]; then \
-		pytest soil_id -m "not api_snapshot"; \
+		$(PYTHON) -m pytest soil_id -m "not api_snapshot"; \
 	else \
-		pytest soil_id -m "not api_snapshot" -k "$(PATTERN)"; \
+		$(PYTHON) -m pytest soil_id -m "not api_snapshot" -k "$(PATTERN)"; \
 	fi
 
 # All tests except api_snapshot and integration (no live external APIs)
 test_unit:
-	pytest soil_id -m "not api_snapshot and not integration"
+	@if [ "$(NATIVE_OK)" != "1" ]; then exec ./scripts/run_in_container.sh test_unit; fi; \
+	$(PYTHON) -m pytest soil_id -m "not api_snapshot and not integration"
 
 # update the unit test snapshots (but not the API snapshots)
 test_update_unit_snapshots:
-	pytest soil_id -m "not api_snapshot and not integration" --snapshot-update; \
+	@if [ "$(NATIVE_OK)" != "1" ]; then exec ./scripts/run_in_container.sh test_update_unit_snapshots; fi; \
+	$(PYTHON) -m pytest soil_id -m "not api_snapshot and not integration" --snapshot-update
+
+# Regenerate the unit-test output snapshots reproducibly in a GDAL-capable
+# container, against the pinned soil-id-db image CI uses. Wraps
+# test_update_unit_snapshots for machines where GDAL won't build natively
+# (e.g. macOS). See scripts/regen_snapshots.sh.
+regen_snapshots:
+	./scripts/regen_snapshots.sh
 
 # Integration smoke tests only (full live API run, no output validation)
 test_integration:

README.md

@@ -86,6 +86,17 @@ There are several smaller test suites:
 - The unit and integration tests can be run together with `make test` for convenience: this is what must pass for a PR to be mergeable.
 - The API snapshots themselves can be checked against the live API for drift using `make test_api_snapshot`. They can be updated to the new live API values using `make test_update_api_snapshots`.
 
+> **macOS note:** GDAL doesn't build under uv/pip on macOS, so `make test`,
+> `make test_unit`, and `make test_update_unit_snapshots` automatically re-run
+> themselves in a GDAL-capable container against the pinned `soil-id-db` image
+> (the one CI uses) — the command is the same as on Linux/CI, it just needs
+> Docker. They run natively only when the local interpreter has pytest **and the
+> pinned GDAL version** (a mismatched Homebrew GDAL would produce snapshots that
+> disagree with CI, so it's routed to the container instead). Set `NATIVE=1` to
+> force the local interpreter, or `PYTHON=…` to pick one. To regenerate and
+> verify the output snapshots in one step, use `make regen_snapshots`. See
+> `scripts/README.md`.
+
 ### Bulk test
 
 There is a large suite of integration tests which takes many hours to run. It comes in the format of two scripts:

scripts/README.md

@@ -1,5 +1,51 @@
 # soil-id scripts
 
+## Containerized tests / snapshots on macOS
+
+GDAL doesn't build under uv/pip on macOS, and the output snapshots are
+environment-sensitive — they must be produced against an interpreter matching CI
+**and** the exact pinned `soil-id-db` image CI uses (`docker-compose.yml`
+`db.image`), **not** the `:latest` your local terraso-backend runs. Two pieces make
+this transparent:
+
+- **`snapshot-runner.Dockerfile`** — a runner image mirroring the CI `test` job
+  (Ubuntu + ubuntugis + Python 3.13 + the pinned requirements). Built once, layer-cached
+  on the requirements files.
+- **`run_in_container.sh`** — starts the pinned DB on a private network (no host port, so
+  it can't collide with a running terraso-backend on 5432), then runs the given Makefile
+  target(s) in the runner with the repo bind-mounted read-write, and tears everything
+  down afterward.
+
+You don't normally call `run_in_container.sh` directly: the Makefile's `test`,
+`test_unit`, and `test_update_unit_snapshots` targets **auto-detect** whether the local
+interpreter has pytest **and the pinned GDAL version** and, if not, delegate to it — so
+the same `make …` command works on macOS and CI. The version gate matters because
+snapshots are GDAL-version-sensitive: a Homebrew GDAL of a different version is routed to
+the container rather than silently producing snapshots that disagree with CI. Set
+`NATIVE=1` to force the local interpreter (set automatically inside the runner to avoid
+recursion), or `PYTHON=…` to choose the interpreter.
+
+## `regen_snapshots.sh`
+
+Regenerates the unit-test **output** snapshots
+(`soil_id/tests/{us,global}/__snapshots__/test_soil_location[...].json`) **and verifies
+them** in one DB session: it runs `make test_update_unit_snapshots test_unit` (update,
+then re-run without `--snapshot-update`; the second pass must pass clean or the snapshots
+are non-deterministic), then shows `git status` for the snapshot dirs.
+
+```sh
+make regen_snapshots            # regenerate + verify
+./scripts/regen_snapshots.sh --no-verify
+REBUILD=1 make regen_snapshots  # force-rebuild the runner image (after a deps bump)
+```
+
+> Not to be confused with `make test_update_api_snapshots`, which refreshes the
+> **input** fixtures (raw SoilWeb/elevation API responses under
+> `__snapshots__/test_api_snapshots`) by hitting the **live** external APIs. That's a
+> different, drift-prone operation and is not what re-baselining algorithm output needs.
+
+---
+
 ## `wrb_descriptions_sync.py`
 
 Syncs the WRB soil **Description**/**Management** narratives from the

scripts/regen_snapshots.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Copyright © 2024 Technology Matters
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see https://www.gnu.org/licenses/.
+
+# ---------------------------------------------------------------------------
+# Regenerate the unit-test *output* snapshots (soil_id/tests/{us,global}/
+# __snapshots__/test_soil_location[...].json) reproducibly, then verify them.
+#
+# The snapshot command is just `make test_update_unit_snapshots`; the fiddly
+# part is the environment (GDAL, which won't build on macOS, + the exact pinned
+# soil-id-db image CI uses). scripts/run_in_container.sh supplies that. Here we:
+#   1. Regenerate + verify in one container/DB session:
+#      `make test_update_unit_snapshots test_unit` — update, then re-run without
+#      --snapshot-update. The verify pass MUST pass clean, or the snapshots are
+#      non-deterministic and untrustworthy.
+#   2. Show `git status` for the snapshot dirs.
+#
+# Usage:
+#   scripts/regen_snapshots.sh            # regenerate + verify (default)
+#   scripts/regen_snapshots.sh --no-verify
+#   REBUILD=1 scripts/regen_snapshots.sh  # force rebuild of the runner image
+# ---------------------------------------------------------------------------
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$REPO_ROOT"
+
+VERIFY=1
+for arg in "$@"; do
+  case "$arg" in
+    --no-verify) VERIFY=0 ;;
+    -h|--help) sed -n '19,40p' "$0"; exit 0 ;;
+    *) echo "unknown arg: $arg" >&2; exit 2 ;;
+  esac
+done
+
+log() { printf '\n\033[1;34m==> %s\033[0m\n' "$*"; }
+
+# Update, then (unless --no-verify) re-run the same tests without
+# --snapshot-update to prove determinism — both in one DB session.
+if [ "$VERIFY" -eq 1 ]; then
+  log "Regenerating unit snapshots + verifying determinism"
+  if ./scripts/run_in_container.sh test_update_unit_snapshots test_unit; then
+    echo "  verification passed — snapshots are deterministic"
+  else
+    printf '\033[1;31m%s\033[0m\n' \
+      "verification FAILED: snapshots differ on a second run (non-determinism). Do not commit." >&2
+    exit 1
+  fi
+else
+  log "Regenerating unit snapshots (no verify)"
+  ./scripts/run_in_container.sh test_update_unit_snapshots
+fi
+
+log "Snapshot changes:"
+git -c color.status=always status --short \
+  soil_id/tests/us/__snapshots__ soil_id/tests/global/__snapshots__ || true
+echo
+echo "Review the diff, then commit. Snapshots were generated against the pinned"
+echo "image in docker-compose.yml, so they should match CI."

scripts/run_in_container.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Copyright © 2024 Technology Matters
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see https://www.gnu.org/licenses/.
+
+# ---------------------------------------------------------------------------
+# Run one or more Makefile targets inside a GDAL-capable container, against the
+# pinned soil-id-db image, on machines where the native env can't run the tests
+# (GDAL doesn't build under uv/pip on macOS).
+#
+# The Makefile's test targets call this automatically when the local interpreter
+# doesn't have pytest + the *pinned* GDAL version (see the NATIVE_OK guard in the
+# Makefile), so `make test_unit`, `make test_update_unit_snapshots`, etc. work
+# identically on macOS and CI. You normally don't invoke it directly.
+#
+# It:
+#   1. Builds the CI-faithful runner image (scripts/snapshot-runner.Dockerfile),
+#      layer-cached on the requirements files.
+#   2. Starts the pinned soil-id-db (docker-compose.yml `db.image` — the one CI
+#      uses, NOT your local :latest) on a private throwaway network with no host
+#      port published (avoids colliding with a running terraso-backend on 5432).
+#   3. Runs `make <targets> NATIVE=1` in the runner with the repo bind-mounted
+#      read-write (so snapshots land in your working tree). NATIVE=1 makes the
+#      in-container make run pytest directly instead of recursing back here.
+#   4. Tears the DB + network down (including the anonymous data volume).
+#
+# Usage:
+#   scripts/run_in_container.sh test_unit
+#   scripts/run_in_container.sh test_update_unit_snapshots test_unit
+#   REBUILD=1 scripts/run_in_container.sh test_unit   # force runner-image rebuild
+# ---------------------------------------------------------------------------
+
+[ "$#" -ge 1 ] || { echo "usage: $0 <make-target> [make-target...]" >&2; exit 2; }
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$REPO_ROOT"
+
+RUNNER_IMAGE="${RUNNER_IMAGE:-soil-id-snapshot-runner:latest}"
+RUNNER_DOCKERFILE="scripts/snapshot-runner.Dockerfile"
+PLATFORM="${PLATFORM:-linux/amd64}"
+
+# The pinned DB and runner run on a private, throwaway network and talk by
+# container name. We deliberately avoid `docker compose up` / the compose host
+# port (5432), which collides with a running terraso-backend Postgres.
+NETWORK="${NETWORK:-soil-id-snapshot-net}"
+DB_CONTAINER="${DB_CONTAINER:-soil-id-snapshot-db}"
+# Pinned image read straight from docker-compose.yml so it stays in lockstep
+# with the pin CI uses (do not point at your local :latest).
+DB_IMAGE="$(awk '/^[[:space:]]*image:/{print $2; exit}' docker-compose.yml)"
+
+log() { printf '\n\033[1;34m==> %s\033[0m\n' "$*"; }
+die() { printf '\033[1;31merror: %s\033[0m\n' "$*" >&2; exit 1; }
+
+command -v docker >/dev/null || die "docker not found on PATH"
+
+# Data/ must be present (US path reads shapefiles + Munsell CSV).
+if [ ! -f Data/gsmsoilmu_a_us.shp ] || [ ! -f Data/LandPKS_munsell_rgb_lab.csv ]; then
+  die "Data/ is missing required files. Run 'make download_soil_data' first."
+fi
+
+# --- Build the runner image (cheap when the deps layer is cached) --------------
+if [ -n "${REBUILD:-}" ] || ! docker image inspect "$RUNNER_IMAGE" >/dev/null 2>&1; then
+  log "Building runner image $RUNNER_IMAGE (mirrors CI: Ubuntu + ubuntugis + Python 3.13)"
+  docker build --platform "$PLATFORM" -f "$RUNNER_DOCKERFILE" -t "$RUNNER_IMAGE" .
+else
+  log "Reusing runner image $RUNNER_IMAGE (set REBUILD=1 to force a rebuild)"
+fi
+
+# --- Start the pinned soil-id-db (private network, no host port) ----------------
+cleanup() {
+  # -v removes the anonymous data volume too; without it each run leaks ~3.7GB.
+  docker rm -fv "$DB_CONTAINER" >/dev/null 2>&1 || true
+  docker network rm "$NETWORK" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+# Fresh network + DB each run so a stale container can't serve wrong data.
+docker rm -fv "$DB_CONTAINER" >/dev/null 2>&1 || true
+docker network inspect "$NETWORK" >/dev/null 2>&1 || docker network create "$NETWORK" >/dev/null
+
+log "Starting pinned soil-id-db ($DB_IMAGE)"
+docker run -d --name "$DB_CONTAINER" --platform "$PLATFORM" \
+  --network "$NETWORK" \
+  -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=soil_id \
+  "$DB_IMAGE" >/dev/null
+
+log "Waiting for the DB to accept connections"
+for i in $(seq 1 60); do
+  if docker exec "$DB_CONTAINER" pg_isready -U postgres -d soil_id >/dev/null 2>&1; then
+    echo "  db ready"
+    break
+  fi
+  [ "$i" -eq 60 ] && die "DB did not become ready in time"
+  sleep 2
+done
+
+# --- Run the requested make target(s) in the runner -----------------------------
+# NATIVE=1 so the in-container make runs pytest directly (no recursion). DB_* is
+# what soil_id/config.py reads; the runner reaches the DB by name on NETWORK.
+log "Running in container: make $* NATIVE=1"
+docker run --rm \
+  --platform "$PLATFORM" \
+  --network "$NETWORK" \
+  -v "$REPO_ROOT:/src" -w /src \
+  -e DB_HOST="$DB_CONTAINER" -e DB_PORT=5432 -e DB_NAME=soil_id \
+  -e DB_USERNAME=postgres -e DB_PASSWORD=postgres \
+  "$RUNNER_IMAGE" make "$@" NATIVE=1

scripts/snapshot-runner.Dockerfile

@@ -0,0 +1,56 @@
+# Copyright © 2024 Technology Matters
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see https://www.gnu.org/licenses/.
+
+# Runner image for regenerating soil-id test snapshots off the machine's native
+# environment (GDAL does not build under uv/pip on macOS). This mirrors the CI
+# "test" job (.github/workflows/build.yml) as closely as practical so the
+# snapshots it produces match what CI validates:
+#   - Ubuntu 24.04 + ubuntugis-unstable  -> same libgdal as CI
+#   - Python 3.13                         -> same interpreter series as CI (3.13.7)
+#   - `make install` / `make install_dev` -> the exact pinned requirements
+#
+# Only requirements are baked in (layer-cached on the requirements files); the
+# soil-id source and Data/ are bind-mounted at run time by regen_snapshots.sh.
+FROM ubuntu:24.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PATH="/root/.local/bin:${PATH}"
+# uv --system installs must target the 3.13 we register as default python3 below.
+ENV UV_SYSTEM_PYTHON=1
+ENV UV_BREAK_SYSTEM_PACKAGES=1
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      software-properties-common curl ca-certificates git make gcc g++ \
+ && add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable \
+ && add-apt-repository -y ppa:deadsnakes/ppa \
+ && apt-get update && apt-get install -y --no-install-recommends \
+      libgdal-dev gdal-bin \
+      python3.13 python3.13-dev python3.13-venv \
+ && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.13 1 \
+ && update-alternatives --install /usr/bin/python  python  /usr/bin/python3.13 1 \
+ && rm -rf /var/lib/apt/lists/*
+
+# uv (used by the Makefile install targets)
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+
+WORKDIR /src
+
+# Install the pinned deps. Keyed on the requirements files so this layer is
+# reused until they change. `DC_ENV=ci` makes the Makefile pass uv `--system`.
+COPY requirements.txt requirements-dev.txt Makefile ./
+COPY requirements/ requirements/
+RUN make install DC_ENV=ci && make install_dev DC_ENV=ci
+
+CMD ["bash"]