feat: export LiteLLM data directly from PostgreSQL

.github/workflows/ci.yml

@@ -52,7 +52,7 @@ jobs:
         run: |
           docker pull ghcr.io/berriai/litellm@sha256:625981c83410a3ea68eb0697590a57ec1d764d634514d54fa5db0591077ee839
           docker pull postgres@sha256:3645570cccdfa447589da9f57dd740faa29b30938e861289a5574b6ca6b03826
-          FLEETDIFF_BIN="$PWD/bin/fleetdiff" FLEETDIFF_SPEND_DOCKER=1 python3 -m unittest discover -s examples/litellm-spend -p 'test_*.py' -v
+          FLEETDIFF_BIN="$PWD/bin/fleetdiff" FLEETDIFF_LINUX_BIN="$RUNNER_TEMP/offline-build/fleetdiff" FLEETDIFF_SPEND_DOCKER=1 python3 -m unittest discover -s examples/litellm-spend -p 'test_*.py' -v
       - name: Offline capture accounting contract
         run: |
           go run ./examples/inspect --variant stock > "$RUNNER_TEMP/inspect-synthetic-traces.json"
@@ -122,6 +122,7 @@ jobs:
       - name: Bounded fuzz smoke
         # Fixed counts avoid timed-fuzz cancellation failures on the release toolchain.
         run: |
+          go test ./internal/postgres -run '^$' -fuzz '^FuzzConnectionProfile$' -fuzztime 1000x -parallel 2 -timeout 2m
           go test ./internal/reprice -run '^$' -fuzz '^FuzzRepriceComponents$' -fuzztime 1000x -parallel 2 -timeout 2m
           go test ./internal/spend -run '^$' -fuzz '^FuzzReaderBoundedFormats$' -fuzztime 1000x -parallel 2 -timeout 2m
           go test ./internal/spend -run '^$' -fuzz '^FuzzRoutingReaderBoundedFormats$' -fuzztime 1000x -parallel 2 -timeout 2m

.github/workflows/release.yml

@@ -58,6 +58,10 @@ jobs:
           jq -e 'any(.packages[]; .name == "github.com/llm-measurement/llm-sketchkit") and
             any(.packages[]; .name == "go.opentelemetry.io/proto/slim/otlp") and
             any(.packages[]; .name == "go.yaml.in/yaml/v3") and
+            any(.packages[]; .name == "github.com/jackc/pgx/v5" and .versionInfo == "v5.11.0") and
+            any(.packages[]; .name == "github.com/jackc/pgpassfile" and .versionInfo == "v1.0.0") and
+            any(.packages[]; .name == "github.com/jackc/pgservicefile" and .versionInfo == "v0.0.0-20240606120523-5a60cdf6a761") and
+            any(.packages[]; .name == "golang.org/x/text" and .versionInfo == "v0.41.0") and
             any(.packages[]; .name == "google.golang.org/protobuf")' sbom.spdx.json >/dev/null
           mv sbom.spdx.json dist/
           cd dist
@@ -108,6 +112,10 @@ jobs:
       - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
         with:
           go-version: '1.26.9'
+      - name: Native export safety on each supported platform
+        run: |
+          go test ./internal/postgres -count=1
+          go test ./internal/cli -run '^TestProductionHasNoNetwork$' -count=1
       - name: Verify spend, summary, trace, configuration, and scan commands from the packaged binary
         env:
           FLEETDIFF_RELEASE_BINARY: ${{ github.workspace }}/unpacked/fleetdiff
@@ -117,6 +125,7 @@ jobs:
         timeout-minutes: 10
         env:
           FLEETDIFF_BIN: ${{ github.workspace }}/unpacked/fleetdiff
+          FLEETDIFF_LINUX_BIN: ${{ github.workspace }}/unpacked/fleetdiff
           FLEETDIFF_SPEND_DOCKER: '1'
         run: |
           docker pull ghcr.io/berriai/litellm@sha256:625981c83410a3ea68eb0697590a57ec1d764d634514d54fa5db0591077ee839

CHANGELOG.md

@@ -2,6 +2,30 @@
 
 Notable user-visible changes are recorded here, newest first.
 
+## [0.12.0] - 2026-10-18
+
+See the [release notes](docs/releases/v0.12.0.md) and
+[native export guide](docs/NATIVE_EXPORT.md).
+
+- Add explicit native `export litellm` for private CSV/JSONL snapshots, optional
+  daily counters and routing details, with the pinned LiteLLM v1.104.0 projection.
+- Add `--litellm-db` to `investigate` and `what-if`, with automatic private-export
+  cleanup or `--keep-export NEWDIR`. Default to the last 14 complete UTC days;
+  preserve what-if `--from` / `--to` as model selectors and `--period` as time.
+- Introduce an explicit runtime network boundary for database modes only; file
+  commands remain offline. Require dedicated credential sources, verified remote
+  TLS, bounded read-only snapshots, sanitized errors and new private output paths.
+- Stage exports in a private mode-0700 sibling `.fleetdiff-stage-...` directory, then
+  atomically publish the complete directory without replacement. Keep the final
+  destination absent until publication; crashes beforehand may leave sibling
+  staging data, never a partial final directory.
+- Order SQL exports deterministically by `"startTime", request_id
+  COLLATE "C"` and daily `date COLLATE "C"` for native/psql byte equivalence.
+  Compared with the earlier unsorted recipe, bytes and first-seen/tied report
+  ordering may change; recorded totals and identity hashing are unchanged.
+- Pin `pgx/v5 v5.11.0`, `pgpassfile v1.0.0` and
+  `pgservicefile v0.0.0-20240606120523-5a60cdf6a761`; retain `x/text v0.41.0`.
+
 ## [0.11.0] - 2026-10-17
 
 See the [release notes](docs/releases/v0.11.0.md) for routing evidence and

README.md

@@ -7,69 +7,77 @@ compares LLM usage summaries, flags unusual windows, and checks trace captures
 and collector configurations.
 
 Use it to answer **why LLM token usage spiked, and which users or sessions drove
-the change.** It reads your files and prints a report: no account, upload,
-collector or model API key is needed for the LiteLLM path.
+the change.** File-based commands stay offline. The native LiteLLM
+path reads your database explicitly, then reports locally: no fleetdiff account,
+upload, collector or model API key is needed.
 
 | | |
 | --- | --- |
-| Commands | `investigate`, `scan`, `compare`, `inspect`, `diagnose` |
+| Commands | `investigate`, `what-if`, `scan`, `compare`, `inspect`, `diagnose`, `export litellm` |
 | Platforms | Linux and macOS, AMD64 and ARM64 |
 | Status | Pre-1.0; see the [changelog](CHANGELOG.md) |
 | License | Apache-2.0 |
 
 ## Using LiteLLM?
 
-**Export -> install -> investigate.** Use the request logs you already retain to
-compare two periods, see which models and keys drove the change, and check
-whether the usage records are complete.
-
-### 1. Export Request-Level Rows
-
-Use the [read-only SQL projection](examples/litellm-spend/export.sql) and your
-existing PostgreSQL access. The [export guide](examples/litellm-spend/README.md#export-request-level-rows)
-covers connection settings and version-pinned cache fields. Once your approved
-read-only connection is named `litellm_readonly`, this example exports 16 UTC days:
-
-```sh
-umask 077
-export_dir="$HOME/litellm-export"
-mkdir "$export_dir"
-curl --fail --location --proto '=https' --tlsv1.2 \
-  https://raw.githubusercontent.com/llm-measurement/fleetdiff/v0.11.0/examples/litellm-spend/export.sql \
-  --output "$export_dir/export.sql"
-psql 'service=litellm_readonly' -Xq -v ON_ERROR_STOP=1 \
-  -v start_utc=2026-09-23T00:00:00Z -v end_utc=2026-10-09T00:00:00Z \
-  -v csv_file="$export_dir/spend.csv" -f "$export_dir/export.sql" \
-  >"$export_dir/private-psql.out" 2>"$export_dir/private-psql.err"
-```
-
-Choose dates for your own data and continue only after the export succeeds.
-The SQL writes CSV in one pass and excludes prompts and responses. Keep the
-export directory private. Existing request-level JSON and JSONL work too.
+**One command to the first report, after setup.** Compare two periods, see which
+models and keys drove the change, and check the usage records you already retain.
 
 <a id="2-install-v060"></a>
 <a id="2-install-v061"></a>
 <a id="2-install-v070"></a>
 <a id="2-install-v080"></a>
 <a id="2-install-v090"></a>
 <a id="2-install-v0100"></a>
+<a id="2-install-v0110"></a>
 
-### 2. Install v0.11.0
+### 1. Install v0.12.0
 
-Run this in Bash with `curl`, `gh`, `shasum` and `tar` available:
+Run this in Bash with `curl`, `gh`, `shasum` and `tar` available; `gh` need not be signed in:
 
 ```sh
-bash -o pipefail -c 'curl --fail --location --proto "=https" --tlsv1.2 https://raw.githubusercontent.com/llm-measurement/fleetdiff/7773f212e4ae6e800f066b394e5507c0df8ada5f/scripts/install.sh | sh -s -- v0.11.0 ./fleetdiff-install'
+bash -o pipefail -c 'curl --fail --location --proto "=https" --tlsv1.2 https://raw.githubusercontent.com/llm-measurement/fleetdiff/65990e302913fb960db3ba3a53cb4633bef63117/scripts/install.sh | sh -s -- v0.12.0 ./fleetdiff-install'
 ./fleetdiff-install/fleetdiff --version
 ```
 
 The installer checks signed release provenance and checksums before extracting
-the binary into a new directory. Prefer to inspect the script first? Follow
-[Install and verify](docs/OPERATIONS.md#install-and-verify).
+into a new directory. No Go compiler is needed. Follow
+[Install and verify](docs/OPERATIONS.md#install-and-verify) for manual verification.
 
-### 3. Investigate
+### 2. Investigate Your Database
+
+Have your administrator provision the [read-only role and credential file](docs/NATIVE_EXPORT.md#credentials).
+Set `DATABASE_URL_FILE` to that existing, owned mode-0600 regular file. Leave
+`FLEETDIFF_LITELLM_DATABASE_URL` unset when using the file, then run:
 
 ```sh
+: "${DATABASE_URL_FILE:?Set the path to your approved database URL file}"
+./fleetdiff-install/fleetdiff investigate --litellm-db \
+  --database-url-file "$DATABASE_URL_FILE" --group-by key
+```
+
+This connects only for the requested export, compares the last 14 complete UTC
+days as two seven-day periods, and removes its private temporary export after
+analysis. Add `--daily` for row coverage or `--routing-audit` for route evidence;
+routing details are included automatically. Use `--keep-export NEWDIR` to retain
+the private files, or `export litellm --out NEWDIR` to export without a report.
+The [native guide](docs/NATIVE_EXPORT.md) covers credentials, periods and what-if.
+
+<a id="1-export-request-level-rows"></a>
+
+### Manual Alternative: Export With psql
+
+For client certificates, PostgreSQL services and other native-unsupported
+connections, follow the [read-only SQL recipe](examples/litellm-spend/README.md#export-request-level-rows).
+Keep the export private; it excludes prompts and responses but retains identities.
+Existing request-level CSV, JSON and JSONL also work with the same installed binary.
+
+<a id="3-investigate"></a>
+
+Investigate the exported file offline:
+
+```sh
+export_dir="$HOME/litellm-export"
 ./fleetdiff-install/fleetdiff investigate \
   --litellm-spend "$export_dir/spend.csv" --group-by key
 ```
@@ -94,7 +102,7 @@ Download the synthetic spend file and use the installed binary:
 
 ```sh
 curl --fail --location --proto '=https' --tlsv1.2 \
-  https://raw.githubusercontent.com/llm-measurement/fleetdiff/v0.11.0/examples/litellm-spend/synthetic.csv \
+  https://raw.githubusercontent.com/llm-measurement/fleetdiff/v0.12.0/examples/litellm-spend/synthetic.csv \
   --output synthetic.csv
 ./fleetdiff-install/fleetdiff investigate --litellm-spend synthetic.csv \
   --before-period 2026-10-07 --after-period 2026-10-08
@@ -127,15 +135,15 @@ The summary-file path also shows session concentration:
 
 ## Install
 
-The verified installer above selects the native binary for your machine.
-For manual downloads, use the [v0.11.0 archives](https://github.com/llm-measurement/fleetdiff/releases/tag/v0.11.0)
+The verified v0.12.0 installer selects the platform binary for your machine.
+For manual downloads, use the [v0.12.0 archives](https://github.com/llm-measurement/fleetdiff/releases/tag/v0.12.0)
 and [verification instructions](docs/OPERATIONS.md#install-and-verify).
 An administrator can verify and transfer artifacts through an approved internal mirror.
 
 With a security-patched Go 1.26 toolchain, install the same version from its module:
 
 ```sh
-go install github.com/llm-measurement/fleetdiff/cmd/fleetdiff@v0.11.0
+go install github.com/llm-measurement/fleetdiff/cmd/fleetdiff@v0.12.0
 ```
 
 The executable goes to `GOBIN`, or `$(go env GOPATH)/bin` when unset.
@@ -150,13 +158,14 @@ See [source builds and version reporting](docs/OPERATIONS.md#build-from-source).
   receive instructions for getting those rows. The reader accepts up to 10M rows
   and 16 GiB per file, with [per-record limits](examples/litellm-spend/README.md#data-and-limits).
 - By default, private names use aliases or pseudonymous hashes; public catalog model IDs are readable.
-  keep source files private and share reports under your organization's policy.
+  Keep source files private and share reports under your organization's policy.
 - Rankings carry deterministic bounds; distinct counts are statistical estimates.
   A flagged contributor identifies something to investigate, rather than its cause.
   Compatible summary files need aligned windows, keys and disjoint producer coverage.
 
 ## Documentation
 
+- [Native LiteLLM export](docs/NATIVE_EXPORT.md): setup, credentials and connection contract.
 - [LiteLLM spend exports](examples/litellm-spend/README.md): SQL, periods, groupings and source accounting.
 - [Session-budget review](docs/SESSION_BUDGET.md): single-period recorded totals and cap review.
 - [Model-mix what-if](docs/WHAT_IF.md) (v0.10.0): re-price covered usage under explicit assumptions.
@@ -168,7 +177,7 @@ See [source builds and version reporting](docs/OPERATIONS.md#build-from-source).
 - <a id="questions-it-answers"></a><a id="user-and-session-contributors"></a> [Questions and contributor rankings](docs/WORKFLOWS.md#questions-it-answers).
 - <a id="how-it-works"></a><a id="use-your-own-data"></a><a id="reading-the-results"></a> [How summary comparison works](docs/WORKFLOWS.md#how-it-works), [comparison contract](docs/COMPARISON.md) and [investigation contract](docs/INVESTIGATION.md).
 - <a id="built-to-be-checked"></a> [Operations](docs/OPERATIONS.md), [resource measurements](docs/BENCHMARKS.md) and [checks](docs/WORKFLOWS.md#built-to-be-checked).
-- <a id="docs"></a> [FAQ](docs/FAQ.md), [security](SECURITY.md), [release notes](docs/releases/v0.11.0.md) and [release procedure](docs/RELEASING.md).
+- <a id="docs"></a> [FAQ](docs/FAQ.md), [security](SECURITY.md), [release notes](docs/releases/v0.12.0.md) and [release procedure](docs/RELEASING.md).
 
 Questions or feedback: [open an issue](https://github.com/llm-measurement/fleetdiff/issues).
 See [Contributing](CONTRIBUTING.md) for checks and signed, signed-off commits.

SECURITY.md

@@ -2,7 +2,7 @@
 
 ## Supported Versions
 
-Security fixes target the latest patch in the `0.11.x` release line and `main`.
+Security fixes target the latest patch in the `0.12.x` release line and `main`.
 Use the newest patch and record its version and revision when running trials.
 Older patches and development snapshots do not receive routine backports.
 Dependencies and the Go toolchain are reviewed on an ongoing basis.
@@ -42,9 +42,45 @@ contractual response-time commitment.
   per-run key and aliases for shared reports, and remove captures after inspection.
   Raw values are transiently decoded in process memory; this is not secure erasure.
 
-The comparison and inspection commands read local files and write stdout/stderr
-without network access. Inspection also accepts stdin. Initial builds may download dependencies, and the live example starts
-local Docker collectors and may download its pinned image. See the
+File-based commands read local files and write stdout/stderr without network
+access; inspection also accepts stdin. Merely setting database credentials does
+not enable a connection. Initial builds may download dependencies, and the live
+example starts local Docker collectors and may download its pinned image.
+
+## Native Database Export
+
+Only explicit `export litellm`, `investigate --litellm-db` and `what-if --litellm-db`
+connect to PostgreSQL. This adds a network boundary, not telemetry, update checks
+or provider calls.
+
+- Supply exactly one credential source: `FLEETDIFF_LITELLM_DATABASE_URL` or
+  `--database-url-file` pointing to an owned regular mode-0600 file. Literal
+  URL arguments and symlink components are rejected. Native export ignores
+  inherited `PG*` settings, `~/.pgpass`, PostgreSQL service files and implicit
+  client certificates. It never prints connection details or raw driver errors.
+- Remote connections require `sslmode=verify-full`; `verify-ca` is not accepted.
+  Only numeric loopback addresses or Unix sockets may use `sslmode=disable`.
+  The query allowlist is `sslmode`, optional `sslrootcert`, and `host` only for
+  an absolute Unix socket directory. Client-certificate and service-based setups
+  use the separately reviewed `psql` recipe, not a native TLS bypass.
+- Use a dedicated SELECT-only role. Every native export uses one read-only,
+  repeatable-read snapshot with bounded statements and idle time. Role creation
+  and grants are administrator actions; fleetdiff never changes the database.
+- Native export writes private local files. Destinations must be new, with no
+  symlink components; existing directories, even empty ones, and overwrites are
+  refused. Files are built in a mode-0700 sibling `.fleetdiff-stage-...` directory,
+  with mode-0600 files. The final destination stays absent until the complete
+  directory is atomically published without replacement, including when another
+  process creates the destination during export. Convenience modes remove
+  temporary exports unless `--keep-export NEWDIR` is selected.
+- Exports retain raw identities. Reports keep existing aliases and keyed-hash
+  rules; optional label/hash disclosure remains explicit. A crash before
+  publication may leave private sibling `.fleetdiff-stage-...` staging data, never a
+  partial final directory. After publication, the final directory is complete.
+  Remove only the run's abandoned staging data under your retention policy.
+
+See the [native export contract](docs/NATIVE_EXPORT.md) for setup, exact grants,
+limits and failure handling. See the
 [comparison contract](docs/COMPARISON.md) for input limits and trust assumptions.
 See [inspection](docs/INSPECT.md) for bounded decoding, source semantics, and key handling.
 See [operations](docs/OPERATIONS.md) for installation verification, isolated

cmd/fleetdiff/main.go

@@ -4,9 +4,46 @@
 package main
 
 import (
+	"context"
+	"io"
 	"os"
+	"os/signal"
+	"syscall"
 
 	"github.com/llm-measurement/fleetdiff/internal/cli"
+	"github.com/llm-measurement/fleetdiff/internal/exportdata"
+	"github.com/llm-measurement/fleetdiff/internal/postgres"
 )
 
-func main() { os.Exit(cli.Run(os.Args[1:], os.Stdout, os.Stderr)) }
+func main() { os.Exit(run()) }
+
+func run() int {
+	return runWithInput(os.Stdin)
+}
+
+func runWithInput(in io.Reader) int {
+	return cli.RunWithExporter(context.Background(), os.Args[1:], in, os.Stdout, os.Stderr, withExportSignals(postgres.Export))
+}
+
+func withExportSignals(exporter exportdata.Exporter) exportdata.Exporter {
+	return func(ctx context.Context, opts exportdata.Options) (exportdata.Result, error) {
+		ctx, cancel := context.WithCancel(ctx)
+		interrupts := make(chan os.Signal, 1)
+		signal.Notify(interrupts, os.Interrupt, syscall.SIGTERM)
+		stopped := make(chan struct{})
+		go func() {
+			// The first signal requests cleanup. A repeated signal may terminate
+			// an unresponsive exporter and leave owner-only temporary data.
+			select {
+			case <-interrupts:
+				signal.Stop(interrupts)
+				cancel()
+			case <-ctx.Done():
+				signal.Stop(interrupts)
+			}
+			close(stopped)
+		}()
+		defer func() { cancel(); <-stopped }()
+		return exporter(ctx, opts)
+	}
+}