From 65990e302913fb960db3ba3a53cb4633bef63117 Mon Sep 17 00:00:00 2001 From: Vijay Date: Sun, 18 Oct 2026 13:29:34 -0400 Subject: [PATCH 1/3] feat: add secure native LiteLLM database exports Signed-off-by: Vijay --- .github/workflows/ci.yml | 3 +- .github/workflows/release.yml | 9 + CHANGELOG.md | 25 + README.md | 74 +- SECURITY.md | 42 +- cmd/fleetdiff/main.go | 39 +- cmd/fleetdiff/main_test.go | 123 +++ docs/COMPARISON.md | 5 +- docs/INSPECT.md | 6 +- docs/NATIVE_EXPORT.md | 269 +++++++ docs/OPERATIONS.md | 68 +- docs/calibration/manifest.json | 2 +- docs/releases/v0.12.0.md | 61 ++ examples/litellm-spend/README.md | 69 +- examples/litellm-spend/TESTING.md | 54 +- .../coverage_runtime_expected.json | 11 +- .../litellm-spend/coverage_runtime_spend.csv | 18 +- examples/litellm-spend/daily-projection.sql | 1 + examples/litellm-spend/export.sql | 3 + examples/litellm-spend/projection.go | 13 + examples/litellm-spend/test_native_export.py | 738 ++++++++++++++++++ .../test_row_coverage_runtime.py | 59 +- examples/litellm-spend/test_spend.py | 5 +- go.mod | 7 +- go.sum | 6 + internal/cli/database.go | 338 ++++++++ internal/cli/database_files_test.go | 113 +++ internal/cli/database_test.go | 536 +++++++++++++ internal/cli/network_test.go | 226 +++++- internal/cli/run.go | 90 ++- internal/cli/whatif.go | 46 +- internal/exportdata/export.go | 64 ++ internal/postgres/config.go | 180 +++++ internal/postgres/config_test.go | 236 ++++++ internal/postgres/errors_test.go | 52 ++ internal/postgres/export.go | 204 +++++ internal/postgres/files.go | 267 +++++++ internal/postgres/files_test.go | 424 ++++++++++ internal/postgres/query.go | 66 ++ internal/postgres/query_test.go | 98 +++ internal/postgres/rename_darwin.go | 48 ++ internal/postgres/rename_linux.go | 46 ++ internal/postgres/rename_linux_amd64.go | 6 + internal/postgres/rename_linux_arm64.go | 8 + internal/postgres/schema.go | 70 ++ internal/postgres/tls_test.go | 265 +++++++ internal/spend/reader.go | 29 +- internal/spend/row_coverage.go | 16 +- 48 files changed, 5038 insertions(+), 100 deletions(-) create mode 100644 cmd/fleetdiff/main_test.go create mode 100644 docs/NATIVE_EXPORT.md create mode 100644 docs/releases/v0.12.0.md create mode 100644 examples/litellm-spend/projection.go create mode 100644 examples/litellm-spend/test_native_export.py create mode 100644 internal/cli/database.go create mode 100644 internal/cli/database_files_test.go create mode 100644 internal/cli/database_test.go create mode 100644 internal/exportdata/export.go create mode 100644 internal/postgres/config.go create mode 100644 internal/postgres/config_test.go create mode 100644 internal/postgres/errors_test.go create mode 100644 internal/postgres/export.go create mode 100644 internal/postgres/files.go create mode 100644 internal/postgres/files_test.go create mode 100644 internal/postgres/query.go create mode 100644 internal/postgres/query_test.go create mode 100644 internal/postgres/rename_darwin.go create mode 100644 internal/postgres/rename_linux.go create mode 100644 internal/postgres/rename_linux_amd64.go create mode 100644 internal/postgres/rename_linux_arm64.go create mode 100644 internal/postgres/schema.go create mode 100644 internal/postgres/tls_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54ccebb..addaec9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: run: | docker pull ghcr.io/berriai/litellm@sha256:625981c83410a3ea68eb0697590a57ec1d764d634514d54fa5db0591077ee839 docker pull postgres@sha256:3645570cccdfa447589da9f57dd740faa29b30938e861289a5574b6ca6b03826 - FLEETDIFF_BIN="$PWD/bin/fleetdiff" FLEETDIFF_SPEND_DOCKER=1 python3 -m unittest discover -s examples/litellm-spend -p 'test_*.py' -v + FLEETDIFF_BIN="$PWD/bin/fleetdiff" FLEETDIFF_LINUX_BIN="$RUNNER_TEMP/offline-build/fleetdiff" FLEETDIFF_SPEND_DOCKER=1 python3 -m unittest discover -s examples/litellm-spend -p 'test_*.py' -v - name: Offline capture accounting contract run: | go run ./examples/inspect --variant stock > "$RUNNER_TEMP/inspect-synthetic-traces.json" @@ -122,6 +122,7 @@ jobs: - name: Bounded fuzz smoke # Fixed counts avoid timed-fuzz cancellation failures on the release toolchain. run: | + go test ./internal/postgres -run '^$' -fuzz '^FuzzConnectionProfile$' -fuzztime 1000x -parallel 2 -timeout 2m go test ./internal/reprice -run '^$' -fuzz '^FuzzRepriceComponents$' -fuzztime 1000x -parallel 2 -timeout 2m go test ./internal/spend -run '^$' -fuzz '^FuzzReaderBoundedFormats$' -fuzztime 1000x -parallel 2 -timeout 2m go test ./internal/spend -run '^$' -fuzz '^FuzzRoutingReaderBoundedFormats$' -fuzztime 1000x -parallel 2 -timeout 2m diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4d14aea..d42efca 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -58,6 +58,10 @@ jobs: jq -e 'any(.packages[]; .name == "github.com/llm-measurement/llm-sketchkit") and any(.packages[]; .name == "go.opentelemetry.io/proto/slim/otlp") and any(.packages[]; .name == "go.yaml.in/yaml/v3") and + any(.packages[]; .name == "github.com/jackc/pgx/v5" and .versionInfo == "v5.11.0") and + any(.packages[]; .name == "github.com/jackc/pgpassfile" and .versionInfo == "v1.0.0") and + any(.packages[]; .name == "github.com/jackc/pgservicefile" and .versionInfo == "v0.0.0-20240606120523-5a60cdf6a761") and + any(.packages[]; .name == "golang.org/x/text" and .versionInfo == "v0.41.0") and any(.packages[]; .name == "google.golang.org/protobuf")' sbom.spdx.json >/dev/null mv sbom.spdx.json dist/ cd dist @@ -108,6 +112,10 @@ jobs: - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: '1.26.9' + - name: Native export safety on each supported platform + run: | + go test ./internal/postgres -count=1 + go test ./internal/cli -run '^TestProductionHasNoNetwork$' -count=1 - name: Verify spend, summary, trace, configuration, and scan commands from the packaged binary env: FLEETDIFF_RELEASE_BINARY: ${{ github.workspace }}/unpacked/fleetdiff @@ -117,6 +125,7 @@ jobs: timeout-minutes: 10 env: FLEETDIFF_BIN: ${{ github.workspace }}/unpacked/fleetdiff + FLEETDIFF_LINUX_BIN: ${{ github.workspace }}/unpacked/fleetdiff FLEETDIFF_SPEND_DOCKER: '1' run: | docker pull ghcr.io/berriai/litellm@sha256:625981c83410a3ea68eb0697590a57ec1d764d634514d54fa5db0591077ee839 diff --git a/CHANGELOG.md b/CHANGELOG.md index 20fe2b5..741732f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,31 @@ Notable user-visible changes are recorded here, newest first. +## [Unreleased] - v0.12.0 Candidate + +See the [unreleased notes](docs/releases/v0.12.0.md) and +[native export guide](docs/NATIVE_EXPORT.md). These features are not in the +released v0.11.0 installer. + +- Add explicit native `export litellm` for private CSV/JSONL snapshots, optional + daily counters and routing details, with the pinned LiteLLM v1.104.0 projection. +- Add `--litellm-db` to `investigate` and `what-if`, with automatic private-export + cleanup or `--keep-export NEWDIR`. Default to the last 14 complete UTC days; + preserve what-if `--from` / `--to` as model selectors and `--period` as time. +- Introduce an explicit runtime network boundary for database modes only; file + commands remain offline. Require dedicated credential sources, verified remote + TLS, bounded read-only snapshots, sanitized errors and new private output paths. +- Stage exports in a private mode-0700 sibling `.fleetdiff-stage-...` directory, then + atomically publish the complete directory without replacement. Keep the final + destination absent until publication; crashes beforehand may leave sibling + staging data, never a partial final directory. +- Order candidate SQL exports deterministically by `"startTime", request_id + COLLATE "C"` and daily `date COLLATE "C"` for native/psql byte equivalence. + Compared with the earlier unsorted recipe, bytes and first-seen/tied report + ordering may change; recorded totals and identity hashing are unchanged. +- Pin `pgx/v5 v5.11.0`, `pgpassfile v1.0.0` and + `pgservicefile v0.0.0-20240606120523-5a60cdf6a761`; retain `x/text v0.41.0`. + ## [0.11.0] - 2026-10-17 See the [release notes](docs/releases/v0.11.0.md) for routing evidence and diff --git a/README.md b/README.md index 5e04398..be378a9 100644 --- a/README.md +++ b/README.md @@ -7,45 +7,61 @@ compares LLM usage summaries, flags unusual windows, and checks trace captures and collector configurations. Use it to answer **why LLM token usage spiked, and which users or sessions drove -the change.** It reads your files and prints a report: no account, upload, -collector or model API key is needed for the LiteLLM path. +the change.** File-based commands stay offline. The unreleased native LiteLLM +path reads your database explicitly, then reports locally: no fleetdiff account, +upload, collector or model API key is needed. | | | | --- | --- | -| Commands | `investigate`, `scan`, `compare`, `inspect`, `diagnose` | +| Commands | `investigate`, `what-if`, `scan`, `compare`, `inspect`, `diagnose`; candidate: `export litellm` | | Platforms | Linux and macOS, AMD64 and ARM64 | | Status | Pre-1.0; see the [changelog](CHANGELOG.md) | | License | Apache-2.0 | ## Using LiteLLM? -**Export -> install -> investigate.** Use the request logs you already retain to -compare two periods, see which models and keys drove the change, and check -whether the usage records are complete. +**One command to the first report, after setup.** Compare two periods, see which +models and keys drove the change, and check the usage records you already retain. +The native path below is an **unreleased v0.12 candidate**, not a feature of the +released v0.11.0 installer. +For a released binary, use the fallback below. -### 1. Export Request-Level Rows +### Native Candidate Setup (Development Only) -Use the [read-only SQL projection](examples/litellm-spend/export.sql) and your -existing PostgreSQL access. The [export guide](examples/litellm-spend/README.md#export-request-level-rows) -covers connection settings and version-pinned cache fields. Once your approved -read-only connection is named `litellm_readonly`, this example exports 16 UTC days: +From a reviewed candidate source checkout with a patched Go 1.26 toolchain, +build into a new directory; stop if that directory already exists: ```sh -umask 077 -export_dir="$HOME/litellm-export" -mkdir "$export_dir" -curl --fail --location --proto '=https' --tlsv1.2 \ - https://raw.githubusercontent.com/llm-measurement/fleetdiff/v0.11.0/examples/litellm-spend/export.sql \ - --output "$export_dir/export.sql" -psql 'service=litellm_readonly' -Xq -v ON_ERROR_STOP=1 \ - -v start_utc=2026-09-23T00:00:00Z -v end_utc=2026-10-09T00:00:00Z \ - -v csv_file="$export_dir/spend.csv" -f "$export_dir/export.sql" \ - >"$export_dir/private-psql.out" 2>"$export_dir/private-psql.err" +mkdir ./fleetdiff-install && +go build -o ./fleetdiff-install/fleetdiff ./cmd/fleetdiff +``` + +Have your administrator provision the [read-only role and credential file](docs/NATIVE_EXPORT.md#credentials). +Set `DATABASE_URL_FILE` to that existing, owned mode-0600 regular file. Leave +`FLEETDIFF_LITELLM_DATABASE_URL` unset when using the file, then run: + +```sh +: "${DATABASE_URL_FILE:?Set the path to your approved database URL file}" +./fleetdiff-install/fleetdiff investigate --litellm-db \ + --database-url-file "$DATABASE_URL_FILE" --group-by key ``` -Choose dates for your own data and continue only after the export succeeds. -The SQL writes CSV in one pass and excludes prompts and responses. Keep the -export directory private. Existing request-level JSON and JSONL work too. +This connects only for the requested export, compares the last 14 complete UTC +days as two seven-day periods, and removes its private temporary export after +analysis. Add `--daily` for row coverage or `--routing-audit` for route evidence; +routing details are included automatically. Use `--keep-export NEWDIR` to retain +the private files, or `export litellm --out NEWDIR` to export without a report. +The [native guide](docs/NATIVE_EXPORT.md) covers credentials, periods and what-if. + + + +### Released Fallback: Export With psql + +For v0.11.0, or client certificates, PostgreSQL services and other native-unsupported +connections, follow the [read-only SQL recipe](examples/litellm-spend/README.md#export-request-level-rows). +Keep the export private; it excludes prompts and responses but retains identities. +Existing request-level CSV, JSON and JSONL also work. Install the released binary +in a separate new directory from any candidate build, then investigate the file. @@ -63,13 +79,14 @@ bash -o pipefail -c 'curl --fail --location --proto "=https" --tlsv1.2 https://r ./fleetdiff-install/fleetdiff --version ``` -The installer checks signed release provenance and checksums before extracting -the binary into a new directory. Prefer to inspect the script first? Follow +The v0.11.0 installer does not provide native database export. It checks signed +release provenance and checksums before extracting into a new directory. Follow [Install and verify](docs/OPERATIONS.md#install-and-verify). ### 3. Investigate ```sh +export_dir="$HOME/litellm-export" ./fleetdiff-install/fleetdiff investigate \ --litellm-spend "$export_dir/spend.csv" --group-by key ``` @@ -127,7 +144,7 @@ The summary-file path also shows session concentration: ## Install -The verified installer above selects the native binary for your machine. +The verified v0.11.0 installer selects the platform binary for your machine. For manual downloads, use the [v0.11.0 archives](https://github.com/llm-measurement/fleetdiff/releases/tag/v0.11.0) and [verification instructions](docs/OPERATIONS.md#install-and-verify). An administrator can verify and transfer artifacts through an approved internal mirror. @@ -150,13 +167,14 @@ See [source builds and version reporting](docs/OPERATIONS.md#build-from-source). receive instructions for getting those rows. The reader accepts up to 10M rows and 16 GiB per file, with [per-record limits](examples/litellm-spend/README.md#data-and-limits). - By default, private names use aliases or pseudonymous hashes; public catalog model IDs are readable. - keep source files private and share reports under your organization's policy. + Keep source files private and share reports under your organization's policy. - Rankings carry deterministic bounds; distinct counts are statistical estimates. A flagged contributor identifies something to investigate, rather than its cause. Compatible summary files need aligned windows, keys and disjoint producer coverage. ## Documentation +- [Native LiteLLM export](docs/NATIVE_EXPORT.md): unreleased v0.12 candidate setup and connection contract. - [LiteLLM spend exports](examples/litellm-spend/README.md): SQL, periods, groupings and source accounting. - [Session-budget review](docs/SESSION_BUDGET.md): single-period recorded totals and cap review. - [Model-mix what-if](docs/WHAT_IF.md) (v0.10.0): re-price covered usage under explicit assumptions. diff --git a/SECURITY.md b/SECURITY.md index 9824334..ebb11a3 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -42,9 +42,45 @@ contractual response-time commitment. per-run key and aliases for shared reports, and remove captures after inspection. Raw values are transiently decoded in process memory; this is not secure erasure. -The comparison and inspection commands read local files and write stdout/stderr -without network access. Inspection also accepts stdin. Initial builds may download dependencies, and the live example starts -local Docker collectors and may download its pinned image. See the +File-based commands read local files and write stdout/stderr without network +access; inspection also accepts stdin. Merely setting database credentials does +not enable a connection. Initial builds may download dependencies, and the live +example starts local Docker collectors and may download its pinned image. + +## Native Database Export (Unreleased v0.12 Candidate) + +Only explicit `export litellm`, `investigate --litellm-db` and `what-if --litellm-db` +connect to PostgreSQL. This adds a network boundary, not telemetry, update checks +or provider calls. The released v0.11.0 binary does not include these modes. + +- Supply exactly one credential source: `FLEETDIFF_LITELLM_DATABASE_URL` or + `--database-url-file` pointing to an owned regular mode-0600 file. Literal + URL arguments and symlink components are rejected. Native export ignores + inherited `PG*` settings, `~/.pgpass`, PostgreSQL service files and implicit + client certificates. It never prints connection details or raw driver errors. +- Remote connections require `sslmode=verify-full`; `verify-ca` is not accepted. + Only numeric loopback addresses or Unix sockets may use `sslmode=disable`. + The query allowlist is `sslmode`, optional `sslrootcert`, and `host` only for + an absolute Unix socket directory. Client-certificate and service-based setups + use the separately reviewed `psql` recipe, not a native TLS bypass. +- Use a dedicated SELECT-only role. Every native export uses one read-only, + repeatable-read snapshot with bounded statements and idle time. Role creation + and grants are administrator actions; fleetdiff never changes the database. +- Native export writes private local files. Destinations must be new, with no + symlink components; existing directories, even empty ones, and overwrites are + refused. Files are built in a mode-0700 sibling `.fleetdiff-stage-...` directory, + with mode-0600 files. The final destination stays absent until the complete + directory is atomically published without replacement, including when another + process creates the destination during export. Convenience modes remove + temporary exports unless `--keep-export NEWDIR` is selected. +- Exports retain raw identities. Reports keep existing aliases and keyed-hash + rules; optional label/hash disclosure remains explicit. A crash before + publication may leave private sibling `.fleetdiff-stage-...` staging data, never a + partial final directory. After publication, the final directory is complete. + Remove only the run's abandoned staging data under your retention policy. + +See the [native export contract](docs/NATIVE_EXPORT.md) for setup, exact grants, +limits and failure handling. See the [comparison contract](docs/COMPARISON.md) for input limits and trust assumptions. See [inspection](docs/INSPECT.md) for bounded decoding, source semantics, and key handling. See [operations](docs/OPERATIONS.md) for installation verification, isolated diff --git a/cmd/fleetdiff/main.go b/cmd/fleetdiff/main.go index e02762d..41e07b8 100644 --- a/cmd/fleetdiff/main.go +++ b/cmd/fleetdiff/main.go @@ -4,9 +4,46 @@ package main import ( + "context" + "io" "os" + "os/signal" + "syscall" "github.com/llm-measurement/fleetdiff/internal/cli" + "github.com/llm-measurement/fleetdiff/internal/exportdata" + "github.com/llm-measurement/fleetdiff/internal/postgres" ) -func main() { os.Exit(cli.Run(os.Args[1:], os.Stdout, os.Stderr)) } +func main() { os.Exit(run()) } + +func run() int { + return runWithInput(os.Stdin) +} + +func runWithInput(in io.Reader) int { + return cli.RunWithExporter(context.Background(), os.Args[1:], in, os.Stdout, os.Stderr, withExportSignals(postgres.Export)) +} + +func withExportSignals(exporter exportdata.Exporter) exportdata.Exporter { + return func(ctx context.Context, opts exportdata.Options) (exportdata.Result, error) { + ctx, cancel := context.WithCancel(ctx) + interrupts := make(chan os.Signal, 1) + signal.Notify(interrupts, os.Interrupt, syscall.SIGTERM) + stopped := make(chan struct{}) + go func() { + // The first signal requests cleanup. A repeated signal may terminate + // an unresponsive exporter and leave owner-only temporary data. + select { + case <-interrupts: + signal.Stop(interrupts) + cancel() + case <-ctx.Done(): + signal.Stop(interrupts) + } + close(stopped) + }() + defer func() { cancel(); <-stopped }() + return exporter(ctx, opts) + } +} diff --git a/cmd/fleetdiff/main_test.go b/cmd/fleetdiff/main_test.go new file mode 100644 index 0000000..b35b84d --- /dev/null +++ b/cmd/fleetdiff/main_test.go @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package main + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/llm-measurement/fleetdiff/internal/cli" + "github.com/llm-measurement/fleetdiff/internal/exportdata" +) + +func TestSignalsOfflineAndDatabase(t *testing.T) { + for _, phase := range []string{"offline", "after-export", "cancel-export", "stuck-export"} { + for _, signal := range []os.Signal{os.Interrupt, syscall.SIGTERM} { + t.Run(phase+"/"+signal.String(), func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestSignalProcessHelper$") + cmd.Env = append(os.Environ(), "FLEETDIFF_SIGNAL_TEST="+phase) + var stderr bytes.Buffer + cmd.Stderr = &stderr + stdout, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + stdin, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + defer stdin.Close() + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer cmd.Process.Kill() + r := bufio.NewReader(stdout) + ready, err := r.ReadString('\n') + if err != nil || !strings.HasPrefix(ready, "ready") { + t.Fatalf("helper not ready: %q %v", ready, err) + } + if err := cmd.Process.Signal(signal); err != nil { + t.Fatal(err) + } + if phase == "stuck-export" { + line, err := r.ReadString('\n') + if err != nil || line != "canceled\n" { + t.Fatalf("signal not delivered to exporter: %q %v", line, err) + } + if err := cmd.Process.Signal(signal); err != nil { + t.Fatal(err) + } + } + err = cmd.Wait() + if err == nil || ctx.Err() != nil { + t.Fatalf("signal did not terminate helper: %v timeout=%v stderr=%s", err, ctx.Err(), &stderr) + } + if phase == "cancel-export" || phase == "stuck-export" { + directory := strings.TrimSpace(strings.TrimPrefix(ready, "ready ")) + if phase == "cancel-export" { + if _, err := os.Stat(filepath.Dir(directory)); !os.IsNotExist(err) { + t.Fatalf("handled export cancellation left temporary data: %v", err) + } + if strings.Contains(stderr.String(), directory) { + t.Fatal("cancellation exposed temporary path") + } + } else { + _ = os.RemoveAll(filepath.Dir(directory)) + } + } + }) + } + } +} + +type signalReadyReader struct{ io.Reader } + +func (r signalReadyReader) Read(p []byte) (int, error) { + fmt.Println("ready") + return r.Reader.Read(p) +} + +func TestSignalProcessHelper(t *testing.T) { + phase := os.Getenv("FLEETDIFF_SIGNAL_TEST") + if phase == "" { + return + } + if phase == "offline" || phase == "after-export" { + if phase == "after-export" { + _, _ = withExportSignals(func(context.Context, exportdata.Options) (exportdata.Result, error) { + return exportdata.Result{}, nil + })(context.Background(), exportdata.Options{}) + } + os.Args = []string{"fleetdiff", "inspect", "-"} + os.Exit(runWithInput(signalReadyReader{os.Stdin})) + } + exporter := withExportSignals(func(ctx context.Context, opts exportdata.Options) (exportdata.Result, error) { + if err := os.Mkdir(opts.Directory, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(opts.Directory, "spend.csv"), []byte("private test data"), 0600); err != nil { + t.Fatal(err) + } + fmt.Println("ready " + opts.Directory) + <-ctx.Done() + if phase == "stuck-export" { + fmt.Println("canceled") + select {} + } + return exportdata.Result{}, exportdata.ErrInterrupted + }) + os.Exit(cli.RunWithExporter(context.Background(), []string{"investigate", "--litellm-db"}, os.Stdin, os.Stdout, os.Stderr, exporter)) +} diff --git a/docs/COMPARISON.md b/docs/COMPARISON.md index 4de361f..431cba3 100644 --- a/docs/COMPARISON.md +++ b/docs/COMPARISON.md @@ -83,7 +83,10 @@ does not bypass scope, key, accounting, replay, or structural checks. ## Trust And Effects -The runtime reads local files and writes stdout/stderr, offline and read-only. +File-based comparison reads local files and writes stdout/stderr, offline and +without modifying inputs. The unreleased v0.12 candidate's explicit +[native database modes](NATIVE_EXPORT.md) have a separate network and private-file +write boundary; they do not change this summary comparison contract. Transfer inputs through authenticated channels and follow [Safe Use](../SECURITY.md#safe-use) for producer and metadata trust. diff --git a/docs/INSPECT.md b/docs/INSPECT.md index a7408d3..e76939f 100644 --- a/docs/INSPECT.md +++ b/docs/INSPECT.md @@ -122,8 +122,10 @@ interval. Frequent-item bounds are deterministic for retained observed weight. ## Privacy -The CLI makes no network requests, opens no receiver, and writes no files. It -holds decoded raw input transiently in memory to compute the report. A fresh +The file-based `inspect` command makes no network requests, opens no receiver, +and writes no files. The unreleased v0.12 candidate's explicit +[native database modes](NATIVE_EXPORT.md) do not change inspection's offline +behavior. Inspection holds decoded raw input transiently in memory to compute the report. A fresh cryptographic key is created for each run; aggregate state retains hashes and counts rather than raw values. Garbage collection is not a secure memory eraser. diff --git a/docs/NATIVE_EXPORT.md b/docs/NATIVE_EXPORT.md new file mode 100644 index 0000000..e5512e7 --- /dev/null +++ b/docs/NATIVE_EXPORT.md @@ -0,0 +1,269 @@ +# Native LiteLLM Export + +**Get a spend investigation directly from your existing LiteLLM database.** +fleetdiff exports a private read-only snapshot, analyzes it locally, and removes +the temporary files unless you ask to keep them. No `psql` installation is needed +for the native path. + +This is an **unreleased v0.12 candidate**. Build a reviewed candidate checkout +using the [development setup](OPERATIONS.md#native-candidate-build-development-only). +The released v0.11.0 installer does not include these commands; use the +[manual SQL export](../examples/litellm-spend/README.md#export-request-level-rows) +with that binary. The examples below run from the candidate checkout. + +## First Report + +After the administrator setup below, set `DATABASE_URL_FILE` to the approved +credential file's path. Leave `FLEETDIFF_LITELLM_DATABASE_URL` unset when using it: + +```sh +: "${DATABASE_URL_FILE:?Set the path to your approved database URL file}" +./fleetdiff-install/fleetdiff investigate --litellm-db \ + --database-url-file "$DATABASE_URL_FILE" --group-by key +``` + +The default compares the last 14 complete UTC days as two adjacent seven-day +periods, ending at today's UTC midnight. This is based on the clock, not the +first and last rows found. Empty or sparse logs do not establish complete coverage. +Add `--daily` to compare against daily counters, or `--routing-audit` to review +the models serving each original route. + +For chosen comparison windows, use both `--before-period` and `--after-period` +with UTC dates or RFC3339 `start/end` intervals. They must be complete, ordered, +nonoverlapping and equal-duration. Single-period session review uses `--period` +and/or `--session-budget-usd`, as in the [session guide](SESSION_BUDGET.md). + +## Credentials + +Choose exactly one source: + +- `--database-url-file FILE`: an existing regular file owned by the effective + user, with mode exactly 0600 and no symlink path components. Have your approved + secret-delivery process provision it in a private directory outside the checkout. + `DATABASE_URL_FILE` in the examples is only a shell variable holding that path. +- `FLEETDIFF_LITELLM_DATABASE_URL`: the URL supplied by an approved environment + secret mechanism. Omit `--database-url-file` in this case. Do not type a + password-bearing assignment into shell history or print the environment. + +Literal URL arguments are rejected, including a URL passed to a file option. +Do not put credentials in scripts, terminal recordings, reports or issue text. +The examples contain no stored password. This password-free URL illustrates the +shape only; it is not a working credential or a command-line argument: + +```text +postgresql://fleetdiff_reader@db.example.invalid/litellm?sslmode=verify-full +``` + +The URL must name one user, database and host. If the server requires a password, +the approved secret mechanism supplies it, URL-encoded, in the credential source. +Native export does not recover a missing password from any other file. + +Remote connections require `sslmode=verify-full`, verifying both the certificate +chain and the database hostname. Optional `sslrootcert` selects a local CA file; +otherwise system trust roots apply. `verify-ca`, opportunistic TLS and remote +plaintext are rejected. Only a numeric loopback address, such as `127.0.0.1` +or `::1`, or a Unix socket may use explicit `sslmode=disable`; the name `localhost` +does not qualify for the plaintext exception. + +The only URL query keys are `sslmode`, optional `sslrootcert`, and `host` for an +absolute Unix socket directory only. A Unix socket URL omits the authority host +and uses `sslmode=disable`. Duplicate or other query keys are rejected. No native +client-certificate, service, passfile, arbitrary connection-option or multi-host +fallback is supported. Use the manual `psql` recipe with administrator-approved +settings for those setups, not a native security override. + +Native configuration neutralizes all inherited `PG*` variables and does not load +`~/.pgpass`, `~/.pg_service.conf`, system service files or implicit client +certificates. Installing a driver dependency does not enable these settings. +Diagnostics omit the URL, host, user, password and raw driver/server errors. + +## Administrator Setup + +Have the database administrator review and run these statements in the intended +LiteLLM database, replacing the example database name `litellm` as needed. +**These are administrator provisioning actions, never SQL executed by fleetdiff.** +The new login owns no database objects and needs no privileged role memberships. + +```sql +CREATE ROLE fleetdiff_reader LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE + NOINHERIT NOREPLICATION NOBYPASSRLS; +GRANT CONNECT ON DATABASE litellm TO fleetdiff_reader; +GRANT USAGE ON SCHEMA public TO fleetdiff_reader; +GRANT SELECT ( + request_id, call_type, api_key, "user", team_id, organization_id, end_user, + model, model_id, model_group, custom_llm_provider, + prompt_tokens, completion_tokens, total_tokens, spend, status, + session_id, agent_id, "startTime", "endTime", metadata +) ON TABLE public."LiteLLM_SpendLogs" TO fleetdiff_reader; +ALTER ROLE fleetdiff_reader SET default_transaction_read_only = on; +``` + +Only when daily counters are needed, also grant: + +```sql +GRANT SELECT (date, api_requests, successful_requests, failed_requests) + ON TABLE public."LiteLLM_DailyUserSpend" TO fleetdiff_reader; +``` + +No sequence, write, schema-create or all-table grants are needed. Column-level +SELECT excludes message/response columns, but `metadata` itself is sensitive; +only the fixed projection is exported. The administrator must review existing +`PUBLIC` grants, memberships and row-security policies: these grants do not +remove privileges received elsewhere. Configure authentication through the +normal approved process; no password is set in this SQL. The role default is +defense in depth; every tool transaction independently selects read-only mode. +See PostgreSQL's [GRANT](https://www.postgresql.org/docs/17/sql-grant.html) and +[CREATE ROLE](https://www.postgresql.org/docs/17/sql-createrole.html) documentation. + +## Export Without Analysis + +Choose a destination that does not yet exist under an existing, trusted parent. +Replace these historical UTC bounds with your own complete interval: + +```sh +export_dir="$HOME/litellm-export" +./fleetdiff-install/fleetdiff export litellm \ + --database-url-file "$DATABASE_URL_FILE" --out "$export_dir" \ + --from 2026-10-01T00:00:00Z --to 2026-10-15T00:00:00Z \ + --format csv --daily --routing-details +``` + +Omit both `--from` and `--to` for the default last 14 complete UTC days. Supply +both to override it; the start is inclusive and the end exclusive. Requests are +selected by `startTime`, interpreted as UTC under the pinned schema. + +| Option | Output | +| --- | --- | +| `--format csv` (default) | `spend.csv` | +| `--format jsonl` | `spend.jsonl`, instead of CSV | +| `--daily` | Adds `daily.csv`, always CSV, from the same snapshot | +| `--routing-details` | Adds the three pinned routing columns; default off for standalone export | +| `--usage-details=false` | Omits default-on cache and reasoning detail columns | + +Native export does not emit JSON arrays. Existing JSON arrays remain accepted by +file-based readers. `--daily` requires midnight-to-midnight UTC bounds. Review +[routing fields](../examples/litellm-spend/README.md#routing-details-opt-in), +[usage details](../examples/litellm-spend/README.md#usage-details-default-on) and +[daily coverage](ROW_COVERAGE.md) for their accounting meaning. + +For later offline analysis of the example above: + +```sh +./fleetdiff-install/fleetdiff investigate --litellm-spend "$export_dir/spend.csv" \ + --litellm-daily-spend "$export_dir/daily.csv" \ + --before-period 2026-10-01T00:00:00Z/2026-10-08T00:00:00Z \ + --after-period 2026-10-08T00:00:00Z/2026-10-15T00:00:00Z +``` + +## Keep An Export Or Run What-If + +Both convenience modes create private temporary CSV exports and remove them +after analysis, including handled failures. `--keep-export NEWDIR` retains a +completed export, even if later analysis fails; the destination must be new. +Do not combine `--litellm-db` with local spend, daily-spend or summary inputs. + +```sh +./fleetdiff-install/fleetdiff investigate --litellm-db \ + --database-url-file "$DATABASE_URL_FILE" --routing-audit \ + --keep-export "$HOME/litellm-routing-export" +``` + +Convenience exports include routing details automatically, so routing audits, +route/agent scopes, scenario selectors and workload maps have those fields +without another export option. Missing source metadata stays missing. Existing +scope rules still apply: scoped investigations require explicit before/after +periods and cannot combine scope with daily coverage or session grouping. + +For what-if, set `FROM_MODEL` and `TO_MODEL` to reviewed model IDs supported by +your price catalog. The flags `--from` and `--to` **always name models** on this +command; **`--period` selects the UTC date or date interval**, not those flags: + +```sh +: "${FROM_MODEL:?Set the source model ID}" "${TO_MODEL:?Set the target model ID}" +./fleetdiff-install/fleetdiff what-if --litellm-db \ + --database-url-file "$DATABASE_URL_FILE" \ + --period 2026-10-01T00:00:00Z/2026-10-15T00:00:00Z \ + --from "$FROM_MODEL" --to "$TO_MODEL" --fraction 0.5 +``` + +Without `--period`, database what-if uses the default export interval. It also +accepts `--keep-export NEWDIR`. Its `--daily` option retains daily counters in the +export; it does not add row-coverage analysis to what-if. `--format text|json` +on either convenience command controls the report, not the private CSV format. +See [what-if accounting](WHAT_IF.md) for coverage, prices and assumptions. + +## Snapshot And Ordering + +The exporter checks the required table/column shapes and spend request-ID +primary key against the LiteLLM v1.104.0 profile in `public`. Unsupported shapes +fail rather than guessing. All reads use one +`BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY` transaction with UTC settings; +fleetdiff issues no database writes or schema migrations. + +The candidate's shared SQL orders spend rows by `"startTime", request_id COLLATE +"C"` and daily rows by `date COLLATE "C"`. For the same database snapshot, bounds, +format and detail options, native and candidate `psql` exports must be +byte-for-byte identical. Sequential exports from a changing live database are +not the same snapshot. This ordering is new relative to the v0.11.0 recipe: +it can change bytes, first-seen aliases and tied report order, not recorded +totals. Database sorting can also affect export time. + +## Limits And Failures + +| Native export bound | Value | +| --- | --- | +| Spend rows | 10,000,000 | +| Spend file | 16 GiB | +| Encoded row / field | 64 KiB / 8 KiB | +| Statement / idle-in-transaction timeout | 2 minutes each | +| Daily interval / file | 3,660 complete UTC days / 1 MiB | + +The existing [spend reader limits](../examples/litellm-spend/README.md#data-and-limits) +also apply before publication. A bound or timeout fails the export, never silently +truncates it. Narrow the interval with your database operator when needed. + +The exporter uses a private mode-0700 sibling `.fleetdiff-stage-...` +staging directory under the destination's parent, with mode-0600 files. Staging +is not inside `--out`. The final destination stays absent while all files are +written and validated. Only the complete directory is then published in one +atomic no-replace operation; files are not published individually. Existing +destinations, including empty directories, and symlink components are rejected. +The parent and its ancestors must be owned by you or root, and must not be +writable by others unless protected by the sticky bit (as in a standard `/tmp`). +The filesystem must support an atomic rename that refuses existing destinations; +there is no overwriting fallback. +If another process creates the final destination during export, publication +fails without replacing or removing that destination. + +Handled failures before publication remove staging data, not a partial final +directory. A crash before publication may leave only the private sibling +`.fleetdiff-stage-...` directory; it never exposes a partial export at the final path. +After publication, the final directory is complete, even if the process exits +before reporting success. Do not consume staging files; remove only the run's +abandoned staging directory under your retention policy. Publication does not +establish that a subsequent investigation or what-if analysis succeeded. +Exit 0 indicates command success, 1 an export/analysis/output failure, and 2 +invalid command options. + +Errors deliberately suppress connection details and raw database diagnostics. +Check credential source and permissions, verified TLS, read grants, schema pin, +bounds and a new output path; use approved private server logs for investigation. +Do not publish credentials, raw exports or server diagnostics in bug reports. + +## Privacy And Verification + +File-based commands stay offline, even with credentials in the environment. +Only explicit native database modes connect; there are no telemetry, update or +provider requests. DNS may resolve the chosen database host. Native exports +retain private keys, users, sessions and routing identities, while excluding +prompts, responses, raw error messages and whole metadata objects. Reports keep +existing aliases and keyed-hash rules; public catalog model IDs remain readable. +Keep exports private and review reports under your organization's sharing policy. +Use `umask 077` when redirecting reports; shell output permissions are separate +from native export permissions. + +The candidate pins `pgx/v5 v5.11.0`, `pgpassfile v1.0.0` and +`pgservicefile v0.0.0-20240606120523-5a60cdf6a761`; existing `x/text v0.41.0` +normalization is retained. Dependency names do not imply passfile/service support. +The [candidate verification checklist](../examples/litellm-spend/TESTING.md#native-export-candidate-checks) +lists the database, security and equivalence tests. diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index cb936c5..6fe826d 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -1,8 +1,10 @@ # Operating fleetdiff -fleetdiff is one local binary. It reads LiteLLM spend exports, summary files, -OTLP captures or collector configurations and prints a report, offline and -unprivileged. All input and output stays on your machine. +fleetdiff is one local binary. File-based commands read LiteLLM spend exports, +summary files, OTLP captures or collector configurations and report offline, +without administrator privileges. The **unreleased v0.12 candidate** also offers +explicit native database export; that mode connects to PostgreSQL and writes +private local exports before analysis. It is not in the v0.11.0 installer. ## Install And Verify @@ -148,17 +150,61 @@ record the commit and local changes separately. Explicit packaging stamps take precedence over module metadata. `sh scripts/build-release.sh dev` explicitly stamps `dev`, unlike the unstamped build above. -Builds can download Go dependencies and toolchains. Runtime comparison cannot. +Builds can download Go dependencies and toolchains. File-based runtime commands +cannot. For an offline build, prepare the approved toolchain and module cache in a connected build environment; use `GOTOOLCHAIN=local GOPROXY=off GOSUMDB=off` only after verifying those cached dependencies there. Do not disable checksum verification for ordinary connected builds. An offline user normally needs only the reviewed binary, not Go, Git, Docker, or a compiler. +### Native Candidate Build (Development Only) + +Use a reviewed candidate checkout and a new build directory, separate from a +released installation. Stop if directory creation or the build fails: + +```sh +mkdir ./fleetdiff-install && +go build -o ./fleetdiff-install/fleetdiff ./cmd/fleetdiff +``` + +This is development setup, not a verified release installation. The v0.11.0 +installer and tagged Go install above do not contain native export. Follow +[Native LiteLLM Export](NATIVE_EXPORT.md) for administrator-approved credentials, +read-only grants and a first report. Dependency downloads are build-time access, +separate from the explicit runtime database connection. + +## Native Database Runs + +Only `export litellm` and `investigate` / `what-if` with `--litellm-db` connect. +File inputs remain offline even when database credentials are present. Restrict +network access to the approved database endpoint and any required DNS resolution; +there are no provider calls, telemetry or update checks. + +Provision a dedicated read-only role and supply credentials through the +[native credential contract](NATIVE_EXPORT.md#credentials). Remote connections +require verified TLS. PostgreSQL services and client-certificate authentication +belong on the manual `psql` path; native export does not inherit those settings. + +Native destinations must be new private directories, not existing empty ones. +Files are built in a private mode-0700 sibling `.fleetdiff-stage-...` directory, +then the complete export is published atomically as one +directory without replacing anything. The final destination stays absent until +successful publication; overwrites and symlink components are refused. A crash +before publication may leave sibling staging data, never a partial final +directory. After publication, any final directory is complete. + +Temporary convenience exports are cleaned up after analysis; `--keep-export +NEWDIR` retains a completed export for later offline use. Reports redirected by +the shell still need `umask 077`. After an externally killed run, remove only its +abandoned sibling staging directory; do not consume staging files as an export. See +[limits and failures](NATIVE_EXPORT.md#limits-and-failures) before scheduling runs. + ## Run On Real Exports -1. Use a dedicated, unprivileged account. Grant it read access only to approved - completed exports. Do not grant credentials to the model or collector. +1. For offline analysis, use a dedicated, unprivileged account. Grant it read + access only to approved completed exports. Do not grant credentials to the + model or collector. 2. Freeze a copy of both windows in directories that other users cannot rewrite during comparison. Symlinks and special files are rejected; parent-directory ownership and a trustworthy filesystem remain the operator's responsibility. @@ -192,7 +238,8 @@ The limits below apply to `compare` and summary-file `investigate`. The other commands have separate guides: [diagnose](DIAGNOSE.md) bounds static YAML inspection; [scan](SCAN.md) applies the summary input budget to the whole history and also limits candidate comparisons. The optional [archive helper](SUMMARY_ARCHIVE.md) -is a separate, explicitly writing process; fleetdiff itself stays read-only. +is a separate, explicitly writing process. File-based analysis does not modify +inputs; native database export writes only its private local export files. Each side permits at most 512 JSON files, 1,024 directory entries, and 32 MiB of encoded input. One summary is limited to 8 MiB. Expected producers are limited @@ -207,9 +254,10 @@ boundary. See [measured input cases](BENCHMARKS.md); size from your real fixture and retain headroom. Start with one comparison process, not unlimited parallel jobs. The demonstration's 256 MiB container test covers its small sample only. -CI checks the packaged Linux binary with no network, non-root UID, no Linux -capabilities, a read-only root, and read-only input mounts. Apply your organization's -software approval process to the binary and deployment configuration. +CI checks file-based commands in the packaged Linux binary with no network, +non-root UID, no Linux capabilities, a read-only root, and read-only input mounts. +Apply your organization's software approval process to the binary and deployment +configuration. ## Automation diff --git a/docs/calibration/manifest.json b/docs/calibration/manifest.json index 829cd56..5f29422 100644 --- a/docs/calibration/manifest.json +++ b/docs/calibration/manifest.json @@ -1,7 +1,7 @@ { "schema": "fleetdiff-calibration-publication/v1", "sha256": { - ".github/workflows/ci.yml": "5812e4a690ebbbdbd9ca450a9aaf5aba6c524befdf8d6ef4a052dd37a192859d", + ".github/workflows/ci.yml": "41b15ed949e951cfd799238db5ba3ade092e51ec20644aad7c68edfcf9424be2", "docs/WHAT_IF.md": "bf5be3a7eade73aabc3d1492b7408eeb051e09f5b2a2abaa82507c5dbbccb5c6", "docs/WHAT_IF_CALIBRATION.md": "9e929d6793a8f243950896b7c4672b732e9579687e2ac40dfed9af38f44920da", "docs/calibration/LICENSES.md": "fce7715504d899724474f0519b77ab0ddc5fabe95278ddb437c200b2ed2fdd7d", diff --git a/docs/releases/v0.12.0.md b/docs/releases/v0.12.0.md new file mode 100644 index 0000000..6ef5d34 --- /dev/null +++ b/docs/releases/v0.12.0.md @@ -0,0 +1,61 @@ +# v0.12.0 Release Notes (Unreleased) + +**Investigate LiteLLM spend directly from a read-only database snapshot.** This +candidate adds native export and a one-command export-then-analyze path, keeping +existing file-based analysis offline. + +This version is **unreleased**. Use a reviewed checkout and the +[development-only candidate build](../OPERATIONS.md#native-candidate-build-development-only) +to try it. The released v0.11.0 installer does not provide these features. + +## Native LiteLLM Export + +- `export litellm --out NEWDIR` writes CSV or JSONL with optional daily counters + and routing details, using the pinned LiteLLM v1.104.0 SQL projection. Usage + details are on by default; `--usage-details=false` omits them. +- `investigate --litellm-db` and `what-if --litellm-db` export privately, analyze + locally and clean up automatically. `--keep-export NEWDIR` retains the files. + Routing details are included automatically for audit and scope use. +- Database periods default to the last 14 complete UTC days. Standalone export + uses `--from` / `--to` for UTC bounds. What-if keeps those flags for model IDs + and uses `--period` for its date or interval. + +See the [native guide](../NATIVE_EXPORT.md) for first-run examples, exact +administrator-only read grants and the connection contract. + +## Explicit Network Boundary + +Native database modes explicitly connect to PostgreSQL; file commands remain +offline. There are no telemetry, update checks or provider calls. Credentials +come only from the dedicated environment variable or an owned mode-0600 regular +file, never literal URL arguments. Inherited `PG*`, passfiles and service files +do not configure native connections; raw driver errors are not printed. + +Remote TLS requires `verify-full`. Only numeric loopback and Unix sockets may +use plaintext; client-certificate and service-based setups keep the manual +`psql` path. Exports use one bounded read-only, repeatable-read snapshot and new +private directories without overwrites or symlink components. It +builds files in a mode-0700 sibling `.fleetdiff-stage-...` directory, +then atomically publishes the complete directory without replacement. The final +destination stays absent until publication; a crash beforehand may leave sibling +staging data, never a partial final directory. The +[security policy](../../SECURITY.md#native-database-export-unreleased-v012-candidate) +and [limits](../NATIVE_EXPORT.md#limits-and-failures) describe these controls. + +## Ordering Change + +The candidate SQL recipe now orders spend rows by `"startTime", request_id +COLLATE "C"` and daily rows by `date COLLATE "C"`. Native and candidate `psql` +exports must match byte for byte for the same snapshot and options. The earlier +recipe did not define spend-row order: export bytes, first-seen aliases and tied +report order may change, while recorded totals and identity hashing do not. +Sorting may affect export time. Compare against the candidate recipe, not a +historical unsorted export, when checking equivalence. + +## Candidate Validation + +The dependency set adds `pgx/v5 v5.11.0`, `pgpassfile v1.0.0` and +`pgservicefile v0.0.0-20240606120523-5a60cdf6a761`, retaining `x/text v0.41.0`. +The [verification checklist](../../examples/litellm-spend/TESTING.md#native-export-candidate-checks) +covers pinned-database byte equivalence, negative/security tests and an outsider +first run. diff --git a/examples/litellm-spend/README.md b/examples/litellm-spend/README.md index 9c6bb0f..b302763 100644 --- a/examples/litellm-spend/README.md +++ b/examples/litellm-spend/README.md @@ -31,10 +31,32 @@ The per-model split is available for model-1; the quality counts point to rows to review in the local LiteLLM logs. Try the same 20 safe, synthetic records as [CSV](synthetic.csv), [JSON array](synthetic.json), or [JSONL](synthetic.jsonl). +## Native Candidate: One Command + +The **unreleased v0.12 candidate** can export and investigate directly. Use the +[development-only build](../../docs/OPERATIONS.md#native-candidate-build-development-only), +not the v0.11.0 installer. After administrator-approved read-only access and +[credential-file setup](../../docs/NATIVE_EXPORT.md#credentials): + +```sh +: "${DATABASE_URL_FILE:?Set the path to your approved database URL file}" +./fleetdiff-install/fleetdiff investigate --litellm-db \ + --database-url-file "$DATABASE_URL_FILE" --group-by key +``` + +It compares the last 14 complete UTC days and cleans up its private temporary +export. Add `--daily`, `--routing-audit`, or `--keep-export NEWDIR` as needed. +For export only, use `export litellm --out NEWDIR`; see the +[native guide](../../docs/NATIVE_EXPORT.md) for formats, bounds, security and +what-if's separate model and period flags. Native exports retain private +identities; reports keep their existing aliasing rules. + ## Export Request-Level Rows -Use [export.sql](export.sql) with `psql` and your existing read-only database -access. Configure a PostgreSQL service named `litellm_readonly` using approved +This manual path remains available with released v0.11.0 and for connection +setups outside the native contract, including client certificates and PostgreSQL +services. Use [export.sql](export.sql) with `psql` and your existing read-only +database access. Configure a PostgreSQL service named `litellm_readonly` using approved connection settings and your normal credential mechanism. From the repository root, choose a new private destination directory outside the checkout: @@ -49,8 +71,11 @@ psql 'service=litellm_readonly' -Xq -v ON_ERROR_STOP=1 \ >"$export_dir/private-psql.out" 2>"$export_dir/private-psql.err" ``` -The recipe writes **CSV only by default**, using one `COPY` pass with no -database sort, in a **REPEATABLE READ, READ ONLY transaction**. The historical +The candidate recipe writes **CSV only by default**, using one `COPY` pass in a +**REPEATABLE READ, READ ONLY transaction**. It orders spend rows by `"startTime", +request_id COLLATE "C"`; optional daily rows use `date COLLATE "C"`. This is an +unreleased ordering change from the v0.11.0 recipe, enabling byte-for-byte parity +with native export for the same snapshot and options. The historical 16-day interval leaves room for the importer's default two seven-day windows inside the file's edge days. Sparse logs may still need explicit periods; the interval alone does not certify coverage. @@ -84,9 +109,11 @@ To check for rows missing relative to LiteLLM's daily counters, use the optional ### Routing Details (Opt In) -The source-checkout SQL recipe accepts `-v routing_details=true` on the same -private-file export command. Omitted or `false` leaves the default columns and -output bytes unchanged. This option is independent of `usage_details` and adds +The SQL recipe accepts `-v routing_details=true` on the same private-file export +command; standalone native export uses `--routing-details`. Convenience database +modes include these details automatically. Within the candidate recipe, omitted +or `false` leaves the default columns and bytes unchanged. This option is +independent of `usage_details` and adds exactly three columns: | Export alias | Pinned path in `metadata` | @@ -133,7 +160,8 @@ For the **v0.11.0** routing audit and route/agent selectors, see the The SQL includes these three projection aliases by default. Add `-v usage_details=false` to the `psql` command to omit all three columns; -`-v usage_details=true` explicitly selects the default behavior. +`-v usage_details=true` explicitly selects the default behavior. Native export +uses `--usage-details=false` or `--usage-details=true` for the same selection. | Export alias | Path below `metadata.additional_usage_values` | | --- | --- | @@ -223,7 +251,7 @@ Released v0.7.0 ignores both added columns; it does not interpret rejection evidence or provide the new session-budget review. See the reproducible [compatibility checks](TESTING.md#released-reader-compatibility). -### Claude Code And Codex Requests +### Native Messages And Responses Requests The importer reads `anthropic_messages` / `aanthropic_messages` and `responses` / `aresponses`, alongside Chat Completions and text completions. The pinned @@ -253,8 +281,9 @@ the call types and asks for a supported request-level export. array, or JSONL. LiteLLM Usage dashboard CSV/JSON exports are daily aggregates: the importer recognizes them and requests row-level data. API page envelopes and session-grouped views are unsupported; no API export recipe is included. -- fleetdiff reads a local file without a collector, database connection, model - key, or provider call. Counts describe supplied logged model requests, not +- File-based fleetdiff commands read locally without a collector, database + connection, model key, or provider call. Only explicit native database modes + connect. Counts describe supplied logged model requests, not proven provider sends. Hidden retries and missing records cannot be recovered. - Supported call types are Chat Completions, text completions, Anthropic Messages and Responses, including their asynchronous names. Other call types remain @@ -289,6 +318,8 @@ the call types and asks for a supported request-level export. session ID alone within the supplied application/cache namespace. In either mode, session IDs may be generated trace IDs, not conversations. - Tied contributors and models follow their first appearance in the file. + The candidate's deterministic SQL ordering can therefore change aliases and + tied report order relative to older unsorted exports, without changing totals. Concentration uses the full tracked candidate set before `--top` truncates display rows. Its denominator is the net recorded increase, so offsets from declining contributors appear as added, offsetting and net token totals in @@ -296,17 +327,23 @@ the call types and asks for a supported request-level export. - The SQL excludes raw error messages, prompts, responses, code, request bodies, IP addresses, API bases, and whole metadata objects. Exports and database diagnostics still contain sensitive information: keep them private. Use only successful exports; - failed SQL can leave partial files. Retry into a new directory because `psql` - can overwrite files. A consistent snapshot does not establish complete logging. + failed manual SQL can leave partial files. Retry into a new directory because + `psql` can overwrite files. The native exporter stages in a private + mode-0700 sibling `.fleetdiff-stage-...` directory and atomically publishes the complete + directory without replacement. The final path stays absent until publication; + a crash beforehand may leave sibling staging data, never a partial final + directory. A consistent snapshot does not + establish complete logging. - Shared input limits: **10,000,000 rows; 16 GiB/file; 64 KiB/row; 8 KiB/field; 64 fields/row; JSON depth 8.** Input limits also apply to ignored content. Two-period comparisons additionally limit context to **128 models and 64 call types**. The single-period session-budget review instead has the [session and identity-membership limits](../../docs/SESSION_BUDGET.md) described in its guide. Select a narrower SQL interval when necessary. - The recipe writes one selected format - without sorting or aggregating rows and has a two-minute per-statement - timeout; review large exports with your database operator. + The recipe writes one selected format, orders rows without JSON-array + aggregation, and has two-minute statement and idle-in-transaction timeouts. + Review large exports with your database operator; the native guide lists + [export failure handling](../../docs/NATIVE_EXPORT.md#limits-and-failures). ## Synthetic Session-Budget Review diff --git a/examples/litellm-spend/TESTING.md b/examples/litellm-spend/TESTING.md index 8781686..fd28c44 100644 --- a/examples/litellm-spend/TESTING.md +++ b/examples/litellm-spend/TESTING.md @@ -3,6 +3,52 @@ # LiteLLM Spend Export Checks +## Documentation Checks + +Run the standard-library documentation checks without building or connecting to +a database: + +```sh +python3 -B -m unittest discover -s examples/litellm-spend -p 'test_docs.py' -v +``` + +These check the README line budget, legacy anchors, linked documents and pinned +usage table. They do not establish native export, TLS or database equivalence. + +## Native Export Candidate Checks + +The [native guide](../../docs/NATIVE_EXPORT.md) describes an **unreleased v0.12 +candidate**. Run the acceptance checks below on the exact candidate binary. + +| Gate | Required evidence | +| --- | --- | +| Native / psql equivalence | Byte-identical spend CSV and JSONL and daily CSV on the same frozen pinned database contents, bounds and options; every usage/routing combination, empty periods and equal-timestamp request-ID ties | +| Ordering | Spend `"startTime", request_id COLLATE "C"`; daily `date COLLATE "C"`, independent of database default collation | +| Snapshot and schema | One read-only repeatable snapshot for spend/daily, SELECT-only role, pinned v1.104.0 column types and request primary key; incompatible schema fails | +| Credentials and transport | Reject literal URL arguments, multiple credential sources, wrong owner/mode, nonregular or symlinked credential paths, remote plaintext, `verify-ca`, invalid certificates/hostnames and unknown URL keys | +| Ambient settings | Hostile `PG*`, passfiles, service files and implicit client certificates cannot change native destinations or authentication | +| Atomic publication | Build mode-0600 files in a private mode-0700 sibling `.fleetdiff-stage-...` directory; final destination absent until one atomic no-replace complete-directory publication; never publish files individually | +| Publication failures and crashes | Reject existing/empty destinations and symlink components; a destination created concurrently is neither replaced nor removed; interrupt before and across publication to check absent-or-complete final paths, with only sibling staging debris before publication | +| Bounds and cleanup | Row/byte/field limits, timeouts and handled cancellation remove staging without publishing a partial final directory; convenience cleanup and retained-export behavior remain correct | +| Privacy and offline behavior | Sentinels absent from reports/errors; no raw driver diagnostics; local-file commands never connect even with database credentials set | +| Convenience modes | Default UTC windows, explicit periods, automatic routing details, cleanup and retained exports; what-if model `--from` / `--to` remain distinct from `--period` | +| Release readiness | Full race/vet checks, vulnerability and license review, dependency inventory, reproducible builds and an outsider following the candidate README from a fresh environment | + +Reuse the cached immutable images below; native candidate validation requires no +new image or paid call. Existing writer/psql tests alone do not prove that the +host binary connected through the native driver. Record the exact binary, +snapshot/fixture, image digests, option matrix and byte comparisons for that run. +Tests requiring a local database connection must explicitly describe that +boundary; the older `--network=none` writer harness description is not a claim +that all native integration tests are network-free. + +The dependency approval covers `pgx/v5 v5.11.0`, `pgpassfile v1.0.0` and +`pgservicefile v0.0.0-20240606120523-5a60cdf6a761`, retaining `x/text v0.41.0`. +It does not substitute for the release checks above. Keep historical verification +results below scoped to their original binaries and harnesses. + +## Existing Fixture Suite + Build the current CLI and run Python 3.10+ standard-library tests from the repository root: @@ -106,7 +152,8 @@ or deployment configuration. and Responses operation names are exercised. Unpinned operations are inventoried separately and make the selected-period comparison incomplete. Expected export membership comes from actual writer timestamps and the - half-open UTC bounds, not a fixed row count or database row order. + half-open UTC bounds, not a fixed row count. The candidate recipe additionally + fixes row order for native/psql byte equivalence. - The real writer produces the usage detail metadata. SQL aliases are compared to each actual payload's pinned metadata paths, including native usage details and absent values. Base token counts must remain unchanged; @@ -131,8 +178,9 @@ or deployment configuration. CSV; setting `jsonl_file` must create only JSONL, with or without `csv_file`. These checks cover both default-detail and opt-out projections. Records match across both SQL formats and the locally assembled JSON array. Empty intervals - also produce valid files. A fixture-only check rejects database sorting, - JSON-array aggregation, and multiple CSV `COPY` passes in the recipe. + also produce valid files. The candidate fixture-only check requires the + deterministic SQL order and rejects JSON-array aggregation and multiple CSV + `COPY` passes in the recipe. - Fixed-secret JSON reports match for all five groups, with and without displayed hashes. Importer totals and individual usage-detail counters are checked against exported row arithmetic without adding subsets to totals. diff --git a/examples/litellm-spend/coverage_runtime_expected.json b/examples/litellm-spend/coverage_runtime_expected.json index 3251375..34109e8 100644 --- a/examples/litellm-spend/coverage_runtime_expected.json +++ b/examples/litellm-spend/coverage_runtime_expected.json @@ -29,14 +29,15 @@ } }, "export_source_sha256": { - "export.sql": "0abae1edc8187812ae579596b5f166e7c236e3c79a95586122910db43c0943bd", - "daily-projection.sql": "f72e96674a760b7ccf2664bc5f11f5be38af935be99a72015f0dd5476f36f9e5", + "export.sql": "0b57656b137d1216f9e657535434800a6ebede545230a4a55a271235ffd1fe2c", + "daily-projection.sql": "efd20f10a06a237782664101e36536df94ee4c80bcb47cf7188a6c77222fe1e5", "export-coverage.sh": "d44304d135a6785358503ed16b04782f8eb31034b6477519cb642e11256b7688" }, "runtime_harness_sha256": { "coverage_runtime_probe.py": "b629b42453080a453e7b47e5878b44ce9ff0b549716a357f6dfdf20bb458f951", - "test_row_coverage_runtime.py": "5dcabcc935403777dd5a669c974889c49ea6d1cdf031ff08c640b16f59b337b5", - "coverage_runtime_pg_service.conf": "1477e5d37ec583fd693d585666af6ba67e754178a834605f6b9a5105558d42bd" + "test_row_coverage_runtime.py": "d3574c9e7ea9fa42ac44050ea7719dcecfcd1c35956e680f0e5bdd25911ffa0a", + "coverage_runtime_pg_service.conf": "1477e5d37ec583fd693d585666af6ba67e754178a834605f6b9a5105558d42bd", + "test_native_export.py": "76fba62fe4c146297fa9b0d6a91dd0c69ff3f5c4dcac5e6348de228cb9479ee7" }, "fixture_generator_sha256": "c85c13d53bdd7a15decf512702758a28f3bee71903eca8ec83657bd37b1423fb", "export_helper": "export-coverage.sh; one psql invocation with inherited distinct file descriptors", @@ -71,7 +72,7 @@ "files": { "spend": { "path": "coverage_runtime_spend.csv", - "sha256": "7ebb6dd44eb1bad22c0d0b5843d19e47652a167d05d51498e7deeb5fbee1fb5a" + "sha256": "d581125757bbc9fc748c4c79e994ff94457ea111f53b1a7e669373bf5fbf37c0" }, "daily": { "path": "coverage_runtime_daily.csv", diff --git a/examples/litellm-spend/coverage_runtime_spend.csv b/examples/litellm-spend/coverage_runtime_spend.csv index 3e408af..d7b2cca 100644 --- a/examples/litellm-spend/coverage_runtime_spend.csv +++ b/examples/litellm-spend/coverage_runtime_spend.csv @@ -1,10 +1,10 @@ request_id,call_type,api_key,user,team_id,organization_id,end_user,model,model_id,model_group,custom_llm_provider,prompt_tokens,completion_tokens,total_tokens,spend,status,session_id,agent_id,rejection_kind,startTime,endTime,cache_read_input_tokens,cache_write_input_tokens,reasoning_output_tokens -3199c0e4-129f-4abe-8b86-125cb553dd9b,acompletion,39dcf56c6a879ccf4f7f341b1f4dac26f25f0a14048c93fb262e451febbb5c26,coverage-user,"","","",openai/gpt-4o-mini,b839acdde25eef275df507f390befc41d80285472acf1cfc8698eb19ab7f0ac7,coverage-model,openai,13,0,13,0,failure,4d01ea73-d124-405b-955f-a8440b3b6a45,,,2026-10-07T12:00:00.000Z,2026-10-07T12:00:01.000Z,,, -chatcmpl-coverage-absent_user,acompletion,7a8dad512531765ebf78ccfd53f662a9fcd19dd2e661a5527a41a0170c474e44,"","","","",openai/gpt-4o-mini,b839acdde25eef275df507f390befc41d80285472acf1cfc8698eb19ab7f0ac7,coverage-model,openai,12,3,15,3.6e-06,success,2e992118-9e78-4bf9-bb02-09f36843bde3,,,2026-10-07T12:00:00.000Z,2026-10-07T12:00:01.000Z,,, -chatcmpl-coverage-cache_fill,acompletion,39dcf56c6a879ccf4f7f341b1f4dac26f25f0a14048c93fb262e451febbb5c26,coverage-user,"","","",openai/gpt-4o-mini,b839acdde25eef275df507f390befc41d80285472acf1cfc8698eb19ab7f0ac7,coverage-model,openai,12,3,15,3.6e-06,success,9089078c-500e-4f7b-8aa1-a5bc539e1ce0,,,2026-10-07T12:00:00.000Z,2026-10-07T12:00:01.000Z,,, -chatcmpl-coverage-cache_fill_cache_hit1791997595.4835541,acompletion,39dcf56c6a879ccf4f7f341b1f4dac26f25f0a14048c93fb262e451febbb5c26,coverage-user,"","","",openai/gpt-4o-mini,b839acdde25eef275df507f390befc41d80285472acf1cfc8698eb19ab7f0ac7,coverage-model,openai,12,3,15,0,success,cd6ce630-6658-4854-a6a4-832c8e50091f,,,2026-10-07T12:00:00.000Z,2026-10-07T12:00:00.000Z,,, -chatcmpl-coverage-collision,acompletion,39dcf56c6a879ccf4f7f341b1f4dac26f25f0a14048c93fb262e451febbb5c26,coverage-user,"","","",openai/gpt-4o-mini,b839acdde25eef275df507f390befc41d80285472acf1cfc8698eb19ab7f0ac7,coverage-model,openai,12,3,15,3.6e-06,success,cc9445b3-47ca-420d-9781-ebe689d687de,,,2026-10-07T12:00:00.000Z,2026-10-07T12:00:01.000Z,,, -chatcmpl-coverage-stream,acompletion,39dcf56c6a879ccf4f7f341b1f4dac26f25f0a14048c93fb262e451febbb5c26,coverage-user,"","","",openai/gpt-4o-mini,b839acdde25eef275df507f390befc41d80285472acf1cfc8698eb19ab7f0ac7,coverage-model,openai,12,3,15,3.6e-06,success,c6343ad6-2196-4158-b4fb-f63f97e693c3,,,2026-10-07T12:00:00.000Z,2026-10-07T12:00:01.000Z,,,0 -chatcmpl-coverage-success,acompletion,39dcf56c6a879ccf4f7f341b1f4dac26f25f0a14048c93fb262e451febbb5c26,coverage-user,"","","",openai/gpt-4o-mini,b839acdde25eef275df507f390befc41d80285472acf1cfc8698eb19ab7f0ac7,coverage-model,openai,12,3,15,3.6e-06,success,752e686b-4ac3-4148-8da6-ffc71a72c20c,,,2026-10-07T12:00:00.000Z,2026-10-07T12:00:01.000Z,,, -chatcmpl-coverage-cross_midnight,acompletion,39dcf56c6a879ccf4f7f341b1f4dac26f25f0a14048c93fb262e451febbb5c26,coverage-user,"","","",openai/gpt-4o-mini,b839acdde25eef275df507f390befc41d80285472acf1cfc8698eb19ab7f0ac7,coverage-model,openai,12,3,15,3.6e-06,success,5c34cdea-6b13-4ef7-bd5a-a8c49f698ab1,,,2026-10-07T23:59:59.000Z,2026-10-08T00:00:01.000Z,,, -chatcmpl-coverage-after_midnight,acompletion,39dcf56c6a879ccf4f7f341b1f4dac26f25f0a14048c93fb262e451febbb5c26,coverage-user,"","","",openai/gpt-4o-mini,b839acdde25eef275df507f390befc41d80285472acf1cfc8698eb19ab7f0ac7,coverage-model,openai,12,3,15,3.6e-06,success,38059217-ff89-4f42-865c-430ed089c042,,,2026-10-08T00:00:01.000Z,2026-10-08T00:00:02.000Z,,, +6095f301-80c2-45d5-bb78-d23874efd92b,acompletion,0ebdfbf01eef0436b211d81404b383a0a3f30c3696ee39f2f8b4f63e7e6a12f6,coverage-user,"","","",openai/gpt-4o-mini,b839acdde25eef275df507f390befc41d80285472acf1cfc8698eb19ab7f0ac7,coverage-model,openai,13,0,13,0,failure,1b523b23-9b0c-4d9a-b4bc-2a022099306f,,,2026-10-07T12:00:00.000Z,2026-10-07T12:00:01.000Z,,, +chatcmpl-coverage-absent_user,acompletion,327c97d62a52e148eb6fe663feabe20fee731c57350656948ff6012043d454b1,"","","","",openai/gpt-4o-mini,b839acdde25eef275df507f390befc41d80285472acf1cfc8698eb19ab7f0ac7,coverage-model,openai,12,3,15,3.6e-06,success,d67e66cd-8d7b-4457-9cb4-5f13a59008c0,,,2026-10-07T12:00:00.000Z,2026-10-07T12:00:01.000Z,,, +chatcmpl-coverage-cache_fill,acompletion,0ebdfbf01eef0436b211d81404b383a0a3f30c3696ee39f2f8b4f63e7e6a12f6,coverage-user,"","","",openai/gpt-4o-mini,b839acdde25eef275df507f390befc41d80285472acf1cfc8698eb19ab7f0ac7,coverage-model,openai,12,3,15,3.6e-06,success,2e53ef92-9478-4258-90a3-7dff0897e767,,,2026-10-07T12:00:00.000Z,2026-10-07T12:00:01.000Z,,, +chatcmpl-coverage-cache_fill_cache_hit1792344351.4393437,acompletion,0ebdfbf01eef0436b211d81404b383a0a3f30c3696ee39f2f8b4f63e7e6a12f6,coverage-user,"","","",openai/gpt-4o-mini,b839acdde25eef275df507f390befc41d80285472acf1cfc8698eb19ab7f0ac7,coverage-model,openai,12,3,15,0,success,58034215-a1f6-44f8-9e8f-c1603197ec4a,,,2026-10-07T12:00:00.000Z,2026-10-07T12:00:00.000Z,,, +chatcmpl-coverage-collision,acompletion,0ebdfbf01eef0436b211d81404b383a0a3f30c3696ee39f2f8b4f63e7e6a12f6,coverage-user,"","","",openai/gpt-4o-mini,b839acdde25eef275df507f390befc41d80285472acf1cfc8698eb19ab7f0ac7,coverage-model,openai,12,3,15,3.6e-06,success,1794a576-1b9d-457d-a3a8-3a622e9ba5c8,,,2026-10-07T12:00:00.000Z,2026-10-07T12:00:01.000Z,,, +chatcmpl-coverage-stream,acompletion,0ebdfbf01eef0436b211d81404b383a0a3f30c3696ee39f2f8b4f63e7e6a12f6,coverage-user,"","","",openai/gpt-4o-mini,b839acdde25eef275df507f390befc41d80285472acf1cfc8698eb19ab7f0ac7,coverage-model,openai,12,3,15,3.6e-06,success,af4e985e-48aa-43d8-9435-6b587f4d8dad,,,2026-10-07T12:00:00.000Z,2026-10-07T12:00:01.000Z,,,0 +chatcmpl-coverage-success,acompletion,0ebdfbf01eef0436b211d81404b383a0a3f30c3696ee39f2f8b4f63e7e6a12f6,coverage-user,"","","",openai/gpt-4o-mini,b839acdde25eef275df507f390befc41d80285472acf1cfc8698eb19ab7f0ac7,coverage-model,openai,12,3,15,3.6e-06,success,fc0e7767-7e7f-4d32-ae8e-2c5a9e616e46,,,2026-10-07T12:00:00.000Z,2026-10-07T12:00:01.000Z,,, +chatcmpl-coverage-cross_midnight,acompletion,0ebdfbf01eef0436b211d81404b383a0a3f30c3696ee39f2f8b4f63e7e6a12f6,coverage-user,"","","",openai/gpt-4o-mini,b839acdde25eef275df507f390befc41d80285472acf1cfc8698eb19ab7f0ac7,coverage-model,openai,12,3,15,3.6e-06,success,6d8102c7-9467-4953-bbc0-6c0585e00fb4,,,2026-10-07T23:59:59.000Z,2026-10-08T00:00:01.000Z,,, +chatcmpl-coverage-after_midnight,acompletion,0ebdfbf01eef0436b211d81404b383a0a3f30c3696ee39f2f8b4f63e7e6a12f6,coverage-user,"","","",openai/gpt-4o-mini,b839acdde25eef275df507f390befc41d80285472acf1cfc8698eb19ab7f0ac7,coverage-model,openai,12,3,15,3.6e-06,success,ab621925-f1e1-478a-9dbe-144cc0536b26,,,2026-10-08T00:00:01.000Z,2026-10-08T00:00:02.000Z,,, diff --git a/examples/litellm-spend/daily-projection.sql b/examples/litellm-spend/daily-projection.sql index 4c42a8f..65ecd88 100644 --- a/examples/litellm-spend/daily-projection.sql +++ b/examples/litellm-spend/daily-projection.sql @@ -11,5 +11,6 @@ COPY ( WHERE date >= to_char(:'start_utc'::timestamptz, 'YYYY-MM-DD') AND date < to_char(:'end_utc'::timestamptz, 'YYYY-MM-DD') GROUP BY date + ORDER BY date COLLATE "C" ) TO STDOUT WITH (FORMAT CSV, HEADER true) \g :daily_csv_file diff --git a/examples/litellm-spend/export.sql b/examples/litellm-spend/export.sql index e73b682..07342c0 100644 --- a/examples/litellm-spend/export.sql +++ b/examples/litellm-spend/export.sql @@ -18,6 +18,8 @@ SET LOCAL TIME ZONE 'UTC'; SET LOCAL search_path = pg_catalog; SET LOCAL statement_timeout = '120s'; SET LOCAL lock_timeout = '5s'; +SET LOCAL idle_in_transaction_session_timeout = '120s'; +SET LOCAL client_encoding = 'UTF8'; -- Require explicit UTC RFC3339 bounds, ascending and half-open. SELECT 1 / CASE WHEN @@ -115,6 +117,7 @@ SELECT format($projection$ FROM public."LiteLLM_SpendLogs" WHERE "startTime" >= (%L::timestamptz AT TIME ZONE 'UTC') AND "startTime" < (%L::timestamptz AT TIME ZONE 'UTC') + ORDER BY "startTime", request_id COLLATE "C" $projection$, :'detail_projection' || :'routing_projection', :'start_utc', :'end_utc') AS export_query \gset diff --git a/examples/litellm-spend/projection.go b/examples/litellm-spend/projection.go new file mode 100644 index 0000000..24fc80d --- /dev/null +++ b/examples/litellm-spend/projection.go @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +// Package litellmsql embeds the public psql recipe, the sole projection source. +package litellmsql + +import _ "embed" + +//go:embed export.sql +var Recipe string + +//go:embed daily-projection.sql +var DailyRecipe string diff --git a/examples/litellm-spend/test_native_export.py b/examples/litellm-spend/test_native_export.py new file mode 100644 index 0000000..d18993e --- /dev/null +++ b/examples/litellm-spend/test_native_export.py @@ -0,0 +1,738 @@ +# SPDX-License-Identifier: Apache-2.0 +# Code authors: Vijay and Codex +"""Offline pinned writer/PostgreSQL byte oracle for the native exporter. + +FLEETDIFF_SPEND_DOCKER=1 FLEETDIFF_LINUX_BIN=/absolute/linux/fleetdiff \ + python3 -B -m unittest discover -s examples/litellm-spend \ + -p test_native_export.py -v + +Uses existing local image pins only. The binary is mounted read-only at +/native/fleetdiff, not executed from the database container's noexec /tmp. +Spend usage comes from the actual pinned writer. Daily counters replay the +existing recorded runtime fixture; this is export parity, not a new proxy run. +Failures never include subprocess output, credentials, rows, or byte diffs. +FLEETDIFF_NATIVE_EXPORT_LARGE=1 additionally enables a 10,000,001-row test +requiring multi-GB Docker volume space; it does not fit the fixture's /tmp. +""" +from contextlib import contextmanager, ExitStack +import csv +import io +import json +import os +from pathlib import Path +import re +import subprocess +import time +import unittest +from unittest import mock +from urllib.parse import urlencode +import uuid + +import test_spend as spend + + +LINUX_BIN = os.environ.get("FLEETDIFF_LINUX_BIN") +URL = "postgres://fixture_reader@/fixture?host=/var/run/postgresql&sslmode=disable" +START, END = "2026-10-07T00:00:00Z", "2026-10-09T00:00:00Z" +PASSWORD = "SENTINEL_NATIVE_PASSWORD_20261018" +FIELD_BYTES = 8 << 10 # internal/spend.MaxFieldBytes, measured in UTF-8 bytes. +MAX_ROWS = 10_000_000 +FINAL_FILES = {"spend.csv", "spend.jsonl", "daily.csv"} +require = spend.require +docker, psql, run = spend.docker, spend.psql, spend.run + + +@contextmanager +def native_database(binary): + """Compose the existing pinned fixture with one exec-capable binary mount.""" + def mounted(*args, **kwargs): + if args and args[0] == "run": + index = args.index(spend.PINS["postgres"]) + args = (*args[:index], "--mount", + f"type=bind,src={binary},dst=/native/fleetdiff,readonly", *args[index:]) + return docker(*args, **kwargs) + + with ExitStack() as cleanup: + # Patch only fixture startup; setup/cleanup remains in the existing helper. + with mock.patch.object(spend, "docker", side_effect=mounted): + container = cleanup.enter_context(spend.database()) + yield container + + +def private_file(container, path, data, mode="600"): + docker("exec", "-i", container, "sh", "-c", + 'umask 077; cat > "$1" && chmod "$2" "$1"', "fixture", path, mode, + data=data) + + +def scalar(container, sql): + return psql(container, "-At", "-c", sql).strip() + + +def wait_for(container, sql, message, process=None): + deadline = time.monotonic() + 15 + while scalar(container, sql) != b"t": + require(process is None or process.poll() is None, + "test process exited before database synchronization") + require(time.monotonic() < deadline, message) + time.sleep(0.05) # Poll explicit database state; never infer readiness. + + +def finish(process, timeout=20): + try: + stdout, stderr = process.communicate(timeout=timeout) + except (OSError, subprocess.SubprocessError): + raise AssertionError("test process did not finish; details suppressed") from None + return subprocess.CompletedProcess([], process.returncode, stdout, stderr) + + +@contextmanager +def background(command): + try: + process = subprocess.Popen(command, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except (OSError, subprocess.SubprocessError): + raise AssertionError("test process could not start; details suppressed") from None + try: + yield process + finally: + if process.poll() is None: + process.kill() + finish(process) + + +def assert_bytes(actual, expected, label): + require(actual == expected, label + " bytes differ from the canonical psql export") + + +def apply_documented_grants(container): + document = (spend.ROOT.parents[1] / "docs" / "NATIVE_EXPORT.md").read_text() + fence = chr(96) * 3 + blocks = re.findall(fence + r"sql\n(.*?)" + fence, document, re.DOTALL) + grants = re.findall( + r'GRANT SELECT\s*\([^;]+\)\s+ON TABLE public\."LiteLLM_(?:SpendLogs|DailyUserSpend)"' + r"\s+TO fleetdiff_reader;", "\n".join(blocks)) + require(len(grants) == 2 and + sum('public."LiteLLM_SpendLogs"' in grant for grant in grants) == 1, + "documented export column grants are missing or ambiguous") + psql(container, "-c", '''REVOKE SELECT ON TABLE public."LiteLLM_SpendLogs", + public."LiteLLM_DailyUserSpend" FROM fixture_reader;''' + + "\n".join(grant.replace("TO fleetdiff_reader;", "TO fixture_reader;") for grant in grants)) + require(scalar(container, '''SELECT + NOT has_table_privilege('fixture_reader', 'public."LiteLLM_SpendLogs"', 'SELECT') + AND NOT has_table_privilege('fixture_reader', 'public."LiteLLM_DailyUserSpend"', 'SELECT') + AND NOT has_column_privilege('fixture_reader', 'public."LiteLLM_SpendLogs"', 'messages', 'SELECT') + AND NOT has_column_privilege('fixture_reader', 'public."LiteLLM_SpendLogs"', 'response', 'SELECT') + AND NOT has_column_privilege('fixture_reader', 'public."LiteLLM_SpendLogs"', 'proxy_server_request', 'SELECT') + ''') == b"t", "documented grants retain broad or content-column access") + + +@unittest.skipUnless(spend.DOCKER and LINUX_BIN, + "set FLEETDIFF_SPEND_DOCKER=1 and FLEETDIFF_LINUX_BIN") +class NativeExportTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + try: + binary = Path(LINUX_BIN).resolve(strict=True) + with binary.open("rb") as source: + magic = source.read(4) + require(binary.is_file() and os.access(binary, os.X_OK) and magic == b"\x7fELF", + "FLEETDIFF_LINUX_BIN must be an executable Linux ELF file") + except OSError: + raise AssertionError("cannot read the Linux test binary; details suppressed") from None + for key in ("litellm", "postgres"): + inspected = json.loads(docker("image", "inspect", spend.PINS[key])) + require(any(ref.endswith(spend.PINS[key].split("@", 1)[1]) + for ref in inspected[0].get("RepoDigests", [])), + "local test image digest mismatch") + cls.records = (spend.writer_payloads() + spend.writer_payloads("--routing") + + spend.writer_payloads("--rejections")) + context = native_database(binary) + cls.db = context.__enter__() + cls.addClassCleanup(context.__exit__, None, None, None) + state = json.loads(docker("inspect", cls.db))[0] + require(state["HostConfig"]["NetworkMode"] == "none", + "native fixture must have no external network") + require(any(m["Destination"] == "/native/fleetdiff" and not m["RW"] + for m in state["Mounts"]), "native binary is not mounted read-only") + + def setUp(self): + # Malformed projection controls are isolated by resetting the schema. + psql(self.db, data=b'''DROP SCHEMA public CASCADE; CREATE SCHEMA public; + DO $$ BEGIN + IF EXISTS (SELECT FROM pg_roles WHERE rolname='fixture_reader') THEN + DROP OWNED BY fixture_reader; DROP ROLE fixture_reader; + END IF; + END $$;''') + spend.seed_database(self.db, list(reversed(self.records))) + # Pinned Prisma: date String; all three counters BigInt, not Int. + psql(self.db, data=b'''CREATE TABLE public."LiteLLM_DailyUserSpend" ( + date text NOT NULL, api_requests bigint NOT NULL DEFAULT 0, + successful_requests bigint NOT NULL DEFAULT 0, + failed_requests bigint NOT NULL DEFAULT 0); + GRANT SELECT ON public."LiteLLM_DailyUserSpend" TO fixture_reader; + ALTER ROLE fixture_reader SET default_transaction_read_only=off; + ALTER ROLE fixture_reader SET default_transaction_isolation='read committed'; + ALTER ROLE fixture_reader SET timezone='Pacific/Honolulu';''') + daily = list(csv.reader(io.StringIO( + (spend.ROOT / "coverage_runtime_daily.csv").read_text())))[1:] + data = io.StringIO() + data.write('COPY public."LiteLLM_DailyUserSpend" FROM STDIN WITH (FORMAT CSV);\n') + csv.writer(data, lineterminator="\n").writerows(reversed(daily)) + data.write("\\.\n") + psql(self.db, data=data.getvalue().encode()) + self.parent = "/tmp/native-" + uuid.uuid4().hex + docker("exec", self.db, "mkdir", "-m", "700", self.parent) + self.addCleanup(docker, "exec", self.db, "rm", "-rf", self.parent) + + def destination(self): + return self.parent + "/out-" + uuid.uuid4().hex + + def command(self, directory, *, fmt=None, daily=False, routing=False, usage=True, + credential_file=None, url=URL, environment=None, start=START, end=END, + extra=(), pid_file=None, user=None): + env = {"PATH": "/usr/local/bin:/usr/bin:/bin", "HOME": self.parent, + "LC_ALL": "C", "TZ": "Pacific/Honolulu"} + if url is not None: + env["FLEETDIFF_LITELLM_DATABASE_URL"] = url + env.update(environment or {}) + args = ["/native/fleetdiff", "export", "litellm", "--out", directory, + "--from", start, "--to", end] + if fmt is not None: + args.extend(("--format", fmt)) + if daily: + args.append("--daily") + if routing: + args.append("--routing-details") + if usage is not None: + args.append("--usage-details=" + str(usage).lower()) + if credential_file is not None: + args.extend(("--database-url-file", credential_file)) + args.extend(extra) + # A permissive umask exercises the exporter's own permission policy. + script = 'umask 000; exec "$@"' + shell_args = [] + if pid_file: + script = 'umask 000; printf "%s\\n" "$$" > "$1"; shift; exec "$@"' + shell_args.append(pid_file) + return ["docker", "exec", *(["--user", user] if user else []), self.db, "env", "-i", + *(key + "=" + value for key, value in env.items()), + "sh", "-c", script, "native-export", *shell_args, *args] + + def check_result(self, result, success): + spend.assert_private(result, records=self.records) + require(PASSWORD.encode() not in result.stdout + result.stderr, + "credential appeared in native export diagnostics") + require((result.returncode == 0) == success, "unexpected native export exit status") + if not success: + require(not result.stdout, "failed native export emitted success output") + + def assert_no_complete_outputs(self, directory): + result = run(["docker", "exec", self.db, "sh", "-c", + 'if [ -d "$1" ]; then ls -A1 -- "$1"; fi', "fixture", directory]) + require(result.returncode == 0, "cannot inspect failed export output") + require(not FINAL_FILES.intersection(result.stdout.decode().splitlines()), + "failed or interrupted export published a complete output") + + def assert_no_output_directory(self, directory): + result = run(["docker", "exec", self.db, "sh", "-c", + '[ ! -e "$1" ] && [ ! -L "$1" ]', "fixture", directory]) + require(result.returncode == 0, "failed or canceled export left its output directory") + + def read_export(self, directory, fmt="csv", daily=False): + expected = {"spend." + fmt} | ({"daily.csv"} if daily else set()) + names = set(docker("exec", self.db, "ls", "-A1", directory).decode().splitlines()) + require(names == expected, "native export created missing or unexpected files") + require(docker("exec", self.db, "stat", "-c", "%a:%u", directory).strip() == b"700:0", + "native output directory is not owned mode 0700") + files, inodes = {}, set() + for name in sorted(expected): + path = directory + "/" + name + require(docker("exec", self.db, "stat", "-c", "%a:%u", path).strip() == b"600:0", + "native output file is not owned mode 0600") + inodes.add(docker("exec", self.db, "stat", "-c", "%d:%i", path).strip()) + files[name] = docker("exec", self.db, "cat", path) + for sentinel in (b"SENTINEL_PROMPT_", b"SENTINEL_RESPONSE_", b"SENTINEL_CODE_", + b"SENTINEL_PATH_", PASSWORD.encode()): + require(sentinel not in files[name], "unprojected private content escaped into export") + require(len(inodes) == len(expected), "native output files alias each other") + return files + + def export(self, *, success=True, directory=None, timeout=40, **options): + directory = directory or self.destination() + result = run(self.command(directory, **options), timeout=timeout) + self.check_result(result, success) + if not success: + self.assert_no_complete_outputs(directory) + return None + return self.read_export(directory, options.get("fmt") or "csv", options.get("daily", False)) + + def daily_oracle(self, start=START, end=END): + directory = self.destination() + result = run(["docker", "exec", "-e", + "PGSERVICEFILE=/example/coverage_runtime_pg_service.conf", self.db, + "sh", "/example/export-coverage.sh", directory, start, end, "coverage_fixture"]) + spend.assert_private(result, records=self.records) + require(result.returncode == 0, "guarded canonical daily export failed") + for name in ("private-psql.out", "private-psql.err"): + require(not docker("exec", self.db, "cat", directory + "/" + name), + "guarded canonical export produced private diagnostics") + return {name: docker("exec", self.db, "cat", directory + "/" + name) + for name in ("spend.csv", "daily.csv")} + + def test_all_format_routing_usage_combinations_exact_bytes(self): + daily = self.daily_oracle()["daily.csv"] + for routing in (False, True): + for usage in (False, True): + oracle = spend.export_database(self.db, details=usage, routing=routing) + for fmt in ("csv", "jsonl"): + for include_daily in (False, True): + with self.subTest(format=fmt, routing=routing, usage=usage, daily=include_daily): + actual = self.export(fmt=fmt, daily=include_daily, routing=routing, usage=usage) + assert_bytes(actual["spend." + fmt], oracle[fmt], "spend") + if include_daily: + assert_bytes(actual["daily.csv"], daily, "daily") + default = self.export(usage=None) + assert_bytes(default["spend.csv"], spend.export_database(self.db)["csv"], "default spend") + + def test_exact_documented_column_grants_preserve_export_bytes(self): + expected = spend.export_database(self.db, details=True, routing=True) + daily = self.daily_oracle()["daily.csv"] + apply_documented_grants(self.db) + for fmt in ("csv", "jsonl"): + actual = self.export(fmt=fmt, daily=True, routing=True) + assert_bytes(actual["spend." + fmt], expected[fmt], "column-granted spend") + assert_bytes(actual["daily.csv"], daily, "column-granted daily") + + def test_half_open_bounds_empty_windows_and_repeatability(self): + for start, end in ((START, "2026-10-08T00:00:00Z"), + ("2026-10-08T00:00:00Z", END), + ("2026-10-06T23:59:59.999Z", START), + ("2026-10-10T00:00:00Z", "2026-10-11T00:00:00Z")): + oracle = spend.export_database(self.db, start, end) + for fmt in ("csv", "jsonl"): + with self.subTest(format=fmt, start=start): + first = self.export(fmt=fmt, start=start, end=end) + second = self.export(fmt=fmt, start=start, end=end) + assert_bytes(first["spend." + fmt], oracle[fmt], "bounded spend") + require(first == second, "repeated native exports changed bytes") + + def test_c_collation_ties_and_csv_json_escaping(self): + # Projection-only controls change identifiers, not writer-generated usage. + psql(self.db, data=b'''UPDATE public."LiteLLM_SpendLogs" + SET request_id=CASE WHEN request_id='synthetic-000' THEN 'Z-tie' + WHEN request_id='synthetic-001' THEN 'a-tie' + ELSE request_id END, + "startTime"='2026-10-07 12:34:56.789', + "endTime"='2026-10-07 12:34:57.789', + model=E'SENTINEL_ESCAPED,"quoted"\\nline\\r\\n\\t\\\\tail', + team_id=NULL, organization_id='', "user"=U&'SENTINEL_\\00e9\\96ea';''') + oracle = spend.export_database(self.db, details=True, routing=True) + rows = list(csv.DictReader(io.StringIO(oracle["csv"].decode()))) + ids = [row["request_id"] for row in rows] + require(ids == sorted(ids, key=lambda value: value.encode()), + "canonical spend ordering is not request_id COLLATE C for timestamp ties") + for fmt in ("csv", "jsonl"): + actual = self.export(fmt=fmt, routing=True) + assert_bytes(actual["spend." + fmt], oracle[fmt], "escaped spend") + + def test_owned_0600_credential_file_and_ambiguous_sources(self): + path = self.parent + "/SENTINEL_DATABASE_URL" + private_file(self.db, path, (URL + "\r\n").encode()) + expected = spend.export_database(self.db)["csv"] + actual = self.export(url=None, credential_file=path) + assert_bytes(actual["spend.csv"], expected, "credential-file spend") + self.export(success=False, credential_file=path) + self.export(success=False, url=None) + self.export(success=False, url="") + self.export(success=False, url=None, environment={"DATABASE_URL": URL}) + for mode in ("644", "660", "400", "700"): + with self.subTest(mode=mode): + docker("exec", self.db, "chmod", mode, path) + self.export(success=False, url=None, credential_file=path) + docker("exec", self.db, "chmod", "600", path) + docker("exec", self.db, "chown", "65534:65534", path) + self.export(success=False, url=None, credential_file=path) + + def test_credential_symlink_directory_fifo_and_oversize_rejected(self): + path = self.parent + "/SENTINEL_CREDENTIAL" + private_file(self.db, path, URL.encode()) + link = path + "-link" + docker("exec", self.db, "ln", "-s", path, link) + fifo = path + "-fifo" + docker("exec", self.db, "mkfifo", "-m", "600", fifo) + for candidate in (link, fifo, self.parent, path + "-missing"): + self.export(success=False, url=None, credential_file=candidate) + private_file(self.db, path, b"SENTINEL_OVERSIZE" * 5000) + self.export(success=False, url=None, credential_file=path) + + def test_inherited_pg_environment_and_service_files_are_ignored(self): + service = self.parent + "/.pg_service.conf" + private_file(self.db, service, b'''[SENTINEL_SERVICE] +host=/SENTINEL_SOCKET +dbname=SENTINEL_DATABASE +user=SENTINEL_USER +password=SENTINEL_PASSWORD +options=-c search_path=SENTINEL_SCHEMA +''') + passfile = self.parent + "/.pgpass" + private_file(self.db, passfile, b"*:*:*:*:SENTINEL_PASSWORD\n") + poison = {"PGHOST": "/SENTINEL_SOCKET", "PGHOSTADDR": "192.0.2.1", + "PGPORT": "1", "PGDATABASE": "SENTINEL_DATABASE", "PGUSER": "SENTINEL_USER", + "PGPASSWORD": PASSWORD, "PGSERVICE": "SENTINEL_SERVICE", + "PGSERVICEFILE": service, "PGSYSCONFDIR": self.parent, "PGPASSFILE": passfile, + "PGSSLMODE": "SENTINEL_INVALID_MODE", "PGSSLROOTCERT": "/SENTINEL_CA", + "PGOPTIONS": "-c search_path=SENTINEL_SCHEMA -c default_transaction_read_only=off", + "PGCLIENTENCODING": "LATIN1", "PGAPPNAME": "SENTINEL_APPLICATION", + "PGCONNECT_TIMEOUT": "1", "DATABASE_URL": "SENTINEL_OTHER_DATABASE"} + oracle = self.daily_oracle() + actual = self.export(daily=True, environment=poison) + require(actual == oracle, "inherited environment or home files altered native export bytes") + self.export(success=False, url=None, environment=poison) + + def test_pgpass_and_password_environment_cannot_supply_missing_url_password(self): + # Unlike trust authentication, SCRAM makes implicit-password use observable. + expected = spend.export_database(self.db)["csv"] + psql(self.db, "-c", "ALTER ROLE fixture_reader PASSWORD '" + PASSWORD + "'") + hba = scalar(self.db, "SHOW hba_file").decode() + require(hba.startswith("/var/lib/postgresql/data/"), "unexpected fixture auth file location") + original = docker("exec", self.db, "cat", hba) + + def reload_auth(data): + before = scalar(self.db, "SELECT extract(epoch FROM pg_conf_load_time())").decode() + require(before.replace(".", "").isdigit(), "invalid config reload observation") + private_file(self.db, hba, data) + require(scalar(self.db, "SELECT pg_reload_conf()") == b"t", "test auth reload failed") + wait_for(self.db, "SELECT extract(epoch FROM pg_conf_load_time()) > " + before, + "test auth reload was not observed") + + try: + reload_auth(b"local fixture fixture_reader scram-sha-256\n" + original) + explicit = URL.replace("fixture_reader@", "fixture_reader:" + PASSWORD + "@") + # psql's shared reader helper intentionally has no password. The + # positive native control proves authentication is otherwise usable. + actual = self.export(url=explicit) + assert_bytes(actual["spend.csv"], expected, "password-authenticated spend") + self.export(success=False) + passfile = self.parent + "/.pgpass" + private_file(self.db, passfile, ("*:*:fixture:fixture_reader:" + PASSWORD + "\n").encode()) + for environment in ({}, {"PGPASSFILE": passfile}, {"PGPASSWORD": PASSWORD}): + self.export(success=False, environment=environment) + service = self.parent + "/.pg_service.conf" + private_file(self.db, service, ("[SENTINEL_SERVICE]\n" + "host=/var/run/postgresql\ndbname=fixture\nuser=fixture_reader\npassword=" + + PASSWORD + "\nsslmode=disable\n").encode()) + self.export(success=False, environment={"PGSERVICE": "SENTINEL_SERVICE", + "PGSERVICEFILE": service}) + self.export(url=explicit, environment={"PGPASSWORD": "SENTINEL_WRONG_PASSWORD", + "PGPASSFILE": passfile}) + finally: + reload_auth(original) + + def test_invalid_bounds_formats_and_argv_credentials_are_private(self): + cases = ( + {"start": "SENTINEL_INVALID_BOUND"}, {"start": END, "end": START}, + {"start": START, "end": START}, {"fmt": "SENTINEL_INVALID_FORMAT"}, + {"start": "2026-10-07T00:00:00-04:00"}, + {"daily": True, "start": "2026-10-07T00:00:00.001Z"}, + {"daily": True, "start": "2000-01-01T00:00:00Z"}, + {"extra": ("--database-url", URL.replace("fixture_reader@", "fixture_reader:" + PASSWORD + "@"))}, + {"extra": ("--SENTINEL_UNKNOWN_FLAG",)}, + ) + for index, options in enumerate(cases): + with self.subTest(invalid_options=index): + self.export(success=False, **options) + + def test_url_options_and_connection_errors_do_not_leak_sentinels(self): + for key, value in (("service", "SENTINEL_SERVICE"), ("servicefile", "/SENTINEL_SERVICE"), + ("passfile", "/SENTINEL_PASSWORD_FILE"), + ("options", "-c search_path=SENTINEL_SCHEMA"), + ("hostaddr", "192.0.2.1"), ("sslkey", "/SENTINEL_KEY"), + ("application_name", "SENTINEL_APP")): + with self.subTest(option=key): + self.export(success=False, url=URL + "&" + urlencode({key: value})) + for url in (URL + "&sslmode=disable", URL + "#SENTINEL_FRAGMENT", + URL.replace("fixture_reader@", "SENTINEL_MISSING:" + PASSWORD + "@"), + URL.replace("/fixture?", "/SENTINEL_MISSING_DATABASE?"), + "host=/var/run/postgresql user=fixture_reader dbname=fixture password=" + PASSWORD, + "https://SENTINEL_HOST/fixture", URL.replace("fixture_reader@", "")): + self.export(success=False, url=url) + + def test_existing_directory_and_symlink_outputs_are_not_modified(self): + directory = self.destination() + original = self.export(directory=directory, daily=True) + link = self.parent + "/existing-link" + docker("exec", self.db, "ln", "-s", directory, link) + dangling = self.parent + "/dangling-link" + docker("exec", self.db, "ln", "-s", self.parent + "/absent", dangling) + parent_link = self.parent + "/parent-link" + docker("exec", self.db, "ln", "-s", self.parent, parent_link) + for candidate in (directory, link, dangling, parent_link + "/new"): + result = run(self.command(candidate, daily=True)) + self.check_result(result, False) + require(self.read_export(directory, daily=True) == original, + "rejected export changed preexisting output bytes") + require(run(["docker", "exec", self.db, "test", "-e", self.parent + "/absent"]).returncode == 1, + "rejected dangling link created its target") + + def test_schema_type_precision_missing_column_and_primary_key_mismatches(self): + # Identity-only fill permits testing a wrongly required optional column. + psql(self.db, "-c", '''UPDATE public."LiteLLM_SpendLogs" + SET agent_id='' WHERE agent_id IS NULL''') + cases = ( + ('ALTER COLUMN "startTime" TYPE timestamptz(3) USING "startTime" AT TIME ZONE \'UTC\'', + 'ALTER COLUMN "startTime" TYPE timestamp(3) USING "startTime" AT TIME ZONE \'UTC\''), + ('ALTER COLUMN "startTime" TYPE timestamp(6)', 'ALTER COLUMN "startTime" TYPE timestamp(3)'), + ('ALTER COLUMN prompt_tokens TYPE bigint', 'ALTER COLUMN prompt_tokens TYPE integer'), + ('ALTER COLUMN metadata TYPE json', 'ALTER COLUMN metadata TYPE jsonb'), + ('RENAME COLUMN agent_id TO "SENTINEL_MISSING_COLUMN"', + 'RENAME COLUMN "SENTINEL_MISSING_COLUMN" TO agent_id'), + ('DROP CONSTRAINT "LiteLLM_SpendLogs_pkey"', 'ADD PRIMARY KEY (request_id)'), + ('ALTER COLUMN call_type DROP NOT NULL', 'ALTER COLUMN call_type SET NOT NULL'), + ('ALTER COLUMN agent_id SET NOT NULL', 'ALTER COLUMN agent_id DROP NOT NULL'), + ) + for index, (change, restore) in enumerate(cases): + with self.subTest(schema_case=index): + psql(self.db, "-c", 'ALTER TABLE public."LiteLLM_SpendLogs" ' + change) + try: + self.export(success=False, daily=True) + finally: + psql(self.db, "-c", 'ALTER TABLE public."LiteLLM_SpendLogs" ' + restore) + for column in ("date", "api_requests", "successful_requests", "failed_requests"): + with self.subTest(daily_nullable=column): + prefix = 'ALTER TABLE public."LiteLLM_DailyUserSpend" ALTER COLUMN ' + column + psql(self.db, "-c", prefix + " DROP NOT NULL") + try: + self.export(success=False, daily=True) + finally: + psql(self.db, "-c", prefix + " SET NOT NULL") + psql(self.db, "-c", 'ALTER TABLE public."LiteLLM_DailyUserSpend" ALTER COLUMN date TYPE date USING date::date') + self.export(success=False, daily=True) + self.export() # A daily mismatch must not disable spend-only export. + + def test_database_cast_error_does_not_publish_spend_or_daily(self): + psql(self.db, data=b'''UPDATE public."LiteLLM_SpendLogs" SET metadata= + jsonb_set(COALESCE(metadata, '{}'::jsonb), '{additional_usage_values}', + '{"cache_read_input_tokens":"SENTINEL_BAD_USAGE"}'::jsonb) + WHERE request_id='synthetic-000';''') + for fmt in ("csv", "jsonl"): + self.export(success=False, fmt=fmt, daily=True) + oracle = spend.export_database(self.db, details=False) + actual = self.export(fmt=fmt, daily=True, usage=False) + assert_bytes(actual["spend." + fmt], oracle[fmt], "usage-opt-out spend") + + def test_known_field_limits_apply_to_utf8_bytes(self): + for width, expression in ((1, "'x'"), (4, "chr(128578)")): + for delta in (0, 1): + length = FIELD_BYTES // width + delta + psql(self.db, "-c", 'UPDATE public."LiteLLM_SpendLogs" SET model=repeat(' + + expression + f", {length}) WHERE request_id='synthetic-000'") + for fmt in ("csv", "jsonl"): + with self.subTest(bytes_per_character=width, over_limit=bool(delta), format=fmt): + actual = self.export(success=not delta, fmt=fmt, daily=True) + if not delta: + oracle = spend.export_database(self.db) + assert_bytes(actual["spend." + fmt], oracle[fmt], "field-boundary spend") + + @unittest.skipUnless(os.environ.get("FLEETDIFF_NATIVE_EXPORT_LARGE") == "1", + "set FLEETDIFF_NATIVE_EXPORT_LARGE=1 for the multi-GB 10M-row gate") + def test_real_postgresql_row_limit(self): + disk = docker("exec", self.db, "df", "-Pk", "/var/lib/postgresql/data").splitlines()[-1].split() + require(len(disk) >= 6 and disk[3].isdigit(), "cannot measure Docker fixture free space") + if int(disk[3]) * 1024 < 25 * (1 << 30): + self.skipTest("10M-row gate requires at least 25 GiB free in the Docker data volume") + # Clone a real writer row; only request_id changes to preserve its PK. + # First prove this payload works, so malformed-field failures cannot pass. + for fmt in ("csv", "jsonl"): + self.export(fmt=fmt) + fields = (*spend.COLUMNS, "agent_id", "metadata", "messages", "response", + "proxy_server_request") + columns = ",".join('"' + name + '"' for name in fields) + selected = ",".join(("'native-limit-' || lpad(g::text, 8, '0')" + if name == "request_id" else 't."' + name + '"') + for name in fields) + sql = f'''CREATE TEMP TABLE native_limit_source AS + SELECT * FROM public."LiteLLM_SpendLogs" WHERE request_id='synthetic-000'; + TRUNCATE public."LiteLLM_SpendLogs"; + INSERT INTO public."LiteLLM_SpendLogs" ({columns}) + SELECT {selected} FROM generate_series(1, {MAX_ROWS + 1}) g + CROSS JOIN native_limit_source t; + CREATE INDEX native_limit_order ON public."LiteLLM_SpendLogs" + ("startTime", request_id COLLATE "C"); + ANALYZE public."LiteLLM_SpendLogs";''' + # The ordinary psql helper has a 90-second subprocess cap, too short for + # deliberately loading this scale. Keep its exact psql options here. + started = time.monotonic() + docker("exec", "-i", self.db, "psql", "-Xq", "-v", "ON_ERROR_STOP=1", + "-U", "postgres", "-d", "fixture", data=sql.encode(), timeout=900) + seed_seconds = time.monotonic() - started + require(scalar(self.db, 'SELECT count(*) FROM public."LiteLLM_SpendLogs"') == + str(MAX_ROWS + 1).encode(), "large row-limit fixture has the wrong size") + source_bytes = scalar(self.db, """SELECT pg_total_relation_size('public."LiteLLM_SpendLogs"')""") + require(source_bytes.isdigit(), "cannot measure large fixture storage") + print(f"native large fixture: rows={MAX_ROWS + 1} source_bytes={int(source_bytes)} " + f"seed_seconds={seed_seconds:.2f}", flush=True) + parent = "/var/lib/postgresql/data/native-limit-" + uuid.uuid4().hex + docker("exec", "--user", "postgres", self.db, "mkdir", "-m", "700", parent) + self.addCleanup(docker, "exec", self.db, "rm", "-rf", parent) + for fmt in ("csv", "jsonl"): + with self.subTest(format=fmt): + directory = parent + "/" + fmt + started = time.monotonic() + try: + result = run(self.command(directory, fmt=fmt, user="postgres"), timeout=330) + finally: + print(f"native large export: format={fmt} " + f"elapsed_seconds={time.monotonic() - started:.2f}", flush=True) + self.check_result(result, False) + self.assert_no_complete_outputs(directory) + self.assert_no_output_directory(directory) + require(b"request, file, row or field limit" in result.stderr, + "large fixture failed before reaching the export limit") + + def install_gate(self, key, *, action=None): + if action is None: + action = f"PERFORM pg_catalog.pg_advisory_xact_lock_shared({key});" + psql(self.db, "-c", '''CREATE FUNCTION public.native_export_gate() RETURNS boolean + LANGUAGE plpgsql VOLATILE AS $$ BEGIN + IF current_setting('transaction_read_only') <> 'on' + OR current_setting('transaction_isolation') <> 'repeatable read' THEN + RAISE EXCEPTION 'SENTINEL_TRANSACTION_SETTINGS'; + END IF; + ''' + action + ''' RETURN true; END $$; + ALTER TABLE public."LiteLLM_SpendLogs" ENABLE ROW LEVEL SECURITY; + CREATE POLICY native_export_gate ON public."LiteLLM_SpendLogs" + FOR SELECT TO fixture_reader USING (public.native_export_gate());''') + + @contextmanager + def lock_gate(self, key): + application = "native-lock-" + uuid.uuid4().hex + command = ["docker", "exec", "-e", "PGAPPNAME=" + application, self.db, + "psql", "-Xq", "-v", "ON_ERROR_STOP=1", "-U", "postgres", "-d", "fixture", + "-c", f"SELECT pg_advisory_lock({key}); SELECT pg_sleep(60);"] + with background(command) as process: + try: + wait_for(self.db, f'''SELECT EXISTS (SELECT FROM pg_locks l + JOIN pg_stat_activity a USING (pid) WHERE a.application_name='{application}' + AND l.locktype='advisory' AND l.objid={key} AND l.granted)''', + "test advisory lock was not acquired", process) + yield + finally: + scalar(self.db, f"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name='{application}'") + + @contextmanager + def active_export(self, directory, **options): + pid_file = self.parent + "/native.pid" + with background(self.command(directory, pid_file=pid_file, **options)) as process: + try: + yield process, pid_file + finally: + if process.poll() is None: + raw = run(["docker", "exec", self.db, "cat", pid_file]).stdout.strip() + if raw.isdigit(): + run(["docker", "exec", self.db, "sh", "-c", + 'kill -s KILL "$1"', "fixture", raw.decode()]) + scalar(self.db, "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename='fixture_reader'") + + def wait_blocked(self, key, process): + wait_for(self.db, f'''SELECT EXISTS (SELECT FROM pg_locks l + JOIN pg_stat_activity a USING (pid) WHERE a.usename='fixture_reader' + AND l.locktype='advisory' AND l.objid={key} AND NOT l.granted + AND a.backend_xmin IS NOT NULL AND a.xact_start IS NOT NULL)''', + "native export did not reach its snapshot lock", process) + + def test_repeatable_read_read_only_single_snapshot_during_concurrent_mutation(self): + key = 730018 + for fmt in ("csv", "jsonl"): + before_spend = spend.export_database(self.db)[fmt] + before_daily = self.daily_oracle()["daily.csv"] + self.install_gate(key) + directory = self.destination() + try: + with self.lock_gate(key): + with self.active_export(directory, fmt=fmt, daily=True) as (process, _): + self.wait_blocked(key, process) + psql(self.db, data=b'''BEGIN; + UPDATE public."LiteLLM_SpendLogs" SET model=model || '-changed'; + UPDATE public."LiteLLM_DailyUserSpend" + SET api_requests=api_requests+1, successful_requests=successful_requests+1; + COMMIT;''') + # Release only the blocker, leaving the export transaction intact. + scalar(self.db, f'''SELECT pg_terminate_backend(a.pid) FROM pg_stat_activity a + JOIN pg_locks l USING (pid) WHERE l.locktype='advisory' + AND l.objid={key} AND l.granted AND a.usename='postgres' ''') + self.check_result(finish(process), True) + actual = self.read_export(directory, fmt, daily=True) + assert_bytes(actual["spend." + fmt], before_spend, "snapshot spend") + assert_bytes(actual["daily.csv"], before_daily, "snapshot daily") + finally: + psql(self.db, data=b'''DROP POLICY native_export_gate ON public."LiteLLM_SpendLogs"; + ALTER TABLE public."LiteLLM_SpendLogs" DISABLE ROW LEVEL SECURITY; + DROP FUNCTION public.native_export_gate();''') + require(spend.export_database(self.db)[fmt] != before_spend and + self.daily_oracle()["daily.csv"] != before_daily, + "concurrent control did not change both database projections") + + def test_read_only_role_and_server_error_privacy(self): + require(scalar(self.db, '''SELECT has_table_privilege('fixture_reader', + 'public."LiteLLM_SpendLogs"', 'SELECT') AND NOT has_table_privilege('fixture_reader', + 'public."LiteLLM_SpendLogs"', 'INSERT,UPDATE,DELETE')''') == b"t", + "fixture reader is not SELECT-only") + oracle = spend.export_database(self.db) + self.install_gate(730019) + for fmt in ("csv", "jsonl"): + actual = self.export(fmt=fmt) + assert_bytes(actual["spend." + fmt], oracle[fmt], "read-only transaction spend") + psql(self.db, "-c", """CREATE OR REPLACE FUNCTION public.native_export_gate() + RETURNS boolean LANGUAGE plpgsql VOLATILE AS $$ BEGIN + RAISE EXCEPTION 'SENTINEL_SERVER_DIAGNOSTIC'; END $$;""") + for fmt in ("csv", "jsonl"): + self.export(success=False, fmt=fmt) + + def test_interruption_never_publishes_complete_outputs(self): + key = 730020 + self.install_gate(key) + for signal in ("INT", "TERM"): + for fmt in ("csv", "jsonl"): + with self.subTest(signal=signal, format=fmt): + directory = self.destination() + with self.lock_gate(key): + with self.active_export(directory, fmt=fmt, daily=True) as (process, pid_file): + self.wait_blocked(key, process) + self.assert_no_output_directory(directory) + pid = docker("exec", self.db, "cat", pid_file).strip() + require(pid.isdigit(), "native process ID unavailable") + docker("exec", self.db, "sh", "-c", + 'kill -s "$1" "$2"', "fixture", signal, pid.decode()) + self.check_result(finish(process), False) + self.assert_no_complete_outputs(directory) + self.assert_no_output_directory(directory) + + def test_actual_statement_timeout_never_publishes_complete_outputs(self): + # Sleep occurs in the actual SELECT, not in the test runner or mock client. + self.install_gate(730021, action="""IF current_setting('statement_timeout') <> '2min' THEN + RAISE EXCEPTION 'SENTINEL_STATEMENT_TIMEOUT'; END IF; + PERFORM pg_catalog.pg_sleep(180);""") + directory = self.destination() + with self.active_export(directory) as (process, _): + wait_for(self.db, """SELECT EXISTS (SELECT FROM pg_stat_activity + WHERE usename='fixture_reader' AND wait_event='PgSleep')""", + "native export did not reach the statement-timeout control", process) + # The observed PgSleep proves it got past schema/config validation. + self.check_result(finish(process, timeout=150), False) + self.assert_no_complete_outputs(directory) + self.assert_no_output_directory(directory) + + +class HarnessPrivacyTests(unittest.TestCase): + def test_byte_mismatch_does_not_print_either_row(self): + try: + assert_bytes(b"SENTINEL_PRIVATE_ROW_A", b"SENTINEL_PRIVATE_ROW_B", "spend") + except AssertionError as error: + require("SENTINEL" not in str(error), "byte assertion leaked row contents") + else: + self.fail("byte mismatch was accepted") + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/litellm-spend/test_row_coverage_runtime.py b/examples/litellm-spend/test_row_coverage_runtime.py index 77182bd..58332a7 100644 --- a/examples/litellm-spend/test_row_coverage_runtime.py +++ b/examples/litellm-spend/test_row_coverage_runtime.py @@ -17,7 +17,7 @@ import unittest import uuid -from test_spend import DOCKER, PINS, ROOT, docker, psql, require, run +from test_spend import DOCKER, PINS, ROOT, assert_private, docker, psql, require, run CASES = ("success", "failure", "stream", "cache_fill", "cache_hit", "collision_first", "collision_second", "absent_user", "cross_midnight", "after_midnight") @@ -47,6 +47,16 @@ def totals(snapshot): def runtime(output, failure_logging="default"): name = "fleetdiff-coverage-" + uuid.uuid4().hex gateway, volume = name + "-proxy", name + "-data" + native_mount = [] + if os.environ.get("FLEETDIFF_LINUX_BIN"): + try: + binary = Path(os.environ["FLEETDIFF_LINUX_BIN"]).resolve(strict=True) + with binary.open("rb") as source: + require(source.read(4) == b"\x7fELF" and os.access(binary, os.X_OK), + "native runtime parity requires an executable Linux ELF binary") + except OSError: + raise AssertionError("cannot read native runtime binary; details suppressed") from None + native_mount = ["--mount", f"type=bind,src={binary},dst=/native/fleetdiff,readonly"] docker("volume", "create", volume) try: docker("run", "-d", "--name", name, "--pull=never", "--network=none", "--read-only", @@ -56,7 +66,7 @@ def runtime(output, failure_logging="default"): "--mount", f"type=bind,src={ROOT},dst=/example,readonly", "--pids-limit=128", "--memory=512m", "--log-driver=none", "-e", "POSTGRES_DB=fixture", "-e", "POSTGRES_HOST_AUTH_METHOD=trust", - PINS["postgres"], "-c", "listen_addresses=127.0.0.1", + *native_mount, PINS["postgres"], "-c", "listen_addresses=127.0.0.1", "-c", "log_statement=none", "-c", "log_min_error_statement=panic") deadline = time.monotonic() + 60 while True: @@ -104,6 +114,46 @@ def runtime(output, failure_logging="default"): class RuntimeCoverageTests(unittest.TestCase): + def check_native_export(self, database, spend_bytes, daily_bytes, records): + from test_native_export import apply_documented_grants + + apply_documented_grants(database) + directory = "/tmp/coverage-export-private/native" + inspected = json.loads(docker("inspect", database))[0] + require(inspected["HostConfig"]["NetworkMode"] == "none" and + any(m["Destination"] == "/native/fleetdiff" and not m["RW"] + for m in inspected["Mounts"]), + "native runtime export requires network-none and a read-only binary mount") + identity = psql(database, "-At", "-c", '''SELECT current_user, + has_column_privilege(current_user, 'public."LiteLLM_SpendLogs"', 'request_id', 'SELECT'), + has_table_privilege(current_user, 'public."LiteLLM_SpendLogs"', 'INSERT,UPDATE,DELETE'), + has_column_privilege(current_user, 'public."LiteLLM_DailyUserSpend"', 'api_requests', 'SELECT'), + has_table_privilege(current_user, 'public."LiteLLM_DailyUserSpend"', 'INSERT,UPDATE,DELETE');''', + reader=True) + require(identity.strip() == b"fixture_reader|t|f|t|f", "native runtime reader privileges differ") + result = run(["docker", "exec", database, "env", "-i", + "PATH=/usr/local/bin:/usr/bin:/bin", "HOME=/tmp", "LC_ALL=C", + "FLEETDIFF_LITELLM_DATABASE_URL=postgres://fixture_reader@/fixture?host=/var/run/postgresql&sslmode=disable", + "sh", "-c", 'umask 000; exec "$@"', "native-runtime", + "/native/fleetdiff", "export", "litellm", "--out", directory, + "--from", "2026-10-07T00:00:00Z", "--to", "2026-10-09T00:00:00Z", + "--format", "csv", "--daily"]) + assert_private(result, records=records) + require(result.returncode == 0, "native export of the full pinned Prisma database failed") + require(docker("exec", database, "ls", "-A1", directory).splitlines() == + [b"daily.csv", b"spend.csv"], "native runtime export created unexpected files") + require(docker("exec", database, "stat", "-c", "%a", directory).strip() == b"700", + "native runtime output directory is not private") + for name, expected in (("spend.csv", spend_bytes), ("daily.csv", daily_bytes)): + path = directory + "/" + name + require(docker("exec", database, "stat", "-c", "%a", path).strip() == b"600", + "native runtime output file is not private") + require(docker("exec", database, "cat", path) == expected, + "native runtime output bytes differ from the paired psql oracle") + return {"full_pinned_prisma_schema": True, "reader_role": "fixture_reader", + "documented_column_grants_only": True, + "spend_csv_byte_equivalent": True, "daily_csv_byte_equivalent": True} + def export_with_helper(self, database, output): parent = "/tmp/coverage-export-private" directory = parent + "/paired" @@ -299,6 +349,9 @@ def run_cases(self, cases, failure_logging): self.assertEqual(len(exported), len(snapshot["spend"])) spend_bytes, daily_bytes, safety = self.export_with_helper(database, output) evidence["export_safety"] = safety + if os.environ.get("FLEETDIFF_LINUX_BIN"): + evidence["native_export"] = self.check_native_export( + database, spend_bytes, daily_bytes, exported) daily_export = list(csv.DictReader(io.StringIO(daily_bytes.decode()))) exported_again = list(csv.DictReader(io.StringIO(spend_bytes.decode()))) self.assertEqual(sorted(exported, key=lambda r: r["request_id"]), @@ -317,7 +370,7 @@ def run_cases(self, cases, failure_logging): for name in ("export.sql", "daily-projection.sql", "export-coverage.sh")} evidence["harness_sha256"] = {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() for name in ("coverage_runtime_probe.py", "test_row_coverage_runtime.py", - "coverage_runtime_pg_service.conf")} + "coverage_runtime_pg_service.conf", "test_native_export.py")} if output: for fmt, data in exports.items(): (output / ("spend." + fmt)).write_bytes(data) diff --git a/examples/litellm-spend/test_spend.py b/examples/litellm-spend/test_spend.py index 7fa4b3d..4b13405 100644 --- a/examples/litellm-spend/test_spend.py +++ b/examples/litellm-spend/test_spend.py @@ -344,10 +344,11 @@ def export_database(container, start="2026-10-07T00:00:00Z", end="2026-10-09T00: class FixtureTests(unittest.TestCase): - def test_sql_recipe_uses_one_unsorted_export_without_array_aggregation(self): + def test_sql_recipe_uses_one_ordered_export_without_array_aggregation(self): sql = (ROOT / "export.sql").read_text().upper() require(sql.count("COPY (") == 1, "SQL recipe must have one CSV COPY pass") - require("ORDER BY" not in sql, "SQL export must not require a database sort") + require('ORDER BY "STARTTIME", REQUEST_ID COLLATE "C"' in sql, + "SQL export must use the shared deterministic row order") require("JSON_AGG" not in sql and "JSONB_AGG" not in sql, "SQL export must not aggregate a JSON array") require("JSON_FILE" not in sql, "SQL recipe must not export a JSON array") diff --git a/go.mod b/go.mod index c99c7a1..338cf2d 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,15 @@ module github.com/llm-measurement/fleetdiff go 1.26.0 require ( + github.com/jackc/pgx/v5 v5.11.0 github.com/llm-measurement/llm-sketchkit v0.2.4 go.opentelemetry.io/proto/slim/otlp v1.11.1 go.yaml.in/yaml/v3 v3.0.5 google.golang.org/protobuf v1.36.12 ) -require golang.org/x/text v0.41.0 // indirect +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + golang.org/x/text v0.41.0 // indirect +) diff --git a/go.sum b/go.sum index 6110f7e..489a286 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,11 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/llm-measurement/llm-sketchkit v0.2.4 h1:H8VfYhQpo6N0sg4gNj7a1t28V2Uw5xyrhWgZZvgcWlU= github.com/llm-measurement/llm-sketchkit v0.2.4/go.mod h1:FfKeuXT6lD9G4wv8Z5KBcHF+q+3odiJfo7mNDZpX52M= go.opentelemetry.io/proto/slim/otlp v1.11.1 h1:qf2OA3kQcIDXh3q7b6kc0IzRe8BFXlVIlH5uaOACrVQ= diff --git a/internal/cli/database.go b/internal/cli/database.go new file mode 100644 index 0000000..90daf34 --- /dev/null +++ b/internal/cli/database.go @@ -0,0 +1,338 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package cli + +import ( + "context" + "crypto/rand" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/llm-measurement/fleetdiff/internal/exportdata" + "github.com/llm-measurement/fleetdiff/internal/spend" +) + +const exportHelp = `Export a private, read-only LiteLLM database snapshot. + +Usage: + fleetdiff export litellm --out NEW_DIRECTORY [options] + +Options: + --from UTC Inclusive UTC date or RFC3339 timestamp + --to UTC Exclusive UTC date or RFC3339 timestamp + --format csv|jsonl Spend file format (default csv) + --daily Include daily row counters + --routing-details Include original-route and agent details + --usage-details=false Omit default-on cache and usage details + --database-url-file FILE Read credentials from an owner-only regular file + +The default range is the last 14 complete UTC days. Supply both bounds to +override it. Credentials otherwise come from FLEETDIFF_LITELLM_DATABASE_URL, +never command-line URLs. Remote databases require TLS with verify-full. +The output directory must be new; directory mode is 0700 and files are 0600. +Exports contain private identities. Keep them private. +Handled export cancellation cleans temporary data. Forced termination can leave +private temporary data behind. +` + +type databaseFlags struct { + enabled, daily, routing, usage bool + keep, credentials string +} + +const databaseURLMessage = "URLs are not accepted in database command arguments; use FLEETDIFF_LITELLM_DATABASE_URL or --database-url-file" + +func (d *databaseFlags) register(f *flag.FlagSet) { + f.BoolVar(&d.enabled, "litellm-db", false, "") + f.StringVar(&d.keep, "keep-export", "", "") + f.StringVar(&d.credentials, "database-url-file", "", "") + f.BoolVar(&d.daily, "daily", false, "") + f.BoolVar(&d.routing, "routing-details", false, "") + f.BoolVar(&d.usage, "usage-details", true, "") +} + +func (d databaseFlags) validate(f *flag.FlagSet) error { + var err error + f.Visit(func(v *flag.Flag) { + switch v.Name { + case "keep-export", "database-url-file", "daily", "routing-details", "usage-details": + if !d.enabled { + err = errors.New("database export options require --litellm-db") + } else if (v.Name == "keep-export" || v.Name == "database-url-file") && strings.TrimSpace(v.Value.String()) == "" { + err = errors.New("database file and directory options must be nonempty") + } + case "litellm-spend", "litellm-daily-spend", "summaries", "expected", "model-usage-contract", "before", "after": + if d.enabled { + err = errors.New("choose --litellm-db or local inputs, not both") + } + } + }) + return err +} + +func databaseURLArgument(args []string) bool { + for _, arg := range args { + _, value, ok := strings.Cut(arg, "=") + if !ok { + value = arg + } + value = strings.ToLower(strings.TrimSpace(value)) + if strings.Contains(arg, "://") || strings.HasPrefix(value, "postgres:") || strings.HasPrefix(value, "postgresql:") { + return true + } + } + return false +} + +func defaultDatabasePeriod(now time.Time) spend.Period { + end := now.UTC().Truncate(24 * time.Hour) + return spend.Period{Start: end.AddDate(0, 0, -14), End: end} +} + +func periodText(p spend.Period) string { + return p.Start.UTC().Format(time.RFC3339Nano) + "/" + p.End.UTC().Format(time.RFC3339Nano) +} + +func databasePeriod(value string, now time.Time) (spend.Period, error) { + if value == "" { + return defaultDatabasePeriod(now), nil + } + p, err := spend.ParsePeriod(value) + if err != nil || p.End.After(now) { + return spend.Period{}, errors.New("database period must be a completed UTC date or RFC3339 start/end interval") + } + return p, nil +} + +func databaseInvestigationPeriods(before, after string, now time.Time) (spend.Period, string, string, error) { + if before == "" && after == "" { + p := defaultDatabasePeriod(now) + middle := p.Start.AddDate(0, 0, 7) + return p, periodText(spend.Period{Start: p.Start, End: middle}), periodText(spend.Period{Start: middle, End: p.End}), nil + } + a, e1 := spend.ParsePeriod(before) + b, e2 := spend.ParsePeriod(after) + if e1 != nil || e2 != nil || a.End.After(b.Start) || a.End.Sub(a.Start) != b.End.Sub(b.Start) || b.End.After(now) { + return spend.Period{}, "", "", errors.New("select two completed, nonoverlapping periods of equal duration") + } + return spend.Period{Start: a.Start, End: b.End}, before, after, nil +} + +func validateDatabaseDaily(p spend.Period, daily bool) error { + if daily && (!p.Start.Equal(p.Start.UTC().Truncate(24*time.Hour)) || !p.End.Equal(p.End.UTC().Truncate(24*time.Hour)) || p.End.Sub(p.Start) > 3660*24*time.Hour) { + return errors.New("daily export needs complete UTC days spanning at most 3,660 days") + } + return nil +} + +func databaseExportError(ctx context.Context, err error) error { + if ctx.Err() != nil { + return errors.New("database export canceled") + } + var failure exportdata.Failure + if errors.As(err, &failure) { + return failure + } + return exportdata.ErrDatabase +} + +// Keep a private parent because the exporter requires a destination that does +// not exist yet. Cleanup owns only that parent, never a user-supplied directory. +func (d databaseFlags) export(ctx context.Context, p spend.Period, exporter exportdata.Exporter) (exportdata.Result, func(), error) { + cleanup := func() {} + if exporter == nil { + return exportdata.Result{}, cleanup, errors.New("database export is unavailable in this offline caller") + } + if ctx.Err() != nil { + return exportdata.Result{}, cleanup, databaseExportError(ctx, ctx.Err()) + } + directory := d.keep + if directory == "" { + var err error + directory, cleanup, err = databaseTemporaryExport() + if err != nil { + return exportdata.Result{}, func() {}, err + } + } + r, err := exporter(ctx, exportdata.Options{ + Directory: directory, CredentialFile: d.credentials, Start: p.Start, End: p.End, + Format: "csv", Daily: d.daily, Routing: true, UsageDetails: d.usage, + }) + if err != nil || ctx.Err() != nil { + cleanup() + return exportdata.Result{}, func() {}, databaseExportError(ctx, err) + } + return r, cleanup, nil +} + +func databaseTemporaryExport() (string, func(), error) { + failure := errors.New("cannot create private temporary export directory under a trusted parent") + // Resolve the system temporary alias before creating anything private. + base, err := filepath.EvalSymlinks(os.TempDir()) + if err != nil { + return "", nil, failure + } + base, err = filepath.Abs(base) + if err != nil { + return "", nil, failure + } + parent, err := os.OpenRoot("/") + if err != nil { + return "", nil, failure + } + // Trust every ancestor, not just TMPDIR: an unprotected writable ancestor + // could replace the private directory's name before the exporter sees it. + for _, part := range append([]string{"."}, strings.FieldsFunc(base, func(r rune) bool { return r == '/' })...) { + info, err := parent.Lstat(part) + if err != nil || !info.IsDir() { + parent.Close() + return "", nil, failure + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || stat.Uid != 0 && stat.Uid != uint32(os.Geteuid()) || info.Mode().Perm()&0022 != 0 && info.Mode()&os.ModeSticky == 0 { + parent.Close() + return "", nil, failure + } + next, err := parent.OpenRoot(part) + parent.Close() + if err != nil { + return "", nil, failure + } + opened, err := next.Stat(".") + if err != nil || !os.SameFile(info, opened) { + next.Close() + return "", nil, failure + } + parent = next + } + name := "fleetdiff-export-" + rand.Text() + if err := parent.Mkdir(name, 0700); err != nil { + parent.Close() + return "", nil, failure + } + info, err := parent.Lstat(name) + if err != nil || !info.IsDir() || info.Mode().Perm() != 0700 { + parent.Close() + return "", nil, failure + } + root, err := parent.OpenRoot(name) + if err != nil { + parent.Close() + return "", nil, failure + } + opened, err := root.Stat(".") + stat, owned := info.Sys().(*syscall.Stat_t) + if err != nil || !os.SameFile(info, opened) || !owned || stat.Uid != uint32(os.Geteuid()) { + root.Close() + parent.Close() + return "", nil, failure + } + path := filepath.Join(base, name) + cleanup := func() { + if root == nil { + return + } + // Recursion is confined to the known export child of the held private + // root. Never recursively remove its replaceable public pathname. + _ = root.RemoveAll("export") + root.Close() + root = nil + if current, err := parent.Lstat(name); err == nil && os.SameFile(info, current) { + // Rmdir cannot delete a replacement file, symlink, or nonempty tree. + _ = syscall.Rmdir(path) + } + parent.Close() + } + return filepath.Join(path, "export"), cleanup, nil +} + +func runExport(ctx context.Context, args []string, out, errout io.Writer, exporter exportdata.Exporter) int { + fail := func(code int, message string) int { fmt.Fprintln(errout, message); return code } + if databaseURLArgument(args) { + return fail(2, databaseURLMessage) + } + if len(args) == 0 || args[0] == "--help" || args[0] == "-h" { + if _, err := io.WriteString(out, exportHelp); err != nil { + return fail(1, "cannot write export help") + } + return 0 + } + if args[0] != "litellm" { + return fail(2, "export supports litellm; run fleetdiff export --help") + } + f := flag.NewFlagSet("export litellm", flag.ContinueOnError) + f.SetOutput(io.Discard) + var opts exportdata.Options + var from, to string + f.StringVar(&opts.Directory, "out", "", "") + f.StringVar(&opts.CredentialFile, "database-url-file", "", "") + f.StringVar(&from, "from", "", "") + f.StringVar(&to, "to", "", "") + f.StringVar(&opts.Format, "format", "csv", "") + f.BoolVar(&opts.Daily, "daily", false, "") + f.BoolVar(&opts.Routing, "routing-details", false, "") + f.BoolVar(&opts.UsageDetails, "usage-details", true, "") + if err := f.Parse(args[1:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + if _, err := io.WriteString(out, exportHelp); err != nil { + return fail(1, "cannot write export help") + } + return 0 + } + return fail(2, "invalid export options; run fleetdiff export --help") + } + empty := false + f.Visit(func(v *flag.Flag) { + if v.Name == "from" || v.Name == "to" || v.Name == "database-url-file" { + empty = empty || strings.TrimSpace(v.Value.String()) == "" + } + }) + if f.NArg() != 0 || strings.TrimSpace(opts.Directory) == "" || empty || opts.Format != "csv" && opts.Format != "jsonl" || (from == "") != (to == "") { + return fail(2, "export needs a new output directory, csv or jsonl, and both UTC bounds when supplied") + } + p := defaultDatabasePeriod(time.Now()) + if from != "" { + bounds := []*time.Time{&p.Start, &p.End} + for i, value := range []string{from, to} { + layout := time.RFC3339Nano + if len(value) == 10 { + layout = time.DateOnly + } + t, err := time.Parse(layout, value) + _, offset := t.Zone() + if err != nil || offset != 0 || t.Year() < 1970 || t.Year() > 2200 { + return fail(2, "export bounds must be UTC dates or RFC3339 timestamps between 1970 and 2200") + } + *bounds[i] = t.UTC() + } + if !p.Start.Before(p.End) || p.End.After(time.Now()) { + return fail(2, "export needs an increasing, completed UTC interval") + } + } + opts.Start, opts.End = p.Start, p.End + if err := validateDatabaseDaily(p, opts.Daily); err != nil { + return fail(2, err.Error()) + } + if exporter == nil { + return fail(1, "database export is unavailable in this offline caller") + } + if ctx.Err() != nil { + return fail(1, databaseExportError(ctx, ctx.Err()).Error()) + } + r, err := exporter(ctx, opts) + if err != nil || ctx.Err() != nil { + return fail(1, databaseExportError(ctx, err).Error()) + } + if _, err = fmt.Fprintf(out, "Exported %d spend rows (%s to %s, end exclusive). Keep the export private.\n", r.Rows, opts.Start.Format(time.RFC3339Nano), opts.End.Format(time.RFC3339Nano)); err != nil { + return fail(1, "cannot write export result") + } + return 0 +} diff --git a/internal/cli/database_files_test.go b/internal/cli/database_files_test.go new file mode 100644 index 0000000..ca86aa8 --- /dev/null +++ b/internal/cli/database_files_test.go @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package cli + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/llm-measurement/fleetdiff/internal/exportdata" +) + +func TestDatabaseTemporaryCleanupPreservesReplacement(t *testing.T) { + for _, kind := range []string{"file", "directory", "symlink"} { + for _, failure := range []string{"export", "analysis"} { + t.Run(kind+"/"+failure, func(t *testing.T) { + base, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + t.Setenv("TMPDIR", base) + target := filepath.Join(base, "unrelated") + if err := os.Mkdir(target, 0700); err != nil { + t.Fatal(err) + } + sentinel := filepath.Join(target, "sentinel") + if err := os.WriteFile(sentinel, []byte("unrelated contents"), 0600); err != nil { + t.Fatal(err) + } + var original, moved string + var replacement os.FileInfo + code, out, errs := databaseRun([]string{"investigate", "--litellm-db"}, func(_ context.Context, opts exportdata.Options) (exportdata.Result, error) { + result := databaseFixture(t, opts, []byte("invalid CSV"), nil) + original = filepath.Dir(opts.Directory) + moved = original + "-moved" + if err := os.Rename(original, moved); err != nil { + t.Fatal(err) + } + switch kind { + case "file": + err = os.WriteFile(original, []byte("replacement"), 0600) + case "directory": + err = os.Mkdir(original, 0700) + if err == nil { + err = os.WriteFile(filepath.Join(original, "sentinel"), []byte("replacement"), 0600) + } + case "symlink": + err = os.Symlink(target, original) + } + if err != nil { + t.Fatal(err) + } + replacement, err = os.Lstat(original) + if err != nil { + t.Fatal(err) + } + if failure == "export" { + return result, errors.New("SENTINEL exporter error") + } + result.SpendPath = filepath.Join(moved, "export", "spend.csv") + return result, nil + }) + if code != 1 || out != "" || strings.Contains(errs, "SENTINEL") || strings.Contains(errs, base) { + t.Fatalf("exit=%d out=%q errors=%q", code, out, errs) + } + after, err := os.Lstat(original) + if err != nil || !os.SameFile(replacement, after) { + t.Fatalf("cleanup removed replacement: %v", err) + } + data, err := os.ReadFile(sentinel) + if err != nil || string(data) != "unrelated contents" { + t.Fatalf("cleanup touched symlink target: %q %v", data, err) + } + if kind == "directory" { + data, err := os.ReadFile(filepath.Join(original, "sentinel")) + if err != nil || string(data) != "replacement" { + t.Fatalf("cleanup traversed replacement: %q %v", data, err) + } + } + if _, err := os.Lstat(filepath.Join(moved, "export")); !os.IsNotExist(err) { + t.Fatalf("owned export was not cleaned through held root: %v", err) + } + }) + } + } +} + +func TestDatabaseTemporaryParentPolicy(t *testing.T) { + for _, ancestor := range []bool{false, true} { + base := t.TempDir() + if err := os.Chmod(base, 0777); err != nil { + t.Fatal(err) + } + if ancestor { + base = filepath.Join(base, "private") + if err := os.Mkdir(base, 0700); err != nil { + t.Fatal(err) + } + } + t.Setenv("TMPDIR", base) + code, out, errs := databaseRun([]string{"investigate", "--litellm-db"}, func(context.Context, exportdata.Options) (exportdata.Result, error) { + t.Fatal("untrusted temporary parent reached exporter") + return exportdata.Result{}, nil + }) + if code != 1 || out != "" || !strings.Contains(errs, "trusted parent") || strings.Contains(errs, base) { + t.Fatalf("exit=%d out=%q errors=%q", code, out, errs) + } + } +} diff --git a/internal/cli/database_test.go b/internal/cli/database_test.go new file mode 100644 index 0000000..4ed29bd --- /dev/null +++ b/internal/cli/database_test.go @@ -0,0 +1,536 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package cli + +import ( + "bytes" + "context" + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/llm-measurement/fleetdiff/internal/exportdata" + "github.com/llm-measurement/fleetdiff/internal/reprice" + "github.com/llm-measurement/fleetdiff/internal/spend" +) + +func databaseRun(args []string, exporter exportdata.Exporter) (int, string, string) { + var out, errs bytes.Buffer + code := RunWithExporter(context.Background(), args, strings.NewReader(""), &out, &errs, exporter) + return code, out.String(), errs.String() +} + +func databaseFixture(t *testing.T, opts exportdata.Options, contents, daily []byte) exportdata.Result { + t.Helper() + if _, err := os.Lstat(opts.Directory); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("export destination must not exist: %v", err) + } + if strings.HasPrefix(filepath.Base(filepath.Dir(opts.Directory)), "fleetdiff-export-") { + resolved, err := filepath.EvalSymlinks(filepath.Dir(opts.Directory)) + if err != nil || resolved != filepath.Dir(opts.Directory) { + t.Fatalf("temporary path still has symlink components: %q %v", resolved, err) + } + parent, err := os.Stat(filepath.Dir(opts.Directory)) + if err != nil || parent.Mode().Perm() != 0700 { + t.Fatalf("temporary export parent must be private: %v %v", parent, err) + } + } + if err := os.Mkdir(opts.Directory, 0700); err != nil { + t.Fatal(err) + } + r := exportdata.Result{Directory: opts.Directory, SpendPath: filepath.Join(opts.Directory, "spend.csv"), Start: opts.Start, End: opts.End, Rows: 2} + if err := os.WriteFile(r.SpendPath, contents, 0600); err != nil { + t.Fatal(err) + } + if opts.Daily { + r.DailyPath = filepath.Join(opts.Directory, "daily.csv") + if err := os.WriteFile(r.DailyPath, daily, 0600); err != nil { + t.Fatal(err) + } + } + return r +} + +func databaseSparseRows(t *testing.T, opts exportdata.Options) []byte { + t.Helper() + var b bytes.Buffer + w := csv.NewWriter(&b) + rows := [][]string{{"request_id", "startTime", "endTime", "call_type", "status", "api_key", "session_id", "model", "prompt_tokens", "completion_tokens", "total_tokens", "spend", "cache_read_input_tokens", "cache_write_input_tokens"}} + for i, at := range []time.Time{opts.Start.Add(time.Hour), opts.Start.Add(opts.End.Sub(opts.Start) / 2).Add(time.Hour)} { + rows = append(rows, []string{[]string{"SENTINEL_REQUEST_A", "SENTINEL_REQUEST_B"}[i], at.Format(time.RFC3339Nano), at.Add(time.Second).Format(time.RFC3339Nano), "completion", "success", "SENTINEL_KEY", "SENTINEL_SESSION", "claude-sonnet-5-5", "100", "20", "120", "0.001", "0", "0"}) + } + if err := w.WriteAll(rows); err != nil { + t.Fatal(err) + } + return b.Bytes() +} + +func TestDatabaseExportOptions(t *testing.T) { + for _, explicit := range []bool{false, true} { + t.Run(map[bool]string{false: "defaults", true: "explicit"}[explicit], func(t *testing.T) { + args := []string{"export", "litellm", "--out", filepath.Join(t.TempDir(), "SENTINEL_DIRECTORY")} + if explicit { + args = append(args, "--from", "2026-10-01", "--to", "2026-10-15T00:00:00Z", "--format", "jsonl", "--daily", "--routing-details", "--usage-details=false", "--database-url-file", "SENTINEL_CREDENTIAL_FILE") + } + var got exportdata.Options + calls := 0 + before := time.Now() + code, out, errs := databaseRun(args, func(ctx context.Context, opts exportdata.Options) (exportdata.Result, error) { + calls++ + got = opts + return exportdata.Result{Rows: 23}, nil + }) + if code != 0 || calls != 1 || errs != "" || !strings.Contains(out, "Exported 23 spend rows") || strings.Contains(out, "SENTINEL") { + t.Fatalf("exit=%d calls=%d out=%q errors=%q", code, calls, out, errs) + } + if got.Directory != args[3] || got.End.Sub(got.Start) != 14*24*time.Hour { + t.Fatalf("incorrect range/destination: %+v", got) + } + if explicit { + if got.Format != "jsonl" || !got.Daily || !got.Routing || got.UsageDetails || got.CredentialFile != "SENTINEL_CREDENTIAL_FILE" || got.Start.Format(time.DateOnly) != "2026-10-01" { + t.Fatalf("explicit options lost: %+v", got) + } + } else if got.Format != "csv" || got.Daily || got.Routing || !got.UsageDetails || !got.End.Equal(before.UTC().Truncate(24*time.Hour)) && !got.End.Equal(time.Now().UTC().Truncate(24*time.Hour)) { + t.Fatalf("wrong defaults: %+v", got) + } + }) + } +} + +func TestDatabaseNeverExportsOfflineCommands(t *testing.T) { + cases := [][]string{ + nil, {"--help"}, {"--version"}, {"export", "--help"}, {"export", "litellm", "--help"}, + {"investigate", "--litellm-db", "--help"}, {"what-if", "--litellm-db", "--help"}, + {"inspect", "--help"}, {"diagnose", "--help"}, {"scan", "--help"}, + {"compare", "--before", "../../examples/two-systems/data/before", "--after", "../../examples/two-systems/data/after", "--expected", "owned,partner"}, + {"investigate", "--litellm-spend", "../../examples/litellm-spend/synthetic.csv", "--before-period", "2026-10-07", "--after-period", "2026-10-08"}, + {"inspect", "../../testdata/inspect-contract/v1/litellm-normal-no-usage-provenance.json"}, + {"diagnose", "../../examples/diagnose/safe.yaml"}, + {"scan", "../../examples/what-if-summaries/snapshots", "--expected", "demo-a,demo-b"}, + whatIfArgs(), + } + for _, args := range cases { + t.Run(strings.Join(args, " "), func(t *testing.T) { + calls := 0 + exporter := func(context.Context, exportdata.Options) (exportdata.Result, error) { + calls++ + return exportdata.Result{}, errors.New("SENTINEL_DATABASE") + } + code, _, errs := databaseRun(args, exporter) + if calls != 0 || code != 0 && (len(args) == 0 || args[0] != "scan") { + t.Fatalf("offline command exit=%d calls=%d: %s", code, calls, errs) + } + }) + } +} + +func TestDatabaseInvalidOptionsNeverExport(t *testing.T) { + t.Setenv("SENTINEL_WEAK_SECRET", "weak") + for command, cases := range map[string][][]string{ + "export": { + {"litellm"}, {"other"}, {"litellm", "--out", ""}, {"litellm", "--out", "new", "--format", "text"}, + {"litellm", "--out", "new", "--from", "2026-10-01"}, + {"litellm", "--out", "new", "--from", "", "--to", ""}, + {"litellm", "--out", "new", "--from", "SENTINEL_DATE", "--to", "2026-10-02"}, + {"litellm", "--out", "new", "--from", "2026-10-03", "--to", "2026-10-02"}, + {"litellm", "--out", "new", "--from", "2026-10-01T00:00:00+01:00", "--to", "2026-10-02"}, + {"litellm", "--out", "new", "--from", "2199-10-01", "--to", "2199-10-02"}, + {"litellm", "--out", "new", "--database-url-file", ""}, + {"litellm", "--out", "new", "extra"}, {"litellm", "--out", "new", "--usage-details=bad"}, + {"litellm", "--out", "new", "--daily", "--from", "2026-10-01T01:00:00Z", "--to", "2026-10-02"}, + {"litellm", "--out", "new", "--daily", "--from", "2000-10-01", "--to", "2026-10-02"}, + }, + "investigate": { + {"--daily"}, {"--daily=false"}, {"--routing-details"}, {"--usage-details=false"}, {"--keep-export", "SENTINEL_PATH"}, {"--database-url-file", "SENTINEL_PATH"}, + {"--litellm-db", "--litellm-spend", "SENTINEL_PATH"}, {"--litellm-db", "--litellm-spend="}, + {"--litellm-db", "--litellm-daily-spend", "SENTINEL_PATH"}, {"--litellm-db", "--before", "SENTINEL_PATH"}, + {"--litellm-db", "--expected", "SENTINEL_PRODUCER"}, {"--litellm-db", "--allow-partial"}, + {"--litellm-db", "--format", "csv"}, {"--litellm-db", "--top", "0"}, {"--litellm-db", "--top", "101"}, + {"--litellm-db", "--group-by", "SENTINEL_GROUP"}, {"--litellm-db", "--hash-secret-env", "SENTINEL_WEAK_SECRET"}, + {"--litellm-db", "--before-period", "2026-10-01"}, {"--litellm-db", "--before-period=", "--after-period="}, + {"--litellm-db", "--before-period", "2026-10-02", "--after-period", "2026-10-01"}, + {"--litellm-db", "--before-period", "2026-10-01", "--after-period", "2026-10-01"}, + {"--litellm-db", "--before-period", "2199-10-01", "--after-period", "2199-10-02"}, + {"--litellm-db", "--period", "SENTINEL_PERIOD"}, {"--litellm-db", "--period="}, + {"--litellm-db", "--period", "2026-10-01", "--before-period", "2026-10-01", "--after-period", "2026-10-02"}, + {"--litellm-db", "--session-budget-usd", "-1"}, {"--litellm-db", "--session-budget-usd", "1", "--session-counter-window", "1ms"}, + {"--litellm-db", "--session-budget-usd", "1", "--group-by", "key"}, + {"--litellm-db", "--routing-by", "route"}, {"--litellm-db", "--show-labels"}, + {"--litellm-db", "--scope-route", "SENTINEL_ROUTE"}, + {"--litellm-db", "--daily", "--scope-route", "SENTINEL_ROUTE", "--before-period", "2026-10-01", "--after-period", "2026-10-02"}, + {"--litellm-db", "--routing-audit", "--period", "2026-10-01"}, + {"--litellm-db", "--keep-export="}, {"--litellm-db", "--database-url-file="}, + {"--litellm-db", "SENTINEL_EXTRA"}, + {"--litellm-db", "--daily", "--period", "2026-10-01T01:00:00Z/2026-10-02T00:00:00Z"}, + }, + "what-if": { + {"--daily"}, {"--keep-export", "SENTINEL_PATH"}, {"--database-url-file", "SENTINEL_PATH"}, + {"--litellm-db", "--litellm-spend", "SENTINEL_PATH"}, {"--litellm-db", "--summaries", "SENTINEL_PATH"}, + {"--litellm-db", "--expected", "SENTINEL_PRODUCER"}, {"--litellm-db", "--model-usage-contract", "SENTINEL_CONTRACT"}, + {"--litellm-db"}, {"--litellm-db", "--from", "SENTINEL_MODEL", "--to", "gpt-6.1-sol", "--fraction", "0.5"}, + {"--litellm-db", "--scenario", "SENTINEL_MISSING_FILE"}, + {"--litellm-db", "--scenario", "../../examples/what-if/scenario.json", "--period", "SENTINEL_PERIOD"}, + {"--litellm-db", "--scenario", "../../examples/what-if/scenario.json", "--period="}, + {"--litellm-db", "--scenario", "../../examples/what-if/scenario.json", "--period", "2199-10-01"}, + {"--litellm-db", "--scenario", "../../examples/what-if/scenario.json", "--fraction", "0.5"}, + {"--litellm-db", "--scenario", "../../examples/what-if/scenario.json", "--prices", "SENTINEL_MISSING_FILE"}, + {"--litellm-db", "--scenario", "../../examples/what-if/scenario.json", "--workload-map", "SENTINEL_MISSING_FILE"}, + {"--litellm-db", "--scenario", "../../examples/what-if/scenario.json", "--daily", "--period", "2026-10-01T01:00:00Z/2026-10-02T00:00:00Z"}, + }, + "compare": {{"--litellm-db"}}, + } { + for _, flags := range cases { + args := append([]string{command}, flags...) + t.Run(strings.Join(args, " "), func(t *testing.T) { + calls := 0 + code, out, errs := databaseRun(args, func(context.Context, exportdata.Options) (exportdata.Result, error) { + calls++ + return exportdata.Result{}, errors.New("SENTINEL_DRIVER_ERROR") + }) + if code != 2 || calls != 0 || out != "" || strings.Contains(errs, "SENTINEL") { + t.Fatalf("exit=%d calls=%d out=%q errors=%q", code, calls, out, errs) + } + }) + } + } +} + +func TestDatabaseURLArgumentsArePrivate(t *testing.T) { + for _, args := range [][]string{ + {"export", "litellm", "postgres://SENTINEL_USER:SENTINEL_PASSWORD@SENTINEL_HOST/db"}, + {"investigate", "--litellm-db", "postgresql://SENTINEL_HOST/db"}, + {"what-if", "--litellm-db", "--database-url-file=postgres://SENTINEL_HOST/db"}, + {"export", "--help", "https://SENTINEL_HOST"}, + {"export", "litellm", "postgres:SENTINEL_CREDENTIAL"}, + } { + code, out, errs := databaseRun(args, func(context.Context, exportdata.Options) (exportdata.Result, error) { + t.Fatal("URL argument reached exporter") + return exportdata.Result{}, nil + }) + if code != 2 || out != "" || !strings.Contains(errs, "URLs are not accepted") || strings.Contains(errs, "SENTINEL") { + t.Fatalf("exit=%d out=%q errors=%q", code, out, errs) + } + } +} + +func TestDatabaseLibraryCallersStayOffline(t *testing.T) { + for _, args := range [][]string{ + {"export", "litellm", "--out", "new"}, {"investigate", "--litellm-db"}, + {"what-if", "--litellm-db", "--from", "claude-sonnet-5-5", "--to", "gpt-6.1-sol", "--fraction", "0.5"}, + } { + var out, errs bytes.Buffer + if code := Run(args, &out, &errs); code != 1 || out.Len() != 0 || !strings.Contains(errs.String(), "offline caller") { + t.Fatalf("Run attempted database access: exit=%d %s %s", code, &out, &errs) + } + out.Reset() + errs.Reset() + if code := RunWithInput(args, strings.NewReader(""), &out, &errs); code != 1 || out.Len() != 0 || !strings.Contains(errs.String(), "offline caller") { + t.Fatalf("RunWithInput attempted database access: exit=%d %s %s", code, &out, &errs) + } + } +} + +func TestDatabaseDefaultInvestigationKeepsFullWindow(t *testing.T) { + var opts exportdata.Options + code, out, errs := databaseRun([]string{"investigate", "--litellm-db", "--format", "json"}, func(_ context.Context, o exportdata.Options) (exportdata.Result, error) { + opts = o + return databaseFixture(t, o, databaseSparseRows(t, o), nil), nil + }) + if code != 0 || errs != "" { + t.Fatalf("exit=%d: %s", code, errs) + } + var report spend.Report + if err := json.Unmarshal([]byte(out), &report); err != nil { + t.Fatal(err) + } + if !report.Before.Period.Start.Equal(opts.Start) || !report.After.Period.End.Equal(opts.End) || report.Before.Period.End.Sub(report.Before.Period.Start) != 7*24*time.Hour || !report.Before.Period.End.Equal(report.After.Period.Start) || report.Before.Requests != 1 || report.After.Requests != 1 { + t.Fatalf("sparse export changed requested periods: before=%+v after=%+v", report.Before, report.After) + } + if _, err := os.Stat(filepath.Dir(opts.Directory)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("temporary parent remains: %v", err) + } + if !opts.Routing || !opts.UsageDetails || strings.Contains(out, "SENTINEL") { + t.Fatalf("missing export details or identity leak: %+v %s", opts, out) + } +} + +func TestDatabaseReportsMatchExistingFilePaths(t *testing.T) { + t.Setenv("DATABASE_TEST_HASH", "fixed-database-test-hash-secret-at-least-16-bytes") + for _, tc := range []struct { + name, command, fixture, daily string + flags []string + }{ + {"spend", "investigate", "litellm-spend/synthetic.csv", "", []string{"--before-period", "2026-10-07", "--after-period", "2026-10-08"}}, + {"coverage", "investigate", "litellm-spend/coverage_runtime_spend.csv", "litellm-spend/coverage_runtime_daily.csv", []string{"--before-period", "2026-10-07", "--after-period", "2026-10-08"}}, + {"routing", "investigate", "routing/synthetic.csv", "", []string{"--before-period", "2026-10-01", "--after-period", "2026-10-02", "--routing-audit"}}, + {"scope", "investigate", "routing/synthetic.csv", "", []string{"--before-period", "2026-10-01", "--after-period", "2026-10-02", "--scope-route", "coding-default"}}, + {"session", "investigate", "litellm-spend/session-budget.csv", "", []string{"--period", "2026-10-03", "--session-budget-usd", "1.00"}}, + {"whatif", "what-if", "what-if/spend.csv", "", []string{"--period", "2026-10-07", "--scenario", "../../examples/what-if/scenario.json"}}, + {"whatif-routing", "what-if", "routing/synthetic.csv", "", []string{"--period", "2026-10-02", "--from", "claude-sonnet-5-5", "--to", "gpt-6.1-sol", "--fraction", "0.5", "--scope-route", "coding-default"}}, + } { + t.Run(tc.name, func(t *testing.T) { + flags := append(append([]string{}, tc.flags...), "--format", "json") + if tc.command == "investigate" { + flags = append(flags, "--hash-secret-env", "DATABASE_TEST_HASH") + } + fileArgs := append([]string{tc.command, "--litellm-spend", "../../examples/" + tc.fixture}, flags...) + dbArgs := append([]string{tc.command, "--litellm-db"}, flags...) + contents, err := os.ReadFile("../../examples/" + tc.fixture) + if err != nil { + t.Fatal(err) + } + var daily []byte + if tc.daily != "" { + fileArgs = append(fileArgs, "--litellm-daily-spend", "../../examples/"+tc.daily) + dbArgs = append(dbArgs, "--daily") + daily, err = os.ReadFile("../../examples/" + tc.daily) + if err != nil { + t.Fatal(err) + } + } + fileCode, want, fileErr := databaseRun(fileArgs, nil) + var opts exportdata.Options + code, got, errs := databaseRun(dbArgs, func(_ context.Context, o exportdata.Options) (exportdata.Result, error) { + opts = o + return databaseFixture(t, o, contents, daily), nil + }) + if code != 0 || fileCode != 0 || got != want || errs != fileErr { + t.Fatalf("database/file parity: exit=%d/%d errors=%q/%q\n%s\nwant:\n%s", code, fileCode, errs, fileErr, got, want) + } + if !opts.Routing || !opts.UsageDetails || opts.Daily != (tc.daily != "") { + t.Fatalf("wrong options: %+v", opts) + } + if _, err := os.Stat(filepath.Dir(opts.Directory)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("temporary parent remains: %v", err) + } + }) + } +} + +func TestDatabaseWhatIfDefaultRangeAndModelFlags(t *testing.T) { + var opts exportdata.Options + code, out, errs := databaseRun([]string{"what-if", "--litellm-db", "--from", "claude-sonnet-5-5", "--to", "gpt-6.1-sol", "--fraction", "0.5", "--format", "json", "--usage-details=false", "--database-url-file", "SENTINEL_FILE"}, func(_ context.Context, o exportdata.Options) (exportdata.Result, error) { + opts = o + return databaseFixture(t, o, databaseSparseRows(t, o), nil), nil + }) + if code != 0 || errs != "" { + t.Fatalf("exit=%d: %s", code, errs) + } + var report reprice.Report + if err := json.Unmarshal([]byte(out), &report); err != nil { + t.Fatal(err) + } + if report.Covered != 2 || !report.Source.Start.Equal(opts.Start) || !report.Source.End.Equal(opts.End) || opts.End.Sub(opts.Start) != 14*24*time.Hour || opts.UsageDetails || opts.CredentialFile != "SENTINEL_FILE" { + t.Fatalf("wrong what-if range/model options: %+v %+v", opts, report) + } +} + +func TestDatabaseWhatIfRoutingMapWithoutScope(t *testing.T) { + path := whatIfRoutingFile(t, []map[string]any{whatIfRoutingRow("one")}) + var encoded bytes.Buffer + if err := spend.ReadRouting(path, func(row spend.Row) error { + return json.NewEncoder(&encoded).Encode(row.Values) + }); err != nil { + t.Fatal(err) + } + mapping := whatIfRoutingFile(t, reprice.WorkloadMap{Version: 1, Mappings: []reprice.Mapping{{Selector: reprice.Selector{Kind: "route", Value: "private-route-sentinel"}, Workload: "qa"}}}) + args := []string{"what-if", "--litellm-db", "--period", "2020-01-02", "--from", "claude-sonnet-5-5", "--to", "gpt-6.1-sol", "--fraction", "0.5", "--workload-map", mapping, "--format", "json"} + code, out, errs := databaseRun(args, func(_ context.Context, opts exportdata.Options) (exportdata.Result, error) { + if !opts.Routing { + t.Fatal("route workload map needs routing columns even without a scope flag") + } + r := databaseFixture(t, opts, nil, nil) + r.SpendPath = filepath.Join(opts.Directory, "spend.jsonl") + if err := os.WriteFile(r.SpendPath, encoded.Bytes(), 0600); err != nil { + t.Fatal(err) + } + return r, nil + }) + var report reprice.Report + if code != 0 || errs != "" || json.Unmarshal([]byte(out), &report) != nil || report.Unmapped != 0 || report.Source.Attribution == nil || report.Covered != 1 || len(report.Level2) != 1 || report.Level2[0].Workload != "mapped" { + t.Fatalf("route mapping lost: exit=%d errors=%q report=%s", code, errs, out) + } +} + +func TestDatabasePeriodsUseUTCAndExplicitEnvelope(t *testing.T) { + now := time.Date(2026, 10, 18, 0, 30, 0, 0, time.FixedZone("east", 12*3600)) + p := defaultDatabasePeriod(now) + if p.End.Format(time.RFC3339) != "2026-10-17T00:00:00Z" || p.Start.Format(time.RFC3339) != "2026-10-03T00:00:00Z" { + t.Fatalf("default uses local midnight: %+v", p) + } + p, before, after, err := databaseInvestigationPeriods("2026-10-01", "2026-10-05", now) + if err != nil || before != "2026-10-01" || after != "2026-10-05" || p.Start.Format(time.DateOnly) != "2026-10-01" || p.End.Format(time.DateOnly) != "2026-10-06" { + t.Fatalf("gapped period envelope is wrong: %+v %q %q %v", p, before, after, err) + } + if _, _, _, err := databaseInvestigationPeriods("2026-10-01", "2026-10-03T00:00:00Z/2026-10-05T00:00:00Z", now); err == nil { + t.Fatal("unequal durations accepted") + } +} + +func TestDatabaseDefaultSessionRange(t *testing.T) { + var opts exportdata.Options + code, out, errs := databaseRun([]string{"investigate", "--litellm-db", "--session-budget-usd", "1", "--format", "json"}, func(_ context.Context, o exportdata.Options) (exportdata.Result, error) { + opts = o + return databaseFixture(t, o, databaseSparseRows(t, o), nil), nil + }) + var report spend.SessionReport + if code != 0 || errs != "" || json.Unmarshal([]byte(out), &report) != nil || !report.Period.Start.Equal(opts.Start) || !report.Period.End.Equal(opts.End) || report.AutomaticPeriod { + t.Fatalf("session range inferred from file edges: exit=%d errors=%q report=%s", code, errs, out) + } +} + +func TestDatabaseStandaloneErrorsArePrivate(t *testing.T) { + args := []string{"export", "litellm", "--out", "SENTINEL_PATH", "--database-url-file", "SENTINEL_FILE"} + code, out, errs := databaseRun(args, func(context.Context, exportdata.Options) (exportdata.Result, error) { + return exportdata.Result{}, errors.New("postgres://SENTINEL_USER:SENTINEL_PASSWORD@SENTINEL_HOST SENTINEL_PATH SENTINEL_FILE") + }) + if code != 1 || out != "" || strings.Contains(errs, "SENTINEL") { + t.Fatalf("driver error exposed: exit=%d out=%q errors=%q", code, out, errs) + } + var errorsOut bytes.Buffer + code = RunWithExporter(context.Background(), args, nil, databaseBrokenWriter{}, &errorsOut, func(context.Context, exportdata.Options) (exportdata.Result, error) { + return exportdata.Result{Rows: 1}, nil + }) + if code != 1 || strings.Contains(errorsOut.String(), "SENTINEL") { + t.Fatalf("writer error exposed: exit=%d %s", code, &errorsOut) + } +} + +func TestDatabaseFixedFailuresReachUser(t *testing.T) { + for _, failure := range []exportdata.Failure{exportdata.ErrCredentials, exportdata.ErrConfiguration, exportdata.ErrTransport, exportdata.ErrOutput, exportdata.ErrSchema, exportdata.ErrLimit, exportdata.ErrInterrupted, exportdata.ErrDatabase} { + for _, args := range [][]string{{"export", "litellm", "--out", "new"}, {"investigate", "--litellm-db"}, {"what-if", "--litellm-db", "--scenario", "../../examples/what-if/scenario.json"}} { + code, out, errs := databaseRun(args, func(context.Context, exportdata.Options) (exportdata.Result, error) { + return exportdata.Result{}, fmt.Errorf("SENTINEL_DRIVER_CONTEXT: %w", failure) + }) + if code != 1 || out != "" || errs != failure.Error()+"\n" { + t.Fatalf("fixed failure suppressed or wrapper leaked: exit=%d out=%q errors=%q", code, out, errs) + } + } + } +} + +func TestDatabaseURLGuardPreservesOfflineIdentityLabels(t *testing.T) { + for _, kind := range []string{"route", "agent"} { + row := whatIfRoutingRow("one") + field := "original_model_group" + if kind == "agent" { + field = "agent_id" + } + label := "https://private-" + kind + "-sentinel.invalid" + row[field] = label + path := whatIfRoutingFile(t, []map[string]any{row}) + for _, args := range [][]string{ + append(whatIfRoutingArgs(path), "--scope-"+kind, label), + {"investigate", "--litellm-spend", path, "--before-period", "2020-01-01", "--after-period", "2020-01-02", "--scope-" + kind, label}, + } { + code, out, errs := databaseRun(args, func(context.Context, exportdata.Options) (exportdata.Result, error) { + t.Fatal("offline identity label invoked exporter") + return exportdata.Result{}, nil + }) + if code != 0 || errs != "" || strings.Contains(out, label) { + t.Fatalf("offline URL identity changed semantics: exit=%d out=%q errors=%q", code, out, errs) + } + } + } +} + +type databaseBrokenWriter struct{} + +func (databaseBrokenWriter) Write([]byte) (int, error) { + return 0, errors.New("SENTINEL_WRITE_FAILURE") +} + +func TestDatabaseTemporaryCleanupAndPrivateErrors(t *testing.T) { + for _, command := range []string{"investigate", "what-if"} { + for _, failure := range []string{"export", "parse", "write", "cancel", "cancel-after-export"} { + t.Run(command+"/"+failure, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + args := []string{command, "--litellm-db"} + if command == "what-if" { + args = append(args, "--from", "claude-sonnet-5-5", "--to", "gpt-6.1-sol", "--fraction", "0.5") + } + var directory string + exporter := func(gotCtx context.Context, o exportdata.Options) (exportdata.Result, error) { + if gotCtx != ctx { + t.Fatal("context was not propagated") + } + directory = o.Directory + contents := databaseSparseRows(t, o) + if failure == "parse" { + contents = []byte("SENTINEL_INVALID_FILE") + } + r := databaseFixture(t, o, contents, nil) + if failure == "cancel" { + cancel() + return r, ctx.Err() + } + if failure == "cancel-after-export" { + cancel() + } + if failure == "export" { + return r, errors.New("postgres://SENTINEL_USER:SENTINEL_PASSWORD@SENTINEL_HOST " + o.Directory) + } + return r, nil + } + var out, errs bytes.Buffer + var writer io.Writer = &out + if failure == "write" { + writer = databaseBrokenWriter{} + } + code := RunWithExporter(ctx, args, strings.NewReader(""), writer, &errs, exporter) + if code != 1 || out.Len() != 0 || strings.Contains(errs.String(), "SENTINEL") || strings.Contains(errs.String(), directory) { + t.Fatalf("exit=%d out=%q errors=%q", code, &out, &errs) + } + if _, err := os.Stat(filepath.Dir(directory)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("temporary parent remains after %s: %v", failure, err) + } + }) + } + } +} + +func TestDatabaseKeepExport(t *testing.T) { + for _, failure := range []bool{false, true} { + directory := filepath.Join(t.TempDir(), "SENTINEL_PRIVATE_EXPORT") + code, out, errs := databaseRun([]string{"investigate", "--litellm-db", "--keep-export", directory}, func(_ context.Context, o exportdata.Options) (exportdata.Result, error) { + if o.Directory != directory { + t.Fatal("keep destination not honored") + } + contents := databaseSparseRows(t, o) + if failure { + contents = []byte("SENTINEL_INVALID_CSV") + } + return databaseFixture(t, o, contents, nil), nil + }) + if code != map[bool]int{false: 0, true: 1}[failure] || strings.Contains(out+errs, "SENTINEL") { + t.Fatalf("exit=%d out=%q errors=%q", code, out, errs) + } + if _, err := os.Stat(filepath.Join(directory, "spend.csv")); err != nil { + t.Fatalf("explicitly retained export removed: %v", err) + } + } +} + +func TestDatabaseCanceledBeforeExport(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + for _, args := range [][]string{{"export", "litellm", "--out", "new"}, {"investigate", "--litellm-db"}, {"what-if", "--litellm-db", "--scenario", "../../examples/what-if/scenario.json"}} { + var out, errs bytes.Buffer + code := RunWithExporter(ctx, args, nil, &out, &errs, func(context.Context, exportdata.Options) (exportdata.Result, error) { + t.Fatal("canceled request reached exporter") + return exportdata.Result{}, nil + }) + if code != 1 || out.Len() != 0 || !strings.Contains(errs.String(), "canceled") { + t.Fatalf("exit=%d out=%q errors=%q", code, &out, &errs) + } + } +} diff --git a/internal/cli/network_test.go b/internal/cli/network_test.go index b914958..c1618af 100644 --- a/internal/cli/network_test.go +++ b/internal/cli/network_test.go @@ -9,6 +9,7 @@ import ( "encoding/json" "fmt" "go/ast" + "go/constant" "go/importer" "go/parser" "go/token" @@ -34,6 +35,9 @@ import ( // scripts/restricted-smoke.sh also compares built Linux CLI outputs with Docker // bridge networking versus --network none in CI. Assembly, reflection and // dynamic loading still require review; neither test proves absence of all I/O. +// Explicit database export permits only the reviewed adapter/pgconn dial and +// resolver APIs below. All packages still receive the HTTP/process/cgo checks; +// offline packages cannot import the adapter or its driver runtime at all. func TestProductionHasNoNetwork(t *testing.T) { packages := listNetworkPackages(t, "../..") for _, finding := range networkFindings(t, packages) { @@ -48,6 +52,64 @@ type networkPackage struct { Standard bool GoFiles []string CgoFiles []string + Imports []string + Module *networkModule +} + +type networkModule struct { + Path, Version string + Replace *networkModule +} + +const databaseAdapter = "github.com/llm-measurement/fleetdiff/internal/postgres" +const databaseMain = "github.com/llm-measurement/fleetdiff/cmd/fleetdiff" + +// Exact runtime packages, not a module-prefix exemption. New driver packages +// and upgrades require review; shared x/text continues to be scanned normally. +func databaseRuntimeModule(path string) (string, string) { + switch path { + case "github.com/jackc/pgpassfile": + return path, "v1.0.0" + case "github.com/jackc/pgservicefile": + return path, "v0.0.0-20240606120523-5a60cdf6a761" + case "github.com/jackc/pgx/v5/pgconn", + "github.com/jackc/pgx/v5/pgconn/ctxwatch", + "github.com/jackc/pgx/v5/pgconn/internal/bgreader", + "github.com/jackc/pgx/v5/pgproto3", + "github.com/jackc/pgx/v5/pgtype", + "github.com/jackc/pgx/v5/internal/pgio", + "github.com/jackc/pgx/v5/internal/pgdatetime", + "github.com/jackc/pgx/v5/internal/iobufpool": + return "github.com/jackc/pgx/v5", "v5.11.0" + } + return "", "" +} + +func databaseBoundaryFindings(packages []networkPackage) []string { + var findings []string + for _, p := range packages { + module, version := databaseRuntimeModule(p.ImportPath) + if strings.HasPrefix(p.ImportPath, "github.com/jackc/") && module == "" { + findings = append(findings, p.ImportPath+": unreviewed database runtime package") + } + if module != "" && (p.Module == nil || p.Module.Path != module || p.Module.Version != version || p.Module.Replace != nil) { + findings = append(findings, p.ImportPath+": database runtime differs from approved module pin") + } + for _, imported := range p.Imports { + if imported == databaseAdapter && p.ImportPath != databaseMain { + findings = append(findings, p.ImportPath+": offline code imports database adapter") + } + if strings.HasPrefix(imported, "github.com/jackc/") && p.ImportPath != databaseAdapter && module == "" { + findings = append(findings, p.ImportPath+": offline code imports database runtime") + } + } + } + return findings +} + +func databaseNetworkFunction(owner, path, name string) bool { + return (owner == databaseAdapter || owner == "github.com/jackc/pgx/v5/pgconn") && + path == "net" && (name == "DialContext" || name == "LookupHost") } func listNetworkPackages(t *testing.T, dir string) []networkPackage { @@ -90,7 +152,7 @@ func networkFindings(t *testing.T, packages []networkPackage) []string { imp := importer.ForCompiler(fset, "gc", func(path string) (io.ReadCloser, error) { return os.Open(exports[path]) }) - var findings []string + findings := databaseBoundaryFindings(packages) for _, p := range packages { if networkImport(p.ImportPath) { findings = append(findings, p.ImportPath+": forbidden dependency in production graph") @@ -121,11 +183,12 @@ func networkFindings(t *testing.T, packages []networkPackage) []string { } } } - info := &types.Info{Uses: make(map[*ast.Ident]types.Object)} + info := &types.Info{Uses: make(map[*ast.Ident]types.Object), Types: make(map[ast.Expr]types.TypeAndValue)} config := types.Config{Importer: imp, Sizes: types.SizesFor("gc", runtime.GOARCH)} if _, err := config.Check(p.ImportPath, fset, files, info); err != nil { t.Fatalf("type-check %s for network guard: %v", p.ImportPath, err) } + filesystemCalls := reviewedFilesystemSyscalls(p.ImportPath, files, fset, info) // Resolve symbols, not selector spelling. This catches import aliases, // dot imports, method expressions, and saved function values while // leaving shadowed names and unrelated Dial/Listen methods alone. @@ -136,7 +199,7 @@ func networkFindings(t *testing.T, packages []networkPackage) []string { if _, ok := obj.(*types.Func); !ok { continue } - if networkFunction(obj.Pkg().Path(), obj.Name()) { + if networkFunction(obj.Pkg().Path(), obj.Name()) && !databaseNetworkFunction(p.ImportPath, obj.Pkg().Path(), obj.Name()) && !filesystemCalls[id] { findings = append(findings, fmt.Sprintf("%s: %s: forbidden network/process API %s.%s", p.ImportPath, fset.Position(id.Pos()), obj.Pkg().Path(), obj.Name())) } } @@ -145,6 +208,72 @@ func networkFindings(t *testing.T, packages []networkPackage) []string { return findings } +// Permit only direct, constant-operation calls in the two audited filesystem +// wrappers. Saved syscall functions, variable operation/flag values, other +// files, and all socket/process operations remain forbidden. +func reviewedFilesystemSyscalls(owner string, files []*ast.File, fset *token.FileSet, info *types.Info) map[*ast.Ident]bool { + allowed := make(map[*ast.Ident]bool) + if owner != databaseAdapter { + return allowed + } + for _, file := range files { + filename := filepath.Base(fset.Position(file.Pos()).Filename) + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Body == nil { + continue + } + ast.Inspect(fn.Body, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok || len(call.Args) != 7 || call.Ellipsis.IsValid() { + return true + } + selector, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return true + } + obj, ok := info.Uses[selector.Sel].(*types.Func) + if !ok || obj.Pkg() == nil || obj.Pkg().Path() != "syscall" || obj.Name() != "Syscall6" { + return true + } + is := func(index int, want int64) bool { + value := info.Types[call.Args[index]].Value + if value == nil || value.Kind() != constant.Int { + return false + } + got, ok := constant.Int64Val(value) + return ok && got == want + } + var rename, unlink, noReplace, removeDir int64 + switch filename { + case "rename_darwin.go": + rename, unlink, noReplace, removeDir = 488, 472, 4, 0x80 + case "rename_linux.go": + noReplace, removeDir = 1, 0x200 + switch runtime.GOARCH { + case "amd64": + rename, unlink = 316, 263 + case "arm64": + rename, unlink = 276, 35 + default: + return true + } + default: + return true + } + switch fn.Name.Name { + case "renameDirectoryNoReplace": + allowed[selector.Sel] = is(0, rename) && is(5, noReplace) && is(6, 0) + case "removeEmptyDirectoryAt": + allowed[selector.Sel] = is(0, unlink) && is(3, removeDir) && is(4, 0) && is(5, 0) && is(6, 0) + } + return true + }) + } + } + return allowed +} + func networkImport(path string) bool { // Keep parsing/value packages (net/url, net/netip, net/mail) available. return path == "net/http" || strings.HasPrefix(path, "net/http/") || @@ -331,6 +460,97 @@ func TestNetworkGuardLowLevelPolicy(t *testing.T) { } } +func TestNetworkGuardDatabaseBoundary(t *testing.T) { + pgconn := "github.com/jackc/pgx/v5/pgconn" + module, version := databaseRuntimeModule(pgconn) + approved := []networkPackage{ + {ImportPath: databaseMain, Imports: []string{databaseAdapter}}, + {ImportPath: databaseAdapter, Imports: []string{pgconn}}, + {ImportPath: pgconn, Module: &networkModule{Path: module, Version: version}}, + } + if got := databaseBoundaryFindings(approved); len(got) != 0 { + t.Fatalf("reviewed database boundary rejected: %v", got) + } + for _, p := range []networkPackage{ + {ImportPath: "github.com/llm-measurement/fleetdiff/internal/cli", Imports: []string{databaseAdapter}}, + {ImportPath: "github.com/llm-measurement/fleetdiff/internal/spend", Imports: []string{pgconn}}, + {ImportPath: databaseMain, Imports: []string{pgconn}}, + {ImportPath: "example.com/transitive", Imports: []string{pgconn}}, + {ImportPath: "github.com/jackc/pgx/v5/unreviewed"}, + {ImportPath: pgconn, Module: &networkModule{Path: module, Version: "v5.99.0"}}, + {ImportPath: pgconn, Module: &networkModule{Path: module, Version: version, Replace: &networkModule{Path: "./local"}}}, + } { + if got := databaseBoundaryFindings([]networkPackage{p}); len(got) == 0 { + t.Errorf("database boundary escape accepted: %+v", p) + } + } + for _, owner := range []string{databaseAdapter, pgconn, databaseMain, "github.com/llm-measurement/fleetdiff/internal/cli"} { + for _, api := range [][2]string{{"net", "DialContext"}, {"net", "LookupHost"}, {"net", "Listen"}, {"os", "StartProcess"}, {"syscall", "Socket"}, {"net/http", "Get"}} { + want := (owner == databaseAdapter || owner == pgconn) && api[0] == "net" && (api[1] == "DialContext" || api[1] == "LookupHost") + if got := databaseNetworkFunction(owner, api[0], api[1]); got != want { + t.Errorf("database API exception %s %v = %v, want %v", owner, api, got, want) + } + } + } +} + +func TestNetworkGuardFilesystemSyscallExceptions(t *testing.T) { + linuxRename, linuxUnlink := 316, 263 + if runtime.GOARCH == "arm64" { + linuxRename, linuxUnlink = 276, 35 + } + for _, tc := range []struct { + name, file, function, declarations, body string + want bool + }{ + {"darwin_rename", "rename_darwin.go", "renameDirectoryNoReplace", "", "syscall.Syscall6(488, 0, 0, 0, 0, 4, 0)", true}, + {"darwin_rmdir", "rename_darwin.go", "removeEmptyDirectoryAt", "", "syscall.Syscall6(472, 0, 0, 128, 0, 0, 0)", true}, + {"linux_rename", "rename_linux.go", "renameDirectoryNoReplace", fmt.Sprintf("const operation = %d", linuxRename), "syscall.Syscall6(operation, 0, 0, 0, 0, 1, 0)", true}, + {"linux_rmdir", "rename_linux.go", "removeEmptyDirectoryAt", "", fmt.Sprintf("syscall.Syscall6(%d, 0, 0, 512, 0, 0, 0)", linuxUnlink), true}, + {"wrong_file", "files.go", "renameDirectoryNoReplace", "", "syscall.Syscall6(488, 0, 0, 0, 0, 4, 0)", false}, + {"wrong_function", "rename_darwin.go", "other", "", "syscall.Syscall6(488, 0, 0, 0, 0, 4, 0)", false}, + {"variable_operation", "rename_darwin.go", "renameDirectoryNoReplace", "var operation uintptr = 488", "syscall.Syscall6(operation, 0, 0, 0, 0, 4, 0)", false}, + {"variable_flags", "rename_darwin.go", "renameDirectoryNoReplace", "var flags uintptr = 4", "syscall.Syscall6(488, 0, 0, 0, 0, flags, 0)", false}, + {"overwriting_rename", "rename_darwin.go", "renameDirectoryNoReplace", "", "syscall.Syscall6(488, 0, 0, 0, 0, 0, 0)", false}, + {"unlink_file", "rename_darwin.go", "removeEmptyDirectoryAt", "", "syscall.Syscall6(472, 0, 0, 0, 0, 0, 0)", false}, + {"saved_function", "rename_darwin.go", "renameDirectoryNoReplace", "var saved = syscall.Syscall6", "saved(488, 0, 0, 0, 0, 4, 0)", false}, + {"raw_syscall", "rename_darwin.go", "renameDirectoryNoReplace", "", "syscall.RawSyscall6(488, 0, 0, 0, 0, 4, 0)", false}, + {"socket", "rename_darwin.go", "renameDirectoryNoReplace", "", "syscall.Syscall6(syscall.SYS_SOCKET, 0, 0, 0, 0, 4, 0)", false}, + {"process", "rename_linux.go", "renameDirectoryNoReplace", "", "syscall.Syscall6(syscall.SYS_EXECVE, 0, 0, 0, 0, 1, 0)", false}, + } { + t.Run(tc.name, func(t *testing.T) { + fset := token.NewFileSet() + source := "package postgres\nimport \"syscall\"\n" + tc.declarations + "\nfunc " + tc.function + "() { " + tc.body + " }" + file, err := parser.ParseFile(fset, tc.file, source, 0) + if err != nil { + t.Fatal(err) + } + info := &types.Info{Uses: make(map[*ast.Ident]types.Object), Types: make(map[ast.Expr]types.TypeAndValue)} + config := types.Config{Importer: importer.Default()} + if _, err := config.Check(databaseAdapter, fset, []*ast.File{file}, info); err != nil { + t.Fatal(err) + } + for _, owner := range []string{databaseAdapter, "offline/package"} { + allowed := reviewedFilesystemSyscalls(owner, []*ast.File{file}, fset, info) + permitted, denied := 0, 0 + for id, obj := range info.Uses { + if obj.Pkg() != nil && obj.Pkg().Path() == "syscall" && networkFunction(obj.Pkg().Path(), obj.Name()) { + if allowed[id] { + permitted++ + } else { + denied++ + } + } + } + want := tc.want && owner == databaseAdapter + if want && (permitted != 1 || denied != 0) || !want && (permitted != 0 || denied == 0) { + t.Fatalf("owner=%s permitted=%d denied=%d want exception=%t", owner, permitted, denied, want) + } + } + }) + } +} + func writeNetworkFixture(t *testing.T, root, name, source string) { t.Helper() path := filepath.Join(root, filepath.FromSlash(name)) diff --git a/internal/cli/run.go b/internal/cli/run.go index 8134f1f..a067a6e 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -6,6 +6,7 @@ package cli import ( "bytes" + "context" "encoding/json" "errors" "flag" @@ -15,11 +16,16 @@ import ( "os" "runtime" "runtime/debug" + "slices" "strconv" "strings" "time" + "unicode/utf8" + "github.com/llm-measurement/fleetdiff/internal/accounting" + "github.com/llm-measurement/fleetdiff/internal/attribution" "github.com/llm-measurement/fleetdiff/internal/compare" + "github.com/llm-measurement/fleetdiff/internal/exportdata" "github.com/llm-measurement/fleetdiff/internal/spend" ) @@ -38,6 +44,9 @@ const help = `fleetdiff shows what changed between two windows of agent measurem Local and read-only. Usage: + fleetdiff export litellm --out NEW_DIRECTORY [options] + fleetdiff investigate --litellm-db [--daily] [--routing-audit] [options] + fleetdiff what-if --litellm-db --from MODEL --to MODEL --fraction 0.5 [options] fleetdiff inspect [options] CAPTURE fleetdiff diagnose [options] CONFIG fleetdiff scan [options] PATH --expected PRODUCER[,PRODUCER] @@ -61,6 +70,12 @@ Options: --allow-partial Permit missing producers or incomplete observation intervals --show-hashes Include pseudonymous, linkable hashes in output --litellm-spend FILE Read a request-level SQL export: CSV, JSON array or JSONL + --litellm-db Export a private database snapshot, then analyze locally + --database-url-file FILE Read database credentials from an owner-only regular file + --keep-export NEW_DIRECTORY Keep the private export; otherwise remove it after analysis + --daily Export daily counters for database row coverage + --routing-details Include routing details in the database export + --usage-details=false Omit default-on cache and usage details from the export --litellm-daily-spend FILE Compare row counts with the daily SQL-projection CSV --before-period PERIOD UTC date or RFC3339 start/end (requires after-period) --after-period PERIOD UTC date or RFC3339 start/end (equal duration) @@ -79,7 +94,9 @@ Options: inspect reads local OTLP captures; run fleetdiff inspect --help for capture options. diagnose checks supported collector configuration; scan checks retained summary history. -Runs offline using local files. Default output uses aliases for identities. +File commands run offline. Only explicit database commands connect to Postgres. +Database periods default to the last 14 complete UTC days; export --help has details. +Default output uses aliases for identities. Spend periods default to the latest two complete 7-day UTC intervals in the file. With fewer than 14 complete days, supply both period flags. Daily UI aggregates need a request-level SQL export; see examples/litellm-spend/README.md. @@ -93,9 +110,18 @@ func Run(args []string, out, errout io.Writer) int { // RunWithInput makes stdin explicit for callers embedding the command. func RunWithInput(args []string, in io.Reader, out, errout io.Writer) int { + return RunWithExporter(context.Background(), args, in, out, errout, nil) +} + +// RunWithExporter injects the only database capability. Run and RunWithInput +// remain offline, including when their callers supply database options. +func RunWithExporter(ctx context.Context, args []string, in io.Reader, out, errout io.Writer, exporter exportdata.Exporter) int { fail := func(code int, message string) int { fmt.Fprintln(errout, message); return code } + if len(args) > 0 && args[0] == "export" { + return runExport(ctx, args[1:], out, errout, exporter) + } if len(args) > 0 && args[0] == "what-if" { - return runWhatIf(args[1:], out, errout) + return runWhatIf(ctx, args[1:], out, errout, exporter) } if len(args) > 0 && args[0] == "inspect" { return runInspect(args[1:], in, out, errout) @@ -124,6 +150,8 @@ func RunWithInput(args []string, in io.Reader, out, errout io.Writer) int { } flags := flag.NewFlagSet("compare", flag.ContinueOnError) flags.SetOutput(io.Discard) + var database databaseFlags + database.register(flags) before := flags.String("before", "", "") after := flags.String("after", "", "") expected := flags.String("expected", "", "") @@ -176,13 +204,20 @@ func RunWithInput(args []string, in io.Reader, out, errout io.Writer) int { } return fail(2, "invalid compare options; run fleetdiff compare --help") } - spendMode, incompatible := false, false + if database.enabled && databaseURLArgument(args) { + return fail(2, databaseURLMessage) + } + if err := database.validate(flags); err != nil { + return fail(2, err.Error()) + } + spendMode, incompatible := database.enabled, false sessionMode, windowSet, emptySessionOption, twoPeriods := false, false, false, false groupExplicit := false emptyDailyOption := false routingSet, routingBySet := false, false flags.Visit(func(f *flag.Flag) { switch f.Name { + case "litellm-db", "keep-export", "database-url-file", "daily", "routing-details", "usage-details": case "routing-audit", "routing-by", "scope-route", "scope-agent", "show-labels": spendMode, routingSet = true, true routingBySet = routingBySet || f.Name == "routing-by" @@ -210,7 +245,7 @@ func RunWithInput(args []string, in io.Reader, out, errout io.Writer) int { if emptyDailyOption { return fail(2, "litellm-daily-spend needs a daily SQL-projection CSV file") } - if args[0] != "investigate" || *spendPath == "" || incompatible || flags.NArg() != 0 { + if args[0] != "investigate" || *spendPath == "" && !database.enabled || incompatible || flags.NArg() != 0 { return fail(2, "use investigate --litellm-spend FILE with period flags, not summary paths or expected producers") } if routingSet { @@ -223,11 +258,11 @@ func RunWithInput(args []string, in io.Reader, out, errout io.Writer) int { if *showLabels && !*routingAudit { return fail(2, "show-labels requires routing-audit") } - if routeSet && agentSet || routeSet && (strings.TrimSpace(scopeRoute) == "" || len(scopeRoute) > spend.MaxFieldBytes) || agentSet && (strings.TrimSpace(scopeAgent) == "" || len(scopeAgent) > spend.MaxFieldBytes) { + if routeSet && agentSet || routeSet && (strings.TrimSpace(scopeRoute) == "" || len(scopeRoute) > spend.MaxFieldBytes || database.enabled && !utf8.ValidString(scopeRoute)) || agentSet && (strings.TrimSpace(scopeAgent) == "" || len(scopeAgent) > spend.MaxFieldBytes || database.enabled && !utf8.ValidString(scopeAgent)) { return fail(2, "choose one nonempty route or agent scope of at most 8192 bytes") } if routeSet || agentSet { - if *beforePeriod == "" || *afterPeriod == "" || *dailySpendPath != "" || *group == "session" { + if *beforePeriod == "" || *afterPeriod == "" || *dailySpendPath != "" || database.daily || *group == "session" { return fail(2, "route or agent scopes require both explicit periods; do not combine with daily coverage or session grouping") } } @@ -236,6 +271,49 @@ func RunWithInput(args []string, in io.Reader, out, errout io.Writer) int { if !sessionMode || groupExplicit && *group != "session" || twoPeriods || emptySessionOption || *counterWindow <= 0 { return fail(2, "session review needs --period or --session-budget-usd; grouping is session; do not combine with other groups or two-period flags") } + } + if database.enabled { + if *format != "text" && *format != "json" || *top < 1 || *top > 100 || !slices.Contains([]string{"key", "team", "user", "end-user", "session"}, *group) { + return fail(2, "use text or json output, top between 1 and 100, and a supported spend group") + } + key, _, err := attribution.Secret(*secret) + clear(key) + if err != nil { + return fail(2, err.Error()) + } + var p spend.Period + if sessionMode || windowSet { + if *counterWindow < time.Second || *counterWindow > 365*24*time.Hour { + return fail(2, "session-counter-window must be between 1s and 8760h") + } + if *budget != "" { + cap, ok := accounting.Cost(*budget) + if !ok || cap.Sign() <= 0 { + return fail(2, "session-budget-usd must be a positive decimal no greater than 1e18") + } + } + p, err = databasePeriod(*period, time.Now()) + *period = periodText(p) + } else { + if twoPeriods && (*beforePeriod == "" || *afterPeriod == "") { + return fail(2, "supply both nonempty before-period and after-period") + } + p, *beforePeriod, *afterPeriod, err = databaseInvestigationPeriods(*beforePeriod, *afterPeriod, time.Now()) + } + if err != nil { + return fail(2, err.Error()) + } + if err := validateDatabaseDaily(p, database.daily); err != nil { + return fail(2, err.Error()) + } + r, cleanup, err := database.export(ctx, p, exporter) + defer cleanup() + if err != nil { + return fail(1, err.Error()) + } + *spendPath, *dailySpendPath = r.SpendPath, r.DailyPath + } + if sessionMode || windowSet { return runSessionBudget(*spendPath, *period, *budget, *secret, *format, *dailySpendPath, *top, *hashes, *counterWindow, out, errout) } return runSpend(*spendPath, *format, spend.Options{ diff --git a/internal/cli/whatif.go b/internal/cli/whatif.go index 15bc3ed..03116f4 100644 --- a/internal/cli/whatif.go +++ b/internal/cli/whatif.go @@ -5,6 +5,7 @@ package cli import ( "bytes" + "context" "encoding/json" "errors" "flag" @@ -12,8 +13,10 @@ import ( "io" "math/big" "strings" + "time" "github.com/llm-measurement/fleetdiff/internal/compare" + "github.com/llm-measurement/fleetdiff/internal/exportdata" "github.com/llm-measurement/fleetdiff/internal/reprice" "github.com/llm-measurement/fleetdiff/internal/spend" ) @@ -21,11 +24,18 @@ import ( const whatIfHelp = `Re-price a period of covered usage under explicit model-mix assumptions. Usage: + fleetdiff what-if --litellm-db --from MODEL --to MODEL --fraction 0.5 [--period PERIOD] fleetdiff what-if --litellm-spend FILE --period PERIOD --from MODEL --to MODEL --fraction 0.5 fleetdiff what-if --litellm-spend FILE --period PERIOD --scenario scenario.json fleetdiff what-if --summaries PATH --expected PRODUCER[,PRODUCER] --period PERIOD --scenario scenario.json Options: + --litellm-db Export a private database snapshot, then analyze locally + --database-url-file FILE Read database credentials from an owner-only regular file + --keep-export NEW_DIRECTORY Keep the private export; otherwise remove it after analysis + --daily Include daily counters in the private export + --routing-details Include routing details (always included in database mode) + --usage-details=false Omit default-on cache and usage details from the export --scope-key ID Restrict a simple move to a recorded API key --scope-team ID Restrict a simple move to a recorded team --scope-route ID Restrict a simple move to its recorded original route @@ -38,13 +48,17 @@ Options: PERIOD is a UTC date or RFC3339 start/end. Fractions are uniform shares of original covered token volume. Raw identities are never printed. Put selectors in a protected scenario file to avoid shell history. See docs/WHAT_IF.md. -No network requests are made. Missing model or token splits remain unknown. +File commands run offline. Database mode defaults to the last 14 complete UTC +days; --period selects the database range. --from and --to always name models. +Missing model or token splits remain unknown. ` -func runWhatIf(args []string, out, errout io.Writer) int { +func runWhatIf(ctx context.Context, args []string, out, errout io.Writer, exporter exportdata.Exporter) int { fail := func(code int, s string) int { fmt.Fprintln(errout, s); return code } f := flag.NewFlagSet("what-if", flag.ContinueOnError) f.SetOutput(io.Discard) + var database databaseFlags + database.register(f) spendFile := f.String("litellm-spend", "", "") summaryFile := f.String("summaries", "", "") expected := f.String("expected", "", "") @@ -76,7 +90,13 @@ func runWhatIf(args []string, out, errout io.Writer) int { } return fail(2, "invalid what-if options; run fleetdiff what-if --help") } - if f.NArg() != 0 || (*spendFile == "") == (*summaryFile == "") || *period == "" || (*format != "text" && *format != "json") || *spendFile != "" && *expected != "" || *summaryFile != "" && *expected == "" { + if database.enabled && databaseURLArgument(args) { + return fail(2, databaseURLMessage) + } + if err := database.validate(f); err != nil { + return fail(2, err.Error()) + } + if f.NArg() != 0 || !database.enabled && ((*spendFile == "") == (*summaryFile == "") || *period == "") || (*format != "text" && *format != "json") || *spendFile != "" && *expected != "" || *summaryFile != "" && *expected == "" { return fail(2, "choose one input, a period, and text or json; summaries also require expected producers") } if *spendFile != "" && *modelContract != "" { @@ -135,6 +155,26 @@ func runWhatIf(args []string, out, errout io.Writer) int { return fail(2, err.Error()) } defer c.Close() + if database.enabled { + emptyPeriod := false + f.Visit(func(v *flag.Flag) { emptyPeriod = emptyPeriod || v.Name == "period" && v.Value.String() == "" }) + if emptyPeriod { + return fail(2, "database period must be nonempty when supplied") + } + p, err := databasePeriod(*period, time.Now()) + if err != nil { + return fail(2, err.Error()) + } + if err := validateDatabaseDaily(p, database.daily); err != nil { + return fail(2, err.Error()) + } + r, cleanup, err := database.export(ctx, p, exporter) + defer cleanup() + if err != nil { + return fail(1, err.Error()) + } + *spendFile, *period = r.SpendPath, periodText(p) + } var source reprice.Source if *spendFile != "" { read := spend.ReadReprice diff --git a/internal/exportdata/export.go b/internal/exportdata/export.go new file mode 100644 index 0000000..413b146 --- /dev/null +++ b/internal/exportdata/export.go @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +// Package exportdata defines the boundary between offline reports and an explicit export. +package exportdata + +import ( + "context" + "time" +) + +type Options struct { + Directory, CredentialFile string + Start, End time.Time + Format string + Daily, Routing, UsageDetails bool +} + +type Result struct { + Directory, SpendPath, DailyPath string + Start, End time.Time + Rows int64 +} + +type Exporter func(context.Context, Options) (Result, error) + +// Failure has no untrusted text fields. The CLI may render these fixed messages, +// while suppressing errors from arbitrary exporter implementations. +type Failure uint8 + +const ( + ErrCredentials Failure = iota + 1 + ErrConfiguration + ErrTransport + ErrOutput + ErrSchema + ErrLimit + ErrInterrupted + ErrDatabase + ErrStorage +) + +func (e Failure) Error() string { + switch e { + case ErrCredentials: + return "provide one database URL through FLEETDIFF_LITELLM_DATABASE_URL or a regular, owned mode-0600 database-url-file" + case ErrConfiguration: + return "invalid database URL; specify one host, database and user, with only sslmode and optional sslrootcert (or host for a Unix socket)" + case ErrTransport: + return "remote databases require sslmode=verify-full; only numeric loopback or Unix sockets permit sslmode=disable" + case ErrOutput: + return "export needs a new output directory under an existing directory, without symlink components" + case ErrSchema: + return "database schema does not match the LiteLLM v1.104.0 export profile; verify the pinned schema or use the psql recipe" + case ErrLimit: + return "export exceeds the request, file, row or field limit; select a shorter period" + case ErrInterrupted: + return "database export interrupted or timed out; no complete export was published" + case ErrStorage: + return "cannot finish private export files; check available disk space and output access" + default: + return "database export failed; check read access, connection settings and the LiteLLM v1.104.0 export profile" + } +} diff --git a/internal/postgres/config.go b/internal/postgres/config.go new file mode 100644 index 0000000..1b5619b --- /dev/null +++ b/internal/postgres/config.go @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +// Package postgres is the explicit network boundary for read-only LiteLLM exports. +package postgres + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "io" + "net" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/llm-measurement/fleetdiff/internal/exportdata" + "github.com/llm-measurement/fleetdiff/internal/localfile" +) + +const credentialEnv = "FLEETDIFF_LITELLM_DATABASE_URL" + +var errCredentials = exportdata.ErrCredentials +var errURL = exportdata.ErrConfiguration +var configMu sync.Mutex + +func credentials(path string) (string, error) { + value, set := os.LookupEnv(credentialEnv) + if path != "" { + if set { + return "", errCredentials + } + root, name, err := openParent(path) + if err != nil { + return "", errCredentials + } + defer root.Close() + data, err := readCredentials(root, name) + if err != nil { + return "", errCredentials + } + value = strings.TrimSuffix(strings.TrimSuffix(string(data), "\n"), "\r") + } + if len(value) == 0 || len(value) > 64<<10 { + return "", errCredentials + } + return value, nil +} + +func readCredentials(root *os.Root, name string) ([]byte, error) { + f, err := root.OpenFile(name, os.O_RDONLY|syscall.O_NONBLOCK|syscall.O_NOFOLLOW, 0) + if err != nil { + return nil, errCredentials + } + defer f.Close() + return readCredentialFile(f) +} + +func readCredentialFile(f *os.File) ([]byte, error) { + info, err := f.Stat() + if err != nil || !owned(info) || info.Mode() != 0600 || info.Size() > 64<<10 { + return nil, errCredentials + } + data, err := io.ReadAll(io.LimitReader(f, (64<<10)+1)) + if err != nil || len(data) > 64<<10 { + return nil, errCredentials + } + after, err := f.Stat() + if err != nil || !owned(after) || after.Mode() != 0600 || after.Size() != info.Size() || !after.ModTime().Equal(info.ModTime()) { + return nil, errCredentials + } + return data, nil +} + +func connectionConfig(raw string) (*pgconn.Config, error) { + u, err := url.Parse(raw) + if err != nil || u.Scheme != "postgres" && u.Scheme != "postgresql" || u.Opaque != "" || u.Fragment != "" || u.User == nil || u.User.Username() == "" { + return nil, errURL + } + q, err := url.ParseQuery(u.RawQuery) + if err != nil { + return nil, errURL + } + for key, values := range q { + if len(values) != 1 || key != "sslmode" && key != "sslrootcert" && key != "host" { + return nil, errURL + } + } + host := u.Hostname() + if socket := q.Get("host"); socket != "" { + if host != "" || u.Port() != "" || !filepath.IsAbs(socket) || filepath.Clean(socket) != socket { + return nil, errURL + } + host = socket + } + db := strings.TrimPrefix(u.Path, "/") + password, _ := u.User.Password() + for _, field := range []string{host, db, u.User.Username(), password} { + if strings.IndexFunc(field, func(r rune) bool { return r == 0 || r == '\r' || r == '\n' }) >= 0 { + return nil, errURL + } + } + if host == "" || db == "" || strings.ContainsAny(db, "/\\") || strings.Contains(host, ",") || strings.Contains(host, "%") { + return nil, errURL + } + port := uint64(5432) + if u.Port() != "" { + port, err = strconv.ParseUint(u.Port(), 10, 16) + if err != nil || port == 0 { + return nil, errURL + } + } + local := filepath.IsAbs(host) || net.ParseIP(host) != nil && net.ParseIP(host).IsLoopback() + mode := q.Get("sslmode") + if mode != "verify-full" && (!local || mode != "disable") { + return nil, exportdata.ErrTransport + } + if filepath.IsAbs(host) && mode != "disable" { + return nil, errURL + } + c, err := neutralConfig() + if err != nil { + return nil, errors.New("cannot initialize database configuration") + } + c.Host, c.Port, c.Database, c.User, c.Password = host, uint16(port), db, u.User.Username(), password + c.Fallbacks = nil + c.ConnectTimeout = 10 * time.Second + c.MaxProtocolMessageBodyLen = (64 << 10) + 256 + c.RuntimeParams = map[string]string{"application_name": "fleetdiff", "client_encoding": "UTF8", "default_transaction_read_only": "on"} + if mode == "verify-full" { + c.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS12, ServerName: host} + if ca := q.Get("sslrootcert"); ca != "" { + root, name, err := openParent(ca) + if err != nil { + return nil, errors.New("cannot read configured database CA file") + } + pem, err := localfile.Read(root, name, 1<<20) + root.Close() + pool := x509.NewCertPool() + if err != nil || !pool.AppendCertsFromPEM(pem) { + return nil, errors.New("invalid database CA file") + } + c.TLSConfig.RootCAs = pool + } + } else if q.Get("sslrootcert") != "" { + return nil, errURL + } + return c, nil +} + +// pgconn requires its parser's initialized Config. Parse a constant template, +// not the supplied URL, with all inherited PG* settings temporarily removed. +// Explicit empty certificate paths and a nonempty placeholder password suppress +// home-file defaults. Replace every destination/credential before connecting. +func neutralConfig() (*pgconn.Config, error) { + configMu.Lock() + defer configMu.Unlock() + previous := map[string]string{} + defer func() { + for key, value := range previous { + _ = os.Setenv(key, value) + } + }() + for _, entry := range os.Environ() { + key, value, _ := strings.Cut(entry, "=") + if strings.HasPrefix(key, "PG") { + previous[key] = value + if err := os.Unsetenv(key); err != nil { + return nil, errors.New("cannot isolate database environment") + } + } + } + return pgconn.ParseConfig("host=127.0.0.1 port=5432 dbname=fleetdiff user=fleetdiff password=no-inherited-credentials sslmode=disable sslrootcert='' sslcert='' sslkey='' sslpassword='' connect_timeout=10") +} diff --git a/internal/postgres/config_test.go b/internal/postgres/config_test.go new file mode 100644 index 0000000..03ace33 --- /dev/null +++ b/internal/postgres/config_test.go @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package postgres + +import ( + "context" + "os" + osexec "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/llm-measurement/fleetdiff/internal/exportdata" +) + +const secretURL = "postgres://SENTINEL_USER:SENTINEL_PASSWORD@SENTINEL_HOST/fixture?sslmode=verify-full" + +func TestConnectionProfile(t *testing.T) { + for _, raw := range []string{ + secretURL, + "postgres://user@127.0.0.1:5433/fixture?sslmode=disable", + "postgres://user@[::1]/fixture?sslmode=disable", + "postgres://user@/fixture?host=/var/run/postgresql&sslmode=disable", + } { + c, err := connectionConfig(raw) + if err != nil { + t.Fatalf("accepted profile rejected: %v", err) + } + if len(c.Fallbacks) != 0 || c.RuntimeParams["default_transaction_read_only"] != "on" || c.MaxProtocolMessageBodyLen == 0 { + t.Fatal("unsafe connection defaults") + } + if strings.Contains(raw, "verify-full") && (c.TLSConfig == nil || c.TLSConfig.InsecureSkipVerify || c.TLSConfig.ServerName != c.Host) { + t.Fatal("remote host is not verified") + } + } + for _, raw := range []string{ + "postgres://user:secret@remote/fixture", "postgres://user:secret@remote/fixture?sslmode=disable", + "postgres://user:secret@remote/fixture?sslmode=verify-ca", "postgres://user:secret@localhost/fixture?sslmode=disable", + "postgres://user:secret@remote/fixture?sslmode=prefer", "postgres://user@one,two/fixture?sslmode=verify-full", + "postgres://user@remote/fixture?sslmode=verify-full&sslmode=disable", "postgres://user@remote/fixture?ssl=true", + "postgres://user@remote/fixture?sslmode=verify-full&host=127.0.0.1", "postgres://user@remote/fixture?sslmode=verify-full&options=-c%20role=postgres", + "postgres://user@remote/fixture?sslmode=verify-full&service=evil", "postgres://user@remote/fixture?sslmode=verify-full&passfile=/tmp/secret", + "postgres://user@remote/fixture?sslmode=verify-full&servicefile=/tmp/secret", "postgres://user@remote/fixture?sslmode=verify-full&sslkey=/tmp/secret", + "postgres://user@remote/fixture?sslmode=verify-full&%73slmode=disable", "postgres://user@remote:0/fixture?sslmode=verify-full", + "postgres://user@remote:5432,5433/fixture?sslmode=verify-full", "postgres://user@remote/fixture#secret", + "postgres://user%00other@remote/fixture?sslmode=verify-full", "postgres://user@remote/fi%0axture?sslmode=verify-full", + "host=remote password=SENTINEL_PASSWORD", "postgres://user@/fixture?host=relative&sslmode=disable", + "postgres:///fixture?sslmode=verify-full", "postgres://user@remote/?sslmode=verify-full", + } { + if _, err := connectionConfig(raw); err == nil { + t.Fatal("unsafe URL accepted") + } else if strings.Contains(err.Error(), "SENTINEL") || strings.Contains(err.Error(), "secret") { + t.Fatal("unsafe diagnostic") + } + } +} + +func TestInheritedPostgresEnvironment(t *testing.T) { + for _, key := range []string{"PGHOST", "PGHOSTADDR", "PGPORT", "PGDATABASE", "PGUSER", "PGPASSWORD", "PGPASSFILE", "PGSERVICE", "PGSERVICEFILE", "PGSSLMODE", "PGSSLCERT", "PGSSLKEY", "PGSSLROOTCERT", "PGSSLPASSWORD", "PGOPTIONS", "PGAPPNAME", "PGCONNECT_TIMEOUT", "PGTARGETSESSIONATTRS", "PGTZ", "PGMINPROTOCOLVERSION", "PGMAXPROTOCOLVERSION", "PGCHANNELBINDING", "PGREQUIREAUTH", "PGFUTUREOPTION"} { + t.Run(key, func(t *testing.T) { + t.Setenv(key, "SENTINEL_INHERITED_SETTING") + c, err := connectionConfig("postgres://explicit@127.0.0.1/fixture?sslmode=disable") + if err != nil { + t.Fatal(err) + } + if c.Host != "127.0.0.1" || c.User != "explicit" || c.Database != "fixture" || c.Password != "" || c.Port != 5432 || c.TLSConfig != nil || len(c.RuntimeParams) != 3 || len(c.Fallbacks) != 0 { + t.Fatal("inherited configuration escaped isolation") + } + if os.Getenv(key) != "SENTINEL_INHERITED_SETTING" { + t.Fatal("caller environment not restored") + } + }) + } +} + +func TestHomeCredentialAndServiceFilesIgnored(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + for _, item := range []struct{ name, data string }{ + {".pgpass", "127.0.0.1:5432:fixture:explicit:SENTINEL_HOME_PASSWORD\n"}, + {".pg_service.conf", "[evil]\nhost=SENTINEL_HOME_HOST\npassword=SENTINEL_HOME_PASSWORD\nsslmode=disable\n"}, + } { + if err := os.WriteFile(filepath.Join(home, item.name), []byte(item.data), 0600); err != nil { + t.Fatal(err) + } + } + t.Setenv("PGSERVICE", "evil") + c, err := connectionConfig("postgres://explicit@127.0.0.1/fixture?sslmode=disable") + if err != nil || c.Password != "" || c.Host != "127.0.0.1" { + t.Fatal("home files affected explicit credentials") + } +} + +func TestHomeCertificateFilesIgnored(t *testing.T) { + if os.Getenv("FLEETDIFF_TEST_HOME_CERT") == "child" { + if _, err := neutralConfig(); err != nil { + t.Fatal(err) + } + return + } + for _, fifo := range []bool{false, true} { + home := t.TempDir() + dir := filepath.Join(home, ".postgresql") + if err := os.Mkdir(dir, 0700); err != nil { + t.Fatal(err) + } + for _, name := range []string{"root.crt", "postgresql.crt", "postgresql.key"} { + path := filepath.Join(dir, name) + var err error + if fifo { + err = syscall.Mkfifo(path, 0600) + } else { + err = os.WriteFile(path, []byte("SENTINEL_INVALID_HOME_CERTIFICATE"), 0600) + } + if err != nil { + t.Fatal(err) + } + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + cmd := osexec.CommandContext(ctx, os.Args[0], "-test.run=^TestHomeCertificateFilesIgnored$") + cmd.Env = append(os.Environ(), "HOME="+home, "FLEETDIFF_TEST_HOME_CERT=child") + out, err := cmd.CombinedOutput() + cancel() + if err != nil { + t.Fatalf("home certificates affected isolated parser: %v %s", err, out) + } + } +} + +func TestCredentialChecksFollowReadDescriptor(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "credentials") + if err := os.WriteFile(path, []byte(secretURL), 0644); err != nil { + t.Fatal(err) + } + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + if err = os.Rename(path, path+".old"); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(path, []byte(secretURL), 0600); err != nil { + t.Fatal(err) + } + if _, err := readCredentialFile(f); err == nil { + t.Fatal("permissions checked on replacement path instead of read descriptor") + } +} + +func TestCredentialFiles(t *testing.T) { + t.Setenv(credentialEnv, "") + if err := os.Unsetenv(credentialEnv); err != nil { + t.Fatal(err) + } + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "credentials") + if err := os.WriteFile(path, []byte(secretURL+"\n"), 0600); err != nil { + t.Fatal(err) + } + if got, err := credentials(path); err != nil || got != secretURL { + t.Fatal("owner-only credential file rejected") + } + link := filepath.Join(dir, "link") + if err := os.Symlink(path, link); err != nil { + t.Fatal(err) + } + if _, err := credentials(link); err == nil { + t.Fatal("symlink credentials accepted") + } + if err := os.Chmod(path, 0644); err != nil { + t.Fatal(err) + } + if _, err := credentials(path); err == nil { + t.Fatal("public credential file accepted") + } + if err := os.Chmod(path, 0600); err != nil { + t.Fatal(err) + } + t.Setenv(credentialEnv, secretURL) + if _, err := credentials(path); err == nil { + t.Fatal("ambiguous credential sources accepted") + } +} + +func TestExportErrorPrivacy(t *testing.T) { + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + t.Setenv(credentialEnv, "postgres://SENTINEL_USER:SENTINEL_PASSWORD@127.0.0.1:1/fixture?sslmode=disable") + o := exportdata.Options{Directory: filepath.Join(root, "SENTINEL_PATH"), Start: time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC), End: time.Date(2026, 10, 9, 0, 0, 0, 0, time.UTC), Format: "csv", UsageDetails: true} + _, err = Export(context.Background(), o) + if err == nil || strings.Contains(err.Error(), "SENTINEL") || strings.Contains(err.Error(), "127.0.0.1") || strings.Contains(err.Error(), root) { + t.Fatal("database diagnostics were not safe") + } + if _, err := os.Stat(o.Directory); !os.IsNotExist(err) { + t.Fatal("failed export left an output directory") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err = Export(ctx, o); !strings.Contains(err.Error(), "interrupted") { + t.Fatal("cancellation not reported") + } +} + +func FuzzConnectionProfile(f *testing.F) { + for _, raw := range []string{secretURL, "postgres://u@127.0.0.1/d?sslmode=disable", "postgres://u@remote/d?sslmode=verify-full&service=evil", "postgres://u@remote/d?sslmode=verify-full&sslmode=disable"} { + f.Add(raw) + } + f.Fuzz(func(t *testing.T, raw string) { + if len(raw) > 65536 || strings.Contains(raw, "sslrootcert") || strings.Contains(raw, "%") { + return + } + c, err := connectionConfig(raw) + if err != nil { + if strings.Contains(err.Error(), "SENTINEL") || strings.Contains(err.Error(), raw) && len(raw) > 16 { + t.Fatal("connection input escaped into an error") + } + return + } + if len(c.Fallbacks) != 0 || c.RuntimeParams["default_transaction_read_only"] != "on" { + t.Fatal("unsafe config") + } + if c.TLSConfig != nil && (c.TLSConfig.InsecureSkipVerify || c.TLSConfig.ServerName != c.Host) { + t.Fatal("host verification lost") + } + }) +} diff --git a/internal/postgres/errors_test.go b/internal/postgres/errors_test.go new file mode 100644 index 0000000..eedb77b --- /dev/null +++ b/internal/postgres/errors_test.go @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package postgres + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgproto3" + "github.com/llm-measurement/fleetdiff/internal/exportdata" +) + +func TestDatabaseFailureClassifications(t *testing.T) { + for _, item := range []struct{ in, want error }{ + {&pgproto3.ExceededMaxBodyLenErr{MaxExpectedBodyLen: 10, ActualBodyLen: 11}, errLimit}, + {fmt.Errorf("SENTINEL: %w", errLimit), errLimit}, + {&pgconn.PgError{Code: "57014", Message: "SENTINEL statement"}, errInterrupted}, + {&pgconn.PgError{Code: "55P03", Detail: "SENTINEL relation"}, errInterrupted}, + {&pgconn.PgError{Code: "25P03", Message: "SENTINEL idle transaction"}, errInterrupted}, + {context.Canceled, errInterrupted}, + {context.DeadlineExceeded, errInterrupted}, + {&pgconn.PgError{Code: "42P01", Message: "SENTINEL table"}, errDatabase}, + {errors.New("SENTINEL connection string"), errDatabase}, + {fmt.Errorf("SENTINEL: %w", exportdata.ErrStorage), exportdata.ErrStorage}, + } { + if got := databaseFailure(item.in); got != item.want { + t.Errorf("got %v, want %v", got, item.want) + } + } +} + +type failedOutput struct{ short bool } + +func (w failedOutput) Write(p []byte) (int, error) { + if w.short { + return len(p) - 1, nil + } + return 0, errors.New("SENTINEL_OUTPUT_PATH") +} + +func TestOutputStorageFailureIsSafe(t *testing.T) { + for _, short := range []bool{false, true} { + w := byteLimit{w: failedOutput{short: short}, remaining: 100} + if _, err := w.Write([]byte("fixture")); err != exportdata.ErrStorage { + t.Fatal("storage failure was not classified safely") + } + } +} diff --git a/internal/postgres/export.go b/internal/postgres/export.go new file mode 100644 index 0000000..e7402ba --- /dev/null +++ b/internal/postgres/export.go @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package postgres + +import ( + "context" + "errors" + "io" + "path/filepath" + "strconv" + "time" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgproto3" + "github.com/llm-measurement/fleetdiff/internal/exportdata" + "github.com/llm-measurement/fleetdiff/internal/spend" +) + +var errDatabase = exportdata.ErrDatabase +var errLimit = exportdata.ErrLimit +var errInterrupted = exportdata.ErrInterrupted + +func Export(parent context.Context, o exportdata.Options) (result exportdata.Result, err error) { + if o.Format != "csv" && o.Format != "jsonl" { + return result, errors.New("export format must be csv or jsonl") + } + query, daily, err := queries(o) + if err != nil { + return result, err + } + if o.Daily && (!o.Start.Equal(o.Start.UTC().Truncate(24*time.Hour)) || !o.End.Equal(o.End.UTC().Truncate(24*time.Hour)) || o.End.Sub(o.Start) > 3660*24*time.Hour) { + return result, errors.New("daily export needs complete UTC days spanning at most 3,660 days") + } + raw, err := credentials(o.CredentialFile) + if err != nil { + return result, err + } + config, err := connectionConfig(raw) + raw = "" + if err != nil { + return result, err + } + dir, err := newOutput(o.Directory) + if err != nil { + return result, err + } + defer dir.close() + ctx, cancel := context.WithTimeout(parent, 5*time.Minute) + defer cancel() + defer func() { + if err != nil && ctx.Err() != nil { + err = errInterrupted + } + }() + conn, err := pgconn.ConnectConfig(ctx, config) + config.Password = "" + if err != nil { + return result, databaseFailure(err) + } + defer func() { + closeCtx, stop := context.WithTimeout(context.Background(), time.Second) + defer stop() + _ = conn.Close(closeCtx) + }() + if err = exec(ctx, conn, `BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY; +SET LOCAL TIME ZONE 'UTC'; SET LOCAL search_path = pg_catalog; +SET LOCAL statement_timeout = '120s'; SET LOCAL lock_timeout = '5s'; +SET LOCAL idle_in_transaction_session_timeout = '120s'; +SET LOCAL client_encoding = 'UTF8';`); err != nil { + return result, err + } + if err = checkSchema(ctx, conn, o.Daily); err != nil { + return result, err + } + name := "spend." + o.Format + file, err := dir.create(name) + if err != nil { + return result, err + } + // One extra row detects the cap; it is never published as a truncated export. + query += " LIMIT " + strconv.Itoa(spend.MaxRows+1) + w := &byteLimit{w: file, remaining: spend.MaxFileBytes} + var count int64 + if o.Format == "csv" { + var tag pgconn.CommandTag + tag, err = conn.CopyTo(ctx, w, "COPY ("+query+") TO STDOUT WITH (FORMAT CSV, HEADER true)") + count = tag.RowsAffected() + } else { + r := conn.ExecParams(ctx, "SELECT row_to_json(r) FROM ("+query+") AS r", nil, nil, nil, nil) + for r.NextRow() { + values := r.Values() + count++ + if count > spend.MaxRows || len(values) != 1 || len(values[0])+1 > spend.MaxRowBytes { + err = errLimit + break + } + if _, err = w.Write(values[0]); err != nil { + break + } + if _, err = w.Write([]byte{'\n'}); err != nil { + break + } + } + if err == nil { + _, err = r.Close() + } + } + if err != nil || count > spend.MaxRows { + file.Close() + if count > spend.MaxRows || errors.Is(err, errLimit) { + return result, errLimit + } + return result, databaseFailure(err) + } + if err = file.Sync(); err != nil { + file.Close() + return result, exportdata.ErrStorage + } + if err = file.Close(); err != nil { + return result, exportdata.ErrStorage + } + if o.Daily { + f, e := dir.create("daily.csv") + if e != nil { + return result, e + } + tag, e := conn.CopyTo(ctx, &byteLimit{w: f, remaining: 1 << 20}, daily) + if e != nil || tag.RowsAffected() > 3660 { + f.Close() + if tag.RowsAffected() > 3660 { + return result, errLimit + } + return result, databaseFailure(e) + } + if e = f.Sync(); e != nil { + f.Close() + return result, exportdata.ErrStorage + } + if e = f.Close(); e != nil { + return result, exportdata.ErrStorage + } + } + if err = exec(ctx, conn, "COMMIT"); err != nil { + return result, err + } + // Reuse the same bounded readers as every offline command before publication. + err = spend.ReadRoutingRoot(dir.root, name, func(spend.Row) error { return ctx.Err() }) + if err != nil { + return result, errLimit + } + if o.Daily { + if err = spend.ValidateDailyExportRoot(dir.root, "daily.csv"); err != nil { + return result, errors.New("daily export is outside the supported coverage format or limits") + } + } + if err = dir.publish(ctx); err != nil { + return result, err + } + result = exportdata.Result{Directory: o.Directory, SpendPath: filepath.Join(o.Directory, name), Start: o.Start, End: o.End, Rows: count} + if o.Daily { + result.DailyPath = filepath.Join(o.Directory, "daily.csv") + } + return result, nil +} + +type byteLimit struct { + w io.Writer + remaining int64 +} + +func (w *byteLimit) Write(p []byte) (int, error) { + if int64(len(p)) > w.remaining { + return 0, errLimit + } + n, err := w.w.Write(p) + w.remaining -= int64(n) + if err != nil || n != len(p) { + return n, exportdata.ErrStorage + } + return n, err +} + +func exec(ctx context.Context, c *pgconn.PgConn, sql string) error { + if _, err := c.Exec(ctx, sql).ReadAll(); err != nil { + return databaseFailure(err) + } + return nil +} + +func databaseFailure(err error) error { + if errors.Is(err, exportdata.ErrStorage) { + return exportdata.ErrStorage + } + var oversized *pgproto3.ExceededMaxBodyLenErr + if errors.Is(err, errLimit) || errors.As(err, &oversized) { + return errLimit + } + var server *pgconn.PgError + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) || errors.As(err, &server) && (server.Code == "57014" || server.Code == "55P03" || server.Code == "25P03") { + return errInterrupted + } + return errDatabase +} diff --git a/internal/postgres/files.go b/internal/postgres/files.go new file mode 100644 index 0000000..95c7fc6 --- /dev/null +++ b/internal/postgres/files.go @@ -0,0 +1,267 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package postgres + +import ( + "context" + "crypto/rand" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/llm-measurement/fleetdiff/internal/exportdata" +) + +var errOutput = exportdata.ErrOutput + +func owned(info os.FileInfo) bool { + stat, ok := info.Sys().(*syscall.Stat_t) + return ok && stat.Uid == uint32(os.Geteuid()) +} + +// Walk from an open root, checking each component and its opened identity. +func openParent(path string) (*os.Root, string, error) { + return walkParent(path, nil) +} + +func openOutputParent(path string) (*os.Root, string, error) { + for _, part := range strings.Split(path, string(filepath.Separator)) { + if part == ".." { + return nil, "", errOutput + } + } + return walkParent(path, trustedOutputParent) +} + +func trustedOutputParent(info os.FileInfo) bool { + stat, ok := info.Sys().(*syscall.Stat_t) + return ok && info.IsDir() && (stat.Uid == 0 || stat.Uid == uint32(os.Geteuid())) && + (info.Mode().Perm()&0022 == 0 || info.Mode()&os.ModeSticky != 0) +} + +func walkParent(path string, trusted func(os.FileInfo) bool) (*os.Root, string, error) { + if path == "" { + return nil, "", errOutput + } + abs, err := filepath.Abs(path) + if err != nil || abs == "/" { + return nil, "", errOutput + } + root, err := os.OpenRoot("/") + if err != nil { + return nil, "", errOutput + } + if trusted != nil { + info, err := root.Stat(".") + if err != nil || !trusted(info) { + root.Close() + return nil, "", errOutput + } + } + parts := strings.Split(strings.TrimPrefix(abs, "/"), "/") + for _, part := range parts[:len(parts)-1] { + info, e := root.Lstat(part) + if e != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + root.Close() + return nil, "", errOutput + } + next, e := root.OpenRoot(part) + if e != nil { + root.Close() + return nil, "", errOutput + } + opened, e := next.Stat(".") + root.Close() + if e != nil || !os.SameFile(info, opened) || trusted != nil && !trusted(opened) { + next.Close() + return nil, "", errOutput + } + root = next + } + return root, parts[len(parts)-1], nil +} + +type outputDir struct { + parent, container, root *os.Root + parentFile, containerFile, rootFile *os.File + path, name, stage string + stageInfo, parentInfo os.FileInfo + created map[string]os.FileInfo + complete bool +} + +func newOutput(path string) (*outputDir, error) { + parent, name, err := openOutputParent(path) + if err != nil { + return nil, err + } + if _, err = parent.Lstat(name); !os.IsNotExist(err) { + parent.Close() + return nil, errOutput + } + d := &outputDir{parent: parent, path: path, name: name, created: make(map[string]os.FileInfo)} + ok := false + defer func() { + if !ok { + d.close() + } + }() + d.parentInfo, err = parent.Stat(".") + if err != nil { + return nil, errOutput + } + d.parentFile, err = parent.Open(".") + if err != nil { + return nil, errOutput + } + for attempts := 0; attempts < 32; attempts++ { + d.stage = ".fleetdiff-stage-" + rand.Text() + if err = parent.Mkdir(d.stage, 0700); !os.IsExist(err) { + break + } + } + if err != nil { + d.stage = "" + return nil, errOutput + } + d.stageInfo, err = parent.Lstat(d.stage) + if err != nil || !d.stageInfo.IsDir() || !owned(d.stageInfo) || d.stageInfo.Mode().Perm() != 0700 { + return nil, errOutput + } + d.container, err = parent.OpenRoot(d.stage) + if err != nil { + return nil, errOutput + } + opened, err := d.container.Stat(".") + if err != nil || !os.SameFile(d.stageInfo, opened) { + return nil, errOutput + } + d.containerFile, err = d.container.Open(".") + if err != nil { + return nil, errOutput + } + // The public sibling name is never the rename source. An attacker may + // replace that name, but cannot redirect operations on this private root. + if err = d.container.Mkdir("export", 0700); err != nil { + return nil, errOutput + } + d.root, err = d.container.OpenRoot("export") + if err != nil { + return nil, errOutput + } + d.rootFile, err = d.root.Open(".") + if err != nil { + return nil, errOutput + } + ok = true + return d, nil +} + +func (d *outputDir) close() { + if d.parent == nil { + return + } + if !d.complete && d.root != nil { + for name := range d.created { + _ = d.root.Remove(name) + } + } + if d.rootFile != nil { + d.rootFile.Close() + } + if d.root != nil { + d.root.Close() + } + if !d.complete && d.containerFile != nil { + _ = removeEmptyDirectoryAt(d.containerFile, "export") + } + if d.containerFile != nil { + d.containerFile.Close() + } + if d.container != nil { + d.container.Close() + } + // Only rmdir is safe through this mutable public name. A last-moment + // replacement can at most lose an empty directory, never files or a tree. + if d.stageInfo != nil && d.parentFile != nil { + if now, err := d.parent.Lstat(d.stage); err == nil && os.SameFile(d.stageInfo, now) { + _ = removeEmptyDirectoryAt(d.parentFile, d.stage) + } + } + if d.parentFile != nil { + d.parentFile.Close() + } + d.parent.Close() + d.parent = nil +} + +func (d *outputDir) create(name string) (*os.File, error) { + if d.complete || d.parent == nil || name != "spend.csv" && name != "spend.jsonl" && name != "daily.csv" { + return nil, errOutput + } + f, err := d.root.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL|syscall.O_NOFOLLOW, 0600) + if err != nil { + return nil, errOutput + } + d.created[name] = nil + info, err := f.Stat() + if err != nil || !info.Mode().IsRegular() || !owned(info) || info.Mode().Perm() != 0600 { + f.Close() + return nil, errOutput + } + d.created[name] = info + return f, nil +} + +// publish has one commit point: a successful no-replace directory rename. +// Cancellation after that point must not roll back or report an absent export. +func (d *outputDir) publish(ctx context.Context) error { + if d.parent == nil || d.complete || len(d.created) == 0 { + return errOutput + } + _, csv := d.created["spend.csv"] + _, jsonl := d.created["spend.jsonl"] + if csv == jsonl { + return errOutput + } + // Detect a caller-path rebind before committing. The syscall still targets + // the held parent if an external actor renames an ancestor after this check. + parent, name, err := openOutputParent(d.path) + if err != nil { + return errOutput + } + current, err := parent.Stat(".") + parent.Close() + if err != nil || name != d.name || !os.SameFile(d.parentInfo, current) { + return errOutput + } + source, err := d.container.Lstat("export") + held, heldErr := d.root.Stat(".") + if err != nil || heldErr != nil || !source.IsDir() || !os.SameFile(source, held) { + return errOutput + } + entries, err := d.rootFile.ReadDir(-1) + if err != nil || len(entries) != len(d.created) { + return errOutput + } + for _, entry := range entries { + original, ok := d.created[entry.Name()] + current, err := d.root.Lstat(entry.Name()) + if !ok || original == nil || err != nil || !os.SameFile(original, current) || !current.Mode().IsRegular() || current.Mode().Perm() != 0600 { + return errOutput + } + } + if err := d.rootFile.Sync(); err != nil { + return errOutput + } + if ctx.Err() != nil { + return exportdata.ErrInterrupted + } + if err := renameDirectoryNoReplace(d.containerFile, "export", d.parentFile, d.name); err != nil { + return errOutput + } + d.complete = true + return nil +} diff --git a/internal/postgres/files_test.go b/internal/postgres/files_test.go new file mode 100644 index 0000000..1dd0bbd --- /dev/null +++ b/internal/postgres/files_test.go @@ -0,0 +1,424 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package postgres + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "os" + process "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/llm-measurement/fleetdiff/internal/exportdata" + "github.com/llm-measurement/fleetdiff/internal/spend" +) + +const stagedSpend = "request_id,call_type,startTime,prompt_tokens,completion_tokens,total_tokens\nprivate-id,completion,2026-10-01T01:00:00Z,1,2,3\n" +const stagedDaily = "date,api_requests,successful_requests,failed_requests\n2026-10-01,1,1,0\n" + +func privateOutputPath(t *testing.T) string { + t.Helper() + parent, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + return filepath.Join(parent, "export") +} + +func stageOutput(t *testing.T, path string) *outputDir { + t.Helper() + d, err := newOutput(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(d.close) + for name, body := range map[string]string{"spend.csv": stagedSpend, "daily.csv": stagedDaily} { + f, err := d.create(name) + if err != nil { + t.Fatal(err) + } + if _, err = f.WriteString(body); err != nil { + t.Fatal(err) + } + if err = f.Sync(); err != nil { + t.Fatal(err) + } + if err = f.Close(); err != nil { + t.Fatal(err) + } + } + return d +} + +func assertCompleteOutput(t *testing.T, path string) { + t.Helper() + entries, err := os.ReadDir(path) + if err != nil || len(entries) != 2 { + t.Fatalf("not one complete two-file export: %v %v", entries, err) + } + for name, want := range map[string]string{"spend.csv": stagedSpend, "daily.csv": stagedDaily} { + got, err := os.ReadFile(filepath.Join(path, name)) + if err != nil || string(got) != want { + t.Fatalf("incomplete %s: %q %v", name, got, err) + } + info, err := os.Stat(filepath.Join(path, name)) + if err != nil || info.Mode().Perm() != 0600 { + t.Fatalf("file is not private: %v %v", info, err) + } + } + info, err := os.Stat(path) + if err != nil || info.Mode().Perm() != 0700 { + t.Fatalf("directory is not private: %v %v", info, err) + } +} + +func makeReplacement(t *testing.T, path, kind string) { + t.Helper() + switch kind { + case "file": + if err := os.WriteFile(path, []byte("unrelated sentinel"), 0600); err != nil { + t.Fatal(err) + } + case "empty-directory", "directory": + if err := os.Mkdir(path, 0700); err != nil { + t.Fatal(err) + } + if kind == "directory" { + if err := os.WriteFile(filepath.Join(path, "sentinel"), []byte("unrelated sentinel"), 0600); err != nil { + t.Fatal(err) + } + } + case "symlink": + if err := os.Symlink("not-an-export", path); err != nil { + t.Fatal(err) + } + default: + t.Fatalf("unknown replacement kind %q", kind) + } +} + +func assertReplacement(t *testing.T, path, kind string, before os.FileInfo) { + t.Helper() + after, err := os.Lstat(path) + if err != nil || !os.SameFile(before, after) { + t.Fatalf("replacement changed: %s %v", kind, err) + } + if kind == "file" || kind == "directory" { + if kind == "directory" { + path = filepath.Join(path, "sentinel") + } + data, err := os.ReadFile(path) + if err != nil || string(data) != "unrelated sentinel" { + t.Fatalf("replacement data changed: %s %q %v", kind, data, err) + } + } +} + +func TestOutputDirectoryAtomicNoReplace(t *testing.T) { + for _, kind := range []string{"absent", "file", "empty-directory", "directory", "symlink"} { + t.Run(kind, func(t *testing.T) { + path := privateOutputPath(t) + d := stageOutput(t, path) + if _, err := os.Lstat(path); !os.IsNotExist(err) { + t.Fatal("staging exposed final output") + } + var before os.FileInfo + if kind != "absent" { + makeReplacement(t, path, kind) + before, _ = os.Lstat(path) + } + err := d.publish(context.Background()) + if kind == "absent" { + if err != nil || !d.complete { + t.Fatalf("publish: %v", err) + } + d.close() + assertCompleteOutput(t, path) + } else { + if !errors.Is(err, errOutput) || d.complete { + t.Fatalf("existing destination accepted: %v", err) + } + d.close() + assertReplacement(t, path, kind, before) + } + }) + } +} + +func TestOutputRejectsExistingBeforeStaging(t *testing.T) { + for _, kind := range []string{"file", "empty-directory", "directory", "symlink"} { + t.Run(kind, func(t *testing.T) { + path := privateOutputPath(t) + makeReplacement(t, path, kind) + before, _ := os.Lstat(path) + if _, err := newOutput(path); !errors.Is(err, errOutput) { + t.Fatalf("existing output accepted: %v", err) + } + assertReplacement(t, path, kind, before) + entries, err := os.ReadDir(filepath.Dir(path)) + if err != nil || len(entries) != 1 { + t.Fatalf("existing output created staging directory: %v %v", entries, err) + } + }) + } +} + +func TestOutputCleanupOnlyRemovesEmptyDirectory(t *testing.T) { + for _, kind := range []string{"file", "empty-directory", "directory", "symlink"} { + t.Run(kind, func(t *testing.T) { + path := privateOutputPath(t) + makeReplacement(t, path, kind) + before, _ := os.Lstat(path) + parent, err := os.Open(filepath.Dir(path)) + if err != nil { + t.Fatal(err) + } + defer parent.Close() + err = removeEmptyDirectoryAt(parent, filepath.Base(path)) + if kind == "empty-directory" { + if err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(path); !os.IsNotExist(err) { + t.Fatal("empty directory not removed") + } + } else { + if err == nil { + t.Fatal("directory-only cleanup accepted another file type or a nonempty directory") + } + assertReplacement(t, path, kind, before) + } + }) + } +} + +func TestOutputWrapperReplacementCannotRedirectValidationOrPublication(t *testing.T) { + for _, publish := range []bool{false, true} { + for _, kind := range []string{"file", "empty-directory", "directory", "symlink"} { + t.Run(fmt.Sprintf("publish=%t/%s", publish, kind), func(t *testing.T) { + path := privateOutputPath(t) + d := stageOutput(t, path) + original := filepath.Join(filepath.Dir(path), d.stage) + moved := original + "-moved" + if err := os.Rename(original, moved); err != nil { + t.Fatal(err) + } + makeReplacement(t, original, kind) + before, _ := os.Lstat(original) + rows := 0 + if err := spend.ReadRoutingRoot(d.root, "spend.csv", func(spend.Row) error { rows++; return nil }); err != nil || rows != 1 { + t.Fatalf("held-root spend validation redirected: %d %v", rows, err) + } + if err := spend.ValidateDailyExportRoot(d.root, "daily.csv"); err != nil { + t.Fatalf("held-root daily validation redirected: %v", err) + } + if publish { + if err := d.publish(context.Background()); err != nil { + t.Fatal(err) + } + } + d.close() + assertReplacement(t, original, kind, before) + if publish { + assertCompleteOutput(t, path) + } else { + if _, err := os.Lstat(path); !os.IsNotExist(err) { + t.Fatal("failed export exposed final path") + } + if _, err := os.Lstat(filepath.Join(moved, "export", "spend.csv")); !os.IsNotExist(err) { + t.Fatal("private staged data not cleaned through held root") + } + } + }) + } + } +} + +func TestOutputParentAndInnerSourceReplacementsFailClosed(t *testing.T) { + for _, source := range []bool{false, true} { + path := privateOutputPath(t) + d := stageOutput(t, path) + if source { + if err := d.container.Rename("export", "moved"); err != nil { + t.Fatal(err) + } + if err := d.container.Mkdir("export", 0700); err != nil { + t.Fatal(err) + } + } else { + parent := filepath.Dir(path) + if err := os.Rename(parent, parent+"-moved"); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(parent + "-moved") }) + if err := os.Mkdir(parent, 0700); err != nil { + t.Fatal(err) + } + } + if err := d.publish(context.Background()); !errors.Is(err, errOutput) { + t.Fatalf("replaced source/parent accepted: %v", err) + } + d.close() + if _, err := os.Lstat(path); !os.IsNotExist(err) { + t.Fatal("replacement path received an export") + } + } +} + +func TestOutputParentPolicyIsOutputOnly(t *testing.T) { + path := privateOutputPath(t) + parent := filepath.Dir(path) + for _, mode := range []os.FileMode{0777, 0770} { + if err := os.Chmod(parent, mode); err != nil { + t.Fatal(err) + } + if _, err := newOutput(path); !errors.Is(err, errOutput) { + t.Fatalf("untrusted writable parent accepted: %o %v", mode, err) + } + root, _, err := openParent(path) + if err != nil { + t.Fatalf("credential parent policy changed: %v", err) + } + root.Close() + } + if err := os.Chmod(parent, 0777|os.ModeSticky); err != nil { + t.Fatal(err) + } + d, err := newOutput(path) + if err != nil { + t.Fatalf("trusted sticky parent rejected: %v", err) + } + d.close() + if err := os.Chmod(parent, 0700); err != nil { + t.Fatal(err) + } + ancestor := filepath.Join(parent, "untrusted") + if err := os.Mkdir(ancestor, 0700); err != nil { + t.Fatal(err) + } + if err := os.Chmod(ancestor, 0777); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(ancestor, "private"), 0700); err != nil { + t.Fatal(err) + } + if _, err := newOutput(filepath.Join(ancestor, "private", "export")); !errors.Is(err, errOutput) { + t.Fatal("untrusted ancestor accepted") + } +} + +func TestOutputCancellationAndCommitPoint(t *testing.T) { + path := privateOutputPath(t) + d := stageOutput(t, path) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := d.publish(ctx); !errors.Is(err, exportdata.ErrInterrupted) { + t.Fatalf("canceled publication: %v", err) + } + d.close() + if _, err := os.Lstat(path); !os.IsNotExist(err) { + t.Fatal("canceled export was published") + } + d = stageOutput(t, path) + ctx, cancel = context.WithCancel(context.Background()) + if err := d.publish(ctx); err != nil { + t.Fatal(err) + } + cancel() + d.close() + assertCompleteOutput(t, path) +} + +func TestOutputHeldRootRejectsSymlinksAndInvalidData(t *testing.T) { + path := privateOutputPath(t) + d := stageOutput(t, path) + if err := d.root.Rename("spend.csv", "moved.csv"); err != nil { + t.Fatal(err) + } + if err := d.root.Symlink("moved.csv", "spend.csv"); err != nil { + t.Fatal(err) + } + if err := spend.ReadRoutingRoot(d.root, "spend.csv", func(spend.Row) error { return nil }); err == nil { + t.Fatal("held-root validator accepted symlink") + } + if err := d.root.WriteFile("daily.csv", []byte("invalid daily"), 0600); err != nil { + t.Fatal(err) + } + if err := spend.ValidateDailyExportRoot(d.root, "daily.csv"); err == nil { + t.Fatal("held-root validator skipped daily parser") + } +} + +func TestOutputKillPublicationIsAllOrNothing(t *testing.T) { + for _, phase := range []string{"staged", "committing", "published"} { + t.Run(phase, func(t *testing.T) { + path := privateOutputPath(t) + cmd := process.Command(os.Args[0], "-test.run=^TestOutputProcessHelper$") + cmd.Env = append(os.Environ(), "FLEETDIFF_OUTPUT_TEST_PATH="+path, "FLEETDIFF_OUTPUT_TEST_PHASE="+phase) + stdout, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + input, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + defer input.Close() + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer cmd.Process.Kill() + line, err := bufio.NewReader(stdout).ReadString('\n') + if err != nil || strings.TrimSpace(line) != "ready" { + t.Fatalf("subprocess did not stage: %q %v", line, err) + } + if phase == "committing" { + _, _ = input.Write([]byte{1}) + } + if err := cmd.Process.Kill(); err != nil { + t.Fatal(err) + } + if err := cmd.Wait(); err == nil { + t.Fatal("subprocess was not interrupted") + } + _, err = os.Lstat(path) + if phase == "staged" && !os.IsNotExist(err) { + t.Fatal("staging created final directory") + } + if phase == "published" || err == nil { + assertCompleteOutput(t, path) + } + }) + } +} + +func TestOutputProcessHelper(t *testing.T) { + path := os.Getenv("FLEETDIFF_OUTPUT_TEST_PATH") + if path == "" { + return + } + d := stageOutput(t, path) + phase := os.Getenv("FLEETDIFF_OUTPUT_TEST_PHASE") + if phase == "published" { + if err := d.publish(context.Background()); err != nil { + t.Fatal(err) + } + } + fmt.Println("ready") + var b [1]byte + if _, err := io.ReadFull(os.Stdin, b[:]); err != nil { + t.Fatal(err) + } + if phase == "committing" { + if err := d.publish(context.Background()); err != nil { + t.Fatal(err) + } + } + _, _ = io.ReadFull(os.Stdin, b[:]) +} diff --git a/internal/postgres/query.go b/internal/postgres/query.go new file mode 100644 index 0000000..901fb14 --- /dev/null +++ b/internal/postgres/query.go @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package postgres + +import ( + "errors" + "strings" + "time" + + litellmsql "github.com/llm-measurement/fleetdiff/examples/litellm-spend" + "github.com/llm-measurement/fleetdiff/internal/exportdata" +) + +var errProjection = errors.New("embedded LiteLLM projection is invalid") + +func recipeBlock(marker string) (string, error) { + parts := strings.Split(litellmsql.Recipe, marker) + if len(parts) != 3 { + return "", errProjection + } + return parts[1], nil +} + +// COPY has no bind parameters. Only UTC timestamps formatted from checked +// time.Time values and fixed, embedded fragments may enter its SQL text. +func queries(o exportdata.Options) (string, string, error) { + if o.Start.IsZero() || o.Start.Year() < 1970 || o.End.Year() > 2200 || !o.Start.Before(o.End) { + return "", "", errors.New("export needs ascending UTC bounds between 1970 and 2200") + } + query, err := recipeBlock("$projection$") + if err != nil { + return "", "", err + } + details, err := recipeBlock("$details$") + if err != nil { + return "", "", err + } + routing, err := recipeBlock("$routing$") + if err != nil { + return "", "", err + } + if !o.UsageDetails { + details = "" + } + if !o.Routing { + routing = "" + } + if strings.Count(query, "%s") != 1 || strings.Count(query, "%L") != 2 { + return "", "", errProjection + } + query = strings.Replace(query, "%s", details+routing, 1) + query = strings.Replace(query, "%L", sqlTime(o.Start), 1) + query = strings.Replace(query, "%L", sqlTime(o.End), 1) + daily := litellmsql.DailyRecipe + daily = strings.ReplaceAll(daily, ":'start_utc'", sqlTime(o.Start)) + daily = strings.ReplaceAll(daily, ":'end_utc'", sqlTime(o.End)) + index := strings.Index(daily, "\n\\g ") + if index < 0 { + return "", "", errProjection + } + daily = daily[:index] + return query, daily, nil +} + +func sqlTime(t time.Time) string { return "'" + t.UTC().Format(time.RFC3339Nano) + "'" } diff --git a/internal/postgres/query_test.go b/internal/postgres/query_test.go new file mode 100644 index 0000000..d917321 --- /dev/null +++ b/internal/postgres/query_test.go @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package postgres + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/llm-measurement/fleetdiff/internal/exportdata" +) + +func TestQueriesOnlyUseParsedTimesAndEmbeddedFragments(t *testing.T) { + for _, details := range []bool{false, true} { + for _, routing := range []bool{false, true} { + o := exportdata.Options{Start: time.Date(2026, 10, 7, 0, 0, 0, 123000000, time.FixedZone("SENTINEL_ZONE'); DROP TABLE x; --", 3600)), End: time.Date(2026, 10, 9, 0, 0, 0, 0, time.UTC), UsageDetails: details, Routing: routing, Directory: "SENTINEL_PATH';DROP TABLE x;--", CredentialFile: "SENTINEL_CREDENTIAL"} + q, d, err := queries(o) + if err != nil { + t.Fatal(err) + } + for _, query := range []string{q, d} { + if strings.Contains(query, "SENTINEL") || strings.Contains(query, "%L") || strings.Contains(query, ":'start_utc'") { + t.Fatal("unformatted or caller-controlled SQL") + } + } + if !strings.Contains(q, "'2026-10-06T23:00:00.123Z'") || !strings.Contains(q, `ORDER BY "startTime", request_id COLLATE "C"`) { + t.Fatal("typed UTC/order contract changed") + } + if strings.Contains(q, "AS cache_read_input_tokens") != details || strings.Contains(q, "AS original_model_group") != routing { + t.Fatal("projection option mismatch") + } + } + } + if _, _, err := queries(exportdata.Options{}); err == nil { + t.Fatal("unbounded query accepted") + } +} + +func TestNewOutputAndCleanup(t *testing.T) { + parent, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(parent, "export") + d, err := newOutput(path) + if err != nil { + t.Fatal(err) + } + f, err := d.create("spend.csv") + if err != nil { + t.Fatal(err) + } + if _, err = f.WriteString("incomplete"); err != nil { + t.Fatal(err) + } + f.Close() + if _, err := os.Lstat(path); !os.IsNotExist(err) { + t.Fatal("final output exists before publication") + } + if info, err := d.root.Stat("."); err != nil || info.Mode().Perm() != 0700 { + t.Fatal("directory permissions") + } + if info, err := d.root.Stat("spend.csv"); err != nil || info.Mode().Perm() != 0600 { + t.Fatal("file permissions") + } + d.close() + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatal("incomplete output retained") + } + if err := os.Symlink(parent, filepath.Join(parent, "symlink")); err != nil { + t.Fatal(err) + } + if _, err := newOutput(filepath.Join(parent, "symlink", "outside")); err == nil { + t.Fatal("symlink parent accepted") + } + if err := os.Mkdir(path, 0700); err != nil { + t.Fatal(err) + } + if _, err := newOutput(path); err == nil { + t.Fatal("empty existing directory accepted") + } +} + +func TestOutputByteBound(t *testing.T) { + var b bytes.Buffer + w := byteLimit{w: &b, remaining: 3} + if n, err := w.Write([]byte("abc")); n != 3 || err != nil { + t.Fatal("exact limit rejected") + } + if n, err := w.Write([]byte("d")); n != 0 || !errors.Is(err, errLimit) || b.String() != "abc" { + t.Fatal("limit truncated or exceeded") + } +} diff --git a/internal/postgres/rename_darwin.go b/internal/postgres/rename_darwin.go new file mode 100644 index 0000000..8592253 --- /dev/null +++ b/internal/postgres/rename_darwin.go @@ -0,0 +1,48 @@ +//go:build darwin && (amd64 || arm64) + +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package postgres + +import ( + "os" + "runtime" + "syscall" + "unsafe" +) + +func renameDirectoryNoReplace(from *os.File, old string, to *os.File, name string) error { + a, err := syscall.BytePtrFromString(old) + if err != nil { + return err + } + b, err := syscall.BytePtrFromString(name) + if err != nil { + return err + } + // Darwin SYS_renameatx_np, RENAME_EXCL; the Go amd64 wrapper adds + // the syscall class itself. Never fall back to an overwriting rename. + _, _, errno := syscall.Syscall6(488, from.Fd(), uintptr(unsafe.Pointer(a)), to.Fd(), uintptr(unsafe.Pointer(b)), 4, 0) + runtime.KeepAlive(from) + runtime.KeepAlive(to) + if errno != 0 { + return errno + } + return nil +} + +func removeEmptyDirectoryAt(parent *os.File, name string) error { + p, err := syscall.BytePtrFromString(name) + if err != nil { + return err + } + // Darwin SYS_unlinkat, AT_REMOVEDIR: never unlink a replacement file + // or symlink, and never remove a nonempty replacement directory. + _, _, errno := syscall.Syscall6(472, parent.Fd(), uintptr(unsafe.Pointer(p)), 0x80, 0, 0, 0) + runtime.KeepAlive(parent) + if errno != 0 { + return errno + } + return nil +} diff --git a/internal/postgres/rename_linux.go b/internal/postgres/rename_linux.go new file mode 100644 index 0000000..3d859cc --- /dev/null +++ b/internal/postgres/rename_linux.go @@ -0,0 +1,46 @@ +//go:build linux && (amd64 || arm64) + +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package postgres + +import ( + "os" + "runtime" + "syscall" + "unsafe" +) + +func renameDirectoryNoReplace(from *os.File, old string, to *os.File, name string) error { + a, err := syscall.BytePtrFromString(old) + if err != nil { + return err + } + b, err := syscall.BytePtrFromString(name) + if err != nil { + return err + } + // Linux renameat2, RENAME_NOREPLACE. Unsupported kernels/filesystems + // fail closed; ordinary Rename would overwrite an existing directory. + _, _, errno := syscall.Syscall6(renameat2Trap, from.Fd(), uintptr(unsafe.Pointer(a)), to.Fd(), uintptr(unsafe.Pointer(b)), 1, 0) + runtime.KeepAlive(from) + runtime.KeepAlive(to) + if errno != 0 { + return errno + } + return nil +} + +func removeEmptyDirectoryAt(parent *os.File, name string) error { + p, err := syscall.BytePtrFromString(name) + if err != nil { + return err + } + _, _, errno := syscall.Syscall6(syscall.SYS_UNLINKAT, parent.Fd(), uintptr(unsafe.Pointer(p)), 0x200, 0, 0, 0) + runtime.KeepAlive(parent) + if errno != 0 { + return errno + } + return nil +} diff --git a/internal/postgres/rename_linux_amd64.go b/internal/postgres/rename_linux_amd64.go new file mode 100644 index 0000000..b70d219 --- /dev/null +++ b/internal/postgres/rename_linux_amd64.go @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package postgres + +const renameat2Trap = 316 diff --git a/internal/postgres/rename_linux_arm64.go b/internal/postgres/rename_linux_arm64.go new file mode 100644 index 0000000..d074efd --- /dev/null +++ b/internal/postgres/rename_linux_arm64.go @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package postgres + +import "syscall" + +const renameat2Trap = syscall.SYS_RENAMEAT2 diff --git a/internal/postgres/schema.go b/internal/postgres/schema.go new file mode 100644 index 0000000..7ef7f2d --- /dev/null +++ b/internal/postgres/schema.go @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package postgres + +import ( + "context" + "strconv" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/llm-measurement/fleetdiff/internal/exportdata" +) + +var errSchema = exportdata.ErrSchema + +func checkSchema(ctx context.Context, c *pgconn.PgConn, daily bool) error { + spend := map[string]uint32{} + for _, name := range []string{"request_id", "call_type", "api_key", "user", "team_id", "organization_id", "end_user", "model", "model_id", "model_group", "custom_llm_provider", "status", "session_id", "agent_id"} { + spend[name] = 25 + } + for _, name := range []string{"prompt_tokens", "completion_tokens", "total_tokens"} { + spend[name] = 23 + } + spend["spend"], spend["metadata"], spend["startTime"], spend["endTime"] = 701, 3802, 1114, 1114 + notNull := map[string]bool{} + for _, name := range []string{"request_id", "call_type", "api_key", "model", "prompt_tokens", "completion_tokens", "total_tokens", "spend", "startTime", "endTime"} { + notNull[name] = true + } + if err := schemaColumns(ctx, c, `public."LiteLLM_SpendLogs"`, spend, notNull); err != nil { + return err + } + r := c.ExecParams(ctx, `SELECT count(*) FROM pg_catalog.pg_index i JOIN pg_catalog.pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=i.indkey[0] WHERE i.indrelid='public."LiteLLM_SpendLogs"'::regclass AND i.indisprimary AND i.indnkeyatts=1 AND a.attname='request_id'`, nil, nil, nil, nil).Read() + if r.Err != nil { + return databaseFailure(r.Err) + } + if len(r.Rows) != 1 || len(r.Rows[0]) != 1 || string(r.Rows[0][0]) != "1" { + return errSchema + } + if daily { + return schemaColumns(ctx, c, `public."LiteLLM_DailyUserSpend"`, map[string]uint32{"date": 25, "api_requests": 20, "successful_requests": 20, "failed_requests": 20}, map[string]bool{"date": true, "api_requests": true, "successful_requests": true, "failed_requests": true}) + } + return nil +} + +func schemaColumns(ctx context.Context, c *pgconn.PgConn, table string, expected map[string]uint32, notNull map[string]bool) error { + r := c.ExecParams(ctx, `SELECT a.attname, a.atttypid::text, a.atttypmod::text, a.attnotnull::text FROM pg_catalog.pg_attribute a JOIN pg_catalog.pg_class c ON c.oid=a.attrelid WHERE a.attrelid=pg_catalog.to_regclass($1) AND a.attnum>0 AND NOT a.attisdropped AND c.relkind IN ('r','p')`, [][]byte{[]byte(table)}, nil, nil, nil).Read() + if r.Err != nil { + return databaseFailure(r.Err) + } + seen := map[string]bool{} + for _, row := range r.Rows { + if len(row) != 4 { + return errSchema + } + name := string(row[0]) + want, ok := expected[name] + if !ok { + continue + } + got, err := strconv.ParseUint(string(row[1]), 10, 32) + if err != nil || got != uint64(want) || want == 1114 && string(row[2]) != "3" || string(row[3]) != strconv.FormatBool(notNull[name]) { + return errSchema + } + seen[name] = true + } + if len(seen) != len(expected) { + return errSchema + } + return nil +} diff --git a/internal/postgres/tls_test.go b/internal/postgres/tls_test.go new file mode 100644 index 0000000..d5e5d65 --- /dev/null +++ b/internal/postgres/tls_test.go @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: Apache-2.0 +// Code authors: Vijay and Codex + +package postgres + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "io" + "math/big" + "net" + "net/url" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgproto3" +) + +const tlsTransportStop = "TLS transport test stops before authentication" + +func TestVerifyFullTLSTransport(t *testing.T) { + const hostname = "database.example.invalid" + ca, key, trustedRoot := newTransportTestCA(t) + _, _, unrelatedRoot := newTransportTestCA(t) + certificate := newTransportTestCertificate(t, ca, key, hostname) + for _, tc := range []struct { + name, host, root string + refuseTLS bool + }{ + {name: "trusted_hostname", host: hostname, root: trustedRoot}, + {name: "untrusted_ca", host: hostname, root: unrelatedRoot}, + {name: "hostname_mismatch", host: "other.example.invalid", root: trustedRoot}, + {name: "server_refuses_tls", host: hostname, root: trustedRoot, refuseTLS: true}, + } { + t.Run(tc.name, func(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = listener.Close() }) + _, port, err := net.SplitHostPort(listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + u := url.URL{ + Scheme: "postgresql", User: url.User("tls_probe"), + Host: net.JoinHostPort(tc.host, port), Path: "/fixture", + RawQuery: url.Values{"sslmode": {"verify-full"}, "sslrootcert": {tc.root}}.Encode(), + } + config, err := connectionConfig(u.String()) + if err != nil { + t.Fatal(err) + } + if config.TLSConfig == nil || config.TLSConfig.InsecureSkipVerify || config.TLSConfig.ServerName != tc.host || len(config.Fallbacks) != 0 { + t.Fatal("verify-full configuration lost hostname verification or added a fallback") + } + // Keep the remote-style hostname and its TLS policy, but never use DNS + // or dial any endpoint other than this test's loopback listener. + config.LookupFunc = func(_ context.Context, host string) ([]string, error) { + if host != tc.host { + return nil, errors.New("unexpected TLS test hostname lookup") + } + return []string{"127.0.0.1"}, nil + } + var attempts atomic.Int32 + config.DialFunc = func(ctx context.Context, network, address string) (net.Conn, error) { + if attempts.Add(1) != 1 || network != "tcp" || address != listener.Addr().String() { + return nil, errors.New("unexpected TLS test dial or fallback attempt") + } + return (&net.Dialer{}).DialContext(ctx, network, address) + } + result := make(chan tlsTransportObservation, 1) + go func() { + conn, err := listener.Accept() + if err != nil { + result <- tlsTransportObservation{err: err} + return + } + result <- observeTLSTransport(conn, certificate, tc.refuseTLS) + }() + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + conn, connectErr := pgconn.ConnectConfig(ctx, config) + if conn != nil { + _ = conn.Close(ctx) + t.Fatal("transport-only server unexpectedly authenticated a connection") + } + if connectErr == nil { + t.Fatal("transport-only server unexpectedly completed PostgreSQL startup") + } + var observed tlsTransportObservation + select { + case observed = <-result: + case <-ctx.Done(): + t.Fatal("TLS test server did not finish") + } + if count := attempts.Load(); count != 1 { + t.Fatalf("connection attempts = %d, want exactly one without fallback", count) + } + if !observed.sslRequested { + t.Fatalf("PostgreSQL SSLRequest was not received: %v", observed.err) + } + switch tc.name { + case "trusted_hostname": + if observed.err != nil || !observed.tlsComplete || !observed.startupReceived || observed.serverName != hostname { + t.Fatalf("trusted TLS did not reach encrypted startup: %+v", observed) + } + var protocolStop *pgconn.PgError + if !errors.As(connectErr, &protocolStop) || protocolStop.Code != "08004" || protocolStop.Message != tlsTransportStop { + t.Fatalf("expected deliberate PostgreSQL startup stop after valid TLS: %v", connectErr) + } + case "untrusted_ca": + var unknownAuthority x509.UnknownAuthorityError + if !errors.As(connectErr, &unknownAuthority) { + t.Fatalf("expected untrusted CA rejection, got %v", connectErr) + } + case "hostname_mismatch": + var hostnameError x509.HostnameError + if !errors.As(connectErr, &hostnameError) || hostnameError.Host != tc.host { + t.Fatalf("expected hostname verification rejection, got %v", connectErr) + } + case "server_refuses_tls": + if observed.err != nil || !strings.Contains(connectErr.Error(), "server refused TLS connection") { + t.Fatalf("TLS refusal did not close without plaintext startup: server=%v client=%v", observed.err, connectErr) + } + } + if tc.name != "trusted_hostname" && (observed.tlsComplete || observed.startupReceived) { + t.Fatal("rejected transport reached TLS completion or PostgreSQL startup") + } + }) + } +} + +type tlsTransportObservation struct { + sslRequested, tlsComplete, startupReceived bool + serverName string + err error +} + +func observeTLSTransport(conn net.Conn, certificate tls.Certificate, refuseTLS bool) (result tlsTransportObservation) { + defer conn.Close() + if result.err = conn.SetDeadline(time.Now().Add(5 * time.Second)); result.err != nil { + return result + } + message, err := pgproto3.NewBackend(conn, conn).ReceiveStartupMessage() + if err != nil { + result.err = err + return result + } + if _, ok := message.(*pgproto3.SSLRequest); !ok { + result.err = fmt.Errorf("expected SSLRequest before any startup data, got %T", message) + return result + } + result.sslRequested = true + response := byte('S') + if refuseTLS { + response = 'N' + } + if _, result.err = conn.Write([]byte{response}); result.err != nil { + return result + } + if refuseTLS { + var next [1]byte + n, err := conn.Read(next[:]) + if n != 0 || !errors.Is(err, io.EOF) { + result.err = fmt.Errorf("expected client close after TLS refusal, received %d bytes: %v", n, err) + } + return result + } + secured := tls.Server(conn, &tls.Config{ + MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{certificate}, + }) + defer secured.Close() + if result.err = secured.Handshake(); result.err != nil { + return result + } + state := secured.ConnectionState() + result.tlsComplete, result.serverName = state.HandshakeComplete, state.ServerName + backend := pgproto3.NewBackend(secured, secured) + message, result.err = backend.ReceiveStartupMessage() + if result.err != nil { + return result + } + startup, ok := message.(*pgproto3.StartupMessage) + if !ok || startup.Parameters["user"] != "tls_probe" || startup.Parameters["database"] != "fixture" { + result.err = errors.New("unexpected encrypted PostgreSQL startup") + return result + } + result.startupReceived = true + // A protocol error ends this transport-only probe without pgconn's + // asynchronous cancellation dial on unexpected EOF. No authentication stub. + backend.Send(&pgproto3.ErrorResponse{Severity: "FATAL", Code: "08004", Message: tlsTransportStop}) + result.err = backend.Flush() + return result +} + +func newTransportTestCA(t *testing.T) (*x509.Certificate, *ecdsa.PrivateKey, string) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + t.Fatal(err) + } + now := time.Now() + template := &x509.Certificate{ + SerialNumber: serial, Subject: pkix.Name{CommonName: "fleetdiff ephemeral TLS test CA"}, + NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), + IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + ca, err := x509.ParseCertificate(der) + if err != nil { + t.Fatal(err) + } + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "ca.pem") + if err := os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0600); err != nil { + t.Fatal(err) + } + return ca, key, path +} + +func newTransportTestCertificate(t *testing.T, ca *x509.Certificate, caKey *ecdsa.PrivateKey, hostname string) tls.Certificate { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + t.Fatal(err) + } + template := &x509.Certificate{ + SerialNumber: serial, DNSNames: []string{hostname}, + NotBefore: ca.NotBefore, NotAfter: ca.NotAfter, + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, template, ca, &key.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} +} diff --git a/internal/spend/reader.go b/internal/spend/reader.go index 31e64ae..ce77bc6 100644 --- a/internal/spend/reader.go +++ b/internal/spend/reader.go @@ -106,6 +106,29 @@ func ReadRouting(path string, visit func(Row) error) error { return readSpend(path, visit, true) } +// ReadRoutingRoot validates an export through an already-held directory. It +// keeps ReadRouting's file checks and limits without reopening a public path. +func ReadRoutingRoot(root *os.Root, name string, visit func(Row) error) error { + if visit == nil { + return errors.New("spend row visitor is required; configure an importer before reading") + } + if root == nil { + return errSpendFile + } + info, err := root.Lstat(name) + if err != nil || !info.Mode().IsRegular() { + return errSpendFile + } + if info.Size() > MaxFileBytes { + return errSpendSize + } + ext := strings.ToLower(filepath.Ext(name)) + if ext != ".csv" && ext != ".json" && ext != ".jsonl" { + return errSpendType + } + return readSpendRoot(root, name, info, ext, visit, true) +} + func readSpend(path string, visit func(Row) error, routing bool) error { if visit == nil { return errors.New("spend row visitor is required; configure an importer before reading") @@ -126,7 +149,11 @@ func readSpend(path string, visit func(Row) error, routing bool) error { return errSpendFile } defer root.Close() - f, err := root.OpenFile(filepath.Base(path), os.O_RDONLY|syscall.O_NONBLOCK|syscall.O_NOFOLLOW, 0) + return readSpendRoot(root, filepath.Base(path), info, ext, visit, routing) +} + +func readSpendRoot(root *os.Root, name string, info os.FileInfo, ext string, visit func(Row) error, routing bool) error { + f, err := root.OpenFile(name, os.O_RDONLY|syscall.O_NONBLOCK|syscall.O_NOFOLLOW, 0) if err != nil { return errSpendFile } diff --git a/internal/spend/row_coverage.go b/internal/spend/row_coverage.go index 7aee6f3..b8e967c 100644 --- a/internal/spend/row_coverage.go +++ b/internal/spend/row_coverage.go @@ -26,6 +26,16 @@ var errCoverageFile = errors.New("daily spend input must be a readable regular C var errCoverageCSV = errors.New("invalid daily spend CSV; use the four-column read-only daily SQL projection") var errCoverageLimit = errors.New("daily spend coverage exceeds 3,660 days, 1 MiB or 512 bytes per line; select a shorter period") +// ValidateDailyExportRoot applies the same bounded coverage parser through an +// already-held export directory, without resolving its public pathname again. +func ValidateDailyExportRoot(root *os.Root, name string) error { + if root == nil { + return errCoverageFile + } + _, err := readDailyCountsRoot(root, name) + return err +} + // RowCoverage compares records with a second accounting path. Matching counts // do not establish that requests missing from both paths were recorded. type RowCoverage struct { @@ -56,7 +66,11 @@ func readDailyCounts(path string) (map[string]dailyCounts, error) { return nil, errCoverageFile } defer root.Close() - data, err := localfile.Read(root, filepath.Base(path), maxCoverageBytes) + return readDailyCountsRoot(root, filepath.Base(path)) +} + +func readDailyCountsRoot(root *os.Root, name string) (map[string]dailyCounts, error) { + data, err := localfile.Read(root, name, maxCoverageBytes) if err != nil { return nil, fmt.Errorf("daily spend input: %w", err) } From 86223c57c98b588bb2b41f81e78cd7d99ab421c8 Mon Sep 17 00:00:00 2001 From: Vijay Date: Sun, 18 Oct 2026 14:45:36 -0400 Subject: [PATCH 2/3] release: prepare v0.12.0 verified one-command workflow Signed-off-by: Vijay --- CHANGELOG.md | 9 ++-- README.md | 71 +++++++++++++---------------- SECURITY.md | 6 +-- docs/COMPARISON.md | 2 +- docs/INSPECT.md | 2 +- docs/NATIVE_EXPORT.md | 17 ++++--- docs/OPERATIONS.md | 40 ++++++---------- docs/RELEASING.md | 19 ++++++-- docs/ROUTING.md | 4 +- docs/WHAT_IF.md | 4 +- docs/releases/v0.12.0.md | 25 +++++----- examples/litellm-spend/README.md | 23 +++++----- examples/litellm-spend/TESTING.md | 23 ++++++---- examples/litellm-spend/test_docs.py | 29 ++++++++++++ scripts/install.sh | 4 +- scripts/install_test.go | 57 +++++++++++++++++++++-- 16 files changed, 205 insertions(+), 130 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 741732f..1c3bfcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,10 @@ Notable user-visible changes are recorded here, newest first. -## [Unreleased] - v0.12.0 Candidate +## [0.12.0] - 2026-10-18 -See the [unreleased notes](docs/releases/v0.12.0.md) and -[native export guide](docs/NATIVE_EXPORT.md). These features are not in the -released v0.11.0 installer. +See the [release notes](docs/releases/v0.12.0.md) and +[native export guide](docs/NATIVE_EXPORT.md). - Add explicit native `export litellm` for private CSV/JSONL snapshots, optional daily counters and routing details, with the pinned LiteLLM v1.104.0 projection. @@ -20,7 +19,7 @@ released v0.11.0 installer. atomically publish the complete directory without replacement. Keep the final destination absent until publication; crashes beforehand may leave sibling staging data, never a partial final directory. -- Order candidate SQL exports deterministically by `"startTime", request_id +- Order SQL exports deterministically by `"startTime", request_id COLLATE "C"` and daily `date COLLATE "C"` for native/psql byte equivalence. Compared with the earlier unsorted recipe, bytes and first-seen/tied report ordering may change; recorded totals and identity hashing are unchanged. diff --git a/README.md b/README.md index be378a9..b400677 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,13 @@ compares LLM usage summaries, flags unusual windows, and checks trace captures and collector configurations. Use it to answer **why LLM token usage spiked, and which users or sessions drove -the change.** File-based commands stay offline. The unreleased native LiteLLM +the change.** File-based commands stay offline. The native LiteLLM path reads your database explicitly, then reports locally: no fleetdiff account, upload, collector or model API key is needed. | | | | --- | --- | -| Commands | `investigate`, `what-if`, `scan`, `compare`, `inspect`, `diagnose`; candidate: `export litellm` | +| Commands | `investigate`, `what-if`, `scan`, `compare`, `inspect`, `diagnose`, `export litellm` | | Platforms | Linux and macOS, AMD64 and ARM64 | | Status | Pre-1.0; see the [changelog](CHANGELOG.md) | | License | Apache-2.0 | @@ -22,20 +22,30 @@ upload, collector or model API key is needed. **One command to the first report, after setup.** Compare two periods, see which models and keys drove the change, and check the usage records you already retain. -The native path below is an **unreleased v0.12 candidate**, not a feature of the -released v0.11.0 installer. -For a released binary, use the fallback below. -### Native Candidate Setup (Development Only) + + + + + + + -From a reviewed candidate source checkout with a patched Go 1.26 toolchain, -build into a new directory; stop if that directory already exists: +### 1. Install v0.12.0 + +Run this in Bash with `curl`, `gh`, `shasum` and `tar` available; `gh` need not be signed in: ```sh -mkdir ./fleetdiff-install && -go build -o ./fleetdiff-install/fleetdiff ./cmd/fleetdiff +bash -o pipefail -c 'curl --fail --location --proto "=https" --tlsv1.2 https://raw.githubusercontent.com/llm-measurement/fleetdiff/65990e302913fb960db3ba3a53cb4633bef63117/scripts/install.sh | sh -s -- v0.12.0 ./fleetdiff-install' +./fleetdiff-install/fleetdiff --version ``` +The installer checks signed release provenance and checksums before extracting +into a new directory. No Go compiler is needed. Follow +[Install and verify](docs/OPERATIONS.md#install-and-verify) for manual verification. + +### 2. Investigate Your Database + Have your administrator provision the [read-only role and credential file](docs/NATIVE_EXPORT.md#credentials). Set `DATABASE_URL_FILE` to that existing, owned mode-0600 regular file. Leave `FLEETDIFF_LITELLM_DATABASE_URL` unset when using the file, then run: @@ -55,35 +65,16 @@ The [native guide](docs/NATIVE_EXPORT.md) covers credentials, periods and what-i -### Released Fallback: Export With psql +### Manual Alternative: Export With psql -For v0.11.0, or client certificates, PostgreSQL services and other native-unsupported +For client certificates, PostgreSQL services and other native-unsupported connections, follow the [read-only SQL recipe](examples/litellm-spend/README.md#export-request-level-rows). Keep the export private; it excludes prompts and responses but retains identities. -Existing request-level CSV, JSON and JSONL also work. Install the released binary -in a separate new directory from any candidate build, then investigate the file. - - - - - - - - -### 2. Install v0.11.0 - -Run this in Bash with `curl`, `gh`, `shasum` and `tar` available: - -```sh -bash -o pipefail -c 'curl --fail --location --proto "=https" --tlsv1.2 https://raw.githubusercontent.com/llm-measurement/fleetdiff/7773f212e4ae6e800f066b394e5507c0df8ada5f/scripts/install.sh | sh -s -- v0.11.0 ./fleetdiff-install' -./fleetdiff-install/fleetdiff --version -``` +Existing request-level CSV, JSON and JSONL also work with the same installed binary. -The v0.11.0 installer does not provide native database export. It checks signed -release provenance and checksums before extracting into a new directory. Follow -[Install and verify](docs/OPERATIONS.md#install-and-verify). + -### 3. Investigate +Investigate the exported file offline: ```sh export_dir="$HOME/litellm-export" @@ -111,7 +102,7 @@ Download the synthetic spend file and use the installed binary: ```sh curl --fail --location --proto '=https' --tlsv1.2 \ - https://raw.githubusercontent.com/llm-measurement/fleetdiff/v0.11.0/examples/litellm-spend/synthetic.csv \ + https://raw.githubusercontent.com/llm-measurement/fleetdiff/v0.12.0/examples/litellm-spend/synthetic.csv \ --output synthetic.csv ./fleetdiff-install/fleetdiff investigate --litellm-spend synthetic.csv \ --before-period 2026-10-07 --after-period 2026-10-08 @@ -144,15 +135,15 @@ The summary-file path also shows session concentration: ## Install -The verified v0.11.0 installer selects the platform binary for your machine. -For manual downloads, use the [v0.11.0 archives](https://github.com/llm-measurement/fleetdiff/releases/tag/v0.11.0) +The verified v0.12.0 installer selects the platform binary for your machine. +For manual downloads, use the [v0.12.0 archives](https://github.com/llm-measurement/fleetdiff/releases/tag/v0.12.0) and [verification instructions](docs/OPERATIONS.md#install-and-verify). An administrator can verify and transfer artifacts through an approved internal mirror. With a security-patched Go 1.26 toolchain, install the same version from its module: ```sh -go install github.com/llm-measurement/fleetdiff/cmd/fleetdiff@v0.11.0 +go install github.com/llm-measurement/fleetdiff/cmd/fleetdiff@v0.12.0 ``` The executable goes to `GOBIN`, or `$(go env GOPATH)/bin` when unset. @@ -174,7 +165,7 @@ See [source builds and version reporting](docs/OPERATIONS.md#build-from-source). ## Documentation -- [Native LiteLLM export](docs/NATIVE_EXPORT.md): unreleased v0.12 candidate setup and connection contract. +- [Native LiteLLM export](docs/NATIVE_EXPORT.md): setup, credentials and connection contract. - [LiteLLM spend exports](examples/litellm-spend/README.md): SQL, periods, groupings and source accounting. - [Session-budget review](docs/SESSION_BUDGET.md): single-period recorded totals and cap review. - [Model-mix what-if](docs/WHAT_IF.md) (v0.10.0): re-price covered usage under explicit assumptions. @@ -186,7 +177,7 @@ See [source builds and version reporting](docs/OPERATIONS.md#build-from-source). - [Questions and contributor rankings](docs/WORKFLOWS.md#questions-it-answers). - [How summary comparison works](docs/WORKFLOWS.md#how-it-works), [comparison contract](docs/COMPARISON.md) and [investigation contract](docs/INVESTIGATION.md). - [Operations](docs/OPERATIONS.md), [resource measurements](docs/BENCHMARKS.md) and [checks](docs/WORKFLOWS.md#built-to-be-checked). -- [FAQ](docs/FAQ.md), [security](SECURITY.md), [release notes](docs/releases/v0.11.0.md) and [release procedure](docs/RELEASING.md). +- [FAQ](docs/FAQ.md), [security](SECURITY.md), [release notes](docs/releases/v0.12.0.md) and [release procedure](docs/RELEASING.md). Questions or feedback: [open an issue](https://github.com/llm-measurement/fleetdiff/issues). See [Contributing](CONTRIBUTING.md) for checks and signed, signed-off commits. diff --git a/SECURITY.md b/SECURITY.md index ebb11a3..c22b0d9 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,7 +2,7 @@ ## Supported Versions -Security fixes target the latest patch in the `0.11.x` release line and `main`. +Security fixes target the latest patch in the `0.12.x` release line and `main`. Use the newest patch and record its version and revision when running trials. Older patches and development snapshots do not receive routine backports. Dependencies and the Go toolchain are reviewed on an ongoing basis. @@ -47,11 +47,11 @@ access; inspection also accepts stdin. Merely setting database credentials does not enable a connection. Initial builds may download dependencies, and the live example starts local Docker collectors and may download its pinned image. -## Native Database Export (Unreleased v0.12 Candidate) +## Native Database Export Only explicit `export litellm`, `investigate --litellm-db` and `what-if --litellm-db` connect to PostgreSQL. This adds a network boundary, not telemetry, update checks -or provider calls. The released v0.11.0 binary does not include these modes. +or provider calls. - Supply exactly one credential source: `FLEETDIFF_LITELLM_DATABASE_URL` or `--database-url-file` pointing to an owned regular mode-0600 file. Literal diff --git a/docs/COMPARISON.md b/docs/COMPARISON.md index 431cba3..7edffe8 100644 --- a/docs/COMPARISON.md +++ b/docs/COMPARISON.md @@ -84,7 +84,7 @@ does not bypass scope, key, accounting, replay, or structural checks. ## Trust And Effects File-based comparison reads local files and writes stdout/stderr, offline and -without modifying inputs. The unreleased v0.12 candidate's explicit +without modifying inputs. The explicit [native database modes](NATIVE_EXPORT.md) have a separate network and private-file write boundary; they do not change this summary comparison contract. Transfer inputs through authenticated channels and follow diff --git a/docs/INSPECT.md b/docs/INSPECT.md index e76939f..00d7893 100644 --- a/docs/INSPECT.md +++ b/docs/INSPECT.md @@ -123,7 +123,7 @@ interval. Frequent-item bounds are deterministic for retained observed weight. ## Privacy The file-based `inspect` command makes no network requests, opens no receiver, -and writes no files. The unreleased v0.12 candidate's explicit +and writes no files. The explicit [native database modes](NATIVE_EXPORT.md) do not change inspection's offline behavior. Inspection holds decoded raw input transiently in memory to compute the report. A fresh cryptographic key is created for each run; aggregate state retains hashes and diff --git a/docs/NATIVE_EXPORT.md b/docs/NATIVE_EXPORT.md index e5512e7..c11777f 100644 --- a/docs/NATIVE_EXPORT.md +++ b/docs/NATIVE_EXPORT.md @@ -5,11 +5,10 @@ fleetdiff exports a private read-only snapshot, analyzes it locally, and removes the temporary files unless you ask to keep them. No `psql` installation is needed for the native path. -This is an **unreleased v0.12 candidate**. Build a reviewed candidate checkout -using the [development setup](OPERATIONS.md#native-candidate-build-development-only). -The released v0.11.0 installer does not include these commands; use the +Install [fleetdiff v0.12.0](../README.md#1-install-v0120) with the verified +installer; the examples below use that binary. The [manual SQL export](../examples/litellm-spend/README.md#export-request-level-rows) -with that binary. The examples below run from the candidate checkout. +remains available for connection setups outside the native contract. ## First Report @@ -200,13 +199,13 @@ fail rather than guessing. All reads use one `BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY` transaction with UTC settings; fleetdiff issues no database writes or schema migrations. -The candidate's shared SQL orders spend rows by `"startTime", request_id COLLATE +The v0.12.0 shared SQL orders spend rows by `"startTime", request_id COLLATE "C"` and daily rows by `date COLLATE "C"`. For the same database snapshot, bounds, -format and detail options, native and candidate `psql` exports must be +format and detail options, native and v0.12.0 `psql` exports must be byte-for-byte identical. Sequential exports from a changing live database are not the same snapshot. This ordering is new relative to the v0.11.0 recipe: it can change bytes, first-seen aliases and tied report order, not recorded -totals. Database sorting can also affect export time. +totals or identity hashing. Database sorting can also affect export time. ## Limits And Failures @@ -262,8 +261,8 @@ Keep exports private and review reports under your organization's sharing policy Use `umask 077` when redirecting reports; shell output permissions are separate from native export permissions. -The candidate pins `pgx/v5 v5.11.0`, `pgpassfile v1.0.0` and +Version v0.12.0 pins `pgx/v5 v5.11.0`, `pgpassfile v1.0.0` and `pgservicefile v0.0.0-20240606120523-5a60cdf6a761`; existing `x/text v0.41.0` normalization is retained. Dependency names do not imply passfile/service support. -The [candidate verification checklist](../examples/litellm-spend/TESTING.md#native-export-candidate-checks) +The [verification checklist](../examples/litellm-spend/TESTING.md#native-export-checks) lists the database, security and equivalence tests. diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 6fe826d..1998e28 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -2,15 +2,15 @@ fleetdiff is one local binary. File-based commands read LiteLLM spend exports, summary files, OTLP captures or collector configurations and report offline, -without administrator privileges. The **unreleased v0.12 candidate** also offers +without administrator privileges. Version v0.12.0 also offers explicit native database export; that mode connects to PostgreSQL and writes -private local exports before analysis. It is not in the v0.11.0 installer. +private local exports before analysis. ## Install And Verify The initial supported targets are Linux and macOS on AMD64 and ARM64. Native Windows binaries are not provided. Download a binary from the -[release page](https://github.com/llm-measurement/fleetdiff/releases/tag/v0.11.0). +[release page](https://github.com/llm-measurement/fleetdiff/releases/tag/v0.12.0). Go is not needed to run it. The commands below use `curl` for downloading and the [GitHub CLI](https://cli.github.com/) for provenance verification. @@ -22,7 +22,7 @@ subshell stops on any download or verification failure: ```sh ( set -eu -version=v0.11.0 +version=v0.12.0 case "$(uname -s)" in Linux) os=linux ;; Darwin) os=darwin ;; @@ -56,7 +56,7 @@ From a reviewed repository checkout, the same download and verification steps are available as a script (no Go compiler needed): ```sh -sh scripts/install.sh v0.11.0 ./fleetdiff-install +sh scripts/install.sh v0.12.0 ./fleetdiff-install ./fleetdiff-install/fleetdiff --version ``` @@ -64,14 +64,18 @@ To download and inspect the pinned installer before running it: ```sh curl --fail --location --proto '=https' --tlsv1.2 \ - https://raw.githubusercontent.com/llm-measurement/fleetdiff/e75518e1b62ead77a6870cfeabd3ddae944a1e0a/scripts/install.sh \ + https://raw.githubusercontent.com/llm-measurement/fleetdiff/65990e302913fb960db3ba3a53cb4633bef63117/scripts/install.sh \ --output fleetdiff-install.sh less fleetdiff-install.sh -sh fleetdiff-install.sh v0.11.0 ./fleetdiff-install +sh fleetdiff-install.sh v0.12.0 ./fleetdiff-install ./fleetdiff-install/fleetdiff --version ``` -The v0.11.0 binary includes [model-mix what-if repricing](WHAT_IF.md), the +The immutable script pin is a reviewed installer; the explicit `v0.12.0` +argument selects the release independently of that script's older default. + +The v0.12.0 binary includes [native LiteLLM export](NATIVE_EXPORT.md), +[model-mix what-if repricing](WHAT_IF.md), the [LiteLLM spend importer](../examples/litellm-spend/README.md) and optional [daily row coverage](ROW_COVERAGE.md), alongside summary, trace and configuration commands. @@ -119,7 +123,7 @@ Install a published version with a currently patched Go 1.26 or later toolchain: go install github.com/llm-measurement/fleetdiff/cmd/fleetdiff@latest ``` -Use `@v0.11.0` instead of `@latest` to pin that release. The binary is installed +Use `@v0.12.0` instead of `@latest` to pin that release. The binary is installed under `GOBIN`, or `$(go env GOPATH)/bin` when unset; put that directory on `PATH`. This path uses Go module verification rather than the archive attestation above, and does not include unreleased checkout features. @@ -140,7 +144,7 @@ Check `--version` on the binary you actually run: | Build | Version | Revision | |---|---|---| | Unstamped checkout build above | Go's embedded module/VCS version, or `dev` when unavailable | `unknown` | -| Tagged `go install ...@v0.11.0` | `v0.11.0`, from Go's module metadata | `unknown` | +| Tagged `go install ...@v0.12.0` | `v0.12.0`, from Go's module metadata | `unknown` | | Verified release archive | Stamped release version | Stamped source revision | A checkout may report a pseudo-version and `+dirty`, depending on the Go toolchain @@ -158,22 +162,6 @@ only after verifying those cached dependencies there. Do not disable checksum verification for ordinary connected builds. An offline user normally needs only the reviewed binary, not Go, Git, Docker, or a compiler. -### Native Candidate Build (Development Only) - -Use a reviewed candidate checkout and a new build directory, separate from a -released installation. Stop if directory creation or the build fails: - -```sh -mkdir ./fleetdiff-install && -go build -o ./fleetdiff-install/fleetdiff ./cmd/fleetdiff -``` - -This is development setup, not a verified release installation. The v0.11.0 -installer and tagged Go install above do not contain native export. Follow -[Native LiteLLM Export](NATIVE_EXPORT.md) for administrator-approved credentials, -read-only grants and a first report. Dependency downloads are build-time access, -separate from the explicit runtime database connection. - ## Native Database Runs Only `export litellm` and `investigate` / `what-if` with `--litellm-db` connect. diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 32872ee..c8c8b07 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -25,6 +25,11 @@ archive is then checksum-verified and exercised on its native platform. Linux also runs the offline/non-root smoke test. Dependency inventory alone is not a vulnerability verdict; the source reachability scan is a separate check. +Each of the four platform jobs also runs `go test ./internal/postgres -count=1` +and `go test ./internal/cli -run '^TestProductionHasNoNetwork$' -count=1`. +These check platform-specific atomic publication and the source network boundary; +they are source tests, separate from the packaged-binary tests below. + The native archive check also runs `TestReleaseBinaryTopK` against the unpacked executable. `TestReleaseBinaryInspect` checks stdin inspection, token accounting, label filtering, and opt-in name display. Together they check user/session attribution, @@ -44,6 +49,8 @@ JSONL on each platform, including raw-identity suppression and quality counts. The Linux AMD64 archive also runs the pinned LiteLLM writer -> PostgreSQL -> read-only SQL -> packaged importer checks. This path uses synthetic data, no provider keys and no paid calls. +The same job binds both `FLEETDIFF_BIN` and `FLEETDIFF_LINUX_BIN` to the unpacked +archive executable, so the native-driver database tests exercise that artifact too. `TestReleaseBinarySessionBudget` checks the cap-only command, observed-span fallback, explicit periods and clipping in each native archive. The existing @@ -52,7 +59,7 @@ sentinels with the packaged importer. `TestReleaseBinaryWhatIf` checks the simple and scenario commands against the checked text/JSON output, plus workload mappings, summary gaps and error privacy. -Before packaging v0.11.0, reproduce its public numeric bundle offline: +Before packaging v0.12.0, reproduce its public numeric bundle offline: ```sh python3 -B scripts/reproduce-model-mix.py --check @@ -90,7 +97,7 @@ verify that every advertised version is downloadable. Do not replace immutable t or artifacts to correct a failed publication; fix forward with a new version if any release content must change. -## After Publishing v0.11.0 +## After Publishing v0.12.0 From an anonymous clone in a fresh directory outside any synced workspace: @@ -99,7 +106,7 @@ From an anonymous clone in a fresh directory outside any synced workspace: [OPERATIONS](OPERATIONS.md#install-and-verify). Confirm the signed tag and recorded build revision match the approved commit. 2. Run the README's verified installer into a new directory. Confirm the native - executable reports `v0.11.0`, then run the packaged-binary tests above. + executable reports `v0.12.0`, then run the packaged-binary tests above. 3. Set `FLEETDIFF_BIN` to that freshly installed executable and run the exact [pinned database test](../examples/litellm-spend/TESTING.md#pinned-database-integration). It exercises the real pinned writer, a SELECT-only SQL export and the released @@ -141,6 +148,12 @@ From an anonymous clone in a fresh directory outside any synced workspace: and catalog provider both equal the prefix. Repeat with mismatched and missing providers; those rows remain unknown. The packaged `TestReleaseBinaryStructuredModelPricing` covers scoped and unscoped pricing. +12. Follow the [native export checks](../examples/litellm-spend/TESTING.md#native-export-checks) + with the installed binary. Compare native CSV/JSONL and daily output against + the v0.12.0 `psql` recipe on the same frozen database contents and options. + Exercise the README's database investigation, private cleanup and retained + exports. Record the exact executable and distinguish native-driver tests + from the older writer/psql tests; skipped database checks do not pass this gate. The release workflow tests every native platform before publication. The local outsider run proves the post-publication download and database path on its own diff --git a/docs/ROUTING.md b/docs/ROUTING.md index 5158199..51c255d 100644 --- a/docs/ROUTING.md +++ b/docs/ROUTING.md @@ -1,6 +1,6 @@ # Which Models Served Each Route? -Available in v0.11.0: the optional routing audit compares the model distribution behind each +Introduced in v0.11.0: the optional routing audit compares the model distribution behind each original LiteLLM route, with request, token and recorded-spend shares and explicit attribution coverage. Classifier overhead is counted separately. @@ -44,7 +44,7 @@ requests. Recorded spend is the source's value, not a recomputed invoice. ## No Data Handy? Try The Sample -From a v0.11.0 checkout, use the [verified binary](../README.md#2-install-v0100): +From a v0.12.0 checkout, use the [verified binary](../README.md#1-install-v0120): ```sh ./fleetdiff-install/fleetdiff investigate --litellm-spend examples/routing/synthetic.csv \ diff --git a/docs/WHAT_IF.md b/docs/WHAT_IF.md index 4a176cf..7fee8a5 100644 --- a/docs/WHAT_IF.md +++ b/docs/WHAT_IF.md @@ -3,7 +3,7 @@ Re-price a period of recorded usage under a proposed model mix, with the price date, measured factors and covered observations shown beside the result. -Available in **v0.11.0**. Install the [verified binary](../README.md#2-install-v0100) +Available in **v0.12.0**. Install the [verified binary](../README.md#1-install-v0120) before running the commands below. ## Run it on your data @@ -39,7 +39,7 @@ and [JSON](../examples/what-if/expected.json). ## No data handy? Try the sample -From a v0.11.0 checkout, using the verified binary installed above: +From a v0.12.0 checkout, using the verified binary installed above: ```sh ./fleetdiff-install/fleetdiff what-if --litellm-spend examples/what-if/spend.csv \ diff --git a/docs/releases/v0.12.0.md b/docs/releases/v0.12.0.md index 6ef5d34..95f7aa0 100644 --- a/docs/releases/v0.12.0.md +++ b/docs/releases/v0.12.0.md @@ -1,12 +1,15 @@ -# v0.12.0 Release Notes (Unreleased) +# v0.12.0 Release Notes + +2026-10-18 **Investigate LiteLLM spend directly from a read-only database snapshot.** This -candidate adds native export and a one-command export-then-analyze path, keeping +version adds native export and a one-command export-then-analyze path, keeping existing file-based analysis offline. -This version is **unreleased**. Use a reviewed checkout and the -[development-only candidate build](../OPERATIONS.md#native-candidate-build-development-only) -to try it. The released v0.11.0 installer does not provide these features. +Use the [verified installer or manual verification](../OPERATIONS.md#install-and-verify) +with `v0.12.0`. Linux and macOS archives cover AMD64 and ARM64. The +[manual SQL export](../../examples/litellm-spend/README.md#export-request-level-rows) +remains available for client certificates and other native-unsupported setups. ## Native LiteLLM Export @@ -39,23 +42,23 @@ builds files in a mode-0700 sibling `.fleetdiff-stage-...` directory, then atomically publishes the complete directory without replacement. The final destination stays absent until publication; a crash beforehand may leave sibling staging data, never a partial final directory. The -[security policy](../../SECURITY.md#native-database-export-unreleased-v012-candidate) +[security policy](../../SECURITY.md#native-database-export) and [limits](../NATIVE_EXPORT.md#limits-and-failures) describe these controls. ## Ordering Change -The candidate SQL recipe now orders spend rows by `"startTime", request_id -COLLATE "C"` and daily rows by `date COLLATE "C"`. Native and candidate `psql` +The v0.12.0 SQL recipe now orders spend rows by `"startTime", request_id +COLLATE "C"` and daily rows by `date COLLATE "C"`. Native and v0.12.0 `psql` exports must match byte for byte for the same snapshot and options. The earlier recipe did not define spend-row order: export bytes, first-seen aliases and tied report order may change, while recorded totals and identity hashing do not. -Sorting may affect export time. Compare against the candidate recipe, not a +Sorting may affect export time. Compare against the v0.12.0 recipe, not a historical unsorted export, when checking equivalence. -## Candidate Validation +## Verification The dependency set adds `pgx/v5 v5.11.0`, `pgpassfile v1.0.0` and `pgservicefile v0.0.0-20240606120523-5a60cdf6a761`, retaining `x/text v0.41.0`. -The [verification checklist](../../examples/litellm-spend/TESTING.md#native-export-candidate-checks) +The [verification checklist](../../examples/litellm-spend/TESTING.md#native-export-checks) covers pinned-database byte equivalence, negative/security tests and an outsider first run. diff --git a/examples/litellm-spend/README.md b/examples/litellm-spend/README.md index b302763..ef00199 100644 --- a/examples/litellm-spend/README.md +++ b/examples/litellm-spend/README.md @@ -7,7 +7,7 @@ increase, and two keys account for all of it.** The report separates request count from request size where the usage records support that comparison. -Install [fleetdiff v0.11.0](../../README.md#2-install-v0100), then run this fixture +Install [fleetdiff v0.12.0](../../README.md#1-install-v0120), then run this fixture from the repository root with the verified binary: ```sh @@ -31,11 +31,10 @@ The per-model split is available for model-1; the quality counts point to rows to review in the local LiteLLM logs. Try the same 20 safe, synthetic records as [CSV](synthetic.csv), [JSON array](synthetic.json), or [JSONL](synthetic.jsonl). -## Native Candidate: One Command +## Native Export: One Command -The **unreleased v0.12 candidate** can export and investigate directly. Use the -[development-only build](../../docs/OPERATIONS.md#native-candidate-build-development-only), -not the v0.11.0 installer. After administrator-approved read-only access and +Version v0.12.0 can export and investigate directly. Use the +[verified installer](../../README.md#1-install-v0120). After administrator-approved read-only access and [credential-file setup](../../docs/NATIVE_EXPORT.md#credentials): ```sh @@ -53,7 +52,7 @@ identities; reports keep their existing aliasing rules. ## Export Request-Level Rows -This manual path remains available with released v0.11.0 and for connection +This manual path remains available for offline analysis and for connection setups outside the native contract, including client certificates and PostgreSQL services. Use [export.sql](export.sql) with `psql` and your existing read-only database access. Configure a PostgreSQL service named `litellm_readonly` using approved @@ -71,10 +70,10 @@ psql 'service=litellm_readonly' -Xq -v ON_ERROR_STOP=1 \ >"$export_dir/private-psql.out" 2>"$export_dir/private-psql.err" ``` -The candidate recipe writes **CSV only by default**, using one `COPY` pass in a +The recipe writes **CSV only by default**, using one `COPY` pass in a **REPEATABLE READ, READ ONLY transaction**. It orders spend rows by `"startTime", request_id COLLATE "C"`; optional daily rows use `date COLLATE "C"`. This is an -unreleased ordering change from the v0.11.0 recipe, enabling byte-for-byte parity +ordering change from the v0.11.0 recipe, enabling byte-for-byte parity with native export for the same snapshot and options. The historical 16-day interval leaves room for the importer's default two seven-day windows inside the file's edge days. Sparse logs may still need explicit periods; @@ -111,7 +110,7 @@ To check for rows missing relative to LiteLLM's daily counters, use the optional The SQL recipe accepts `-v routing_details=true` on the same private-file export command; standalone native export uses `--routing-details`. Convenience database -modes include these details automatically. Within the candidate recipe, omitted +modes include these details automatically. Within this recipe, omitted or `false` leaves the default columns and bytes unchanged. This option is independent of `usage_details` and adds exactly three columns: @@ -153,7 +152,7 @@ catalog provider both match the prefix; pricing uses the same rule. The routing guide describes the explicit local label option. Released v0.10.1 and v0.7.0 readers ignore the added columns; they do not implement routing-audit features. See the per-column [compatibility checks](TESTING.md#routing-projection). -For the **v0.11.0** routing audit and route/agent selectors, see the +For the routing audit and route/agent selectors introduced in **v0.11.0**, see the [routing guide](../../docs/ROUTING.md). ### Usage Details (Default On) @@ -318,7 +317,7 @@ the call types and asks for a supported request-level export. session ID alone within the supplied application/cache namespace. In either mode, session IDs may be generated trace IDs, not conversations. - Tied contributors and models follow their first appearance in the file. - The candidate's deterministic SQL ordering can therefore change aliases and + The v0.12.0 deterministic SQL ordering can therefore change aliases and tied report order relative to older unsorted exports, without changing totals. Concentration uses the full tracked candidate set before `--top` truncates display rows. Its denominator is the net recorded increase, so offsets from @@ -350,7 +349,7 @@ the call types and asks for a supported request-level export. [session-budget.csv](session-budget.csv) is a newly authored, deterministic Apache-2.0 fixture with four sessions and no production data. The companion [license notice](session-budget.csv.license) keeps the CSV directly importable. -Use fleetdiff v0.11.0 for the session-budget command: +Use fleetdiff v0.12.0 for the session-budget command: ```sh fleetdiff investigate --litellm-spend examples/litellm-spend/session-budget.csv \ diff --git a/examples/litellm-spend/TESTING.md b/examples/litellm-spend/TESTING.md index fd28c44..d5f477c 100644 --- a/examples/litellm-spend/TESTING.md +++ b/examples/litellm-spend/TESTING.md @@ -15,10 +15,10 @@ python3 -B -m unittest discover -s examples/litellm-spend -p 'test_docs.py' -v These check the README line budget, legacy anchors, linked documents and pinned usage table. They do not establish native export, TLS or database equivalence. -## Native Export Candidate Checks +## Native Export Checks -The [native guide](../../docs/NATIVE_EXPORT.md) describes an **unreleased v0.12 -candidate**. Run the acceptance checks below on the exact candidate binary. +The [native guide](../../docs/NATIVE_EXPORT.md) describes the v0.12.0 database +path. Run the acceptance checks below on the exact binary being verified. | Gate | Required evidence | | --- | --- | @@ -32,16 +32,23 @@ candidate**. Run the acceptance checks below on the exact candidate binary. | Bounds and cleanup | Row/byte/field limits, timeouts and handled cancellation remove staging without publishing a partial final directory; convenience cleanup and retained-export behavior remain correct | | Privacy and offline behavior | Sentinels absent from reports/errors; no raw driver diagnostics; local-file commands never connect even with database credentials set | | Convenience modes | Default UTC windows, explicit periods, automatic routing details, cleanup and retained exports; what-if model `--from` / `--to` remain distinct from `--period` | -| Release readiness | Full race/vet checks, vulnerability and license review, dependency inventory, reproducible builds and an outsider following the candidate README from a fresh environment | +| Release readiness | Full race/vet checks, vulnerability and license review, dependency inventory, reproducible builds and an outsider following the versioned README from a fresh environment | -Reuse the cached immutable images below; native candidate validation requires no +Reuse the cached immutable images below; native validation requires no new image or paid call. Existing writer/psql tests alone do not prove that the -host binary connected through the native driver. Record the exact binary, +supplied binary ran the native exporter. Record the exact binary, snapshot/fixture, image digests, option matrix and byte comparisons for that run. Tests requiring a local database connection must explicitly describe that boundary; the older `--network=none` writer harness description is not a claim that all native integration tests are network-free. +For native database tests, also set `FLEETDIFF_LINUX_BIN` to a verified Linux +executable matching Docker's architecture. The harness mounts it read-only and +runs it inside the pinned database container. On Linux this can be the same +executable as `FLEETDIFF_BIN`; on macOS use a separately verified Linux archive. +The release workflow binds both variables to the unpacked Linux AMD64 binary. +Unset `FLEETDIFF_LINUX_BIN` skips native database tests, not just optional detail. + The dependency approval covers `pgx/v5 v5.11.0`, `pgpassfile v1.0.0` and `pgservicefile v0.0.0-20240606120523-5a60cdf6a761`, retaining `x/text v0.41.0`. It does not substitute for the release checks above. Keep historical verification @@ -152,7 +159,7 @@ or deployment configuration. and Responses operation names are exercised. Unpinned operations are inventoried separately and make the selected-period comparison incomplete. Expected export membership comes from actual writer timestamps and the - half-open UTC bounds, not a fixed row count. The candidate recipe additionally + half-open UTC bounds, not a fixed row count. The v0.12.0 recipe additionally fixes row order for native/psql byte equivalence. - The real writer produces the usage detail metadata. SQL aliases are compared to each actual payload's pinned metadata paths, including native @@ -178,7 +185,7 @@ or deployment configuration. CSV; setting `jsonl_file` must create only JSONL, with or without `csv_file`. These checks cover both default-detail and opt-out projections. Records match across both SQL formats and the locally assembled JSON array. Empty intervals - also produce valid files. The candidate fixture-only check requires the + also produce valid files. The native fixture-only check requires the deterministic SQL order and rejects JSON-array aggregation and multiple CSV `COPY` passes in the recipe. - Fixed-secret JSON reports match for all five groups, with and without diff --git a/examples/litellm-spend/test_docs.py b/examples/litellm-spend/test_docs.py index bfbe296..6cf72ab 100644 --- a/examples/litellm-spend/test_docs.py +++ b/examples/litellm-spend/test_docs.py @@ -59,8 +59,33 @@ def test_readme_shape_and_previous_anchors(self): "questions-it-answers", "user-and-session-contributors", "how-it-works", "run-it-through-real-collectors", "use-your-own-data", "reading-the-results", "built-to-be-checked", "docs", + "1-export-request-level-rows", "3-investigate", + "2-install-v060", "2-install-v061", "2-install-v070", + "2-install-v080", "2-install-v090", "2-install-v0100", + "2-install-v0110", "1-install-v0120", } <= anchors(readme)) + def test_native_release_promotion(self): + readme = (ROOT / "README.md").read_text() + self.assertLess(readme.index("### 1. Install v0.12.0"), + readme.index("investigate --litellm-db")) + self.assertIn("Manual Alternative: Export With psql", readme) + self.assertNotIn("go build", readme) + notes = (ROOT / "docs/releases/v0.12.0.md").read_text() + self.assertIn("2026-10-18", notes) + for phrase in ("first-seen aliases", "tied", "identity hashing"): + self.assertIn(phrase, notes) + self.assertIn("## [0.12.0] - 2026-10-18", + (ROOT / "CHANGELOG.md").read_text()) + for name in ("README.md", "SECURITY.md", "docs/NATIVE_EXPORT.md", + "docs/OPERATIONS.md", "docs/INSPECT.md", "docs/COMPARISON.md", + "docs/releases/v0.12.0.md", "examples/litellm-spend/README.md", + "examples/litellm-spend/TESTING.md"): + with self.subTest(document=name): + content = (ROOT / name).read_text() + self.assertNotRegex(content.lower(), + r"unreleased v0\.12|native candidate|candidate (?:build|recipe|checkout)") + def test_document_links(self): paths = [ROOT / "README.md", ROOT / "docs/WORKFLOWS.md", ROOT / "docs/OPERATIONS.md", ROOT / "docs/RELEASING.md", @@ -69,6 +94,10 @@ def test_document_links(self): ROOT / "docs/CACHE.md", Path(__file__).with_name("README.md")] paths.extend([ROOT / "docs/ROW_COVERAGE.md", ROOT / "docs/releases/v0.9.0.md", ROOT / "docs/WHAT_IF.md", ROOT / "docs/releases/v0.10.0.md"]) + paths.extend(ROOT / name for name in ( + "SECURITY.md", "CHANGELOG.md", "docs/NATIVE_EXPORT.md", "docs/ROUTING.md", + "docs/INSPECT.md", "docs/COMPARISON.md", "docs/releases/v0.11.0.md", + "docs/releases/v0.12.0.md", "examples/litellm-spend/TESTING.md")) for path in paths: for link in re.findall(r"\]\(([^)\s]+)\)", path.read_text()): parsed = urlsplit(link) diff --git a/scripts/install.sh b/scripts/install.sh index c424a80..19d79bf 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -6,11 +6,11 @@ umask 077 usage() { printf '%s\n' 'Usage: sh scripts/install.sh [VERSION [NEW_DIRECTORY]]' \ - 'Defaults: v0.11.0 ./fleetdiff-install. Requires curl, gh, shasum, and tar.' + 'Defaults: v0.12.0 ./fleetdiff-install. Requires curl, gh, shasum, and tar.' } case "${1:-}" in --help|-h) usage; exit 0 ;; esac if [ "$#" -gt 2 ]; then usage >&2; exit 2; fi -version=${1-v0.11.0} +version=${1-v0.12.0} destination=${2-./fleetdiff-install} case "$version" in ''|*[!a-zA-Z0-9.-]*) printf '%s\n' 'Supply a release version such as v0.3.0.' >&2; exit 2 ;; diff --git a/scripts/install_test.go b/scripts/install_test.go index 401cc60..2334105 100644 --- a/scripts/install_test.go +++ b/scripts/install_test.go @@ -15,15 +15,17 @@ import ( "testing" ) -const installTestVersion = "v0.11.0" +const installTestVersion = "v0.12.0" +const installTestURL = "https://raw.githubusercontent.com/llm-measurement/fleetdiff/65990e302913fb960db3ba3a53cb4633bef63117/scripts/install.sh" func TestInstallDocumentationVersionPins(t *testing.T) { parts := strings.Split(strings.TrimPrefix(installTestVersion, "v"), ".") for path, want := range map[string][]string{ - "../README.md": {"### 2. Install " + installTestVersion, "sh -s -- " + installTestVersion, - "cmd/fleetdiff@" + installTestVersion, "docs/releases/" + installTestVersion + ".md"}, + "../README.md": {"### 1. Install " + installTestVersion, "sh -s -- " + installTestVersion, + "cmd/fleetdiff@" + installTestVersion, "docs/releases/" + installTestVersion + ".md", installTestURL, + "`gh` need not be signed in"}, "../SECURITY.md": {"`" + strings.Join(parts[:2], ".") + ".x` release line"}, - "../docs/OPERATIONS.md": {"version=" + installTestVersion, "sh scripts/install.sh " + installTestVersion}, + "../docs/OPERATIONS.md": {"version=" + installTestVersion, "sh scripts/install.sh " + installTestVersion, installTestURL}, "../docs/RELEASING.md": {"## After Publishing " + installTestVersion}, "../docs/WHAT_IF.md": {"Available in **" + installTestVersion + "**"}, "../docs/releases/" + installTestVersion + ".md": {"# " + installTestVersion + " Release Notes"}, @@ -41,6 +43,47 @@ func TestInstallDocumentationVersionPins(t *testing.T) { } } +func TestDocumentedReadmeInstall(t *testing.T) { + doc, err := os.ReadFile("../README.md") + if err != nil { + t.Fatal(err) + } + _, rest, ok := strings.Cut(string(doc), "```sh\n") + if !ok { + t.Fatal("missing README install example") + } + block, _, ok := strings.Cut(rest, "\n```") + if !ok { + t.Fatal("unterminated README install example") + } + script, err := os.ReadFile("install.sh") + if err != nil { + t.Fatal(err) + } + for _, failure := range []string{"", "download", "archive", "manifest", "checksum", "missing"} { + t.Run("failure="+failure, func(t *testing.T) { + root, env := installEnvironment(t, "Darwin", "arm64", "darwin_arm64", failure) + installer := filepath.Join(root, "reviewed-install.sh") + // The reviewed pin has an older default; the README must select its version explicitly. + writeTestFile(t, installer, strings.ReplaceAll(string(script), installTestVersion, "v0.11.0"), 0600) + env = append(env, "TEST_INSTALLER_URL="+installTestURL, "TEST_INSTALLER="+installer) + cmd := exec.Command("bash", "-c", block) + cmd.Dir, cmd.Env = root, env + output, err := cmd.CombinedOutput() + if (err != nil) != (failure != "") { + t.Fatalf("README install example: %v\n%s", err, output) + } + _, statErr := os.Stat(filepath.Join(root, "fleetdiff-install", "fleetdiff")) + if failure == "" && statErr != nil { + t.Fatal(statErr) + } + if failure != "" && !os.IsNotExist(statErr) { + t.Fatalf("README extracts on failed checks: %v", statErr) + } + }) + } +} + func TestInstallDefaultVersion(t *testing.T) { root, env := installEnvironment(t, "Darwin", "arm64", "darwin_arm64", "") script, err := filepath.Abs("install.sh") @@ -223,7 +266,11 @@ func installEnvironment(t *testing.T, system, arch, target, failure string) (str set -eu [ "$TEST_FAILURE" != download ] || exit 1 for url do :; done -case "$url" in "https://github.com/llm-measurement/fleetdiff/releases/download/$TEST_VERSION/"*) ;; *) exit 1 ;; esac +case "$url" in + "${TEST_INSTALLER_URL:-}") cat "$TEST_INSTALLER"; exit 0 ;; + "https://github.com/llm-measurement/fleetdiff/releases/download/$TEST_VERSION/"*) ;; + *) exit 1 ;; +esac asset=${url##*/} if [ "$TEST_FAILURE" = missing ] && [ "$asset" = "$TEST_ARCHIVE" ]; then exit 0; fi cp "$TEST_ASSETS/$asset" "$asset" From e84a9fb6f4379433cc5ad0acb5a272a6672493c4 Mon Sep 17 00:00:00 2001 From: Vijay Date: Sun, 18 Oct 2026 14:49:39 -0400 Subject: [PATCH 3/3] docs: refresh calibration inventory for release links Signed-off-by: Vijay --- docs/calibration/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/calibration/manifest.json b/docs/calibration/manifest.json index 5f29422..8cd4184 100644 --- a/docs/calibration/manifest.json +++ b/docs/calibration/manifest.json @@ -2,7 +2,7 @@ "schema": "fleetdiff-calibration-publication/v1", "sha256": { ".github/workflows/ci.yml": "41b15ed949e951cfd799238db5ba3ade092e51ec20644aad7c68edfcf9424be2", - "docs/WHAT_IF.md": "bf5be3a7eade73aabc3d1492b7408eeb051e09f5b2a2abaa82507c5dbbccb5c6", + "docs/WHAT_IF.md": "29ca0eee0f40b968222e4dd4fc26670ffb0d02f0684b9b4b2b751668c3479449", "docs/WHAT_IF_CALIBRATION.md": "9e929d6793a8f243950896b7c4672b732e9579687e2ac40dfed9af38f44920da", "docs/calibration/LICENSES.md": "fce7715504d899724474f0519b77ab0ddc5fabe95278ddb437c200b2ed2fdd7d", "docs/calibration/README.md": "d6feb2af08b3fef21f1eca1f18e096cf9c15c8dc1e44c79d3fc5c44ed409c5f7",