From 8b1055bc42017d47169303f9c294ed0394dacd40 Mon Sep 17 00:00:00 2001 From: Pablo Ontiveros Date: Thu, 15 Oct 2026 11:09:16 -0600 Subject: [PATCH] feat(search)!: adopt truthful lexical-only contract --- .coverage-floors.toml | 2 - AGENTS.md | 16 +- README.md | 9 +- cmd/backscroll/backscroll_skill.md | 11 +- cmd/backscroll/codex_e2e_test.go | 4 +- cmd/backscroll/codex_lifecycle_test.go | 2 +- cmd/backscroll/codex_wrapper_e2e_test.go | 2 +- .../echo_shell_zero_query_gap_test.go | 2 +- cmd/backscroll/echo_zero_replay_e2e_test.go | 2 +- cmd/backscroll/legacy_sources_test.go | 2 +- cmd/backscroll/main_test.go | 40 +- cmd/backscroll/omp_e2e_test.go | 4 +- cmd/backscroll/output_failure_test.go | 4 +- cmd/backscroll/preflight_contract_test.go | 8 +- cmd/backscroll/preflight_numbers.go | 4 +- cmd/backscroll/project_proof_test.go | 2 +- cmd/backscroll/search.go | 77 ++- .../search_context_contract_test.go | 2 +- .../search_relaxation_echo_idf_e2e_test.go | 2 +- cmd/backscroll/search_relaxation_test.go | 6 +- cmd/backscroll/search_test.go | 2 +- cmd/backscroll/testdata/required_output.json | 24 +- docs/input-contract.md | 4 +- docs/search.md | 13 +- internal/chunking/chunking.go | 150 ------ internal/chunking/chunking_test.go | 206 -------- internal/config/config.go | 54 +-- internal/config/config_test.go | 87 ---- internal/embedding/mock.go | 42 -- internal/embedding/mock_test.go | 87 ---- internal/embedding/onnx_provider.go | 47 -- internal/embedding/onnx_provider_test.go | 43 -- internal/embedding/provider.go | 15 - internal/models/models.go | 24 +- internal/models/models_test.go | 44 +- internal/services/query.go | 4 +- internal/services/query_test.go | 1 - internal/storage/hybrid.go | 183 ------- internal/storage/query_cancellation_test.go | 164 ------- .../storage/query_failure_contract_test.go | 2 +- internal/storage/storage.go | 6 +- internal/storage/unit_test.go | 448 +----------------- 42 files changed, 146 insertions(+), 1705 deletions(-) delete mode 100644 internal/chunking/chunking.go delete mode 100644 internal/chunking/chunking_test.go delete mode 100644 internal/embedding/mock.go delete mode 100644 internal/embedding/mock_test.go delete mode 100644 internal/embedding/onnx_provider.go delete mode 100644 internal/embedding/onnx_provider_test.go delete mode 100644 internal/embedding/provider.go delete mode 100644 internal/storage/hybrid.go diff --git a/.coverage-floors.toml b/.coverage-floors.toml index 4d50f797..e4ea50dd 100644 --- a/.coverage-floors.toml +++ b/.coverage-floors.toml @@ -2,9 +2,7 @@ default = 85 packages = [ "cmd/backscroll", "internal/autoupdate", - "internal/chunking", "internal/config", - "internal/embedding", "internal/hybrid", "internal/input_config", "internal/models", diff --git a/AGENTS.md b/AGENTS.md index ada87e52..232812ea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,7 @@ Backscroll is a Go CLI tool that indexes Claude Code, Pi, OMP, OpenCode, Codex C **Status**: Go port complete — `main` branch is the active Go implementation. The Rust implementation is frozen in the `v0` branch. - Implemented: `internal/autoupdate`, `internal/config`, `internal/input_config`, `internal/models`, `internal/readers`, `internal/sync`, `internal/tagging`, `internal/plans`, `internal/sources`, `internal/storage`, `internal/services`, `internal/projects`, `internal/templates`, `internal/corrections`, `internal/categories`, `internal/sequences`, `internal/chunking`, `internal/embedding`, `internal/hybrid`, `internal/compat`, `internal/recovery`, `internal/startuplock`. CLI commands in `cmd/backscroll/` (10 v2 commands via cobra). + Implemented: `internal/autoupdate`, `internal/config`, `internal/input_config`, `internal/models`, `internal/readers`, `internal/sync`, `internal/tagging`, `internal/plans`, `internal/sources`, `internal/storage`, `internal/services`, `internal/projects`, `internal/templates`, `internal/corrections`, `internal/categories`, `internal/sequences`, `internal/hybrid`, `internal/compat`, `internal/recovery`, `internal/startuplock`. CLI commands in `cmd/backscroll/` (10 v2 commands via cobra). github.com/pablontiv/backscroll/internal/autoupdate — Private context-aware release staging and safe replacement Stack: cobra, go-toml/v2, goldmark, modernc.org/sqlite (pure Go, no CGO), stdlib testing. @@ -43,7 +43,7 @@ Tests use stdlib `testing` + subprocess or direct `run()` invocation. Unit tests cmd/backscroll/ ├── main.go — entrypoint; run(stdout, stderr, args) for testability ├── list.go — list command (v2: --project, --all-projects, --recent N, --order, --limit, --offset, --json, --robot) -├── search.go — search command (v2: --text, --project, --all-projects, --source, --source-path, --after, --before, --role, --content-type, --tag, --fields, --max-tokens, --lexical-only, --relax, --similarity-threshold, --json, --robot) +├── search.go — search command (v2: --text, --project, --all-projects, --source, --source-path, --after, --before, --role, --content-type, --tag, --fields, --max-tokens, --relax, --json, --robot) ├── context.go — exact indexed neighborhood command (--uuid or exact --source-path + --ordinal, --before, --after, --max-tokens, --json, --robot) ├── patterns.go — patterns command (v2: --kind commands|failures|templates|sequences|corrections [--origin human|assistant|system|automation|unknown] [--pending] [--batch N] [--trend], --project, --all-projects, --tag, --min-support, --min-confidence, --min-length, --max-length, --json, --robot) ├── annotate.go — annotate command (F3b: --uuid --kind --label; validates message existence; upsert semantics) @@ -71,9 +71,7 @@ internal/ ├── templates/ — F2 Drain-inspired miner: Miner, ProcessLine, ExtractErrorLines, deterministic signature via SHA256 ├── corrections/ — F3 correction detection: bilingual lexicon, interrupt flags, denial heuristics, rephrase-similarity; detector registry + implementations ├── categories/ — F4 category map loader (rule engine, versioning, tool→category mapping) with embedded default preset -├── chunking/ — Token-aware text chunking helpers for embedding preparation -├── embedding/ — Embedding provider interfaces, mock provider, and ONNX provider implementation -├── hybrid/ — Reciprocal Rank Fusion helpers for merged lexical/vector retrieval +├── hybrid/ — Reciprocal Rank Fusion helpers for merged lexical retrieval ├── sequences/ — F4 PrefixSpan mining (deterministic discovery of frequent tool-call sequences per session) ├── services/ — Concrete typed read-only query orchestration over storage └── storage/ — SQLite adapter (dual FTS5 indexes: tool_fts + messages_fts, BM25, WAL mode, migrations v1–v17, search_items, session_tags, tool_events, message_templates, template_matches, correction_signals, annotations, AggregateCommands, AggregateFailures, AggregateTemplates, AggregateCorrections, UpsertAnnotation, LoadToolSequences) @@ -81,7 +79,7 @@ scripts/ └── recall-eval/ — isolated legacy/synthetic recall evaluator and cohort reporter ``` -Ten v2 CLI commands: `list [--project] [--all-projects] [--recent N] [--order timestamp:desc|asc] [--limit] [--offset] [--json] [--robot]`, `search [--text ] [--project] [--all-projects] [--source] [--source-path] [--after] [--before] [--role] [--content-type] [--tag] [--limit] [--offset] [--fields minimal|full] [--max-tokens N] [--lexical-only] [--relax] [--similarity-threshold F] [--json] [--robot]`, `context (--uuid | --source-path --ordinal ) [--before 0..50] [--after 0..50] [--max-tokens 64..16384] [--json] [--robot]`, `patterns --kind commands|failures|templates|sequences|corrections [--origin human|assistant|system|automation|unknown] [--pending] [--batch N] [--project] [--all-projects] [--tag] [--trend] [--after] [--before] [--min-support N] [--min-confidence F] [--min-length N] [--max-length N] [--limit] [--offset] [--json] [--robot]`, `annotate --uuid --kind --label [--path

--ordinal ]`, `recover --from [--dry-run]`, `status [--json]`, `validate [--json]`, `rebuild`, `config [--json]`. +Ten v2 CLI commands: `list [--project] [--all-projects] [--recent N] [--order timestamp:desc|asc] [--limit] [--offset] [--json] [--robot]`, `search [--text ] [--project] [--all-projects] [--source] [--source-path] [--after] [--before] [--role] [--content-type] [--tag] [--limit] [--offset] [--fields minimal|full] [--max-tokens N] [--relax] [--json] [--robot]`, `context (--uuid | --source-path --ordinal ) [--before 0..50] [--after 0..50] [--max-tokens 64..16384] [--json] [--robot]`, `patterns --kind commands|failures|templates|sequences|corrections [--origin human|assistant|system|automation|unknown] [--pending] [--batch N] [--project] [--all-projects] [--tag] [--trend] [--after] [--before] [--min-support N] [--min-confidence F] [--min-length N] [--max-length N] [--limit] [--offset] [--json] [--robot]`, `annotate --uuid --kind --label [--path

--ordinal ]`, `recover --from [--dry-run]`, `status [--json]`, `validate [--json]`, `rebuild`, `config [--json]`. The `SearchEngine` interface is the port; `internal/storage` is the adapter. Database opened lazily. `OpenReadOnly()` provides read-only access for external consumers. @@ -255,16 +253,14 @@ github.com/pablontiv/backscroll/internal/sources — External source parse github.com/pablontiv/backscroll/internal/templates — F2 Drain-inspired miner: Miner, ProcessLine, ExtractErrorLines github.com/pablontiv/backscroll/internal/corrections — F3 correction-signal detectors: lexicon (es+en), interrupt, denial, rephrase-similarity; registry pattern for deterministic, pluggable detectors github.com/pablontiv/backscroll/internal/categories — F4 category map loader (rule engine, versioning, tool→category mapping) with embedded default preset -github.com/pablontiv/backscroll/internal/chunking — Token-aware text chunking helpers for embedding preparation -github.com/pablontiv/backscroll/internal/embedding — Embedding provider interface, mock provider, and ONNX provider implementation -github.com/pablontiv/backscroll/internal/hybrid — Reciprocal Rank Fusion helpers for merged lexical/vector retrieval +github.com/pablontiv/backscroll/internal/hybrid — Reciprocal Rank Fusion helpers for merged lexical retrieval github.com/pablontiv/backscroll/internal/sequences — F4 PrefixSpan mining (deterministic pattern discovery per session) github.com/pablontiv/backscroll/internal/services — Concrete typed read-only query orchestration over storage github.com/pablontiv/backscroll/internal/storage — Database schema, migrations v1–v17, FTS5 indexes github.com/pablontiv/backscroll/internal/projects — Project identity registry github.com/pablontiv/backscroll/internal/readers — SessionReader interface, Registry, ClaudeReader (text+tool_use+tool_result), PiReader (text+toolCall+custom results), OMPReader (native v3 text+thinking+toolCall+toolResult; parser-backed identities and origin), OpenCodeReader (text+tool state.input+state.output), CodexReader (rollout response items; docs/input-contract.md), MarkdownDocumentReader (`markdown_document`), MarkdownSectionsReader (`markdown_sections`); toolfmt serializer github.com/pablontiv/backscroll/internal/recovery — Stranded database recovery orchestration, durable backup, atomic replacement, and post-install sync - Implemented: `internal/autoupdate`, `internal/config`, `internal/input_config`, `internal/models`, `internal/readers`, `internal/sync`, `internal/tagging`, `internal/plans`, `internal/sources`, `internal/storage`, `internal/services`, `internal/projects`, `internal/templates`, `internal/corrections`, `internal/categories`, `internal/sequences`, `internal/chunking`, `internal/embedding`, `internal/hybrid`, `internal/compat`, `internal/recovery`, `internal/startuplock`. CLI commands in `cmd/backscroll/` (10 v2 commands via cobra). + Implemented: `internal/autoupdate`, `internal/config`, `internal/input_config`, `internal/models`, `internal/readers`, `internal/sync`, `internal/tagging`, `internal/plans`, `internal/sources`, `internal/storage`, `internal/services`, `internal/projects`, `internal/templates`, `internal/corrections`, `internal/categories`, `internal/sequences`, `internal/hybrid`, `internal/compat`, `internal/recovery`, `internal/startuplock`. CLI commands in `cmd/backscroll/` (10 v2 commands via cobra). github.com/pablontiv/backscroll/internal/autoupdate — Private context-aware release staging and safe replacement github.com/pablontiv/backscroll/internal/startuplock — Canonical database lock coordination via persistent OS advisory sidecar github.com/pablontiv/backscroll/scripts/recall-eval — Isolated recall evaluation runner and cohort reporter diff --git a/README.md b/README.md index e5d3a56b..8fc535d7 100644 --- a/README.md +++ b/README.md @@ -176,11 +176,10 @@ A context UUID is opaque and is never synthesized for a result that lacks one. T Filters worth knowing: `--after` / `--before` for a date window, `--tag` for auto-detected session categories (debugging, refactoring, testing…), `--source-path` to pin one stored input path, `--source` to keep one source class, and `--role` to filter the stored role field. The role field is not a semantic-origin guarantee. -`--lexical-only` explicitly selects BM25-only retrieval. The CLI currently has no -wired embedding provider, so ordinary search also falls back to lexical retrieval. -`--similarity-threshold F` accepts a finite value in `0..1` (default `0.3`, -`0` disables a vector threshold), but applies only to vector results; it does not -filter BM25 results or `--relax`. These flags do not enable semantic expansion. +Search is lexical: prose uses Porter-stemmed BM25, tool text uses trigram BM25, +and unfiltered searches fuse those two lexical rankings with RRF. `--relax` +optionally drops low-IDF terms after a strict zero-result search without changing +scope or introducing semantic expansion. ### Discover what recurs diff --git a/cmd/backscroll/backscroll_skill.md b/cmd/backscroll/backscroll_skill.md index 77ab79aa..01967a03 100644 --- a/cmd/backscroll/backscroll_skill.md +++ b/cmd/backscroll/backscroll_skill.md @@ -59,7 +59,7 @@ Input manifests are loaded only from: where `` is non-empty `BACKSCROLL_CONFIG_DIR`, otherwise `$HOME/.config` on every OS (Windows: `\.config`). `XDG_CONFIG_HOME`, macOS Application Support, and `%APPDATA%` do not select this directory. The app config file is for -database and embedding settings, not ingestion sources. +database settings, not ingestion sources. Inspect the effective database, legacy session directories, and active inputs: @@ -125,11 +125,10 @@ If output is truncated, treat it as evidence that more indexed data exists. Refi For explicit lexical term-overload recall, add `--relax` (for example `backscroll search --text '+violet handshake technique adaptation' --relax --robot --fields minimal --max-tokens 2000`); `+term`/quoted phrases stay protected, scope never widens, and relaxed results report `match_stage`/`dropped_terms`—this does not recover missing vocabulary. -`--lexical-only` explicitly requests BM25-only retrieval. The CLI currently has -no wired embedding provider and falls back to lexical retrieval even without -that flag. `--similarity-threshold F` accepts finite `0..1` (default `0.3`; -`0` disables the vector threshold) but applies only to vector results, not BM25 -or `--relax`. It does not enable semantic expansion. +Search is always lexical: prose uses Porter-stemmed BM25, tool text uses trigram +BM25, and unfiltered searches fuse those lexical rankings with RRF. `--relax` +only drops low-IDF terms after a strict zero-result search; it never performs +semantic expansion. ## 5) Query patterns by use case diff --git a/cmd/backscroll/codex_e2e_test.go b/cmd/backscroll/codex_e2e_test.go index 5f348b55..411c7636 100644 --- a/cmd/backscroll/codex_e2e_test.go +++ b/cmd/backscroll/codex_e2e_test.go @@ -52,7 +52,7 @@ format = "codex" {"codexafterquartz", "assistant", "text"}, } { t.Run(tc.marker, func(t *testing.T) { - out, stderr, err := runCmd("search", tc.marker, "--project", "codex-fixture-project", "--source", "session", "--source-path", source, "--role", tc.role, "--content-type", tc.kind, "--after", "2026-09-01", "--before", "2026-09-02", "--lexical-only", "--json") + out, stderr, err := runCmd("search", tc.marker, "--project", "codex-fixture-project", "--source", "session", "--source-path", source, "--role", tc.role, "--content-type", tc.kind, "--after", "2026-09-01", "--before", "2026-09-02", "--json") if err != nil { t.Fatalf("search: %v stderr=%s", err, stderr) } @@ -69,7 +69,7 @@ format = "codex" "codexignoredinstructions", "codexignoredimage", "codexignoredencrypted", "codexignoredcompact", "codexignoredfuture", "codexignoredmalformed", "codexreasonquartz", } { - out, stderr, err := runCmd("search", marker, "--all-projects", "--lexical-only", "--json") + out, stderr, err := runCmd("search", marker, "--all-projects", "--json") if err != nil { t.Fatalf("negative search: %v %s", err, stderr) } diff --git a/cmd/backscroll/codex_lifecycle_test.go b/cmd/backscroll/codex_lifecycle_test.go index 92f64dab..b9821acb 100644 --- a/cmd/backscroll/codex_lifecycle_test.go +++ b/cmd/backscroll/codex_lifecycle_test.go @@ -42,7 +42,7 @@ func TestCodexPresetIncrementalAndPerennialRecall(t *testing.T) { } recall := func(marker string) { t.Helper() - out, stderr, err := runCmd("search", marker, "--all-projects", "--lexical-only", "--json") + out, stderr, err := runCmd("search", marker, "--all-projects", "--json") if err != nil { t.Fatalf("recall %s: %v %s", marker, err, stderr) } diff --git a/cmd/backscroll/codex_wrapper_e2e_test.go b/cmd/backscroll/codex_wrapper_e2e_test.go index fb0cb485..e566f5c3 100644 --- a/cmd/backscroll/codex_wrapper_e2e_test.go +++ b/cmd/backscroll/codex_wrapper_e2e_test.go @@ -46,7 +46,7 @@ func TestCodexInjectedWrappersV1E2E(t *testing.T) { {"codexblockkeepquartz", "user", 1}, } { t.Run(tc.marker, func(t *testing.T) { - out, stderr, err := runCmd("search", tc.marker, "--all-projects", "--source-path", sourcePath, "--role", tc.role, "--content-type", "text", "--lexical-only", "--json") + out, stderr, err := runCmd("search", tc.marker, "--all-projects", "--source-path", sourcePath, "--role", tc.role, "--content-type", "text", "--json") if err != nil { t.Fatalf("search: %v stderr=%s", err, stderr) } diff --git a/cmd/backscroll/echo_shell_zero_query_gap_test.go b/cmd/backscroll/echo_shell_zero_query_gap_test.go index c421da72..c4336aa1 100644 --- a/cmd/backscroll/echo_shell_zero_query_gap_test.go +++ b/cmd/backscroll/echo_shell_zero_query_gap_test.go @@ -124,7 +124,7 @@ func TestZeroValuedCodexShellEchoExcludedBeforeReplay(t *testing.T) { } } - out := e.run("search", "--text", "violet handshake adaptation", "--all-projects", "--lexical-only", "--relax", "--robot", "--fields", "minimal", "--max-tokens", "200") + out := e.run("search", "--text", "violet handshake adaptation", "--all-projects", "--relax", "--robot", "--fields", "minimal", "--max-tokens", "200") if !strings.Contains(out, "result_0_filepath="+filepath.Join(e.fixtures, "target.jsonl")+"\n") || !strings.Contains(out, `result_0_dropped_terms=["adaptation"]`) { t.Errorf("zero-valued shell query echoes inverted unfiltered --relax IDF\nstdout=%s", out) } diff --git a/cmd/backscroll/echo_zero_replay_e2e_test.go b/cmd/backscroll/echo_zero_replay_e2e_test.go index 2946f8c0..30ed16a8 100644 --- a/cmd/backscroll/echo_zero_replay_e2e_test.go +++ b/cmd/backscroll/echo_zero_replay_e2e_test.go @@ -64,7 +64,7 @@ func TestZeroValuedCodexExecCommandEchoExcludedBeforeReplay(t *testing.T) { t.Errorf("zero-valued Codex fallback leaks=%d want %d: %v", codexLeaks, wantCodexLeaks, queryEchoShape(got)) } - out := e.run("search", "--text", "violet handshake adaptation", "--all-projects", "--lexical-only", "--relax", "--robot", "--fields", "minimal", "--max-tokens", "200") + out := e.run("search", "--text", "violet handshake adaptation", "--all-projects", "--relax", "--robot", "--fields", "minimal", "--max-tokens", "200") if !strings.Contains(out, "result_0_filepath="+filepath.Join(e.fixtures, "target.jsonl")+"\n") || !strings.Contains(out, `result_0_dropped_terms=["adaptation"]`) { t.Errorf("zero-valued query echoes changed unfiltered --relax IDF\nstdout=%s", out) } diff --git a/cmd/backscroll/legacy_sources_test.go b/cmd/backscroll/legacy_sources_test.go index 96f974f5..1a120339 100644 --- a/cmd/backscroll/legacy_sources_test.go +++ b/cmd/backscroll/legacy_sources_test.go @@ -88,7 +88,7 @@ func TestLegacySourcesAllowsHelpAndVersion(t *testing.T) { }{ {name: "root help", args: []string{"--help"}, want: "Available Commands:"}, {name: "root version", args: []string{"--version"}, want: "backscroll"}, - {name: "subcommand help", args: []string{"search", "--help"}, want: "Search performs a hybrid search"}, + {name: "subcommand help", args: []string{"search", "--help"}, want: "Search performs lexical full-text search"}, } for _, tt := range tests { diff --git a/cmd/backscroll/main_test.go b/cmd/backscroll/main_test.go index 31088447..f26cf3bb 100644 --- a/cmd/backscroll/main_test.go +++ b/cmd/backscroll/main_test.go @@ -471,40 +471,22 @@ func TestSearchHelp(t *testing.T) { if err != nil { t.Fatalf("search --help error: %v", err) } - for _, flag := range []string{"--project", "--source", "--after", "--before", "--role", "--limit", "--json", "--robot", "--lexical-only", "--similarity-threshold"} { + for _, flag := range []string{"--project", "--source", "--after", "--before", "--role", "--limit", "--json", "--robot", "--relax"} { if !strings.Contains(out, flag) { t.Errorf("search --help missing flag %q", flag) } } } -func TestSearchLexicalOnly(t *testing.T) { - _, cleanup := testEnv(t) - defer cleanup() - - fixture := filepath.Join(fixturesDir(), "claude.jsonl") - _, _, _ = syncForTest(t, "sync", "--path", filepath.Dir(fixture)) - - out, _, err := runCmd("search", "--lexical-only", "session") - if err != nil { - t.Fatalf("search --lexical-only error: %v", err) - } - _ = out -} - -func TestSearchSimilarityThreshold(t *testing.T) { - _, cleanup := testEnv(t) - defer cleanup() - - fixture := filepath.Join(fixturesDir(), "claude.jsonl") - _, _, _ = syncForTest(t, "sync", "--path", filepath.Dir(fixture)) - - // With no vectors in DB, falls back to BM25 regardless of threshold - out, _, err := runCmd("search", "--similarity-threshold", "0.5", "session") - if err != nil { - t.Fatalf("search --similarity-threshold error: %v", err) +func TestSearchRejectsRemovedSemanticFlags(t *testing.T) { + for _, flag := range []string{"--lexical-only", "--similarity-threshold"} { + t.Run(flag, func(t *testing.T) { + _, _, err := runCmd("search", "session", flag) + if err == nil || !strings.Contains(err.Error(), "unknown flag: "+flag) { + t.Fatalf("search %s error = %v, want unknown flag", flag, err) + } + }) } - _ = out } func TestListTextSynced(t *testing.T) { @@ -1359,7 +1341,7 @@ func TestSearchFindsPionRecordTypeAfterEmptyIndex(t *testing.T) { t.Fatalf("close seeded index: %v", err) } - out, _, err := runCmd("search", "pion_recordtype_search_token", "--all-projects", "--lexical-only") + out, _, err := runCmd("search", "pion_recordtype_search_token", "--all-projects") if err != nil { t.Fatalf("search Pion recordType: %v", err) } @@ -1440,7 +1422,7 @@ func TestPiPionIngestionRegressionMatrixE2E(t *testing.T) { searchCount := func(query string) int { t.Helper() - out, stderr, err := runCmd("search", "--text", query, "--all-projects", "--lexical-only", "--json") + out, stderr, err := runCmd("search", "--text", query, "--all-projects", "--json") if err != nil { t.Fatalf("search %q: %v\nstdout=%s\nstderr=%s", query, err, out, stderr) } diff --git a/cmd/backscroll/omp_e2e_test.go b/cmd/backscroll/omp_e2e_test.go index c6bcfd75..52d19bb4 100644 --- a/cmd/backscroll/omp_e2e_test.go +++ b/cmd/backscroll/omp_e2e_test.go @@ -37,7 +37,7 @@ func installOMPFixture(t *testing.T, reasoning bool) (string, string, []byte) { func ompRecall(t *testing.T, marker string, extra ...string) []map[string]any { t.Helper() - args := append([]string{"search", "--text", marker, "--all-projects", "--lexical-only", "--json"}, extra...) + args := append([]string{"search", "--text", marker, "--all-projects", "--json"}, extra...) out, stderr, err := runCmd(args...) if err != nil { t.Fatalf("recall %s: %v %s", marker, err, stderr) @@ -70,7 +70,7 @@ func TestOMPV3E2E(t *testing.T) { {"ompagentquartz", "user", "text"}, {"ompunknownquartz", "user", "text"}, {"ompbranchquartz", "assistant", "text"}, } { // Project identity comes from the header cwd, never the bucket directory. - out, stderr, err := runCmd("search", "--text", tc.marker, "--project", "omp-fixture-project", "--source", "session", "--source-path", source, "--role", tc.role, "--content-type", tc.kind, "--after", "2026-09-01", "--before", "2026-09-02", "--lexical-only", "--json") + out, stderr, err := runCmd("search", "--text", tc.marker, "--project", "omp-fixture-project", "--source", "session", "--source-path", source, "--role", tc.role, "--content-type", tc.kind, "--after", "2026-09-01", "--before", "2026-09-02", "--json") if err != nil { t.Fatalf("%s: %v %s", tc.marker, err, stderr) } diff --git a/cmd/backscroll/output_failure_test.go b/cmd/backscroll/output_failure_test.go index 8e283725..a5246754 100644 --- a/cmd/backscroll/output_failure_test.go +++ b/cmd/backscroll/output_failure_test.go @@ -89,11 +89,11 @@ func TestRequiredOutputFailures(t *testing.T) { if populated { seedOutputBranches(t, path) } - commands := [][]string{{"config"}, {"status"}, {"validate"}, {"list", "--all-projects"}, {"search", "--text", "database", "--all-projects", "--lexical-only"}} + commands := [][]string{{"config"}, {"status"}, {"validate"}, {"list", "--all-projects"}, {"search", "--text", "database", "--all-projects"}} for _, kind := range []string{"commands", "failures", "templates", "corrections", "sequences"} { commands = append(commands, []string{"patterns", "--kind", kind, "--all-projects", "--min-support", "1"}) } - commands = append(commands, []string{"search", "--text", "database", "--all-projects", "--lexical-only", "--fields", "full"}, []string{"list", "--project", "absent"}, []string{"patterns", "--kind", "corrections", "--origin", "human", "--all-projects"}) + commands = append(commands, []string{"search", "--text", "database", "--all-projects", "--fields", "full"}, []string{"list", "--project", "absent"}, []string{"patterns", "--kind", "corrections", "--origin", "human", "--all-projects"}) for _, kind := range []string{"commands", "failures"} { commands = append(commands, []string{"patterns", "--kind", kind, "--all-projects", "--trend"}) } diff --git a/cmd/backscroll/preflight_contract_test.go b/cmd/backscroll/preflight_contract_test.go index 6a8b2153..78773e5c 100644 --- a/cmd/backscroll/preflight_contract_test.go +++ b/cmd/backscroll/preflight_contract_test.go @@ -36,10 +36,6 @@ func TestInvalidInvocationBeforeStartup(t *testing.T) { }{args, "all-projects"}) } for _, value := range []string{"NaN", "+Inf", "-Inf", "-0.1", "1.1"} { - cases = append(cases, struct { - args []string - want string - }{[]string{"search", "query", "--similarity-threshold", value}, "--similarity-threshold"}) cases = append(cases, struct { args []string want string @@ -112,8 +108,8 @@ func TestPreflightBoundariesPreservePayloads(t *testing.T) { args []string want string }{ - {[]string{"search", "needle", "--all-projects", "--similarity-threshold", "0", "--max-tokens", "0", "--json"}, "[]"}, - {[]string{"search", "--text", "needle", "--all-projects", "--similarity-threshold", "1", "--limit", "0", "--offset", "0", "--json"}, "[]"}, + {[]string{"search", "needle", "--all-projects", "--max-tokens", "0", "--json"}, "[]"}, + {[]string{"search", "--text", "needle", "--all-projects", "--limit", "0", "--offset", "0", "--json"}, "[]"}, {[]string{"patterns", "--kind", "sequences", "--all-projects", "--min-support", "0", "--min-length", "0", "--max-length", "0", "--limit", "0", "--batch", "0", "--json"}, `{"count":0,"kind":"sequences","patterns":[]}`}, {[]string{"patterns", "--kind", "corrections", "--all-projects", "--min-confidence", "0", "--json"}, `{"count":0,"kind":"corrections","patterns":[]}`}, {[]string{"patterns", "--kind", "corrections", "--all-projects", "--min-confidence", "1", "--json"}, `{"count":0,"kind":"corrections","patterns":[]}`}, diff --git a/cmd/backscroll/preflight_numbers.go b/cmd/backscroll/preflight_numbers.go index 699390ab..004c44d9 100644 --- a/cmd/backscroll/preflight_numbers.go +++ b/cmd/backscroll/preflight_numbers.go @@ -12,11 +12,11 @@ func validateUnitInterval(flag string, value float64) error { return nil } -func validateSearchNumbers(maxTokens int, threshold float64) error { +func validateSearchNumbers(maxTokens int) error { if maxTokens < 0 { return fmt.Errorf("--max-tokens must be >= 0") } - return validateUnitInterval("similarity-threshold", threshold) + return nil } func validateListNumbers(recent, limit, offset int) error { diff --git a/cmd/backscroll/project_proof_test.go b/cmd/backscroll/project_proof_test.go index a8ca54ab..6b019adc 100644 --- a/cmd/backscroll/project_proof_test.go +++ b/cmd/backscroll/project_proof_test.go @@ -22,7 +22,7 @@ func TestClaudeProjectProofSurvivesExpiredSourceRebuild(t *testing.T) { t.Fatal(err) } writeFile(t, filepath.Join(os.Getenv("BACKSCROLL_CONFIG_DIR"), "backscroll", "inputs", "claude.inputs.toml"), replaceManifestRoots(t, "inputs/claude.inputs.toml", string(preset), root)) - if _, stderr, err := runCmd("search", "--text", "nativeproofquartz", "--all-projects", "--lexical-only", "--json"); err != nil { + if _, stderr, err := runCmd("search", "--text", "nativeproofquartz", "--all-projects", "--json"); err != nil { t.Fatalf("%v %s", err, stderr) } read := func() (int64, string, string, string, string) { diff --git a/cmd/backscroll/search.go b/cmd/backscroll/search.go index 53040f60..3025bad4 100644 --- a/cmd/backscroll/search.go +++ b/cmd/backscroll/search.go @@ -20,33 +20,31 @@ import ( func newSearchCmd(stdout, stderr io.Writer) *cobra.Command { var ( - project string - allProjects bool - jsonFormat bool - robotFormat bool - source string - sourcePath string - after string - before string - role string - limit int - offset int - contentType string - tag string - fields string - maxTokens int - lexicalOnly bool - relax bool - similarityThreshold float64 - text string + project string + allProjects bool + jsonFormat bool + robotFormat bool + source string + sourcePath string + after string + before string + role string + limit int + offset int + contentType string + tag string + fields string + maxTokens int + relax bool + text string ) cmd := &cobra.Command{ Use: "search []", Short: "Full-text search indexed content", SilenceUsage: true, - Long: `Search performs a hybrid search (BM25 + vector embeddings with RRF fusion) -across all indexed sessions, plans, and external sources. + Long: `Search performs lexical full-text search (BM25) across the prose and +tool indexes for all indexed sessions, plans, and external sources. Use --text (v2 preferred) or positional (legacy) for search text. Use --project to filter to a single project (default: auto-detect from cwd). @@ -62,11 +60,11 @@ Use --fields to choose machine-readable detail: minimal (default) or full. Use --max-tokens to limit output size (approximate token count). Use --relax for opt-in lexical recall: after zero rows, drop low-IDF terms while keeping at least two unprotected terms. Leading +term and quoted phrases are -protected. All scope filters remain fixed; no OR or semantic expansion is used.`, +protected. All scope filters remain fixed.`, Args: func(cmd *cobra.Command, args []string) error { return validateCommandBeforeStartup(cmd, args, cobra.MaximumNArgs(1), func() error { query := searchQuery(text, args) - if err := validateSearchNumbers(maxTokens, similarityThreshold); err != nil { + if err := validateSearchNumbers(maxTokens); err != nil { return err } _, _, err := validateAndParseSearchRequest(query, fields, contentType, after, before, limit, offset, relax) @@ -88,7 +86,7 @@ protected. All scope filters remain fixed; no OR or semantic expansion is used.` } return runSearch(cmd.Context(), stdout, stderr, startup.Config, query, project, allProjects, invocationOutputMode(cmd), source, sourcePath, after, before, role, limit, offset, contentType, tag, - fields, maxTokens, lexicalOnly, similarityThreshold, relax) + fields, maxTokens, relax) }, } @@ -107,9 +105,7 @@ protected. All scope filters remain fixed; no OR or semantic expansion is used.` cmd.Flags().StringVar(&tag, "tag", "", "Filter sessions by tag") cmd.Flags().StringVar(&fields, "fields", "minimal", "Machine-readable fields to emit: minimal or full") cmd.Flags().IntVar(&maxTokens, "max-tokens", 0, "Max tokens in output (0=unlimited)") - cmd.Flags().BoolVar(&lexicalOnly, "lexical-only", false, "Use BM25 only, skip vector search") - cmd.Flags().BoolVar(&relax, "relax", false, "Lexical-only recall: after zero rows, drop low-IDF terms within the same scope (+term protects a term); ignores vector similarity thresholds") - cmd.Flags().Float64Var(&similarityThreshold, "similarity-threshold", 0.3, "Minimum cosine similarity for vector results (finite 0..1; 0=no threshold)") + cmd.Flags().BoolVar(&relax, "relax", false, "Lexical recall: after zero rows, drop low-IDF terms within the same scope (+term protects a term)") cmd.Flags().StringVar(&text, "text", "", "Search text (v2 preferred grammar)") cmd.MarkFlagsMutuallyExclusive("project", "all-projects") @@ -170,8 +166,7 @@ func runSearch(ctx context.Context, stdout, stderr io.Writer, cfg *config.Config project string, allProjects bool, mode outputMode, source, sourcePath, after, before, role string, limit, offset int, contentType, tag string, - fields string, maxTokens int, - lexicalOnly bool, similarityThreshold float64, relax bool) (retErr error) { + fields string, maxTokens int, relax bool) (retErr error) { afterTime, beforeTime, err := validateAndParseSearchRequest(query, fields, contentType, after, before, limit, offset, relax) if err != nil { @@ -196,19 +191,17 @@ func runSearch(ctx context.Context, stdout, stderr io.Writer, cfg *config.Config // Build search options opts := models.SearchOptions{ - Project: project, - AllProjects: allProjects, - Source: source, - SourcePath: sourcePath, - After: afterTime, - Before: beforeTime, - Role: role, - Limit: limit, - Offset: offset, - ContentType: contentType, - Tag: tag, - LexicalOnly: lexicalOnly, - SimilarityThreshold: similarityThreshold, + Project: project, + AllProjects: allProjects, + Source: source, + SourcePath: sourcePath, + After: afterTime, + Before: beforeTime, + Role: role, + Limit: limit, + Offset: offset, + ContentType: contentType, + Tag: tag, } queryService := services.QueryService{DB: db} diff --git a/cmd/backscroll/search_context_contract_test.go b/cmd/backscroll/search_context_contract_test.go index 2def462e..f083e146 100644 --- a/cmd/backscroll/search_context_contract_test.go +++ b/cmd/backscroll/search_context_contract_test.go @@ -248,7 +248,7 @@ func runSearchContract(t *testing.T, cfg *config.Config, query, fields string, j err := runSearch(context.Background(), &stdout, &stderr, cfg, query, "", true, mode, "", "", "", "", "", 20, 0, "text", "", - fields, 0, true, 0.3, false) + fields, 0, false) if err != nil { t.Fatalf("run search %q: %v\nstderr: %s", query, err, stderr.String()) } diff --git a/cmd/backscroll/search_relaxation_echo_idf_e2e_test.go b/cmd/backscroll/search_relaxation_echo_idf_e2e_test.go index 340abad5..68966000 100644 --- a/cmd/backscroll/search_relaxation_echo_idf_e2e_test.go +++ b/cmd/backscroll/search_relaxation_echo_idf_e2e_test.go @@ -44,7 +44,7 @@ func TestSearchRelaxationQueryEchoesDoNotInvertIDF(t *testing.T) { t.Fatal(err) } - control := []string{"search", "--text", "violet handshake adaptation", "--all-projects", "--lexical-only", "--robot", "--fields", "minimal", "--max-tokens", "200"} + control := []string{"search", "--text", "violet handshake adaptation", "--all-projects", "--robot", "--fields", "minimal", "--max-tokens", "200"} strict, stderr, err := runCmd(control...) if err != nil || strict != "" { t.Fatalf("strict must miss: err=%v stdout=%s stderr=%s", err, strict, stderr) diff --git a/cmd/backscroll/search_relaxation_test.go b/cmd/backscroll/search_relaxation_test.go index ca1b9655..9d9e4921 100644 --- a/cmd/backscroll/search_relaxation_test.go +++ b/cmd/backscroll/search_relaxation_test.go @@ -39,7 +39,7 @@ func TestSearchRelaxationRecoverableE2E(t *testing.T) { if err := os.WriteFile(filepath.Join(inputs, "relaxation.inputs.toml"), []byte(manifest), 0600); err != nil { t.Fatal(err) } - args := []string{"search", "--text", "violet handshake adaptation", "--all-projects", "--lexical-only", "--robot", "--fields", "minimal", "--max-tokens", "200"} + args := []string{"search", "--text", "violet handshake adaptation", "--all-projects", "--robot", "--fields", "minimal", "--max-tokens", "200"} strict, stderr, err := runCmd(args...) if err != nil || strict != "" { t.Fatalf("strict must miss target: err=%v stdout=%s stderr=%s", err, strict, stderr) @@ -54,7 +54,7 @@ func TestSearchRelaxationRecoverableE2E(t *testing.T) { if !strings.Contains(out, "result_0_match_stage=drop-terms\n") || !strings.Contains(out, "result_0_dropped_terms=[\"adaptation\"]\n") { t.Fatalf("relaxed target must carry provenance: %s", out) } - positive := []string{"search", "--text", "violet handshake", "--all-projects", "--lexical-only", "--robot", "--fields", "minimal"} + positive := []string{"search", "--text", "violet handshake", "--all-projects", "--robot", "--fields", "minimal"} before, _, err := runCmd(positive...) if err != nil { t.Fatal(err) @@ -91,7 +91,7 @@ func TestSearchRelaxationRecoverableE2E(t *testing.T) { } } for _, format := range [][]string{{}, {"--robot"}, {"--json"}, {"--json", "--fields", "full"}} { - base := append([]string{"search", "--text", "violet handshake", "--all-projects", "--lexical-only"}, format...) + base := append([]string{"search", "--text", "violet handshake", "--all-projects"}, format...) before, beforeErr, err := runCmd(base...) if err != nil { t.Fatal(err) diff --git a/cmd/backscroll/search_test.go b/cmd/backscroll/search_test.go index bd6692f5..ca6fa00b 100644 --- a/cmd/backscroll/search_test.go +++ b/cmd/backscroll/search_test.go @@ -389,7 +389,7 @@ func TestRunSearchTextBudgetBoundary(t *testing.T) { err := runSearch(context.Background(), &stdout, &stderr, cfg, "presentselector", "", true, outputText, "", "", "", "", "", 20, 0, "text", "", - "minimal", maxTokens, true, 0.3, false) + "minimal", maxTokens, false) if err != nil { t.Fatalf("runSearch with budget %d: %v\nstderr: %s", maxTokens, err, stderr.String()) } diff --git a/cmd/backscroll/testdata/required_output.json b/cmd/backscroll/testdata/required_output.json index 0ff23575..fcd47fdb 100644 --- a/cmd/backscroll/testdata/required_output.json +++ b/cmd/backscroll/testdata/required_output.json @@ -31,12 +31,12 @@ "false/patterns --kind templates --all-projects --min-support 1": "No templates found.\n", "false/patterns --kind templates --all-projects --min-support 1 --json": "{\"count\":0,\"patterns\":[]}\n", "false/patterns --kind templates --all-projects --min-support 1 --robot": "", - "false/search --text database --all-projects --lexical-only": "", - "false/search --text database --all-projects --lexical-only --fields full": "", - "false/search --text database --all-projects --lexical-only --fields full --json": "null\n", - "false/search --text database --all-projects --lexical-only --fields full --robot": "", - "false/search --text database --all-projects --lexical-only --json": "[]\n", - "false/search --text database --all-projects --lexical-only --robot": "", + "false/search --text database --all-projects": "", + "false/search --text database --all-projects --fields full": "", + "false/search --text database --all-projects --fields full --json": "null\n", + "false/search --text database --all-projects --fields full --robot": "", + "false/search --text database --all-projects --json": "[]\n", + "false/search --text database --all-projects --robot": "", "false/status": "Backscroll Status\n=================\n\nDatabase:\n Path: /test.db\n Size: MB\n\nIndex:\n Files indexed: 0\n Messages indexed: 0\n Chunks stored: 0\n Embeddings: 0\n Vectors stored: 0\n\nConfiguration:\n Inputs: legacy (session_dirs)\n - /empty-inputs\n", "false/status --json": "{\"config\":{\"active_inputs\":[],\"session_dirs\":[\"/empty-inputs\"],\"using_declarative_inputs\":false},\"database\":{\"exists\":true,\"path\":\"/test.db\",\"size\":},\"index\":{\"indexed_at\":\"0001-01-01T00:00:00Z\",\"total_chunks\":0,\"total_embeddings\":0,\"total_files\":0,\"total_messages\":0,\"total_vectors\":0,\"usable\":false}}\n", "false/validate": "✓ Index validation passed\n✓ All required tables exist\n✓ FTS5 virtual table is set up correctly\n✓ No orphaned records found\n", @@ -76,12 +76,12 @@ "true/patterns --kind templates --all-projects --min-support 1": "Found 1 templates (min_support=1):\n\n1. [043a5b5e]\n Text: error: database locked (errno 5)\n Occurrences: 3\n Normalization: v2\n Projects: [output]\n Sample UUIDs: [output-tool-0 output-tool-1 output-tool-2]\n\n", "true/patterns --kind templates --all-projects --min-support 1 --json": "{\"count\":1,\"kind\":\"templates\",\"patterns\":[{\"template_id\":1,\"signature\":\"043a5b5e1a7b3c6bb635c7358b2c97a23bee5447858c10c53b780c66dd611262\",\"template_text\":\"error: database locked (errno 5)\",\"occurrence_count\":3,\"projects_affected\":[\"output\"],\"sample_uuids\":[\"output-tool-0\",\"output-tool-1\",\"output-tool-2\"],\"first_seen\":\"2026-01-01 00:00:00\",\"last_seen\":\"2026-01-01 00:00:00\",\"normalization_version\":2}]}\n", "true/patterns --kind templates --all-projects --min-support 1 --robot": "*** Templates ***\nresult_0_template_id=1\nresult_0_template=\"error: database locked (errno 5)\"\nresult_0_occurrence_count=3\nresult_0_normalization_version=2\n*** Total: 1 patterns ***\n", - "true/search --text database --all-projects --lexical-only": "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\nRank: 1 | Source: session | Role: assistant | Score: 0.02\nPath: /output/session.jsonl\nTime: 2026-01-01 00:00:03\nProject: output\nUUID: output-tool-0\nOrdinal: 3\nerror: database locked (errno 5)\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\nRank: 2 | Source: session | Role: assistant | Score: 0.02\nPath: /output/session.jsonl\nTime: 2026-01-01 00:00:04\nProject: output\nUUID: output-tool-1\nOrdinal: 4\nerror: database locked (errno 5)\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\nRank: 3 | Source: session | Role: assistant | Score: 0.02\nPath: /output/session.jsonl\nTime: 2026-01-01 00:00:05\nProject: output\nUUID: output-tool-2\nOrdinal: 5\nerror: database locked (errno 5)", - "true/search --text database --all-projects --lexical-only --fields full": "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\nRank: 1 | Source: session | Role: assistant | Score: 0.02\nPath: /output/session.jsonl\nTime: 2026-01-01 00:00:03\nProject: output\nUUID: output-tool-0\nOrdinal: 3\nerror: database locked (errno 5)\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\nRank: 2 | Source: session | Role: assistant | Score: 0.02\nPath: /output/session.jsonl\nTime: 2026-01-01 00:00:04\nProject: output\nUUID: output-tool-1\nOrdinal: 4\nerror: database locked (errno 5)\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\nRank: 3 | Source: session | Role: assistant | Score: 0.02\nPath: /output/session.jsonl\nTime: 2026-01-01 00:00:05\nProject: output\nUUID: output-tool-2\nOrdinal: 5\nerror: database locked (errno 5)", - "true/search --text database --all-projects --lexical-only --fields full --json": "[\n {\n \"Source\": \"session\",\n \"Role\": \"assistant\",\n \"UUID\": \"output-tool-0\",\n \"Ordinal\": 3,\n \"Content\": \"error: database locked (errno 5)\",\n \"FilePath\": \"/output/session.jsonl\",\n \"Timestamp\": \"2026-01-01T00:00:03Z\",\n \"SessionID\": \"\",\n \"ProjectPath\": \"output\",\n \"Score\": 0.01639344262295082,\n \"Tags\": null,\n \"ContentType\": \"tool\",\n \"Rank\": 1\n },\n {\n \"Source\": \"session\",\n \"Role\": \"assistant\",\n \"UUID\": \"output-tool-1\",\n \"Ordinal\": 4,\n \"Content\": \"error: database locked (errno 5)\",\n \"FilePath\": \"/output/session.jsonl\",\n \"Timestamp\": \"2026-01-01T00:00:04Z\",\n \"SessionID\": \"\",\n \"ProjectPath\": \"output\",\n \"Score\": 0.016129032258064516,\n \"Tags\": null,\n \"ContentType\": \"tool\",\n \"Rank\": 2\n },\n {\n \"Source\": \"session\",\n \"Role\": \"assistant\",\n \"UUID\": \"output-tool-2\",\n \"Ordinal\": 5,\n \"Content\": \"error: database locked (errno 5)\",\n \"FilePath\": \"/output/session.jsonl\",\n \"Timestamp\": \"2026-01-01T00:00:05Z\",\n \"SessionID\": \"\",\n \"ProjectPath\": \"output\",\n \"Score\": 0.015873015873015872,\n \"Tags\": null,\n \"ContentType\": \"tool\",\n \"Rank\": 3\n }\n]\n", - "true/search --text database --all-projects --lexical-only --fields full --robot": "result_0_source=session\nresult_0_role=assistant\nresult_0_filepath=/output/session.jsonl\nresult_0_uuid=output-tool-0\nresult_0_ordinal=3\nresult_0_content=error: database locked (errno 5)\nresult_0_project=output\nresult_0_content_type=tool\nresult_0_timestamp=2026-01-01T00:00:03Z\nresult_0_score=0.02\nresult_0_rank=1\nresult_1_source=session\nresult_1_role=assistant\nresult_1_filepath=/output/session.jsonl\nresult_1_uuid=output-tool-1\nresult_1_ordinal=4\nresult_1_content=error: database locked (errno 5)\nresult_1_project=output\nresult_1_content_type=tool\nresult_1_timestamp=2026-01-01T00:00:04Z\nresult_1_score=0.02\nresult_1_rank=2\nresult_2_source=session\nresult_2_role=assistant\nresult_2_filepath=/output/session.jsonl\nresult_2_uuid=output-tool-2\nresult_2_ordinal=5\nresult_2_content=error: database locked (errno 5)\nresult_2_project=output\nresult_2_content_type=tool\nresult_2_timestamp=2026-01-01T00:00:05Z\nresult_2_score=0.02\nresult_2_rank=3\n", - "true/search --text database --all-projects --lexical-only --json": "[\n {\n \"source_path\": \"/output/session.jsonl\",\n \"uuid\": \"output-tool-0\",\n \"ordinal\": 3,\n \"snippet\": \"error: \\u003cb\\u003edatabase\\u003c/b\\u003e locked (errno 5)\",\n \"score\": 0.01639344262295082,\n \"role\": \"assistant\",\n \"timestamp\": \"2026-01-01T00:00:03Z\"\n },\n {\n \"source_path\": \"/output/session.jsonl\",\n \"uuid\": \"output-tool-1\",\n \"ordinal\": 4,\n \"snippet\": \"error: \\u003cb\\u003edatabase\\u003c/b\\u003e locked (errno 5)\",\n \"score\": 0.016129032258064516,\n \"role\": \"assistant\",\n \"timestamp\": \"2026-01-01T00:00:04Z\"\n },\n {\n \"source_path\": \"/output/session.jsonl\",\n \"uuid\": \"output-tool-2\",\n \"ordinal\": 5,\n \"snippet\": \"error: \\u003cb\\u003edatabase\\u003c/b\\u003e locked (errno 5)\",\n \"score\": 0.015873015873015872,\n \"role\": \"assistant\",\n \"timestamp\": \"2026-01-01T00:00:05Z\"\n }\n]\n", - "true/search --text database --all-projects --lexical-only --robot": "result_0_filepath=/output/session.jsonl\nresult_0_uuid=output-tool-0\nresult_0_ordinal=3\nresult_0_content=error: database locked (errno 5)\nresult_0_score=0.02\nresult_0_role=assistant\nresult_0_timestamp=2026-01-01T00:00:03Z\nresult_1_filepath=/output/session.jsonl\nresult_1_uuid=output-tool-1\nresult_1_ordinal=4\nresult_1_content=error: database locked (errno 5)\nresult_1_score=0.02\nresult_1_role=assistant\nresult_1_timestamp=2026-01-01T00:00:04Z\nresult_2_filepath=/output/session.jsonl\nresult_2_uuid=output-tool-2\nresult_2_ordinal=5\nresult_2_content=error: database locked (errno 5)\nresult_2_score=0.02\nresult_2_role=assistant\nresult_2_timestamp=2026-01-01T00:00:05Z\n", + "true/search --text database --all-projects": "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\nRank: 1 | Source: session | Role: assistant | Score: 0.02\nPath: /output/session.jsonl\nTime: 2026-01-01 00:00:03\nProject: output\nUUID: output-tool-0\nOrdinal: 3\nerror: database locked (errno 5)\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\nRank: 2 | Source: session | Role: assistant | Score: 0.02\nPath: /output/session.jsonl\nTime: 2026-01-01 00:00:04\nProject: output\nUUID: output-tool-1\nOrdinal: 4\nerror: database locked (errno 5)\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\nRank: 3 | Source: session | Role: assistant | Score: 0.02\nPath: /output/session.jsonl\nTime: 2026-01-01 00:00:05\nProject: output\nUUID: output-tool-2\nOrdinal: 5\nerror: database locked (errno 5)", + "true/search --text database --all-projects --fields full": "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\nRank: 1 | Source: session | Role: assistant | Score: 0.02\nPath: /output/session.jsonl\nTime: 2026-01-01 00:00:03\nProject: output\nUUID: output-tool-0\nOrdinal: 3\nerror: database locked (errno 5)\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\nRank: 2 | Source: session | Role: assistant | Score: 0.02\nPath: /output/session.jsonl\nTime: 2026-01-01 00:00:04\nProject: output\nUUID: output-tool-1\nOrdinal: 4\nerror: database locked (errno 5)\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\nRank: 3 | Source: session | Role: assistant | Score: 0.02\nPath: /output/session.jsonl\nTime: 2026-01-01 00:00:05\nProject: output\nUUID: output-tool-2\nOrdinal: 5\nerror: database locked (errno 5)", + "true/search --text database --all-projects --fields full --json": "[\n {\n \"Source\": \"session\",\n \"Role\": \"assistant\",\n \"UUID\": \"output-tool-0\",\n \"Ordinal\": 3,\n \"Content\": \"error: database locked (errno 5)\",\n \"FilePath\": \"/output/session.jsonl\",\n \"Timestamp\": \"2026-01-01T00:00:03Z\",\n \"SessionID\": \"\",\n \"ProjectPath\": \"output\",\n \"Score\": 0.01639344262295082,\n \"Tags\": null,\n \"ContentType\": \"tool\",\n \"Rank\": 1\n },\n {\n \"Source\": \"session\",\n \"Role\": \"assistant\",\n \"UUID\": \"output-tool-1\",\n \"Ordinal\": 4,\n \"Content\": \"error: database locked (errno 5)\",\n \"FilePath\": \"/output/session.jsonl\",\n \"Timestamp\": \"2026-01-01T00:00:04Z\",\n \"SessionID\": \"\",\n \"ProjectPath\": \"output\",\n \"Score\": 0.016129032258064516,\n \"Tags\": null,\n \"ContentType\": \"tool\",\n \"Rank\": 2\n },\n {\n \"Source\": \"session\",\n \"Role\": \"assistant\",\n \"UUID\": \"output-tool-2\",\n \"Ordinal\": 5,\n \"Content\": \"error: database locked (errno 5)\",\n \"FilePath\": \"/output/session.jsonl\",\n \"Timestamp\": \"2026-01-01T00:00:05Z\",\n \"SessionID\": \"\",\n \"ProjectPath\": \"output\",\n \"Score\": 0.015873015873015872,\n \"Tags\": null,\n \"ContentType\": \"tool\",\n \"Rank\": 3\n }\n]\n", + "true/search --text database --all-projects --fields full --robot": "result_0_source=session\nresult_0_role=assistant\nresult_0_filepath=/output/session.jsonl\nresult_0_uuid=output-tool-0\nresult_0_ordinal=3\nresult_0_content=error: database locked (errno 5)\nresult_0_project=output\nresult_0_content_type=tool\nresult_0_timestamp=2026-01-01T00:00:03Z\nresult_0_score=0.02\nresult_0_rank=1\nresult_1_source=session\nresult_1_role=assistant\nresult_1_filepath=/output/session.jsonl\nresult_1_uuid=output-tool-1\nresult_1_ordinal=4\nresult_1_content=error: database locked (errno 5)\nresult_1_project=output\nresult_1_content_type=tool\nresult_1_timestamp=2026-01-01T00:00:04Z\nresult_1_score=0.02\nresult_1_rank=2\nresult_2_source=session\nresult_2_role=assistant\nresult_2_filepath=/output/session.jsonl\nresult_2_uuid=output-tool-2\nresult_2_ordinal=5\nresult_2_content=error: database locked (errno 5)\nresult_2_project=output\nresult_2_content_type=tool\nresult_2_timestamp=2026-01-01T00:00:05Z\nresult_2_score=0.02\nresult_2_rank=3\n", + "true/search --text database --all-projects --json": "[\n {\n \"source_path\": \"/output/session.jsonl\",\n \"uuid\": \"output-tool-0\",\n \"ordinal\": 3,\n \"snippet\": \"error: \\u003cb\\u003edatabase\\u003c/b\\u003e locked (errno 5)\",\n \"score\": 0.01639344262295082,\n \"role\": \"assistant\",\n \"timestamp\": \"2026-01-01T00:00:03Z\"\n },\n {\n \"source_path\": \"/output/session.jsonl\",\n \"uuid\": \"output-tool-1\",\n \"ordinal\": 4,\n \"snippet\": \"error: \\u003cb\\u003edatabase\\u003c/b\\u003e locked (errno 5)\",\n \"score\": 0.016129032258064516,\n \"role\": \"assistant\",\n \"timestamp\": \"2026-01-01T00:00:04Z\"\n },\n {\n \"source_path\": \"/output/session.jsonl\",\n \"uuid\": \"output-tool-2\",\n \"ordinal\": 5,\n \"snippet\": \"error: \\u003cb\\u003edatabase\\u003c/b\\u003e locked (errno 5)\",\n \"score\": 0.015873015873015872,\n \"role\": \"assistant\",\n \"timestamp\": \"2026-01-01T00:00:05Z\"\n }\n]\n", + "true/search --text database --all-projects --robot": "result_0_filepath=/output/session.jsonl\nresult_0_uuid=output-tool-0\nresult_0_ordinal=3\nresult_0_content=error: database locked (errno 5)\nresult_0_score=0.02\nresult_0_role=assistant\nresult_0_timestamp=2026-01-01T00:00:03Z\nresult_1_filepath=/output/session.jsonl\nresult_1_uuid=output-tool-1\nresult_1_ordinal=4\nresult_1_content=error: database locked (errno 5)\nresult_1_score=0.02\nresult_1_role=assistant\nresult_1_timestamp=2026-01-01T00:00:04Z\nresult_2_filepath=/output/session.jsonl\nresult_2_uuid=output-tool-2\nresult_2_ordinal=5\nresult_2_content=error: database locked (errno 5)\nresult_2_score=0.02\nresult_2_role=assistant\nresult_2_timestamp=2026-01-01T00:00:05Z\n", "true/status": "Backscroll Status\n=================\n\nDatabase:\n Path: /test.db\n Size: MB\n\nIndex:\n Files indexed: 1\n Messages indexed: 6\n Chunks stored: 0\n Embeddings: 0\n Vectors stored: 0\n Last indexed: 2026-01-01 00:00:00 UTC\n\nConfiguration:\n Inputs: legacy (session_dirs)\n - /empty-inputs\n", "true/status --json": "{\"config\":{\"active_inputs\":[],\"session_dirs\":[\"/empty-inputs\"],\"using_declarative_inputs\":false},\"database\":{\"exists\":true,\"path\":\"/test.db\",\"size\":},\"index\":{\"indexed_at\":\"2026-01-01T00:00:00Z\",\"total_chunks\":0,\"total_embeddings\":0,\"total_files\":1,\"total_messages\":6,\"total_vectors\":0,\"usable\":true}}\n", "true/validate": "✓ Index validation passed\n✓ All required tables exist\n✓ FTS5 virtual table is set up correctly\n✓ No orphaned records found\n", diff --git a/docs/input-contract.md b/docs/input-contract.md index 0f216a2d..34b8de9a 100644 --- a/docs/input-contract.md +++ b/docs/input-contract.md @@ -327,7 +327,7 @@ Verify with an installed manifest (use your actual custom source path if needed) ```bash backscroll config --json -backscroll search --text "a phrase you remember" --all-projects --source session --source-path '*omp*' --lexical-only --json +backscroll search --text "a phrase you remember" --all-projects --source session --source-path '*omp*' --json backscroll search --text "a command fragment" --all-projects --source session --content-type tool --source-path '*omp*' --robot ``` @@ -368,7 +368,7 @@ Codex files remain read-only): ```bash backscroll config --json -backscroll search --text "a phrase you remember" --all-projects --source session --source-path '*codex*' --lexical-only --json +backscroll search --text "a phrase you remember" --all-projects --source session --source-path '*codex*' --json backscroll search --text "a command fragment" --all-projects --content-type tool --source-path '*codex*' --robot --fields minimal --max-tokens 1000 backscroll validate --json ``` diff --git a/docs/search.md b/docs/search.md index 02b1cc39..d51609bf 100644 --- a/docs/search.md +++ b/docs/search.md @@ -29,14 +29,6 @@ backscroll search "artifact literal" --source-path "*/session.jsonl" --robot | `--max-tokens ` | Approximate token budget for text and robot output; JSON is not contractually truncated | | `--source-path ` | Filter a normal text query by indexed `source_path`; exact paths or `*`/SQL `LIKE` patterns | | `--relax` | Opt in to bounded lexical term dropping after zero rows; all scope filters stay fixed | -| `--lexical-only` | Force BM25-only search; skip vector retrieval | -| `--similarity-threshold ` | Finite `0..1` vector cosine threshold (default `0.3`; `0` disables it); not a BM25 filter | - -The CLI has no wired embedding provider, so search currently falls back to -lexical retrieval even without `--lexical-only`. `--similarity-threshold` only -affects vector candidates when a provider is available; it does not filter -BM25 results and is ignored by lexical `--relax`. Neither flag enables -semantic expansion in the deployed CLI. ## Output Formats @@ -250,7 +242,7 @@ If all tokens in a query are stopwords, the original query is used unfiltered as ## Opt-in lexical relaxation -Ordinary searches, including `--relax=false`, retain the existing behavior and output. `--relax` is a lexical-only option, not a new default and not semantic search: +Ordinary searches, including `--relax=false`, retain the existing behavior and output. `--relax` is an opt-in lexical option: ```bash backscroll search --text 'violet handshake adaptation' --relax --robot --fields minimal --max-tokens 2000 @@ -300,8 +292,7 @@ commands accept no positional operands. Search, list, and patterns reject simultaneous `--project` and `--all-projects` rather than overriding a scope. Their independent default scope remains unchanged. -Before startup, search requires a finite `--similarity-threshold` in [0, 1] -(0 still means no threshold) and nonnegative `--max-tokens` (0 is unlimited). +Before startup, search requires nonnegative `--max-tokens` (0 is unlimited). List requires nonnegative `--recent`, `--limit`, and `--offset`. List defaults to 20 results; explicit `--limit 0` or `--recent 0` is unlimited. `--recent` cannot be combined with explicit `--limit` or `--order`; diff --git a/internal/chunking/chunking.go b/internal/chunking/chunking.go deleted file mode 100644 index 7d0d87f1..00000000 --- a/internal/chunking/chunking.go +++ /dev/null @@ -1,150 +0,0 @@ -// Package chunking provides text splitting for embedding generation. -package chunking - -import "strings" - -// TokenCount estimates the number of tokens in text. -// Approximation: 1 token ≈ 0.75 words (GPT/BERT-family models). -func TokenCount(text string) int { - words := len(strings.Fields(text)) - if words == 0 { - return 0 - } - // ceil(words / 0.75) = ceil(words * 4 / 3) - return (words*4 + 2) / 3 -} - -func tokenCount(text string) int { return TokenCount(text) } - -// ChunkText splits text into chunks of at most maxTokens tokens each, with -// optional token overlap between consecutive chunks. -// Split preference order: paragraph boundary > sentence boundary > word boundary. -// Words are never split. -func ChunkText(text string, maxTokens, overlap int) []string { - if maxTokens <= 0 { - return []string{text} - } - text = strings.TrimSpace(text) - if text == "" { - return nil - } - if tokenCount(text) <= maxTokens { - return []string{text} - } - - // Split into paragraphs first - paragraphs := splitParagraphs(text) - chunks := chunkSegments(paragraphs, maxTokens, overlap, splitSentences) - return chunks -} - -// splitParagraphs splits text on blank lines. -func splitParagraphs(text string) []string { - var paras []string - for _, p := range strings.Split(text, "\n\n") { - p = strings.TrimSpace(p) - if p != "" { - paras = append(paras, p) - } - } - return paras -} - -// splitSentences splits a paragraph into sentences on '. ', '! ', '? ' or newlines. -func splitSentences(para string) []string { - var sentences []string - // Use naive sentence splitting on sentence-ending punctuation - var buf strings.Builder - runes := []rune(para) - for i, r := range runes { - buf.WriteRune(r) - if (r == '.' || r == '!' || r == '?') && i+1 < len(runes) { - next := runes[i+1] - if next == ' ' || next == '\n' { - s := strings.TrimSpace(buf.String()) - if s != "" { - sentences = append(sentences, s) - } - buf.Reset() - } - } else if r == '\n' { - s := strings.TrimSpace(buf.String()) - if s != "" { - sentences = append(sentences, s) - } - buf.Reset() - } - } - if s := strings.TrimSpace(buf.String()); s != "" { - sentences = append(sentences, s) - } - return sentences -} - -// chunkSegments groups segments (paragraphs or sentences) into chunks ≤ maxTokens. -// If a single segment exceeds maxTokens, it is split by words. -// overlap is the number of tokens to repeat from the previous chunk. -func chunkSegments(segments []string, maxTokens, overlap int, subSplit func(string) []string) []string { - var chunks []string - var current []string - currentTokens := 0 - - flush := func() { - if len(current) == 0 { - return - } - chunk := strings.Join(current, " ") - chunks = append(chunks, chunk) - - // Compute overlap: keep enough trailing words from current - if overlap > 0 { - words := strings.Fields(chunk) - // overlap tokens → approx overlap * 3/4 words (inverse of tokenCount) - keepWords := (overlap * 3) / 4 - if keepWords >= len(words) { - keepWords = len(words) / 2 - } - if keepWords > 0 { - overlapText := strings.Join(words[len(words)-keepWords:], " ") - current = []string{overlapText} - currentTokens = tokenCount(overlapText) - return - } - } - current = nil - currentTokens = 0 - } - - for _, seg := range segments { - segTokens := tokenCount(seg) - - if segTokens > maxTokens { - // Segment too large: flush current, then split segment further - flush() - if subSplit != nil { - sub := subSplit(seg) - subChunks := chunkSegments(sub, maxTokens, overlap, splitWords) - chunks = append(chunks, subChunks...) - } else { - // Word-level split - subChunks := chunkSegments(splitWords(seg), maxTokens, overlap, nil) - chunks = append(chunks, subChunks...) - } - continue - } - - if currentTokens+segTokens > maxTokens { - flush() - } - - current = append(current, seg) - currentTokens += segTokens - } - flush() - return chunks -} - -// splitWords splits text into individual words. -func splitWords(text string) []string { - return strings.Fields(text) -} diff --git a/internal/chunking/chunking_test.go b/internal/chunking/chunking_test.go deleted file mode 100644 index a9f2bea9..00000000 --- a/internal/chunking/chunking_test.go +++ /dev/null @@ -1,206 +0,0 @@ -package chunking - -import ( - "strings" - "testing" -) - -// generateText creates a text with approximately n words. -func generateText(n int) string { - words := make([]string, n) - for i := range words { - words[i] = "word" - } - return strings.Join(words, " ") -} - -func TestChunkText_ShortText(t *testing.T) { - text := "Hello world" - chunks := ChunkText(text, 512, 0) - if len(chunks) != 1 { - t.Errorf("short text: got %d chunks, want 1", len(chunks)) - } - if chunks[0] != text { - t.Errorf("short text: got %q, want %q", chunks[0], text) - } -} - -func TestChunkText_Empty(t *testing.T) { - chunks := ChunkText("", 512, 0) - if len(chunks) != 0 { - t.Errorf("empty text: got %d chunks, want 0", len(chunks)) - } -} - -func TestChunkText_ZeroMaxTokens(t *testing.T) { - text := "some text here" - chunks := ChunkText(text, 0, 0) - if len(chunks) != 1 { - t.Errorf("zero maxTokens: got %d chunks, want 1", len(chunks)) - } -} - -func TestChunkText_LongText_ProducesMultipleChunks(t *testing.T) { - // ~2000 words → ~2667 tokens; with maxTokens=512 expect ≥3 chunks - text := generateText(2000) - chunks := ChunkText(text, 512, 0) - if len(chunks) < 3 { - t.Errorf("2000-word text with maxTokens=512: got %d chunks, want ≥3", len(chunks)) - } -} - -func TestChunkText_MaxTokensRespected(t *testing.T) { - text := generateText(2000) - maxTokens := 512 - chunks := ChunkText(text, maxTokens, 0) - for i, chunk := range chunks { - tc := tokenCount(chunk) - if tc > maxTokens+10 { // allow small overshoot due to approximation - t.Errorf("chunk[%d] has %d tokens, exceeds maxTokens=%d", i, tc, maxTokens) - } - } -} - -func TestChunkText_NoWordSplit(t *testing.T) { - // Words should not be split mid-word - text := generateText(500) - chunks := ChunkText(text, 100, 0) - for _, chunk := range chunks { - for _, word := range strings.Fields(chunk) { - if word != "word" { - t.Errorf("unexpected word fragment: %q", word) - } - } - } -} - -func TestChunkText_ParagraphBoundary(t *testing.T) { - para1 := generateText(100) // ~133 tokens - para2 := generateText(100) - para3 := generateText(100) - text := para1 + "\n\n" + para2 + "\n\n" + para3 - - // maxTokens=200 → should split on paragraph boundaries - chunks := ChunkText(text, 200, 0) - if len(chunks) < 2 { - t.Errorf("expected ≥2 chunks for 3 paragraphs with maxTokens=200, got %d", len(chunks)) - } -} - -func TestChunkText_Overlap(t *testing.T) { - text := generateText(300) // ~400 tokens - chunks := ChunkText(text, 150, 30) - if len(chunks) < 2 { - t.Fatalf("expected ≥2 chunks, got %d", len(chunks)) - } - // With overlap, consecutive chunks should share some words - words1 := strings.Fields(chunks[0]) - words2 := strings.Fields(chunks[1]) - - // The beginning of chunk[1] should appear near the end of chunk[0] - if len(words1) > 0 && len(words2) > 0 { - last := words1[len(words1)-1] - foundOverlap := false - for _, w := range words2[:min(20, len(words2))] { - if w == last { - foundOverlap = true - break - } - } - // With word="word" everywhere, just verify chunks differ - if chunks[0] == chunks[1] { - t.Error("chunks should not be identical") - } - _ = foundOverlap - } -} - -func TestChunkText_AllWordsCovered(t *testing.T) { - // All words from input should appear in at least one chunk - words := make([]string, 200) - for i := range words { - words[i] = "token" - } - text := strings.Join(words, " ") - chunks := ChunkText(text, 50, 0) - - allChunksText := strings.Join(chunks, " ") - totalWordCount := len(strings.Fields(allChunksText)) - // With no overlap, total words across chunks ≥ input words - if totalWordCount < len(words) { - t.Errorf("words lost: input %d, chunks total %d", len(words), totalWordCount) - } -} - -func TestChunkText_TableDriven(t *testing.T) { - cases := []struct { - name string - wordCount int - maxTokens int - minChunks int - }{ - {"small/512", 50, 512, 1}, - {"medium/512", 500, 512, 1}, - {"large/512", 2000, 512, 3}, - {"large/200", 2000, 200, 5}, - {"huge/100", 5000, 100, 10}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - text := generateText(tc.wordCount) - chunks := ChunkText(text, tc.maxTokens, 0) - if len(chunks) < tc.minChunks { - t.Errorf("got %d chunks, want ≥%d", len(chunks), tc.minChunks) - } - }) - } -} - -func TestSplitSentences_Punctuation(t *testing.T) { - cases := []struct { - name string - input string - want int // number of sentences - }{ - {"period", "Hello world. How are you.", 2}, - {"exclamation", "Hello! How are you!", 2}, - {"question", "Who are you? Where am I?", 2}, - {"newline", "Line one\nLine two", 2}, - {"empty", "", 0}, - {"no break", "No punctuation here", 1}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got := splitSentences(tc.input) - if len(got) != tc.want { - t.Errorf("splitSentences(%q) = %d sentences, want %d: %v", tc.input, len(got), tc.want, got) - } - }) - } -} - -func TestTokenCount(t *testing.T) { - cases := []struct { - text string - want int - }{ - {"", 0}, - {"hello", 2}, // 1 word → ceil(1*4/3) = 2 - {"hello world", 3}, // 2 words → ceil(2*4/3) = 3 - {"a b c d", 6}, // 4 words → ceil(4*4/3) = 6 - } - for _, tc := range cases { - got := tokenCount(tc.text) - if got != tc.want { - t.Errorf("tokenCount(%q) = %d, want %d", tc.text, got, tc.want) - } - } -} - -func min(a, b int) int { - if a < b { - return a - } - return b -} diff --git a/internal/config/config.go b/internal/config/config.go index 484da2d2..5ac65688 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,15 +9,6 @@ import ( "github.com/pelletier/go-toml/v2" ) -// EmbeddingConfig contains settings for the vector embedding system. -type EmbeddingConfig struct { - Enabled bool `toml:"enabled"` - ModelName string `toml:"model_name"` // e.g., "all-MiniLM-L6-v2" - ModelPath string `toml:"model_path"` // local path or "" for auto-download - SimilarityThreshold float32 `toml:"similarity_threshold"` // default: 0.7 - TopK int `toml:"top_k"` // default: 10 -} - // SourcesConfig contains paths for different source types. type SourcesConfig struct { KE []string `toml:"ke"` @@ -30,12 +21,11 @@ type SourcesConfig struct { // Config represents the complete application configuration. type Config struct { - DatabasePath string `toml:"database_path"` - SessionDirs []string `toml:"session_dirs"` - SessionDir string `toml:"session_dir"` // backward compat: single string - Sources SourcesConfig `toml:"sources"` - Embedding EmbeddingConfig `toml:"embedding"` - SessionDirsExplicit bool `toml:"-"` // true if session_dirs was set in config + DatabasePath string `toml:"database_path"` + SessionDirs []string `toml:"session_dirs"` + SessionDir string `toml:"session_dir"` // backward compat: single string + Sources SourcesConfig `toml:"sources"` + SessionDirsExplicit bool `toml:"-"` // true if session_dirs was set in config } // Load loads the configuration from files and environment variables. @@ -50,12 +40,6 @@ func LoadValidated(inputsDir string) (*Config, error) { cfg := &Config{ DatabasePath: defaultDatabasePath(), SessionDirs: []string{"."}, - Embedding: EmbeddingConfig{ - Enabled: false, - ModelName: "all-MiniLM-L6-v2", - SimilarityThreshold: 0.7, - TopK: 10, - }, } var legacyFiles []LegacySourcesFile @@ -110,16 +94,15 @@ func loadConfigFileSnapshot(path string, cfg *Config) (*LegacySourcesFile, error // A pointer distinguishes an absent sources key from even an empty table. var parsed struct { - DatabasePath string `toml:"database_path"` - SessionDirs []string `toml:"session_dirs"` - SessionDir string `toml:"session_dir"` - Sources *SourcesConfig `toml:"sources"` - Embedding EmbeddingConfig `toml:"embedding"` + DatabasePath string `toml:"database_path"` + SessionDirs []string `toml:"session_dirs"` + SessionDir string `toml:"session_dir"` + Sources *SourcesConfig `toml:"sources"` } if err := toml.Unmarshal(data, &parsed); err != nil { return nil, fmt.Errorf("failed to parse %s: %w", path, err) } - tempCfg := Config{DatabasePath: parsed.DatabasePath, SessionDirs: parsed.SessionDirs, SessionDir: parsed.SessionDir, Embedding: parsed.Embedding} + tempCfg := Config{DatabasePath: parsed.DatabasePath, SessionDirs: parsed.SessionDirs, SessionDir: parsed.SessionDir} var legacy *LegacySourcesFile if parsed.Sources != nil { tempCfg.Sources = *parsed.Sources @@ -160,23 +143,6 @@ func loadConfigFileSnapshot(path string, cfg *Config) (*LegacySourcesFile, error cfg.Sources.Backlog = tempCfg.Sources.Backlog } - // Merge embedding config if enabled - if tempCfg.Embedding.Enabled { - cfg.Embedding.Enabled = true - } - if tempCfg.Embedding.ModelName != "" { - cfg.Embedding.ModelName = tempCfg.Embedding.ModelName - } - if tempCfg.Embedding.ModelPath != "" { - cfg.Embedding.ModelPath = tempCfg.Embedding.ModelPath - } - if tempCfg.Embedding.SimilarityThreshold > 0 { - cfg.Embedding.SimilarityThreshold = tempCfg.Embedding.SimilarityThreshold - } - if tempCfg.Embedding.TopK > 0 { - cfg.Embedding.TopK = tempCfg.Embedding.TopK - } - return legacy, nil } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 5bee7d02..190f9b87 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -6,93 +6,6 @@ import ( "testing" ) -func TestEmbeddingConfig_Defaults(t *testing.T) { - _ = os.Unsetenv("BACKSCROLL_DATABASE_PATH") - _ = os.Unsetenv("BACKSCROLL_SESSION_DIRS") - - cfg, err := Load() - if err != nil { - t.Fatalf("Load: %v", err) - } - if cfg.Embedding.Enabled { - t.Error("Embedding.Enabled default should be false") - } - if cfg.Embedding.ModelName != "all-MiniLM-L6-v2" { - t.Errorf("ModelName = %q, want all-MiniLM-L6-v2", cfg.Embedding.ModelName) - } - if cfg.Embedding.SimilarityThreshold != 0.7 { - t.Errorf("SimilarityThreshold = %v, want 0.7", cfg.Embedding.SimilarityThreshold) - } - if cfg.Embedding.TopK != 10 { - t.Errorf("TopK = %d, want 10", cfg.Embedding.TopK) - } -} - -func TestEmbeddingConfig_LoadFromTOML(t *testing.T) { - dir := t.TempDir() - toml := `[embedding] -enabled = true -model_name = "custom-model" -model_path = "/path/to/model.onnx" -similarity_threshold = 0.85 -top_k = 20 -` - tomlPath := filepath.Join(dir, "backscroll.toml") - if err := os.WriteFile(tomlPath, []byte(toml), 0o644); err != nil { - t.Fatal(err) - } - - orig, _ := os.Getwd() - _ = os.Chdir(dir) - defer func() { _ = os.Chdir(orig) }() - - cfg, err := Load() - if err != nil { - t.Fatalf("Load: %v", err) - } - if !cfg.Embedding.Enabled { - t.Error("Embedding.Enabled should be true") - } - if cfg.Embedding.ModelName != "custom-model" { - t.Errorf("ModelName = %q", cfg.Embedding.ModelName) - } - if cfg.Embedding.ModelPath != "/path/to/model.onnx" { - t.Errorf("ModelPath = %q", cfg.Embedding.ModelPath) - } - if cfg.Embedding.SimilarityThreshold != 0.85 { - t.Errorf("SimilarityThreshold = %v", cfg.Embedding.SimilarityThreshold) - } - if cfg.Embedding.TopK != 20 { - t.Errorf("TopK = %d", cfg.Embedding.TopK) - } -} - -func TestEmbeddingConfig_LegacyTOMLUnaffected(t *testing.T) { - dir := t.TempDir() - // No [embedding] section - toml := `database_path = "/tmp/test.db" -` - tomlPath := filepath.Join(dir, "backscroll.toml") - if err := os.WriteFile(tomlPath, []byte(toml), 0o644); err != nil { - t.Fatal(err) - } - - orig, _ := os.Getwd() - _ = os.Chdir(dir) - defer func() { _ = os.Chdir(orig) }() - - cfg, err := Load() - if err != nil { - t.Fatalf("Load: %v", err) - } - if cfg.Embedding.Enabled { - t.Error("Embedding.Enabled should default to false when not in TOML") - } - if cfg.Embedding.TopK != 10 { - t.Errorf("TopK should default to 10, got %d", cfg.Embedding.TopK) - } -} - func TestLoadDefaults(t *testing.T) { // Save current env vars oldDbPath := os.Getenv("BACKSCROLL_DATABASE_PATH") diff --git a/internal/embedding/mock.go b/internal/embedding/mock.go deleted file mode 100644 index da900c89..00000000 --- a/internal/embedding/mock.go +++ /dev/null @@ -1,42 +0,0 @@ -package embedding - -import ( - "context" - "crypto/sha256" - "encoding/binary" -) - -// MockEmbeddingProvider returns deterministic vectors for testing without -// requiring a real ONNX model. Different texts produce different vectors. -type MockEmbeddingProvider struct { - dims int -} - -// NewMockProvider creates a MockEmbeddingProvider with the given vector dimension. -func NewMockProvider(dims int) *MockEmbeddingProvider { - return &MockEmbeddingProvider{dims: dims} -} - -// Embed returns a deterministic vector derived from the SHA-256 hash of the text. -// The first min(8, dims) values are derived from the hash; the rest are zero. -func (m *MockEmbeddingProvider) Embed(_ context.Context, text string) ([]float32, error) { - vec := make([]float32, m.dims) - if m.dims == 0 { - return vec, nil - } - - h := sha256.Sum256([]byte(text)) - // Fill as many float32 values as we have hash bytes for (up to dims) - for i := 0; i+4 <= len(h) && i/4 < m.dims; i += 4 { - bits := binary.LittleEndian.Uint32(h[i : i+4]) - // Normalize to [-1, 1] - vec[i/4] = (float32(bits)/float32(^uint32(0)))*2 - 1 - } - return vec, nil -} - -// Dimensions returns the configured vector dimension. -func (m *MockEmbeddingProvider) Dimensions() int { return m.dims } - -// Close is a no-op for the mock provider. -func (m *MockEmbeddingProvider) Close() error { return nil } diff --git a/internal/embedding/mock_test.go b/internal/embedding/mock_test.go deleted file mode 100644 index a91d9afb..00000000 --- a/internal/embedding/mock_test.go +++ /dev/null @@ -1,87 +0,0 @@ -package embedding - -import ( - "context" - "testing" -) - -func TestMockProvider_ImplementsInterface(t *testing.T) { - var _ EmbeddingProvider = &MockEmbeddingProvider{} -} - -func TestMockProvider_Dimensions(t *testing.T) { - m := NewMockProvider(384) - if m.Dimensions() != 384 { - t.Errorf("Dimensions() = %d, want 384", m.Dimensions()) - } -} - -func TestMockProvider_Embed_Length(t *testing.T) { - m := NewMockProvider(384) - vec, err := m.Embed(context.Background(), "hello") - if err != nil { - t.Fatalf("Embed: %v", err) - } - if len(vec) != 384 { - t.Errorf("Embed len = %d, want 384", len(vec)) - } -} - -func TestMockProvider_Embed_Deterministic(t *testing.T) { - m := NewMockProvider(384) - v1, _ := m.Embed(context.Background(), "hello") - v2, _ := m.Embed(context.Background(), "hello") - if len(v1) != len(v2) { - t.Fatal("lengths differ") - } - for i := range v1 { - if v1[i] != v2[i] { - t.Errorf("v1[%d]=%v != v2[%d]=%v (not deterministic)", i, v1[i], i, v2[i]) - } - } -} - -func TestMockProvider_Embed_Distinct(t *testing.T) { - m := NewMockProvider(384) - vh, _ := m.Embed(context.Background(), "hello") - vw, _ := m.Embed(context.Background(), "world") - - same := true - for i := range vh { - if vh[i] != vw[i] { - same = false - break - } - } - if same { - t.Error("Embed('hello') == Embed('world'), expected distinct vectors") - } -} - -func TestMockProvider_Embed_ZeroDims(t *testing.T) { - m := NewMockProvider(0) - vec, err := m.Embed(context.Background(), "hello") - if err != nil { - t.Fatalf("Embed: %v", err) - } - if len(vec) != 0 { - t.Errorf("Embed len = %d, want 0", len(vec)) - } -} - -func TestMockProvider_Close(t *testing.T) { - m := NewMockProvider(384) - if err := m.Close(); err != nil { - t.Errorf("Close: %v", err) - } -} - -func TestMockProvider_Embed_Range(t *testing.T) { - m := NewMockProvider(8) - vec, _ := m.Embed(context.Background(), "test") - for i, v := range vec { - if v < -1.0 || v > 1.0 { - t.Errorf("vec[%d] = %f, want in [-1, 1]", i, v) - } - } -} diff --git a/internal/embedding/onnx_provider.go b/internal/embedding/onnx_provider.go deleted file mode 100644 index c042cac6..00000000 --- a/internal/embedding/onnx_provider.go +++ /dev/null @@ -1,47 +0,0 @@ -package embedding - -import ( - "context" - "errors" -) - -// ErrOnnxNotAvailable is returned when the ONNX provider is used but the -// binary was not compiled with ONNX runtime support. -// -// Decision: both github.com/knights-analytics/hugot and -// github.com/yalue/onnxruntime_go require CGO or a native ONNX Runtime -// shared library at build time. The project rule is "no CGO if avoidable" -// (using modernc.org/sqlite for the same reason). Until a pure-Go ONNX -// inference path is available (e.g., github.com/owulveryck/onnx-go with -// full operator support), the ONNX provider is implemented as a stub that -// returns this error. Use MockEmbeddingProvider for all tests. -// -// To enable the real ONNX provider: add a build-tagged file -// onnx_provider_real.go with `//go:build hugot` and wire hugot's pipeline -// behind NewOnnxProvider. CI without the native library continues to use -// the stub. -var ErrOnnxNotAvailable = errors.New("ONNX provider not available: rebuild with -tags hugot and install the ONNX Runtime library") - -// OnnxProvider generates embeddings using an ONNX model (all-MiniLM-L6-v2, -// 384 dimensions). Requires the ONNX Runtime native library at runtime. -// -// In this build, all methods return ErrOnnxNotAvailable. -// Use NewMockProvider for testing. -type OnnxProvider struct{} - -// NewOnnxProvider loads an ONNX embedding model from modelPath. -// Returns ErrOnnxNotAvailable in the default (no ONNX runtime) build. -func NewOnnxProvider(_ string) (*OnnxProvider, error) { - return nil, ErrOnnxNotAvailable -} - -// Embed returns ErrOnnxNotAvailable. -func (p *OnnxProvider) Embed(_ context.Context, _ string) ([]float32, error) { - return nil, ErrOnnxNotAvailable -} - -// Dimensions returns 384 (all-MiniLM-L6-v2 output dimension). -func (p *OnnxProvider) Dimensions() int { return 384 } - -// Close is a no-op. -func (p *OnnxProvider) Close() error { return nil } diff --git a/internal/embedding/onnx_provider_test.go b/internal/embedding/onnx_provider_test.go deleted file mode 100644 index da7a5ba1..00000000 --- a/internal/embedding/onnx_provider_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package embedding - -import ( - "context" - "errors" - "testing" -) - -func TestOnnxProvider_ImplementsInterface(t *testing.T) { - var _ EmbeddingProvider = &OnnxProvider{} -} - -func TestOnnxProvider_NewReturnsError(t *testing.T) { - _, err := NewOnnxProvider("/nonexistent/model.onnx") - if err == nil { - t.Fatal("expected error from NewOnnxProvider") - } - if !errors.Is(err, ErrOnnxNotAvailable) { - t.Errorf("expected ErrOnnxNotAvailable, got %v", err) - } -} - -func TestOnnxProvider_Embed_ReturnsError(t *testing.T) { - p := &OnnxProvider{} - _, err := p.Embed(context.Background(), "hello") - if !errors.Is(err, ErrOnnxNotAvailable) { - t.Errorf("expected ErrOnnxNotAvailable, got %v", err) - } -} - -func TestOnnxProvider_Dimensions(t *testing.T) { - p := &OnnxProvider{} - if p.Dimensions() != 384 { - t.Errorf("Dimensions() = %d, want 384", p.Dimensions()) - } -} - -func TestOnnxProvider_Close(t *testing.T) { - p := &OnnxProvider{} - if err := p.Close(); err != nil { - t.Errorf("Close: %v", err) - } -} diff --git a/internal/embedding/provider.go b/internal/embedding/provider.go deleted file mode 100644 index 7fafa5f1..00000000 --- a/internal/embedding/provider.go +++ /dev/null @@ -1,15 +0,0 @@ -// Package embedding provides the EmbeddingProvider interface and implementations -// for generating vector embeddings from text. -package embedding - -import "context" - -// EmbeddingProvider generates fixed-dimension vector embeddings for text. -type EmbeddingProvider interface { - // Embed generates a vector for the given text. - Embed(ctx context.Context, text string) ([]float32, error) - // Dimensions returns the output vector dimension (e.g., 384). - Dimensions() int - // Close releases resources held by the provider. - Close() error -} diff --git a/internal/models/models.go b/internal/models/models.go index 742394c6..9da2435d 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -59,19 +59,17 @@ type Stats struct { // SearchOptions contains options for searching. type SearchOptions struct { - Project string - AllProjects bool - Source string - SourcePath string // exact path, SQL LIKE pattern, or * glob - After *time.Time - Before *time.Time - Role string - Limit int - Offset int - ContentType string - Tag string - LexicalOnly bool // skip vector search, BM25 only - SimilarityThreshold float64 // minimum cosine similarity (0 = no threshold) + Project string + AllProjects bool + Source string + SourcePath string // exact path, SQL LIKE pattern, or * glob + After *time.Time + Before *time.Time + Role string + Limit int + Offset int + ContentType string + Tag string } // SyncOptions contains options for syncing. diff --git a/internal/models/models_test.go b/internal/models/models_test.go index 240ab045..6700105a 100644 --- a/internal/models/models_test.go +++ b/internal/models/models_test.go @@ -433,18 +433,16 @@ func TestSearchOptionsCreation(t *testing.T) { before := time.Date(2024, 12, 31, 23, 59, 59, 0, time.UTC) opts := SearchOptions{ - Project: "myproject", - AllProjects: false, - Source: "session", - After: &after, - Before: &before, - Role: "assistant", - Limit: 10, - Offset: 5, - ContentType: "text", - Tag: "feature", - LexicalOnly: true, - SimilarityThreshold: 0.75, + Project: "myproject", + AllProjects: false, + Source: "session", + After: &after, + Before: &before, + Role: "assistant", + Limit: 10, + Offset: 5, + ContentType: "text", + Tag: "feature", } if opts.Project != "myproject" { @@ -477,12 +475,6 @@ func TestSearchOptionsCreation(t *testing.T) { if opts.Tag != "feature" { t.Errorf("expected Tag 'feature', got %q", opts.Tag) } - if opts.LexicalOnly != true { - t.Errorf("expected LexicalOnly true, got %v", opts.LexicalOnly) - } - if opts.SimilarityThreshold != 0.75 { - t.Errorf("expected SimilarityThreshold 0.75, got %f", opts.SimilarityThreshold) - } } // TestSearchOptionsZeroValues tests SearchOptions with zero values. @@ -504,9 +496,6 @@ func TestSearchOptionsZeroValues(t *testing.T) { if opts.Before != nil { t.Errorf("expected nil Before, got %v", opts.Before) } - if opts.LexicalOnly != false { - t.Errorf("expected LexicalOnly false, got %v", opts.LexicalOnly) - } } // TestSearchOptionsAllProjects tests SearchOptions with AllProjects true. @@ -533,19 +522,6 @@ func TestSearchOptionsVariousSources(t *testing.T) { } } -// TestSearchOptionsSimilarityThreshold tests SearchOptions with various similarity thresholds. -func TestSearchOptionsSimilarityThreshold(t *testing.T) { - thresholds := []float64{0.0, 0.25, 0.5, 0.75, 1.0} - for _, threshold := range thresholds { - opts := SearchOptions{ - SimilarityThreshold: threshold, - } - if opts.SimilarityThreshold != threshold { - t.Errorf("expected SimilarityThreshold %f, got %f", threshold, opts.SimilarityThreshold) - } - } -} - // TestSyncOptionsCreation tests SyncOptions struct creation and field access. func TestSyncOptionsCreation(t *testing.T) { opts := SyncOptions{ diff --git a/internal/services/query.go b/internal/services/query.go index e4122000..b91abe71 100644 --- a/internal/services/query.go +++ b/internal/services/query.go @@ -85,7 +85,7 @@ func ValidateExplicitSearchScope(request SearchRequest) error { return nil } -// Search runs either ordinary hybrid retrieval or opt-in lexical relaxation. +// Search runs ordinary lexical retrieval or opt-in lexical relaxation. func (s QueryService) Search(ctx context.Context, request SearchRequest) (SearchResponse, error) { if err := ctx.Err(); err != nil { return SearchResponse{}, err @@ -104,7 +104,7 @@ func (s QueryService) Search(ctx context.Context, request SearchRequest) (Search }, err } - results, err := s.DB.HybridSearchContext(ctx, request.Query, request.Options) + results, err := s.DB.SearchContext(ctx, request.Query, request.Options) return SearchResponse{Results: results}, err } diff --git a/internal/services/query_test.go b/internal/services/query_test.go index f65fcdcc..f35b730f 100644 --- a/internal/services/query_test.go +++ b/internal/services/query_test.go @@ -21,7 +21,6 @@ func TestQueryServiceSearch(t *testing.T) { Options: models.SearchOptions{ SourcePath: "/target.jsonl", ContentType: "text", - LexicalOnly: true, Limit: 10, }, }) diff --git a/internal/storage/hybrid.go b/internal/storage/hybrid.go deleted file mode 100644 index a3a7ccc8..00000000 --- a/internal/storage/hybrid.go +++ /dev/null @@ -1,183 +0,0 @@ -package storage - -import ( - "context" - "errors" - "strconv" - - "github.com/pablontiv/backscroll/internal/embedding" - "github.com/pablontiv/backscroll/internal/hybrid" - "github.com/pablontiv/backscroll/internal/models" -) - -// SetEmbeddingProvider attaches an EmbeddingProvider used by HybridSearch. -func (d *Database) SetEmbeddingProvider(p embedding.EmbeddingProvider) { - d.embeddingProvider = p -} - -// HybridSearch performs BM25 search, optionally fused with vector search via RRF. -// Falls back to BM25-only when: -// - opts.LexicalOnly is true -// - no embedding provider is set -// - no vector embeddings exist in the database -// - the provider fails to embed the query -func (d *Database) HybridSearch(query string, opts models.SearchOptions) ([]SearchResult, error) { - return d.HybridSearchContext(context.Background(), query, opts) -} - -// HybridSearchContext is HybridSearch with cancellation propagated through the lexical, -// provider, and vector paths. Cancellation never falls back to lexical results. -func (d *Database) HybridSearchContext(ctx context.Context, query string, opts models.SearchOptions) ([]SearchResult, error) { - if err := ctx.Err(); err != nil { - return nil, err - } - bm25Results, err := d.SearchContext(ctx, query, opts) - if err != nil { - return nil, err - } - - // Skip vector path when explicitly requested or no provider configured. - if opts.LexicalOnly || d.embeddingProvider == nil { - return lexicalFallback(ctx, bm25Results) - } - - // Check if any vectors are stored. - vectorCount, err := d.getVectorCountContext(ctx) - if err != nil { - if cancellation := contextCancellation(ctx, err); cancellation != nil { - return nil, cancellation - } - return lexicalFallback(ctx, bm25Results) - } - if vectorCount == 0 { - return lexicalFallback(ctx, bm25Results) - } - - // Embed the query. - queryVec, err := d.embeddingProvider.Embed(ctx, query) - if err != nil { - if cancellation := contextCancellation(ctx, err); cancellation != nil { - return nil, cancellation - } - // Provider unavailable (e.g. ONNX stub) — fall back to BM25. - return lexicalFallback(ctx, bm25Results) - } - if err := ctx.Err(); err != nil { - return nil, err - } - - // Vector search: fetch more candidates than limit for RRF fusion. - topK := opts.Limit * 2 - if topK <= 0 { - topK = 200 - } - vecResults, err := d.vectorSearchContext(ctx, queryVec, topK) - if err != nil { - if cancellation := contextCancellation(ctx, err); cancellation != nil { - return nil, cancellation - } - return lexicalFallback(ctx, bm25Results) - } - - vecResults, err = filterVectorResultsContext(ctx, vecResults, opts.SimilarityThreshold) - if err != nil { - return nil, err - } - if len(vecResults) == 0 { - return lexicalFallback(ctx, bm25Results) - } - - // Convert to RRF ranking lists - bm25Ranking := make([]hybrid.RankResult, len(bm25Results)) - for i, r := range bm25Results { - bm25Ranking[i] = hybrid.RankResult{ID: strconv.Itoa(r.ID), Score: r.Score} - } - vecRanking := make([]hybrid.RankResult, len(vecResults)) - for i, vr := range vecResults { - vecRanking[i] = hybrid.RankResult{ID: strconv.FormatInt(vr.ItemID, 10), Score: vr.Similarity} - } - - fused := hybrid.ReciprocatRankFusion(60, bm25Ranking, vecRanking) - - // Map BM25 results by ID for fast lookup - byID := make(map[string]SearchResult, len(bm25Results)) - for _, r := range bm25Results { - byID[strconv.Itoa(r.ID)] = r - } - - // Build final result list: items from BM25 re-ranked by RRF score - // Items only in vector results (not in BM25) are omitted (no snippet available) - final := make([]SearchResult, 0, len(fused)) - seen := make(map[string]bool) - for _, f := range fused { - if seen[f.ID] { - continue - } - seen[f.ID] = true - if r, ok := byID[f.ID]; ok { - r.Score = f.Score - final = append(final, r) - } - } - - // Apply limit - if opts.Limit > 0 && len(final) > opts.Limit { - final = final[:opts.Limit] - } - if err := ctx.Err(); err != nil { - return nil, err - } - - return final, nil -} - -func lexicalFallback(ctx context.Context, results []SearchResult) ([]SearchResult, error) { - if err := ctx.Err(); err != nil { - return nil, err - } - return results, nil -} - -func filterVectorResultsContext(ctx context.Context, results []VectorResult, threshold float64) ([]VectorResult, error) { - if err := ctx.Err(); err != nil { - return nil, err - } - if threshold <= 0 { - if err := ctx.Err(); err != nil { - return nil, err - } - return results, nil - } - - filtered := make([]VectorResult, 0, len(results)) - for _, result := range results { - if err := ctx.Err(); err != nil { - return nil, err - } - if result.Similarity >= threshold { - filtered = append(filtered, result) - } - } - if err := ctx.Err(); err != nil { - return nil, err - } - return filtered, nil -} - -func contextCancellation(ctx context.Context, err error) error { - if ctxErr := ctx.Err(); ctxErr != nil { - return ctxErr - } - if errors.Is(err, context.Canceled) { - return context.Canceled - } - if errors.Is(err, context.DeadlineExceeded) { - return context.DeadlineExceeded - } - return nil -} - -// HasEmbeddingProvider returns true if an embedding provider has been set. -func (d *Database) HasEmbeddingProvider() bool { - return d.embeddingProvider != nil -} diff --git a/internal/storage/query_cancellation_test.go b/internal/storage/query_cancellation_test.go index e2e70cbb..822310e3 100644 --- a/internal/storage/query_cancellation_test.go +++ b/internal/storage/query_cancellation_test.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "testing" - "time" "github.com/pablontiv/backscroll/internal/models" ) @@ -48,139 +47,6 @@ func TestCanceledQueryContextsReturnNoPartialResults(t *testing.T) { }) } -type blockingEmbeddingProvider struct { - entered chan struct{} -} - -func (p *blockingEmbeddingProvider) Embed(ctx context.Context, _ string) ([]float32, error) { - close(p.entered) - <-ctx.Done() - return nil, ctx.Err() -} - -func (*blockingEmbeddingProvider) Dimensions() int { return 1 } -func (*blockingEmbeddingProvider) Close() error { return nil } - -type failingEmbeddingProvider struct { - err error -} - -func (p failingEmbeddingProvider) Embed(context.Context, string) ([]float32, error) { - return nil, p.err -} - -func (failingEmbeddingProvider) Dimensions() int { return 1 } -func (failingEmbeddingProvider) Close() error { return nil } - -func TestHybridSearchContextCancellationDoesNotFallBackToLexicalResults(t *testing.T) { - db, cleanup := newTestDB(t) - defer cleanup() - seedCancelableHybridSearch(t, db) - - provider := &blockingEmbeddingProvider{entered: make(chan struct{})} - db.SetEmbeddingProvider(provider) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - type outcome struct { - results []SearchResult - err error - } - done := make(chan outcome, 1) - go func() { - results, err := db.HybridSearchContext(ctx, "cancellation", models.SearchOptions{AllProjects: true}) - done <- outcome{results: results, err: err} - }() - - select { - case <-provider.entered: - cancel() - case <-time.After(5 * time.Second): - t.Fatal("embedding provider was not called") - } - - select { - case got := <-done: - if !errors.Is(got.err, context.Canceled) { - t.Fatalf("HybridSearchContext error = %v, want context.Canceled", got.err) - } - if got.results != nil { - t.Fatalf("HybridSearchContext fell back to partial lexical results: %+v", got.results) - } - case <-time.After(5 * time.Second): - t.Fatal("HybridSearchContext did not return after cancellation") - } -} - -func TestHybridSearchContextProviderFailureKeepsLexicalFallback(t *testing.T) { - db, cleanup := newTestDB(t) - defer cleanup() - seedCancelableHybridSearch(t, db) - - providerErr := errors.New("provider unavailable") - db.SetEmbeddingProvider(failingEmbeddingProvider{err: providerErr}) - results, err := db.HybridSearchContext(context.Background(), "cancellation", models.SearchOptions{AllProjects: true}) - if err != nil { - t.Fatalf("HybridSearchContext provider fallback error = %v", err) - } - if len(results) != 1 || results[0].Text != "cancellation lexical fallback" { - t.Fatalf("HybridSearchContext provider fallback results = %+v", results) - } -} - -func TestHybridSearchContextProviderDeadlineDoesNotFallBack(t *testing.T) { - db, cleanup := newTestDB(t) - defer cleanup() - seedCancelableHybridSearch(t, db) - - db.SetEmbeddingProvider(failingEmbeddingProvider{err: context.DeadlineExceeded}) - results, err := db.HybridSearchContext(context.Background(), "cancellation", models.SearchOptions{AllProjects: true}) - if !errors.Is(err, context.DeadlineExceeded) { - t.Fatalf("HybridSearchContext error = %v, want context.DeadlineExceeded", err) - } - if results != nil { - t.Fatalf("HybridSearchContext fell back to partial lexical results: %+v", results) - } -} - -type cancellationAfterChecksContext struct { - context.Context - cancelAt int - checks int - err error -} - -func (c *cancellationAfterChecksContext) Err() error { - c.checks++ - if c.checks >= c.cancelAt { - return c.err - } - return nil -} - -func TestFilterVectorResultsContextCancellationReturnsNoPartialResults(t *testing.T) { - for _, cancellation := range []error{context.Canceled, context.DeadlineExceeded} { - t.Run(cancellation.Error(), func(t *testing.T) { - ctx := &cancellationAfterChecksContext{ - Context: context.Background(), - cancelAt: 3, - err: cancellation, - } - results, err := filterVectorResultsContext(ctx, []VectorResult{ - {ItemID: 1, Similarity: 0.9}, - {ItemID: 2, Similarity: 0.8}, - {ItemID: 3, Similarity: 0.7}, - }, 0.5) - if !errors.Is(err, cancellation) { - t.Fatalf("filterVectorResultsContext error = %v, want %v", err, cancellation) - } - if results != nil { - t.Fatalf("filterVectorResultsContext returned partial results: %+v", results) - } - }) - } -} - func TestOptionalStatsQueryCancellationClassifiesDriverErrors(t *testing.T) { for _, cancellation := range []error{context.Canceled, context.DeadlineExceeded} { t.Run(cancellation.Error(), func(t *testing.T) { @@ -195,33 +61,3 @@ func TestOptionalStatsQueryCancellationClassifiesDriverErrors(t *testing.T) { t.Fatalf("genuine optional-schema error was not tolerated: %v", got) } } - -func seedCancelableHybridSearch(t *testing.T, db *Database) { - t.Helper() - const sourcePath = "/test/query-cancellation.jsonl" - if err := db.SyncFiles([]IndexedFile{{ - SourcePath: sourcePath, - Source: "session", - Hash: "query-cancellation", - Project: "test", - Messages: []IndexedMessage{{ - Ordinal: 0, - Role: "user", - Text: "cancellation lexical fallback", - ContentType: "text", - }}, - }}); err != nil { - t.Fatalf("seed search item: %v", err) - } - ids, err := db.InsertChunks(sourcePath, []ChunkRecord{{ - ChunkIdx: 0, - Content: "cancellation lexical fallback", - TokenCount: 3, - }}, time.Now().Unix()) - if err != nil { - t.Fatalf("seed chunk: %v", err) - } - if err := db.InsertChunkEmbedding(ids[0], []float32{1}); err != nil { - t.Fatalf("seed embedding: %v", err) - } -} diff --git a/internal/storage/query_failure_contract_test.go b/internal/storage/query_failure_contract_test.go index ea89db05..9a337383 100644 --- a/internal/storage/query_failure_contract_test.go +++ b/internal/storage/query_failure_contract_test.go @@ -178,7 +178,7 @@ func TestDisposedIndexConsumersReportFailure(t *testing.T) { return e }, "search": func() error { - r, e := db.HybridSearch("retained", models.SearchOptions{Limit: 10, LexicalOnly: true}) + r, e := db.Search("retained", models.SearchOptions{Limit: 10}) if r != nil { t.Errorf("partial search: %+v", r) } diff --git a/internal/storage/storage.go b/internal/storage/storage.go index ac360817..34d392f6 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -13,15 +13,13 @@ import ( "time" "github.com/pablontiv/backscroll/internal/compat" - "github.com/pablontiv/backscroll/internal/embedding" "modernc.org/sqlite" ) // Database represents a SQLite database connection with FTS5 support. type Database struct { - db *sql.DB - path string - embeddingProvider embedding.EmbeddingProvider + db *sql.DB + path string } var ( diff --git a/internal/storage/unit_test.go b/internal/storage/unit_test.go index 1c5b4b9c..70bd4b62 100644 --- a/internal/storage/unit_test.go +++ b/internal/storage/unit_test.go @@ -1,7 +1,6 @@ package storage import ( - "context" "fmt" "os" "path/filepath" @@ -9,7 +8,6 @@ import ( "testing" "time" - "github.com/pablontiv/backscroll/internal/embedding" "github.com/pablontiv/backscroll/internal/models" ) @@ -766,12 +764,11 @@ func TestListSessionsAfterSync(t *testing.T) { } } -func TestEmbeddingPipeline_WithMockProvider(t *testing.T) { +func TestEmbeddingMetadataLifecycle(t *testing.T) { db, cleanup := newTestDB(t) defer cleanup() - provider := embedding.NewMockProvider(384) - defer func() { _ = provider.Close() }() + const dimensions = 384 sourcePath := "test/session.jsonl" text := "hello world. this is a test paragraph. it has multiple sentences!" @@ -786,14 +783,7 @@ func TestEmbeddingPipeline_WithMockProvider(t *testing.T) { } for _, id := range ids { - vec, err := provider.Embed(context.Background(), text) - if err != nil { - t.Fatalf("Embed: %v", err) - } - if len(vec) != provider.Dimensions() { - t.Errorf("vector length %d != dimensions %d", len(vec), provider.Dimensions()) - } - if err := db.InsertEmbeddingMetadata(id, "mock", "v1", provider.Dimensions(), now); err != nil { + if err := db.InsertEmbeddingMetadata(id, "historical", "v1", dimensions, now); err != nil { t.Fatalf("InsertEmbeddingMetadata: %v", err) } } @@ -939,152 +929,6 @@ func TestVectorSearch_RoundTrip(t *testing.T) { } } -func TestHybridSearch_FallbackWithoutProvider(t *testing.T) { - db, cleanup := newTestDB(t) - defer cleanup() - - _ = db.SyncFiles([]IndexedFile{{ - SourcePath: "test/b.jsonl", - Source: "session", - Hash: "h2", - Project: "proj", - Messages: []IndexedMessage{ - {Ordinal: 0, Role: "user", Text: "hybrid search fallback test", ContentType: "text"}, - }, - }}) - - // No provider set — should return BM25 results - results, err := db.HybridSearch("hybrid", models.SearchOptions{AllProjects: true}) - if err != nil { - t.Fatalf("HybridSearch: %v", err) - } - if len(results) == 0 { - t.Error("expected BM25 fallback results") - } -} - -func TestHybridSearch_WithMockProvider(t *testing.T) { - db, cleanup := newTestDB(t) - defer cleanup() - - provider := embedding.NewMockProvider(4) - defer func() { _ = provider.Close() }() - db.SetEmbeddingProvider(provider) - - _ = db.SyncFiles([]IndexedFile{{ - SourcePath: "test/c.jsonl", - Source: "session", - Hash: "h3", - Project: "proj", - Messages: []IndexedMessage{ - {Ordinal: 7, Role: "user", Text: "mock provider hybrid search", UUID: "hybrid-uuid", ContentType: "text"}, - }, - }}) - - // Insert chunk + embedding so vector path activates - ids, _ := db.InsertChunks("test/c.jsonl", []ChunkRecord{ - {ChunkIdx: 0, Content: "mock provider hybrid search", TokenCount: 4}, - }, time.Now().Unix()) - vec, _ := provider.Embed(context.Background(), "mock provider hybrid search") - _ = db.InsertChunkEmbedding(ids[0], vec) - - results, err := db.HybridSearch("mock", models.SearchOptions{AllProjects: true}) - if err != nil { - t.Fatalf("HybridSearch with provider: %v", err) - } - if len(results) == 0 { - t.Fatal("expected hybrid results with mock provider") - } - if results[0].UUID == nil || *results[0].UUID != "hybrid-uuid" || results[0].Ordinal != 7 { - t.Fatalf("hybrid copy lost identity: %+v", results[0]) - } -} - -func TestHybridSearch_LexicalOnly(t *testing.T) { - db, cleanup := newTestDB(t) - defer cleanup() - - provider := embedding.NewMockProvider(4) - defer func() { _ = provider.Close() }() - db.SetEmbeddingProvider(provider) - - _ = db.SyncFiles([]IndexedFile{{ - SourcePath: "test/d.jsonl", - Source: "session", - Hash: "h4", - Project: "proj", - Messages: []IndexedMessage{ - {Ordinal: 0, Role: "user", Text: "lexical only search test", ContentType: "text"}, - }, - }}) - - // LexicalOnly=true should skip vector path even with provider - results, err := db.HybridSearch("lexical", models.SearchOptions{ - AllProjects: true, - LexicalOnly: true, - }) - if err != nil { - t.Fatalf("HybridSearch lexical-only: %v", err) - } - if len(results) == 0 { - t.Error("expected BM25 results with lexical-only") - } -} - -func TestHasEmbeddingProvider(t *testing.T) { - db, cleanup := newTestDB(t) - defer cleanup() - - if db.HasEmbeddingProvider() { - t.Error("expected no provider initially") - } - - provider := embedding.NewMockProvider(4) - defer func() { _ = provider.Close() }() - db.SetEmbeddingProvider(provider) - - if !db.HasEmbeddingProvider() { - t.Error("expected provider after SetEmbeddingProvider") - } -} - -func TestHybridSearch_SimilarityThreshold(t *testing.T) { - db, cleanup := newTestDB(t) - defer cleanup() - - provider := embedding.NewMockProvider(4) - defer func() { _ = provider.Close() }() - db.SetEmbeddingProvider(provider) - - _ = db.SyncFiles([]IndexedFile{{ - SourcePath: "test/e.jsonl", - Source: "session", - Hash: "h5", - Project: "proj", - Messages: []IndexedMessage{ - {Ordinal: 0, Role: "user", Text: "threshold filter test query", ContentType: "text"}, - }, - }}) - - ids, _ := db.InsertChunks("test/e.jsonl", []ChunkRecord{ - {ChunkIdx: 0, Content: "threshold filter test query", TokenCount: 4}, - }, time.Now().Unix()) - vec, _ := provider.Embed(context.Background(), "threshold filter test query") - _ = db.InsertChunkEmbedding(ids[0], vec) - - // High threshold — mock provider produces deterministic unit vectors so similarity ≈ 1.0 - // Using very high threshold (e.g. 0.99) should still return results - results, err := db.HybridSearch("threshold", models.SearchOptions{ - AllProjects: true, - SimilarityThreshold: 0.99, - }) - if err != nil { - t.Fatalf("HybridSearch similarity threshold: %v", err) - } - // Results may be empty if similarity < threshold — just verify no error - _ = results -} - func TestCosineSimilarity_ZeroVector(t *testing.T) { zero := []float32{0, 0, 0} unit := []float32{1, 0, 0} @@ -1335,8 +1179,8 @@ func TestGetStatsEmptyDatabase(t *testing.T) { } } -// TestHybridSearchWithAllProjectsFilter covers allprojects path -func TestHybridSearchWithAllProjectsFilter(t *testing.T) { +// TestSearchWithAllProjectsFilter covers the all-projects path. +func TestSearchWithAllProjectsFilter(t *testing.T) { db, cleanup := newTestDB(t) defer cleanup() @@ -1354,9 +1198,9 @@ func TestHybridSearchWithAllProjectsFilter(t *testing.T) { t.Fatalf("SyncFiles: %v", err) } - results, err := db.HybridSearch("search", models.SearchOptions{AllProjects: true}) + results, err := db.Search("search", models.SearchOptions{AllProjects: true}) if err != nil { - t.Fatalf("HybridSearch all projects: %v", err) + t.Fatalf("Search all projects: %v", err) } if len(results) == 0 { t.Error("expected results for AllProjects=true") @@ -1670,62 +1514,6 @@ func TestVectorSearchTopKLimit(t *testing.T) { } } -// TestHybridSearchWithEmptyIndex covers hybrid search on empty DB -func TestHybridSearchWithEmptyIndex(t *testing.T) { - db, cleanup := newTestDB(t) - defer cleanup() - - // Hybrid search on empty database - results, err := db.HybridSearch("test", models.SearchOptions{AllProjects: true}) - if err != nil { - t.Fatalf("HybridSearch empty: %v", err) - } - if len(results) != 0 { - t.Errorf("expected 0 results on empty index, got %d", len(results)) - } -} - -// TestHybridSearchVectorDeduplication covers the deduplication logic -func TestHybridSearchVectorDeduplication(t *testing.T) { - db, cleanup := newTestDB(t) - defer cleanup() - - // Set a provider - provider := embedding.NewMockProvider(4) - defer func() { _ = provider.Close() }() - db.SetEmbeddingProvider(provider) - - // Index content - if err := db.SyncFiles([]IndexedFile{ - { - SourcePath: "test/v.jsonl", - Source: "session", - Hash: "h1", - Project: "test", - Messages: []IndexedMessage{ - {Ordinal: 0, Role: "user", Text: "dedup vector test content", UUID: getTestUUID(), ContentType: "text"}, - }, - }, - }); err != nil { - t.Fatalf("SyncFiles: %v", err) - } - - // Insert chunk with matching embedding - ids, _ := db.InsertChunks("test/v.jsonl", []ChunkRecord{ - {ChunkIdx: 0, Content: "dedup vector test content", TokenCount: 5}, - }, time.Now().Unix()) - - vec, _ := provider.Embed(context.Background(), "dedup vector test content") - _ = db.InsertChunkEmbedding(ids[0], vec) - - // Search should handle deduplication correctly - results, err := db.HybridSearch("dedup", models.SearchOptions{AllProjects: true}) - if err != nil { - t.Fatalf("HybridSearch dedup: %v", err) - } - _ = results -} - // TestLoadChunkEmbeddingsError covers error path in LoadChunkEmbeddings func TestLoadChunkEmbeddingsEmpty(t *testing.T) { db, cleanup := newTestDB(t) @@ -1849,48 +1637,6 @@ func TestInsertChunksMultipleBatches(t *testing.T) { } } -// TestHybridSearchSimilarityFilter covers the similarity threshold filtering -func TestHybridSearchSimilarityFilter(t *testing.T) { - db, cleanup := newTestDB(t) - defer cleanup() - - provider := embedding.NewMockProvider(4) - defer func() { _ = provider.Close() }() - db.SetEmbeddingProvider(provider) - - // Index content - if err := db.SyncFiles([]IndexedFile{ - { - SourcePath: "test/sim.jsonl", - Source: "session", - Hash: "h1", - Project: "test", - Messages: []IndexedMessage{ - {Ordinal: 0, Role: "user", Text: "similarity filtering test", UUID: getTestUUID(), ContentType: "text"}, - }, - }, - }); err != nil { - t.Fatalf("SyncFiles: %v", err) - } - - // Insert chunk with embedding - ids, _ := db.InsertChunks("test/sim.jsonl", []ChunkRecord{ - {ChunkIdx: 0, Content: "similarity filtering test", TokenCount: 3}, - }, time.Now().Unix()) - vec, _ := provider.Embed(context.Background(), "similarity filtering test") - _ = db.InsertChunkEmbedding(ids[0], vec) - - // Search with high similarity threshold - results, err := db.HybridSearch("similarity", models.SearchOptions{ - AllProjects: true, - SimilarityThreshold: 0.5, - }) - if err != nil { - t.Fatalf("HybridSearch similarity: %v", err) - } - _ = results -} - // TestSyncFilesSessionTagsNodup covers session-specific tag insertion func TestSyncFilesSessionTagsDedupPath(t *testing.T) { db, cleanup := newTestDB(t) @@ -2031,178 +1777,6 @@ func TestSearchProjectFiltering(t *testing.T) { } } -// TestHybridSearchRRFFusion covers the RRF fusion path with both BM25 and vector results -func TestHybridSearchRRFFusion(t *testing.T) { - db, cleanup := newTestDB(t) - defer cleanup() - - provider := embedding.NewMockProvider(4) - defer func() { _ = provider.Close() }() - db.SetEmbeddingProvider(provider) - - // Index multiple documents - if err := db.SyncFiles([]IndexedFile{ - { - SourcePath: "test/rrf1.jsonl", - Source: "session", - Hash: "h1", - Project: "test", - Messages: []IndexedMessage{ - {Ordinal: 0, Role: "user", Text: "machine learning algorithms", UUID: getTestUUID(), ContentType: "text"}, - }, - }, - { - SourcePath: "test/rrf2.jsonl", - Source: "session", - Hash: "h2", - Project: "test", - Messages: []IndexedMessage{ - {Ordinal: 0, Role: "user", Text: "deep learning neural networks", UUID: getTestUUID(), ContentType: "text"}, - }, - }, - }); err != nil { - t.Fatalf("SyncFiles: %v", err) - } - - // Insert chunks and embeddings for both - for i, path := range []string{"test/rrf1.jsonl", "test/rrf2.jsonl"} { - text := []string{"machine learning algorithms", "deep learning neural networks"}[i] - ids, _ := db.InsertChunks(path, []ChunkRecord{ - {ChunkIdx: 0, Content: text, TokenCount: 3}, - }, time.Now().Unix()) - vec, _ := provider.Embed(context.Background(), text) - _ = db.InsertChunkEmbedding(ids[0], vec) - } - - // Hybrid search with limit to trigger RRF path - results, err := db.HybridSearch("learning", models.SearchOptions{ - AllProjects: true, - Limit: 10, - }) - if err != nil { - t.Fatalf("HybridSearch RRF: %v", err) - } - if len(results) == 0 { - t.Error("expected RRF fusion results") - } -} - -// TestHybridSearchVectorCountZero covers zero vector count path -func TestHybridSearchZeroVectorCount(t *testing.T) { - db, cleanup := newTestDB(t) - defer cleanup() - - provider := embedding.NewMockProvider(4) - defer func() { _ = provider.Close() }() - db.SetEmbeddingProvider(provider) - - // Index without vectors - if err := db.SyncFiles([]IndexedFile{ - { - SourcePath: "test/novector.jsonl", - Source: "session", - Hash: "h1", - Project: "test", - Messages: []IndexedMessage{ - {Ordinal: 0, Role: "user", Text: "no vector embedding", UUID: getTestUUID(), ContentType: "text"}, - }, - }, - }); err != nil { - t.Fatalf("SyncFiles: %v", err) - } - - // HybridSearch should fallback to BM25 when no vectors exist - results, err := db.HybridSearch("vector", models.SearchOptions{AllProjects: true}) - if err != nil { - t.Fatalf("HybridSearch no vectors: %v", err) - } - if len(results) == 0 { - t.Error("expected BM25 fallback results") - } -} - -// TestHybridSearchProviderEmbedError covers provider embed error -func TestHybridSearchProviderEmbedError(t *testing.T) { - db, cleanup := newTestDB(t) - defer cleanup() - - // Mock provider that succeeds - provider := embedding.NewMockProvider(4) - defer func() { _ = provider.Close() }() - db.SetEmbeddingProvider(provider) - - // Index with vector embeddings - if err := db.SyncFiles([]IndexedFile{ - { - SourcePath: "test/embed.jsonl", - Source: "session", - Hash: "h1", - Project: "test", - Messages: []IndexedMessage{ - {Ordinal: 0, Role: "user", Text: "embedding test query", UUID: getTestUUID(), ContentType: "text"}, - }, - }, - }); err != nil { - t.Fatalf("SyncFiles: %v", err) - } - - ids, _ := db.InsertChunks("test/embed.jsonl", []ChunkRecord{ - {ChunkIdx: 0, Content: "embedding test query", TokenCount: 3}, - }, time.Now().Unix()) - vec, _ := provider.Embed(context.Background(), "embedding test query") - _ = db.InsertChunkEmbedding(ids[0], vec) - - // Search should handle embedding success gracefully - results, err := db.HybridSearch("embedding", models.SearchOptions{AllProjects: true}) - if err != nil { - t.Fatalf("HybridSearch with embed: %v", err) - } - _ = results -} - -// TestHybridSearchEmptyVectorResults covers empty vector results path -func TestHybridSearchEmptyVectorResults(t *testing.T) { - db, cleanup := newTestDB(t) - defer cleanup() - - provider := embedding.NewMockProvider(4) - defer func() { _ = provider.Close() }() - db.SetEmbeddingProvider(provider) - - // Index document - if err := db.SyncFiles([]IndexedFile{ - { - SourcePath: "test/emptyvec.jsonl", - Source: "session", - Hash: "h1", - Project: "test", - Messages: []IndexedMessage{ - {Ordinal: 0, Role: "user", Text: "text content here", UUID: getTestUUID(), ContentType: "text"}, - }, - }, - }); err != nil { - t.Fatalf("SyncFiles: %v", err) - } - - // Add vector embedding with specific vector - ids, _ := db.InsertChunks("test/emptyvec.jsonl", []ChunkRecord{ - {ChunkIdx: 0, Content: "text content here", TokenCount: 3}, - }, time.Now().Unix()) - // Insert vector of different dimension to cause mismatch - vec := make([]float32, 4) - for i := range vec { - vec[i] = float32(i+1) * 10 - } - _ = db.InsertChunkEmbedding(ids[0], vec) - - // Search with query that won't match vectors (different embedding) - results, err := db.HybridSearch("different", models.SearchOptions{AllProjects: true}) - if err != nil { - t.Fatalf("HybridSearch empty vectors: %v", err) - } - _ = results -} - // TestOpenReadOnlyPingError covers read-only ping success path func TestOpenReadOnlyPingSuccess(t *testing.T) { dir := t.TempDir() @@ -2941,12 +2515,12 @@ func TestIntegrationFullWorkflow(t *testing.T) { } _ = vresults - // 12. Hybrid search - hresults, err := db.HybridSearch("workflow", models.SearchOptions{AllProjects: true}) + // 12. Lexical search + searchResults, err := db.Search("workflow", models.SearchOptions{AllProjects: true}) if err != nil { - t.Fatalf("HybridSearch: %v", err) + t.Fatalf("Search: %v", err) } - _ = hresults + _ = searchResults // 13. ResolveSessionPath path, err := db.ResolveSessionPath("/s1.jsonl")