From eef659274598e1d882097ff0050d65b6c228a0f9 Mon Sep 17 00:00:00 2001 From: Zhiqiang ZHOU Date: Sat, 25 Jul 2026 13:06:13 -0700 Subject: [PATCH 1/3] feat: discover NDJSON logs via message projection Per-file format detection routes NDJSON entries through a projected message line for Drain while samples keep the raw structured lines, per ADR 0006. Envelope requires ts, severity, and payload; anything less projects whole. --- pkg/ndjson/detect.go | 48 +++++++++ pkg/ndjson/detect_test.go | 60 ++++++++++++ pkg/ndjson/project.go | 80 +++++++++++++++ pkg/ndjson/project_test.go | 52 ++++++++++ .../detect/blank-lines-only.expect-text.log | 0 .../detect/json-arrays.expect-text.log | 2 + .../testdata/detect/mixed.expect-text.log | 4 + .../detect/plain-text.expect-text.log | 3 + .../detect/pure-ndjson.expect-ndjson.log | 5 + .../testdata/project/arbitrary-shape.golden | 4 + .../project/arbitrary-shape.input.ndjson | 4 + pkg/ndjson/testdata/project/envelope.golden | 3 + .../testdata/project/envelope.input.ndjson | 3 + .../project/no-message-fallback.golden | 2 + .../project/no-message-fallback.input.ndjson | 2 + .../project/numeric-level-ignored.golden | 2 + .../numeric-level-ignored.input.ndjson | 2 + .../project/severity-via-level.golden | 2 + .../project/severity-via-level.input.ndjson | 2 + pkg/workspace/builder.go | 2 +- pkg/workspace/pipeline.go | 57 ++++++++--- pkg/workspace/pipeline_ndjson_test.go | 97 +++++++++++++++++++ pkg/workspace/workspace.go | 12 +++ 23 files changed, 436 insertions(+), 12 deletions(-) create mode 100644 pkg/ndjson/detect.go create mode 100644 pkg/ndjson/detect_test.go create mode 100644 pkg/ndjson/project.go create mode 100644 pkg/ndjson/project_test.go create mode 100644 pkg/ndjson/testdata/detect/blank-lines-only.expect-text.log create mode 100644 pkg/ndjson/testdata/detect/json-arrays.expect-text.log create mode 100644 pkg/ndjson/testdata/detect/mixed.expect-text.log create mode 100644 pkg/ndjson/testdata/detect/plain-text.expect-text.log create mode 100644 pkg/ndjson/testdata/detect/pure-ndjson.expect-ndjson.log create mode 100644 pkg/ndjson/testdata/project/arbitrary-shape.golden create mode 100644 pkg/ndjson/testdata/project/arbitrary-shape.input.ndjson create mode 100644 pkg/ndjson/testdata/project/envelope.golden create mode 100644 pkg/ndjson/testdata/project/envelope.input.ndjson create mode 100644 pkg/ndjson/testdata/project/no-message-fallback.golden create mode 100644 pkg/ndjson/testdata/project/no-message-fallback.input.ndjson create mode 100644 pkg/ndjson/testdata/project/numeric-level-ignored.golden create mode 100644 pkg/ndjson/testdata/project/numeric-level-ignored.input.ndjson create mode 100644 pkg/ndjson/testdata/project/severity-via-level.golden create mode 100644 pkg/ndjson/testdata/project/severity-via-level.input.ndjson create mode 100644 pkg/workspace/pipeline_ndjson_test.go diff --git a/pkg/ndjson/detect.go b/pkg/ndjson/detect.go new file mode 100644 index 0000000..c8abac4 --- /dev/null +++ b/pkg/ndjson/detect.go @@ -0,0 +1,48 @@ +// Package ndjson classifies log files as NDJSON and projects structured +// entries to text lines for pattern mining, per ADR 0006. +package ndjson + +import ( + "encoding/json" + "strings" +) + +// Format classifies the on-disk format of a log file. +type Format string + +const ( + // FormatText marks a file for the plain text pipeline. + FormatText Format = "text" + // FormatNDJSON marks a file whose lines are JSON objects, one per line. + FormatNDJSON Format = "ndjson" +) + +// DetectFormat classifies a file's lines. A file is NDJSON only when it has +// at least one non-empty line and every non-empty line parses as a JSON +// object. Files mixing text with JSON lines stay on the text path, so the +// projection never has to guess on a half-structured file. +func DetectFormat(lines []string) Format { + sampled := 0 + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed == "" { + continue + } + if !isJSONObject(trimmed) { + return FormatText + } + sampled++ + } + if sampled == 0 { + return FormatText + } + return FormatNDJSON +} + +func isJSONObject(trimmed string) bool { + if !strings.HasPrefix(trimmed, "{") { + return false + } + var obj map[string]any + return json.Unmarshal([]byte(trimmed), &obj) == nil +} diff --git a/pkg/ndjson/detect_test.go b/pkg/ndjson/detect_test.go new file mode 100644 index 0000000..964a33a --- /dev/null +++ b/pkg/ndjson/detect_test.go @@ -0,0 +1,60 @@ +package ndjson + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// TestDetectFormatFixtures classifies every fixture under testdata/detect/. +// The expected format is encoded in the file name as .expect-.log, +// so adding a case only means adding one fixture file. +func TestDetectFormatFixtures(t *testing.T) { + paths, err := filepath.Glob(filepath.Join("testdata", "detect", "*.log")) + if err != nil { + t.Fatalf("glob detect fixtures: %v", err) + } + if len(paths) == 0 { + t.Fatal("no detect fixtures found under testdata/detect") + } + + for _, path := range paths { + name, want := parseDetectFixtureName(t, path) + t.Run(name, func(t *testing.T) { + got := DetectFormat(readFixtureLines(t, path)) + if got != want { + t.Fatalf("DetectFormat(%s) = %q, want %q", path, got, want) + } + }) + } +} + +func TestDetectFormatEmptyInput(t *testing.T) { + if got := DetectFormat(nil); got != FormatText { + t.Fatalf("DetectFormat(nil) = %q, want %q", got, FormatText) + } +} + +func parseDetectFixtureName(t *testing.T, path string) (string, Format) { + t.Helper() + base := strings.TrimSuffix(filepath.Base(path), ".log") + name, expected, ok := strings.Cut(base, ".expect-") + if !ok { + t.Fatalf("detect fixture %s must be named .expect-.log", path) + } + format := Format(expected) + if format != FormatText && format != FormatNDJSON { + t.Fatalf("detect fixture %s has unknown expected format %q", path, expected) + } + return name, format +} + +func readFixtureLines(t *testing.T, path string) []string { + t.Helper() + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read fixture %s: %v", path, err) + } + return strings.Split(strings.TrimSuffix(string(content), "\n"), "\n") +} diff --git a/pkg/ndjson/project.go b/pkg/ndjson/project.go new file mode 100644 index 0000000..fca09f5 --- /dev/null +++ b/pkg/ndjson/project.go @@ -0,0 +1,80 @@ +package ndjson + +import ( + "bytes" + "encoding/json" + "strings" +) + +// messageFields are checked in order; the first string value wins. +var messageFields = []string{"message", "msg", "log", "error"} + +// severityFields are checked in order; the first string value wins. +// Numeric levels are ignored in v1. +var severityFields = []string{"severity", "level"} + +// Project converts one NDJSON line into the text line fed to pattern mining. +// Envelope entries {"ts": ..., "severity": ..., "payload": {...}} project +// from the payload; any other object is the payload itself. The projection is +// " ", just "" when no severity-like string field +// exists, or the compact payload JSON when no message-like string field +// exists. A line that does not parse as a JSON object is returned unchanged. +func Project(line string) string { + var entry map[string]any + if err := json.Unmarshal([]byte(line), &entry); err != nil || entry == nil { + return line + } + + payload, severity := splitEnvelope(entry) + if severity == "" { + severity = firstStringField(payload, severityFields) + } + + message := firstStringField(payload, messageFields) + if message == "" { + return compactJSON(payload) + } + if severity == "" { + return message + } + return severity + " " + message +} + +// splitEnvelope returns the payload object and the envelope severity when the +// entry matches the fixed envelope shape, otherwise the entry itself. The +// envelope is the importer's fixed contract (ADR 0006): ts, a string severity, +// and a payload object must all be present; anything less is an arbitrary +// user shape and is projected as a whole. +func splitEnvelope(entry map[string]any) (payload map[string]any, severity string) { + nested, ok := entry["payload"].(map[string]any) + if !ok { + return entry, "" + } + if _, hasTS := entry["ts"]; !hasTS { + return entry, "" + } + envelopeSeverity, ok := entry["severity"].(string) + if !ok || envelopeSeverity == "" { + return entry, "" + } + return nested, envelopeSeverity +} + +func firstStringField(obj map[string]any, fields []string) string { + for _, field := range fields { + if value, ok := obj[field].(string); ok && value != "" { + return value + } + } + return "" +} + +func compactJSON(payload map[string]any) string { + var buf bytes.Buffer + encoder := json.NewEncoder(&buf) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(payload); err != nil { + return "" + } + return strings.TrimSuffix(buf.String(), "\n") +} diff --git a/pkg/ndjson/project_test.go b/pkg/ndjson/project_test.go new file mode 100644 index 0000000..e3ae89b --- /dev/null +++ b/pkg/ndjson/project_test.go @@ -0,0 +1,52 @@ +package ndjson + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// TestProjectGoldenFixtures projects every line of every input fixture under +// testdata/project/ and compares the result against its committed golden +// file. Each .input.ndjson pairs with .golden, so adding a case +// only means adding one fixture pair. +func TestProjectGoldenFixtures(t *testing.T) { + inputs, err := filepath.Glob(filepath.Join("testdata", "project", "*.input.ndjson")) + if err != nil { + t.Fatalf("glob projection fixtures: %v", err) + } + if len(inputs) == 0 { + t.Fatal("no projection fixtures found under testdata/project") + } + + for _, inputPath := range inputs { + name := strings.TrimSuffix(filepath.Base(inputPath), ".input.ndjson") + goldenPath := filepath.Join("testdata", "project", name+".golden") + t.Run(name, func(t *testing.T) { + var projected []string + for _, line := range readFixtureLines(t, inputPath) { + if strings.TrimSpace(line) == "" { + continue + } + projected = append(projected, Project(line)) + } + got := strings.Join(projected, "\n") + "\n" + + golden, err := os.ReadFile(goldenPath) + if err != nil { + t.Fatalf("read golden %s: %v", goldenPath, err) + } + if got != string(golden) { + t.Fatalf("projection mismatch for %s\ngot:\n%swant:\n%s", inputPath, got, golden) + } + }) + } +} + +func TestProjectNonJSONLineIsReturnedUnchanged(t *testing.T) { + line := "2026-06-06 10:00:00 INFO plain text line" + if got := Project(line); got != line { + t.Fatalf("Project(%q) = %q, want unchanged", line, got) + } +} diff --git a/pkg/ndjson/testdata/detect/blank-lines-only.expect-text.log b/pkg/ndjson/testdata/detect/blank-lines-only.expect-text.log new file mode 100644 index 0000000..e69de29 diff --git a/pkg/ndjson/testdata/detect/json-arrays.expect-text.log b/pkg/ndjson/testdata/detect/json-arrays.expect-text.log new file mode 100644 index 0000000..57e30db --- /dev/null +++ b/pkg/ndjson/testdata/detect/json-arrays.expect-text.log @@ -0,0 +1,2 @@ +[1,2,3] +["a","b"] diff --git a/pkg/ndjson/testdata/detect/mixed.expect-text.log b/pkg/ndjson/testdata/detect/mixed.expect-text.log new file mode 100644 index 0000000..5f930ce --- /dev/null +++ b/pkg/ndjson/testdata/detect/mixed.expect-text.log @@ -0,0 +1,4 @@ +{"ts":"2026-06-06T10:00:00Z","severity":"ERROR","payload":{"message":"db timeout user=42"}} +{"level":"warn","msg":"queue depth 100"} +2026-06-06 10:00:02 ERROR db timeout user=43 +{"event":"gc","duration_ms":12} diff --git a/pkg/ndjson/testdata/detect/plain-text.expect-text.log b/pkg/ndjson/testdata/detect/plain-text.expect-text.log new file mode 100644 index 0000000..eb7a778 --- /dev/null +++ b/pkg/ndjson/testdata/detect/plain-text.expect-text.log @@ -0,0 +1,3 @@ +2026-06-06 10:00:00 INFO server started port=8080 +2026-06-06 10:00:01 ERROR db timeout user=42 +2026-06-06 10:00:02 ERROR db timeout user=43 diff --git a/pkg/ndjson/testdata/detect/pure-ndjson.expect-ndjson.log b/pkg/ndjson/testdata/detect/pure-ndjson.expect-ndjson.log new file mode 100644 index 0000000..b4476ff --- /dev/null +++ b/pkg/ndjson/testdata/detect/pure-ndjson.expect-ndjson.log @@ -0,0 +1,5 @@ +{"ts":"2026-06-06T10:00:00Z","severity":"ERROR","payload":{"message":"db timeout user=42"}} +{"level":"warn","msg":"queue depth 100"} + +{"event":"gc","duration_ms":12} +{"ts":"2026-06-06T10:00:03Z","severity":"INFO","payload":{"message":"server started port=8080"}} diff --git a/pkg/ndjson/testdata/project/arbitrary-shape.golden b/pkg/ndjson/testdata/project/arbitrary-shape.golden new file mode 100644 index 0000000..1be2c14 --- /dev/null +++ b/pkg/ndjson/testdata/project/arbitrary-shape.golden @@ -0,0 +1,4 @@ +WARN disk usage high volume=/var +connection reset peer=10.0.0.5 +context deadline exceeded +primary text diff --git a/pkg/ndjson/testdata/project/arbitrary-shape.input.ndjson b/pkg/ndjson/testdata/project/arbitrary-shape.input.ndjson new file mode 100644 index 0000000..aff73dd --- /dev/null +++ b/pkg/ndjson/testdata/project/arbitrary-shape.input.ndjson @@ -0,0 +1,4 @@ +{"severity":"WARN","message":"disk usage high volume=/var"} +{"log":"connection reset peer=10.0.0.5"} +{"error":"context deadline exceeded","op":"fetch"} +{"message":"primary text","msg":"secondary text"} diff --git a/pkg/ndjson/testdata/project/envelope.golden b/pkg/ndjson/testdata/project/envelope.golden new file mode 100644 index 0000000..1e28b54 --- /dev/null +++ b/pkg/ndjson/testdata/project/envelope.golden @@ -0,0 +1,3 @@ +ERROR db timeout user=42 +INFO server started port=8080 +{"payload":{"message":"heartbeat ok"},"ts":"2026-06-06T10:00:02Z"} diff --git a/pkg/ndjson/testdata/project/envelope.input.ndjson b/pkg/ndjson/testdata/project/envelope.input.ndjson new file mode 100644 index 0000000..8dcb081 --- /dev/null +++ b/pkg/ndjson/testdata/project/envelope.input.ndjson @@ -0,0 +1,3 @@ +{"ts":"2026-06-06T10:00:00Z","severity":"ERROR","payload":{"message":"db timeout user=42"}} +{"ts":"2026-06-06T10:00:01Z","severity":"INFO","payload":{"msg":"server started port=8080"}} +{"ts":"2026-06-06T10:00:02Z","payload":{"message":"heartbeat ok"}} diff --git a/pkg/ndjson/testdata/project/no-message-fallback.golden b/pkg/ndjson/testdata/project/no-message-fallback.golden new file mode 100644 index 0000000..1b0a172 --- /dev/null +++ b/pkg/ndjson/testdata/project/no-message-fallback.golden @@ -0,0 +1,2 @@ +{"duration_ms":12,"event":"gc"} +{"event":"cache_evict","keys":120} diff --git a/pkg/ndjson/testdata/project/no-message-fallback.input.ndjson b/pkg/ndjson/testdata/project/no-message-fallback.input.ndjson new file mode 100644 index 0000000..2811209 --- /dev/null +++ b/pkg/ndjson/testdata/project/no-message-fallback.input.ndjson @@ -0,0 +1,2 @@ +{"event":"gc","duration_ms":12} +{"ts":"2026-06-06T10:00:05Z","severity":"DEBUG","payload":{"event":"cache_evict","keys":120}} diff --git a/pkg/ndjson/testdata/project/numeric-level-ignored.golden b/pkg/ndjson/testdata/project/numeric-level-ignored.golden new file mode 100644 index 0000000..255b7fb --- /dev/null +++ b/pkg/ndjson/testdata/project/numeric-level-ignored.golden @@ -0,0 +1,2 @@ +request done status=200 +upstream unavailable diff --git a/pkg/ndjson/testdata/project/numeric-level-ignored.input.ndjson b/pkg/ndjson/testdata/project/numeric-level-ignored.input.ndjson new file mode 100644 index 0000000..2b05d51 --- /dev/null +++ b/pkg/ndjson/testdata/project/numeric-level-ignored.input.ndjson @@ -0,0 +1,2 @@ +{"level":30,"msg":"request done status=200"} +{"level":50,"error":"upstream unavailable"} diff --git a/pkg/ndjson/testdata/project/severity-via-level.golden b/pkg/ndjson/testdata/project/severity-via-level.golden new file mode 100644 index 0000000..a701c83 --- /dev/null +++ b/pkg/ndjson/testdata/project/severity-via-level.golden @@ -0,0 +1,2 @@ +warn queue depth 100 +info user login user=alice diff --git a/pkg/ndjson/testdata/project/severity-via-level.input.ndjson b/pkg/ndjson/testdata/project/severity-via-level.input.ndjson new file mode 100644 index 0000000..f26a5eb --- /dev/null +++ b/pkg/ndjson/testdata/project/severity-via-level.input.ndjson @@ -0,0 +1,2 @@ +{"level":"warn","msg":"queue depth 100"} +{"level":"info","message":"user login user=alice"} diff --git a/pkg/workspace/builder.go b/pkg/workspace/builder.go index f52e276..d313fcf 100644 --- a/pkg/workspace/builder.go +++ b/pkg/workspace/builder.go @@ -107,7 +107,7 @@ func (b *Builder) computePatterns() { } matches := make([]lineWithTemplate, 0, len(b.tagged)) for _, tl := range b.tagged { - t, ok := pattern.MatchTemplate(tl.Content, b.templates) + t, ok := pattern.MatchTemplate(tl.DrainLine(), b.templates) id := "" if ok { id = t.ID.String() diff --git a/pkg/workspace/pipeline.go b/pkg/workspace/pipeline.go index 3dc8429..ee4ede3 100644 --- a/pkg/workspace/pipeline.go +++ b/pkg/workspace/pipeline.go @@ -6,12 +6,14 @@ import ( "os" "path/filepath" "sort" + "strings" "sync" "time" "github.com/go-errors/errors" "github.com/google/uuid" "github.com/strrl/lapp/pkg/multiline" + "github.com/strrl/lapp/pkg/ndjson" "github.com/strrl/lapp/pkg/pattern" "github.com/strrl/lapp/pkg/semantic" "go.opentelemetry.io/otel" @@ -298,24 +300,57 @@ func mergeAllLogs(ctx context.Context, dir string) (tagged []TaggedLine, content var allTagged []TaggedLine var allContent []string for _, fileName := range fileNames { - lines := allLogs[fileName] - detector, err := multiline.NewDetector(multiline.DetectorConfig{}) + fileTagged, err := tagFileLines(ctx, fileName, allLogs[fileName]) if err != nil { - return nil, nil, 0, errors.Errorf("multiline detector: %w", err) + return nil, nil, 0, err } - merged := multiline.MergeSlice(ctx, lines, detector) - for _, m := range merged { - allTagged = append(allTagged, TaggedLine{ - Content: m.Content, - FileName: fileName, - LineNum: m.StartLine, - }) - allContent = append(allContent, m.Content) + for _, tl := range fileTagged { + allTagged = append(allTagged, tl) + allContent = append(allContent, tl.DrainLine()) } } return allTagged, allContent, len(allLogs), nil } +// tagFileLines converts one log file into tagged entries. NDJSON files keep +// the raw JSON line as Content and carry a text projection for pattern +// mining; plain text files go through multiline merging unchanged. +func tagFileLines(ctx context.Context, fileName string, lines []string) ([]TaggedLine, error) { + if ndjson.DetectFormat(lines) == ndjson.FormatNDJSON { + return tagNDJSONLines(fileName, lines), nil + } + detector, err := multiline.NewDetector(multiline.DetectorConfig{}) + if err != nil { + return nil, errors.Errorf("multiline detector: %w", err) + } + merged := multiline.MergeSlice(ctx, lines, detector) + tagged := make([]TaggedLine, 0, len(merged)) + for _, m := range merged { + tagged = append(tagged, TaggedLine{ + Content: m.Content, + FileName: fileName, + LineNum: m.StartLine, + }) + } + return tagged, nil +} + +func tagNDJSONLines(fileName string, lines []string) []TaggedLine { + var tagged []TaggedLine + for i, line := range lines { + if strings.TrimSpace(line) == "" { + continue + } + tagged = append(tagged, TaggedLine{ + Content: line, + FileName: fileName, + LineNum: i + 1, + Projection: ndjson.Project(line), + }) + } + return tagged +} + func discoverRepeatedPatterns(ctx context.Context, content []string) ([]pattern.DrainCluster, error) { drainParser, err := pattern.NewDrainParser() if err != nil { diff --git a/pkg/workspace/pipeline_ndjson_test.go b/pkg/workspace/pipeline_ndjson_test.go new file mode 100644 index 0000000..e7ea94d --- /dev/null +++ b/pkg/workspace/pipeline_ndjson_test.go @@ -0,0 +1,97 @@ +package workspace + +import ( + "context" + "encoding/json" + "path/filepath" + "strings" + "testing" + + "github.com/strrl/lapp/pkg/semantic" +) + +func TestDiscoverMixedTextAndNDJSONWorkspace(t *testing.T) { + dir := t.TempDir() + mustMkdir(t, filepath.Join(dir, "logs")) + textLines := []string{ + "2026-06-06 10:00:00 ERROR db timeout user=42", + "2026-06-06 10:00:01 ERROR db timeout user=43", + } + mustWrite(t, filepath.Join(dir, "logs", "app.log"), strings.Join(textLines, "\n")+"\n") + ndjsonLines := []string{ + `{"ts":"2026-06-06T10:00:02Z","severity":"ERROR","payload":{"message":"payment declined order=1001"}}`, + `{"ts":"2026-06-06T10:00:03Z","severity":"ERROR","payload":{"message":"payment declined order=1002"}}`, + } + unmatchedLine := `{"status_code":901,"details":{"state":"only_once_marker"}}` + mustWrite(t, filepath.Join(dir, "logs", "service.ndjson"), strings.Join(append(append([]string{}, ndjsonLines...), unmatchedLine), "\n")+"\n") + + result, err := Discover(context.Background(), DiscoveryConfig{ + Dir: dir, + RunID: "01900000-0000-7000-8000-000000000002", + Labeler: func(_ context.Context, _ semantic.Config, inputs []semantic.PatternInput) ([]semantic.SemanticLabel, error) { + labels := make([]semantic.SemanticLabel, 0, len(inputs)) + for _, input := range inputs { + labels = append(labels, semantic.SemanticLabel{ + PatternUUIDString: input.PatternUUIDString, + SemanticID: "pattern-" + input.PatternUUIDString[:8], + Description: "Labeled by test", + }) + } + return labels, nil + }, + }) + if err != nil { + t.Fatalf("Discover: %v", err) + } + + if result.FileCount != 2 || result.LineCount != 5 || result.PatternCount != 2 || result.UnmatchedCount != 1 { + t.Fatalf("unexpected result: %+v", result) + } + + record, err := ReadDiscoveryRunRecord(dir, result.RunID) + if err != nil { + t.Fatalf("ReadDiscoveryRunRecord: %v", err) + } + + textPattern := findPatternForFile(record.Patterns, "app.log") + if textPattern == nil { + t.Fatalf("expected a pattern originating from app.log, got %+v", record.Patterns) + } + jsonPattern := findPatternForFile(record.Patterns, "service.ndjson") + if jsonPattern == nil { + t.Fatalf("expected a pattern originating from service.ndjson, got %+v", record.Patterns) + } + + runDir := DiscoveryRunDir(dir, result.RunID) + textSamples := mustRead(t, filepath.Join(runDir, "patterns", textPattern.DirName, "samples.log")) + assertContains(t, textSamples, "ERROR db timeout user=42") + + jsonSamples := mustRead(t, filepath.Join(runDir, "patterns", jsonPattern.DirName, "samples.log")) + sampleLines := strings.Split(strings.TrimSuffix(jsonSamples, "\n"), "\n") + if len(sampleLines) != len(ndjsonLines) { + t.Fatalf("expected %d NDJSON samples, got %d:\n%s", len(ndjsonLines), len(sampleLines), jsonSamples) + } + for i, line := range sampleLines { + var entry map[string]any + if err := json.Unmarshal([]byte(line), &entry); err != nil { + t.Fatalf("sample line %d is not a JSON object: %v\nline: %s", i+1, err, line) + } + if line != ndjsonLines[i] { + t.Fatalf("sample line %d is not the raw NDJSON line\ngot: %s\nwant: %s", i+1, line, ndjsonLines[i]) + } + } + + unmatchedSamples := mustRead(t, filepath.Join(runDir, "patterns", "unmatched", "samples.log")) + assertContains(t, unmatchedSamples, unmatchedLine) +} + +func findPatternForFile(patterns []PatternInfo, fileName string) *PatternInfo { + for i := range patterns { + for _, ref := range patterns[i].LineRefs { + if ref.FileName == fileName { + return &patterns[i] + } + } + } + return nil +} diff --git a/pkg/workspace/workspace.go b/pkg/workspace/workspace.go index 631b112..8eb4d22 100644 --- a/pkg/workspace/workspace.go +++ b/pkg/workspace/workspace.go @@ -12,6 +12,18 @@ type TaggedLine struct { Content string FileName string LineNum int + // Projection is the text fed to pattern mining for NDJSON entries. + // Empty for plain text entries, whose Content is mined directly. + Projection string `json:",omitempty"` +} + +// DrainLine returns the text used for pattern mining and template matching: +// the projection for NDJSON entries, otherwise the raw content. +func (t TaggedLine) DrainLine() string { + if t.Projection != "" { + return t.Projection + } + return t.Content } // LineRef identifies a line's location in a source file. From bf955cd91ea25524aa74e2cc9cfe5803660c44a7 Mon Sep 17 00:00:00 2001 From: Zhiqiang ZHOU Date: Sat, 25 Jul 2026 13:11:28 -0700 Subject: [PATCH 2/3] test: rename golden files to expected.txt --- pkg/ndjson/project_test.go | 20 +++++++++---------- ...pe.golden => arbitrary-shape.expected.txt} | 0 ...{envelope.golden => envelope.expected.txt} | 0 ...olden => no-message-fallback.expected.txt} | 0 ...den => numeric-level-ignored.expected.txt} | 0 ...golden => severity-via-level.expected.txt} | 0 6 files changed, 10 insertions(+), 10 deletions(-) rename pkg/ndjson/testdata/project/{arbitrary-shape.golden => arbitrary-shape.expected.txt} (100%) rename pkg/ndjson/testdata/project/{envelope.golden => envelope.expected.txt} (100%) rename pkg/ndjson/testdata/project/{no-message-fallback.golden => no-message-fallback.expected.txt} (100%) rename pkg/ndjson/testdata/project/{numeric-level-ignored.golden => numeric-level-ignored.expected.txt} (100%) rename pkg/ndjson/testdata/project/{severity-via-level.golden => severity-via-level.expected.txt} (100%) diff --git a/pkg/ndjson/project_test.go b/pkg/ndjson/project_test.go index e3ae89b..9699b06 100644 --- a/pkg/ndjson/project_test.go +++ b/pkg/ndjson/project_test.go @@ -7,11 +7,11 @@ import ( "testing" ) -// TestProjectGoldenFixtures projects every line of every input fixture under -// testdata/project/ and compares the result against its committed golden -// file. Each .input.ndjson pairs with .golden, so adding a case -// only means adding one fixture pair. -func TestProjectGoldenFixtures(t *testing.T) { +// TestProjectFixtures projects every line of every input fixture under +// testdata/project/ and compares the result against its committed expected +// output. Each .input.ndjson pairs with .expected.txt, so adding +// a case only means adding one fixture pair. +func TestProjectFixtures(t *testing.T) { inputs, err := filepath.Glob(filepath.Join("testdata", "project", "*.input.ndjson")) if err != nil { t.Fatalf("glob projection fixtures: %v", err) @@ -22,7 +22,7 @@ func TestProjectGoldenFixtures(t *testing.T) { for _, inputPath := range inputs { name := strings.TrimSuffix(filepath.Base(inputPath), ".input.ndjson") - goldenPath := filepath.Join("testdata", "project", name+".golden") + expectedPath := filepath.Join("testdata", "project", name+".expected.txt") t.Run(name, func(t *testing.T) { var projected []string for _, line := range readFixtureLines(t, inputPath) { @@ -33,12 +33,12 @@ func TestProjectGoldenFixtures(t *testing.T) { } got := strings.Join(projected, "\n") + "\n" - golden, err := os.ReadFile(goldenPath) + expected, err := os.ReadFile(expectedPath) if err != nil { - t.Fatalf("read golden %s: %v", goldenPath, err) + t.Fatalf("read expected output %s: %v", expectedPath, err) } - if got != string(golden) { - t.Fatalf("projection mismatch for %s\ngot:\n%swant:\n%s", inputPath, got, golden) + if got != string(expected) { + t.Fatalf("projection mismatch for %s\ngot:\n%swant:\n%s", inputPath, got, expected) } }) } diff --git a/pkg/ndjson/testdata/project/arbitrary-shape.golden b/pkg/ndjson/testdata/project/arbitrary-shape.expected.txt similarity index 100% rename from pkg/ndjson/testdata/project/arbitrary-shape.golden rename to pkg/ndjson/testdata/project/arbitrary-shape.expected.txt diff --git a/pkg/ndjson/testdata/project/envelope.golden b/pkg/ndjson/testdata/project/envelope.expected.txt similarity index 100% rename from pkg/ndjson/testdata/project/envelope.golden rename to pkg/ndjson/testdata/project/envelope.expected.txt diff --git a/pkg/ndjson/testdata/project/no-message-fallback.golden b/pkg/ndjson/testdata/project/no-message-fallback.expected.txt similarity index 100% rename from pkg/ndjson/testdata/project/no-message-fallback.golden rename to pkg/ndjson/testdata/project/no-message-fallback.expected.txt diff --git a/pkg/ndjson/testdata/project/numeric-level-ignored.golden b/pkg/ndjson/testdata/project/numeric-level-ignored.expected.txt similarity index 100% rename from pkg/ndjson/testdata/project/numeric-level-ignored.golden rename to pkg/ndjson/testdata/project/numeric-level-ignored.expected.txt diff --git a/pkg/ndjson/testdata/project/severity-via-level.golden b/pkg/ndjson/testdata/project/severity-via-level.expected.txt similarity index 100% rename from pkg/ndjson/testdata/project/severity-via-level.golden rename to pkg/ndjson/testdata/project/severity-via-level.expected.txt From 5bc9bd65de2450e08af91a543fb2a96404a83367 Mon Sep 17 00:00:00 2001 From: Zhiqiang ZHOU Date: Sat, 25 Jul 2026 13:13:38 -0700 Subject: [PATCH 3/3] refactor: rename projection concept to extraction The word project reads as both verb and noun and its Chinese translations collide. Extraction is unambiguous. --- CONTEXT.md | 6 ++-- ...json-log-files-with-message-extraction.md} | 8 +++--- pkg/ndjson/detect.go | 6 ++-- pkg/ndjson/{project.go => extract.go} | 10 +++---- .../{project_test.go => extract_test.go} | 28 +++++++++---------- .../arbitrary-shape.expected.txt | 0 .../arbitrary-shape.input.ndjson | 0 .../envelope.expected.txt | 0 .../envelope.input.ndjson | 0 .../no-message-fallback.expected.txt | 0 .../no-message-fallback.input.ndjson | 0 .../numeric-level-ignored.expected.txt | 0 .../numeric-level-ignored.input.ndjson | 0 .../severity-via-level.expected.txt | 0 .../severity-via-level.input.ndjson | 0 pkg/workspace/pipeline.go | 10 +++---- pkg/workspace/workspace.go | 10 +++---- 17 files changed, 39 insertions(+), 39 deletions(-) rename docs/adr/{0006-ndjson-log-files-with-message-projection.md => 0006-ndjson-log-files-with-message-extraction.md} (69%) rename pkg/ndjson/{project.go => extract.go} (92%) rename pkg/ndjson/{project_test.go => extract_test.go} (52%) rename pkg/ndjson/testdata/{project => extract}/arbitrary-shape.expected.txt (100%) rename pkg/ndjson/testdata/{project => extract}/arbitrary-shape.input.ndjson (100%) rename pkg/ndjson/testdata/{project => extract}/envelope.expected.txt (100%) rename pkg/ndjson/testdata/{project => extract}/envelope.input.ndjson (100%) rename pkg/ndjson/testdata/{project => extract}/no-message-fallback.expected.txt (100%) rename pkg/ndjson/testdata/{project => extract}/no-message-fallback.input.ndjson (100%) rename pkg/ndjson/testdata/{project => extract}/numeric-level-ignored.expected.txt (100%) rename pkg/ndjson/testdata/{project => extract}/numeric-level-ignored.input.ndjson (100%) rename pkg/ndjson/testdata/{project => extract}/severity-via-level.expected.txt (100%) rename pkg/ndjson/testdata/{project => extract}/severity-via-level.input.ndjson (100%) diff --git a/CONTEXT.md b/CONTEXT.md index 43da09e..34b85f9 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -27,6 +27,6 @@ _Avoid_: Sync, connection, integration, LogImport An external logging service LAPP can import from, such as GCP Cloud Logging or Vercel. LAPP uses credentials already present on the machine and never manages provider authentication itself. _Avoid_: Source, backend -**Projection**: -The text line derived from a structured log entry for pattern discovery. Discovery reads the projection; investigation material keeps the full structured entry. -_Avoid_: Flattening, rendering +**Extraction**: +The text line pulled out of a structured log entry for pattern discovery. Discovery reads the extraction; investigation material keeps the full structured entry. +_Avoid_: Projection, flattening, rendering diff --git a/docs/adr/0006-ndjson-log-files-with-message-projection.md b/docs/adr/0006-ndjson-log-files-with-message-extraction.md similarity index 69% rename from docs/adr/0006-ndjson-log-files-with-message-projection.md rename to docs/adr/0006-ndjson-log-files-with-message-extraction.md index 03d44b3..e06955c 100644 --- a/docs/adr/0006-ndjson-log-files-with-message-projection.md +++ b/docs/adr/0006-ndjson-log-files-with-message-extraction.md @@ -1,16 +1,16 @@ -# Structured logs are stored as NDJSON and discovered via message projection +# Structured logs are stored as NDJSON and discovered via message extraction Log files can be NDJSON (one JSON entry per line) as well as plain text, detected per file by the pipeline — the capability belongs to the pipeline, not to any import provider, so hand-uploaded NDJSON files get the same treatment as imported ones. Imported GCP entries land in a fixed envelope: `{"ts": ..., "severity": ..., "payload": {...}}` with the jsonPayload nested untouched (textPayload becomes `payload.message`); provider metadata such as labels, trace, and resource stays in the ImportRun record, not in the log lines. -For pattern discovery, JSON entries are projected to a text line rather than fed to Drain whole: the first string field among `payload.message`, `msg`, `log`, `error` becomes the projection (` `); if none exists, the compact payload JSON is the fallback. Timestamps are excluded from the projection to keep Drain templates clean. +For pattern discovery, a text line is extracted from each JSON entry rather than feeding Drain the whole entry: the first string field among `payload.message`, `msg`, `log`, `error` becomes the extraction (` `); if none exists, the compact payload JSON is the fallback. Timestamps are excluded from the extraction to keep Drain templates clean. **Considered Options** - Flatten everything to text at import time (structure lost) - NDJSON storage, discovery on compact-JSON lines (structure kept, dirty templates) -- NDJSON storage with message projection (chosen) +- NDJSON storage with message extraction (chosen) - JSON-native pattern discovery by structure/key-set (a second discovery engine — deferred, not rejected) **Consequences** -The envelope and the projection rule are part of the workspace file contract (ADR 0001). Payload fields are never flattened to the top level, so user fields named `severity` or `ts` cannot collide with the envelope. Analysis agents can query structure with jq-style tools instead of grepping flattened text. If real usage shows dirty templates for payloads without a message-like field, the projection rule is the extension point. +The envelope and the extraction rule are part of the workspace file contract (ADR 0001). Payload fields are never flattened to the top level, so user fields named `severity` or `ts` cannot collide with the envelope. Analysis agents can query structure with jq-style tools instead of grepping flattened text. If real usage shows dirty templates for payloads without a message-like field, the extraction rule is the extension point. diff --git a/pkg/ndjson/detect.go b/pkg/ndjson/detect.go index c8abac4..630c0fb 100644 --- a/pkg/ndjson/detect.go +++ b/pkg/ndjson/detect.go @@ -1,5 +1,5 @@ -// Package ndjson classifies log files as NDJSON and projects structured -// entries to text lines for pattern mining, per ADR 0006. +// Package ndjson classifies log files as NDJSON and extracts the text lines +// fed to pattern mining from structured entries, per ADR 0006. package ndjson import ( @@ -20,7 +20,7 @@ const ( // DetectFormat classifies a file's lines. A file is NDJSON only when it has // at least one non-empty line and every non-empty line parses as a JSON // object. Files mixing text with JSON lines stay on the text path, so the -// projection never has to guess on a half-structured file. +// extraction never has to guess on a half-structured file. func DetectFormat(lines []string) Format { sampled := 0 for _, line := range lines { diff --git a/pkg/ndjson/project.go b/pkg/ndjson/extract.go similarity index 92% rename from pkg/ndjson/project.go rename to pkg/ndjson/extract.go index fca09f5..bdd6fa4 100644 --- a/pkg/ndjson/project.go +++ b/pkg/ndjson/extract.go @@ -13,13 +13,13 @@ var messageFields = []string{"message", "msg", "log", "error"} // Numeric levels are ignored in v1. var severityFields = []string{"severity", "level"} -// Project converts one NDJSON line into the text line fed to pattern mining. -// Envelope entries {"ts": ..., "severity": ..., "payload": {...}} project -// from the payload; any other object is the payload itself. The projection is +// Extract converts one NDJSON line into the text line fed to pattern mining. +// Envelope entries {"ts": ..., "severity": ..., "payload": {...}} extract +// from the payload; any other object is the payload itself. The extracted line is // " ", just "" when no severity-like string field // exists, or the compact payload JSON when no message-like string field // exists. A line that does not parse as a JSON object is returned unchanged. -func Project(line string) string { +func Extract(line string) string { var entry map[string]any if err := json.Unmarshal([]byte(line), &entry); err != nil || entry == nil { return line @@ -44,7 +44,7 @@ func Project(line string) string { // entry matches the fixed envelope shape, otherwise the entry itself. The // envelope is the importer's fixed contract (ADR 0006): ts, a string severity, // and a payload object must all be present; anything less is an arbitrary -// user shape and is projected as a whole. +// user shape and is extracted as a whole. func splitEnvelope(entry map[string]any) (payload map[string]any, severity string) { nested, ok := entry["payload"].(map[string]any) if !ok { diff --git a/pkg/ndjson/project_test.go b/pkg/ndjson/extract_test.go similarity index 52% rename from pkg/ndjson/project_test.go rename to pkg/ndjson/extract_test.go index 9699b06..721e27c 100644 --- a/pkg/ndjson/project_test.go +++ b/pkg/ndjson/extract_test.go @@ -7,46 +7,46 @@ import ( "testing" ) -// TestProjectFixtures projects every line of every input fixture under -// testdata/project/ and compares the result against its committed expected +// TestExtractFixtures extracts every line of every input fixture under +// testdata/extract/ and compares the result against its committed expected // output. Each .input.ndjson pairs with .expected.txt, so adding // a case only means adding one fixture pair. -func TestProjectFixtures(t *testing.T) { - inputs, err := filepath.Glob(filepath.Join("testdata", "project", "*.input.ndjson")) +func TestExtractFixtures(t *testing.T) { + inputs, err := filepath.Glob(filepath.Join("testdata", "extract", "*.input.ndjson")) if err != nil { - t.Fatalf("glob projection fixtures: %v", err) + t.Fatalf("glob extraction fixtures: %v", err) } if len(inputs) == 0 { - t.Fatal("no projection fixtures found under testdata/project") + t.Fatal("no extraction fixtures found under testdata/extract") } for _, inputPath := range inputs { name := strings.TrimSuffix(filepath.Base(inputPath), ".input.ndjson") - expectedPath := filepath.Join("testdata", "project", name+".expected.txt") + expectedPath := filepath.Join("testdata", "extract", name+".expected.txt") t.Run(name, func(t *testing.T) { - var projected []string + var extracted []string for _, line := range readFixtureLines(t, inputPath) { if strings.TrimSpace(line) == "" { continue } - projected = append(projected, Project(line)) + extracted = append(extracted, Extract(line)) } - got := strings.Join(projected, "\n") + "\n" + got := strings.Join(extracted, "\n") + "\n" expected, err := os.ReadFile(expectedPath) if err != nil { t.Fatalf("read expected output %s: %v", expectedPath, err) } if got != string(expected) { - t.Fatalf("projection mismatch for %s\ngot:\n%swant:\n%s", inputPath, got, expected) + t.Fatalf("extraction mismatch for %s\ngot:\n%swant:\n%s", inputPath, got, expected) } }) } } -func TestProjectNonJSONLineIsReturnedUnchanged(t *testing.T) { +func TestExtractNonJSONLineIsReturnedUnchanged(t *testing.T) { line := "2026-06-06 10:00:00 INFO plain text line" - if got := Project(line); got != line { - t.Fatalf("Project(%q) = %q, want unchanged", line, got) + if got := Extract(line); got != line { + t.Fatalf("Extract(%q) = %q, want unchanged", line, got) } } diff --git a/pkg/ndjson/testdata/project/arbitrary-shape.expected.txt b/pkg/ndjson/testdata/extract/arbitrary-shape.expected.txt similarity index 100% rename from pkg/ndjson/testdata/project/arbitrary-shape.expected.txt rename to pkg/ndjson/testdata/extract/arbitrary-shape.expected.txt diff --git a/pkg/ndjson/testdata/project/arbitrary-shape.input.ndjson b/pkg/ndjson/testdata/extract/arbitrary-shape.input.ndjson similarity index 100% rename from pkg/ndjson/testdata/project/arbitrary-shape.input.ndjson rename to pkg/ndjson/testdata/extract/arbitrary-shape.input.ndjson diff --git a/pkg/ndjson/testdata/project/envelope.expected.txt b/pkg/ndjson/testdata/extract/envelope.expected.txt similarity index 100% rename from pkg/ndjson/testdata/project/envelope.expected.txt rename to pkg/ndjson/testdata/extract/envelope.expected.txt diff --git a/pkg/ndjson/testdata/project/envelope.input.ndjson b/pkg/ndjson/testdata/extract/envelope.input.ndjson similarity index 100% rename from pkg/ndjson/testdata/project/envelope.input.ndjson rename to pkg/ndjson/testdata/extract/envelope.input.ndjson diff --git a/pkg/ndjson/testdata/project/no-message-fallback.expected.txt b/pkg/ndjson/testdata/extract/no-message-fallback.expected.txt similarity index 100% rename from pkg/ndjson/testdata/project/no-message-fallback.expected.txt rename to pkg/ndjson/testdata/extract/no-message-fallback.expected.txt diff --git a/pkg/ndjson/testdata/project/no-message-fallback.input.ndjson b/pkg/ndjson/testdata/extract/no-message-fallback.input.ndjson similarity index 100% rename from pkg/ndjson/testdata/project/no-message-fallback.input.ndjson rename to pkg/ndjson/testdata/extract/no-message-fallback.input.ndjson diff --git a/pkg/ndjson/testdata/project/numeric-level-ignored.expected.txt b/pkg/ndjson/testdata/extract/numeric-level-ignored.expected.txt similarity index 100% rename from pkg/ndjson/testdata/project/numeric-level-ignored.expected.txt rename to pkg/ndjson/testdata/extract/numeric-level-ignored.expected.txt diff --git a/pkg/ndjson/testdata/project/numeric-level-ignored.input.ndjson b/pkg/ndjson/testdata/extract/numeric-level-ignored.input.ndjson similarity index 100% rename from pkg/ndjson/testdata/project/numeric-level-ignored.input.ndjson rename to pkg/ndjson/testdata/extract/numeric-level-ignored.input.ndjson diff --git a/pkg/ndjson/testdata/project/severity-via-level.expected.txt b/pkg/ndjson/testdata/extract/severity-via-level.expected.txt similarity index 100% rename from pkg/ndjson/testdata/project/severity-via-level.expected.txt rename to pkg/ndjson/testdata/extract/severity-via-level.expected.txt diff --git a/pkg/ndjson/testdata/project/severity-via-level.input.ndjson b/pkg/ndjson/testdata/extract/severity-via-level.input.ndjson similarity index 100% rename from pkg/ndjson/testdata/project/severity-via-level.input.ndjson rename to pkg/ndjson/testdata/extract/severity-via-level.input.ndjson diff --git a/pkg/workspace/pipeline.go b/pkg/workspace/pipeline.go index ee4ede3..3663c73 100644 --- a/pkg/workspace/pipeline.go +++ b/pkg/workspace/pipeline.go @@ -313,7 +313,7 @@ func mergeAllLogs(ctx context.Context, dir string) (tagged []TaggedLine, content } // tagFileLines converts one log file into tagged entries. NDJSON files keep -// the raw JSON line as Content and carry a text projection for pattern +// the raw JSON line as Content and carry an extracted text line for pattern // mining; plain text files go through multiline merging unchanged. func tagFileLines(ctx context.Context, fileName string, lines []string) ([]TaggedLine, error) { if ndjson.DetectFormat(lines) == ndjson.FormatNDJSON { @@ -342,10 +342,10 @@ func tagNDJSONLines(fileName string, lines []string) []TaggedLine { continue } tagged = append(tagged, TaggedLine{ - Content: line, - FileName: fileName, - LineNum: i + 1, - Projection: ndjson.Project(line), + Content: line, + FileName: fileName, + LineNum: i + 1, + ExtractedLine: ndjson.Extract(line), }) } return tagged diff --git a/pkg/workspace/workspace.go b/pkg/workspace/workspace.go index 8eb4d22..78f934e 100644 --- a/pkg/workspace/workspace.go +++ b/pkg/workspace/workspace.go @@ -12,16 +12,16 @@ type TaggedLine struct { Content string FileName string LineNum int - // Projection is the text fed to pattern mining for NDJSON entries. + // ExtractedLine is the text fed to pattern mining for NDJSON entries. // Empty for plain text entries, whose Content is mined directly. - Projection string `json:",omitempty"` + ExtractedLine string `json:",omitempty"` } // DrainLine returns the text used for pattern mining and template matching: -// the projection for NDJSON entries, otherwise the raw content. +// the extracted line for NDJSON entries, otherwise the raw content. func (t TaggedLine) DrainLine() string { - if t.Projection != "" { - return t.Projection + if t.ExtractedLine != "" { + return t.ExtractedLine } return t.Content }