From f121acfea4596e393239fd2aaca621a4cdf310ef Mon Sep 17 00:00:00 2001 From: Jens Papenhagen Date: Sun, 9 Aug 2026 08:15:15 +0200 Subject: [PATCH 1/5] adding new conformance file for decode commants change in the specs 4.1.1 --- .../conformance/decode/comments.json | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/test/resources/conformance/decode/comments.json b/src/test/resources/conformance/decode/comments.json index ac6432d..e351082 100644 --- a/src/test/resources/conformance/decode/comments.json +++ b/src/test/resources/conformance/decode/comments.json @@ -139,6 +139,28 @@ "specSection": "5.1", "note": "Only spaces (U+0020) may precede the #; the tab makes this a regular line, and tabs in indentation error in strict mode (§12)" }, + { + "name": "decodes tab-indented hash row as data in non-strict mode", + "input": "items[3]{id}:\n 1\n\t#x\n 2", + "expected": { + "items": [ + { + "id": 1 + }, + { + "id": "#x" + }, + { + "id": 2 + } + ] + }, + "options": { + "strict": false + }, + "specSection": "5.1", + "note": "Comment removal precedes the §12 tab leniency, so the tab keeps the line out of the pre-pass" + }, { "name": "parses quoted hash-leading first cell as data, not comment", "input": "items[1]{tag}:\n \"#a\"", From 7db79105eb57a64a22c2ac7c64afc539b7c876fe Mon Sep 17 00:00:00 2001 From: Jens Papenhagen Date: Sun, 9 Aug 2026 08:25:58 +0200 Subject: [PATCH 2/5] update docs --- CHANGELOG.md | 2 +- README.md | 2 +- docs/FORMAT.md | 6 ++++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b136021..340eda6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ This project adheres to Semantic Versioning and follows a Keep a Changelog-like ### Changed -- Full TOON Spec 4.1 conformance: canonical number formatting, BOM stripping, comment pre-pass (§5.1), strict header validation (§5, §6, §7.3, §7.4), nested field groups in tabular arrays (§9.3), and keyed tabular form for objects of uniform objects, including the keyless root form and keyed headers on list-item hyphen lines (§9.5, §10). Conformance suite: 95/95 passing. +- Full TOON Spec 4.1.1 conformance: canonical number formatting, BOM stripping, comment pre-pass (§5.1), strict header validation (§5, §6, §7.3, §7.4), nested field groups in tabular arrays (§9.3), and keyed tabular form for objects of uniform objects, including the keyless root form and keyed headers on list-item hyphen lines (§9.5, §10). Non-strict tab leniency: leading tabs are accepted as indentation and expanded to `indent` spaces before classification (§12); a tab-indented `#` line is data, not a comment. Conformance suite: 95/95 passing. ## [2.0.1] - 2026-07-11 diff --git a/README.md b/README.md index 83facda..fb5a087 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Release](https://github.com/toon-format/toon-java/actions/workflows/release.yml/badge.svg)](https://github.com/toon-format/toon-java/actions/workflows/release.yml) [![Maven Central](https://img.shields.io/maven-central/v/dev.toonformat/jtoon.svg)](https://central.sonatype.com/artifact/dev.toonformat/jtoon) ![Coverage](.github/badges/jacoco.svg) -[![SPEC v4.1](https://img.shields.io/badge/spec-v4.1-fef3c0?labelColor=1b1b1f)](https://github.com/toon-format/spec) +[![SPEC v4.1.1](https://img.shields.io/badge/spec-v4.1.1-fef3c0?labelColor=1b1b1f)](https://github.com/toon-format/spec) [![License: MIT](https://img.shields.io/badge/license-MIT-fef3c0?labelColor=1b1b1f)](./LICENSE) Compact, human-readable serialization format for LLM contexts with **30-60% token reduction** vs JSON. Combines YAML-like indentation with CSV-like tabular arrays. Working towards full compatibility with the [official TOON specification](https://github.com/toon-format/spec). diff --git a/docs/FORMAT.md b/docs/FORMAT.md index 0e392d8..2cc9cc7 100644 --- a/docs/FORMAT.md +++ b/docs/FORMAT.md @@ -483,6 +483,12 @@ level1: - Tabs not allowed in indentation - Mixing spaces and tabs causes errors +**Non-strict mode (§12 leniency):** + +- Depth may be computed as `floor(leadingSpaces / indent)` +- Leading tabs are accepted as indentation and removed from the line's content before classification (§5.2). Depth computation for tabs is implementation-defined: JToon expands each leading tab to `indent` spaces, so a leading tab contributes exactly one indentation level. +- Because comment detection precedes the tab leniency (§5.1), a tab-indented `#` line is data, not a comment. + --- ## Array Length Indicators From dd132de5a098d4fce59e657efbc584b7f83586ee Mon Sep 17 00:00:00 2001 From: Jens Papenhagen Date: Sun, 9 Aug 2026 08:26:22 +0200 Subject: [PATCH 3/5] update valueDecoder --- .../jtoon/decoder/ValueDecoder.java | 47 +++++++++++++++++-- .../jtoon/decoder/DecodeHelperTest.java | 4 +- .../jtoon/decoder/ValueDecoderTest.java | 44 +++++++++++++++++ 3 files changed, 90 insertions(+), 5 deletions(-) diff --git a/src/main/java/dev/toonformat/jtoon/decoder/ValueDecoder.java b/src/main/java/dev/toonformat/jtoon/decoder/ValueDecoder.java index 8530a71..35b6a3d 100644 --- a/src/main/java/dev/toonformat/jtoon/decoder/ValueDecoder.java +++ b/src/main/java/dev/toonformat/jtoon/decoder/ValueDecoder.java @@ -89,8 +89,8 @@ private static Object decodeInternal(final String toon, final DecodeOptions opti //set an own decode context final DecodeContext context = new DecodeContext(); - context.lines = buildContentLines(input.split("\r?\n", -1)); context.options = options; + context.lines = buildContentLines(input.split("\r?\n", -1), options); context.delimiter = options.delimiter(); // Spec §5.1: a document of only comments and blank lines is an empty object @@ -127,8 +127,12 @@ private static String stripByteOrderMark(final String input) { /** * Builds the list of content lines: trailing spaces are stripped per line * (§12) and full-line comments are discarded (§5.1). + * + * @param rawLines the raw input lines + * @param options decode options (strict mode, indent size) + * @return the content lines to parse */ - private static String[] buildContentLines(final String... rawLines) { + private static String[] buildContentLines(final String[] rawLines, final DecodeOptions options) { // Spec §12: trailing spaces at the end of a line are not part of its content; // strip them per line before classification. Only characters after the last // non-space character are removed, so trailing spaces inside quoted strings @@ -138,15 +142,50 @@ private static String[] buildContentLines(final String... rawLines) { // interpretation. A tab before '#' disqualifies the line, and a '#' // anywhere else is data, not a comment. final List contentLines = new ArrayList<>(rawLines.length); - for (String rawLine : rawLines) { + for (final String rawLine : rawLines) { final String stripped = rawLine.stripTrailing(); if (!isCommentLine(stripped)) { - contentLines.add(stripped); + contentLines.add(expandLeadingTabs(stripped, options)); } } return contentLines.toArray(new String[0]); } + /** + * Spec §12 (non-strict mode): implementations MAY accept tab characters in + * indentation; when they do, leading tabs are indentation and MUST be + * removed from the line's content before classification (§5.2). The depth + * computation for tabs is implementation-defined. JToon expands each + * leading tab to {@code indentSize} spaces, so a leading tab contributes + * exactly one indentation level. + * + *

In strict mode tabs in indentation are errors (§12), so no expansion + * is performed and the tab is left for {@link DecodeHelper#getDepth} to + * reject. A tab before a leading '#' has already kept the line out of the + * §5.1 comment pre-pass, so a tab-indented hash row is data, not a + * comment.

+ * + * @param line the stripped line to process + * @param options decode options (strict mode, indent size) + * @return the line with leading tabs expanded to indentSize spaces in + * non-strict mode, otherwise the unchanged line + */ + private static String expandLeadingTabs(final String line, final DecodeOptions options) { + if (options.strict()) { + return line; + } + int i = 0; + while (i < line.length() && (line.charAt(i) == ' ' || line.charAt(i) == '\t')) { + i++; + } + final String leading = line.substring(0, i); + if (leading.indexOf('\t') < 0) { + return line; + } + final String expanded = leading.replace("\t", " ".repeat(options.indent())); + return expanded + line.substring(i); + } + private static boolean isEmptyDocument(final String... lines) { for (final String line : lines) { if (!line.isBlank()) { diff --git a/src/test/java/dev/toonformat/jtoon/decoder/DecodeHelperTest.java b/src/test/java/dev/toonformat/jtoon/decoder/DecodeHelperTest.java index 05efc67..480dce3 100644 --- a/src/test/java/dev/toonformat/jtoon/decoder/DecodeHelperTest.java +++ b/src/test/java/dev/toonformat/jtoon/decoder/DecodeHelperTest.java @@ -710,7 +710,9 @@ void testLeadingSpacesStrictValidMultiple() throws Exception { @Test void testTabNonStrictStopsCounting() throws Exception { - // in non-strict mode, indentation stops at first non-space (including tab) + // Direct calls to computeLeadingSpaces stop at the first non-space. + // The §12 tab leniency is applied earlier, in the ValueDecoder + // pre-pass, where leading tabs are expanded to indentSize spaces. assertEquals(2, invokeCompute(" \t text", ctxNonStrict2)); } diff --git a/src/test/java/dev/toonformat/jtoon/decoder/ValueDecoderTest.java b/src/test/java/dev/toonformat/jtoon/decoder/ValueDecoderTest.java index 20a37db..a680ff5 100644 --- a/src/test/java/dev/toonformat/jtoon/decoder/ValueDecoderTest.java +++ b/src/test/java/dev/toonformat/jtoon/decoder/ValueDecoderTest.java @@ -117,6 +117,50 @@ void decode_discardsFullLineComments() { assertEquals("{name=Ada}", result.toString()); } + @Test + @DisplayName("throws on a tab-indented hash line in strict mode, which is not a comment") + void decode_throwsOnTabIndentedHashLineInStrictMode() { + // Given + // Spec §5.1: only U+0020 spaces may precede the '#', so a tab keeps the + // line out of the comment pre-pass; §12 then rejects the tab as + // indentation in strict mode. + final String input = "items[1]{tag}:\n\t#a"; + + // When + assertThrows(IllegalArgumentException.class, + () -> ValueDecoder.decode(input, DecodeOptions.DEFAULT)); + } + + @Test + @DisplayName("decodes a tab-indented hash row as data in non-strict mode") + void decode_tabIndentedHashRowInNonStrictMode() { + // Given + // Spec §12 non-strict: leading tabs are accepted as indentation and + // removed from the line's content before classification (§5.2). + final String input = "items[3]{id}:\n 1\n\t#x\n 2"; + + // When + final Object result = ValueDecoder.decode(input, DecodeOptions.withStrict(false)); + + // Then + assertEquals("{items=[{id=1}, {id=#x}, {id=2}]}", result.toString()); + } + + @Test + @DisplayName("expands a leading tab to one indentation level in non-strict mode") + void decode_expandsLeadingTabToOneIndentLevelInNonStrictMode() { + // Given + // Spec §12: depth computation for tabs is implementation-defined; JToon + // expands each leading tab to indentSize spaces (one level). + final String input = "outer:\n\tinner: 1"; + + // When + final Object result = ValueDecoder.decode(input, DecodeOptions.withStrict(false)); + + // Then + assertEquals("{outer={inner=1}}", result.toString()); + } + @Test @DisplayName("treats a hash not at line start as data") void decode_hashInsideLineIsData() { From 0f648a8ac17e3d70f1c052b2ffe4a11cc6a366be Mon Sep 17 00:00:00 2001 From: Jens Papenhagen Date: Sun, 9 Aug 2026 08:31:31 +0200 Subject: [PATCH 4/5] update deps --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 69bcad5..86d7b43 100644 --- a/build.gradle +++ b/build.gradle @@ -3,10 +3,10 @@ plugins { id 'maven-publish' id 'signing' id 'jacoco' - id 'com.github.spotbugs' version '6.5.9' + id 'com.github.spotbugs' version '6.5.10' id 'pmd' id 'checkstyle' - id 'org.owasp.dependencycheck' version '12.2.2' + id 'org.owasp.dependencycheck' version '13.0.0' id 'org.cyclonedx.bom' version '3.3.0' id 'info.solidsoft.pitest' version '1.19.0' id 'net.ltgt.errorprone' version '5.1.0' From 0a678f014f69330fc40249b323a6e0061fd42603 Mon Sep 17 00:00:00 2001 From: Jens Papenhagen Date: Sun, 9 Aug 2026 08:31:58 +0200 Subject: [PATCH 5/5] update metadata --- gradle/verification-metadata.xml | 264 +++++++++++++++++++++++++++++++ 1 file changed, 264 insertions(+) diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 3515691..4e971db 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -72,6 +72,11 @@ + + + + + @@ -92,6 +97,11 @@ + + + + + @@ -139,6 +149,14 @@ + + + + + + + + @@ -147,6 +165,14 @@ + + + + + + + + @@ -160,6 +186,14 @@ + + + + + + + + @@ -168,11 +202,24 @@ + + + + + + + + + + + + + @@ -181,6 +228,14 @@ + + + + + + + + @@ -189,16 +244,34 @@ + + + + + + + + + + + + + + + + + + @@ -207,6 +280,14 @@ + + + + + + + + @@ -242,6 +323,11 @@ + + + + + @@ -296,6 +382,14 @@ + + + + + + + + @@ -856,11 +950,24 @@ + + + + + + + + + + + + + @@ -1437,6 +1544,14 @@ + + + + + + + + @@ -1445,6 +1560,14 @@ + + + + + + + + @@ -1460,6 +1583,16 @@ + + + + + + + + + + @@ -1476,6 +1609,14 @@ + + + + + + + + @@ -1492,6 +1633,14 @@ + + + + + + + + @@ -1507,6 +1656,11 @@ + + + + + @@ -2127,6 +2281,14 @@ + + + + + + + + @@ -2216,16 +2378,34 @@ + + + + + + + + + + + + + + + + + + @@ -2324,6 +2504,14 @@ + + + + + + + + @@ -2332,6 +2520,14 @@ + + + + + + + + @@ -2832,6 +3028,14 @@ + + + + + + + + @@ -2840,11 +3044,24 @@ + + + + + + + + + + + + + @@ -2853,11 +3070,24 @@ + + + + + + + + + + + + + @@ -2949,6 +3179,14 @@ + + + + + + + + @@ -2965,6 +3203,14 @@ + + + + + + + + @@ -2973,11 +3219,24 @@ + + + + + + + + + + + + + @@ -2988,6 +3247,11 @@ + + + + +