Sec411

CHANGELOG.md

@@ -8,7 +8,7 @@ This project adheres to Semantic Versioning and follows a Keep a Changelog-like
 
 ### Changed
 
--   Full TOON Spec 4.1 conformance: canonical number formatting, BOM stripping, comment pre-pass (§5.1), strict header validation (§5, §6, §7.3, §7.4), nested field groups in tabular arrays (§9.3), and keyed tabular form for objects of uniform objects, including the keyless root form and keyed headers on list-item hyphen lines (§9.5, §10). Conformance suite: 95/95 passing.
+-   Full TOON Spec 4.1.1 conformance: canonical number formatting, BOM stripping, comment pre-pass (§5.1), strict header validation (§5, §6, §7.3, §7.4), nested field groups in tabular arrays (§9.3), and keyed tabular form for objects of uniform objects, including the keyless root form and keyed headers on list-item hyphen lines (§9.5, §10). Non-strict tab leniency: leading tabs are accepted as indentation and expanded to `indent` spaces before classification (§12); a tab-indented `#` line is data, not a comment. Conformance suite: 95/95 passing.
 
 ## [2.0.1] - 2026-07-11
 

README.md

@@ -4,7 +4,7 @@
 [![Release](https://github.com/toon-format/toon-java/actions/workflows/release.yml/badge.svg)](https://github.com/toon-format/toon-java/actions/workflows/release.yml)
 [![Maven Central](https://img.shields.io/maven-central/v/dev.toonformat/jtoon.svg)](https://central.sonatype.com/artifact/dev.toonformat/jtoon)
 ![Coverage](.github/badges/jacoco.svg)
-[![SPEC v4.1](https://img.shields.io/badge/spec-v4.1-fef3c0?labelColor=1b1b1f)](https://github.com/toon-format/spec)
+[![SPEC v4.1.1](https://img.shields.io/badge/spec-v4.1.1-fef3c0?labelColor=1b1b1f)](https://github.com/toon-format/spec)
 [![License: MIT](https://img.shields.io/badge/license-MIT-fef3c0?labelColor=1b1b1f)](./LICENSE)
 
 Compact, human-readable serialization format for LLM contexts with **30-60% token reduction** vs JSON. Combines YAML-like indentation with CSV-like tabular arrays. Working towards full compatibility with the [official TOON specification](https://github.com/toon-format/spec).

build.gradle

@@ -3,10 +3,10 @@ plugins {
     id 'maven-publish'
     id 'signing'
     id 'jacoco'
-    id 'com.github.spotbugs' version '6.5.9'
+    id 'com.github.spotbugs' version '6.5.10'
     id 'pmd'
     id 'checkstyle'
-    id 'org.owasp.dependencycheck' version '12.2.2'
+    id 'org.owasp.dependencycheck' version '13.0.0'
     id 'org.cyclonedx.bom' version '3.3.0'
     id 'info.solidsoft.pitest' version '1.19.0'
     id 'net.ltgt.errorprone' version '5.1.0'

docs/FORMAT.md

@@ -483,6 +483,12 @@ level1:
 - Tabs not allowed in indentation
 - Mixing spaces and tabs causes errors
 
+**Non-strict mode (§12 leniency):**
+
+- Depth may be computed as `floor(leadingSpaces / indent)`
+- Leading tabs are accepted as indentation and removed from the line's content before classification (§5.2). Depth computation for tabs is implementation-defined: JToon expands each leading tab to `indent` spaces, so a leading tab contributes exactly one indentation level.
+- Because comment detection precedes the tab leniency (§5.1), a tab-indented `#` line is data, not a comment.
+
 ---
 
 ## Array Length Indicators

src/main/java/dev/toonformat/jtoon/decoder/ValueDecoder.java

@@ -89,8 +89,8 @@ private static Object decodeInternal(final String toon, final DecodeOptions opti
 
         //set an own decode context
         final DecodeContext context = new DecodeContext();
-        context.lines = buildContentLines(input.split("\r?\n", -1));
         context.options = options;
+        context.lines = buildContentLines(input.split("\r?\n", -1), options);
         context.delimiter = options.delimiter();
 
         // Spec §5.1: a document of only comments and blank lines is an empty object
@@ -127,8 +127,12 @@ private static String stripByteOrderMark(final String input) {
     /**
      * Builds the list of content lines: trailing spaces are stripped per line
      * (§12) and full-line comments are discarded (§5.1).
+     *
+     * @param rawLines the raw input lines
+     * @param options  decode options (strict mode, indent size)
+     * @return the content lines to parse
      */
-    private static String[] buildContentLines(final String... rawLines) {
+    private static String[] buildContentLines(final String[] rawLines, final DecodeOptions options) {
         // Spec §12: trailing spaces at the end of a line are not part of its content;
         // strip them per line before classification. Only characters after the last
         // non-space character are removed, so trailing spaces inside quoted strings
@@ -138,15 +142,50 @@ private static String[] buildContentLines(final String... rawLines) {
         // interpretation. A tab before '#' disqualifies the line, and a '#'
         // anywhere else is data, not a comment.
         final List<String> contentLines = new ArrayList<>(rawLines.length);
-        for (String rawLine : rawLines) {
+        for (final String rawLine : rawLines) {
             final String stripped = rawLine.stripTrailing();
             if (!isCommentLine(stripped)) {
-                contentLines.add(stripped);
+                contentLines.add(expandLeadingTabs(stripped, options));
             }
         }
         return contentLines.toArray(new String[0]);
     }
 
+    /**
+     * Spec §12 (non-strict mode): implementations MAY accept tab characters in
+     * indentation; when they do, leading tabs are indentation and MUST be
+     * removed from the line's content before classification (§5.2). The depth
+     * computation for tabs is implementation-defined. JToon expands each
+     * leading tab to {@code indentSize} spaces, so a leading tab contributes
+     * exactly one indentation level.
+     *
+     * <p>In strict mode tabs in indentation are errors (§12), so no expansion
+     * is performed and the tab is left for {@link DecodeHelper#getDepth} to
+     * reject. A tab before a leading '#' has already kept the line out of the
+     * §5.1 comment pre-pass, so a tab-indented hash row is data, not a
+     * comment.</p>
+     *
+     * @param line    the stripped line to process
+     * @param options decode options (strict mode, indent size)
+     * @return the line with leading tabs expanded to indentSize spaces in
+     *         non-strict mode, otherwise the unchanged line
+     */
+    private static String expandLeadingTabs(final String line, final DecodeOptions options) {
+        if (options.strict()) {
+            return line;
+        }
+        int i = 0;
+        while (i < line.length() && (line.charAt(i) == ' ' || line.charAt(i) == '\t')) {
+            i++;
+        }
+        final String leading = line.substring(0, i);
+        if (leading.indexOf('\t') < 0) {
+            return line;
+        }
+        final String expanded = leading.replace("\t", " ".repeat(options.indent()));
+        return expanded + line.substring(i);
+    }
+
     private static boolean isEmptyDocument(final String... lines) {
         for (final String line : lines) {
             if (!line.isBlank()) {

src/test/java/dev/toonformat/jtoon/decoder/DecodeHelperTest.java

@@ -710,7 +710,9 @@ void testLeadingSpacesStrictValidMultiple() throws Exception {
 
         @Test
         void testTabNonStrictStopsCounting() throws Exception {
-            // in non-strict mode, indentation stops at first non-space (including tab)
+            // Direct calls to computeLeadingSpaces stop at the first non-space.
+            // The §12 tab leniency is applied earlier, in the ValueDecoder
+            // pre-pass, where leading tabs are expanded to indentSize spaces.
             assertEquals(2, invokeCompute("  \t   text", ctxNonStrict2));
         }
 

src/test/java/dev/toonformat/jtoon/decoder/ValueDecoderTest.java

@@ -117,6 +117,50 @@ void decode_discardsFullLineComments() {
         assertEquals("{name=Ada}", result.toString());
     }
 
+    @Test
+    @DisplayName("throws on a tab-indented hash line in strict mode, which is not a comment")
+    void decode_throwsOnTabIndentedHashLineInStrictMode() {
+        // Given
+        // Spec §5.1: only U+0020 spaces may precede the '#', so a tab keeps the
+        // line out of the comment pre-pass; §12 then rejects the tab as
+        // indentation in strict mode.
+        final String input = "items[1]{tag}:\n\t#a";
+
+        // When
+        assertThrows(IllegalArgumentException.class,
+            () -> ValueDecoder.decode(input, DecodeOptions.DEFAULT));
+    }
+
+    @Test
+    @DisplayName("decodes a tab-indented hash row as data in non-strict mode")
+    void decode_tabIndentedHashRowInNonStrictMode() {
+        // Given
+        // Spec §12 non-strict: leading tabs are accepted as indentation and
+        // removed from the line's content before classification (§5.2).
+        final String input = "items[3]{id}:\n  1\n\t#x\n  2";
+
+        // When
+        final Object result = ValueDecoder.decode(input, DecodeOptions.withStrict(false));
+
+        // Then
+        assertEquals("{items=[{id=1}, {id=#x}, {id=2}]}", result.toString());
+    }
+
+    @Test
+    @DisplayName("expands a leading tab to one indentation level in non-strict mode")
+    void decode_expandsLeadingTabToOneIndentLevelInNonStrictMode() {
+        // Given
+        // Spec §12: depth computation for tabs is implementation-defined; JToon
+        // expands each leading tab to indentSize spaces (one level).
+        final String input = "outer:\n\tinner: 1";
+
+        // When
+        final Object result = ValueDecoder.decode(input, DecodeOptions.withStrict(false));
+
+        // Then
+        assertEquals("{outer={inner=1}}", result.toString());
+    }
+
     @Test
     @DisplayName("treats a hash not at line start as data")
     void decode_hashInsideLineIsData() {

src/test/resources/conformance/decode/comments.json

@@ -139,6 +139,28 @@
       "specSection": "5.1",
       "note": "Only spaces (U+0020) may precede the #; the tab makes this a regular line, and tabs in indentation error in strict mode (§12)"
     },
+    {
+      "name": "decodes tab-indented hash row as data in non-strict mode",
+      "input": "items[3]{id}:\n  1\n\t#x\n  2",
+      "expected": {
+        "items": [
+          {
+            "id": 1
+          },
+          {
+            "id": "#x"
+          },
+          {
+            "id": 2
+          }
+        ]
+      },
+      "options": {
+        "strict": false
+      },
+      "specSection": "5.1",
+      "note": "Comment removal precedes the §12 tab leniency, so the tab keeps the line out of the pre-pass"
+    },
     {
       "name": "parses quoted hash-leading first cell as data, not comment",
       "input": "items[1]{tag}:\n  \"#a\"",