diff --git a/CHANGELOG.md b/CHANGELOG.md index 89bbcec0..2eca6fac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- CSV export renders unsigned integer columns (U8–U64) with their unsigned values — high-half values previously printed as two's-complement negatives (uci-wine `magnesium` U8 132 exported as -124), silent corruption found by the Raincloud conformance suite. ([#208](https://github.com/dfa1/vortex-java/issues/208)) + ## [0.12.0] — 2026-07-04 Identity gets **types**: encoding ids, layout ids, and column names are now validated domain @@ -92,6 +96,7 @@ A **`vortex.zstd` overhaul**: compression now runs through FFM bindings to the n ### Fixed +- CSV export renders unsigned integer columns (U8–U64) with their unsigned values — high-half values previously printed as two's-complement negatives (uci-wine `magnesium` U8 132 exported as -124), silent corruption in every dtype-blind consumer of the typed getters. ([#208](https://github.com/dfa1/vortex-java/issues/208)) - `vortex.zstd` segments compressed with a shared (trained) dictionary now decode, via the native `libzstd` dictionary support, instead of being rejected. The upstream `zstd.vortex` compatibility fixture is read end-to-end and matches the Rust reference. ([#104](https://github.com/dfa1/vortex-java/issues/104)) - Writing a nullable `Utf8`/`Binary` column no longer throws `NullPointerException` (or silently drops nulls): nullable string columns now carry their validity like nullable primitives and round-trip through `vortex.masked`. As a result they decode as `MaskedArray` (validity + values child) rather than a bare `VarBinArray`. ([#168](https://github.com/dfa1/vortex-java/pull/168)) - CSV export now handles nullable columns (`MaskedArray`): null rows export as an empty field instead of failing with "unsupported array type for CSV export". ([#168](https://github.com/dfa1/vortex-java/pull/168)) diff --git a/csv/src/main/java/io/github/dfa1/vortex/csv/CsvExporter.java b/csv/src/main/java/io/github/dfa1/vortex/csv/CsvExporter.java index 0b3bf706..c366728b 100644 --- a/csv/src/main/java/io/github/dfa1/vortex/csv/CsvExporter.java +++ b/csv/src/main/java/io/github/dfa1/vortex/csv/CsvExporter.java @@ -140,10 +140,18 @@ private static void writeChunk(Chunk chunk, List colNames, int colCount, private static String cellValue(Array arr, long rowIdx) { return switch (arr) { - case LongArray la -> Long.toString(la.getLong(rowIdx)); - case IntArray ia -> Integer.toString(ia.getInt(rowIdx)); - case ShortArray sa -> Short.toString(sa.getShort(rowIdx)); - case ByteArray ba -> Byte.toString(ba.getByte(rowIdx)); + // Long/IntArray have no dtype-aware getter, so gate the unsigned rendering on the + // ptype; U64/U32 high-half values would otherwise print as negative. + case LongArray la -> isUnsigned(la) + ? Long.toUnsignedString(la.getLong(rowIdx)) + : Long.toString(la.getLong(rowIdx)); + case IntArray ia -> isUnsigned(ia) + ? Integer.toUnsignedString(ia.getInt(rowIdx)) + : Integer.toString(ia.getInt(rowIdx)); + // Byte/ShortArray.getInt already zero-extends U8/U16 per their dtype, so widening to + // int and printing signed decimal yields the correct unsigned value. + case ShortArray sa -> Integer.toString(sa.getInt(rowIdx)); + case ByteArray ba -> Integer.toString(ba.getInt(rowIdx)); case DoubleArray da -> Double.toString(da.getDouble(rowIdx)); case FloatArray fa -> Float.toString(fa.getFloat(rowIdx)); case BoolArray ba -> Boolean.toString(ba.getBoolean(rowIdx)); @@ -155,4 +163,12 @@ private static String cellValue(Array arr, long rowIdx) { "unsupported array type for CSV export: " + arr.getClass().getSimpleName()); }; } + + /// Whether the array's dtype is an unsigned integer, so high-half values must render as + /// unsigned decimal rather than the two's-complement negative that the raw signed getter + /// returns. Callers only pass leaf value arrays: the [MaskedArray] arm of `cellValue` + /// recurses into the inner array before any integer arm is reached. + private static boolean isUnsigned(Array arr) { + return arr.dtype() instanceof DType.Primitive p && p.ptype().isUnsigned(); + } } diff --git a/csv/src/test/java/io/github/dfa1/vortex/csv/CsvExporterTest.java b/csv/src/test/java/io/github/dfa1/vortex/csv/CsvExporterTest.java index 16923d0b..b5672a56 100644 --- a/csv/src/test/java/io/github/dfa1/vortex/csv/CsvExporterTest.java +++ b/csv/src/test/java/io/github/dfa1/vortex/csv/CsvExporterTest.java @@ -2,8 +2,10 @@ import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.writer.VortexWriter; import io.github.dfa1.vortex.writer.WriteOptions; +import io.github.dfa1.vortex.writer.encode.NullableData; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -91,4 +93,142 @@ void suppressesHeaderWhenConfigured(@TempDir Path tmp) throws Exception { assertThat(lines).hasSize(1); assertThat(lines.getFirst()).isEqualTo("7"); } + + @Test + void rendersU8HighHalfAsUnsigned(@TempDir Path tmp) throws Exception { + // Given a non-nullable U8 column. 132 is 0x84, which is -124 as a signed byte — the + // uci-wine `magnesium` shape (values 70–162) that regressed to negative numbers. + Path vortex = tmp.resolve("u8.vortex"); + DType.Struct schema = new DType.Struct( + List.of(ColumnName.of("magnesium")), + List.of(new DType.Primitive(PType.U8, false)), + false); + try (FileChannel ch = FileChannel.open(vortex, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + VortexWriter writer = VortexWriter.create(ch, schema, WriteOptions.defaults())) { + writer.writeChunk(Map.of(ColumnName.of("magnesium"), new byte[]{70, (byte) 132, (byte) 255})); + } + Path csv = tmp.resolve("out.csv"); + + // When + CsvExporter.exportCsv(vortex, csv); + + // Then + List result = Files.readAllLines(csv); + assertThat(result).containsExactly("magnesium", "70", "132", "255"); + } + + @Test + void rendersNullableU8HighHalfAsUnsigned(@TempDir Path tmp) throws Exception { + // Given a nullable U8 column (dtype U8?), which decodes as a MaskedArray whose inner + // values array carries a non-nullable U8 dtype. This is the exact uci-wine layout; the + // fix must consult the inner array's ptype, not just the class, to widen 132 correctly. + Path vortex = tmp.resolve("u8n.vortex"); + DType.Struct schema = new DType.Struct( + List.of(ColumnName.of("magnesium")), + List.of(new DType.Primitive(PType.U8, true)), + false); + try (FileChannel ch = FileChannel.open(vortex, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + VortexWriter writer = VortexWriter.create(ch, schema, WriteOptions.defaults())) { + writer.writeChunk(Map.of(ColumnName.of("magnesium"), + new NullableData(new byte[]{(byte) 132, 0, (byte) 162}, + new boolean[]{true, false, true}))); + } + Path csv = tmp.resolve("out.csv"); + + // When + CsvExporter.exportCsv(vortex, csv); + + // Then null rows render as empty; high-half valid rows render unsigned. + List result = Files.readAllLines(csv); + assertThat(result).containsExactly("magnesium", "132", "", "162"); + } + + @Test + void keepsSignedI8Negative(@TempDir Path tmp) throws Exception { + // Given a SIGNED I8 column with a negative value — pins that the unsigned-rendering + // fix did not widen signed bytes: -5 must stay "-5", not become "251" + Path vortex = tmp.resolve("i8.vortex"); + DType.Struct schema = new DType.Struct( + List.of(ColumnName.of("delta")), + List.of(new DType.Primitive(PType.I8, false)), + false); + try (FileChannel ch = FileChannel.open(vortex, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + VortexWriter writer = VortexWriter.create(ch, schema, WriteOptions.defaults())) { + writer.writeChunk(Map.of(ColumnName.of("delta"), new byte[]{(byte) -5, (byte) 7})); + } + Path csv = tmp.resolve("out.csv"); + + // When + CsvExporter.exportCsv(vortex, csv); + + // Then + List result = Files.readAllLines(csv); + assertThat(result).containsExactly("delta", "-5", "7"); + } + + @Test + void rendersU16HighHalfAsUnsigned(@TempDir Path tmp) throws Exception { + // Given a non-nullable U16 column. 40000 is 0x9C40, which is -25536 as a signed short. + Path vortex = tmp.resolve("u16.vortex"); + DType.Struct schema = new DType.Struct( + List.of(ColumnName.of("v")), + List.of(new DType.Primitive(PType.U16, false)), + false); + try (FileChannel ch = FileChannel.open(vortex, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + VortexWriter writer = VortexWriter.create(ch, schema, WriteOptions.defaults())) { + writer.writeChunk(Map.of(ColumnName.of("v"), new short[]{1, (short) 40000, (short) 65535})); + } + Path csv = tmp.resolve("out.csv"); + + // When + CsvExporter.exportCsv(vortex, csv); + + // Then + List result = Files.readAllLines(csv); + assertThat(result).containsExactly("v", "1", "40000", "65535"); + } + + @Test + void rendersU32AboveIntMaxAsUnsigned(@TempDir Path tmp) throws Exception { + // Given a non-nullable U32 column. 0x9000_0000 (2415919104) is negative as a signed int. + Path vortex = tmp.resolve("u32.vortex"); + DType.Struct schema = new DType.Struct( + List.of(ColumnName.of("v")), + List.of(new DType.Primitive(PType.U32, false)), + false); + try (FileChannel ch = FileChannel.open(vortex, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + VortexWriter writer = VortexWriter.create(ch, schema, WriteOptions.defaults())) { + writer.writeChunk(Map.of(ColumnName.of("v"), new int[]{1, 0x9000_0000, 0xFFFF_FFFF})); + } + Path csv = tmp.resolve("out.csv"); + + // When + CsvExporter.exportCsv(vortex, csv); + + // Then + List result = Files.readAllLines(csv); + assertThat(result).containsExactly("v", "1", "2415919104", "4294967295"); + } + + @Test + void rendersU64AboveLongMaxAsUnsigned(@TempDir Path tmp) throws Exception { + // Given a non-nullable U64 column. -1L is the all-ones bit pattern: 2^64-1 unsigned. + Path vortex = tmp.resolve("u64.vortex"); + DType.Struct schema = new DType.Struct( + List.of(ColumnName.of("v")), + List.of(new DType.Primitive(PType.U64, false)), + false); + try (FileChannel ch = FileChannel.open(vortex, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + VortexWriter writer = VortexWriter.create(ch, schema, WriteOptions.defaults())) { + writer.writeChunk(Map.of(ColumnName.of("v"), new long[]{1L, Long.MIN_VALUE, -1L})); + } + Path csv = tmp.resolve("out.csv"); + + // When + CsvExporter.exportCsv(vortex, csv); + + // Then + List result = Files.readAllLines(csv); + assertThat(result).containsExactly("v", "1", "9223372036854775808", "18446744073709551615"); + } }