From a2def5b0a11096ef5dfa7ec09207b05126be045c Mon Sep 17 00:00:00 2001 From: "Victor M. Varela" Date: Fri, 3 Jul 2026 18:26:55 +0200 Subject: [PATCH] fix: BLOB hex encoding in SQL output + full Unicode width tables - Add BLOB hex encoding (X'...') in SQL INSERT output instead of silent NULL conversion (issue in writeSqlRow else branch) - Handle zero-length BLOB quirk: check column_bytes() before column_blob() pointer (null for empty blobs) - Integration tests for BLOB round-trip (174j) and empty BLOB (174k) - Replace 4-range isWideCodepoint() with comprehensive sorted [2]u21 range table + binary search (~80 ranges) - Covers CJK A-H, Hiragana/Katakana, Hangul, Yi, Fullwidth, Emoji (terminal convention, noted) - Fix 3 incorrect expected values in existing visual tests - Add tests for Hiragana, Katakana, and emoji width --- build.zig | 18 ++++++ src/format.zig | 18 +++++- src/visual.zig | 152 ++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 180 insertions(+), 8 deletions(-) diff --git a/build.zig b/build.zig index 746da12..61f9aa9 100644 --- a/build.zig +++ b/build.zig @@ -2399,6 +2399,24 @@ pub fn build(b: *std.Build) void { test_sql_quoted_table.step.dependOn(b.getInstallStep()); test_step.dependOn(&test_sql_quoted_table.step); + // Integration test 174j: BLOB values rendered as X'...' hex + const test_sql_blob = b.addSystemCommand(&.{ + "bash", "-c", + \\result=$(printf 'name\nignored\n' | ./zig-out/bin/sql-pipe -O sql 'SELECT x'"'"'0001FF'"'"' AS data FROM t') + \\echo "$result" | grep -Fq "VALUES (X'0001FF');" + }); + test_sql_blob.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_sql_blob.step); + + // Integration test 174k: empty BLOB renders as X'' + const test_sql_blob_empty = b.addSystemCommand(&.{ + "bash", "-c", + \\result=$(printf 'name\nignored\n' | ./zig-out/bin/sql-pipe -O sql 'SELECT x'"'"''"'"' AS data FROM t') + \\echo "$result" | grep -Fq "VALUES (X'');" + }); + test_sql_blob_empty.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_sql_blob_empty.step); + // ─── Fixture-based integration tests ───────────────────────────────────── // These tests use sample files committed in tests/fixtures/ to exercise // the binary end-to-end with realistic data across all supported formats. diff --git a/src/format.zig b/src/format.zig index d7779fb..fa3b839 100644 --- a/src/format.zig +++ b/src/format.zig @@ -255,8 +255,24 @@ pub const OutputWriter = struct { try writer.print("{d}", .{f}); } }, + c.SQLITE_BLOB => { + const len = c.sqlite3_column_bytes(stmt, i); + if (len == 0) { + try writer.writeAll("X''"); + } else { + const blob = c.sqlite3_column_blob(stmt, i) orelse unreachable; + const data = @as([*]const u8, @ptrCast(blob))[0..@intCast(len)]; + const hex = "0123456789ABCDEF"; + try writer.writeAll("X'"); + for (data) |byte| { + try writer.writeByte(hex[byte >> 4]); + try writer.writeByte(hex[byte & 0x0F]); + } + try writer.writeByte('\''); + } + }, else => { - // ponytail: BLOB truncated at first NUL, same as CSV/JSON; X'...' hex if needed + // ponytail: BLOBs handled above for SQL via X'...'; CSV/JSON still truncate at NUL if (sqlite_mod.columnText(stmt, i)) |text| { try writeSqlStringLiteral(writer, text); } else { diff --git a/src/visual.zig b/src/visual.zig index 186c55b..fb2d0e3 100644 --- a/src/visual.zig +++ b/src/visual.zig @@ -7,6 +7,118 @@ const std = @import("std"); +// Ranges with terminal display width 2. Covers Unicode East Asian Width W/F +// plus common Emoji ranges treated as wide by terminals. +// ponytail: emoji ranges are a terminal convention override, not UAX #11 W/F +const wide_ranges = blk: { + var ranges = [_][2]u21{ + // CJK and related blocks (UAX #11 Wide) + .{ 0x1100, 0x115F }, + .{ 0x2329, 0x232A }, + .{ 0x2E80, 0x2EFF }, + .{ 0x2F00, 0x2FDF }, + .{ 0x2FF0, 0x2FFF }, + .{ 0x3000, 0x303E }, + .{ 0x3041, 0x3096 }, + .{ 0x3099, 0x30FF }, + .{ 0x3105, 0x312F }, + .{ 0x3131, 0x318E }, + .{ 0x3190, 0x31E3 }, + .{ 0x31F0, 0x321E }, + .{ 0x3220, 0x3247 }, + .{ 0x3250, 0x4DBF }, + .{ 0x4E00, 0x9FFF }, + .{ 0xA000, 0xA48C }, + .{ 0xA490, 0xA4C6 }, + .{ 0xA960, 0xA97C }, + .{ 0xAC00, 0xD7FF }, + .{ 0xF900, 0xFAFF }, + .{ 0xFE10, 0xFE19 }, + .{ 0xFE30, 0xFE6F }, + .{ 0xFF01, 0xFF60 }, + .{ 0xFFE0, 0xFFE6 }, + + // CJK Extension B and beyond + .{ 0x1B000, 0x1B0FF }, + .{ 0x1B100, 0x1B12F }, + .{ 0x1B130, 0x1B16F }, + .{ 0x1F200, 0x1F2FF }, + .{ 0x20000, 0x2A6DF }, + .{ 0x2A700, 0x2B739 }, + .{ 0x2B740, 0x2B81D }, + .{ 0x2B820, 0x2CEA1 }, + .{ 0x2CEB0, 0x2EBE0 }, + .{ 0x2F800, 0x2FA1F }, + .{ 0x30000, 0x3134A }, + .{ 0x31350, 0x323AF }, + + // Common Emoji ranges (width 2 in terminals) + .{ 0x231A, 0x231B }, + .{ 0x23E9, 0x23EC }, + .{ 0x23F0, 0x23F3 }, + .{ 0x25FD, 0x25FE }, + .{ 0x2614, 0x2615 }, + .{ 0x2648, 0x2653 }, + .{ 0x267F, 0x267F }, + .{ 0x2693, 0x2693 }, + .{ 0x26A1, 0x26A1 }, + .{ 0x26AA, 0x26AB }, + .{ 0x26BD, 0x26BE }, + .{ 0x26C4, 0x26C5 }, + .{ 0x26CE, 0x26CE }, + .{ 0x26D4, 0x26D4 }, + .{ 0x26EA, 0x26EA }, + .{ 0x26F2, 0x26F3 }, + .{ 0x26F5, 0x26F5 }, + .{ 0x26FA, 0x26FA }, + .{ 0x26FD, 0x26FD }, + .{ 0x2702, 0x2702 }, + .{ 0x2705, 0x2705 }, + .{ 0x2708, 0x270D }, + .{ 0x270F, 0x270F }, + .{ 0x2712, 0x2712 }, + .{ 0x2714, 0x2714 }, + .{ 0x2716, 0x2716 }, + .{ 0x271D, 0x271D }, + .{ 0x2721, 0x2721 }, + .{ 0x2728, 0x2728 }, + .{ 0x2733, 0x2734 }, + .{ 0x2744, 0x2744 }, + .{ 0x2747, 0x2747 }, + .{ 0x274C, 0x274C }, + .{ 0x274E, 0x274E }, + .{ 0x2753, 0x2755 }, + .{ 0x2757, 0x2757 }, + .{ 0x2763, 0x2764 }, + .{ 0x2795, 0x2797 }, + .{ 0x27A1, 0x27A1 }, + .{ 0x27B0, 0x27B0 }, + .{ 0x27BF, 0x27BF }, + .{ 0x2934, 0x2935 }, + .{ 0x2B05, 0x2B07 }, + .{ 0x2B1B, 0x2B1C }, + .{ 0x2B50, 0x2B50 }, + .{ 0x2B55, 0x2B55 }, + .{ 0x3030, 0x3030 }, + .{ 0x303D, 0x303D }, + .{ 0x3297, 0x3297 }, + .{ 0x3299, 0x3299 }, + .{ 0x1F000, 0x1F02F }, + .{ 0x1F030, 0x1F09F }, + .{ 0x1F0A0, 0x1F0FF }, + .{ 0x1F100, 0x1F1FF }, + .{ 0x1F300, 0x1FAFF }, + .{ 0x1FB00, 0x1FBFF }, + }; + @setEvalBranchQuota(10000); + std.sort.block([2]u21, &ranges, {}, struct { + fn lessThan(_: void, a: [2]u21, b: [2]u21) bool { + return a[0] < b[0]; + } + }.lessThan); + break :blk ranges; +}; + /// Return the byte length of a UTF-8 character from its leading byte. fn utf8CharLen(first: u8) usize { if (first < 0x80) return 1; @@ -30,10 +142,19 @@ fn utf8DecodeRaw(bytes: []const u8) ?u21 { /// Check whether a codepoint is wide (display width 2 in a terminal). fn isWideCodepoint(cp: u21) bool { - return (cp >= 0x3400 and cp <= 0x4DBF) or - (cp >= 0x4E00 and cp <= 0x9FFF) or - (cp >= 0xAC00 and cp <= 0xD7AF) or - (cp >= 0xFF00 and cp <= 0xFFEF); + var left: usize = 0; + var right: usize = wide_ranges.len; + while (left < right) { + const mid = (left + right) / 2; + if (wide_ranges[mid][0] <= cp) { + left = mid + 1; + } else { + right = mid; + } + } + if (left == 0) return false; + const range = wide_ranges[left - 1]; + return cp <= range[1]; } /// Compute the visual display width of a UTF-8 string. @@ -84,7 +205,7 @@ pub fn writeCharRepeated(writer: *std.Io.Writer, char: []const u8, n: usize) err var remaining = n; while (remaining > 0) { const chunk = @min(remaining, filled / char_len); - try writer.writeAll(buf[0..chunk * char_len]); + try writer.writeAll(buf[0 .. chunk * char_len]); remaining -= chunk; } } @@ -111,7 +232,7 @@ test "visualWidth ASCII" { test "visualWidth CJK" { // Each CJK character has width 2 - try testing.expectEqual(@as(usize, 6), visualWidth("你好世界")); + try testing.expectEqual(@as(usize, 8), visualWidth("你好世界")); try testing.expectEqual(@as(usize, 2), visualWidth("中")); try testing.expectEqual(@as(usize, 4), visualWidth("中文")); } @@ -127,5 +248,22 @@ test "visualWidth invalid UTF-8" { // Invalid continuation byte treated as width 1 try testing.expectEqual(@as(usize, 1), visualWidth(&[_]u8{0x80})); // Overlong encoding (invalid) — width 1 per byte - try testing.expectEqual(@as(usize, 1), visualWidth(&[_]u8{0xC0, 0x80})); + try testing.expectEqual(@as(usize, 2), visualWidth(&[_]u8{ 0xC0, 0x80 })); +} + +test "visualWidth Hiragana and Katakana" { + try testing.expectEqual(@as(usize, 2), visualWidth("あ")); + try testing.expectEqual(@as(usize, 2), visualWidth("カ")); + try testing.expectEqual(@as(usize, 4), visualWidth("あい")); +} + +test "visualWidth emoji" { + try testing.expectEqual(@as(usize, 2), visualWidth("😀")); + try testing.expectEqual(@as(usize, 4), visualWidth("😀😁")); +} + +test "visualWidth misc emoji symbols" { + try testing.expectEqual(@as(usize, 2), visualWidth("❄")); + try testing.expectEqual(@as(usize, 2), visualWidth("⛵")); + try testing.expectEqual(@as(usize, 2), visualWidth("♿")); }