Skip to content

fix: BLOB hex encoding + full Unicode width tables #190

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vmvarela merged 1 commit into from
Jul 3, 2026
Merged

fix: BLOB hex encoding + full Unicode width tables #190

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions build.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2399,6 +2399,24 @@ pub fn build(b: *std.Build) void {
test_sql_quoted_table.step.dependOn(b.getInstallStep());
test_step.dependOn(&test_sql_quoted_table.step);

// Integration test 174j: BLOB values rendered as X'...' hex
const test_sql_blob = b.addSystemCommand(&.{
"bash", "-c",
\\result=$(printf 'name\nignored\n' | ./zig-out/bin/sql-pipe -O sql 'SELECT x'"'"'0001FF'"'"' AS data FROM t')
\\echo "$result" | grep -Fq "VALUES (X'0001FF');"
});
test_sql_blob.step.dependOn(b.getInstallStep());
test_step.dependOn(&test_sql_blob.step);

// Integration test 174k: empty BLOB renders as X''
const test_sql_blob_empty = b.addSystemCommand(&.{
"bash", "-c",
\\result=$(printf 'name\nignored\n' | ./zig-out/bin/sql-pipe -O sql 'SELECT x'"'"''"'"' AS data FROM t')
\\echo "$result" | grep -Fq "VALUES (X'');"
});
test_sql_blob_empty.step.dependOn(b.getInstallStep());
test_step.dependOn(&test_sql_blob_empty.step);

// ─── Fixture-based integration tests ─────────────────────────────────────
// These tests use sample files committed in tests/fixtures/ to exercise
// the binary end-to-end with realistic data across all supported formats.
Expand Down
18 changes: 17 additions & 1 deletion src/format.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,8 +255,24 @@ pub const OutputWriter = struct {
try writer.print("{d}", .{f});
}
},
c.SQLITE_BLOB => {
const len = c.sqlite3_column_bytes(stmt, i);
if (len == 0) {
try writer.writeAll("X''");
} else {
const blob = c.sqlite3_column_blob(stmt, i) orelse unreachable;
const data = @as([*]const u8, @ptrCast(blob))[0..@intCast(len)];
const hex = "0123456789ABCDEF";
try writer.writeAll("X'");
for (data) |byte| {
try writer.writeByte(hex[byte >> 4]);
try writer.writeByte(hex[byte & 0x0F]);
}
try writer.writeByte('\'');
}
},
else => {
// ponytail: BLOB truncated at first NUL, same as CSV/JSON; X'...' hex if needed
// ponytail: BLOBs handled above for SQL via X'...'; CSV/JSON still truncate at NUL
if (sqlite_mod.columnText(stmt, i)) |text| {
try writeSqlStringLiteral(writer, text);
} else {
Expand Down
152 changes: 145 additions & 7 deletions src/visual.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,118 @@

const std = @import("std");

// Ranges with terminal display width 2. Covers Unicode East Asian Width W/F
// plus common Emoji ranges treated as wide by terminals.
// ponytail: emoji ranges are a terminal convention override, not UAX #11 W/F
const wide_ranges = blk: {
var ranges = [_][2]u21{
// CJK and related blocks (UAX #11 Wide)
.{ 0x1100, 0x115F },
.{ 0x2329, 0x232A },
.{ 0x2E80, 0x2EFF },
.{ 0x2F00, 0x2FDF },
.{ 0x2FF0, 0x2FFF },
.{ 0x3000, 0x303E },
.{ 0x3041, 0x3096 },
.{ 0x3099, 0x30FF },
.{ 0x3105, 0x312F },
.{ 0x3131, 0x318E },
.{ 0x3190, 0x31E3 },
.{ 0x31F0, 0x321E },
.{ 0x3220, 0x3247 },
.{ 0x3250, 0x4DBF },
.{ 0x4E00, 0x9FFF },
.{ 0xA000, 0xA48C },
.{ 0xA490, 0xA4C6 },
.{ 0xA960, 0xA97C },
.{ 0xAC00, 0xD7FF },
.{ 0xF900, 0xFAFF },
.{ 0xFE10, 0xFE19 },
.{ 0xFE30, 0xFE6F },
.{ 0xFF01, 0xFF60 },
.{ 0xFFE0, 0xFFE6 },

// CJK Extension B and beyond
.{ 0x1B000, 0x1B0FF },
.{ 0x1B100, 0x1B12F },
.{ 0x1B130, 0x1B16F },
.{ 0x1F200, 0x1F2FF },
.{ 0x20000, 0x2A6DF },
.{ 0x2A700, 0x2B739 },
.{ 0x2B740, 0x2B81D },
.{ 0x2B820, 0x2CEA1 },
.{ 0x2CEB0, 0x2EBE0 },
.{ 0x2F800, 0x2FA1F },
.{ 0x30000, 0x3134A },
.{ 0x31350, 0x323AF },

// Common Emoji ranges (width 2 in terminals)
.{ 0x231A, 0x231B },
.{ 0x23E9, 0x23EC },
.{ 0x23F0, 0x23F3 },
.{ 0x25FD, 0x25FE },
.{ 0x2614, 0x2615 },
.{ 0x2648, 0x2653 },
.{ 0x267F, 0x267F },
.{ 0x2693, 0x2693 },
.{ 0x26A1, 0x26A1 },
.{ 0x26AA, 0x26AB },
.{ 0x26BD, 0x26BE },
.{ 0x26C4, 0x26C5 },
.{ 0x26CE, 0x26CE },
.{ 0x26D4, 0x26D4 },
.{ 0x26EA, 0x26EA },
.{ 0x26F2, 0x26F3 },
.{ 0x26F5, 0x26F5 },
.{ 0x26FA, 0x26FA },
.{ 0x26FD, 0x26FD },
.{ 0x2702, 0x2702 },
.{ 0x2705, 0x2705 },
.{ 0x2708, 0x270D },
.{ 0x270F, 0x270F },
.{ 0x2712, 0x2712 },
.{ 0x2714, 0x2714 },
.{ 0x2716, 0x2716 },
.{ 0x271D, 0x271D },
.{ 0x2721, 0x2721 },
.{ 0x2728, 0x2728 },
.{ 0x2733, 0x2734 },
.{ 0x2744, 0x2744 },
.{ 0x2747, 0x2747 },
.{ 0x274C, 0x274C },
.{ 0x274E, 0x274E },
.{ 0x2753, 0x2755 },
.{ 0x2757, 0x2757 },
.{ 0x2763, 0x2764 },
.{ 0x2795, 0x2797 },
.{ 0x27A1, 0x27A1 },
.{ 0x27B0, 0x27B0 },
.{ 0x27BF, 0x27BF },
.{ 0x2934, 0x2935 },
.{ 0x2B05, 0x2B07 },
.{ 0x2B1B, 0x2B1C },
.{ 0x2B50, 0x2B50 },
.{ 0x2B55, 0x2B55 },
.{ 0x3030, 0x3030 },
.{ 0x303D, 0x303D },
.{ 0x3297, 0x3297 },
.{ 0x3299, 0x3299 },
.{ 0x1F000, 0x1F02F },
.{ 0x1F030, 0x1F09F },
.{ 0x1F0A0, 0x1F0FF },
.{ 0x1F100, 0x1F1FF },
.{ 0x1F300, 0x1FAFF },
.{ 0x1FB00, 0x1FBFF },
};
@setEvalBranchQuota(10000);
std.sort.block([2]u21, &ranges, {}, struct {
fn lessThan(_: void, a: [2]u21, b: [2]u21) bool {
return a[0] < b[0];
}
}.lessThan);
break :blk ranges;
};

/// Return the byte length of a UTF-8 character from its leading byte.
fn utf8CharLen(first: u8) usize {
if (first < 0x80) return 1;
Expand All @@ -30,10 +142,19 @@ fn utf8DecodeRaw(bytes: []const u8) ?u21 {

/// Check whether a codepoint is wide (display width 2 in a terminal).
fn isWideCodepoint(cp: u21) bool {
return (cp >= 0x3400 and cp <= 0x4DBF) or
(cp >= 0x4E00 and cp <= 0x9FFF) or
(cp >= 0xAC00 and cp <= 0xD7AF) or
(cp >= 0xFF00 and cp <= 0xFFEF);
var left: usize = 0;
var right: usize = wide_ranges.len;
while (left < right) {
const mid = (left + right) / 2;
if (wide_ranges[mid][0] <= cp) {
left = mid + 1;
} else {
right = mid;
}
}
if (left == 0) return false;
const range = wide_ranges[left - 1];
return cp <= range[1];
}

/// Compute the visual display width of a UTF-8 string.
Expand Down Expand Up @@ -84,7 +205,7 @@ pub fn writeCharRepeated(writer: *std.Io.Writer, char: []const u8, n: usize) err
var remaining = n;
while (remaining > 0) {
const chunk = @min(remaining, filled / char_len);
try writer.writeAll(buf[0..chunk * char_len]);
try writer.writeAll(buf[0 .. chunk * char_len]);
remaining -= chunk;
}
}
Expand All @@ -111,7 +232,7 @@ test "visualWidth ASCII" {

test "visualWidth CJK" {
// Each CJK character has width 2
try testing.expectEqual(@as(usize, 6), visualWidth("你好世界"));
try testing.expectEqual(@as(usize, 8), visualWidth("你好世界"));
try testing.expectEqual(@as(usize, 2), visualWidth("中"));
try testing.expectEqual(@as(usize, 4), visualWidth("中文"));
}
Expand All @@ -127,5 +248,22 @@ test "visualWidth invalid UTF-8" {
// Invalid continuation byte treated as width 1
try testing.expectEqual(@as(usize, 1), visualWidth(&[_]u8{0x80}));
// Overlong encoding (invalid) — width 1 per byte
try testing.expectEqual(@as(usize, 1), visualWidth(&[_]u8{0xC0, 0x80}));
try testing.expectEqual(@as(usize, 2), visualWidth(&[_]u8{ 0xC0, 0x80 }));
}

test "visualWidth Hiragana and Katakana" {
try testing.expectEqual(@as(usize, 2), visualWidth("あ"));
try testing.expectEqual(@as(usize, 2), visualWidth("カ"));
try testing.expectEqual(@as(usize, 4), visualWidth("あい"));
}

test "visualWidth emoji" {
try testing.expectEqual(@as(usize, 2), visualWidth("😀"));
try testing.expectEqual(@as(usize, 4), visualWidth("😀😁"));
}

test "visualWidth misc emoji symbols" {
try testing.expectEqual(@as(usize, 2), visualWidth("❄"));
try testing.expectEqual(@as(usize, 2), visualWidth("⛵"));
try testing.expectEqual(@as(usize, 2), visualWidth("♿"));
}
Loading