commit b10a8c42ca51a9080cc5df95ecd26eee613eaf87 from: Isaac Meerleo date: Fri Jul 17 16:16:10 2026 UTC Normalize terminal transcript text commit - 5c30449123bfdaea256bb979bfd6955bf21cda49 commit + b10a8c42ca51a9080cc5df95ecd26eee613eaf87 blob - 2ff3486fbd247746fb4e110747f2f0b72b2092d0 blob + 5d8fdb1ce2fe947b2e679eaf92e1ee92cc7b8952 --- regress/ui_layout/ui_layout_test.c +++ regress/ui_layout/ui_layout_test.c @@ -19,12 +19,13 @@ #include #include "buf.h" +#include "utf8.h" #include "ui_layout.h" #include "ui_palette.h" #include "regress.h" static int -line_is(const struct ui_line *line, const char *want) +line_bytes_are(const struct ui_line *line, const void *want, size_t wantlen) { struct buf buf; size_t i; @@ -33,13 +34,40 @@ line_is(const struct ui_line *line, const char *want) buf_init(&buf); for (i = 0; i < line->nspans; i++) buf_add(&buf, line->spans[i].text, line->spans[i].len); - buf_addc(&buf, '\0'); - ok = strcmp((const char *)buf.data, want) == 0; + ok = buf.len == wantlen && memcmp(buf.data, want, wantlen) == 0; buf_free(&buf); return (ok); } static int +line_is(const struct ui_line *line, const char *want) +{ + return (line_bytes_are(line, want, strlen(want))); +} + +static int +line_is_visible_utf8(const struct ui_line *line) +{ + unsigned char enc[4]; + uint32_t cp; + size_t i, j, adv, enclen; + + for (i = 0; i < line->nspans; i++) { + const struct ui_span *span = &line->spans[i]; + + for (j = 0; j < span->len; j += adv) { + adv = utf8_decode((const unsigned char *)span->text + j, + span->len - j, &cp); + enclen = utf8_encode(cp, enc); + if (adv != enclen || memcmp(span->text + j, enc, enclen) != 0 || + cp == 0 || utf8_width(cp) < 0) + return (0); + } + } + return (1); +} + +static int line_attrs_are(const struct ui_line *line, int attr) { size_t i; @@ -66,6 +94,12 @@ main(void) { static const char wide_combining[] = "A\xcc\x81\xe4\xb8\xad" "B"; + static const unsigned char hostile[] = { + 'A', '\0', '\t', '\x1b', 0xc2, 0x85, 0xf0, '(', 0x8c, '(', 'B' + }; + static const char normalized[] = + "A\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd" + "\xef\xbf\xbd(\xef\xbf\xbd(B"; struct ui_layout layout = { 0 }; int attr = 0x1234; @@ -126,6 +160,32 @@ main(void) layout.lines[0].spans[1].width == 4); CHECK(layout.lines[0].fill_width == 6); + /* + * Hostile transcript bytes become visible UTF-8 cells before storage: + * embedded NUL, C0/C1 controls, and each malformed byte are U+FFFD. + */ + ui_layout_reset(&layout); + ui_layout_plain(&layout, hostile, sizeof(hostile), 40, attr, NULL, + NULL, 0); + CHECK(layout.nlines == 1); + CHECK(line_bytes_are(&layout.lines[0], normalized, + sizeof(normalized) - 1)); + CHECK(line_width(&layout.lines[0]) == 10); + CHECK(line_is_visible_utf8(&layout.lines[0])); + + /* Direct styled spans (the assistant-markdown path) share the boundary. */ + ui_layout_reset(&layout); + { + struct ui_line *line = ui_layout_line(&layout); + + ui_layout_span(line, attr, hostile, sizeof(hostile)); + } + CHECK(layout.nlines == 1); + CHECK(line_bytes_are(&layout.lines[0], normalized, + sizeof(normalized) - 1)); + CHECK(line_width(&layout.lines[0]) == 10); + CHECK(line_is_visible_utf8(&layout.lines[0])); + /* Re-rendering at a narrower pane pins resize and drawer reflow. */ ui_layout_reset(&layout); ui_layout_plain(&layout, "abcdef", 6, 5, attr, "> ", " ", 1); blob - d4568d6e529f82144f31100410917050e0ef7a0f blob + bac1fe598e0384069d9d1cf6244497a9fab59110 --- src/fugu-tty/ui.c +++ src/fugu-tty/ui.c @@ -14,6 +14,8 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#define _XOPEN_SOURCE_EXTENDED + #include #include @@ -26,6 +28,7 @@ #include #include #include +#include #include "log.h" #include "xmalloc.h" @@ -257,8 +260,7 @@ render_assist(struct ui *u, const struct titem *it, in for (j = 0; j < d.lines[i].nspans; j++) { struct md_span *s = &d.lines[i].spans[j]; - ui_layout_span(l, md_attr(u, s->attr), s->text, s->len, - s->width); + ui_layout_span(l, md_attr(u, s->attr), s->text, s->len); } } md_free(&d); @@ -382,6 +384,33 @@ transcript_height(const struct ui *u) return (th < 1 ? 1 : th); } +/* Paint normalized UTF-8 through curses' wide-character interface. */ +static int +draw_utf8_cells(const void *vtext, size_t len, int cells) +{ + const unsigned char *text = vtext; + wchar_t wc; + uint32_t cp; + size_t adv, i; + int used = 0, w; + + for (i = 0; i < len; i += adv) { + adv = utf8_decode(text + i, len - i, &cp); + w = utf8_width(cp); + if (cp == 0 || w < 0) { + cp = UTF8_REPLACEMENT; + w = 1; + } + if (w > cells - used) + break; + wc = (wchar_t)cp; + if (addnwstr(&wc, 1) == ERR) + break; + used += w; + } + return (used); +} + static void draw_line(const struct ui *u, int row, const struct ui_line *l, int xoff) { @@ -399,8 +428,7 @@ draw_line(const struct ui *u, int row, const struct ui move(row, col); attrset(s->attr); - addnstr(s->text, (int)s->len); - col += s->width; + col += draw_utf8_cells(s->text, s->len, u->cols - col); } attrset(A_NORMAL); } @@ -443,12 +471,12 @@ draw_input(struct ui *u, int row) adv = utf8_decode(disp.data + i, disp.len - i, &cp); w = utf8_width(cp); - if (w < 0) - w = 0; + if (cp == 0 || w < 0) + w = 1; sc = (int)scanned - (int)start; if (sc >= 0 && sc + w <= u->cols) { move(row, sc); - addnstr((char *)disp.data + i, (int)adv); + (void)draw_utf8_cells(disp.data + i, adv, w); } scanned += w; i += adv; blob - 7e7e63c09138ee4872a6f61b1188b07b44368e93 blob + e930cf754a8196847eda80497a1253fa5efae8c6 --- src/fugu-tty/ui_layout.c +++ src/fugu-tty/ui_layout.c @@ -38,6 +38,8 @@ next_cell_width(const unsigned char *text, size_t len) if (cp == '\n') return (0); w = utf8_width(cp); + if (cp == 0 || w < 0) + w = 1; if (w > 0) return (w); i += adv; @@ -82,20 +84,40 @@ ui_layout_line(struct ui_layout *layout) } void -ui_layout_span(struct ui_line *line, int attr, const void *text, size_t len, - int width) +ui_layout_span(struct ui_line *line, int attr, const void *vtext, size_t len) { + const unsigned char *text = vtext; struct ui_span *span; + struct buf normalized; + unsigned char enc[4]; + uint32_t cp; + size_t adv, enclen, i; + int w, width = 0; + buf_init(&normalized); + for (i = 0; i < len; i += adv) { + adv = utf8_decode(text + i, len - i, &cp); + w = utf8_width(cp); + if (cp == 0 || w < 0) { + cp = UTF8_REPLACEMENT; + w = 1; + } + enclen = utf8_encode(cp, enc); + buf_add(&normalized, enc, enclen); + width += w; + } + line->spans = xreallocarray(line->spans, line->nspans + 1, sizeof(*line->spans)); span = &line->spans[line->nspans++]; span->attr = attr; span->width = width; - span->len = len; - span->text = xmalloc(len + 1); - memcpy(span->text, text, len); - span->text[len] = '\0'; + span->len = normalized.len; + span->text = xmalloc(normalized.len + 1); + if (normalized.len > 0) + memcpy(span->text, normalized.data, normalized.len); + span->text[normalized.len] = '\0'; + buf_free(&normalized); } /* Wrap plain UTF-8 text to width columns, one attribute, into new lines. */ @@ -138,8 +160,7 @@ ui_layout_plain(struct ui_layout *layout, const void * leadlen = (size_t)lead_limit; if (leadlen > 0) { - ui_layout_span(line, attr, lead, leadlen, - (int)leadlen); + ui_layout_span(line, attr, lead, leadlen); col = (int)leadlen; } } @@ -151,8 +172,9 @@ ui_layout_plain(struct ui_layout *layout, const void * break_len = 0; break_width = 0; while (i < len) { + unsigned char enc[4]; uint32_t cp; - size_t adv; + size_t adv, enclen; int w; adv = utf8_decode(text + i, len - i, &cp); @@ -161,8 +183,10 @@ ui_layout_plain(struct ui_layout *layout, const void * break; } w = utf8_width(cp); - if (w < 0) - w = 0; + if (cp == 0 || w < 0) { + cp = UTF8_REPLACEMENT; + w = 1; + } if (col + w > width) { if (segw > 0) { if (last_break != (size_t)-1) { @@ -182,18 +206,19 @@ ui_layout_plain(struct ui_layout *layout, const void * i += adv; continue; } - buf_add(&seg, text + i, adv); + enclen = utf8_encode(cp, enc); + buf_add(&seg, enc, enclen); segw += w; col += w; - if (cp == ' ' && seg.len > adv) { + if (cp == ' ' && seg.len > enclen) { last_break = i; - break_len = seg.len - adv; + break_len = seg.len - enclen; break_width = segw - w; } i += adv; } if (seg.len > 0) - ui_layout_span(line, attr, seg.data, seg.len, segw); + ui_layout_span(line, attr, seg.data, seg.len); buf_free(&seg); } while (i < len); } blob - 85c490f435a7c16cfa3993077783f5fb53324520 blob + 2ca324091c9504b1a0b68bf62b5670cca635284d --- src/fugu-tty/ui_layout.h +++ src/fugu-tty/ui_layout.h @@ -42,8 +42,7 @@ struct ui_layout { void ui_layout_reset(struct ui_layout *); void ui_layout_free(struct ui_layout *); struct ui_line *ui_layout_line(struct ui_layout *); -void ui_layout_span(struct ui_line *, int, const void *, size_t, - int); +void ui_layout_span(struct ui_line *, int, const void *, size_t); void ui_layout_plain(struct ui_layout *, const void *, size_t, int, int, const char *, const char *, int); int ui_layout_visible_fill(const struct ui_line *, int, int);