commit 91e58933b7fe2e7ffb5f1943b2bdada616824b69 from: Isaac Meerleo date: Wed Jul 15 16:41:07 2026 UTC harden canonical generation results commit - 36f2bf37d293ba47b0c2c529451cfae948bb8cf0 commit + 91e58933b7fe2e7ffb5f1943b2bdada616824b69 blob - d54ec5cd03dff275f96e5e7eb5facf99b6a53e1b blob + 509f8d4740d072eea5ffd48d487799572573190a --- regress/generation/generation_test.c +++ regress/generation/generation_test.c @@ -113,7 +113,10 @@ test_fragmented_binary_text(void) GENERATION_TERMINAL); CHECK(generation_take(g, &r) == 0); CHECK(r.outcome == GENERATION_NORMAL && r.data_used == sizeof(want) && - r.tool_calls == 0 && memcmp(r.reason, "stop", 5) == 0); + r.tool_calls == 0 && + r.invalid_cause == GENERATION_INVALID_NONE && + r.synthesized_text.data == NULL && r.synthesized_text.len == 0 && + memcmp(r.reason, "stop", 5) == 0); generation_free(g); text_is(r.assistant, want, sizeof(want)); msg_free(r.assistant); @@ -192,7 +195,9 @@ test_noargs_uses_exact_budget(void) GENERATION_TERMINAL); CHECK(generation_take(g, &r) == 0); CHECK(r.outcome == GENERATION_NORMAL && r.data_used == 2 && - r.tool_calls == 1); + r.tool_calls == 1 && + r.invalid_cause == GENERATION_INVALID_NONE && + r.synthesized_text.data == NULL && r.synthesized_text.len == 0); b = r.assistant == NULL ? NULL : TAILQ_FIRST(&r.assistant->blocks); CHECK(b != NULL && b->type == BLOCK_TOOL_USE && b->tool_input_len == 2 && memcmp(b->tool_input, "{}", 2) == 0); @@ -207,6 +212,8 @@ test_noargs_uses_exact_budget(void) GENERATION_TERMINAL); CHECK(generation_take(g, &r) == 0); CHECK(r.outcome == GENERATION_INVALID && r.assistant == NULL && + r.invalid_cause == GENERATION_INVALID_DATA_BOUND && + r.synthesized_text.data == NULL && r.synthesized_text.len == 0 && r.data_used == 0 && strcmp(r.reason, "Generation data bound exceeded") == 0); generation_free(g); @@ -241,6 +248,9 @@ test_malformed_arguments_are_never_accepted(void) i == 0 ? 10 : 8, NULL) == GENERATION_TERMINAL); CHECK(generation_take(g, &r) == 0); CHECK(r.outcome == GENERATION_INVALID && r.assistant == NULL && + r.invalid_cause == GENERATION_INVALID_TOOL_ARGUMENTS && + r.synthesized_text.data == NULL && + r.synthesized_text.len == 0 && strcmp(r.reason, "provider sent malformed tool arguments") == 0); generation_free(g); @@ -260,11 +270,20 @@ test_length_outcome_and_empty_placeholder(void) GENERATION_TERMINAL); CHECK(generation_take(g, &r) == 0); CHECK(r.outcome == GENERATION_LENGTH && + r.invalid_cause == GENERATION_INVALID_NONE && r.data_used == sizeof(placeholder) - 1); + b = r.assistant == NULL ? NULL : TAILQ_FIRST(&r.assistant->blocks); + CHECK(b != NULL && r.synthesized_text.data == b->text && + r.synthesized_text.len == sizeof(placeholder) - 1 && + memcmp(r.synthesized_text.data, placeholder, + sizeof(placeholder) - 1) == 0); + generation_free(g); + CHECK(r.synthesized_text.data != NULL && + memcmp(r.synthesized_text.data, placeholder, + sizeof(placeholder) - 1) == 0); text_is(r.assistant, placeholder, sizeof(placeholder) - 1); if (r.assistant != NULL) msg_free(r.assistant); - generation_free(g); memset(&r, 0, sizeof(r)); g = generation_new(sizeof(placeholder) - 2); @@ -272,6 +291,8 @@ test_length_outcome_and_empty_placeholder(void) GENERATION_TERMINAL); CHECK(generation_take(g, &r) == 0); CHECK(r.outcome == GENERATION_INVALID && r.assistant == NULL && + r.invalid_cause == GENERATION_INVALID_DATA_BOUND && + r.synthesized_text.data == NULL && r.synthesized_text.len == 0 && strcmp(r.reason, "Generation data bound exceeded") == 0); generation_free(g); @@ -282,7 +303,9 @@ test_length_outcome_and_empty_placeholder(void) CHECK(generation_feed(g, GENERATION_DONE, "length", 6, NULL) == GENERATION_TERMINAL); CHECK(generation_take(g, &r) == 0); - CHECK(r.outcome == GENERATION_LENGTH && r.data_used == 1); + CHECK(r.outcome == GENERATION_LENGTH && r.data_used == 1 && + r.invalid_cause == GENERATION_INVALID_NONE && + r.synthesized_text.data == NULL && r.synthesized_text.len == 0); text_is(r.assistant, "x", 1); if (r.assistant != NULL) msg_free(r.assistant); @@ -294,6 +317,8 @@ test_length_outcome_and_empty_placeholder(void) GENERATION_TERMINAL); CHECK(generation_take(g, &r) == 0); CHECK(r.outcome == GENERATION_INVALID && r.assistant == NULL && + r.invalid_cause == GENERATION_INVALID_EMPTY_REPLY && + r.synthesized_text.data == NULL && r.synthesized_text.len == 0 && strcmp(r.reason, "provider sent an empty reply") == 0); generation_free(g); @@ -369,6 +394,8 @@ test_usage_retry_and_provider_error(void) memset(error, 'x', sizeof(error) - 1); CHECK(generation_take(g, &r) == 0); CHECK(r.outcome == GENERATION_PROVIDER_ERROR && r.assistant == NULL && + r.invalid_cause == GENERATION_INVALID_NONE && + r.synthesized_text.data == NULL && r.synthesized_text.len == 0 && r.data_used == 3 && strcmp(r.reason, "provider broke") == 0); generation_free(g); } @@ -379,34 +406,28 @@ test_usage_overflow_drains_to_terminal(void) struct generation *g; struct generation_event ev; struct generation_result r = {0}; - struct generation_usage u = { GENERATION_USAGE_EVENT_MAX, 0, 0, 0 }; - struct generation_usage seven = { 7, 0, 0, 0 }; + struct generation_usage u = { GENERATION_USAGE_MAX, 0, 0, 0 }; struct generation_usage one = { 1, 0, 0, 0 }; - int i; - /* The per-event cap is not an aggregate cap. */ + /* The exact per-Generation maximum remains valid. */ g = generation_new(1); CHECK(generation_feed(g, GENERATION_USAGE, &u, sizeof(u), NULL) == GENERATION_MORE); - CHECK(generation_feed(g, GENERATION_USAGE, &one, sizeof(one), NULL) == - GENERATION_MORE); CHECK(generation_feed(g, GENERATION_TEXT, "x", 1, NULL) == GENERATION_MORE); CHECK(generation_feed(g, GENERATION_DONE, "stop", 4, NULL) == GENERATION_TERMINAL); CHECK(generation_take(g, &r) == 0); CHECK(r.outcome == GENERATION_NORMAL && r.assistant != NULL && - r.usage.input_tokens == GENERATION_USAGE_EVENT_MAX + 1); + r.invalid_cause == GENERATION_INVALID_NONE && + r.usage.input_tokens == GENERATION_USAGE_MAX); msg_free(r.assistant); generation_free(g); memset(&r, 0, sizeof(r)); g = generation_new(10); - for (i = 0; i < 100; i++) - CHECK(generation_feed(g, GENERATION_USAGE, &u, sizeof(u), - NULL) == GENERATION_MORE); - CHECK(generation_feed(g, GENERATION_USAGE, &seven, sizeof(seven), - NULL) == GENERATION_MORE); + CHECK(generation_feed(g, GENERATION_USAGE, &u, sizeof(u), NULL) == + GENERATION_MORE); memset(&ev, 0xa5, sizeof(ev)); CHECK(generation_feed(g, GENERATION_USAGE, &one, sizeof(one), &ev) == GENERATION_DRAIN); @@ -417,7 +438,8 @@ test_usage_overflow_drains_to_terminal(void) NULL) == GENERATION_TERMINAL); CHECK(generation_take(g, &r) == 0); CHECK(r.outcome == GENERATION_INVALID && r.assistant == NULL && - r.usage.input_tokens == INT64_MAX && + r.invalid_cause == GENERATION_INVALID_USAGE_BOUND && + r.usage.input_tokens == GENERATION_USAGE_MAX && strcmp(r.reason, "Generation usage bound exceeded") == 0); generation_free(g); } @@ -429,7 +451,7 @@ test_usage_values_are_bounded(void) struct generation_result r = {0}; struct generation_usage bad[] = { { -1, 0, 0, 0 }, - { 0, GENERATION_USAGE_EVENT_MAX + 1, 0, 0 } + { 0, GENERATION_USAGE_MAX + 1, 0, 0 } }; size_t i; @@ -545,6 +567,8 @@ test_call_bound_is_64(void) GENERATION_TERMINAL); CHECK(generation_take(g, &r) == 0); CHECK(r.outcome == GENERATION_INVALID && r.assistant == NULL && + r.invalid_cause == GENERATION_INVALID_TOOL_CALL_BOUND && + r.synthesized_text.data == NULL && r.synthesized_text.len == 0 && r.tool_calls == GENERATION_CALL_MAX && r.data_used == 0 && strcmp(r.reason, "Generation tool-call bound exceeded") == 0); generation_free(g); blob - e0c5c4c60c7b8e667a2abca96e8e692961c06e20 blob + beec2eba89ea957330a8f57292790d96aad1f3ff --- src/common/generation.c +++ src/common/generation.c @@ -41,12 +41,15 @@ struct generation { struct generation_usage usage; struct msg *assistant; enum generation_outcome outcome; + enum generation_invalid_cause invalid_cause; + struct generation_span synthesized_text; char reason[GENERATION_ERROR_MAX]; char error[GENERATION_ERROR_MAX]; int terminal; int drain; int protocol; int taken; + int synthesized; }; static void @@ -61,6 +64,17 @@ generation_set_error(struct generation *g, const char g->error[len] = '\0'; } +static void +generation_invalidate(struct generation *g, + enum generation_invalid_cause cause, const char *reason) +{ + g->drain = 1; + g->outcome = GENERATION_INVALID; + g->invalid_cause = cause; + generation_set_error(g, reason); + memcpy(g->reason, g->error, strlen(g->error) + 1); +} + static enum generation_feed_status generation_protocol(struct generation *g, const char *s) { @@ -105,9 +119,8 @@ static int generation_account(struct generation *g, size_t len) { if (len > g->data_available - g->data_used) { - g->drain = 1; - g->outcome = GENERATION_INVALID; - generation_set_error(g, "Generation data bound exceeded"); + generation_invalidate(g, GENERATION_INVALID_DATA_BOUND, + "Generation data bound exceeded"); return (-1); } g->data_used += len; @@ -120,8 +133,7 @@ generation_usage_add(struct generation *g, const struc struct generation_usage sum = g->usage; #define ADD_USAGE(field) do { \ - if (u->field < 0 || u->field > GENERATION_USAGE_EVENT_MAX || \ - u->field > INT64_MAX - sum.field) \ + if (u->field < 0 || u->field > GENERATION_USAGE_MAX - sum.field) \ return (-1); \ sum.field += u->field; \ } while (0) @@ -138,11 +150,18 @@ static void generation_build_assistant(struct generation *g) { struct generation_call *call; + struct block *text; size_t i; g->assistant = msg_new(ROLE_ASSISTANT); - if (g->text.len > 0) - msg_add_text(g->assistant, (const char *)g->text.data, g->text.len); + if (g->text.len > 0) { + text = msg_add_text(g->assistant, (const char *)g->text.data, + g->text.len); + if (g->synthesized) { + g->synthesized_text.data = text->text; + g->synthesized_text.len = text->textlen; + } + } for (i = 0; i < g->ncalls; i++) { call = &g->calls[i]; msg_add_tool_use(g->assistant, call->id, call->name, @@ -225,9 +244,9 @@ generation_feed(struct generation *g, enum generation_ if (g->drain) return (generation_drain_event(ev)); if (g->ncalls >= GENERATION_CALL_MAX) { - g->drain = 1; - g->outcome = GENERATION_INVALID; - generation_set_error(g, "Generation tool-call bound exceeded"); + generation_invalidate(g, + GENERATION_INVALID_TOOL_CALL_BOUND, + "Generation tool-call bound exceeded"); return (generation_drain_event(ev)); } call = &g->calls[g->ncalls++]; @@ -269,9 +288,7 @@ generation_feed(struct generation *g, enum generation_ if (ev != NULL) ev->usage = u; if (generation_usage_add(g, &u) == -1) { - g->drain = 1; - g->outcome = GENERATION_INVALID; - generation_set_error(g, + generation_invalidate(g, GENERATION_INVALID_USAGE_BOUND, "Generation usage bound exceeded"); return (generation_drain_event(ev)); } @@ -296,44 +313,38 @@ generation_feed(struct generation *g, enum generation_ ev->payload.data = data; ev->payload.len = len; } - memcpy(g->reason, data, len); - g->reason[len] = '\0'; g->terminal = 1; limited = (len == 10 && memcmp(data, "max_tokens", 10) == 0) || (len == 6 && memcmp(data, "length", 6) == 0); - if (g->drain) { - memcpy(g->reason, g->error, strlen(g->error) + 1); + if (g->drain) return (GENERATION_TERMINAL); - } + memcpy(g->reason, data, len); + g->reason[len] = '\0'; if (g->text.len == 0 && g->ncalls == 0 && limited) { if (generation_account(g, sizeof(GENERATION_LENGTH_EMPTY) - 1) == -1) { - memcpy(g->reason, g->error, strlen(g->error) + 1); return (GENERATION_TERMINAL); } buf_add(&g->text, GENERATION_LENGTH_EMPTY, sizeof(GENERATION_LENGTH_EMPTY) - 1); + g->synthesized = 1; } else if (g->text.len == 0 && g->ncalls == 0) { - g->outcome = GENERATION_INVALID; - generation_set_error(g, "provider sent an empty reply"); - memcpy(g->reason, g->error, strlen(g->error) + 1); + generation_invalidate(g, GENERATION_INVALID_EMPTY_REPLY, + "provider sent an empty reply"); return (GENERATION_TERMINAL); } for (i = 0; i < g->ncalls; i++) { if (g->calls[i].input.len == 0) { if (generation_account(g, 2) == -1) { - memcpy(g->reason, g->error, - strlen(g->error) + 1); return (GENERATION_TERMINAL); } buf_addstr(&g->calls[i].input, "{}"); } if (!json_valid_object(g->calls[i].input.data, g->calls[i].input.len)) { - g->outcome = GENERATION_INVALID; - generation_set_error(g, + generation_invalidate(g, + GENERATION_INVALID_TOOL_ARGUMENTS, "provider sent malformed tool arguments"); - memcpy(g->reason, g->error, strlen(g->error) + 1); return (GENERATION_TERMINAL); } } @@ -350,10 +361,8 @@ generation_feed(struct generation *g, enum generation_ ev->payload.len = len; } g->terminal = 1; - if (g->drain) { - memcpy(g->reason, g->error, strlen(g->error) + 1); + if (g->drain) return (GENERATION_TERMINAL); - } memcpy(g->reason, data, len); g->reason[len] = '\0'; g->outcome = GENERATION_PROVIDER_ERROR; @@ -376,11 +385,13 @@ generation_take(struct generation *g, struct generatio } memset(r, 0, sizeof(*r)); r->outcome = g->outcome; + r->invalid_cause = g->invalid_cause; r->assistant = g->assistant; r->usage = g->usage; r->data_used = g->data_used; r->tool_calls = g->ncalls; r->reason = g->reason; + r->synthesized_text = g->synthesized_text; g->assistant = NULL; g->taken = 1; return (0); blob - 2a35d2e3284e55032a69b1dc72a7d0b1673fca21 blob + 769cff33015930a31fafa58fb25c6f572f327144 --- src/common/generation.h +++ src/common/generation.h @@ -30,7 +30,8 @@ struct generation; #define GENERATION_STOP_MAX 64 #define GENERATION_RETRY_MAX 1200 #define GENERATION_ERROR_MAX 1024 -#define GENERATION_USAGE_EVENT_MAX (INT64_MAX / 100) +/* Per-field cumulative maximum for one Generation. */ +#define GENERATION_USAGE_MAX (INT64_MAX / 100) #define GENERATION_LENGTH_EMPTY "Generation stopped before emitting content." /* @@ -72,6 +73,16 @@ enum generation_outcome { GENERATION_INVALID }; +/* Meaningful only when the outcome is GENERATION_INVALID. */ +enum generation_invalid_cause { + GENERATION_INVALID_NONE, + GENERATION_INVALID_DATA_BOUND, + GENERATION_INVALID_USAGE_BOUND, + GENERATION_INVALID_TOOL_CALL_BOUND, + GENERATION_INVALID_EMPTY_REPLY, + GENERATION_INVALID_TOOL_ARGUMENTS +}; + struct generation_usage { int64_t input_tokens; int64_t output_tokens; @@ -103,14 +114,21 @@ struct generation_event { /* * generation_take() transfers assistant to the caller. reason remains * borrowed from the Generation and is valid until generation_free(). + * invalid_cause is non-NONE exactly when outcome is INVALID; an invalid + * result has no assistant. synthesized_text is non-empty only for the + * placeholder created by an otherwise-empty LENGTH outcome. It aliases + * assistant storage and remains valid until that assistant is freed; its + * bytes are already included in data_used. */ struct generation_result { enum generation_outcome outcome; + enum generation_invalid_cause invalid_cause; struct msg *assistant; struct generation_usage usage; size_t data_used; size_t tool_calls; const char *reason; + struct generation_span synthesized_text; }; /* data_available is the Turn's remaining shared text/argument byte budget. */