commit d5e57bf6e1f3359aa57a5535d382d3509e1515a3 from: Isaac Meerleo date: Sat Jul 18 17:38:46 2026 UTC Cancel live tools when Artifact storage fills commit - 1064babd89d73d9e1207d1212c78523c029dbe6a commit + d5e57bf6e1f3359aa57a5535d382d3509e1515a3 blob - 00ff8f0b07ef8f1fe47232c0131c252992729b38 blob + c82aa3744f59ce154cc0ce1ad52633a12258b09c --- regress/artifact_store/artifact_store_test.c +++ regress/artifact_store/artifact_store_test.c @@ -60,6 +60,29 @@ exists(const char *root, const char *session, const ch return (access(path, F_OK) == 0); } +static int +artifact_namespace_empty(const char *root, const char *session) +{ + char path[PATH_MAX]; + struct dirent *de; + DIR *dir; + int empty = 1; + + if ((size_t)snprintf(path, sizeof(path), "%s/%s", root, session) >= + sizeof(path) || (dir = opendir(path)) == NULL) + return (0); + while ((de = readdir(dir)) != NULL) { + if (strncmp(de->d_name, ".pending-", 9) == 0 || + strncmp(de->d_name, ".order-", 7) == 0 || + artifact_id_valid(de->d_name)) { + empty = 0; + break; + } + } + closedir(dir); + return (empty); +} + static void mkdir_session(const char *root, const char *session) { @@ -657,6 +680,53 @@ test_terminal_storage_errors(const char *root) } static void +test_enospc_storage_errors(const char *root) +{ + struct artifact_store *store; + struct artifact_pending *pending = NULL; + struct artifact_stream stream; + struct artifact_terminal terminal; + char name[64]; + int rc, write_fd = -1; + + store = open_ready(root, "enospc-reservation", 1); + CHECK(store != NULL && artifact_store_turn_begin(store) == 0); + artifact_store_test_fail_errno( + ARTIFACT_STORE_FAULT_BEGIN_FTRUNCATE, ENOSPC); + rc = begin_stream(store, 1, &pending, &write_fd, &stream); + CHECK(rc == 1 && stream.status == ARTIFACT_STREAM_UNAVAILABLE && + pending == NULL && write_fd == -1 && + artifact_namespace_empty(root, "enospc-reservation")); + /* The unavailable reservation leaves the same Turn retryable. */ + CHECK(begin_stream(store, 1, &pending, &write_fd, &stream) == 0); + close(write_fd); + CHECK(artifact_store_turn_abort(store) == 0 && + artifact_namespace_empty(root, "enospc-reservation")); + artifact_store_close(store); + + store = open_ready(root, "enospc-write", 1); + CHECK(store != NULL && artifact_store_turn_begin(store) == 0 && + begin_stream(store, 1, &pending, &write_fd, &stream) == 0); + CHECK(write(write_fd, "abc", 3) == 3); + artifact_store_test_fail_errno(ARTIFACT_STORE_FAULT_DRAIN_PWRITE, + ENOSPC); + rc = artifact_pending_drain(pending, &terminal); + close(write_fd); + CHECK(rc == -1 && terminal.status == ARTIFACT_STORE_ERROR && + terminal.retained_bytes == 0); + (void)snprintf(name, sizeof(name), ".pending-%s", stream.id); + CHECK(exists(root, "enospc-write", name) && + !exists(root, "enospc-write", stream.id)); + (void)snprintf(name, sizeof(name), ".order-%s", stream.id); + CHECK(exists(root, "enospc-write", name)); + CHECK(artifact_store_retain(store, 1, stream.id) == -1 && + artifact_store_turn_prepare(store) == -1); + CHECK(artifact_store_turn_abort(store) == 0 && + artifact_namespace_empty(root, "enospc-write")); + artifact_store_close(store); +} + +static void test_abort_retry(const char *root) { static const struct { @@ -1415,6 +1485,7 @@ main(void) test_publication_collision_and_prepared_close(root); #ifdef ARTIFACT_STORE_TESTING test_terminal_storage_errors(root); + test_enospc_storage_errors(root); test_abort_retry(root); test_partial_publication_cleanup(root); test_discard_retry_does_not_close_reused_fd(root); blob - dd3930ae6f3b40c910393b9077d5fdcbd9c562fa blob + b316e27616cc32b9578a41eb88eb89cb87e15434 --- regress/turn/run.sh +++ regress/turn/run.sh @@ -734,6 +734,38 @@ grep -Eq '"id":"toolu_output_read_default","name":"out ok "output_read loop: id-only call applies text and 64 KiB defaults" $? stop_stub +# A real ENOSPC on the first Artifact write is terminal. It cancels the live +# foreground executor, reports storage_error to the next Generation, and rolls +# back every private staging/final name instead of journaling an Artifact. +rm -rf "$HOME/.fugu" +start_stub "STORAGEFULL OK" +write_conf "$stubport" +t0=$(date +%s) +FUGU_SELFTEST=1 FUGU_ARTIFACT_ENOSPC_WRITE=1 \ + FUGU_CONF=$dir/fugu.conf FUGU_LIBEXEC=$dir/libexec/fugu \ + FUGU_CA_FILE=$dir/cert.pem FUGU_RETRY_BASE_MS=50 \ + "$fugu" -p "exercise storage full" "$dir/out" 2>"$dir/errout" +rc=$? +t1=$(date +%s) +ok "Artifact ENOSPC: Turn recovers through the next Generation" \ + $([ "$rc" -eq 0 ]; echo $?) +ok "Artifact ENOSPC: live foreground executor is cancelled promptly" \ + $([ $((t1 - t0)) -lt 15 ]; echo $?) +request_log_entry 2 "$dir/enospc-request" +grep -Fq '\"termination\":\"storage_error\"' "$dir/enospc-request" +ok "Artifact ENOSPC: the continuation receives storage_error" $? +journal=$(ls "$sessions"/*.ndjson 2>/dev/null | head -1) +! grep -q '"t":"artifact"' "$journal" 2>/dev/null +ok "Artifact ENOSPC: no Artifact record commits" $? +if find "$HOME/.fugu/artifacts" -type f 2>/dev/null | \ + grep -Eq '/(\.pending-|\.order-)?[0-9a-f]{32}$'; then + fail "Artifact ENOSPC: no pending, order, or final name leaks" +else + pass "Artifact ENOSPC: no pending, order, or final name leaks" +fi +stop_stub + # A read response remains an exact bounded transaction after Ctrl-C. The # regression-only Role delay widens the cancellation window; Coordinator must # drain DATA/READ_RESULT before Artifact Turn abort reuses the same channel. blob - 4c738bf1b95897ccc589f844b521cd6d3f82746d blob + 6fbb99b07377c4135999435bef4335f28887d4fc --- regress/turn/stub.c +++ regress/turn/stub.c @@ -30,6 +30,7 @@ * page), OUTPUTREADDEFAULT (a defaults-only Artifact page), * BGTOOL (shell_bg sleep 300), BGSHORT (shell_bg sleep 1), * BGFAST (shell_bg immediate output), + * STORAGEFULL (foreground output followed by a long sleep), * ERROR_BG (the provider failure after * that launch), JOBOUTPUT (shell_output job 1), DOUBLE (a hostile error event * appended after a complete OK stream), AGENTCALL (lead delegates to a @@ -1058,6 +1059,29 @@ static const char sse_sleeptool[] = "event: message_stop\n" "data: {\"type\":\"message_stop\"}\n\n"; +/* Emit output before a long foreground sleep. The Artifact ENOSPC fixture + * fails the first retained write and must cancel this process immediately. */ +static const char sse_storagefull[] = +"event: message_start\n" +"data: {\"type\":\"message_start\",\"message\":{\"usage\":" + "{\"input_tokens\":9,\"output_tokens\":1}}}\n\n" +"event: content_block_start\n" +"data: {\"type\":\"content_block_start\",\"index\":0," + "\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_full\"," + "\"name\":\"shell\"}}\n\n" +"event: content_block_delta\n" +"data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":" + "{\"type\":\"input_json_delta\",\"partial_json\":" + "\"{\\\"command\\\":\\\"printf ENOSPC-WRITE-START; sleep 30; " + "printf ENOSPC-WRITE-NOT-CANCELLED\\\"}\"}}\n\n" +"event: content_block_stop\n" +"data: {\"type\":\"content_block_stop\",\"index\":0}\n\n" +"event: message_delta\n" +"data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":" + "\"tool_use\"},\"usage\":{\"output_tokens\":12}}\n\n" +"event: message_stop\n" +"data: {\"type\":\"message_stop\"}\n\n"; + /* Launch a long-lived background job. The turn regression follows this * response with a provider error, so the Coordinator must abort the job's * turn-scoped registration before accepting the next user turn. */ @@ -2101,6 +2125,12 @@ main(int argc, char *argv[]) write_chunk(conn, sse_sleeptool, strlen(sse_sleeptool)); (void)tls_write_all(conn, "0\r\n\r\n", 5); + } else if (strcmp(argv[i], "STORAGEFULL") == 0) { + (void)tls_write_all(conn, body_ok, + strlen(body_ok)); + write_chunk(conn, sse_storagefull, + strlen(sse_storagefull)); + (void)tls_write_all(conn, "0\r\n\r\n", 5); } else if (strcmp(argv[i], "BGTOOL") == 0) { (void)tls_write_all(conn, body_ok, strlen(body_ok)); blob - bf6bb4d01895443dae6e7d077bd2f10781b3725c blob + fcb992463e5e456a295705475380532f909734f5 --- src/fugu-artifact/store.c +++ src/fugu-artifact/store.c @@ -95,11 +95,20 @@ static struct artifact_pending *pending_find_correlati #ifdef ARTIFACT_STORE_TESTING static enum artifact_store_test_fault test_fault; +static int test_fault_errno = EIO; void artifact_store_test_fail(enum artifact_store_test_fault fault) { + artifact_store_test_fail_errno(fault, EIO); +} + +void +artifact_store_test_fail_errno(enum artifact_store_test_fault fault, + int error) +{ test_fault = fault; + test_fault_errno = error; } static int @@ -108,7 +117,7 @@ fault_hit(enum artifact_store_test_fault fault) if (test_fault != fault) return (0); test_fault = ARTIFACT_STORE_FAULT_NONE; - errno = EIO; + errno = test_fault_errno; return (1); } #else @@ -116,6 +125,21 @@ fault_hit(enum artifact_store_test_fault fault) #endif static int +selftest_enospc_write(void) +{ + static int injected; + const char *value; + + if (injected || issetugid() || getenv("FUGU_SELFTEST") == NULL || + (value = getenv("FUGU_ARTIFACT_ENOSPC_WRITE")) == NULL || + strcmp(value, "1") != 0) + return (0); + injected = 1; + errno = ENOSPC; + return (1); +} + +static int session_valid(const char *id) { size_t i, len = strlen(id); @@ -869,6 +893,7 @@ artifact_store_begin(struct artifact_store *s, uint32_ break; } if (p->file_fd == -1 || fchmod(p->file_fd, 0600) == -1 || + fault_hit(ARTIFACT_STORE_FAULT_BEGIN_FTRUNCATE) || ftruncate(p->file_fd, ARTIFACT_RAW_MAX) == -1 || pipe2(pipes, O_CLOEXEC | O_NONBLOCK) == -1) { if (p->file_fd != -1) { @@ -1022,12 +1047,16 @@ artifact_pending_drain(struct artifact_pending *p, terminal->status = ARTIFACT_STORE_CAP; } if (want > 0) { - do { - w = pwrite(p->file_fd, buf, want, - (off_t)p->retained); - } while (w == -1 && errno == EINTR); - if (fault_hit(ARTIFACT_STORE_FAULT_DRAIN_PWRITE) || - w != (ssize_t)want) { + if (fault_hit(ARTIFACT_STORE_FAULT_DRAIN_PWRITE) || + selftest_enospc_write()) + w = -1; + else { + do { + w = pwrite(p->file_fd, buf, want, + (off_t)p->retained); + } while (w == -1 && errno == EINTR); + } + if (w != (ssize_t)want) { result = -1; terminal->status = ARTIFACT_STORE_ERROR; } else blob - 53ad88a2dd0728d10fef8858ecadf4dcace41e3d blob + c5d1f1612b7702914be4702815b135880c694b5d --- src/fugu-artifact/store.h +++ src/fugu-artifact/store.h @@ -55,6 +55,7 @@ int artifact_store_read(struct artifact_store *, /* Deterministic syscall-failure seams used only by the store regression. */ enum artifact_store_test_fault { ARTIFACT_STORE_FAULT_NONE = 0, + ARTIFACT_STORE_FAULT_BEGIN_FTRUNCATE, ARTIFACT_STORE_FAULT_DRAIN_PWRITE, ARTIFACT_STORE_FAULT_DRAIN_FTRUNCATE, ARTIFACT_STORE_FAULT_DRAIN_FUTIMENS, @@ -71,6 +72,7 @@ enum artifact_store_test_fault { }; void artifact_store_test_fail(enum artifact_store_test_fault); +void artifact_store_test_fail_errno(enum artifact_store_test_fault, int); #endif #endif /* ARTIFACT_STORE_H */