sim/aws: fix vibecoding errors in logic

- Entire stdout+stderr was passed to both title and body for the github
  issue, resulting in a failure due to github's validation

Fixes:

- Pass only the line containing "simulation failed:" as title
- Pass max 50 lines following title as body
- Truncate title and body to 255 and 65536 chars respectively
  before posting github issue, just to be sure
This commit is contained in:
Jussi Saurio 2025-06-11 08:35:19 +03:00
parent 72058da9dc
commit f276ff0b72
3 changed files with 17 additions and 12 deletions

View file

@ -107,7 +107,7 @@ const run = async (seed: string, bin: string, args: string[]) => {
type: "assertion",
seed: seedForGithubIssue,
command: args.join(" "),
output: output,
failureInfo,
});
}
} catch (err2) {

View file

@ -1,5 +1,5 @@
import { App } from "octokit";
import { StackTraceInfo } from "./logParse";
import { AssertionFailureInfo, StackTraceInfo } from "./logParse";
import { levenshtein } from "./levenshtein";
type FaultPanic = {
@ -13,7 +13,7 @@ type FaultAssertion = {
type: "assertion"
seed: string
command: string
output: string
failureInfo: AssertionFailureInfo
}
type FaultTimeout = {
@ -27,6 +27,9 @@ type Fault = FaultPanic | FaultTimeout | FaultAssertion;
const MAX_OPEN_SIMULATOR_ISSUES = parseInt(process.env.MAX_OPEN_SIMULATOR_ISSUES || "10", 10);
const GITHUB_ISSUE_TITLE_MAX_LENGTH = 256;
const GITHUB_ISSUE_BODY_MAX_LENGTH = 65536;
export class GithubClient {
/* This is the git hash of the commit that the simulator was built from. */
GIT_HASH: string;
@ -84,15 +87,16 @@ export class GithubClient {
await this.initialize();
}
const title = ((f: Fault) => {
let title = ((f: Fault) => {
if (f.type === "panic") {
return `Simulator panic: "${f.stackTrace.mainError}"`;
} else
if (f.type === "assertion") {
return `Simulator assertion failure: "${f.output}"`;
return `Simulator assertion failure: "${f.failureInfo.mainError}"`;
}
return `Simulator timeout using git hash ${this.GIT_HASH}`;
})(fault);
title = title.slice(0, GITHUB_ISSUE_TITLE_MAX_LENGTH);
for (const existingIssueTitle of this.openIssueTitles) {
const MAGIC_NUMBER = 6;
if (levenshtein(existingIssueTitle, title) < MAGIC_NUMBER) {
@ -101,7 +105,8 @@ export class GithubClient {
}
}
const body = this.createIssueBody(fault);
let body = this.createIssueBody(fault);
body = body.slice(0, GITHUB_ISSUE_BODY_MAX_LENGTH);
if (this.mode === 'dry-run') {
console.log(`Dry-run mode: Would create issue in ${this.GITHUB_REPO} with title: ${title} and body: ${body}`);
@ -133,7 +138,7 @@ export class GithubClient {
private createIssueBody(fault: Fault): string {
const gitShortHash = this.GIT_HASH.substring(0, 7);
return `
## Simulator ${fault.type}
## Simulator failure type:${fault.type}
- **Seed**: ${fault.seed}
- **Git Hash**: ${this.GIT_HASH}
@ -151,7 +156,7 @@ export class GithubClient {
### ${fault.type === "panic" ? "Stack Trace" : "Output"}
\`\`\`
${fault.type === "panic" ? fault.stackTrace.trace : fault.output}
${fault.type === "panic" ? fault.stackTrace.trace : fault.type === "assertion" ? fault.failureInfo.output : fault.output}
\`\`\`
`;
}

View file

@ -9,6 +9,7 @@ export type StackTraceInfo = {
export type AssertionFailureInfo = {
type: "assertion";
output: string;
mainError: string;
}
/**
@ -17,8 +18,6 @@ export type AssertionFailureInfo = {
export function extractFailureInfo(output: string): StackTraceInfo | AssertionFailureInfo {
const lines = output.split('\n');
const panicLineIndex = lines.findIndex(line => line.includes("panic occurred"));
const info = getTraceFromOutput(lines) ?? getAssertionFailureInfo(lines);
if (!info) {
@ -50,9 +49,10 @@ function getAssertionFailureInfo(lines: string[]): AssertionFailureInfo | null {
}
const startIndex = simulationFailedLineIndex;
const endIndex = Math.min(lines.length, startIndex + 1);
const endIndex = Math.min(lines.length, startIndex + 50);
const output = lines.slice(startIndex, endIndex).join('\n');
const mainError = lines[startIndex] ?? "???";
return { type: "assertion", output };
return { type: "assertion", output, mainError };
}