mirror of
https://github.com/tursodatabase/limbo.git
synced 2025-07-07 12:35:00 +00:00
sim/aws: fix vibecoding errors in logic
- Entire stdout+stderr was passed to both title and body for the github issue, resulting in a failure due to github's validation Fixes: - Pass only the line containing "simulation failed:" as title - Pass max 50 lines following title as body - Truncate title and body to 255 and 65536 chars respectively before posting github issue, just to be sure
This commit is contained in:
parent
72058da9dc
commit
f276ff0b72
3 changed files with 17 additions and 12 deletions
|
@ -107,7 +107,7 @@ const run = async (seed: string, bin: string, args: string[]) => {
|
|||
type: "assertion",
|
||||
seed: seedForGithubIssue,
|
||||
command: args.join(" "),
|
||||
output: output,
|
||||
failureInfo,
|
||||
});
|
||||
}
|
||||
} catch (err2) {
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
import { App } from "octokit";
|
||||
import { StackTraceInfo } from "./logParse";
|
||||
import { AssertionFailureInfo, StackTraceInfo } from "./logParse";
|
||||
import { levenshtein } from "./levenshtein";
|
||||
|
||||
type FaultPanic = {
|
||||
|
@ -13,7 +13,7 @@ type FaultAssertion = {
|
|||
type: "assertion"
|
||||
seed: string
|
||||
command: string
|
||||
output: string
|
||||
failureInfo: AssertionFailureInfo
|
||||
}
|
||||
|
||||
type FaultTimeout = {
|
||||
|
@ -27,6 +27,9 @@ type Fault = FaultPanic | FaultTimeout | FaultAssertion;
|
|||
|
||||
const MAX_OPEN_SIMULATOR_ISSUES = parseInt(process.env.MAX_OPEN_SIMULATOR_ISSUES || "10", 10);
|
||||
|
||||
const GITHUB_ISSUE_TITLE_MAX_LENGTH = 256;
|
||||
const GITHUB_ISSUE_BODY_MAX_LENGTH = 65536;
|
||||
|
||||
export class GithubClient {
|
||||
/* This is the git hash of the commit that the simulator was built from. */
|
||||
GIT_HASH: string;
|
||||
|
@ -84,15 +87,16 @@ export class GithubClient {
|
|||
await this.initialize();
|
||||
}
|
||||
|
||||
const title = ((f: Fault) => {
|
||||
let title = ((f: Fault) => {
|
||||
if (f.type === "panic") {
|
||||
return `Simulator panic: "${f.stackTrace.mainError}"`;
|
||||
} else
|
||||
if (f.type === "assertion") {
|
||||
return `Simulator assertion failure: "${f.output}"`;
|
||||
return `Simulator assertion failure: "${f.failureInfo.mainError}"`;
|
||||
}
|
||||
return `Simulator timeout using git hash ${this.GIT_HASH}`;
|
||||
})(fault);
|
||||
title = title.slice(0, GITHUB_ISSUE_TITLE_MAX_LENGTH);
|
||||
for (const existingIssueTitle of this.openIssueTitles) {
|
||||
const MAGIC_NUMBER = 6;
|
||||
if (levenshtein(existingIssueTitle, title) < MAGIC_NUMBER) {
|
||||
|
@ -101,7 +105,8 @@ export class GithubClient {
|
|||
}
|
||||
}
|
||||
|
||||
const body = this.createIssueBody(fault);
|
||||
let body = this.createIssueBody(fault);
|
||||
body = body.slice(0, GITHUB_ISSUE_BODY_MAX_LENGTH);
|
||||
|
||||
if (this.mode === 'dry-run') {
|
||||
console.log(`Dry-run mode: Would create issue in ${this.GITHUB_REPO} with title: ${title} and body: ${body}`);
|
||||
|
@ -133,7 +138,7 @@ export class GithubClient {
|
|||
private createIssueBody(fault: Fault): string {
|
||||
const gitShortHash = this.GIT_HASH.substring(0, 7);
|
||||
return `
|
||||
## Simulator ${fault.type}
|
||||
## Simulator failure type:${fault.type}
|
||||
|
||||
- **Seed**: ${fault.seed}
|
||||
- **Git Hash**: ${this.GIT_HASH}
|
||||
|
@ -151,7 +156,7 @@ export class GithubClient {
|
|||
### ${fault.type === "panic" ? "Stack Trace" : "Output"}
|
||||
|
||||
\`\`\`
|
||||
${fault.type === "panic" ? fault.stackTrace.trace : fault.output}
|
||||
${fault.type === "panic" ? fault.stackTrace.trace : fault.type === "assertion" ? fault.failureInfo.output : fault.output}
|
||||
\`\`\`
|
||||
`;
|
||||
}
|
||||
|
|
|
@ -9,6 +9,7 @@ export type StackTraceInfo = {
|
|||
export type AssertionFailureInfo = {
|
||||
type: "assertion";
|
||||
output: string;
|
||||
mainError: string;
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -17,8 +18,6 @@ export type AssertionFailureInfo = {
|
|||
export function extractFailureInfo(output: string): StackTraceInfo | AssertionFailureInfo {
|
||||
const lines = output.split('\n');
|
||||
|
||||
const panicLineIndex = lines.findIndex(line => line.includes("panic occurred"));
|
||||
|
||||
const info = getTraceFromOutput(lines) ?? getAssertionFailureInfo(lines);
|
||||
|
||||
if (!info) {
|
||||
|
@ -50,9 +49,10 @@ function getAssertionFailureInfo(lines: string[]): AssertionFailureInfo | null {
|
|||
}
|
||||
|
||||
const startIndex = simulationFailedLineIndex;
|
||||
const endIndex = Math.min(lines.length, startIndex + 1);
|
||||
const endIndex = Math.min(lines.length, startIndex + 50);
|
||||
|
||||
const output = lines.slice(startIndex, endIndex).join('\n');
|
||||
const mainError = lines[startIndex] ?? "???";
|
||||
|
||||
return { type: "assertion", output };
|
||||
return { type: "assertion", output, mainError };
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue