Add simulator-docker-runner for running limbo-sim in a loop on AWS

2025-07-07 12:35:00 +00:00 · 2025-06-02 12:13:24 +03:00 · 2025-06-02 12:13:24 +03:00 · d06bb70514
commit d06bb70514
parent e6cfeb9552
12 changed files with 721 additions and 0 deletions
--- a/.github/workflows/build-sim.yml
+++ b/.github/workflows/build-sim.yml
@ -0,0 +1,44 @@
+name: Build and push limbo-sim image
+
+on:
+  push:
+    branches:
+      - main
+
+# Add permissions needed for OIDC authentication with AWS
+permissions:
+  id-token: write # allow getting OIDC token
+  contents: read # allow reading repository contents
+
+# Ensure only one build runs at a time. A new push to main will cancel any in-progress build.
+concurrency:
+  group: "build-sim"
+  cancel-in-progress: true
+
+env:
+  AWS_REGION: ${{ secrets.LIMBO_SIM_AWS_REGION }}
+  IAM_ROLE: ${{ secrets.LIMBO_SIM_DEPLOYER_IAM_ROLE }}
+  ECR_URL: ${{ secrets.LIMBO_SIM_ECR_URL }}
+  ECR_IMAGE_NAME: ${{ secrets.LIMBO_SIM_IMAGE_NAME }}
+  GIT_HASH: ${{ github.sha }}
+
+jobs:
+  deploy:
+    runs-on: blacksmith
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ env.IAM_ROLE }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Build and push limbo-sim docker image
+        run: |
+          docker build -f simulator-docker-runner/Dockerfile.simulator -t ${{ env.ECR_URL }}/${{ env.ECR_IMAGE_NAME }} --build-arg GIT_HASH=${{ env.GIT_HASH }} .
+          docker push ${{ env.ECR_URL }}/${{ env.ECR_IMAGE_NAME }}
--- a/simulator-docker-runner/Dockerfile.simulator
+++ b/simulator-docker-runner/Dockerfile.simulator
@ -0,0 +1,74 @@
+FROM lukemathwalker/cargo-chef:latest-rust-1.87.0 AS chef
+RUN apt update \
+    && apt install -y git libssl-dev pkg-config\
+    && apt clean \
+    && rm -rf /var/lib/apt/lists/*
+WORKDIR /app
+
+#
+# Cache dependencies
+#
+
+FROM chef AS planner
+# cargo-chef requires all this crap to be present in the workspace
+COPY Cargo.toml Cargo.lock ./
+COPY core ./core/
+COPY simulator ./simulator/
+COPY bindings ./bindings/
+COPY extensions ./extensions/
+COPY macros ./macros/
+COPY vendored ./vendored/
+COPY cli ./cli/
+COPY sqlite3 ./sqlite3/
+COPY stress ./stress/
+COPY tests ./tests/
+RUN cargo chef prepare --bin limbo_sim --recipe-path recipe.json
+
+#
+# Build the project.
+#
+
+FROM chef AS builder 
+
+COPY --from=planner /app/recipe.json recipe.json
+RUN cargo chef cook --bin limbo_sim --release --recipe-path recipe.json
+COPY --from=planner /app/core ./core/
+COPY --from=planner /app/vendored ./vendored/
+COPY --from=planner /app/extensions ./extensions/
+COPY --from=planner /app/macros ./macros/
+COPY --from=planner /app/simulator ./simulator/
+
+RUN cargo build --bin limbo_sim --release
+
+#
+# The final image.
+#
+
+FROM debian:bookworm-slim AS runtime
+# Install Bun (we want to use a fancy shell so we can e.g. post github issues when simulator fails)
+RUN apt-get update && apt-get install -y \
+    curl \
+    unzip \
+    ca-certificates \
+    libssl3 \
+    openssl \
+    && curl -fsSL https://bun.sh/install | bash \
+    && ln -s /root/.bun/bin/bun /usr/local/bin/bun \
+    && rm -rf /var/lib/apt/lists/*
+
+# Accept git hash as build argument
+ARG GIT_HASH
+ENV GIT_HASH=${GIT_HASH:-unknown}
+
+WORKDIR /app
+COPY --from=builder /app/target/release/limbo_sim /app/limbo_sim
+COPY simulator-docker-runner/package.json simulator-docker-runner/bun.lock simulator-docker-runner/tsconfig.json ./
+RUN bun install
+COPY simulator-docker-runner/* ./
+
+RUN chmod +x /app/docker-entrypoint.simulator.ts
+RUN chmod +x /app/limbo_sim
+
+ENTRYPOINT ["bun", "/app/docker-entrypoint.simulator.ts"]
+# Arguments can be passed at runtime
+CMD []
--- a/simulator-docker-runner/README.MD
+++ b/simulator-docker-runner/README.MD
@ -0,0 +1,20 @@
+# Limbo Simulator Docker Runner
+
+This directory contains the script that runs inside the `limbo-sim` Docker image. The script continuously runs the `limbo-sim` program in a loop until it encounters a panic, at which point it automatically creates a GitHub issue in the `limbo` repository.
+
+## What it does
+
+1. The limbo-sim image is built and pushed to ECR by [.github/workflows/build-sim.yaml](../.github/workflows/build-sim.yaml) on every main commit
+2. When the container starts, this script:
+   - Runs the [limbo-sim](../simulator/) program with a random seed
+   - If a panic occurs:
+     - Captures the seed value and commit hash
+     - Creates a GitHub issue with reproduction steps
+     - Includes panic output and relevant metadata
+   - Continues running with a new seed until a panic occurs or TIME_LIMIT_MINUTES is reached
+
+The script acts as the entrypoint for the Docker container, automatically starting the simulation loop when the container launches.
+
+## How do I see the open issues created by the simulator?
+
+[GitHub issues](https://github.com/tursodatabase/limbo/issues?q=is%3Aissue+is%3Aopen+label%3A%22automated%22)
--- a/simulator-docker-runner/bun.lock
+++ b/simulator-docker-runner/bun.lock
@ -0,0 +1,93 @@
+{
+  "lockfileVersion": 1,
+  "workspaces": {
+    "": {
+      "name": "simulator-docker-runner",
+      "dependencies": {
+        "octokit": "4.1.2",
+      },
+      "devDependencies": {
+        "@types/node": "22.13.5",
+        "bun-types": "1.2.4",
+        "typescript": "5.7.3",
+      },
+    },
+  },
+  "packages": {
+    "@octokit/app": ["@octokit/app@15.1.5", "", { "dependencies": { "@octokit/auth-app": "^7.1.5", "@octokit/auth-unauthenticated": "^6.1.2", "@octokit/core": "^6.1.4", "@octokit/oauth-app": "^7.1.6", "@octokit/plugin-paginate-rest": "^11.4.2", "@octokit/types": "^13.8.0", "@octokit/webhooks": "^13.6.1" } }, "sha512-6cxLT9U8x7GGQ7lNWsKtFr4ccg9oLkGvowk373sX9HvX5U37kql5d55SzaQUxPE8PwgX2cqkzDm5NF5aPKevqg=="],
+
+    "@octokit/auth-app": ["@octokit/auth-app@7.1.5", "", { "dependencies": { "@octokit/auth-oauth-app": "^8.1.3", "@octokit/auth-oauth-user": "^5.1.3", "@octokit/request": "^9.2.1", "@octokit/request-error": "^6.1.7", "@octokit/types": "^13.8.0", "toad-cache": "^3.7.0", "universal-github-app-jwt": "^2.2.0", "universal-user-agent": "^7.0.0" } }, "sha512-boklS4E6LpbA3nRx+SU2fRKRGZJdOGoSZne/i3Y0B5rfHOcGwFgcXrwDLdtbv4igfDSnAkZaoNBv1GYjPDKRNw=="],
+
+    "@octokit/auth-oauth-app": ["@octokit/auth-oauth-app@8.1.3", "", { "dependencies": { "@octokit/auth-oauth-device": "^7.1.3", "@octokit/auth-oauth-user": "^5.1.3", "@octokit/request": "^9.2.1", "@octokit/types": "^13.6.2", "universal-user-agent": "^7.0.0" } }, "sha512-4e6OjVe5rZ8yBe8w7byBjpKtSXFuro7gqeGAAZc7QYltOF8wB93rJl2FE0a4U1Mt88xxPv/mS+25/0DuLk0Ewg=="],
+
+    "@octokit/auth-oauth-device": ["@octokit/auth-oauth-device@7.1.3", "", { "dependencies": { "@octokit/oauth-methods": "^5.1.4", "@octokit/request": "^9.2.1", "@octokit/types": "^13.6.2", "universal-user-agent": "^7.0.0" } }, "sha512-BECO/N4B/Uikj0w3GCvjf/odMujtYTP3q82BJSjxC2J3rxTEiZIJ+z2xnRlDb0IE9dQSaTgRqUPVOieSbFcVzg=="],
+
+    "@octokit/auth-oauth-user": ["@octokit/auth-oauth-user@5.1.3", "", { "dependencies": { "@octokit/auth-oauth-device": "^7.1.3", "@octokit/oauth-methods": "^5.1.3", "@octokit/request": "^9.2.1", "@octokit/types": "^13.6.2", "universal-user-agent": "^7.0.0" } }, "sha512-zNPByPn9K7TC+OOHKGxU+MxrE9SZAN11UHYEFLsK2NRn3akJN2LHRl85q+Eypr3tuB2GrKx3rfj2phJdkYCvzw=="],
+
+    "@octokit/auth-token": ["@octokit/auth-token@5.1.2", "", {}, "sha512-JcQDsBdg49Yky2w2ld20IHAlwr8d/d8N6NiOXbtuoPCqzbsiJgF633mVUw3x4mo0H5ypataQIX7SFu3yy44Mpw=="],
+
+    "@octokit/auth-unauthenticated": ["@octokit/auth-unauthenticated@6.1.2", "", { "dependencies": { "@octokit/request-error": "^6.1.7", "@octokit/types": "^13.6.2" } }, "sha512-07DlUGcz/AAVdzu3EYfi/dOyMSHp9YsOxPl/MPmtlVXWiD//GlV8HgZsPhud94DEyx+RfrW0wSl46Lx+AWbOlg=="],
+
+    "@octokit/core": ["@octokit/core@6.1.4", "", { "dependencies": { "@octokit/auth-token": "^5.0.0", "@octokit/graphql": "^8.1.2", "@octokit/request": "^9.2.1", "@octokit/request-error": "^6.1.7", "@octokit/types": "^13.6.2", "before-after-hook": "^3.0.2", "universal-user-agent": "^7.0.0" } }, "sha512-lAS9k7d6I0MPN+gb9bKDt7X8SdxknYqAMh44S5L+lNqIN2NuV8nvv3g8rPp7MuRxcOpxpUIATWprO0C34a8Qmg=="],
+
+    "@octokit/endpoint": ["@octokit/endpoint@10.1.3", "", { "dependencies": { "@octokit/types": "^13.6.2", "universal-user-agent": "^7.0.2" } }, "sha512-nBRBMpKPhQUxCsQQeW+rCJ/OPSMcj3g0nfHn01zGYZXuNDvvXudF/TYY6APj5THlurerpFN4a/dQAIAaM6BYhA=="],
+
+    "@octokit/graphql": ["@octokit/graphql@8.2.1", "", { "dependencies": { "@octokit/request": "^9.2.2", "@octokit/types": "^13.8.0", "universal-user-agent": "^7.0.0" } }, "sha512-n57hXtOoHrhwTWdvhVkdJHdhTv0JstjDbDRhJfwIRNfFqmSo1DaK/mD2syoNUoLCyqSjBpGAKOG0BuwF392slw=="],
+
+    "@octokit/oauth-app": ["@octokit/oauth-app@7.1.6", "", { "dependencies": { "@octokit/auth-oauth-app": "^8.1.3", "@octokit/auth-oauth-user": "^5.1.3", "@octokit/auth-unauthenticated": "^6.1.2", "@octokit/core": "^6.1.4", "@octokit/oauth-authorization-url": "^7.1.1", "@octokit/oauth-methods": "^5.1.4", "@types/aws-lambda": "^8.10.83", "universal-user-agent": "^7.0.0" } }, "sha512-OMcMzY2WFARg80oJNFwWbY51TBUfLH4JGTy119cqiDawSFXSIBujxmpXiKbGWQlvfn0CxE6f7/+c6+Kr5hI2YA=="],
+
+    "@octokit/oauth-authorization-url": ["@octokit/oauth-authorization-url@7.1.1", "", {}, "sha512-ooXV8GBSabSWyhLUowlMIVd9l1s2nsOGQdlP2SQ4LnkEsGXzeCvbSbCPdZThXhEFzleGPwbapT0Sb+YhXRyjCA=="],
+
+    "@octokit/oauth-methods": ["@octokit/oauth-methods@5.1.4", "", { "dependencies": { "@octokit/oauth-authorization-url": "^7.0.0", "@octokit/request": "^9.2.1", "@octokit/request-error": "^6.1.7", "@octokit/types": "^13.6.2" } }, "sha512-Jc/ycnePClOvO1WL7tlC+TRxOFtyJBGuTDsL4dzXNiVZvzZdrPuNw7zHI3qJSUX2n6RLXE5L0SkFmYyNaVUFoQ=="],
+
+    "@octokit/openapi-types": ["@octokit/openapi-types@23.0.1", "", {}, "sha512-izFjMJ1sir0jn0ldEKhZ7xegCTj/ObmEDlEfpFrx4k/JyZSMRHbO3/rBwgE7f3m2DHt+RrNGIVw4wSmwnm3t/g=="],
+
+    "@octokit/openapi-webhooks-types": ["@octokit/openapi-webhooks-types@10.1.1", "", {}, "sha512-qBfqQVIDQaCFeGCofXieJDwvXcGgDn17+UwZ6WW6lfEvGYGreLFzTiaz9xjet9Us4zDf8iasoW3ixUj/R5lMhA=="],
+
+    "@octokit/plugin-paginate-graphql": ["@octokit/plugin-paginate-graphql@5.2.4", "", { "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-pLZES1jWaOynXKHOqdnwZ5ULeVR6tVVCMm+AUbp0htdcyXDU95WbkYdU4R2ej1wKj5Tu94Mee2Ne0PjPO9cCyA=="],
+
+    "@octokit/plugin-paginate-rest": ["@octokit/plugin-paginate-rest@11.4.3", "", { "dependencies": { "@octokit/types": "^13.7.0" }, "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-tBXaAbXkqVJlRoA/zQVe9mUdb8rScmivqtpv3ovsC5xhje/a+NOCivs7eUhWBwCApJVsR4G5HMeaLbq7PxqZGA=="],
+
+    "@octokit/plugin-rest-endpoint-methods": ["@octokit/plugin-rest-endpoint-methods@13.3.1", "", { "dependencies": { "@octokit/types": "^13.8.0" }, "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-o8uOBdsyR+WR8MK9Cco8dCgvG13H1RlM1nWnK/W7TEACQBFux/vPREgKucxUfuDQ5yi1T3hGf4C5ZmZXAERgwQ=="],
+
+    "@octokit/plugin-retry": ["@octokit/plugin-retry@7.1.4", "", { "dependencies": { "@octokit/request-error": "^6.1.7", "@octokit/types": "^13.6.2", "bottleneck": "^2.15.3" }, "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-7AIP4p9TttKN7ctygG4BtR7rrB0anZqoU9ThXFk8nETqIfvgPUANTSYHqWYknK7W3isw59LpZeLI8pcEwiJdRg=="],
+
+    "@octokit/plugin-throttling": ["@octokit/plugin-throttling@9.4.0", "", { "dependencies": { "@octokit/types": "^13.7.0", "bottleneck": "^2.15.3" }, "peerDependencies": { "@octokit/core": "^6.1.3" } }, "sha512-IOlXxXhZA4Z3m0EEYtrrACkuHiArHLZ3CvqWwOez/pURNqRuwfoFlTPbN5Muf28pzFuztxPyiUiNwz8KctdZaQ=="],
+
+    "@octokit/request": ["@octokit/request@9.2.2", "", { "dependencies": { "@octokit/endpoint": "^10.1.3", "@octokit/request-error": "^6.1.7", "@octokit/types": "^13.6.2", "fast-content-type-parse": "^2.0.0", "universal-user-agent": "^7.0.2" } }, "sha512-dZl0ZHx6gOQGcffgm1/Sf6JfEpmh34v3Af2Uci02vzUYz6qEN6zepoRtmybWXIGXFIK8K9ylE3b+duCWqhArtg=="],
+
+    "@octokit/request-error": ["@octokit/request-error@6.1.7", "", { "dependencies": { "@octokit/types": "^13.6.2" } }, "sha512-69NIppAwaauwZv6aOzb+VVLwt+0havz9GT5YplkeJv7fG7a40qpLt/yZKyiDxAhgz0EtgNdNcb96Z0u+Zyuy2g=="],
+
+    "@octokit/types": ["@octokit/types@13.8.0", "", { "dependencies": { "@octokit/openapi-types": "^23.0.1" } }, "sha512-x7DjTIbEpEWXK99DMd01QfWy0hd5h4EN+Q7shkdKds3otGQP+oWE/y0A76i1OvH9fygo4ddvNf7ZvF0t78P98A=="],
+
+    "@octokit/webhooks": ["@octokit/webhooks@13.7.4", "", { "dependencies": { "@octokit/openapi-webhooks-types": "10.1.1", "@octokit/request-error": "^6.1.7", "@octokit/webhooks-methods": "^5.1.1" } }, "sha512-f386XyLTieQbgKPKS6ZMlH4dq8eLsxNddwofiKRZCq0bZ2gikoFwMD99K6l1oAwqe/KZNzrEziGicRgnzplplQ=="],
+
+    "@octokit/webhooks-methods": ["@octokit/webhooks-methods@5.1.1", "", {}, "sha512-NGlEHZDseJTCj8TMMFehzwa9g7On4KJMPVHDSrHxCQumL6uSQR8wIkP/qesv52fXqV1BPf4pTxwtS31ldAt9Xg=="],
+
+    "@types/aws-lambda": ["@types/aws-lambda@8.10.147", "", {}, "sha512-nD0Z9fNIZcxYX5Mai2CTmFD7wX7UldCkW2ezCF8D1T5hdiLsnTWDGRpfRYntU6VjTdLQjOvyszru7I1c1oCQew=="],
+
+    "@types/node": ["@types/node@22.13.5", "", { "dependencies": { "undici-types": "~6.20.0" } }, "sha512-+lTU0PxZXn0Dr1NBtC7Y8cR21AJr87dLLU953CWA6pMxxv/UDc7jYAY90upcrie1nRcD6XNG5HOYEDtgW5TxAg=="],
+
+    "@types/ws": ["@types/ws@8.5.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-bd/YFLW+URhBzMXurx7lWByOu+xzU9+kb3RboOteXYDfW+tr+JZa99OyNmPINEGB/ahzKrEuc8rcv4gnpJmxTw=="],
+
+    "before-after-hook": ["before-after-hook@3.0.2", "", {}, "sha512-Nik3Sc0ncrMK4UUdXQmAnRtzmNQTAAXmXIopizwZ1W1t8QmfJj+zL4OA2I7XPTPW5z5TDqv4hRo/JzouDJnX3A=="],
+
+    "bottleneck": ["bottleneck@2.19.5", "", {}, "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw=="],
+
+    "bun-types": ["bun-types@1.2.4", "", { "dependencies": { "@types/node": "*", "@types/ws": "~8.5.10" } }, "sha512-nDPymR207ZZEoWD4AavvEaa/KZe/qlrbMSchqpQwovPZCKc7pwMoENjEtHgMKaAjJhy+x6vfqSBA1QU3bJgs0Q=="],
+
+    "fast-content-type-parse": ["fast-content-type-parse@2.0.1", "", {}, "sha512-nGqtvLrj5w0naR6tDPfB4cUmYCqouzyQiz6C5y/LtcDllJdrcc6WaWW6iXyIIOErTa/XRybj28aasdn4LkVk6Q=="],
+
+    "octokit": ["octokit@4.1.2", "", { "dependencies": { "@octokit/app": "^15.1.4", "@octokit/core": "^6.1.4", "@octokit/oauth-app": "^7.1.6", "@octokit/plugin-paginate-graphql": "^5.2.4", "@octokit/plugin-paginate-rest": "^11.4.2", "@octokit/plugin-rest-endpoint-methods": "^13.3.1", "@octokit/plugin-retry": "^7.1.4", "@octokit/plugin-throttling": "^9.4.0", "@octokit/request-error": "^6.1.7", "@octokit/types": "^13.7.0" } }, "sha512-0kcTxJOK3yQrJsRb8wKa28hlTze4QOz4sLuUnfXXnhboDhFKgv8LxS86tFwbsafDW9JZ08ByuVAE8kQbYJIZkA=="],
+
+    "toad-cache": ["toad-cache@3.7.0", "", {}, "sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw=="],
+
+    "typescript": ["typescript@5.7.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw=="],
+
+    "undici-types": ["undici-types@6.20.0", "", {}, "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg=="],
+
+    "universal-github-app-jwt": ["universal-github-app-jwt@2.2.0", "", {}, "sha512-G5o6f95b5BggDGuUfKDApKaCgNYy2x7OdHY0zSMF081O0EJobw+1130VONhrA7ezGSV2FNOGyM+KQpQZAr9bIQ=="],
+
+    "universal-user-agent": ["universal-user-agent@7.0.2", "", {}, "sha512-0JCqzSKnStlRRQfCdowvqy3cy0Dvtlb8xecj/H8JFZuCze4rwjPZQOgvFvn0Ws/usCHQFGpyr+pB9adaGwXn4Q=="],
+  }
+}
--- a/simulator-docker-runner/docker-entrypoint.simulator.ts
+++ b/simulator-docker-runner/docker-entrypoint.simulator.ts
@ -0,0 +1,174 @@
+#!/usr/bin/env bun
+
+import { spawn } from "bun";
+import { GithubClient } from "./github";
+import { extractFailureInfo } from "./logParse";
+import { randomSeed } from "./random";
+
+// Configuration from environment variables
+const SLEEP_BETWEEN_RUNS_SECONDS = Number.isInteger(Number(process.env.SLEEP_BETWEEN_RUNS_SECONDS)) ? Number(process.env.SLEEP_BETWEEN_RUNS_SECONDS) : 0;
+const TIME_LIMIT_MINUTES = Number.isInteger(Number(process.env.TIME_LIMIT_MINUTES)) ? Number(process.env.TIME_LIMIT_MINUTES) : 24 * 60;
+const PER_RUN_TIMEOUT_SECONDS = Number.isInteger(Number(process.env.PER_RUN_TIMEOUT_SECONDS)) ? Number(process.env.PER_RUN_TIMEOUT_SECONDS) : 10 * 60;
+const LOG_TO_STDOUT = process.env.LOG_TO_STDOUT === "true";
+
+const github = new GithubClient();
+
+process.env.RUST_BACKTRACE = "1";
+
+console.log("Starting limbo_sim in a loop...");
+console.log(`Git hash: ${github.GIT_HASH}`);
+console.log(`GitHub issues enabled: ${github.mode === 'real'}`);
+console.log(`Time limit: ${TIME_LIMIT_MINUTES} minutes`);
+console.log(`Log simulator output to stdout: ${LOG_TO_STDOUT}`);
+console.log(`Sleep between runs: ${SLEEP_BETWEEN_RUNS_SECONDS} seconds`);
+console.log(`Per run timeout: ${PER_RUN_TIMEOUT_SECONDS} seconds`);
+
+process.on("SIGINT", () => {
+  console.log("Received SIGINT, exiting...");
+  process.exit(0);
+});
+process.on("SIGTERM", () => {
+  console.log("Received SIGTERM, exiting...");
+  process.exit(0);
+});
+
+class TimeoutError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'TimeoutError';
+  }
+}
+
+/**
+ * Returns a promise that rejects when the timeout is reached.
+ * Prints a message to the console every 10 seconds.
+ * @param seconds - The number of seconds to timeout.
+ * @param runNumber - The number of the run.
+ * @returns A promise that rejects when the timeout is reached.
+ */
+const timeouter = (seconds: number, runNumber: number) => {
+  const start = new Date();
+  const stdoutNotifyInterval = setInterval(() => {
+    const elapsedSeconds = Math.round((new Date().getTime() - start.getTime()) / 1000);
+    console.log(`Run ${runNumber} - ${elapsedSeconds}s elapsed (timeout: ${seconds}s)`);
+  }, 10 * 1000);
+  let timeout: Timer;
+  const timeouterPromise = new Promise<never>((_, reject) => {
+    timeout = setTimeout(() => {
+      clearInterval(stdoutNotifyInterval);
+      reject(new TimeoutError("Timeout"));
+    }, seconds * 1000);
+  });
+  // @ts-ignore
+  timeouterPromise.clear = () => {
+    clearInterval(stdoutNotifyInterval);
+    if (timeout) {
+      clearTimeout(timeout);
+    }
+  }
+  return timeouterPromise;
+}
+
+const run = async (seed: string, bin: string, args: string[]) => {
+  const proc = spawn([`/app/${bin}`, ...args], {
+    stdout: LOG_TO_STDOUT ? "inherit" : "pipe",
+    stderr: LOG_TO_STDOUT ? "inherit" : "pipe",
+    env: { ...process.env, SIMULATOR_SEED: seed }
+  });
+
+  const timeout = timeouter(PER_RUN_TIMEOUT_SECONDS, runNumber);
+  timeout.catch(async (err) => {
+    if (err instanceof TimeoutError) {
+      console.log(`Timeout on seed ${seed}, exiting...`);
+      proc.kill();
+      const stdout = await new Response(proc.stdout).text();
+      const stderr = await new Response(proc.stderr).text();
+      const output = stdout + '\n' + stderr;
+      const seedForGithubIssue = seed;
+      const lastLines = output.split('\n').slice(-100).join('\n');
+      console.log(`Simulator seed: ${seedForGithubIssue}`);
+      await github.postGitHubIssue({
+        type: "timeout",
+        seed: seedForGithubIssue,
+        command: args.join(" "),
+        output: lastLines,
+      });
+      process.exit(0);
+    }
+  });
+
+  try {
+    const exitCode = await proc.exited;
+    const stdout = await new Response(proc.stdout).text();
+    const stderr = await new Response(proc.stderr).text();
+
+    if (exitCode !== 0) {
+      console.log(`[${new Date().toISOString()}]: ${bin} ${args.join(" ")} exited with code ${exitCode}`);
+      const output = stdout + stderr;
+
+      // Extract simulator seed and stack trace
+      try {
+        const seedForGithubIssue = seed;
+        const failureInfo = extractFailureInfo(output);
+
+        console.log(`Simulator seed: ${seedForGithubIssue}`);
+
+        // Post the issue to Github and continue
+        if (failureInfo.type === "panic") {
+          await github.postGitHubIssue({
+            type: "panic",
+            seed: seedForGithubIssue,
+            command: args.join(" "),
+            stackTrace: failureInfo,
+          });
+        } else {
+          await github.postGitHubIssue({
+            type: "assertion",
+            seed: seedForGithubIssue,
+            command: args.join(" "),
+            output: output,
+          });
+        }
+      } catch (err2) {
+        console.error(`Error extracting simulator seed and stack trace: ${err2}`);
+        console.log("Last 100 lines of stdout: ", (stdout?.toString() || "").split("\n").slice(-100).join("\n"));
+        console.log("Last 100 lines of stderr: ", (stderr?.toString() || "").split("\n").slice(-100).join("\n"));
+        console.log(`Simulator seed: ${seed}`);
+        process.exit(1);
+      }
+    }
+  } catch (err) {
+    throw err;
+  } finally {
+    // @ts-ignore
+    timeout.clear();
+  }
+}
+
+// Main execution loop
+const startTime = new Date();
+const limboSimArgs = process.argv.slice(2);
+let runNumber = 0;
+while (new Date().getTime() - startTime.getTime() < TIME_LIMIT_MINUTES * 60 * 1000) {
+  const timestamp = new Date().toISOString();
+  const args = [...limboSimArgs];
+  const seed = randomSeed();
+  // Reproducible seed
+  args.push('--seed', seed);
+  // Bugbase wants to have .git available, so we disable it
+  args.push("--disable-bugbase");
+  args.push(...["--minimum-tests", "100", "--maximum-tests", "1000"]);
+  const loop = args.includes("loop") ? [] : ["loop", "-n", "10", "--short-circuit"]
+  args.push(...loop);
+
+  console.log(`[${timestamp}]: Running "limbo_sim ${args.join(" ")}" - (seed ${seed}, run number ${runNumber})`);
+  await run(seed, "limbo_sim", args);
+
+  runNumber++;
+
+  SLEEP_BETWEEN_RUNS_SECONDS > 0 && (await sleep(SLEEP_BETWEEN_RUNS_SECONDS));
+}
+
+async function sleep(sec: number) {
+  return new Promise(resolve => setTimeout(resolve, sec * 1000));
+}
--- a/simulator-docker-runner/github.ts
+++ b/simulator-docker-runner/github.ts
@ -0,0 +1,150 @@
+import { App } from "octokit";
+import { StackTraceInfo } from "./logParse";
+import { levenshtein } from "./levenshtein";
+
+type FaultPanic = {
+  type: "panic"
+  seed: string
+  command: string
+  stackTrace: StackTraceInfo
+}
+
+type FaultAssertion = {
+  type: "assertion"
+  seed: string
+  command: string
+  output: string
+}
+
+type FaultTimeout = {
+  type: "timeout"
+  seed: string
+  command: string
+  output: string
+}
+
+type Fault = FaultPanic | FaultTimeout | FaultAssertion;
+
+export class GithubClient {
+  /* This is the git hash of the commit that the simulator was built from. */
+  GIT_HASH: string;
+  /* Github app ID. */
+  GITHUB_APP_ID: string;
+  /* This is the private key of the "Turso Github Handyman" Github App. It's stored in AWS secrets manager and will be injected into the container at runtime. */
+  GITHUB_APP_PRIVATE_KEY: string;
+  /* This is the unique installation id of the above app into the tursodatabase organization. */
+  GITHUB_APP_INSTALLATION_ID: number;
+  GITHUB_REPO: string;
+  GITHUB_ORG: string = "tursodatabase";
+  GITHUB_REPO_NAME: string = "limbo";
+  mode: 'real' | 'dry-run';
+  app: App | null;
+  initialized: boolean = false;
+  openIssueTitles: string[] = [];
+  constructor() {
+    this.GIT_HASH = process.env.GIT_HASH || "unknown";
+    this.GITHUB_APP_PRIVATE_KEY = process.env.GITHUB_APP_PRIVATE_KEY || "";
+    this.GITHUB_APP_ID = process.env.GITHUB_APP_ID || "";
+    this.GITHUB_APP_INSTALLATION_ID = parseInt(process.env.GITHUB_APP_INSTALLATION_ID || "0", 10);
+    this.mode = this.GITHUB_APP_PRIVATE_KEY ? 'real' : 'dry-run';
+    this.GITHUB_REPO = `${this.GITHUB_ORG}/${this.GITHUB_REPO_NAME}`;
+
+    // Initialize GitHub OAuth App
+    this.app = this.mode === 'real' ? new App({
+      appId: this.GITHUB_APP_ID,
+      privateKey: this.GITHUB_APP_PRIVATE_KEY,
+    }) : null;
+  }
+
+  private async getOpenIssues(): Promise<string[]> {
+    const octokit = await this.app!.getInstallationOctokit(this.GITHUB_APP_INSTALLATION_ID);
+    const issues = await octokit.request('GET /repos/{owner}/{repo}/issues', {
+      owner: this.GITHUB_ORG,
+      repo: this.GITHUB_REPO_NAME,
+      state: 'open',
+      creator: 'app/turso-github-handyman',
+    });
+    return issues.data.map((issue) => issue.title);
+  }
+
+  async initialize(): Promise<void> {
+    if (this.mode === 'dry-run') {
+      console.log("Dry-run mode: Skipping initialization");
+      this.initialized = true;
+      return;
+    }
+    this.openIssueTitles = await this.getOpenIssues();
+    this.initialized = true;
+  }
+
+  async postGitHubIssue(fault: Fault): Promise<void> {
+    if (!this.initialized) {
+      await this.initialize();
+    }
+
+    const title = ((f: Fault) => {
+      if (f.type === "panic") {
+        return `Simulator panic: "${f.stackTrace.mainError}"`;
+      } else 
+      if (f.type === "assertion") {
+        return `Simulator assertion failure: "${f.output}"`;
+      }
+      return `Simulator timeout using git hash ${this.GIT_HASH}`;
+    })(fault);
+    for (const existingIssueTitle of this.openIssueTitles) {
+      const MAGIC_NUMBER = 6;
+      if (levenshtein(existingIssueTitle, title) < MAGIC_NUMBER) {
+        console.log(`Not creating issue ${title} because it is too similar to ${existingIssueTitle}`);
+        return;
+      }
+    }
+
+    const body = this.createIssueBody(fault);
+
+    if (this.mode === 'dry-run') {
+      console.log(`Dry-run mode: Would create issue in ${this.GITHUB_REPO} with title: ${title} and body: ${body}`);
+      return;
+    }
+
+    const [owner, repo] = this.GITHUB_REPO.split('/');
+
+    const octokit = await this.app!.getInstallationOctokit(this.GITHUB_APP_INSTALLATION_ID);
+
+    const response = await octokit.request('POST /repos/{owner}/{repo}/issues', {
+      owner,
+      repo,
+      title,
+      body,
+      labels: ['bug', 'simulator', 'automated']
+    });
+
+    console.log(`Successfully created GitHub issue: ${response.data.html_url}`);
+    this.openIssueTitles.push(title);
+  }
+
+  private createIssueBody(fault: Fault): string {
+    const gitShortHash = this.GIT_HASH.substring(0, 7);
+    return `
+ ## Simulator ${fault.type}
+ 
+ - **Seed**: ${fault.seed}
+ - **Git Hash**: ${this.GIT_HASH}
+ - **Command**: \`limbo-sim ${fault.command}\`
+ - **Timestamp**: ${new Date().toISOString()}
+
+ ### Run locally with Docker
+
+ \`\`\`
+ git checkout ${this.GIT_HASH}
+ docker buildx build -t limbo-sim:${gitShortHash} -f simulator-docker-runner/Dockerfile.simulator . --build-arg GIT_HASH=$(git rev-parse HEAD)
+ docker run --network host limbo-sim:${gitShortHash} ${fault.command}
+ \`\`\`
+
+ ### ${fault.type === "panic" ? "Stack Trace" : "Output"}
+ 
+ \`\`\`
+ ${fault.type === "panic" ? fault.stackTrace.trace : fault.output}
+ \`\`\`
+ `;
+  }
+}
--- a/simulator-docker-runner/levenshtein.test.ts
+++ b/simulator-docker-runner/levenshtein.test.ts
@ -0,0 +1,26 @@
+import { test, expect } from "bun:test";
+import { levenshtein } from "./levenshtein";
+
+test("levenshtein distance between empty strings is 0", () => {
+  expect(levenshtein("", "")).toBe(0);
+});
+
+test("levenshtein distance between completely different same length strings is their length", () => {
+  expect(levenshtein("abcde", "fghij")).toBe(5);
+  expect(levenshtein("fghij", "abcde")).toBe(5);
+});
+
+test("levenshtein distance between strings with one edit is 1", () => {
+  expect(levenshtein("hello", "hallo")).toBe(1);
+  expect(levenshtein("hallo", "hello")).toBe(1);
+});
+
+test("levenshtein distance between otherwise identical strings with length difference is their length difference", () => {
+  expect(levenshtein("hello", "hello world")).toBe(6);
+  expect(levenshtein("hello world", "hello")).toBe(6);
+});
+
+test("levenshtein distance between strings with multiple edits is the sum of the edits", () => {
+  expect(levenshtein("hello", "hallu")).toBe(2);
+  expect(levenshtein("hallu", "hello")).toBe(2);
+});
--- a/simulator-docker-runner/levenshtein.ts
+++ b/simulator-docker-runner/levenshtein.ts
@ -0,0 +1,40 @@
+/**
+ * Calculate the Levenshtein distance between two strings.
+ * @param a - The first string.
+ * @param b - The second string.
+ * @returns The Levenshtein distance between the two strings.
+ */
+export function levenshtein(a: string, b: string): number {
+  const an = a ? a.length : 0;
+  const bn = b ? b.length : 0;
+  if (an === 0) {
+    return bn;
+  }
+  if (bn === 0) {
+    return an;
+  }
+  const matrix = new Array<number[]>(bn + 1);
+  for (let i = 0; i <= bn; ++i) {
+    let row = matrix[i] = new Array<number>(an + 1);
+    row[0] = i;
+  }
+  const firstRow = matrix[0];
+  for (let j = 1; j <= an; ++j) {
+    firstRow[j] = j;
+  }
+  for (let i = 1; i <= bn; ++i) {
+    for (let j = 1; j <= an; ++j) {
+      if (b.charAt(i - 1) === a.charAt(j - 1)) {
+        matrix[i][j] = matrix[i - 1][j - 1];
+      }
+      else {
+        matrix[i][j] = Math.min(
+          matrix[i - 1][j - 1], // substitution
+          matrix[i][j - 1], // insertion
+          matrix[i - 1][j] // deletion
+        ) + 1;
+      }
+    }
+  }
+  return matrix[bn][an];
+};
--- a/simulator-docker-runner/logParse.ts
+++ b/simulator-docker-runner/logParse.ts
@ -0,0 +1,58 @@
+// Helper functions
+
+export type StackTraceInfo = {
+  type: "panic";
+  trace: string;
+  mainError: string;
+}
+
+export type AssertionFailureInfo = {
+  type: "assertion";
+  output: string;
+}
+
+/**
+ * Extract failure information from log output
+ */
+export function extractFailureInfo(output: string): StackTraceInfo | AssertionFailureInfo {
+  const lines = output.split('\n');
+
+  const panicLineIndex = lines.findIndex(line => line.includes("panic occurred"));
+
+  const info = getTraceFromOutput(lines) ?? getAssertionFailureInfo(lines);
+
+  if (!info) {
+    throw new Error("No failure information found");
+  }
+
+  return info;
+}
+
+function getTraceFromOutput(lines: string[]): StackTraceInfo | null {
+  const panicLineIndex = lines.findIndex(line => line.includes("panic occurred"));
+  if (panicLineIndex === -1) {
+    return null;
+  }
+
+  const startIndex = panicLineIndex + 1;
+  const endIndex = Math.min(lines.length, startIndex + 50);
+
+  const trace = lines.slice(startIndex, endIndex).join('\n');
+  const mainError = lines[startIndex] ?? "???";
+
+  return { type: "panic", trace, mainError };
+}
+
+function getAssertionFailureInfo(lines: string[]): AssertionFailureInfo | null {
+  const simulationFailedLineIndex = lines.findIndex(line => line.includes("simulation failed:"));
+  if (simulationFailedLineIndex === -1) {
+    return null;
+  }
+
+  const startIndex = simulationFailedLineIndex;
+  const endIndex = Math.min(lines.length, startIndex + 1);
+
+  const output = lines.slice(startIndex, endIndex).join('\n');
+
+  return { type: "assertion", output };
+}
--- a/simulator-docker-runner/package.json
+++ b/simulator-docker-runner/package.json
@ -0,0 +1,14 @@
+{
+  "name": "simulator-docker-runner",
+  "description": "A script to run the simulator in a docker container in a loop and post issues to github when it panics",
+  "module": "docker-entrypoint.simulator.ts",
+  "type": "module",
+  "dependencies": {
+    "octokit": "4.1.2"
+  },
+  "devDependencies": {
+    "@types/node": "22.13.5",
+    "bun-types": "1.2.4",
+    "typescript": "5.7.3"
+  }
+} 
--- a/simulator-docker-runner/random.ts
+++ b/simulator-docker-runner/random.ts
@ -0,0 +1,5 @@
+export function randomSeed() {
+  const high32 = Math.floor(Math.random() * Math.pow(2, 32));
+  const low32 = Math.floor(Math.random() * Math.pow(2, 32));
+  return ((BigInt(high32) << 32n) | BigInt(low32)).toString();
+}
--- a/simulator-docker-runner/tsconfig.json
+++ b/simulator-docker-runner/tsconfig.json
@ -0,0 +1,23 @@
+{
+  "compilerOptions": {
+    "lib": ["ESNext", "DOM"],
+    "module": "esnext",
+    "target": "esnext",
+    "moduleResolution": "bundler",
+    "moduleDetection": "force",
+    "allowImportingTsExtensions": true,
+    "noEmit": true,
+    "composite": true,
+    "strict": true,
+    "downlevelIteration": true,
+    "skipLibCheck": true,
+    "jsx": "preserve",
+    "allowSyntheticDefaultImports": true,
+    "forceConsistentCasingInFileNames": true,
+    "allowJs": true,
+    "types": [
+      "bun-types",
+      "node"
+    ]
+  }
+}