Graphite/.github/workflows/comment-profiling-changes.yaml

name: Profiling Changes

on:
  pull_request:

env:
  CARGO_TERM_COLOR: always

jobs:
  profile:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Install Rust
        uses: actions-rs/toolchain@v1
        with:
          profile: minimal
          toolchain: stable

      - name: Install Valgrind
        run: |
          sudo apt update
          sudo apt install -y valgrind

      - name: Cache dependencies
        uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            target
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}

      - name: Install iai-callgrind
        run: |
          cargo install iai-callgrind-runner@0.12.3

      - name: Checkout master branch
        run: |
          git fetch origin master:master
          git checkout master

      - name: Run baseline benchmarks
        run: |
          cargo bench --bench compile_demo_art_iai -- --save-baseline=master

      - name: Checkout PR branch
        run: |
          git checkout ${{ github.event.pull_request.head.sha }}

      - name: Run PR benchmarks
        id: benchmark
        run: |
          BENCH_OUTPUT=$(cargo bench --bench compile_demo_art_iai -- --baseline=master --output-format=json  | jq -sc | sed 's/\\"//g')
          echo "BENCHMARK_OUTPUT<<EOF" >> $GITHUB_OUTPUT
          echo "$BENCH_OUTPUT" >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT

      - name: Make old comments collapsed by default
        uses: actions/github-script@v6
        with:
          github-token: ${{secrets.GITHUB_TOKEN}}
          script: |
            const { data: comments } = await github.rest.issues.listComments({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
            });

            const botComments = comments.filter((comment) =>
              comment.user.type === 'Bot' && comment.body.includes('Performance Benchmark Results') && comment.body.includes('<details open>')
            );

            for (const comment of botComments) {
              // Edit the comment to remove the "open" attribute from the <details> tag
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: comment.id,
                body: comment.body.replace('<details open>', '<details>')
              });
            }

      - name: Comment PR
        uses: actions/github-script@v6
        with:
          github-token: ${{secrets.GITHUB_TOKEN}}
          script: |
            const benchmarkOutput = JSON.parse(`${{ steps.benchmark.outputs.BENCHMARK_OUTPUT }}`);
            let significantChanges = false;
            let commentBody = "";

            function formatNumber(num) {
              return num.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
            }

            function formatPercentage(pct) {
              const sign = pct >= 0 ? '+' : '';
              return `${sign}${pct.toFixed(2)}%`;
            }

            function padRight(str, len) {
              return str.padEnd(len);
            }

            function padLeft(str, len) {
              return str.padStart(len);
            }

            for (const benchmark of benchmarkOutput) {
              if (benchmark.callgrind_summary && benchmark.callgrind_summary.summaries) {
                const summary = benchmark.callgrind_summary.summaries[0];
                const irDiff = summary.events.Ir;
                
                if (irDiff.diff_pct !== null) {
                  const changePercentage = formatPercentage(irDiff.diff_pct);
                  const color = irDiff.diff_pct > 0 ? "red" : "lime";
                  
                  commentBody += "---\n\n";
                  commentBody += `${benchmark.module_path} ${benchmark.id}:${benchmark.details}\n`;
                  commentBody += `Instructions: \`${formatNumber(irDiff.old)}\` (master) -> \`${formatNumber(irDiff.new)}\` (HEAD) : `;
                  commentBody += `$$\\color{${color}}${changePercentage.replace("%", "\\\\%")}$$\n\n`;
                  
                  commentBody += "<details>\n<summary>Detailed metrics</summary>\n\n```\n";
                  commentBody += `Baselines:                master|       HEAD\n`;
                  
                  for (const [eventKind, costsDiff] of Object.entries(summary.events)) {
                    if (costsDiff.diff_pct !== null) {
                      const changePercentage = formatPercentage(costsDiff.diff_pct);
                      const line = `${padRight(eventKind, 20)} ${padLeft(formatNumber(costsDiff.old), 11)}|${padLeft(formatNumber(costsDiff.new), 11)} ${padLeft(changePercentage, 15)}`;
                      commentBody += `${line}\n`;
                    }
                  }
                  
                  commentBody += "```\n</details>\n\n";
                  
                  if (Math.abs(irDiff.diff_pct) > 5) {
                    significantChanges = true;
                  }
                }
              }
            }

            const output = `
            <details open>

            <summary>Performance Benchmark Results</summary>

            ${commentBody}

            </details>
            `;

            if (significantChanges) {
              github.rest.issues.createComment({
                issue_number: context.issue.number,
                owner: context.repo.owner,
                repo: context.repo.repo,
                body: output
              });
            } else {
              console.log("No significant performance changes detected. Skipping comment.");
              console.log(output);
            }