mirror of
https://github.com/GraphiteEditor/Graphite.git
synced 2025-12-23 10:11:54 +00:00
557df6917d
* Only consider changes significant if they exceed an absolute ir diff * Node graph folder change to trigger ci * Use extra step for detecting significant perf degradations * Reenable expanding of sections with significant changes * Remove temp file
344 lines
15 KiB
YAML
344 lines
15 KiB
YAML
name: Profiling Changes
|
|
|
|
on:
|
|
pull_request:
|
|
paths:
|
|
- 'node-graph/**'
|
|
- 'Cargo.toml'
|
|
- 'Cargo.lock'
|
|
|
|
env:
|
|
CARGO_TERM_COLOR: always
|
|
|
|
jobs:
|
|
profile:
|
|
runs-on: ubuntu-latest
|
|
continue-on-error: true
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Install Rust
|
|
uses: dtolnay/rust-toolchain@stable
|
|
|
|
- name: Install Valgrind
|
|
run: |
|
|
sudo apt update
|
|
sudo apt install -y valgrind
|
|
|
|
- name: Cache Rust dependencies
|
|
uses: Swatinem/rust-cache@v2
|
|
with:
|
|
# Cache on Cargo.lock file
|
|
cache-on-failure: true
|
|
|
|
- name: Cache iai-callgrind binary
|
|
id: cache-iai
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: ~/.cargo/bin/iai-callgrind-runner
|
|
key: ${{ runner.os }}-iai-callgrind-runner-0.16.1
|
|
|
|
- name: Install iai-callgrind
|
|
if: steps.cache-iai.outputs.cache-hit != 'true'
|
|
run: |
|
|
cargo install iai-callgrind-runner@0.16.1
|
|
|
|
- name: Checkout master branch
|
|
run: |
|
|
git fetch origin master:master
|
|
git checkout master
|
|
|
|
- name: Get master commit SHA
|
|
id: master-sha
|
|
run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
|
|
|
|
- name: Get CPU info
|
|
id: cpu-info
|
|
run: |
|
|
# Get CPU model and create a short hash for cache key
|
|
CPU_MODEL=$(cat /proc/cpuinfo | grep "model name" | head -1 | cut -d: -f2 | xargs)
|
|
CPU_HASH=$(echo "$CPU_MODEL" | sha256sum | cut -c1-8)
|
|
echo "cpu-hash=$CPU_HASH" >> $GITHUB_OUTPUT
|
|
echo "CPU: $CPU_MODEL (hash: $CPU_HASH)"
|
|
|
|
- name: Cache benchmark baselines
|
|
id: cache-benchmark-baselines
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: target/iai
|
|
key: ${{ runner.os }}-${{ runner.arch }}-${{ steps.cpu-info.outputs.cpu-hash }}-benchmark-baselines-master-${{ steps.master-sha.outputs.sha }}
|
|
restore-keys: |
|
|
${{ runner.os }}-${{ runner.arch }}-${{ steps.cpu-info.outputs.cpu-hash }}-benchmark-baselines-master-
|
|
|
|
- name: Run baseline benchmarks
|
|
if: steps.cache-benchmark-baselines.outputs.cache-hit != 'true'
|
|
run: |
|
|
# Compile benchmarks
|
|
cargo bench --bench compile_demo_art_iai -- --save-baseline=master
|
|
|
|
# Runtime benchmarks
|
|
cargo bench --bench update_executor_iai -- --save-baseline=master
|
|
cargo bench --bench run_once_iai -- --save-baseline=master
|
|
cargo bench --bench run_cached_iai -- --save-baseline=master
|
|
|
|
- name: Checkout PR branch
|
|
run: |
|
|
git checkout ${{ github.event.pull_request.head.sha }}
|
|
|
|
- name: Run PR benchmarks
|
|
run: |
|
|
# Compile benchmarks
|
|
cargo bench --bench compile_demo_art_iai -- --baseline=master --output-format=json | jq -sc | sed 's/\\"//g' > /tmp/compile_output.json
|
|
|
|
# Runtime benchmarks
|
|
cargo bench --bench update_executor_iai -- --baseline=master --output-format=json | jq -sc | sed 's/\\"//g' > /tmp/update_output.json
|
|
cargo bench --bench run_once_iai -- --baseline=master --output-format=json | jq -sc | sed 's/\\"//g' > /tmp/run_once_output.json
|
|
cargo bench --bench run_cached_iai -- --baseline=master --output-format=json | jq -sc | sed 's/\\"//g' > /tmp/run_cached_output.json
|
|
|
|
- name: Make old comments collapsed by default
|
|
# Only run if we have write permissions (not a fork)
|
|
if: github.event.pull_request.head.repo.full_name == github.repository
|
|
uses: actions/github-script@v7
|
|
with:
|
|
github-token: ${{secrets.GITHUB_TOKEN}}
|
|
script: |
|
|
const { data: comments } = await github.rest.issues.listComments({
|
|
issue_number: context.issue.number,
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
});
|
|
|
|
const botComments = comments.filter((comment) =>
|
|
comment.user.type === 'Bot' && comment.body.includes('Performance Benchmark Results') && comment.body.includes('<details open>')
|
|
);
|
|
|
|
for (const comment of botComments) {
|
|
// Edit the comment to remove the "open" attribute from the <details> tag
|
|
await github.rest.issues.updateComment({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
comment_id: comment.id,
|
|
body: comment.body.replace('<details open>', '<details>')
|
|
});
|
|
}
|
|
|
|
- name: Analyze profiling changes
|
|
id: analyze
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
const fs = require('fs');
|
|
|
|
function isSignificantChange(diffPct, absoluteChange, benchmarkType) {
|
|
const meetsPercentageThreshold = Math.abs(diffPct) > 5;
|
|
const meetsAbsoluteThreshold = absoluteChange > 200000;
|
|
const isCachedExecution = benchmarkType === 'run_cached' ||
|
|
benchmarkType.includes('Cached Execution');
|
|
|
|
return isCachedExecution
|
|
? (meetsPercentageThreshold && meetsAbsoluteThreshold)
|
|
: meetsPercentageThreshold;
|
|
}
|
|
|
|
const allOutputs = [
|
|
JSON.parse(fs.readFileSync('/tmp/compile_output.json', 'utf8')),
|
|
JSON.parse(fs.readFileSync('/tmp/update_output.json', 'utf8')),
|
|
JSON.parse(fs.readFileSync('/tmp/run_once_output.json', 'utf8')),
|
|
JSON.parse(fs.readFileSync('/tmp/run_cached_output.json', 'utf8'))
|
|
];
|
|
const outputNames = ['compile', 'update', 'run_once', 'run_cached'];
|
|
const sectionTitles = ['Compilation', 'Update', 'Run Once', 'Cached Execution'];
|
|
|
|
let hasSignificantChanges = false;
|
|
let regressionDetails = [];
|
|
|
|
for (let i = 0; i < allOutputs.length; i++) {
|
|
const benchmarkOutput = allOutputs[i];
|
|
const outputName = outputNames[i];
|
|
const sectionTitle = sectionTitles[i];
|
|
|
|
for (const benchmark of benchmarkOutput) {
|
|
if (benchmark.profiles?.[0]?.summaries?.parts?.[0]?.metrics_summary?.Callgrind?.Ir?.diffs?.diff_pct) {
|
|
const diffPct = parseFloat(benchmark.profiles[0].summaries.parts[0].metrics_summary.Callgrind.Ir.diffs.diff_pct);
|
|
const oldValue = benchmark.profiles[0].summaries.parts[0].metrics_summary.Callgrind.Ir.metrics.Both[1].Int;
|
|
const newValue = benchmark.profiles[0].summaries.parts[0].metrics_summary.Callgrind.Ir.metrics.Both[0].Int;
|
|
const absoluteChange = Math.abs(newValue - oldValue);
|
|
|
|
if (isSignificantChange(diffPct, absoluteChange, outputName)) {
|
|
hasSignificantChanges = true;
|
|
regressionDetails.push({
|
|
module_path: benchmark.module_path,
|
|
id: benchmark.id,
|
|
diffPct,
|
|
absoluteChange,
|
|
sectionTitle
|
|
});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
core.setOutput('has-significant-changes', hasSignificantChanges);
|
|
core.setOutput('regression-details', JSON.stringify(regressionDetails));
|
|
|
|
- name: Comment PR
|
|
if: github.event.pull_request.head.repo.full_name == github.repository
|
|
uses: actions/github-script@v7
|
|
with:
|
|
github-token: ${{secrets.GITHUB_TOKEN}}
|
|
script: |
|
|
const fs = require('fs');
|
|
|
|
const compileOutput = JSON.parse(fs.readFileSync('/tmp/compile_output.json', 'utf8'));
|
|
const updateOutput = JSON.parse(fs.readFileSync('/tmp/update_output.json', 'utf8'));
|
|
const runOnceOutput = JSON.parse(fs.readFileSync('/tmp/run_once_output.json', 'utf8'));
|
|
const runCachedOutput = JSON.parse(fs.readFileSync('/tmp/run_cached_output.json', 'utf8'));
|
|
|
|
const hasSignificantChanges = '${{ steps.analyze.outputs.has-significant-changes }}' === 'true';
|
|
let commentBody = "";
|
|
|
|
function formatNumber(num) {
|
|
return num.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
|
|
}
|
|
|
|
function formatPercentage(pct) {
|
|
const sign = pct >= 0 ? '+' : '';
|
|
return `${sign}${pct.toFixed(2)}%`;
|
|
}
|
|
|
|
function padRight(str, len) {
|
|
return str.padEnd(len);
|
|
}
|
|
|
|
function padLeft(str, len) {
|
|
return str.padStart(len);
|
|
}
|
|
|
|
function processBenchmarkOutput(benchmarkOutput, sectionTitle, isLast = false) {
|
|
let sectionBody = "";
|
|
let hasResults = false;
|
|
let hasSignificantChanges = false;
|
|
|
|
function isSignificantChange(diffPct, absoluteChange, benchmarkType) {
|
|
const meetsPercentageThreshold = Math.abs(diffPct) > 5;
|
|
const meetsAbsoluteThreshold = absoluteChange > 200000;
|
|
const isCachedExecution = benchmarkType === 'run_cached' ||
|
|
benchmarkType.includes('Cached Execution');
|
|
|
|
return isCachedExecution
|
|
? (meetsPercentageThreshold && meetsAbsoluteThreshold)
|
|
: meetsPercentageThreshold;
|
|
}
|
|
|
|
for (const benchmark of benchmarkOutput) {
|
|
if (benchmark.profiles && benchmark.profiles.length > 0) {
|
|
const profile = benchmark.profiles[0];
|
|
if (profile.summaries && profile.summaries.parts && profile.summaries.parts.length > 0) {
|
|
const part = profile.summaries.parts[0];
|
|
if (part.metrics_summary && part.metrics_summary.Callgrind && part.metrics_summary.Callgrind.Ir) {
|
|
const irData = part.metrics_summary.Callgrind.Ir;
|
|
if (irData.diffs && irData.diffs.diff_pct !== null) {
|
|
const irDiff = {
|
|
diff_pct: parseFloat(irData.diffs.diff_pct),
|
|
old: irData.metrics.Both[1].Int,
|
|
new: irData.metrics.Both[0].Int
|
|
};
|
|
hasResults = true;
|
|
const changePercentage = formatPercentage(irDiff.diff_pct);
|
|
const color = irDiff.diff_pct > 0 ? "red" : "lime";
|
|
|
|
sectionBody += `**${benchmark.module_path} ${benchmark.id}:${benchmark.details}**\n`;
|
|
sectionBody += `Instructions: \`${formatNumber(irDiff.old)}\` (master) → \`${formatNumber(irDiff.new)}\` (HEAD) : `;
|
|
sectionBody += `$$\\color{${color}}${changePercentage.replace("%", "\\\\%")}$$\n\n`;
|
|
|
|
sectionBody += "<details>\n<summary>Detailed metrics</summary>\n\n```\n";
|
|
sectionBody += `Baselines: master| HEAD\n`;
|
|
|
|
for (const [metricName, metricData] of Object.entries(part.metrics_summary.Callgrind)) {
|
|
if (metricData.diffs && metricData.diffs.diff_pct !== null) {
|
|
const changePercentage = formatPercentage(parseFloat(metricData.diffs.diff_pct));
|
|
const oldValue = metricData.metrics.Both[1].Int || metricData.metrics.Both[1].Float;
|
|
const newValue = metricData.metrics.Both[0].Int || metricData.metrics.Both[0].Float;
|
|
const line = `${padRight(metricName, 20)} ${padLeft(formatNumber(Math.round(oldValue)), 11)}|${padLeft(formatNumber(Math.round(newValue)), 11)} ${padLeft(changePercentage, 15)}`;
|
|
sectionBody += `${line}\n`;
|
|
}
|
|
}
|
|
|
|
sectionBody += "```\n</details>\n\n";
|
|
|
|
if (isSignificantChange(irDiff.diff_pct, Math.abs(irDiff.new - irDiff.old), sectionTitle)) {
|
|
significantChanges = true;
|
|
hasSignificantChanges = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (hasResults) {
|
|
// Wrap section in collapsible details, open only if there are significant changes
|
|
const openAttribute = hasSignificantChanges ? " open" : "";
|
|
const ruler = isLast ? "" : "\n\n---";
|
|
return `<details${openAttribute}>\n<summary><h2>${sectionTitle}</h2></summary>\n\n${sectionBody}${ruler}\n</details>`;
|
|
}
|
|
return "";
|
|
}
|
|
|
|
// Process each benchmark category
|
|
const sections = [
|
|
{ output: compileOutput, title: "🔧 Graph Compilation" },
|
|
{ output: updateOutput, title: "🔄 Executor Update" },
|
|
{ output: runOnceOutput, title: "🚀 Render: Cold Execution" },
|
|
{ output: runCachedOutput, title: "⚡ Render: Cached Execution" }
|
|
];
|
|
|
|
// Generate sections and determine which ones have results
|
|
const generatedSections = sections.map(({ output, title }) =>
|
|
processBenchmarkOutput(output, title, true) // temporarily mark all as last
|
|
).filter(section => section.length > 0);
|
|
|
|
// Re-generate with correct isLast flags
|
|
let sectionIndex = 0;
|
|
const finalSections = sections.map(({ output, title }) => {
|
|
const section = processBenchmarkOutput(output, title, true); // check if it has results
|
|
if (section.length > 0) {
|
|
const isLast = sectionIndex === generatedSections.length - 1;
|
|
sectionIndex++;
|
|
return processBenchmarkOutput(output, title, isLast);
|
|
}
|
|
return "";
|
|
}).filter(section => section.length > 0);
|
|
|
|
// Combine all sections
|
|
commentBody = finalSections.join("\n\n");
|
|
|
|
if (commentBody.length > 0) {
|
|
const output = `<details open>\n<summary>Performance Benchmark Results</summary>\n\n${commentBody}\n</details>`;
|
|
|
|
if (hasSignificantChanges) {
|
|
github.rest.issues.createComment({
|
|
issue_number: context.issue.number,
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
body: output
|
|
});
|
|
} else {
|
|
console.log("No significant performance changes detected. Skipping comment.");
|
|
console.log(output);
|
|
}
|
|
} else {
|
|
console.log("No benchmark results to display.");
|
|
}
|
|
|
|
- name: Fail on significant regressions
|
|
if: steps.analyze.outputs.has-significant-changes == 'true'
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
const regressionDetails = JSON.parse('${{ steps.analyze.outputs.regression-details }}');
|
|
const firstRegression = regressionDetails[0];
|
|
|
|
core.setFailed(`Significant performance regression detected: ${firstRegression.module_path} ${firstRegression.id} increased by ${firstRegression.absoluteChange.toLocaleString()} instructions (${firstRegression.diffPct.toFixed(2)}%)`);
|