mirrors/granian
1
0
Fork 0
mirror of https://github.com/emmett-framework/granian.git synced 2025-12-23 05:36:49 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Update benchmarks suite (#266)

Browse source
This commit is contained in:
Giovanni Barillari 2024-04-12 20:27:21 +02:00 • committed by GitHub
parent 8dbf78456d
commit b7156ac8bf
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
23 changed files with 667 additions and 418 deletions
Download patch file Download diff file Expand all files Collapse all files

157
.github/workflows/bechmarks.yml vendored
View file

@ -1,157 +0,0 @@
name: benchmarks
on: workflow_dispatch
env:
MATURIN_VERSION: 1.3.2
jobs:
benchmark-base:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.10'
- uses: messense/maturin-action@v1
with:
maturin-version: v${{ env.MATURIN_VERSION }}
command: build
args: --release --strip --interpreter python3.10
target: x64
manylinux: auto
container: off
- name: Deps
run: |
export _whl=$(ls target/wheels/granian-*.whl)
pip install $_whl
- name: wrk
run: |
git clone https://github.com/wg/wrk.git .wrk
cd .wrk && make && sudo mv wrk /usr/local/bin
- name: Benchmarks
run: |
cd benchmarks && python benchmarks.py
- name: Upload results
uses: actions/upload-artifact@v3
with:
name: results-base
path: benchmarks/results/*
benchmark-concurrency:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.10'
- uses: messense/maturin-action@v1
with:
maturin-version: v${{ env.MATURIN_VERSION }}
command: build
args: --release --strip --interpreter python3.10
target: x64
manylinux: auto
container: off
- name: Deps
run: |
export _whl=$(ls target/wheels/granian-*.whl)
pip install $_whl
- name: wrk
run: |
git clone https://github.com/wg/wrk.git .wrk
cd .wrk && make && sudo mv wrk /usr/local/bin
- name: Benchmarks
env:
BENCHMARK_BASE: false
BENCHMARK_CONCURRENCIES: true
run: |
cd benchmarks && python benchmarks.py
- name: Upload results
uses: actions/upload-artifact@v3
with:
name: results-concurrency
path: benchmarks/results/*
benchmark-vsa:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.10'
- uses: messense/maturin-action@v1
with:
maturin-version: v${{ env.MATURIN_VERSION }}
command: build
args: --release --strip --interpreter python3.10
target: x64
manylinux: auto
container: off
- name: Deps
run: |
export _whl=$(ls target/wheels/granian-*.whl)
pip install $_whl
pip install uvicorn
pip install httptools
pip install hypercorn
- name: wrk
run: |
git clone https://github.com/wg/wrk.git .wrk
cd .wrk && make && sudo mv wrk /usr/local/bin
- name: Benchmarks
env:
BENCHMARK_BASE: false
BENCHMARK_VSA: true
run: |
cd benchmarks && python benchmarks.py
- name: Upload results
uses: actions/upload-artifact@v3
with:
name: results-vsa
path: benchmarks/results/*
benchmark-vss:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.9'
- uses: messense/maturin-action@v1
with:
maturin-version: v${{ env.MATURIN_VERSION }}
command: build
args: --release --strip --interpreter python3.9
target: x64
manylinux: auto
container: off
- name: Deps
run: |
export _whl=$(ls target/wheels/granian-*.whl)
pip install $_whl
pip install gunicorn
pip install meinheld
- name: wrk
run: |
git clone https://github.com/wg/wrk.git .wrk
cd .wrk && make && sudo mv wrk /usr/local/bin
- name: Benchmarks
env:
BENCHMARK_BASE: false
BENCHMARK_VSS: true
run: |
cd benchmarks && python benchmarks.py
- name: Upload results
uses: actions/upload-artifact@v3
with:
name: results-vss
path: benchmarks/results/*

133
.github/workflows/benchmarks.yml vendored Normal file
View file

@ -0,0 +1,133 @@
name: benchmarks
on: workflow_dispatch
jobs:
toolchain:
runs-on: ubuntu-latest
steps:
- uses: actions-rs/toolchain@v1
with:
toolchain: stable
default: true
profile: minimal
- run: |
git clone https://github.com/gi0baro/rewrk.git
cd rewrk && cargo build --release
- uses: actions/upload-artifact@v4
with:
name: rewrk
path: rewrk/target/release/rewrk
benchmark-base:
runs-on: ubuntu-latest
needs: [toolchain]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.11'
- uses: actions/download-artifact@v4
with:
name: rewrk
- run: |
sudo mv rewrk /usr/local/bin && chmod +x /usr/local/bin/rewrk
- uses: pyo3/maturin-action@v1
with:
command: build
args: --release --interpreter python3.11
target: x64
manylinux: auto
container: off
- run: |
export _whl=$(ls target/wheels/granian-*.whl)
pip install $_whl
- name: benchmark
working-directory: ./benchmarks
run: |
python benchmarks.py
- name: upload results
uses: actions/upload-artifact@v4
with:
name: results-base
path: benchmarks/results/*
benchmark-vs:
runs-on: ubuntu-latest
needs: [toolchain]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.11'
- uses: actions/download-artifact@v4
with:
name: rewrk
- run: |
sudo mv rewrk /usr/local/bin && chmod +x /usr/local/bin/rewrk
- uses: pyo3/maturin-action@v1
with:
command: build
args: --release --interpreter python3.11
target: x64
manylinux: auto
container: off
- run: |
export _whl=$(ls target/wheels/granian-*.whl)
pip install $_whl
- name: deps
run: |
pip install -r benchmarks/envs/asgi.txt
pip install -r benchmarks/envs/wsgi.txt
- name: benchmark
env:
BENCHMARK_BASE: false
BENCHMARK_VS: true
working-directory: ./benchmarks
run: |
python benchmarks.py
- name: upload results
uses: actions/upload-artifact@v4
with:
name: results-vs
path: benchmarks/results/*
results:
runs-on: ubuntu-latest
needs: [benchmark-base, benchmark-vs]
steps:
- uses: actions/checkout@v4
- uses: gi0baro/setup-noir@v1
- uses: actions/download-artifact@v4
with:
name: results-base
path: benchmarks/results
- run: |
mv benchmarks/results/data.json benchmarks/results/base.json
- uses: actions/download-artifact@v4
with:
name: results-vs
path: benchmarks/results
- run: |
mv benchmarks/results/data.json benchmarks/results/vs.json
- name: render
working-directory: ./benchmarks
run: |
noir -c data:results/base.json -v 'benv=GHA Linux x86_64' templates/main.md > README.md
noir -c data:results/vs.json -v 'benv=GHA Linux x86_64' templates/vs.md > vs.md
- name: open PR
uses: peter-evans/create-pull-request@v6
with:
branch: benchmarks-update
branch-suffix: timestamp
title: Update benchmark results
body: SSIA
commit-message: |
Update benchmark results
add-paths: |
benchmarks/README.md
benchmarks/vs.md

103
benchmarks/_template.md
View file

@ -1,103 +0,0 @@
# Granian benchmarks
Run at: {{ =data.run_at }}
CPUs: {{ =data.cpu }}
Python version: {{ =data.pyver }}
{{ if "rsgi_body" in data.results: }}
## RSGI response types
| Type | Total requests | RPS | avg latency | max latency |
| --- | --- | --- | --- | --- |
{{ for key, runs in data.results["rsgi_body"].items(): }}
{{ concurrency_values = {runs[ckey]["requests"]["rps"]: ckey for ckey in runs.keys()} }}
{{ max_res = concurrency_values[max(concurrency_values.keys())] }}
{{ run = runs[max_res] }}
| {{ =key }} (c{{ =max_res }}) | {{ =run["requests"]["total"] }} | {{ =run["requests"]["rps"] }} | {{ =int(run["latency"]["avg"]) / 1000 }}ms | {{ =int(run["latency"]["max"]) / 1000 }}ms |
{{ pass }}
{{ pass }}
{{ if "interfaces" in data.results: }}
## Interfaces
| Request | Total requests | RPS | avg latency | max latency |
| --- | --- | --- | --- | --- |
{{ for key, runs in data.results["interfaces"].items(): }}
{{ concurrency_values = {runs[ckey]["requests"]["rps"]: ckey for ckey in runs.keys()} }}
{{ max_res = concurrency_values[max(concurrency_values.keys())] }}
{{ run = runs[max_res] }}
| {{ =key }} (c{{ =max_res }}) | {{ =run["requests"]["total"] }} | {{ =run["requests"]["rps"] }} | {{ =int(run["latency"]["avg"]) / 1000 }}ms | {{ =int(run["latency"]["max"]) / 1000 }}ms |
{{ pass }}
{{ pass }}
{{ if any(key in data.results for key in ["vs_async", "vs_sync"]): }}
## vs 3rd parties
{{ if "vs_async" in data.results: }}
### async
| Mode | Total requests | RPS | avg latency | max latency |
| --- | --- | --- | --- | --- |
{{ for key, runs in data.results["vs_async"].items(): }}
{{ concurrency_values = {runs[ckey]["requests"]["rps"]: ckey for ckey in runs.keys()} }}
{{ max_res = concurrency_values[max(concurrency_values.keys())] }}
{{ run = runs[max_res] }}
| {{ =key }} (c{{ =max_res }}) | {{ =run["requests"]["total"] }} | {{ =run["requests"]["rps"] }} | {{ =int(run["latency"]["avg"]) / 1000 }}ms | {{ =int(run["latency"]["max"]) / 1000 }}ms |
{{ pass }}
{{ pass }}
{{ if "vs_sync" in data.results: }}
### sync
| Mode | Total requests | RPS | avg latency | max latency |
| --- | --- | --- | --- | --- |
{{ for key, runs in data.results["vs_sync"].items(): }}
{{ concurrency_values = {runs[ckey]["requests"]["rps"]: ckey for ckey in runs.keys()} }}
{{ max_res = concurrency_values[max(concurrency_values.keys())] }}
{{ run = runs[max_res] }}
| {{ =key }} (c{{ =max_res }}) | {{ =run["requests"]["total"] }} | {{ =run["requests"]["rps"] }} | {{ =int(run["latency"]["avg"]) / 1000 }}ms | {{ =int(run["latency"]["max"]) / 1000 }}ms |
{{ pass }}
{{ pass }}
{{ if "vs_maxc" in data.results: }}
### concurrency
| Mode | Total requests | RPS | avg latency | max latency |
| --- | --- | --- | --- | --- |
{{ for key, runs in data.results["vs_maxc"].items(): }}
{{ concurrency_values = {runs[ckey]["requests"]["rps"]: ckey for ckey in runs.keys()} }}
{{ max_res = concurrency_values[max(concurrency_values.keys())] }}
{{ run = runs[max_res] }}
| {{ =key }} (c{{ =max_res }}) | {{ =run["requests"]["total"] }} | {{ =run["requests"]["rps"] }} | {{ =int(run["latency"]["avg"]) / 1000 }}ms | {{ =int(run["latency"]["max"]) / 1000 }}ms |
{{ pass }}
{{ pass }}
{{ pass }}
{{ if "concurrencies" in data.results: }}
## Concurrency
{{ for interface in ["asgi", "rsgi", "wsgi"]: }}
### {{ =interface.upper() }}
{{ max_rps = {"runtime": 0, "workers": 0} }}
{{ for runs in data.results["concurrencies"][interface].values(): }}
{{ for crun in runs["res"].values(): }}
{{ max_rps[runs["m"]] = max(int(crun["requests"]["rps"]), max_rps[runs["m"]]) }}
{{ pass }}
{{ pass }}
| Mode | Processes | Threads | Blocking Threads | Total requests | RPS | avg latency | max latency |
| --- | --- | --- | --- | --- | --- | --- | --- |
{{ for runs in data.results["concurrencies"][interface].values(): }}
{{ concurrency_values = {runs["res"][ckey]["requests"]["rps"]: ckey for ckey in runs["res"].keys()} }}
{{ max_res = concurrency_values[max(concurrency_values.keys())] }}
{{ run = runs["res"][max_res] }}
{{ rps = "**" + run["requests"]["rps"] + "**" if int(run["requests"]["rps"]) == max_rps[runs["m"]] else run["requests"]["rps"] }}
| {{ =runs["m"] }} (c{{ =max_res }}) | {{ =runs["p"] }} | {{ =runs["t"] }} | {{ =runs["b"] }} | {{ =run["requests"]["total"] }} | {{ =rps }} | {{ =int(run["latency"]["avg"]) / 1000 }}ms | {{ =int(run["latency"]["max"]) / 1000 }}ms |
{{ pass }}
{{ pass }}
{{ pass }}

33
benchmarks/app/asgi.py
View file

@ -1,3 +1,4 @@
import pathlib
import sys
PLAINTEXT_RESPONSE = {
@ -7,12 +8,19 @@ PLAINTEXT_RESPONSE = {
(b'content-type', b'text/plain; charset=utf-8'),
]
}
MEDIA_RESPONSE = {
'type': 'http.response.start',
'status': 200,
'headers': [[b'content-type', b'image/png'], [b'content-length', b'95']],
}
BODY_BYTES_SHORT = b"Test"
BODY_BYTES_LONG = b"Test" * 20_000
BODY_STR_SHORT = "Test"
BODY_STR_LONG = "Test" * 20_000
MEDIA_PATH = pathlib.Path(__file__).parent.parent / 'files' / 'media.png'
async def b_short(scope, receive, send):
await send(PLAINTEXT_RESPONSE)
@ -60,6 +68,22 @@ async def echo(scope, receive, send):
})
async def file_body(scope, receive, send):
await send(MEDIA_RESPONSE)
with MEDIA_PATH.open('rb') as f:
data = f.read()
await send({
'type': 'http.response.body',
'body': data,
'more_body': False
})
async def file_pathsend(scope, receive, send):
await send(MEDIA_RESPONSE)
await send({'type': 'http.response.pathsend', 'path': str(MEDIA_PATH)})
async def handle_404(scope, receive, send):
content = b'Not found'
await send(PLAINTEXT_RESPONSE)
@ -75,7 +99,9 @@ routes = {
'/bb': b_long,
'/s': s_short,
'/ss': s_long,
'/echo': echo
'/echo': echo,
'/fb': file_body,
'/fp': file_pathsend,
}
@ -84,6 +110,11 @@ def app(scope, receive, send):
return handler(scope, receive, send)
async def async_app(scope, receive, send):
handler = routes.get(scope['path'], handle_404)
return await handler(scope, receive, send)
def granian(wrk, thr):
from granian import Granian
Granian("asgi:app", workers=int(wrk), threads=int(thr), interface="asgi").serve()

15
benchmarks/app/rsgi.py
View file

@ -1,12 +1,16 @@
import pathlib
import sys
HEADERS = [('content-type', 'text/plain; charset=utf-8')]
HEADERS_MEDIA = [('content-type', 'image/png'), ('content-length', '95')]
BODY_BYTES_SHORT = b"Test"
BODY_BYTES_LONG = b"Test" * 20_000
BODY_STR_SHORT = "Test"
BODY_STR_LONG = "Test" * 20_000
MEDIA_PATH = str(pathlib.Path(__file__).parent.parent / 'files' / 'media.png')
async def b_short(scope, proto):
proto.response_bytes(
@ -48,6 +52,14 @@ async def echo(scope, proto):
)
async def file(scope, proto):
proto.response_file(
200,
HEADERS_MEDIA,
MEDIA_PATH
)
async def handle_404(scope, proto):
proto.response_str(
404,
@ -61,7 +73,8 @@ routes = {
'/bb': b_long,
'/s': s_short,
'/ss': s_long,
'/echo': echo
'/echo': echo,
'/fp': file,
}

256
benchmarks/benchmarks.py
View file

@ -4,6 +4,7 @@ import multiprocessing
import os
import signal
import subprocess
import sys
import time
from contextlib import contextmanager
@ -11,86 +12,125 @@ from contextlib import contextmanager
CPU = multiprocessing.cpu_count()
WRK_CONCURRENCIES = [CPU * 2**i for i in range(3, 7)]
APPS = {
"asgi": (
"granian --interface asgi --log-level warning --backlog 2048 "
"--no-ws --http {http} "
"--workers {procs} --threads {threads} --blocking-threads {bthreads} "
"--threading-mode {thmode} app.asgi:app"
),
"rsgi": (
"granian --interface rsgi --log-level warning --backlog 2048 "
"--no-ws --http {http} "
"--workers {procs} --threads {threads} --blocking-threads {bthreads} "
"--threading-mode {thmode} app.rsgi:app"
),
"wsgi": (
"granian --interface wsgi --log-level warning --backlog 2048 "
"--no-ws --http {http} "
"--workers {procs} --threads {threads} --blocking-threads {bthreads} "
"--threading-mode {thmode} app.wsgi:app"
),
"uvicorn_h11": (
"uvicorn --interface asgi3 "
"--no-access-log --log-level warning "
"--http h11 --workers {procs} app.asgi:app"
),
"uvicorn_httptools": (
"uvicorn --interface asgi3 "
"--no-access-log --log-level warning "
"--http httptools --workers {procs} app.asgi:app"
),
"hypercorn": (
"hypercorn -b localhost:8000 -k uvloop --log-level warning --backlog 2048 "
"--workers {procs} asgi:app.asgi:async_app"
),
"gunicorn_gthread": "gunicorn --workers {procs} -k gthread app.wsgi:app",
"gunicorn_gevent": "gunicorn --workers {procs} -k gevent app.wsgi:app",
"uwsgi": (
"uwsgi --http :8000 --master --processes {procs} --enable-threads "
"--disable-logging --die-on-term --single-interpreter --lazy-apps "
"--wsgi-file app/wsgi.py --callable app"
)
}
@contextmanager
def app(name, procs=None, threads=None, bthreads=None, thmode=None):
def app(name, procs=None, threads=None, bthreads=None, thmode=None, http="1"):
procs = procs or 1
threads = threads or 1
bthreads = bthreads or 1
thmode = thmode or "workers"
proc = {
"asgi": (
"granian --interface asgi --log-level warning --backlog 2048 "
"--no-ws --http 1 "
f"--workers {procs} --threads {threads} --blocking-threads {bthreads} "
f"--threading-mode {thmode} app.asgi:app"
),
"rsgi": (
"granian --interface rsgi --log-level warning --backlog 2048 "
"--no-ws --http 1 "
f"--workers {procs} --threads {threads} --blocking-threads {bthreads} "
f"--threading-mode {thmode} app.rsgi:app"
),
"wsgi": (
"granian --interface wsgi --log-level warning --backlog 2048 "
"--no-ws --http 1 "
f"--workers {procs} --threads {threads} --blocking-threads {bthreads} "
f"--threading-mode {thmode} app.wsgi:app"
),
"uvicorn_h11": (
"uvicorn --interface asgi3 "
"--no-access-log --log-level warning "
f"--http h11 --workers {procs} app.asgi:app"
),
"uvicorn_httptools": (
"uvicorn --interface asgi3 "
"--no-access-log --log-level warning "
f"--http httptools --workers {procs} app.asgi:app"
),
"hypercorn": (
"hypercorn -b localhost:8000 -k uvloop --log-level warning --backlog 2048 "
f"--workers {procs} asgi:app.asgi:app"
),
"gunicorn_gthread": f"gunicorn --workers {procs} -k gthread app.wsgi:app",
"gunicorn_gevent": f"gunicorn --workers {procs} -k gevent app.wsgi:app",
}
proc = subprocess.Popen(proc[name], shell=True, preexec_fn=os.setsid)
proc_cmd = APPS[name].format(
procs=procs,
threads=threads,
bthreads=bthreads,
thmode=thmode,
http=http,
)
proc = subprocess.Popen(proc_cmd, shell=True, preexec_fn=os.setsid)
time.sleep(2)
yield proc
os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
def wrk(duration, concurrency, endpoint, post=False):
script = "wrk.post.lua" if post else "wrk.lua"
threads = max(2, CPU // 2)
proc = subprocess.run(
f"wrk -d{duration}s -H \"Connection: keep-alive\" -t{threads} -c{concurrency} "
f"-s {script} http://localhost:8000/{endpoint}",
shell=True,
check=True,
capture_output=True,
)
data = proc.stderr.decode("utf8").split(",")
return {
"requests": {"total": data[1], "rps": data[2]},
"latency": {"avg": data[11], "max": data[10], "stdev": data[12]},
}
def wrk(duration, concurrency, endpoint, post=False, h2=False):
cmd_parts = [
"rewrk",
f"-c {concurrency}",
f"-d {duration}s",
"--json",
]
if h2:
cmd_parts.append("--http2")
else:
cmd_parts.append("-H \"Connection: Keep-Alive\"")
cmd_parts.append("-H \"Keep-Alive: timeout=60'\"")
if post:
cmd_parts.append("-H \"Content-Type: text/plain; charset=utf-8\"")
cmd_parts.append("-H \"Content-Length: 4\"")
cmd_parts.append("-b \"test\"")
cmd_parts.append(f"-h http://127.0.0.1:8000/{endpoint}")
try:
proc = subprocess.run(
" ".join(cmd_parts),
shell=True,
check=True,
capture_output=True,
)
data = json.loads(proc.stdout.decode("utf8"))
return {
"requests": {
"total": data["requests_total"],
"rps": round(data["requests_avg"] or 0)
},
"latency": {
"avg": data["latency_avg"],
"max": data["latency_max"],
"stdev": data["latency_std_deviation"]
},
}
except Exception:
return {
"requests": {"total": 0, "rps": 0},
"latency": {"avg": None, "max": None, "stdev": None},
}
def benchmark(endpoint, post=False):
def benchmark(endpoint, post=False, h2=False):
results = {}
# primer
wrk(5, 8, endpoint, post=post)
wrk(5, 8, endpoint, post=post, h2=h2)
time.sleep(2)
# warm up
wrk(5, max(WRK_CONCURRENCIES), endpoint, post=post)
wrk(5, max(WRK_CONCURRENCIES), endpoint, post=post, h2=h2)
time.sleep(3)
# bench
for concurrency in WRK_CONCURRENCIES:
res = wrk(15, concurrency, endpoint, post=post)
res = wrk(15, concurrency, endpoint, post=post, h2=h2)
results[concurrency] = res
time.sleep(3)
time.sleep(2)
time.sleep(1)
return results
@ -113,8 +153,8 @@ def concurrencies():
"res": benchmark("b")
}
for np in nperm:
for nt in [1, 2, 4]:
for nbt in [1, 2, 4]:
for nt, nbtl in [(1, [1, 2, 4]), (2, [1]), (4, [1])]:
for nbt in nbtl:
for threading_mode in ["workers", "runtime"]:
key = f"P{np} T{nt} B{nbt} {threading_mode[0]}th"
with app("wsgi", np, nt, nbt, threading_mode):
@ -149,10 +189,32 @@ def interfaces():
return results
def vs_3rd_async():
def http2():
results = {}
benches = {"[GET]": ("b", {}), "[POST]": ("echo", {"post": True})}
for fw in ["granian_asgi", "granian_rsgi", "uvicorn_h11", "uvicorn_httptools", "hypercorn"]:
for http2 in [False, True]:
for key, bench_data in benches.items():
route, opts = bench_data
h = "2" if http2 else "1.1"
with app("rsgi", http=h):
results[f"HTTP/{h} {key}"] = benchmark(route, h2=http2, **opts)
return results
def files():
results = {}
with app("rsgi"):
results["RSGI"] = benchmark("fp")
with app("asgi"):
results["ASGI"] = benchmark("fb")
results["ASGI pathsend"] = benchmark("fp")
return results
def vs_asgi():
results = {}
benches = {"[GET]": ("b", {}), "[POST]": ("echo", {"post": True})}
for fw in ["granian_asgi", "uvicorn_h11", "uvicorn_httptools", "hypercorn"]:
for key, bench_data in benches.items():
route, opts = bench_data
fw_app = fw.split("_")[1] if fw.startswith("granian") else fw
@ -162,10 +224,10 @@ def vs_3rd_async():
return results
def vs_3rd_sync():
def vs_wsgi():
results = {}
benches = {"[GET]": ("b", {}), "[POST]": ("echo", {"post": True})}
for fw in ["granian_wsgi", "gunicorn_gthread"]:
for fw in ["granian_wsgi", "gunicorn_gthread", "gunicorn_gevent", "uwsgi"]:
for key, bench_data in benches.items():
route, opts = bench_data
fw_app = fw.split("_")[1] if fw.startswith("granian") else fw
@ -175,45 +237,59 @@ def vs_3rd_sync():
return results
def vs_3rd_maxc():
def vs_http2():
results = {}
procs = {
"asgi": (int(os.environ.get("P_ASGI", 1)), int(os.environ.get("T_ASGI", 1))),
"rsgi": (int(os.environ.get("P_RSGI", 1)), int(os.environ.get("T_RSGI", 1))),
"wsgi": (int(os.environ.get("P_WSGI", 1)), int(os.environ.get("T_WSGI", 1))),
"other": int(os.environ.get("P_OTH", CPU)),
}
with app("asgi", procs["asgi"][0], procs["asgi"][1]):
results["Granian ASGI"] = benchmark("b")
with app("rsgi", procs["rsgi"][0], procs["rsgi"][1]):
results["Granian RSGI"] = benchmark("b")
with app("wsgi", procs["wsgi"][0], procs["wsgi"][1]):
results["Granian WSGI"] = benchmark("b")
with app("uvicorn_httptools", procs["other"]):
results["Uvicorn http-tools"] = benchmark("b")
with app("hypercorn", procs["other"]):
results["Hypercorn"] = benchmark("b")
with app("gunicorn", procs["other"]):
results["Gunicorn (gthread)"] = benchmark("b")
benches = {"[GET]": ("b", {}), "[POST]": ("echo", {"post": True})}
for fw in ["granian_asgi", "hypercorn"]:
for key, bench_data in benches.items():
route, opts = bench_data
fw_app = fw.split("_")[1] if fw.startswith("granian") else fw
title = " ".join(item.title() for item in fw.split("_"))
with app(fw_app, http="2"):
results[f"{title} {key}"] = benchmark(route, h2=True, **opts)
return results
def vs_files():
results = {}
with app("asgi"):
results["Granian (pathsend)"] = benchmark("fp")
for fw in ["uvicorn_h11", "uvicorn_httptools", "hypercorn"]:
title = " ".join(item.title() for item in fw.split("_"))
with app(fw):
results[title] = benchmark("fb")
return results
def _granian_version():
import granian
return granian.__version__
def run():
now = datetime.datetime.utcnow()
results = {}
if os.environ.get("BENCHMARK_BASE", "true") == "true":
results["rsgi_body"] = rsgi_body_type()
results["interfaces"] = interfaces()
results["http2"] = http2()
results["files"] = files()
if os.environ.get("BENCHMARK_CONCURRENCIES") == "true":
results["concurrencies"] = concurrencies()
if os.environ.get("BENCHMARK_VSA") == "true":
results["vs_async"] = vs_3rd_async()
if os.environ.get("BENCHMARK_VSS") == "true":
results["vs_sync"] = vs_3rd_sync()
if os.environ.get("BENCHMARK_VSC") == "true":
results["vs_maxc"] = vs_3rd_maxc()
if os.environ.get("BENCHMARK_VS") == "true":
results["vs_asgi"] = vs_asgi()
results["vs_wsgi"] = vs_wsgi()
results["vs_http2"] = vs_http2()
results["vs_files"] = vs_files()
with open("results/data.json", "w") as f:
f.write(json.dumps({"cpu": CPU, "run_at": now.isoformat(), "results": results}))
pyver = sys.version_info
f.write(json.dumps({
"cpu": CPU,
"run_at": int(now.timestamp()),
"pyver": f"{pyver.major}.{pyver.minor}",
"results": results,
"granian": _granian_version()
}))
if __name__ == "__main__":

173
benchmarks/concurrency.md Normal file
View file

@ -0,0 +1,173 @@
# Granian benchmarks
## Concurrency
Run at: Tue 19 Mar 2024, 15:08
Environment: Apple Silicon M2 Pro (CPUs: 10)
Python version: 3.11
Granian version: 1.2.0
### ASGI
| Mode | Processes | Threads | Blocking Threads | Total requests | RPS | avg latency | max latency |
| --- | --- | --- | --- | --- | --- | --- | --- |
| workers (c80) | 1 | 1 | 1 | 1472105 | 97482 | 0.809ms | 3.893ms |
| runtime (c80) | 1 | 1 | 1 | 929318 | 61534 | 1.284ms | 5.248ms |
| workers (c80) | 1 | 2 | 1 | 1400457 | 92736 | 0.851ms | 4.812ms |
| runtime (c80) | 1 | 2 | 1 | 925279 | 61272 | 1.289ms | 6.294ms |
| workers (c80) | 1 | 4 | 1 | 1290774 | 85475 | 0.94ms | 4.845ms |
| runtime (c80) | 1 | 4 | 1 | 858927 | 57254 | 1.386ms | 6.501ms |
| workers (c320) | 2 | 1 | 1 | 2226178 | 147378 | 2.11ms | 19.675ms |
| runtime (c160) | 2 | 1 | 1 | 1229094 | 81383 | 1.917ms | 9.647ms |
| workers (c640) | 2 | 2 | 1 | 2232483 | 148632 | 3.994ms | 25.4ms |
| runtime (c80) | 2 | 2 | 1 | 1294997 | 85754 | 0.91ms | 7.112ms |
| workers (c640) | 2 | 4 | 1 | 2034157 | 135182 | 4.537ms | 77.928ms |
| runtime (c80) | 2 | 4 | 1 | 1184470 | 78441 | 0.995ms | 9.285ms |
| **workers (c640)** | **4** | **1** | **1** | **2248614** | **149510** | **3.882ms** | **27.427ms** |
| runtime (c160) | 4 | 1 | 1 | 1764747 | 116853 | 1.326ms | 8.362ms |
| workers (c640) | 4 | 2 | 1 | 2241413 | 148522 | 4.233ms | 79.033ms |
| runtime (c160) | 4 | 2 | 1 | 1826775 | 121757 | 1.264ms | 16.418ms |
| workers (c640) | 4 | 4 | 1 | 2080516 | 138148 | 4.197ms | 128.194ms |
| runtime (c640) | 4 | 4 | 1 | 2131702 | 141252 | 3.736ms | 63.085ms |
| workers (c640) | 5 | 1 | 1 | 2231262 | 148035 | 3.86ms | 25.37ms |
| runtime (c320) | 5 | 1 | 1 | 1883522 | 125521 | 2.463ms | 17.453ms |
| workers (c640) | 5 | 2 | 1 | 2186091 | 144838 | 4.32ms | 96.888ms |
| runtime (c640) | 5 | 2 | 1 | 2111729 | 139929 | 3.968ms | 48.927ms |
| workers (c80) | 5 | 4 | 1 | 1483865 | 98288 | 3.024ms | 208.876ms |
| **runtime (c640)** | **5** | **4** | **1** | **2136709** | **141488** | **3.623ms** | **47.921ms** |
| workers (c640) | 10 | 1 | 1 | 2148887 | 143017 | 4.803ms | 112.821ms |
| runtime (c640) | 10 | 1 | 1 | 2020660 | 134116 | 6.486ms | 159.138ms |
| workers (c80) | 10 | 2 | 1 | 1306685 | 86531 | 13.676ms | 254.613ms |
| runtime (c640) | 10 | 2 | 1 | 2068880 | 137170 | 4.371ms | 134.885ms |
| workers (c160) | 10 | 4 | 1 | 1462070 | 96981 | 7.947ms | 375.369ms |
| runtime (c640) | 10 | 4 | 1 | 2071664 | 137285 | 3.948ms | 136.938ms |
### RSGI
| Mode | Processes | Threads | Blocking Threads | Total requests | RPS | avg latency | max latency |
| --- | --- | --- | --- | --- | --- | --- | --- |
| workers (c80) | 1 | 1 | 1 | 1411999 | 93504 | 0.842ms | 4.028ms |
| runtime (c80) | 1 | 1 | 1 | 1117334 | 73994 | 1.063ms | 5.188ms |
| workers (c320) | 1 | 2 | 1 | 2181023 | 145355 | 2.154ms | 18.989ms |
| runtime (c80) | 1 | 2 | 1 | 1422249 | 94181 | 0.833ms | 5.471ms |
| workers (c640) | 1 | 4 | 1 | 1956799 | 130351 | 4.823ms | 22.841ms |
| runtime (c160) | 1 | 4 | 1 | 949719 | 63295 | 2.524ms | 10.488ms |
| workers (c320) | 2 | 1 | 1 | 2157642 | 143791 | 2.177ms | 15.951ms |
| runtime (c80) | 2 | 1 | 1 | 1419978 | 94658 | 0.823ms | 4.059ms |
| **workers (c640)** | **2** | **2** | **1** | **2267309** | **151004** | **3.955ms** | **26.224ms** |
| runtime (c80) | 2 | 2 | 1 | 1341545 | 88836 | 0.877ms | 5.79ms |
| workers (c640) | 2 | 4 | 1 | 2274646 | 150934 | 3.648ms | 23.543ms |
| runtime (c320) | 2 | 4 | 1 | 1348524 | 89862 | 3.539ms | 16.592ms |
| workers (c640) | 4 | 1 | 1 | 2228900 | 148432 | 4.03ms | 22.629ms |
| runtime (c320) | 4 | 1 | 1 | 1837278 | 122439 | 2.532ms | 17.346ms |
| workers (c640) | 4 | 2 | 1 | 2268179 | 150527 | 4.008ms | 73.672ms |
| runtime (c160) | 4 | 2 | 1 | 1961918 | 130743 | 1.156ms | 9.574ms |
| workers (c640) | 4 | 4 | 1 | 2173476 | 144207 | 3.662ms | 87.553ms |
| **runtime (c640)** | **4** | **4** | **1** | **2183501** | **144934** | **3.671ms** | **62.258ms** |
| workers (c320) | 5 | 1 | 1 | 2215795 | 147456 | 1.981ms | 28.612ms |
| runtime (c160) | 5 | 1 | 1 | 1995860 | 133013 | 1.153ms | 10.997ms |
| workers (c640) | 5 | 2 | 1 | 2209641 | 146340 | 5.068ms | 186.313ms |
| runtime (c640) | 5 | 2 | 1 | 2137135 | 141609 | 4.052ms | 54.876ms |
| workers (c640) | 5 | 4 | 1 | 2126848 | 140957 | 3.799ms | 130.24ms |
| runtime (c640) | 5 | 4 | 1 | 2162262 | 143413 | 3.78ms | 98.119ms |
| workers (c640) | 10 | 1 | 1 | 2179120 | 144416 | 4.315ms | 82.551ms |
| runtime (c640) | 10 | 1 | 1 | 2096533 | 138856 | 6.003ms | 125.325ms |
| workers (c80) | 10 | 2 | 1 | 1411899 | 93526 | 9.769ms | 250.795ms |
| runtime (c640) | 10 | 2 | 1 | 2127853 | 141027 | 4.507ms | 143.035ms |
| workers (c160) | 10 | 4 | 1 | 1492358 | 98902 | 7.561ms | 374.871ms |
| runtime (c640) | 10 | 4 | 1 | 2107436 | 139785 | 3.807ms | 68.263ms |
### WSGI
| Mode | Processes | Threads | Blocking Threads | Total requests | RPS | avg latency | max latency |
| --- | --- | --- | --- | --- | --- | --- | --- |
| workers (c80) | 1 | 1 | 1 | 1531035 | 101384 | 0.781ms | 5.844ms |
| runtime (c160) | 1 | 1 | 1 | 1538601 | 101879 | 1.562ms | 8.726ms |
| workers (c640) | 1 | 1 | 2 | 1331197 | 88103 | 7.19ms | 25.729ms |
| runtime (c640) | 1 | 1 | 2 | 1420760 | 94034 | 6.762ms | 23.653ms |
| workers (c160) | 1 | 1 | 4 | 1173581 | 77714 | 2.044ms | 11.113ms |
| runtime (c160) | 1 | 1 | 4 | 1185259 | 78482 | 2.032ms | 9.908ms |
| workers (c640) | 1 | 2 | 1 | 1190184 | 79284 | 8.027ms | 24.066ms |
| runtime (c160) | 1 | 2 | 1 | 2122357 | 140535 | 1.103ms | 8.09ms |
| workers (c640) | 1 | 2 | 2 | 1265033 | 84268 | 7.548ms | 26.557ms |
| runtime (c80) | 1 | 2 | 2 | 1366693 | 90505 | 0.88ms | 4.658ms |
| workers (c320) | 1 | 2 | 4 | 1099795 | 72813 | 4.382ms | 17.27ms |
| runtime (c80) | 1 | 2 | 4 | 1201405 | 79555 | 1.002ms | 5.655ms |
| workers (c80) | 1 | 4 | 1 | 1224406 | 81083 | 1.021ms | 7.549ms |
| runtime (c80) | 1 | 4 | 1 | 1795258 | 118888 | 0.652ms | 5.828ms |
| workers (c80) | 1 | 4 | 2 | 1077639 | 71363 | 1.157ms | 6.695ms |
| runtime (c320) | 1 | 4 | 2 | 1255215 | 83099 | 3.831ms | 19.019ms |
| workers (c160) | 1 | 4 | 4 | 1113421 | 73724 | 2.191ms | 10.89ms |
| runtime (c640) | 1 | 4 | 4 | 1149282 | 76560 | 8.308ms | 24.442ms |
| workers (c80) | 2 | 1 | 1 | 2172513 | 143867 | 0.513ms | 3.766ms |
| runtime (c640) | 2 | 1 | 1 | 2171779 | 143727 | 4.327ms | 22.913ms |
| workers (c640) | 2 | 1 | 2 | 2223326 | 147178 | 4.264ms | 22.327ms |
| runtime (c320) | 2 | 1 | 2 | 2201404 | 145736 | 2.112ms | 17.839ms |
| workers (c320) | 2 | 1 | 4 | 1768073 | 117057 | 2.686ms | 19.285ms |
| runtime (c320) | 2 | 1 | 4 | 1764833 | 117585 | 2.688ms | 15.706ms |
| workers (c640) | 2 | 2 | 1 | 2230845 | 148442 | 3.948ms | 22.191ms |
| runtime (c320) | 2 | 2 | 1 | 2253571 | 149183 | 1.927ms | 19.215ms |
| workers (c640) | 2 | 2 | 2 | 2001567 | 133233 | 4.518ms | 27.879ms |
| runtime (c640) | 2 | 2 | 2 | 2237970 | 148899 | 3.875ms | 24.096ms |
| workers (c160) | 2 | 2 | 4 | 1459247 | 96620 | 1.67ms | 11.944ms |
| runtime (c640) | 2 | 2 | 4 | 2196268 | 146073 | 3.954ms | 24.819ms |
| workers (c640) | 2 | 4 | 1 | 1767834 | 117570 | 5.433ms | 75.389ms |
| runtime (c320) | 2 | 4 | 1 | 2233091 | 148568 | 1.768ms | 19.332ms |
| workers (c320) | 2 | 4 | 2 | 1490862 | 99355 | 3.218ms | 23.559ms |
| runtime (c640) | 2 | 4 | 2 | 2233460 | 147870 | 3.392ms | 32.852ms |
| workers (c160) | 2 | 4 | 4 | 1440232 | 95349 | 1.878ms | 88.268ms |
| runtime (c640) | 2 | 4 | 4 | 1979403 | 131128 | 4.175ms | 25.368ms |
| workers (c320) | 4 | 1 | 1 | 2289337 | 151557 | 1.904ms | 17.528ms |
| runtime (c160) | 4 | 1 | 1 | 2292997 | 151826 | 0.942ms | 11.375ms |
| workers (c640) | 4 | 1 | 2 | 2264994 | 150704 | 3.79ms | 22.417ms |
| runtime (c640) | 4 | 1 | 2 | 2276127 | 151380 | 3.768ms | 24.604ms |
| workers (c640) | 4 | 1 | 4 | 2221442 | 147599 | 3.791ms | 22.506ms |
| runtime (c640) | 4 | 1 | 4 | 2242754 | 148910 | 3.821ms | 26.634ms |
| workers (c640) | 4 | 2 | 1 | 2278037 | 151358 | 3.834ms | 54.723ms |
| runtime (c320) | 4 | 2 | 1 | 2284117 | 151531 | 1.906ms | 48.789ms |
| workers (c640) | 4 | 2 | 2 | 2216991 | 146917 | 4.546ms | 85.882ms |
| runtime (c640) | 4 | 2 | 2 | 2276864 | 150961 | 3.627ms | 38.945ms |
| workers (c640) | 4 | 2 | 4 | 2034538 | 135153 | 4.742ms | 99.635ms |
| runtime (c640) | 4 | 2 | 4 | 2170743 | 143723 | 3.681ms | 59.116ms |
| workers (c80) | 4 | 4 | 1 | 1499563 | 99297 | 13.849ms | 258.47ms |
| runtime (c640) | 4 | 4 | 1 | 2255686 | 149478 | 3.519ms | 39.328ms |
| workers (c80) | 4 | 4 | 2 | 1334184 | 88409 | 10.222ms | 282.733ms |
| runtime (c640) | 4 | 4 | 2 | 2206503 | 146332 | 3.416ms | 70.791ms |
| workers (c160) | 4 | 4 | 4 | 1414347 | 93849 | 8.378ms | 339.89ms |
| runtime (c640) | 4 | 4 | 4 | 2095334 | 138736 | 3.575ms | 51.54ms |
| **workers (c160)** | **5** | **1** | **1** | **2299404** | **152246** | **0.935ms** | **9.328ms** |
| runtime (c320) | 5 | 1 | 1 | 2283078 | 151921 | 1.915ms | 17.687ms |
| workers (c640) | 5 | 1 | 2 | 2277170 | 151155 | 3.777ms | 26.321ms |
| runtime (c640) | 5 | 1 | 2 | 2283252 | 151467 | 3.837ms | 31.866ms |
| workers (c640) | 5 | 1 | 4 | 2230754 | 147775 | 3.996ms | 47.481ms |
| runtime (c640) | 5 | 1 | 4 | 2237231 | 148159 | 4.049ms | 41.956ms |
| workers (c640) | 5 | 2 | 1 | 2263438 | 150183 | 3.832ms | 48.856ms |
| runtime (c640) | 5 | 2 | 1 | 2286014 | 151344 | 3.67ms | 41.313ms |
| workers (c640) | 5 | 2 | 2 | 2125966 | 140932 | 4.76ms | 113.512ms |
| runtime (c640) | 5 | 2 | 2 | 2248842 | 149439 | 3.588ms | 52.371ms |
| workers (c80) | 5 | 2 | 4 | 1434767 | 94984 | 9.639ms | 250.261ms |
| runtime (c640) | 5 | 2 | 4 | 2108567 | 139671 | 3.769ms | 50.505ms |
| workers (c80) | 5 | 4 | 1 | 1503811 | 99552 | 7.747ms | 375.127ms |
| runtime (c640) | 5 | 4 | 1 | 2250059 | 149019 | 3.473ms | 39.089ms |
| workers (c80) | 5 | 4 | 2 | 1361092 | 90112 | 10.666ms | 374.913ms |
| runtime (c640) | 5 | 4 | 2 | 2176351 | 144173 | 3.509ms | 71.264ms |
| workers (c160) | 5 | 4 | 4 | 1469777 | 97288 | 4.499ms | 249.513ms |
| runtime (c640) | 5 | 4 | 4 | 2052086 | 136093 | 3.693ms | 51.207ms |
| workers (c640) | 10 | 1 | 1 | 2287452 | 151760 | 4.493ms | 59.651ms |
| **runtime (c640)** | **10** | **1** | **1** | **2302335** | **152476** | **4.18ms** | **53.213ms** |
| workers (c640) | 10 | 1 | 2 | 2235166 | 147962 | 4.179ms | 75.248ms |
| runtime (c640) | 10 | 1 | 2 | 2216916 | 146880 | 4.312ms | 96.285ms |
| workers (c640) | 10 | 1 | 4 | 2063897 | 136797 | 4.413ms | 73.977ms |
| runtime (c640) | 10 | 1 | 4 | 2067111 | 137054 | 4.612ms | 107.718ms |
| workers (c640) | 10 | 2 | 1 | 2171946 | 143875 | 4.066ms | 86.412ms |
| runtime (c640) | 10 | 2 | 1 | 2204148 | 146054 | 3.561ms | 72.654ms |
| workers (c640) | 10 | 2 | 2 | 2018679 | 133761 | 4.646ms | 119.943ms |
| runtime (c640) | 10 | 2 | 2 | 2106616 | 139680 | 3.79ms | 87.341ms |
| workers (c80) | 10 | 2 | 4 | 1502379 | 99545 | 3.565ms | 250.65ms |
| runtime (c80) | 10 | 2 | 4 | 1434102 | 95160 | 5.538ms | 250.904ms |
| workers (c640) | 10 | 4 | 1 | 1997207 | 132329 | 4.341ms | 103.345ms |
| runtime (c640) | 10 | 4 | 1 | 2154127 | 142812 | 3.593ms | 77.126ms |
| workers (c80) | 10 | 4 | 2 | 1377866 | 91340 | 6.694ms | 374.844ms |
| runtime (c640) | 10 | 4 | 2 | 2041386 | 135175 | 3.782ms | 68.789ms |
| workers (c160) | 10 | 4 | 4 | 1475546 | 97821 | 1.951ms | 75.6ms |
| runtime (c80) | 10 | 4 | 4 | 1424839 | 94454 | 4.106ms | 180.078ms |

3
benchmarks/envs/asgi.txt Normal file
View file

@ -0,0 +1,3 @@
httptools
uvicorn
hypercorn

1
benchmarks/envs/base.txt Normal file
View file

@ -0,0 +1 @@
granian

3
benchmarks/envs/wsgi.txt Normal file
View file

@ -0,0 +1,3 @@
gevent
gunicorn
uwsgi

BIN
benchmarks/files/media.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 95 B

6
benchmarks/templates/_files.md Normal file
View file

@ -0,0 +1,6 @@
## File responses
Comparison between Granian application protocols using 95bytes image.
WSGI is not part of the benchmark since the protocol doesn't implement anything different from returning the file's contents directly.
{{ include './_table.tpl' }}

9
benchmarks/templates/_helpers.tpl Normal file
View file

@ -0,0 +1,9 @@
{{
def get_max_concurrency_run(data):
concurrency_values = {data[ckey]["requests"]["rps"]: ckey for ckey in data.keys()}
maxc = concurrency_values[max(concurrency_values.keys())]
return maxc, data[maxc]
def fmt_ms(v):
return f"{round(v, 3)}ms" if v else "N/A"
}}

5
benchmarks/templates/_http2.md Normal file
View file

@ -0,0 +1,5 @@
## HTTP/2
Comparison between Granian HTTP versions on RSGI using 4bytes plain text response.
{{ include './_table.tpl' }}

8
benchmarks/templates/_ifaces.md Normal file
View file

@ -0,0 +1,8 @@
## Interfaces
Comparison between Granian application protocols using 4bytes plain text response.
Bytes and string response are reported for every protocol just to report the difference with RSGI protocol.
ASGI and WSGI responses are always returned as bytes by the application.
The "echo" request is a 4bytes POST request responding with the same body.
{{ include './_table.tpl' }}

11
benchmarks/templates/_rsgi.md Normal file
View file

@ -0,0 +1,11 @@
## RSGI response types
RSGI plain text response comparison using protocol `response_str` and `response_bytes`.
The "small" response is 4 bytes, the "big" one is 80kbytes.
| Type | Total requests | RPS | avg latency | max latency |
| --- | --- | --- | --- | --- |
{{ for key, runs in _data.items(): }}
{{ max_c, run = get_max_concurrency_run(runs) }}
| {{ =key }} (c{{ =max_c }}) | {{ =run["requests"]["total"] }} | {{ =run["requests"]["rps"] }} | {{ =fmt_ms(run["latency"]["avg"]) }} | {{ =fmt_ms(run["latency"]["max"]) }} |
{{ pass }}

6
benchmarks/templates/_table.tpl Normal file
View file

@ -0,0 +1,6 @@
| Request | Total requests | RPS | avg latency | max latency |
| --- | --- | --- | --- | --- |
{{ for key, runs in _data.items(): }}
{{ max_c, run = get_max_concurrency_run(runs) }}
| {{ =key }} (c{{ =max_c }}) | {{ =run["requests"]["total"] }} | {{ =run["requests"]["rps"] }} | {{ =fmt_ms(run["latency"]["avg"]) }} | {{ =fmt_ms(run["latency"]["max"]) }} |
{{ pass }}

7
benchmarks/templates/_vs_table.tpl Normal file
View file

@ -0,0 +1,7 @@
| Server | Total requests | RPS | avg latency | max latency |
| --- | --- | --- | --- | --- |
{{ for key, runs in _data.items(): }}
{{ max_c, run = get_max_concurrency_run(runs) }}
| {{ =key }} (c{{ =max_c }}) | {{ =run["requests"]["total"] }} | {{ =run["requests"]["rps"] }} | {{ =fmt_ms(run["latency"]["avg"]) }} | {{ =fmt_ms(run["latency"]["max"]) }} |
{{ pass }}
{{ pass }}

32
benchmarks/templates/concurrency.md Normal file
View file

@ -0,0 +1,32 @@
# Granian benchmarks
{{ include './_helpers.tpl' }}
## Concurrency
Run at: {{ =datetime.datetime.fromtimestamp(data.run_at).strftime('%a %d %b %Y, %H:%M') }}
Environment: {{ =benv }} (CPUs: {{ =data.cpu }})
Python version: {{ =data.pyver }}
Granian version: {{ =data.granian }}
{{ for interface in ["asgi", "rsgi", "wsgi"]: }}
### {{ =interface.upper() }}
{{ max_rps = {"runtime": 0, "workers": 0} }}
{{ for runs in data.results["concurrencies"][interface].values(): }}
{{ for crun in runs["res"].values(): }}
{{ max_rps[runs["m"]] = max(int(crun["requests"]["rps"]), max_rps[runs["m"]]) }}
{{ pass }}
{{ pass }}
| Mode | Processes | Threads | Blocking Threads | Total requests | RPS | avg latency | max latency |
| --- | --- | --- | --- | --- | --- | --- | --- |
{{ for runs in data.results["concurrencies"][interface].values(): }}
{{ max_c, run = get_max_concurrency_run(runs["res"]) }}
{{ if int(run["requests"]["rps"]) == max_rps[runs["m"]]: }}
| **{{ =runs["m"] }} (c{{ =max_c }})** | **{{ =runs["p"] }}** | **{{ =runs["t"] }}** | **{{ =runs["b"] }}** | **{{ =run["requests"]["total"] }}** | **{{ =run["requests"]["rps"] }}** | **{{ =fmt_ms(run["latency"]["avg"]) }}** | **{{ =fmt_ms(run["latency"]["max"]) }}** |
{{ else: }}
| {{ =runs["m"] }} (c{{ =max_c }}) | {{ =runs["p"] }} | {{ =runs["t"] }} | {{ =runs["b"] }} | {{ =run["requests"]["total"] }} | {{ =run["requests"]["rps"] }} | {{ =fmt_ms(run["latency"]["avg"]) }} | {{ =fmt_ms(run["latency"]["max"]) }} |
{{ pass }}
{{ pass }}
{{ pass }}

28
benchmarks/templates/main.md Normal file
View file

@ -0,0 +1,28 @@
# Granian benchmarks
{{ include './_helpers.tpl' }}
Run at: {{ =datetime.datetime.fromtimestamp(data.run_at).strftime('%a %d %b %Y, %H:%M') }}
Environment: {{ =benv }} (CPUs: {{ =data.cpu }})
Python version: {{ =data.pyver }}
Granian version: {{ =data.granian }}
{{ _data = data.results["rsgi_body"] }}
{{ include './_rsgi.tpl' }}
{{ _data = data.results["interfaces"] }}
{{ include './_ifaces.tpl' }}
{{ _data = data.results["http2"] }}
{{ include './_http2.tpl' }}
{{ _data = data.results["files"] }}
{{ include './_files.tpl' }}
### Other benchmarks
- [Versus 3rd party servers](./vs.md)
- [Concurrency](./concurrency.md)
{{ if False: }}
- [Python versions](./pyver.md)
{{ pass }}

30
benchmarks/templates/vs.md Normal file
View file

@ -0,0 +1,30 @@
# Granian benchmarks
{{ include './_helpers.tpl' }}
## VS 3rd party comparison
Run at: {{ =datetime.datetime.fromtimestamp(data.run_at).strftime('%a %d %b %Y, %H:%M') }}
Environment: {{ =benv }} (CPUs: {{ =data.cpu }})
Python version: {{ =data.pyver }}
Granian version: {{ =data.granian }}
### ASGI
{{ _data = data.results["vs_asgi"] }}
{{ include './_vs_table.tpl' }}
### WSGI
{{ _data = data.results["vs_wsgi"] }}
{{ include './_vs_table.tpl' }}
### HTTP/2
{{ _data = data.results["vs_http2"] }}
{{ include './_vs_table.tpl' }}
### ASGI file responses
{{ _data = data.results["vs_files"] }}
{{ include './_vs_table.tpl' }}

31
benchmarks/wrk.lua
View file

@ -1,31 +0,0 @@
done = function(summary, latency, requests)
out = {
summary.duration,
summary.requests,
summary.requests/(summary.duration/1000000),
summary.bytes,
summary.errors.connect,
summary.errors.read,
summary.errors.write,
summary.errors.status,
summary.errors.timeout,
latency.min,
latency.max,
latency.mean,
latency.stdev,
latency:percentile(50),
latency:percentile(75),
latency:percentile(90),
latency:percentile(99),
latency:percentile(99.999)
}
for key, value in pairs(out) do
if key > 1 then
io.stderr:write(",")
end
io.stderr:write(string.format("%d", value))
end
end

35
benchmarks/wrk.post.lua
View file

@ -1,35 +0,0 @@
wrk.method = "POST"
wrk.body = "Test"
wrk.headers["Content-Type"] = "text/plain; charset=utf-8"
done = function(summary, latency, requests)
out = {
summary.duration,
summary.requests,
summary.requests/(summary.duration/1000000),
summary.bytes,
summary.errors.connect,
summary.errors.read,
summary.errors.write,
summary.errors.status,
summary.errors.timeout,
latency.min,
latency.max,
latency.mean,
latency.stdev,
latency:percentile(50),
latency:percentile(75),
latency:percentile(90),
latency:percentile(99),
latency:percentile(99.999)
}
for key, value in pairs(out) do
if key > 1 then
io.stderr:write(",")
end
io.stderr:write(string.format("%d", value))
end
end