{"name":"Gemma-4-31B-it","arena":{"id":"gemma-4-31b-bf16","name":"Gemma-4 31B","vram":62,"params":31,"vendor":"google","quality":{"coding":{"bench":"LiveCodeBench v6","value":80},"science":{"bench":"GPQA-Diamond","value":84.3},"knowledge":{"bench":"MMLU-Pro","value":85.2}},"contextK":256,"baseModelId":"gemma-4-31b-it"},"notes":"Model-ID: google/gemma-4-31B-it. Container: vllm/vllm-openai:v0.26.0-aarch64-cu129-ubuntu2404. llama-benchy 0.4.0. Bronrun: 2026-08-06T14:44:40+02:00. KV-cache fp8; max_model_len 131072; gpu_memory_utilization 0,9. Prefix caching uit; async scheduling aan. Extra serverflags: --max-num-batched-tokens 8192. 11 van de 11 tests hebben een meting die volgens de huidige Arena-regels meetelt. Bij deze historische run is geen sanity-check vastgelegd. Dat bewijs ontbreekt, ook wanneer de snelheidsmeting volgens de huidige regels meetelt.","order":11,"engine":"vLLM v0.26.0","vramGb":62,"verdict":"Gemeten op 2026-08-06 met vLLM v0.26.0. In de chattest met tien gelijktijdige verzoeken haalt dit profiel 3,33 tokens/s per gebruiker, met gemiddeld 6,38 seconden tot het eerste token. Bij 25k context stijgt die gemiddelde wachttijd naar 194,42 seconden. Bij deze historische run is geen sanity-check vastgelegd. Dat bewijs ontbreekt, ook wanneer de snelheidsmeting volgens de huidige regels meetelt.","findings":{"broke":{"body":"Bij 25000 inputtokens en 256 outputtokens met tien gelijktijdige verzoeken is de gemiddelde wachttijd voor het eerste token 194,42 seconden. Daarna genereert het model gemiddeld 1,33 tokens/s per gebruiker. De meting alleen toont niet of geheugen, promptverwerking of scheduling de oorzaak van vertraging is.","title":"Wachttijd bij 25k context"},"costly":{"body":"De run verwerkt 0,08 verzoeken/s bij een ingestelde aankomstsnelheid van 1,5 verzoeken/s. 300 van de 300 verzoeken slagen. De p95-wachttijd voor het eerste token is 35,39 seconden.","title":"Verzoeken en wachttijd bij piekbelasting"},"worked":{"body":"Bij 1024 input- en 1024 outputtokens met tien gelijktijdige verzoeken meet ik gemiddeld 3,33 tokens/s per gebruiker. De gemiddelde wachttijd voor het eerste token is 6,38 seconden.","title":"Chat bij tien gelijktijdige verzoeken"},"surprised":{"body":"11 van de 11 tests hebben een meting die volgens de huidige Arena-regels meetelt. Bij deze historische run is geen sanity-check vastgelegd. Dat bewijs ontbreekt, ook wanneer de snelheidsmeting volgens de huidige regels meetelt.","title":"Welke metingen meetellen"}},"hardware":"DGX Spark, NVIDIA GB10, 128 GB unified memory","modelUrl":"https://huggingface.co/google/gemma-4-31B-it","provider":"Google","precision":"BF16","parameters":"31B","provenance":{"modelId":"google/gemma-4-31B-it","runPath":"results/gemma-4/gemma-4-31b-it/bf16","validity":{"tests":{"01-chat":"complete","02-rag-8k":"complete","08-sharegpt":"complete","09-reasoning":"complete","04-multi-turn":"complete","11-rate-sweep":"complete","03-long-output":"complete","05-big-context":"complete","10-monday-peak":"complete","07-office-baseline":"complete","06-concurrency-stress":"complete"},"sanity":"not_recorded"},"sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-31b-it/bf16/","servedName":"gemma-4-31b","generatedAt":"2026-08-06T14:44:40+02:00","runRevision":"a948cec23b054c62fef595bad3cdb284066922ac","vllmVersion":"0.26.0","suiteVersion":"2026-08","repositoryUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks","contractVersion":1,"contractRevision":"a948cec23b054c62fef595bad3cdb284066922ac","commandProvenance":"reconstructed_from_raw_artifacts","llamaBenchyVersion":"0.4.0"},"architecture":"Dense","measuredContextK":128,"benchmarkResults":[{"id":"gemma-4-31b-it-bf16-chat-1k","date":"2026-08-06","runs":3,"metrics":{"ttfrSec":{"mean":6.385,"stddev":2.484},"decodeTokensPerSecondTotal":{"mean":18.31,"stddev":1.62},"prefillTokensPerSecondTotal":{"mean":1055.2,"stddev":16.97},"decodeTokensPerSecondPerUser":{"mean":3.33,"stddev":0.25}},"sourceRun":"Test 01","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-31b-it/bf16/","workloadId":"chat-1k","sourceLabel":"google/gemma-4-31B-it bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"01-chat","prefixCaching":false},{"id":"gemma-4-31b-it-bf16-rag-8k","date":"2026-08-06","runs":3,"metrics":{"ttfrSec":{"mean":45.869,"stddev":23.877},"decodeTokensPerSecondTotal":{"mean":17.94,"stddev":1.17},"prefillTokensPerSecondTotal":{"mean":938.64,"stddev":1.5},"decodeTokensPerSecondPerUser":{"mean":2.61,"stddev":0.46}},"sourceRun":"Test 02","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-31b-it/bf16/","workloadId":"rag-8k","sourceLabel":"google/gemma-4-31B-it bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"02-rag-8k","prefixCaching":false},{"id":"gemma-4-31b-it-bf16-long-output-agents","date":"2026-08-06","runs":3,"metrics":{"ttfrSec":{"mean":2.447,"stddev":0.006},"decodeTokensPerSecondTotal":{"mean":20.22,"stddev":2.35},"prefillTokensPerSecondTotal":{"mean":960.19,"stddev":6.1},"decodeTokensPerSecondPerUser":{"mean":3.57,"stddev":0.02}},"sourceRun":"Test 03","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-31b-it/bf16/","workloadId":"long-output-agents","sourceLabel":"google/gemma-4-31B-it bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"03-long-output","prefixCaching":false},{"id":"gemma-4-31b-it-bf16-multi-turn-office","date":"2026-08-06","runs":3,"metrics":{"ttfrSec":{"mean":12.127,"stddev":5.058},"decodeTokensPerSecondTotal":{"mean":21.74,"stddev":2.48},"prefillTokensPerSecondTotal":{"mean":1066.72,"stddev":6.22},"decodeTokensPerSecondPerUser":{"mean":3.21,"stddev":0.19}},"sourceRun":"Test 04","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-31b-it/bf16/","workloadId":"multi-turn-office","sourceLabel":"google/gemma-4-31B-it bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"04-multi-turn","prefixCaching":false},{"id":"gemma-4-31b-it-bf16-large-context-25k","date":"2026-08-06","runs":3,"metrics":{"ttfrSec":{"mean":194.42,"stddev":98.947},"decodeTokensPerSecondTotal":{"mean":6.29,"stddev":0.15},"prefillTokensPerSecondTotal":{"mean":652.34,"stddev":0.53},"decodeTokensPerSecondPerUser":{"mean":1.33,"stddev":0.7}},"sourceRun":"Test 05","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-31b-it/bf16/","workloadId":"large-context-25k","sourceLabel":"google/gemma-4-31B-it bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"05-big-context","prefixCaching":false},{"id":"gemma-4-31b-it-bf16-concurrency-stress-25k","date":"2026-08-06","runs":3,"metrics":{"ttfrSec":{"mean":367.913,"stddev":199.786},"decodeTokensPerSecondTotal":{"mean":6.4,"stddev":0.11},"prefillTokensPerSecondTotal":{"mean":651.25,"stddev":0.64},"decodeTokensPerSecondPerUser":{"mean":0.77,"stddev":0.49}},"sourceRun":"Test 06","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-31b-it/bf16/","workloadId":"concurrency-stress-25k","sourceLabel":"google/gemma-4-31B-it bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"06-concurrency-stress","prefixCaching":false}],"openLoopResults":[{"id":"gemma-4-31b-it-bf16-office-random-4k","date":"2026-08-06","metrics":{"e2eMs":{"p50":851882.75,"p90":1146090.3,"p95":1169865.36,"p99":1303654.78,"mean":822707.56},"tpotMs":{"p50":1183.84,"p90":1573.85,"p95":1951.32,"p99":2667.75,"mean":1212.77},"ttftMs":{"p50":248552.15,"p90":586820.44,"p95":610894.15,"p99":661729.9,"mean":278210.1}},"sourceRun":"Test 07","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-31b-it/bf16/","scenarioId":"office-random-4k","achievedRps":0.118,"sourceStatus":"complete","sourceTestId":"07-office-baseline","configuredRps":0.3,"prefixCaching":false,"totalRequests":200,"successfulRequests":200,"totalTokenThroughput":536.19,"peakConcurrentRequests":172},{"id":"gemma-4-31b-it-bf16-sharegpt-replay","date":"2026-08-06","metrics":{"e2eMs":{"p50":45682.27,"p90":188422.53,"p95":224412.53,"p99":313655.59,"mean":74182.54},"tpotMs":{"p50":367.59,"p90":383.56,"p95":392.46,"p99":443.15,"mean":360.24},"ttftMs":{"p50":1049.51,"p90":1547.43,"p95":1686.04,"p99":1999.68,"mean":1126.24}},"sourceRun":"Test 08","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-31b-it/bf16/","scenarioId":"sharegpt-replay","achievedRps":0.25,"sourceStatus":"complete","sourceTestId":"08-sharegpt","configuredRps":0.3,"prefixCaching":false,"totalRequests":250,"successfulRequests":250,"totalTokenThroughput":111.21,"peakConcurrentRequests":34},{"id":"gemma-4-31b-it-bf16-reasoning","date":"2026-08-06","metrics":{"e2eMs":{"p50":1763693.65,"p90":2740835.52,"p95":2937623.95,"p99":3004571.68,"mean":1821332.71},"tpotMs":{"p50":449.68,"p90":463.31,"p95":467.94,"p99":471.08,"mean":443.13},"ttftMs":{"p50":2091.18,"p90":2858.29,"p95":3206.86,"p99":3597.44,"mean":2099.62}},"sourceRun":"Test 09","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-31b-it/bf16/","scenarioId":"reasoning","achievedRps":0.016,"sourceStatus":"complete","sourceTestId":"09-reasoning","configuredRps":0.2,"prefixCaching":false,"totalRequests":50,"successfulRequests":50,"totalTokenThroughput":84.44,"peakConcurrentRequests":50},{"id":"gemma-4-31b-it-bf16-monday-peak-random-4k","date":"2026-08-06","metrics":{"e2eMs":{"p50":292448.05,"p90":530039.35,"p95":554412.95,"p99":601429.96,"mean":301175.25},"tpotMs":{"p50":591.37,"p90":638.16,"p95":677.65,"p99":827.38,"mean":591.64},"ttftMs":{"p50":5630.36,"p90":10394.6,"p95":35393.25,"p99":85940,"mean":9225.6}},"sourceRun":"Test 10","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-31b-it/bf16/","scenarioId":"monday-peak-random-4k","achievedRps":0.08,"sourceStatus":"complete","sourceTestId":"10-monday-peak","configuredRps":1.5,"prefixCaching":false,"totalRequests":300,"successfulRequests":300,"totalTokenThroughput":361.8,"peakConcurrentRequests":27}],"rateSweep":{"date":"2026-08-06","capacity":{"slo2000":null,"slo5000":null,"slo10000":null},"sourceRun":"Test 11","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-31b-it/bf16/","abortedAtRps":null,"queueKneeRps":null,"sourceStatus":"complete","sourceTestId":"11-rate-sweep","totalRequests":850,"successfulRequests":850}}