{"name":"Gemma-4-26B-A4B-it","aaUrl":"https://artificialanalysis.ai/models/gemma-4-26b-a4b","arena":{"id":"gemma-4-26b-a4b-it-bf16","name":"Gemma-4 26B-A4B","vram":52,"params":26,"vendor":"google","quality":{"coding":{"bench":"LiveCodeBench v6","value":77.1},"science":{"bench":"GPQA-Diamond","value":82.3},"knowledge":{"bench":"MMLU-Pro","value":82.6}},"contextK":256,"baseModelId":"gemma-4-26b-a4b-it"},"notes":"Model-ID: google/gemma-4-26B-A4B-it. Container: vllm/vllm-openai:v0.26.0-aarch64-cu129-ubuntu2404. llama-benchy 0.4.0. Bronrun: 2026-08-16T12:02:29+02:00. KV-cache fp8; max_model_len 131072; gpu_memory_utilization 0,9. Prefix caching uit; async scheduling aan. Extra serverflags: --max-num-batched-tokens 8192. 11 van de 11 tests hebben een meting die volgens de huidige Arena-regels meetelt. De sanity-check is geslaagd. Dit is een controle op een kort gegenereerd antwoord, geen inhoudelijke evaluatie van de benchmarktaken.","order":0,"engine":"vLLM v0.26.0","vramGb":52,"verdict":"Gemeten op 2026-08-16 met vLLM v0.26.0. In de chattest met tien gelijktijdige verzoeken haalt dit profiel 10,89 tokens/s per gebruiker, met gemiddeld 1,47 seconden tot het eerste token. Bij 25k context stijgt die gemiddelde wachttijd naar 41,17 seconden. Deze runs meten snelheid en wachttijd, niet de kwaliteit van de antwoorden.","findings":{"broke":{"body":"Bij 25000 inputtokens en 256 outputtokens met tien gelijktijdige verzoeken is de gemiddelde wachttijd voor het eerste token 41,17 seconden. Daarna genereert het model gemiddeld 4,99 tokens/s per gebruiker. De meting alleen toont niet of geheugen, promptverwerking of scheduling de oorzaak van vertraging is.","title":"Wachttijd bij 25k context"},"costly":{"body":"De run verwerkt 0,25 verzoeken/s bij een ingestelde aankomstsnelheid van 1,5 verzoeken/s. 300 van de 300 verzoeken slagen. De p95-wachttijd voor het eerste token is 3,22 seconden.","title":"Verzoeken en wachttijd bij piekbelasting"},"worked":{"body":"Bij 1024 input- en 1024 outputtokens met tien gelijktijdige verzoeken meet ik gemiddeld 10,89 tokens/s per gebruiker. De gemiddelde wachttijd voor het eerste token is 1,47 seconden.","title":"Chat bij tien gelijktijdige verzoeken"},"surprised":{"body":"11 van de 11 tests hebben een meting die volgens de huidige Arena-regels meetelt. De sanity-check is geslaagd. Dit is een controle op een kort gegenereerd antwoord, geen inhoudelijke evaluatie van de benchmarktaken.","title":"Welke metingen meetellen"}},"hardware":"DGX Spark, NVIDIA GB10, 128 GB unified memory","modelUrl":"https://huggingface.co/google/gemma-4-26B-A4B-it","provider":"Google","precision":"BF16","parameters":"26B-A4B","provenance":{"modelId":"google/gemma-4-26B-A4B-it","runPath":"results/gemma-4/gemma-4-26b-a4b-it/bf16","validity":{"tests":{"01-chat":"complete","02-rag-8k":"complete","08-sharegpt":"complete","09-reasoning":"complete","04-multi-turn":"complete","11-rate-sweep":"complete","03-long-output":"complete","05-big-context":"complete","10-monday-peak":"complete","07-office-baseline":"complete","06-concurrency-stress":"complete"},"sanity":"passed"},"sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-26b-a4b-it/bf16/","servedName":"gemma-4-26b-a4b-bf16","generatedAt":"2026-08-16T12:02:29+02:00","runRevision":"a948cec23b054c62fef595bad3cdb284066922ac","vllmVersion":"0.26.0","suiteVersion":"2026-08","repositoryUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks","contractVersion":1,"contractRevision":"a948cec23b054c62fef595bad3cdb284066922ac","commandProvenance":"reconstructed_from_raw_artifacts","llamaBenchyVersion":"0.4.0"},"architecture":"MoE","measuredContextK":128,"benchmarkResults":[{"id":"gemma-4-26b-a4b-it-bf16-chat-1k","date":"2026-08-16","runs":3,"metrics":{"ttfrSec":{"mean":1.473,"stddev":0.546},"decodeTokensPerSecondTotal":{"mean":84.56,"stddev":1.7},"prefillTokensPerSecondTotal":{"mean":5077.55,"stddev":24.91},"decodeTokensPerSecondPerUser":{"mean":10.89,"stddev":0.6}},"sourceRun":"Test 01","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-26b-a4b-it/bf16/","workloadId":"chat-1k","sourceLabel":"google/gemma-4-26B-A4B-it bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"01-chat","prefixCaching":false},{"id":"gemma-4-26b-a4b-it-bf16-rag-8k","date":"2026-08-16","runs":3,"metrics":{"ttfrSec":{"mean":8.806,"stddev":4.59},"decodeTokensPerSecondTotal":{"mean":74.55,"stddev":1.22},"prefillTokensPerSecondTotal":{"mean":4836.86,"stddev":9.88},"decodeTokensPerSecondPerUser":{"mean":8.91,"stddev":0.79}},"sourceRun":"Test 02","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-26b-a4b-it/bf16/","workloadId":"rag-8k","sourceLabel":"google/gemma-4-26B-A4B-it bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"02-rag-8k","prefixCaching":false},{"id":"gemma-4-26b-a4b-it-bf16-long-output-agents","date":"2026-08-16","runs":3,"metrics":{"ttfrSec":{"mean":0.524,"stddev":0.003},"decodeTokensPerSecondTotal":{"mean":80.53,"stddev":0.96},"prefillTokensPerSecondTotal":{"mean":4485.15,"stddev":49.76},"decodeTokensPerSecondPerUser":{"mean":11.5,"stddev":0.78}},"sourceRun":"Test 03","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-26b-a4b-it/bf16/","workloadId":"long-output-agents","sourceLabel":"google/gemma-4-26B-A4B-it bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"03-long-output","prefixCaching":false},{"id":"gemma-4-26b-a4b-it-bf16-multi-turn-office","date":"2026-08-16","runs":3,"metrics":{"ttfrSec":{"mean":2.405,"stddev":0.927},"decodeTokensPerSecondTotal":{"mean":89.64,"stddev":1.74},"prefillTokensPerSecondTotal":{"mean":5439.8,"stddev":24.37},"decodeTokensPerSecondPerUser":{"mean":9.85,"stddev":0.26}},"sourceRun":"Test 04","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-26b-a4b-it/bf16/","workloadId":"multi-turn-office","sourceLabel":"google/gemma-4-26B-A4B-it bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"04-multi-turn","prefixCaching":false},{"id":"gemma-4-26b-a4b-it-bf16-large-context-25k","date":"2026-08-16","runs":3,"metrics":{"ttfrSec":{"mean":41.167,"stddev":20.969},"decodeTokensPerSecondTotal":{"mean":27.32,"stddev":0.09},"prefillTokensPerSecondTotal":{"mean":3078.13,"stddev":5.98},"decodeTokensPerSecondPerUser":{"mean":4.99,"stddev":2.01}},"sourceRun":"Test 05","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-26b-a4b-it/bf16/","workloadId":"large-context-25k","sourceLabel":"google/gemma-4-26B-A4B-it bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"05-big-context","prefixCaching":false},{"id":"gemma-4-26b-a4b-it-bf16-concurrency-stress-25k","date":"2026-08-16","runs":3,"metrics":{"ttfrSec":{"mean":77.388,"stddev":42.014},"decodeTokensPerSecondTotal":{"mean":28.66,"stddev":0.06},"prefillTokensPerSecondTotal":{"mean":3089.31,"stddev":1.87},"decodeTokensPerSecondPerUser":{"mean":2.89,"stddev":1.38}},"sourceRun":"Test 06","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-26b-a4b-it/bf16/","workloadId":"concurrency-stress-25k","sourceLabel":"google/gemma-4-26B-A4B-it bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"06-concurrency-stress","prefixCaching":false}],"openLoopResults":[{"id":"gemma-4-26b-a4b-it-bf16-office-random-4k","date":"2026-08-16","metrics":{"e2eMs":{"p50":93331.53,"p90":161293.3,"p95":178313.05,"p99":187416.14,"mean":92922.93},"tpotMs":{"p50":192.04,"p90":212.64,"p95":218.26,"p99":231.23,"mean":191.17},"ttftMs":{"p50":1434.65,"p90":2573.07,"p95":3077.46,"p99":3914.26,"mean":1577.96}},"sourceRun":"Test 07","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-26b-a4b-it/bf16/","scenarioId":"office-random-4k","achievedRps":0.267,"sourceStatus":"complete","sourceTestId":"07-office-baseline","configuredRps":0.3,"prefixCaching":false,"totalRequests":200,"successfulRequests":200,"totalTokenThroughput":1216.5,"peakConcurrentRequests":39},{"id":"gemma-4-26b-a4b-it-bf16-sharegpt-replay","date":"2026-08-16","metrics":{"e2eMs":{"p50":10769.47,"p90":49908.92,"p95":66398.36,"p99":88473.83,"mean":19941.21},"tpotMs":{"p50":96.44,"p90":122.94,"p95":128.57,"p99":138.76,"mean":94.55},"ttftMs":{"p50":364.13,"p90":474.15,"p95":503.97,"p99":585.64,"mean":366.79}},"sourceRun":"Test 08","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-26b-a4b-it/bf16/","scenarioId":"sharegpt-replay","achievedRps":0.298,"sourceStatus":"complete","sourceTestId":"08-sharegpt","configuredRps":0.3,"prefixCaching":false,"totalRequests":250,"successfulRequests":250,"totalTokenThroughput":132.95,"peakConcurrentRequests":17},{"id":"gemma-4-26b-a4b-it-bf16-reasoning","date":"2026-08-16","metrics":{"e2eMs":{"p50":617338.29,"p90":913321.5,"p95":984681.84,"p99":1017991.01,"mean":618158.72},"tpotMs":{"p50":150.81,"p90":164.8,"p95":167.84,"p99":171.72,"mean":150.8},"ttftMs":{"p50":601.74,"p90":755.19,"p95":784.19,"p99":871.84,"mean":589.45}},"sourceRun":"Test 09","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-26b-a4b-it/bf16/","scenarioId":"reasoning","achievedRps":0.044,"sourceStatus":"complete","sourceTestId":"09-reasoning","configuredRps":0.2,"prefixCaching":false,"totalRequests":50,"successfulRequests":50,"totalTokenThroughput":233.52,"peakConcurrentRequests":49},{"id":"gemma-4-26b-a4b-it-bf16-monday-peak-random-4k","date":"2026-08-16","metrics":{"e2eMs":{"p50":97410.01,"p90":175236.03,"p95":181692.7,"p99":190596.89,"mean":98016.19},"tpotMs":{"p50":196.89,"p90":205.91,"p95":212.63,"p99":236.24,"mean":195.28},"ttftMs":{"p50":1245.58,"p90":2193.67,"p95":3216.96,"p99":7778.11,"mean":1521.15}},"sourceRun":"Test 10","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-26b-a4b-it/bf16/","scenarioId":"monday-peak-random-4k","achievedRps":0.247,"sourceStatus":"complete","sourceTestId":"10-monday-peak","configuredRps":1.5,"prefixCaching":false,"totalRequests":300,"successfulRequests":300,"totalTokenThroughput":1111.38,"peakConcurrentRequests":27}],"rateSweep":{"date":"2026-08-16","capacity":{"slo2000":null,"slo5000":{"outputTps":209.7,"ttftP95Ms":4888.5,"achievedRps":0.429,"configuredRps":0.7,"peakConcurrent":129},"slo10000":{"outputTps":261.4,"ttftP95Ms":7627.2,"achievedRps":0.531,"configuredRps":1,"peakConcurrent":221}},"sourceRun":"Test 11","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-26b-a4b-it/bf16/","abortedAtRps":null,"queueKneeRps":null,"sourceStatus":"complete","sourceTestId":"11-rate-sweep","totalRequests":850,"successfulRequests":850}}