{"name":"Gemma-4 E2B","arena":{"id":"gemma-4-e2b-it-bf16","name":"Gemma-4 E2B","vram":5,"params":2.3,"vendor":"google","quality":{"coding":{"bench":"LiveCodeBench v6","value":44},"science":{"bench":"GPQA-Diamond","value":43.4},"knowledge":{"bench":"MMLU-Pro","value":60}},"contextK":128},"notes":"Model-ID: google/gemma-4-E2B-it. Container: vllm/vllm-openai:v0.26.0-aarch64-cu129-ubuntu2404. llama-benchy 0.4.0. Bronrun: 2026-08-17T05:44:25+02:00. KV-cache fp8; max_model_len 131072; gpu_memory_utilization 0,9. Prefix caching uit; async scheduling aan. Extra serverflags: --max-num-batched-tokens 8192. 11 van de 11 tests hebben een meting die volgens de huidige Arena-regels meetelt. De sanity-check is geslaagd. Dit is een controle op een kort gegenereerd antwoord, geen inhoudelijke evaluatie van de benchmarktaken.","order":70,"engine":"vLLM v0.26.0","verdict":"Gemeten op 2026-08-17 met vLLM v0.26.0. In de chattest met tien gelijktijdige verzoeken haalt dit profiel 43,34 tokens/s per gebruiker, met gemiddeld 0,52 seconden tot het eerste token. Bij 25k context stijgt die gemiddelde wachttijd naar 22,58 seconden. Deze runs meten snelheid en wachttijd, niet de kwaliteit van de antwoorden.","hardware":"DGX Spark, NVIDIA GB10, 128 GB unified memory","modelUrl":"https://huggingface.co/google/gemma-4-E2B-it","provider":"google","precision":"BF16","parameters":"2.3B effectief","provenance":{"modelId":"google/gemma-4-E2B-it","runPath":"results/gemma-4/gemma-4-e2b-it/bf16","validity":{"tests":{"01-chat":"complete","02-rag-8k":"complete","08-sharegpt":"complete","09-reasoning":"complete","04-multi-turn":"complete","11-rate-sweep":"complete","03-long-output":"complete","05-big-context":"complete","10-monday-peak":"complete","07-office-baseline":"complete","06-concurrency-stress":"complete"},"sanity":"passed"},"sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-e2b-it/bf16/","servedName":"gemma-4-e2b-it-bf16","generatedAt":"2026-08-17T05:44:25+02:00","runRevision":"a948cec23b054c62fef595bad3cdb284066922ac","vllmVersion":"0.26.0","suiteVersion":"2026-08","repositoryUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks","contractVersion":1,"contractRevision":"a948cec23b054c62fef595bad3cdb284066922ac","commandProvenance":"reconstructed_from_raw_artifacts","llamaBenchyVersion":"0.4.0"},"architecture":"Dense","measuredContextK":128,"benchmarkResults":[{"id":"gemma-4-e2b-it-bf16-chat-1k","date":"2026-08-17","runs":3,"metrics":{"ttfrSec":{"mean":0.515,"stddev":0.226},"decodeTokensPerSecondTotal":{"mean":254.44,"stddev":13.19},"prefillTokensPerSecondTotal":{"mean":12509.86,"stddev":402.28},"decodeTokensPerSecondPerUser":{"mean":43.34,"stddev":0.84}},"sourceRun":"Test 01","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-e2b-it/bf16/","workloadId":"chat-1k","sourceLabel":"google/gemma-4-E2B-it bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"01-chat","prefixCaching":false},{"id":"gemma-4-e2b-it-bf16-rag-8k","date":"2026-08-17","runs":3,"metrics":{"ttfrSec":{"mean":4.32,"stddev":2.204},"decodeTokensPerSecondTotal":{"mean":261.38,"stddev":0.65},"prefillTokensPerSecondTotal":{"mean":10037.79,"stddev":31.47},"decodeTokensPerSecondPerUser":{"mean":32.69,"stddev":4.43}},"sourceRun":"Test 02","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-e2b-it/bf16/","workloadId":"rag-8k","sourceLabel":"google/gemma-4-E2B-it bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"02-rag-8k","prefixCaching":false},{"id":"gemma-4-e2b-it-bf16-long-output-agents","date":"2026-08-17","runs":3,"metrics":{"ttfrSec":{"mean":0.2,"stddev":0.054},"decodeTokensPerSecondTotal":{"mean":268.1,"stddev":15.79},"prefillTokensPerSecondTotal":{"mean":10433.13,"stddev":1211.97},"decodeTokensPerSecondPerUser":{"mean":44.52,"stddev":0.37}},"sourceRun":"Test 03","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-e2b-it/bf16/","workloadId":"long-output-agents","sourceLabel":"google/gemma-4-E2B-it bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"03-long-output","prefixCaching":false},{"id":"gemma-4-e2b-it-bf16-multi-turn-office","date":"2026-08-17","runs":3,"metrics":{"ttfrSec":{"mean":1.005,"stddev":0.411},"decodeTokensPerSecondTotal":{"mean":374.54,"stddev":17.13},"prefillTokensPerSecondTotal":{"mean":12823.88,"stddev":49.86},"decodeTokensPerSecondPerUser":{"mean":41.67,"stddev":1.35}},"sourceRun":"Test 04","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-e2b-it/bf16/","workloadId":"multi-turn-office","sourceLabel":"google/gemma-4-E2B-it bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"04-multi-turn","prefixCaching":false},{"id":"gemma-4-e2b-it-bf16-large-context-25k","date":"2026-08-17","runs":3,"metrics":{"ttfrSec":{"mean":22.582,"stddev":11.601},"decodeTokensPerSecondTotal":{"mean":56.78,"stddev":1.59},"prefillTokensPerSecondTotal":{"mean":5577.71,"stddev":5.2},"decodeTokensPerSecondPerUser":{"mean":13.13,"stddev":8.23}},"sourceRun":"Test 05","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-e2b-it/bf16/","workloadId":"large-context-25k","sourceLabel":"google/gemma-4-E2B-it bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"05-big-context","prefixCaching":false},{"id":"gemma-4-e2b-it-bf16-concurrency-stress-25k","date":"2026-08-17","runs":3,"metrics":{"ttfrSec":{"mean":42.854,"stddev":23.256},"decodeTokensPerSecondTotal":{"mean":58.85,"stddev":0.13},"prefillTokensPerSecondTotal":{"mean":5588.88,"stddev":3.44},"decodeTokensPerSecondPerUser":{"mean":7.71,"stddev":5.73}},"sourceRun":"Test 06","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-e2b-it/bf16/","workloadId":"concurrency-stress-25k","sourceLabel":"google/gemma-4-E2B-it bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"06-concurrency-stress","prefixCaching":false}],"openLoopResults":[{"id":"gemma-4-e2b-it-bf16-office-random-4k","date":"2026-08-17","metrics":{"e2eMs":{"p50":13193.71,"p90":22897.79,"p95":24284.34,"p99":26549.29,"mean":13316.07},"tpotMs":{"p50":26.12,"p90":29.23,"p95":30.33,"p99":35.46,"mean":26.32},"ttftMs":{"p50":493.22,"p90":991.46,"p95":1158.2,"p99":1560.19,"mean":563.98}},"sourceRun":"Test 07","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-e2b-it/bf16/","scenarioId":"office-random-4k","achievedRps":0.295,"sourceStatus":"complete","sourceTestId":"07-office-baseline","configuredRps":0.3,"prefixCaching":false,"totalRequests":200,"successfulRequests":200,"totalTokenThroughput":1346.2,"peakConcurrentRequests":11},{"id":"gemma-4-e2b-it-bf16-sharegpt-replay","date":"2026-08-17","metrics":{"e2eMs":{"p50":2933.79,"p90":11475.97,"p95":15264.77,"p99":19457.4,"mean":4736.74},"tpotMs":{"p50":22.22,"p90":24.94,"p95":25.11,"p99":25.32,"mean":22.77},"ttftMs":{"p50":90.04,"p90":148.03,"p95":170.38,"p99":334.15,"mean":99.67}},"sourceRun":"Test 08","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-e2b-it/bf16/","scenarioId":"sharegpt-replay","achievedRps":0.3,"sourceStatus":"complete","sourceTestId":"08-sharegpt","configuredRps":0.3,"prefixCaching":false,"totalRequests":250,"successfulRequests":250,"totalTokenThroughput":132.66,"peakConcurrentRequests":7},{"id":"gemma-4-e2b-it-bf16-reasoning","date":"2026-08-17","metrics":{"e2eMs":{"p50":111590.02,"p90":186593.49,"p95":204226.39,"p99":213956.09,"mean":116979.05},"tpotMs":{"p50":28.45,"p90":29.76,"p95":29.92,"p99":30.07,"mean":27.86},"ttftMs":{"p50":187.35,"p90":243.25,"p95":250.21,"p99":279.51,"mean":186.91}},"sourceRun":"Test 09","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-e2b-it/bf16/","scenarioId":"reasoning","achievedRps":0.119,"sourceStatus":"complete","sourceTestId":"09-reasoning","configuredRps":0.2,"prefixCaching":false,"totalRequests":50,"successfulRequests":50,"totalTokenThroughput":631.93,"peakConcurrentRequests":25},{"id":"gemma-4-e2b-it-bf16-monday-peak-random-4k","date":"2026-08-17","metrics":{"e2eMs":{"p50":23047.89,"p90":42264.09,"p95":43954.13,"p99":46553.04,"mean":23658.19},"tpotMs":{"p50":46.88,"p90":51.2,"p95":54.21,"p99":61.58,"mean":46.52},"ttftMs":{"p50":504.38,"p90":941.3,"p95":1230.5,"p99":2328.63,"mean":578.93}},"sourceRun":"Test 10","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-e2b-it/bf16/","scenarioId":"monday-peak-random-4k","achievedRps":0.992,"sourceStatus":"complete","sourceTestId":"10-monday-peak","configuredRps":1.5,"prefixCaching":false,"totalRequests":300,"successfulRequests":300,"totalTokenThroughput":4468.21,"peakConcurrentRequests":29}],"rateSweep":{"date":"2026-08-17","capacity":{"slo2000":{"outputTps":451.8,"ttftP95Ms":1690.6,"achievedRps":0.918,"configuredRps":1,"peakConcurrent":35},"slo5000":{"outputTps":451.8,"ttftP95Ms":1690.6,"achievedRps":0.918,"configuredRps":1,"peakConcurrent":35},"slo10000":{"outputTps":451.8,"ttftP95Ms":1690.6,"achievedRps":0.918,"configuredRps":1,"peakConcurrent":35}},"sourceRun":"Test 11","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/gemma-4/gemma-4-e2b-it/bf16/","abortedAtRps":null,"queueKneeRps":null,"sourceStatus":"complete","sourceTestId":"11-rate-sweep","totalRequests":850,"successfulRequests":850}}