{"name":"KAT-Coder V2.5","arena":{"id":"kat-coder-v2-5-bf16","name":"KAT-Coder V2.5","vram":70,"params":35,"vendor":"kwaipilot","quality":{"coding":null,"science":null,"knowledge":null},"contextK":256},"notes":"Model-ID: Kwaipilot/KAT-Coder-V2.5-Dev. Container: vllm/vllm-openai:v0.26.0-aarch64-cu129-ubuntu2404. llama-benchy 0.4.0. Bronrun: 2026-08-13T16:19:56+02:00. KV-cache fp8; max_model_len 131072; gpu_memory_utilization 0,9. Prefix caching uit; async scheduling aan. Extra serverflags: --language-model-only --max-num-batched-tokens 8192 --trust-remote-code. 11 van de 11 tests hebben een meting die volgens de huidige Arena-regels meetelt. De sanity-check is geslaagd. Dit is een controle op een kort gegenereerd antwoord, geen inhoudelijke evaluatie van de benchmarktaken.","order":110,"engine":"vLLM v0.26.0","verdict":"Gemeten op 2026-08-13 met vLLM v0.26.0. In de chattest met tien gelijktijdige verzoeken haalt dit profiel 10,44 tokens/s per gebruiker, met gemiddeld 2,01 seconden tot het eerste token. Bij 25k context stijgt die gemiddelde wachttijd naar 28,91 seconden. Deze runs meten snelheid en wachttijd, niet de kwaliteit van de antwoorden.","hardware":"DGX Spark, NVIDIA GB10, 128 GB unified memory","modelUrl":"https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev","provider":"Kwaipilot","precision":"BF16","parameters":"35B-A3B","provenance":{"modelId":"Kwaipilot/KAT-Coder-V2.5-Dev","runPath":"results/kat-coder/kat-coder-v2.5/bf16","validity":{"tests":{"01-chat":"complete","02-rag-8k":"complete","08-sharegpt":"complete","09-reasoning":"complete","04-multi-turn":"complete","11-rate-sweep":"complete","03-long-output":"complete","05-big-context":"complete","10-monday-peak":"complete","07-office-baseline":"complete","06-concurrency-stress":"complete"},"sanity":"passed"},"sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/kat-coder/kat-coder-v2.5/bf16/","servedName":"kat-coder-v2-5-bf16","generatedAt":"2026-08-13T16:19:56+02:00","runRevision":"a948cec23b054c62fef595bad3cdb284066922ac","vllmVersion":"0.26.0","suiteVersion":"2026-08","repositoryUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks","contractVersion":1,"contractRevision":"a948cec23b054c62fef595bad3cdb284066922ac","commandProvenance":"reconstructed_from_raw_artifacts","llamaBenchyVersion":"0.4.0"},"architecture":"MoE","measuredContextK":128,"benchmarkResults":[{"id":"kat-coder-v2-5-bf16-chat-1k","date":"2026-08-13","runs":3,"metrics":{"ttfrSec":{"mean":2.01,"stddev":0.419},"decodeTokensPerSecondTotal":{"mean":75.79,"stddev":3.47},"prefillTokensPerSecondTotal":{"mean":4020.47,"stddev":286.49},"decodeTokensPerSecondPerUser":{"mean":10.44,"stddev":1.72}},"sourceRun":"Test 01","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/kat-coder/kat-coder-v2.5/bf16/","workloadId":"chat-1k","sourceLabel":"Kwaipilot/KAT-Coder-V2.5-Dev bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"01-chat","prefixCaching":false},{"id":"kat-coder-v2-5-bf16-rag-8k","date":"2026-08-13","runs":3,"metrics":{"ttfrSec":{"mean":9.445,"stddev":4.405},"decodeTokensPerSecondTotal":{"mean":65.52,"stddev":2.84},"prefillTokensPerSecondTotal":{"mean":4884.36,"stddev":7.83},"decodeTokensPerSecondPerUser":{"mean":8.36,"stddev":0.88}},"sourceRun":"Test 02","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/kat-coder/kat-coder-v2.5/bf16/","workloadId":"rag-8k","sourceLabel":"Kwaipilot/KAT-Coder-V2.5-Dev bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"02-rag-8k","prefixCaching":false},{"id":"kat-coder-v2-5-bf16-long-output-agents","date":"2026-08-13","runs":3,"metrics":{"ttfrSec":{"mean":0.739,"stddev":0.123},"decodeTokensPerSecondTotal":{"mean":65.18,"stddev":9.93},"prefillTokensPerSecondTotal":{"mean":3078.71,"stddev":409.81},"decodeTokensPerSecondPerUser":{"mean":10.59,"stddev":2.98}},"sourceRun":"Test 03","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/kat-coder/kat-coder-v2.5/bf16/","workloadId":"long-output-agents","sourceLabel":"Kwaipilot/KAT-Coder-V2.5-Dev bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"03-long-output","prefixCaching":false},{"id":"kat-coder-v2-5-bf16-multi-turn-office","date":"2026-08-13","runs":3,"metrics":{"ttfrSec":{"mean":3.071,"stddev":0.828},"decodeTokensPerSecondTotal":{"mean":77.92,"stddev":1.3},"prefillTokensPerSecondTotal":{"mean":4739.36,"stddev":8.94},"decodeTokensPerSecondPerUser":{"mean":9.36,"stddev":0.73}},"sourceRun":"Test 04","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/kat-coder/kat-coder-v2.5/bf16/","workloadId":"multi-turn-office","sourceLabel":"Kwaipilot/KAT-Coder-V2.5-Dev bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"04-multi-turn","prefixCaching":false},{"id":"kat-coder-v2-5-bf16-large-context-25k","date":"2026-08-13","runs":3,"metrics":{"ttfrSec":{"mean":28.909,"stddev":14.197},"decodeTokensPerSecondTotal":{"mean":34.01,"stddev":0.36},"prefillTokensPerSecondTotal":{"mean":4504.88,"stddev":10.84},"decodeTokensPerSecondPerUser":{"mean":5.43,"stddev":1.57}},"sourceRun":"Test 05","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/kat-coder/kat-coder-v2.5/bf16/","workloadId":"large-context-25k","sourceLabel":"Kwaipilot/KAT-Coder-V2.5-Dev bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"05-big-context","prefixCaching":false},{"id":"kat-coder-v2-5-bf16-concurrency-stress-25k","date":"2026-08-13","runs":3,"metrics":{"ttfrSec":{"mean":54.142,"stddev":28.975},"decodeTokensPerSecondTotal":{"mean":37.9,"stddev":0.38},"prefillTokensPerSecondTotal":{"mean":4482.25,"stddev":2.87},"decodeTokensPerSecondPerUser":{"mean":3.5,"stddev":1.36}},"sourceRun":"Test 06","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/kat-coder/kat-coder-v2.5/bf16/","workloadId":"concurrency-stress-25k","sourceLabel":"Kwaipilot/KAT-Coder-V2.5-Dev bf16 op de DGX Spark","sourceStatus":"complete","sourceTestId":"06-concurrency-stress","prefixCaching":false}],"openLoopResults":[{"id":"kat-coder-v2-5-bf16-office-random-4k","date":"2026-08-13","metrics":{"e2eMs":{"p50":112828.68,"p90":196340.77,"p95":210611.5,"p99":225783.5,"mean":111765.72},"tpotMs":{"p50":237.38,"p90":255.74,"p95":263.46,"p99":293.2,"mean":230.44},"ttftMs":{"p50":1616.75,"p90":2806.84,"p95":3254.51,"p99":4381.01,"mean":1786.24}},"sourceRun":"Test 07","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/kat-coder/kat-coder-v2.5/bf16/","scenarioId":"office-random-4k","achievedRps":0.258,"sourceStatus":"complete","sourceTestId":"07-office-baseline","configuredRps":0.3,"prefixCaching":false,"totalRequests":200,"successfulRequests":200,"totalTokenThroughput":1177.97,"peakConcurrentRequests":46},{"id":"kat-coder-v2-5-bf16-sharegpt-replay","date":"2026-08-13","metrics":{"e2eMs":{"p50":10005.5,"p90":48322.91,"p95":66161.45,"p99":83661.68,"mean":19142.66},"tpotMs":{"p50":87.28,"p90":130.59,"p95":132.36,"p99":143.13,"mean":90.33},"ttftMs":{"p50":386.45,"p90":539.95,"p95":574.37,"p99":727.63,"mean":386.42}},"sourceRun":"Test 08","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/kat-coder/kat-coder-v2.5/bf16/","scenarioId":"sharegpt-replay","achievedRps":0.298,"sourceStatus":"complete","sourceTestId":"08-sharegpt","configuredRps":0.3,"prefixCaching":false,"totalRequests":250,"successfulRequests":250,"totalTokenThroughput":129.7,"peakConcurrentRequests":16},{"id":"kat-coder-v2-5-bf16-reasoning","date":"2026-08-13","metrics":{"e2eMs":{"p50":819852.87,"p90":1173330.64,"p95":1260841.35,"p99":1297081.48,"mean":808741.63},"tpotMs":{"p50":197.56,"p90":222.89,"p95":226.82,"p99":231.46,"mean":198.58},"ttftMs":{"p50":711.6,"p90":934.59,"p95":1018.6,"p99":1073.72,"mean":706.55}},"sourceRun":"Test 09","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/kat-coder/kat-coder-v2.5/bf16/","scenarioId":"reasoning","achievedRps":0.036,"sourceStatus":"complete","sourceTestId":"09-reasoning","configuredRps":0.2,"prefixCaching":false,"totalRequests":50,"successfulRequests":50,"totalTokenThroughput":189.18,"peakConcurrentRequests":49},{"id":"kat-coder-v2-5-bf16-monday-peak-random-4k","date":"2026-08-13","metrics":{"e2eMs":{"p50":106410.66,"p90":192567.26,"p95":199585.98,"p99":206859.41,"mean":107947.95},"tpotMs":{"p50":215.93,"p90":225.81,"p95":232.9,"p99":243.11,"mean":214.54},"ttftMs":{"p50":1366.45,"p90":2424,"p95":3974.68,"p99":8934.38,"mean":1691.98}},"sourceRun":"Test 10","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/kat-coder/kat-coder-v2.5/bf16/","scenarioId":"monday-peak-random-4k","achievedRps":0.224,"sourceStatus":"complete","sourceTestId":"10-monday-peak","configuredRps":1.5,"prefixCaching":false,"totalRequests":300,"successfulRequests":300,"totalTokenThroughput":1008.46,"peakConcurrentRequests":27}],"rateSweep":{"date":"2026-08-13","capacity":{"slo2000":null,"slo5000":{"outputTps":156.6,"ttftP95Ms":4530.1,"achievedRps":0.32,"configuredRps":0.5,"peakConcurrent":84},"slo10000":{"outputTps":232.2,"ttftP95Ms":8228.3,"achievedRps":0.472,"configuredRps":1,"peakConcurrent":223}},"sourceRun":"Test 11","sourceUrl":"https://github.com/djangodevreng/dgx-spark-benchmarks/tree/a948cec23b054c62fef595bad3cdb284066922ac/results/kat-coder/kat-coder-v2.5/bf16/","abortedAtRps":null,"queueKneeRps":null,"sourceStatus":"complete","sourceTestId":"11-rate-sweep","totalRequests":850,"successfulRequests":850}}