[ { "requests": 1, "repeats": 2, "completed": 2, "errors": 0, "peak_running": 1.0, "peak_waiting": 0.0, "preemptions": 0.0, "max_gpu_c": 51.0, "output_tps": 26.588402647201253, "output_tps_min": 26.09826346850626, "output_tps_max": 27.078541825896245, "median_decode_tps": 29.290833052007265, "median_decode_tps_min": 28.93071906739025, "median_decode_tps_max": 29.650947036624284, "median_ttft_s": 0.47919830299997557, "median_ttft_s_min": 0.4437084760011203, "median_ttft_s_max": 0.5146881299988308, "p95_ttft_s": 0.47919830299997557, "p95_ttft_s_min": 0.4437084760011203, "p95_ttft_s_max": 0.5146881299988308, "median_e2e_s": 4.815765504500632, "median_e2e_s_min": 4.726990132001447, "median_e2e_s_max": 4.9045408769998176, "p95_e2e_s": 4.815765504500632, "p95_e2e_s_min": 4.726990132001447, "p95_e2e_s_max": 4.9045408769998176 }, { "requests": 2, "repeats": 2, "completed": 4, "errors": 0, "peak_running": 2.0, "peak_waiting": 0.0, "preemptions": 0.0, "max_gpu_c": 54.0, "output_tps": 44.04508646154517, "output_tps_min": 42.019783418263685, "output_tps_max": 46.07038950482666, "median_decode_tps": 26.483594752299275, "median_decode_tps_min": 24.450841859673226, "median_decode_tps_max": 28.516347644925325, "median_ttft_s": 0.7319777445004547, "median_ttft_s_min": 0.7270799674997761, "median_ttft_s_max": 0.7368755215011333, "p95_ttft_s": 0.7912352758502039, "p95_ttft_s_min": 0.781304094049392, "p95_ttft_s_max": 0.8011664576510157, "median_e2e_s": 5.578561463750702, "median_e2e_s_min": 5.216498923500694, "median_e2e_s_max": 5.940624004000711, "p95_e2e_s": 5.799855170376122, "p95_e2e_s_min": 5.522598261451094, "p95_e2e_s_max": 6.07711207930115 }, { "requests": 4, "repeats": 2, "completed": 8, "errors": 0, "peak_running": 4.0, "peak_waiting": 0.0, "preemptions": 0.0, "max_gpu_c": 55.0, "output_tps": 65.72175667551623, "output_tps_min": 65.65175150434627, "output_tps_max": 65.7917618466862, "median_decode_tps": 18.779103911518135, "median_decode_tps_min": 18.544493923370627, "median_decode_tps_max": 19.013713899665643, "median_ttft_s": 0.9457729060009115, "median_ttft_s_min": 0.9436620735014003, "median_ttft_s_max": 0.9478837385004226, "p95_ttft_s": 0.9460862214257304, "p95_ttft_s_min": 0.9439887555003225, "p95_ttft_s_max": 0.9481836873511383, "median_e2e_s": 7.6731605530003435, "median_e2e_s_min": 7.623876460999782, "median_e2e_s_max": 7.722444645000905, "p95_e2e_s": 7.783148532950054, "p95_e2e_s_min": 7.76800597700003, "p95_e2e_s_max": 7.798291088900077 }, { "requests": 8, "repeats": 2, "completed": 16, "errors": 0, "peak_running": 8.0, "peak_waiting": 0.0, "preemptions": 0.0, "max_gpu_c": 60.0, "output_tps": 96.37335803604057, "output_tps_min": 94.86990010875091, "output_tps_max": 97.87681596333024, "median_decode_tps": 13.939070810502901, "median_decode_tps_min": 13.873033463294039, "median_decode_tps_max": 14.005108157711762, "median_ttft_s": 1.2267478735002442, "median_ttft_s_min": 1.2205320160001065, "median_ttft_s_max": 1.2329637310003818, "p95_ttft_s": 1.227887867401023, "p95_ttft_s_min": 1.2208270144013114, "p95_ttft_s_max": 1.2349487204007346, "median_e2e_s": 10.342395188999944, "median_e2e_s_min": 10.309196992499892, "median_e2e_s_max": 10.375593385499997, "p95_e2e_s": 10.61255840317508, "p95_e2e_s_min": 10.461497604250326, "p95_e2e_s_max": 10.763619202099836 }, { "requests": 16, "repeats": 2, "completed": 32, "errors": 0, "peak_running": 14.0, "peak_waiting": 2.0, "preemptions": 0.0, "max_gpu_c": 63.0, "output_tps": 107.54696366002945, "output_tps_min": 106.54098859703478, "output_tps_max": 108.55293872302411, "median_decode_tps": 10.807145415867973, "median_decode_tps_min": 10.67317624417635, "median_decode_tps_max": 10.941114587559596, "median_ttft_s": 1.6153679170001851, "median_ttft_s_min": 1.6133994940000775, "median_ttft_s_max": 1.6173363400002927, "p95_ttft_s": 13.00677905575003, "p95_ttft_s_min": 12.56879140000001, "p95_ttft_s_max": 13.444766711500051, "median_e2e_s": 13.678209915749903, "median_e2e_s_min": 13.443980604999524, "median_e2e_s_max": 13.912439226500283, "p95_e2e_s": 18.922476432249823, "p95_e2e_s_min": 18.776438152000082, "p95_e2e_s_max": 19.068514712499564 }, { "requests": 32, "repeats": 2, "completed": 64, "errors": 0, "peak_running": 14.0, "peak_waiting": 18.0, "preemptions": 0.0, "max_gpu_c": 63.0, "output_tps": 117.97337626496434, "output_tps_min": 117.78516319941892, "output_tps_max": 118.16158933050977, "median_decode_tps": 10.30230475631996, "median_decode_tps_min": 10.277537503134024, "median_decode_tps_max": 10.327072009505898, "median_ttft_s": 13.307897222250176, "median_ttft_s_min": 13.106756688500354, "median_ttft_s_max": 13.509037755999998, "p95_ttft_s": 26.6005991702496, "p95_ttft_s_min": 26.26832793764952, "p95_ttft_s_max": 26.932870402849677, "median_e2e_s": 25.86491383249995, "median_e2e_s_min": 25.542022181999528, "median_e2e_s_max": 26.187805483000375, "p95_e2e_s": 34.336495081349085, "p95_e2e_s_min": 34.218068052598575, "p95_e2e_s_max": 34.454922110099595 }, { "requests": 64, "repeats": 2, "completed": 128, "errors": 0, "peak_running": 14.0, "peak_waiting": 50.0, "preemptions": 0.0, "max_gpu_c": 66.0, "output_tps": 121.10981430034633, "output_tps_min": 119.63468696347397, "output_tps_max": 122.58494163721869, "median_decode_tps": 9.79557996843394, "median_decode_tps_min": 9.709122703336831, "median_decode_tps_max": 9.882037233531047, "median_ttft_s": 27.85205950574982, "median_ttft_s_min": 27.522013117500137, "median_ttft_s_max": 28.182105893999505, "p95_ttft_s": 56.36821767379978, "p95_ttft_s_min": 55.50022509500004, "p95_ttft_s_max": 57.23621025259953, "median_e2e_s": 41.31077290124995, "median_e2e_s_min": 41.156980110500626, "median_e2e_s_max": 41.46456569199927, "p95_e2e_s": 66.78246951850005, "p95_e2e_s_min": 66.18887185730009, "p95_e2e_s_max": 67.37606717970002 } ]