diff --git a/atom_mtp_profile_tp4_conc64_isl8192_osl1024.json b/atom_mtp_profile_tp4_conc64_isl8192_osl1024.json new file mode 100644 index 0000000000..2e56ab482d --- /dev/null +++ b/atom_mtp_profile_tp4_conc64_isl8192_osl1024.json @@ -0,0 +1 @@ +{"date": "20260804-143300", "backend": "vllm", "model_id": "Qwen/Qwen3.5-397B-A17B-FP8", "tokenizer_id": "Qwen/Qwen3.5-397B-A17B-FP8", "best_of": 1, "num_prompts": 64, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 64, "duration": 24.09488018101547, "benchmark_start_time_unix": 1785853956.324791, "benchmark_end_time_unix": 1785853980.4196706, "completed": 64, "total_input_tokens": 468058, "total_output_tokens": 59194, "request_throughput": 2.6561659372942663, "request_goodput:": null, "output_throughput": 2456.704476440575, "total_token_throughput": 21882.32504328557, "input_lens": [7239, 7114, 7771, 7390, 7318, 7938, 7589, 6832, 7154, 7651, 8052, 7156, 7975, 6869, 7260, 8065, 7107, 6643, 6729, 8180, 7404, 7093, 7401, 6627, 7334, 6671, 7531, 7310, 8010, 7003, 7407, 6654, 7757, 7310, 7353, 7215, 7726, 7465, 6979, 7867, 7844, 7252, 7194, 7099, 7100, 7270, 6800, 6707, 7232, 7066, 8038, 7437, 7762, 6583, 7357, 6683, 6683, 7632, 8129, 8067, 7312, 6828, 6891, 6943], "output_lens": [921, 859, 1002, 848, 936, 818, 883, 988, 873, 852, 915, 938, 828, 990, 898, 906, 945, 958, 915, 868, 958, 938, 985, 900, 1018, 886, 822, 1012, 862, 943, 944, 1022, 915, 997, 894, 957, 965, 1005, 838, 957, 931, 903, 864, 867, 887, 987, 979, 1008, 912, 1012, 913, 818, 927, 993, 851, 978, 865, 909, 949, 911, 858, 1019, 928, 966], "mean_ttft_ms": 5923.282071953508, "median_ttft_ms": 5962.607782508712, "std_ttft_ms": 3115.111853373601, "p90_ttft_ms": 10192.873542697635, "p99_ttft_ms": 11063.288381312741, "p99.9_ttft_ms": 11139.515923125555, "mean_tpot_ms": 17.34632734162186, "median_tpot_ms": 17.390453564193386, "std_tpot_ms": 3.2813547115043504, "p90_tpot_ms": 21.464147023195117, "p99_tpot_ms": 23.255281597324426, "p99.9_tpot_ms": 23.340276868853895, "mean_itl_ms": 50.101673735046504, "median_itl_ms": 34.97572604101151, "std_itl_ms": 339.9492786601427, "p90_itl_ms": 35.71102498099208, "p99_itl_ms": 37.72934671025723, "p99.9_itl_ms": 7162.339345548005, "mean_e2el_ms": 21919.023928949173, "median_e2el_ms": 21965.35529301036, "std_e2el_ms": 895.3565500753016, "p90_e2el_ms": 22911.674276134, "p99_e2el_ms": 23497.431531402977, "p99.9_e2el_ms": 24012.194940689373} \ No newline at end of file diff --git a/atom_mtp_tp4_conc64_isl8192_osl1024.json b/atom_mtp_tp4_conc64_isl8192_osl1024.json new file mode 100644 index 0000000000..f46395fe68 --- /dev/null +++ b/atom_mtp_tp4_conc64_isl8192_osl1024.json @@ -0,0 +1 @@ +{"date": "20260806-132830", "backend": "vllm", "model_id": "Qwen/Qwen3.5-397B-A17B-FP8", "tokenizer_id": "Qwen/Qwen3.5-397B-A17B-FP8", "best_of": 1, "num_prompts": 640, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 64, "duration": 315.2482071389677, "benchmark_start_time_unix": 1786022594.990249, "benchmark_end_time_unix": 1786022910.2384567, "completed": 640, "total_input_tokens": 4729077, "total_output_tokens": 588102, "request_throughput": 2.030146359303085, "request_goodput:": null, "output_throughput": 1865.520522185723, "total_token_throughput": 16866.643107207525, "input_lens": [7239, 7114, 7771, 7390, 7318, 7938, 7588, 6832, 7155, 7649, 8051, 7155, 7975, 6869, 7260, 8066, 7108, 6642, 6731, 8180, 7405, 7092, 7400, 6627, 7332, 6670, 7533, 7311, 8010, 7003, 7405, 6654, 7756, 7311, 7353, 7215, 7727, 7468, 6979, 7868, 7844, 7252, 7194, 7099, 7098, 7268, 6800, 6706, 7230, 7065, 8039, 7438, 7763, 6582, 7358, 6683, 6684, 7634, 8129, 8067, 7312, 6828, 6891, 6943, 7621, 8021, 7098, 7443, 6813, 7901, 6614, 6846, 7698, 7334, 6985, 7662, 6646, 6954, 7166, 8144, 7463, 7190, 6639, 6759, 6879, 7329, 7519, 7627, 7194, 7711, 7527, 7423, 7760, 7723, 7806, 7510, 7346, 7277, 7465, 6928, 7115, 7884, 8161, 6724, 8027, 7674, 7776, 8185, 6812, 7871, 7371, 6942, 8189, 8134, 7528, 6923, 7473, 6756, 6939, 8091, 6925, 7111, 7509, 7962, 6578, 6685, 6932, 6654, 8153, 7486, 7289, 6678, 7149, 7294, 7727, 6764, 7117, 7991, 7337, 7621, 7637, 7260, 7615, 7334, 6641, 7623, 7429, 6566, 7325, 6864, 7659, 6587, 7761, 7708, 7361, 6830, 7753, 8133, 7950, 7763, 6999, 7043, 8169, 6841, 7859, 7192, 7325, 8094, 6650, 7805, 7848, 6853, 7907, 7599, 7162, 7989, 7007, 7827, 6735, 6878, 7559, 7839, 7304, 6694, 7576, 7153, 8041, 7445, 7398, 7252, 8108, 7731, 7481, 6704, 7689, 6580, 8044, 7512, 7441, 7696, 8025, 8174, 7228, 7483, 7807, 8086, 7318, 6677, 7906, 7024, 6842, 7080, 7731, 7147, 6596, 6831, 8131, 8017, 6762, 7335, 6721, 6667, 7929, 7708, 7291, 7367, 6771, 7960, 6579, 6624, 7534, 7563, 6789, 6847, 7578, 7256, 7282, 7112, 6587, 7590, 6768, 7717, 8000, 7178, 8068, 7326, 8118, 7796, 7218, 7376, 6862, 7753, 7728, 7504, 8120, 8158, 7103, 7363, 7972, 6628, 6852, 7475, 7285, 7432, 7334, 7973, 8118, 6985, 6754, 8002, 7641, 6896, 7420, 7623, 7357, 7667, 6588, 7200, 6787, 7322, 6592, 8039, 6846, 6752, 7450, 7834, 7659, 8026, 6691, 6744, 6684, 6764, 7947, 6846, 7957, 7926, 6724, 7360, 6731, 8116, 6878, 7063, 7358, 7693, 7608, 7868, 6701, 7852, 6776, 6896, 7064, 7070, 6625, 7936, 7464, 7112, 7441, 7320, 7902, 6646, 7921, 6979, 8130, 7993, 7266, 7523, 6829, 7415, 6598, 7663, 6989, 7364, 6649, 7754, 7984, 6756, 7053, 7035, 6885, 8119, 6764, 7172, 8154, 7595, 6661, 6719, 7161, 8115, 7184, 7771, 7774, 7886, 6693, 7508, 7414, 7674, 7305, 7668, 6716, 6702, 7141, 7010, 7667, 7592, 7893, 8172, 7700, 7506, 7387, 7270, 7287, 7487, 6836, 7356, 8005, 7026, 7863, 7858, 7903, 8113, 7806, 7446, 7151, 7203, 7128, 7209, 8097, 6813, 7715, 7786, 7018, 7718, 7583, 6978, 6904, 7496, 7861, 7141, 7561, 8012, 7798, 7375, 7294, 7430, 7342, 7952, 7869, 8142, 6994, 8087, 7575, 7341, 7497, 7410, 7004, 7905, 8069, 7632, 7371, 7674, 6614, 6891, 6718, 8149, 7501, 7687, 6756, 7879, 7444, 7105, 7592, 7459, 7571, 7090, 7059, 7788, 8034, 7023, 8052, 8152, 6955, 7635, 7438, 7219, 6835, 7583, 7167, 7559, 6860, 6898, 8122, 8145, 6710, 7932, 6912, 7254, 6962, 7990, 7608, 7484, 7950, 7188, 7503, 7410, 7057, 6562, 7172, 7564, 6693, 7249, 7406, 7956, 7487, 7728, 7564, 7839, 6682, 7132, 8111, 7360, 7904, 7531, 6667, 7206, 6792, 7939, 7545, 8134, 6618, 7009, 7845, 8024, 7158, 7179, 7998, 6894, 7853, 7948, 6781, 7804, 7463, 7808, 6924, 7707, 7363, 7603, 7253, 6847, 6911, 7392, 6690, 6558, 7806, 7444, 7235, 6999, 7998, 7865, 7391, 6584, 7293, 6803, 6920, 7781, 8016, 6752, 7485, 7192, 7162, 6628, 6804, 7407, 7714, 6769, 8054, 7903, 6657, 6883, 7150, 7802, 6812, 7926, 7686, 7481, 8098, 7728, 8056, 7971, 8137, 6557, 7939, 7246, 8129, 7668, 7653, 7054, 7293, 7190, 6898, 7675, 7406, 6837, 7274, 6588, 7695, 7009, 6984, 7384, 7245, 6728, 7712, 8020, 6648, 7927, 7981, 6974, 6959, 7497, 7765, 7531, 6930, 7257, 8095, 7052, 8077, 7784, 7993, 7944, 7294, 6974, 7141, 7797, 8117, 7953, 7602, 7017, 7275, 7933, 8182, 7630, 7553, 8152, 8023, 7370, 6659, 7196, 7973, 6882, 7163, 8097, 7824, 7993, 8106, 6724, 6722, 7888, 7494, 7149, 7970, 7799, 7388, 7389, 7694, 7415, 6889, 7296, 7799, 7825, 6655, 6971, 6776, 7757, 8087, 6730, 7160, 7335, 7403, 7645, 8036, 7566], "output_lens": [941, 954, 925, 970, 999, 934, 882, 1010, 1011, 905, 850, 819, 1005, 981, 984, 905, 884, 964, 947, 844, 932, 950, 881, 906, 918, 917, 938, 1013, 924, 890, 917, 995, 892, 966, 976, 894, 849, 953, 842, 838, 990, 961, 969, 936, 980, 945, 948, 949, 1012, 954, 908, 890, 998, 896, 878, 889, 1017, 863, 950, 866, 1002, 987, 1013, 956, 871, 903, 948, 941, 899, 1012, 1000, 949, 830, 1013, 1008, 911, 970, 948, 885, 984, 982, 875, 996, 996, 965, 979, 962, 914, 952, 996, 863, 944, 979, 894, 976, 827, 934, 864, 934, 850, 851, 930, 900, 939, 968, 987, 924, 869, 924, 878, 892, 934, 891, 960, 821, 823, 948, 864, 892, 855, 935, 865, 925, 996, 866, 889, 908, 991, 844, 818, 891, 903, 883, 935, 834, 858, 846, 971, 903, 996, 865, 1007, 917, 825, 818, 932, 924, 904, 941, 933, 914, 907, 883, 922, 829, 827, 922, 983, 985, 979, 938, 849, 971, 912, 867, 904, 836, 975, 888, 835, 968, 966, 855, 941, 844, 969, 961, 1011, 985, 861, 872, 814, 954, 1010, 947, 912, 1000, 1014, 963, 979, 868, 918, 837, 989, 931, 942, 925, 920, 1008, 957, 831, 931, 848, 970, 954, 952, 854, 816, 826, 824, 941, 823, 919, 982, 891, 930, 902, 945, 931, 996, 864, 818, 895, 973, 826, 931, 1015, 952, 933, 994, 1008, 856, 893, 879, 893, 860, 895, 963, 939, 835, 904, 830, 907, 922, 820, 916, 864, 854, 970, 971, 947, 962, 1007, 854, 877, 932, 910, 913, 850, 878, 954, 953, 909, 878, 971, 909, 967, 962, 949, 852, 896, 941, 820, 911, 1021, 969, 902, 955, 950, 998, 898, 974, 816, 887, 947, 878, 885, 961, 928, 1017, 899, 933, 872, 878, 1002, 864, 964, 931, 999, 825, 1005, 893, 976, 948, 960, 887, 880, 840, 906, 926, 826, 837, 913, 975, 899, 973, 863, 931, 830, 893, 848, 906, 849, 872, 894, 980, 888, 923, 872, 927, 978, 883, 994, 829, 919, 835, 926, 824, 912, 1013, 942, 919, 945, 973, 948, 986, 965, 849, 837, 820, 964, 922, 890, 939, 1011, 875, 911, 987, 855, 1003, 996, 856, 875, 846, 910, 979, 942, 830, 918, 981, 976, 880, 838, 1016, 944, 906, 850, 927, 1009, 967, 902, 947, 915, 858, 961, 928, 862, 927, 1013, 935, 963, 840, 925, 954, 826, 975, 951, 935, 842, 864, 1015, 999, 910, 918, 900, 903, 964, 819, 822, 827, 822, 854, 985, 917, 1009, 933, 878, 979, 950, 822, 855, 981, 906, 994, 902, 932, 824, 951, 993, 838, 911, 944, 898, 947, 892, 827, 958, 897, 970, 1014, 990, 916, 836, 965, 986, 938, 999, 962, 916, 997, 917, 1009, 927, 827, 912, 962, 930, 826, 843, 914, 914, 862, 822, 905, 904, 857, 868, 831, 969, 821, 908, 991, 938, 973, 817, 951, 937, 1015, 833, 891, 982, 998, 920, 934, 822, 994, 918, 996, 975, 997, 930, 954, 1011, 1020, 936, 871, 905, 882, 933, 931, 1004, 911, 917, 956, 899, 816, 981, 948, 846, 895, 830, 1018, 829, 832, 878, 938, 922, 1005, 1005, 945, 962, 997, 847, 1009, 904, 887, 950, 957, 932, 994, 895, 999, 831, 947, 925, 843, 873, 894, 927, 879, 890, 829, 842, 920, 826, 906, 977, 946, 959, 820, 833, 852, 849, 989, 853, 887, 1019, 922, 900, 977, 967, 884, 974, 969, 882, 850, 892, 900, 833, 867, 957, 989, 826, 919, 895, 888, 837, 1012, 893, 877, 945, 866, 912, 922, 900, 1002, 964, 831, 957, 841, 887, 845, 876, 873, 1020, 831, 961, 896, 835, 888, 1000, 863, 907, 861, 833, 903, 1018, 894, 973, 911, 924, 947, 843, 1007, 866, 992, 902, 875, 856, 1012, 989, 831, 839, 983, 879, 951, 964, 896, 893, 979, 986, 980, 968, 852, 932, 975, 987, 942, 861, 914], "mean_ttft_ms": 968.4443868594826, "median_ttft_ms": 398.7352500553243, "std_ttft_ms": 1877.6002546180484, "p90_ttft_ms": 1280.1106976694427, "p99_ttft_ms": 9947.031967156803, "p99.9_ttft_ms": 10884.661080989888, "mean_tpot_ms": 32.20908514635708, "median_tpot_ms": 33.014678370962265, "std_tpot_ms": 3.925546677365811, "p90_tpot_ms": 35.92645567022859, "p99_tpot_ms": 41.355680648257774, "p99.9_tpot_ms": 44.4908373453661, "mean_itl_ms": 92.76455600699641, "median_itl_ms": 59.89688995759934, "std_itl_ms": 143.83485008943575, "p90_itl_ms": 219.5464699761942, "p99_itl_ms": 564.3211746169254, "p99.9_itl_ms": 886.8740316061303, "mean_e2el_ms": 30556.423812185494, "median_e2el_ms": 30514.44219693076, "std_e2el_ms": 4531.694859479395, "p90_e2el_ms": 35588.647377025336, "p99_e2el_ms": 42845.612753537716, "p99.9_e2el_ms": 48260.93381277942} \ No newline at end of file diff --git a/benchmarks/multi_node/amd_utils/docker_sg.sh b/benchmarks/multi_node/amd_utils/docker_sg.sh new file mode 100755 index 0000000000..575c6a74a8 --- /dev/null +++ b/benchmarks/multi_node/amd_utils/docker_sg.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Run `docker` under the 'docker' group. +# +# Why: Slurm launches job steps without the user's 'docker' supplementary group +# in the active credential set. The user IS a docker-group member (getent group +# docker lists them) but the group is missing from `id -G` inside the step, so +# the group-owned socket (/var/run/docker.sock, 0660 root:docker) is unreachable +# via a plain `docker` call. `sg docker -c` re-activates the group for this one +# command — no sudo, no password, no persistent host change. +# +# Used by job.slurm's DOCKER_CMD detection as the fallback when plain `docker` +# fails but `sg docker -c 'docker ps'` succeeds. +# +# argv is passed across the `sg` shell hop by NUL-delimited base64 (NOT string +# re-quoting): naive `printf %q` mangles the big multiline `docker run ... bash +# -lc '