From 012eb3bef05ca5ef5defe324aeecf7ac66bc0191 Mon Sep 17 00:00:00 2001 From: Zimin Li Date: Wed, 24 Jun 2026 03:21:16 +0000 Subject: [PATCH 1/4] refactor: change the directory structure of `examples/` to categorize different types of examples and update `scripts/run_examples.py` - restructure the `examples/` to categorize different types of examples - update `scripts/run_examples.py` to accommodate this change and be able to specify an example directory, specific examples, or fuzzy example names --- examples/CMakeLists.txt | 42 ++++++--- examples/{ => mpi}/all_gather.cc | 0 examples/{ => mpi}/all_reduce.cc | 0 examples/{ => mpi}/all_to_all.cc | 0 examples/{ => mpi}/broadcast.cc | 0 examples/{ => mpi}/gather.cc | 0 examples/{ => mpi}/reduce.cc | 0 examples/{ => mpi}/reduce_scatter.cc | 0 examples/{ => mpi}/scatter.cc | 0 examples/{ => mpi}/send_recv.cc | 0 scripts/run_examples.py | 132 ++++++++++++++++++++------- 11 files changed, 125 insertions(+), 49 deletions(-) rename examples/{ => mpi}/all_gather.cc (100%) rename examples/{ => mpi}/all_reduce.cc (100%) rename examples/{ => mpi}/all_to_all.cc (100%) rename examples/{ => mpi}/broadcast.cc (100%) rename examples/{ => mpi}/gather.cc (100%) rename examples/{ => mpi}/reduce.cc (100%) rename examples/{ => mpi}/reduce_scatter.cc (100%) rename examples/{ => mpi}/scatter.cc (100%) rename examples/{ => mpi}/send_recv.cc (100%) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 168f008..a5eeed2 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,47 +1,61 @@ file(GLOB_RECURSE EXAMPLE_SOURCES "*.cc") foreach(source_file ${EXAMPLE_SOURCES}) - get_filename_component(example_name ${source_file} NAME_WE) + file(RELATIVE_PATH rel_file "${CMAKE_CURRENT_SOURCE_DIR}" "${source_file}") - add_executable(${example_name} ${source_file}) + get_filename_component(file_dir ${rel_file} DIRECTORY) + get_filename_component(file_we ${rel_file} NAME_WE) - target_link_libraries(${example_name} PRIVATE infiniccl) + if(file_dir) + string(REPLACE "/" "_" target_name "${file_dir}_${file_we}") + else() + set(target_name ${file_we}) + endif() + + add_executable(${target_name} ${source_file}) + + set_target_properties(${target_name} PROPERTIES + OUTPUT_NAME "${file_we}" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${file_dir}" + ) + + target_link_libraries(${target_name} PRIVATE infiniccl) # Add runtime and backend dependencies for direct runtime/backend usage. if(WITH_NVIDIA) - target_link_libraries(${example_name} PRIVATE CUDA::cudart) + target_link_libraries(${target_name} PRIVATE CUDA::cudart) endif() if(WITH_ILUVATAR) set_source_files_properties(${source_file} PROPERTIES LANGUAGE CXX) - set_target_properties(${example_name} PROPERTIES + set_target_properties(${target_name} PROPERTIES RULE_LAUNCH_COMPILE "${ILUVATAR_CUDA_COMPILER} " ) - target_compile_options(${example_name} PRIVATE ${ILUVATAR_CUDA_FLAGS} "-Wno-unused-command-line-argument") - target_link_libraries(${example_name} PRIVATE CUDA::cudart CUDA::cuda_driver) + target_compile_options(${target_name} PRIVATE ${ILUVATAR_CUDA_FLAGS} "-Wno-unused-command-line-argument") + target_link_libraries(${target_name} PRIVATE CUDA::cudart CUDA::cuda_driver) endif() if(WITH_METAX) - target_link_libraries(${example_name} PRIVATE ${MACA_RUNTIME_LIB}) - target_compile_options(${example_name} PRIVATE "-x" "maca") + target_link_libraries(${target_name} PRIVATE ${MACA_RUNTIME_LIB}) + target_compile_options(${target_name} PRIVATE "-x" "maca") endif() if(WITH_MOORE) - target_link_libraries(${example_name} PRIVATE ${MUSART_LIB}) - target_compile_options(${example_name} PRIVATE "-x" "musa") + target_link_libraries(${target_name} PRIVATE ${MUSART_LIB}) + target_compile_options(${target_name} PRIVATE "-x" "musa") endif() if(WITH_CAMBRICON) - target_link_libraries(${example_name} PRIVATE ${CAMBRICON_RUNTIME_LIB}) + target_link_libraries(${target_name} PRIVATE ${CAMBRICON_RUNTIME_LIB}) endif() if(WITH_OMPI OR WITH_MPICH) - target_link_libraries(${example_name} PRIVATE MPI::MPI_CXX) + target_link_libraries(${target_name} PRIVATE MPI::MPI_CXX) endif() # Explicitly allow examples to "peek" into the internal `src` and binary dirs. # This is necessary because these were marked `PRIVATE` in the library's CMake. - target_include_directories(${example_name} PRIVATE + target_include_directories(${target_name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} "${PROJECT_SOURCE_DIR}/src" # For internal templates like `runtime.h`. "${CMAKE_BINARY_DIR}/src" # For the generated `backend_manifest.h`. diff --git a/examples/all_gather.cc b/examples/mpi/all_gather.cc similarity index 100% rename from examples/all_gather.cc rename to examples/mpi/all_gather.cc diff --git a/examples/all_reduce.cc b/examples/mpi/all_reduce.cc similarity index 100% rename from examples/all_reduce.cc rename to examples/mpi/all_reduce.cc diff --git a/examples/all_to_all.cc b/examples/mpi/all_to_all.cc similarity index 100% rename from examples/all_to_all.cc rename to examples/mpi/all_to_all.cc diff --git a/examples/broadcast.cc b/examples/mpi/broadcast.cc similarity index 100% rename from examples/broadcast.cc rename to examples/mpi/broadcast.cc diff --git a/examples/gather.cc b/examples/mpi/gather.cc similarity index 100% rename from examples/gather.cc rename to examples/mpi/gather.cc diff --git a/examples/reduce.cc b/examples/mpi/reduce.cc similarity index 100% rename from examples/reduce.cc rename to examples/mpi/reduce.cc diff --git a/examples/reduce_scatter.cc b/examples/mpi/reduce_scatter.cc similarity index 100% rename from examples/reduce_scatter.cc rename to examples/mpi/reduce_scatter.cc diff --git a/examples/scatter.cc b/examples/mpi/scatter.cc similarity index 100% rename from examples/scatter.cc rename to examples/mpi/scatter.cc diff --git a/examples/send_recv.cc b/examples/mpi/send_recv.cc similarity index 100% rename from examples/send_recv.cc rename to examples/mpi/send_recv.cc diff --git a/scripts/run_examples.py b/scripts/run_examples.py index 0fc79ef..3cda028 100755 --- a/scripts/run_examples.py +++ b/scripts/run_examples.py @@ -4,7 +4,70 @@ import subprocess import sys from datetime import datetime -from typing import Optional +from pathlib import Path +from typing import List, Optional + + +# ============================================================================== +# DISCOVERY UTILITIES +# ============================================================================== +def discover_available_examples(examples_root: Path) -> List[dict]: + """ + Scans the `examples/` directory recursively to find all source files. + """ + found = [] + for path in examples_root.rglob("*.cc"): + rel_path = path.relative_to(examples_root) + + file_dir = rel_path.parent + file_we = rel_path.stem + + category = str(file_dir).replace(os.sep, "/") if str(file_dir) != "." else "" + + binary_rel_path = f"{category}/{file_we}" if category else file_we + + found.append({ + "absolute_path": path, + "relative_path": str(rel_path).replace(os.sep, "/"), + "category": category, + "name_we": file_we, + "binary_path": binary_rel_path, + }) + return found + + +def resolve_targets(input_strings: List[str], available: List[dict]) -> List[dict]: + """ + Resolves user-provided strings into targets. + Supports filtering by: + 1. Category directory (e.g. 'mpi') -> Runs everything inside + 2. Exact relative path (e.g. 'mpi/all_reduce') + 3. Short program name matching (e.g. 'all_reduce') -> Runs all instances + """ + resolved = [] + seen_binaries = set() + + for query in input_strings: + query = query.strip().replace(os.sep, "/") + if not query: + continue + + matched_any = False + for item in available: + if (query == item["category"] or + query == item["relative_path"] or + query == item["binary_path"] or + query == item["name_we"]): + + if item["binary_path"] not in seen_binaries: + seen_binaries.add(item["binary_path"]) + resolved.append(item) + matched_any = True + + if not matched_any: + print(f"⚠️ Warning: Input pattern '{query}' did not match any discovered examples.") + + return resolved # ============================================================================== @@ -19,7 +82,7 @@ def setup_log_directory(directory: str) -> str: def run_iccl_example( - example_name: str, + target_info: dict, config_path: str, launcher_opt: Optional[str], log_dir: str, @@ -28,14 +91,16 @@ def run_iccl_example( timeout_duration: int, ) -> bool: """Executes an example via `icclrun` orchestration framework.""" - log_file_path = os.path.join(log_dir, f"{example_name}.log") + binary_path = target_info["binary_path"] + safe_log_name = binary_path.replace("/", "_") + log_file_path = os.path.join(log_dir, f"{safe_log_name}.log") status_msg = ( "🚀 Running via `icclrun` (with build):" if trigger_build else "🚀 Running via `icclrun`: " ) - print(f"{status_msg:<35} {example_name:<20}", end="", flush=True) + print(f"{status_msg:<35} {binary_path:<30}", end="", flush=True) # Base Command Assembly cmd = ["icclrun", "--config", config_path] @@ -43,12 +108,9 @@ def run_iccl_example( cmd.extend(["--launcher", launcher_opt.strip()]) if trigger_build: - cmd.extend(["--build", example_name]) + cmd.extend(["--build", binary_path]) else: - cmd.append(example_name) - - # Format the exact command as a clean string for log documentation. - exact_command_str = " ".join(cmd) + cmd.append(binary_path) # Force environment unbuffered stream states for sub-python instances. custom_env = os.environ.copy() @@ -56,13 +118,12 @@ def run_iccl_example( try: with open(log_file_path, "w", buffering=1) as log_file: - # Write the exact underlying command as the absolute first line of the log. - log_file.write(f"[COMMAND]: {exact_command_str}\n") + log_file.write(f"[COMMAND]: {' '.join(cmd)}\n") log_file.write("=" * 80 + "\n\n") log_file.flush() if verbose: - print(f"\n--- [VERBOSE OUTPUT START: `{example_name}`] ---") + print(f"\n--- [VERBOSE OUTPUT START: `{binary_path}`] ---") # Execute with `Popen` to stream `stdout`/`stderr` live to both terminal and file. process = subprocess.Popen( @@ -80,7 +141,7 @@ def run_iccl_example( process.wait(timeout=timeout_duration) return_code = process.returncode print( - f"--- [VERBOSE OUTPUT END: `{example_name}`] ---\n" + " " * 56, + f"--- [VERBOSE OUTPUT END: `{binary_path}`] ---\n" + " " * 66, end="", ) else: @@ -105,9 +166,7 @@ def run_iccl_example( except subprocess.TimeoutExpired: print(f" ❌ TIMEOUT (Exceeded {timeout_duration} seconds)") with open(log_file_path, "a") as f: - f.write( - f"\n[RUNNER ERROR]: Distributed `icclrun` harness timed out after {timeout_duration} seconds.\n" - ) + f.write(f"\n[RUNNER ERROR]: Harness timed out after {timeout_duration} seconds.\n") return False except FileNotFoundError: print(" ❌ ERROR (`icclrun` executable not found in `PATH`)") @@ -142,8 +201,8 @@ def main(): "-e", "--examples", type=str, - default="all_reduce", - help="Comma-separated list of example target names to execute.", + default="mpi", + help="Comma-separated paths, categories, or short names (fuzzy match). E.g. 'mpi', 'mpi/all_reduce', or 'all_reduce'.", ) parser.add_argument( "-o", @@ -168,18 +227,26 @@ def main(): args = parser.parse_args() - # Parse and clean the comma-separated examples list. - examples_to_run = [ex.strip() for ex in args.examples.split(",") if ex.strip()] + script_dir = Path(__file__).parent.resolve() + examples_root = script_dir / "../examples" + if not examples_root.exists(): + examples_root = Path("examples").resolve() + + if not examples_root.exists(): + print("❌ Error: Could not locate 'examples/' directory tree.") + sys.exit(1) + + all_available = discover_available_examples(examples_root) + + input_queries = [ex.strip() for ex in args.examples.split(",") if ex.strip()] + targets_to_run = resolve_targets(input_queries, all_available) - # Sanity Checks if not os.path.exists(args.config): print(f"❌ Error: Config file matching '{args.config}' could not be located.") sys.exit(1) - if not examples_to_run: - print( - "❌ Error: No target programs defined. Please check your `--examples` list." - ) + if not targets_to_run: + print("❌ Error: No valid targets resolved. Please check your `--examples` query configurations.") sys.exit(1) # Initialize Log Infrastructure @@ -199,9 +266,9 @@ def main(): failed_programs = [] is_first_run = True - for example in examples_to_run: + for target in targets_to_run: success = run_iccl_example( - example_name=example, + target_info=target, config_path=args.config, launcher_opt=args.launcher, log_dir=current_run_log_dir, @@ -216,12 +283,9 @@ def main(): passed_count += 1 else: failed_count += 1 - failed_programs.append(example) + failed_programs.append(target["binary_path"]) - # ============================================================================== - # METRICS REPORTING - # ============================================================================== - total_programs = len(examples_to_run) + total_programs = len(targets_to_run) success_rate = (passed_count / total_programs) * 100 print("\n==================================================================") @@ -234,9 +298,7 @@ def main(): print("==================================================================") if failed_count > 0: - print( - f"⚠️ The following collective validation targets failed: {', '.join(failed_programs)}" - ) + print(f"⚠️ The following collective validation targets failed: {', '.join(failed_programs)}") sys.exit(1) else: print("🎉 All targets executed successfully!") From 102c7e9de6d1ba4cfe61315e4a20fd0febe40cdc Mon Sep 17 00:00:00 2001 From: Zimin Li Date: Wed, 24 Jun 2026 03:27:39 +0000 Subject: [PATCH 2/4] style: ruff format `scripts/run_examples.py` --- scripts/run_examples.py | 53 +++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/scripts/run_examples.py b/scripts/run_examples.py index 3cda028..d938caa 100755 --- a/scripts/run_examples.py +++ b/scripts/run_examples.py @@ -18,21 +18,23 @@ def discover_available_examples(examples_root: Path) -> List[dict]: found = [] for path in examples_root.rglob("*.cc"): rel_path = path.relative_to(examples_root) - + file_dir = rel_path.parent file_we = rel_path.stem - + category = str(file_dir).replace(os.sep, "/") if str(file_dir) != "." else "" - + binary_rel_path = f"{category}/{file_we}" if category else file_we - found.append({ - "absolute_path": path, - "relative_path": str(rel_path).replace(os.sep, "/"), - "category": category, - "name_we": file_we, - "binary_path": binary_rel_path, - }) + found.append( + { + "absolute_path": path, + "relative_path": str(rel_path).replace(os.sep, "/"), + "category": category, + "name_we": file_we, + "binary_path": binary_rel_path, + } + ) return found @@ -51,21 +53,24 @@ def resolve_targets(input_strings: List[str], available: List[dict]) -> List[dic query = query.strip().replace(os.sep, "/") if not query: continue - + matched_any = False for item in available: - if (query == item["category"] or - query == item["relative_path"] or - query == item["binary_path"] or - query == item["name_we"]): - + if ( + query == item["category"] + or query == item["relative_path"] + or query == item["binary_path"] + or query == item["name_we"] + ): if item["binary_path"] not in seen_binaries: seen_binaries.add(item["binary_path"]) resolved.append(item) matched_any = True - + if not matched_any: - print(f"⚠️ Warning: Input pattern '{query}' did not match any discovered examples.") + print( + f"⚠️ Warning: Input pattern '{query}' did not match any discovered examples." + ) return resolved @@ -166,7 +171,9 @@ def run_iccl_example( except subprocess.TimeoutExpired: print(f" ❌ TIMEOUT (Exceeded {timeout_duration} seconds)") with open(log_file_path, "a") as f: - f.write(f"\n[RUNNER ERROR]: Harness timed out after {timeout_duration} seconds.\n") + f.write( + f"\n[RUNNER ERROR]: Harness timed out after {timeout_duration} seconds.\n" + ) return False except FileNotFoundError: print(" ❌ ERROR (`icclrun` executable not found in `PATH`)") @@ -246,7 +253,9 @@ def main(): sys.exit(1) if not targets_to_run: - print("❌ Error: No valid targets resolved. Please check your `--examples` query configurations.") + print( + "❌ Error: No valid targets resolved. Please check your `--examples` query configurations." + ) sys.exit(1) # Initialize Log Infrastructure @@ -298,7 +307,9 @@ def main(): print("==================================================================") if failed_count > 0: - print(f"⚠️ The following collective validation targets failed: {', '.join(failed_programs)}") + print( + f"⚠️ The following collective validation targets failed: {', '.join(failed_programs)}" + ) sys.exit(1) else: print("🎉 All targets executed successfully!") From d15abf18fe58d9e4db5ac4d5cc3a1e5a16e9f631 Mon Sep 17 00:00:00 2001 From: Zimin Li Date: Thu, 25 Jun 2026 06:16:18 +0000 Subject: [PATCH 3/4] docs: update `README.md` to reflect the new file structure of `examples/` --- README.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index df99f2f..fe22f55 100644 --- a/README.md +++ b/README.md @@ -173,18 +173,27 @@ After having a successful build and a complete `cluster.yaml`, we are ready for ### 1. Run Internal Examples #### 1.1. Run a Single Example -To run an internal example program (e.g., `examples/all_reduce.cc`), just run: +The repository organizes test profiles and example suites by collective paradigm layout under `examples/`: +```text +examples/ +├── benmark/ # Performance profiling tools +├── ccl/ # Standalone or Pure CCL test suites +├── ccl_mpi_hybrid/ # Hybrid architecture benchmarks +└── mpi/ # Standard MPI sanity validations +``` + +To run an internal example program (e.g., `examples/mpi/all_reduce.cc`), just run: ```bash # Build and run the executable across the cluster based on the config specified in your `cluster.yaml` -icclrun --config [path-to-your-cluster.yaml] --build all_reduce [program args] +icclrun --config [path-to-your-cluster.yaml] --build mpi/all_reduce [program args] ``` - `--config / -c` : Path to the cluster YAML file. - `--build` : Instructs `icclrun` to compile the library on each node before execution. If omitted, `icclrun` assumes the library is already installed at a consistent location. - `--launcher` : Specify the backend launcher to be used. Default to `ompi`. -The executable (e.g., `all_reduce`) and its arguments follow the options. +The executable (e.g., `mpi/all_reduce`) and its arguments follow the options. For more details, run: @@ -207,7 +216,7 @@ chmod +x ./scripts/run_examples.sh For all the options of the script, see: ```bash -./run_examples --help +./scripts/run_examples.py --help ``` ### 2. Run a Custom User Program From 67dc13c0f36d5d6ef05175c0507fed8d485e0e16 Mon Sep 17 00:00:00 2001 From: Zimin Li Date: Mon, 29 Jun 2026 09:28:37 +0000 Subject: [PATCH 4/4] docs: remove the `examples/` structure diagram --- README.md | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/README.md b/README.md index fe22f55..b9415f9 100644 --- a/README.md +++ b/README.md @@ -173,14 +173,7 @@ After having a successful build and a complete `cluster.yaml`, we are ready for ### 1. Run Internal Examples #### 1.1. Run a Single Example -The repository organizes test profiles and example suites by collective paradigm layout under `examples/`: -```text -examples/ -├── benmark/ # Performance profiling tools -├── ccl/ # Standalone or Pure CCL test suites -├── ccl_mpi_hybrid/ # Hybrid architecture benchmarks -└── mpi/ # Standard MPI sanity validations -``` +The repository organizes example suites by collective paradigm layout under `examples/`. To run an internal example program (e.g., `examples/mpi/all_reduce.cc`), just run: