diff --git a/.github/workflows/ete_test_gpu.yaml b/.github/workflows/ete_test_gpu.yaml index 007c20e4c..89cf0a491 100644 --- a/.github/workflows/ete_test_gpu.yaml +++ b/.github/workflows/ete_test_gpu.yaml @@ -32,6 +32,14 @@ on: Resume phases within one case stay sequential. type: string default: '4' + CI_E2E_DEBUG: + required: false + description: >- + CI_E2E_DEBUG:if true, start "sleep inf &" after install and before training so the + cluster job stays alive for interactive debug (including after OOM). + Default false. + type: boolean + default: false schedule: - cron: '00 16 * * 0-5' env: @@ -69,6 +77,7 @@ jobs: env: GITHUB_RUN_ID: ${{ github.run_id }} CI_REPORTS_RAW_URL_BASE: ${{ env.CI_REPORTS_RAW_URL_BASE }} + CI_E2E_DEBUG: ${{ github.event.inputs.CI_E2E_DEBUG || 'false' }} run: | source /mnt/shared-storage-user/opencompass-shared/qa-llm-cicd/miniconda3/bin/activate conda activate clusterx diff --git a/.github/workflows/ete_test_npu.yaml b/.github/workflows/ete_test_npu.yaml index 9d125c633..01b180613 100644 --- a/.github/workflows/ete_test_npu.yaml +++ b/.github/workflows/ete_test_npu.yaml @@ -24,6 +24,14 @@ on: (e.g. "npu-qwen3-sft,npu-qwen3-rl-vllm"). type: string default: 'all' + CI_E2E_DEBUG: + required: false + description: >- + CI_E2E_DEBUG:if true, start "sleep inf &" after install and before training so the + cluster job stays alive for interactive debug (including after OOM). + Default false. + type: boolean + default: false schedule: - cron: '00 16 * * 0-5' env: @@ -59,6 +67,7 @@ jobs: env: GITHUB_RUN_ID: ${{ github.run_id }} CI_REPORTS_RAW_URL_BASE: ${{ env.CI_REPORTS_RAW_URL_BASE }} + CI_E2E_DEBUG: ${{ github.event.inputs.CI_E2E_DEBUG || 'false' }} run: | source /mnt/hwfile/vc-intern-delivery/qa-llm-cicd/miniconda3/bin/activate source activate npuci diff --git a/autotest/config.yaml b/autotest/config.yaml index 8e9ce222d..91e185010 100644 --- a/autotest/config.yaml +++ b/autotest/config.yaml @@ -824,6 +824,75 @@ case: runtime_info/text_tokens: 0 timeout: 1500 + qwen3-5-sft-ep2-mtp4-vl: + - + type: sft + parameters: + config: autotest/config/qwen3_5_ep2_mtp4_vl.py + output_path: /mnt/shared-storage-user/llmrazor-share/qa-llm-cicd/test_output + resource: + num_nodes: 2 + envs: + - MODEL_PATH=/mnt/shared-storage-user/llmrazor-share/model/Qwen3.5-35B-A3B + - DATA_PATH=/mnt/shared-storage-user/llmrazor-share/data/ci_vl + - MEDIA_ROOT=/mnt/shared-storage-user/llmrazor-share/data/ci_vl + - XTUNER_DETERMINISTIC=true + - TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=0 + assert_info: + base_metric: qwen3-5-sft-ep2-mtp4-vl/tracker.jsonl + check_metrics: + grad_norm: 0.000001 + loss/local_loss: 0.000001 + loss/reduced_balancing_loss: 0.000001 + loss/reduced_llm_loss: 0.000001 + lr: 0 + memory/max_memory_GB: 0.2 + runtime_info/tgs: 0.05 + runtime_info/text_tokens: 0 + timeout: 1500 + + qwen3-5-rl-vl-lmdeploy-mtp-ep: + - + type: rl + parameters: + config: autotest/config/rl_qwen3p5_vl_35B_mtp_ep.py + infer_backend: lmdeploy + output_path: /mnt/shared-storage-user/llmrazor-share/qa-llm-cicd/test_output + resource: + memory_per_task: 1200 + envs: + - MODEL_PATH=/mnt/shared-storage-user/llmrazor-share/model/Qwen3.5-35B-A3B + - DATA_PATH=/mnt/shared-storage-user/llmrazor-share/data/rl_vl_dapomath_geo3k_train.jsonl + - EVAL_DATA_PATH=/mnt/shared-storage-user/llmrazor-share/data/rl_vl_dapomath_geo3k_test.jsonl + - XTUNER_USE_LMDEPLOY=1 + - XTUNER_DETERMINISTIC=true + - LMDEPLOY_FP32_MAMBA_SSM_DTYPE=1 + assert_info: + base_metric: qwen3-5-rl-vl-lmdeploy-mtp-ep/tracker.jsonl + check_metrics: + - + metric: response/rewards/mean + threshold: 0.3 + method: absolute + operator: < + - + metric: mismatch/mismatch_k3_kl + threshold: 0.0005 + method: absolute + operator: <= + - + metric: response/response_len/mean + threshold: 0.25 + method: relative + operator: < + - + metric: time/step + threshold: 0.15 + method: relative + operator: < + aggregate: 80 + timeout: 10800 + qwen3-rl-lmdeploy: - type: rl diff --git a/autotest/config/qwen3_5_ep2_mtp4_vl.py b/autotest/config/qwen3_5_ep2_mtp4_vl.py new file mode 100644 index 000000000..5b88877fe --- /dev/null +++ b/autotest/config/qwen3_5_ep2_mtp4_vl.py @@ -0,0 +1,95 @@ +import os + +from xtuner.v1.config import FSDPConfig, LRConfig, MuonConfig +from xtuner.v1.datasets import Qwen3VLTokenizeFnConfig +from xtuner.v1.datasets.config import DataloaderConfig, DatasetConfig +from xtuner.v1.loss.ce_loss import CELossConfig +from xtuner.v1.model import Qwen3_5_VLMoE35BA3Config +from xtuner.v1.module.mtp import MTPConfig +from xtuner.v1.train import ResumeConfig, TrainerConfig + + +MEDIA_ROOT = os.environ["MEDIA_ROOT"] +MODEL_PATH = os.environ["MODEL_PATH"] +DATA_PATH = os.environ["DATA_PATH"] +WORK_DIR = os.environ["WORK_DIR"] + +sample_max_length = 16 * 1024 +pack_max_length = 64 * 1024 +ep_size = 2 + +moe_cfg = Qwen3_5_VLMoE35BA3Config(only_llm_forward=False, compile_cfg=True) +moe_cfg.text_config.ep_size = ep_size +moe_cfg.text_config.dispatcher = "deepep" +moe_cfg.text_config.mtp_config = MTPConfig( + num_layers=4, + share_weights=True, + loss_scaling_factor=1.0, +) + +optim_cfg = MuonConfig(lr=2e-5, weight_decay=0.05) +lr_cfg = LRConfig(lr_type="cosine", warmup_ratio=0.1, lr_min=1e-6) +fsdp_cfg = FSDPConfig( + recompute_ratio=1.0, + torch_compile=True, + cpu_offload=False, + ep_size=ep_size, + checkpoint_preserve_rng_state=False, +) + +dataset_config = [ + { + "dataset": DatasetConfig( + name="multimodal", + anno_path=DATA_PATH, + class_name="VLMJsonlDataset", + media_root=MEDIA_ROOT, + sample_ratio=1.0, + cache_dir=os.path.join(WORK_DIR, "jsonl_cache"), + cache_tag=f"qwen3p5_vl_{sample_max_length}", + ), + "tokenize_fn": Qwen3VLTokenizeFnConfig( + processor_path=MODEL_PATH, + chat_template="qwen3.5-vl", + llm_pack_weight=-3.2, + visual_pack_weight=5.0, + max_length=sample_max_length, + rand_video_max_frames=24, + max_pixels=16384 * 32 * 32, + ), + }, +] + +dataloader_config = DataloaderConfig( + dataset_config_list=dataset_config, + pack_level="soft", + pack_max_length=pack_max_length, + pack_to_max_length=True, + pack_chunk_size=10000, + pack_workers=4, + global_pack=True, + group_by_length=True, + collator="qwen3_vl_sft_collator", + pack_extra_buffer_size=20, + num_workers=1, +) + +loss_cfg = CELossConfig(mode="chunk", chunk_size=1024, loss_reduction="square") + +trainer = TrainerConfig( + load_from=MODEL_PATH, + model_cfg=moe_cfg, + optim_cfg=optim_cfg, + fsdp_cfg=fsdp_cfg, + dataloader_cfg=dataloader_config, + lr_cfg=lr_cfg, + loss_cfg=loss_cfg, + tokenizer_path=MODEL_PATH, + resume_cfg=ResumeConfig(auto_resume=True), + global_batch_size=16, + total_step=20, + sp_size=4, + work_dir=WORK_DIR, + seed=0, +) + diff --git a/autotest/config/rl_qwen3p5_vl_35B_mtp_ep.py b/autotest/config/rl_qwen3p5_vl_35B_mtp_ep.py new file mode 100644 index 000000000..a86500b9d --- /dev/null +++ b/autotest/config/rl_qwen3p5_vl_35B_mtp_ep.py @@ -0,0 +1,358 @@ +import json +import os + +import ray +from transformers import AutoTokenizer + +from xtuner.v1.config import AdamWConfig, FSDPConfig, LRConfig +from xtuner.v1.data_proto.rl_data import SampleParams +from xtuner.v1.datasets.config import DataloaderConfig, DatasetConfig +from xtuner.v1.datasets.rl_tokenize_fn import RLQwen3VLTokenizeFnConfig +from xtuner.v1.float8 import Float8Config, ScalingGranularity +from xtuner.v1.model import Qwen3_5_VLMoE35BA3Config +from xtuner.v1.module.mtp import MTPConfig +from xtuner.v1.rl.advantage import GRPOAdvantageConfig +from xtuner.v1.rl.agent_loop import SingleTurnAgentLoopConfig +from xtuner.v1.rl.agent_loop_manager import ( + AgentLoopManagerConfig, + AsyncProduceStrategyConfig, + SamplerConfig, + TaskSpecConfig, +) +from xtuner.v1.rl.evaluator import EvaluatorConfig +from xtuner.v1.rl.judger import ComposedJudgerConfig, DapoMathJudgerConfig, GEO3KJudgerConfig +from xtuner.v1.rl.loss import GRPOLossConfig +from xtuner.v1.rl.replay_buffer import AsyncReplayBufferConfig +from xtuner.v1.rl.rollout.worker import RolloutConfig +from xtuner.v1.rl.trainer import RolloutImportanceSampling, WorkerConfig +from xtuner.v1.rl.utils import AcceleratorResourcesConfig, CPUResourcesConfig, get_eos_token +from xtuner.v1.train.rl_trainer import RLColocateTrainerConfig + + +def _as_list(value): + return value if isinstance(value, list) else [value] + + +DATA_JUDGER_MAPPING = { + "openai/gsm8k": {"math_dapo": 1.0}, + "math_dapo": {"math_dapo": 1.0}, + "hiyouga/geometry3k": {"hiyouga/geometry3k": 1.0}, +} + + +work_dir = os.environ["WORK_DIR"] +model_path = os.environ["MODEL_PATH"] +meta_data_path = os.environ["DATA_PATH"] +eval_meta_data_path = os.environ.get("EVAL_DATA_PATH", "") + +debug_rollout_dir = os.environ.get("DEBUG_ROLLOUT_DIR", "") +debug_train = os.environ.get("DEBUG_TRAIN", False) +debug_rollout = os.environ.get("DEBUG_ROLLOUT", False) +swap_optimizer = os.environ.get("SWAP_OPTIMIZER", "0").lower() in ("1", "true", "yes", "on") + +enable_evaluate = eval_meta_data_path is not None and eval_meta_data_path != "" + +# basic settings +experimental_name = "rl_qwen3p5_vl_35B_mtp_ep" +total_epochs = 1 +global_batch_size = 64 +prompt_repeat_k = 4 +rollout_tp_size = 1 +rollout_ep_size = 2 +train_ep_size = 2 +max_prompt_length = 2048 +max_response_length = 4096 +pack_max_length = 32768 +train_optimizer_steps = 8 +hf_interval = 15 + +# 1. resources +resources = AcceleratorResourcesConfig( + accelerator="GPU", + num_workers=8, + num_cpus_per_worker=12, + cpu_memory_per_worker=16 * 1024**3, +) + +# 2. rollout +rollout_config = RolloutConfig( + fp32_lm_head=True, + env=experimental_name, + device=resources.accelerator, + model_path=model_path, + dtype="bfloat16", + skip_load_weights=True, + tensor_parallel_size=rollout_tp_size, + expert_parallel_size=rollout_ep_size, + gpu_memory_utilization=0.6, + context_length=max_response_length + max_prompt_length, + enable_return_routed_experts=True, + rollout_max_batch_size_per_instance=128, + extra_rollout_config=dict( + lmdeploy_log_level="INFO", + lmdeploy_uvicorn_log_level="INFO", + lmdeploy_speculative_algorithm="qwen3_5_mtp", + lmdeploy_speculative_num_draft_tokens=3, + ), + health_check_interval_seconds=300, + health_check_failure_threshold=3, +) + +# sampling params +training_sample_params = SampleParams( + max_tokens=max_response_length, + top_k=0, + top_p=1.0, + temperature=1.0, + min_tokens=0, +) +evaluation_sample_params = SampleParams( + max_tokens=max_response_length, + top_k=1, + top_p=1.0, + temperature=0.0, + min_tokens=0, + return_routed_experts=False, +) + +# 3. datasets +with open(meta_data_path, "r", encoding="utf-8") as f: + ds_collections = json.load(f) + +train_dataset_cfg = [] +for name, data in ds_collections.items(): + annotations = _as_list(data["annotation"]) + for annotation in annotations: + train_dataset_cfg.append( + { + "dataset": DatasetConfig( + name=name, + anno_path=annotation, + media_root=data.get("media_root", ""), + sample_ratio=data.get("sample_ratio", 1.0), + class_name="VLMJsonlDataset", + ), + "tokenize_fn": RLQwen3VLTokenizeFnConfig( + processor_path=model_path, + max_length=max_prompt_length, + system_message=data.get("system_message", None), + chat_template="qwen3.5-vl", + add_generation_prompt=True, + enable_thinking=True, + data_judger_mapping=DATA_JUDGER_MAPPING, + ), + } + ) + +if enable_evaluate: + with open(eval_meta_data_path, "r", encoding="utf-8") as f: + eval_ds_collections = json.load(f) + + eval_dataset_cfg = [] + for name, data in eval_ds_collections.items(): + annotations = _as_list(data["annotation"]) + for annotation in annotations: + eval_dataset_cfg.append( + { + "dataset": DatasetConfig( + name=name, + anno_path=annotation, + media_root=data.get("media_root", ""), + sample_ratio=data.get("sample_ratio", 1.0), + class_name="VLMJsonlDataset", + ), + "tokenize_fn": RLQwen3VLTokenizeFnConfig( + processor_path=model_path, + max_length=max_prompt_length, + system_message=data.get("system_message", None), + chat_template="qwen3.5-vl", + add_generation_prompt=True, + enable_thinking=True, + ignore_multimodal_info=True, + data_judger_mapping=DATA_JUDGER_MAPPING, + ), + } + ) + eval_dataloader_cfg = DataloaderConfig( + dataset_config_list=eval_dataset_cfg, + num_workers=8, + collator="fake_collator", + pack_level="none", + pack_max_length=pack_max_length, + ) + +dataloader_cfg = DataloaderConfig( + dataset_config_list=train_dataset_cfg, + num_workers=8, + collator="fake_collator", + pack_level="none", + pack_max_length=pack_max_length, +) + +# 4. judger +tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) +eos_token_id = get_eos_token(model_path) +eos_token_str = tokenizer.convert_ids_to_tokens(eos_token_id) +judger_config = ComposedJudgerConfig( + branches={ + "math_dapo": DapoMathJudgerConfig( + judger_name="dapo_math", + eos_token=eos_token_str, + enable_overlong_buffer=True, + max_response_len=max_response_length, + overlong_buffer_len=4096, + overlong_penalty_factor=1.0, + tokenizer=tokenizer, + cpu_resources=CPUResourcesConfig(num_workers=1, num_cpus_per_worker=1), + ), + "hiyouga/geometry3k": GEO3KJudgerConfig( + cpu_resources=CPUResourcesConfig(num_workers=1, num_cpus_per_worker=1), + ), + }, +) + +# 5. train worker +model_cfg = Qwen3_5_VLMoE35BA3Config(freeze_vision=True, freeze_projector=True) +model_cfg.float8_cfg = Float8Config( + scaling_granularity_gemm=None, + scaling_granularity_grouped_gemm=ScalingGranularity.TILEWISE, +) +model_cfg.text_config.ep_size = train_ep_size +model_cfg.text_config.z_loss_cfg = None +model_cfg.text_config.balancing_loss_cfg = None +model_cfg.text_config.freeze_routers = True +model_cfg.compile_cfg = None +model_cfg.text_config.mtp_config = MTPConfig( + num_layers=3, + loss_scaling_factor=1.0, + detach_mtp_lm_head_weight=True, + detach_mtp_inputs=True, + share_weights=True, +) +optim_cfg = AdamWConfig( + lr=1e-6, + betas=(0.9, 0.999), + max_grad_norm=1.0, + weight_decay=0.1, + foreach=False, + swap_optimizer=swap_optimizer, +) +loss_cfg = GRPOLossConfig( + policy_loss_cfg=dict( + cliprange_high=0.28, + cliprange_low=0.2, + loss_type="vanilla", + clip_ratio_c=10.0, + log_prob_diff_min=-20.0, + log_prob_diff_max=20.0, + ), + ignore_idx=-100, + use_kl_loss=False, + kl_loss_coef=0.0, + kl_loss_type="low_var_kl", + mode="chunk", + chunk_size=512, + rollout_is=RolloutImportanceSampling( + rollout_is_level="token", + rollout_is_mode="both", + rollout_is_threshold=(5, 0.5), + rollout_is_mask_threshold=(5, 0.5), + rollout_is_veto_threshold=(20, 0), + ), +) +lr_cfg = LRConfig(lr_type="constant", warmup_ratio=0, lr_min=1e-6) +fsdp_cfg = FSDPConfig(torch_compile=False, cpu_offload=False, ep_size=train_ep_size, fp32_lm_head=True) +train_worker_cfg = WorkerConfig( + model_cfg=model_cfg, + load_from=model_path, + optim_cfg=optim_cfg, + loss_cfg=loss_cfg, + lr_cfg=lr_cfg, + fsdp_cfg=fsdp_cfg, + optimizer_steps=train_optimizer_steps, + pack_max_length=pack_max_length, +) + + +def group_sample_filter_func(group_samples): + valid_samples = [] + for s in group_samples: + if s.response_ids is not None: + valid_samples.append(s) + else: + if s.routed_experts is not None: + routed_experts = s.routed_experts + if isinstance(routed_experts, ray.ObjectRef): + ray.internal.free([s.routed_experts], local_only=False) + + rewards = [(d.reward or {}).get("score", 0.0) for d in valid_samples] + if len(set(rewards)) == 1: + print(f"filter all same reward sample: {rewards}") + return False + return True + + +produce_strategy_config = AsyncProduceStrategyConfig( + over_sample_threshold=1, + enable_partial_rollout=1, + is_valid_sample_fn=group_sample_filter_func, + max_staleness=3, +) + +# 6. agent loop managers +agent_loop_config = SingleTurnAgentLoopConfig( + hf_checkpoint=model_path, + sample_params=training_sample_params, +) +agent_loop_manager_cfg = AgentLoopManagerConfig( + tasks=TaskSpecConfig( + task_name="train_task", + agent_loop_config=agent_loop_config, + judger_config=judger_config, + produce_strategy_config=produce_strategy_config, + sampler_config=SamplerConfig(dataloader_cfg=dataloader_cfg, prompt_repeat_k=prompt_repeat_k), + ), +) + +if enable_evaluate: + eval_agent_loop_config = SingleTurnAgentLoopConfig( + hf_checkpoint=model_path, + sample_params=evaluation_sample_params, + ) + eval_agent_loop_manager_cfg = AgentLoopManagerConfig( + tasks=TaskSpecConfig( + task_name="eval_task", + agent_loop_config=eval_agent_loop_config, + judger_config=judger_config, + sampler_config=SamplerConfig(dataloader_cfg=eval_dataloader_cfg, prompt_repeat_k=1), + ), + ) + enable_evaluate = True +else: + eval_agent_loop_manager_cfg = None + enable_evaluate = False + +# 7. trainer +trainer = RLColocateTrainerConfig( + resources=resources, + train_worker_cfg=train_worker_cfg, + rollout_config=rollout_config, + tokenizer_path=model_path, + replay_buffer_config=AsyncReplayBufferConfig(), + agent_loop_manager_cfg=agent_loop_manager_cfg, + eval_agent_loop_manager_cfg=eval_agent_loop_manager_cfg, + evaluator_config=EvaluatorConfig(compute_metric_func=None), + load_from=model_path, + total_epochs=total_epochs, + train_batch_size=global_batch_size, + advantage_estimator_config=GRPOAdvantageConfig(eps=1e-8), + enable_evaluate=enable_evaluate, + enable_initial_evaluate=False, + evaluate_step=1, + work_dir=work_dir, + hf_interval=hf_interval, + debug_rollout_dir=debug_rollout_dir, + debug_train=debug_train, + debug_rollout=debug_rollout, + exp_tracker="jsonl", +) diff --git a/autotest/module/train.py b/autotest/module/train.py index ac89ebe9c..3a93fb096 100644 --- a/autotest/module/train.py +++ b/autotest/module/train.py @@ -42,11 +42,18 @@ def get_cmd(config): "if [[ $TORCH_VERSION == 2.9.1 ]]; then pip install nvidia-cudnn-cu12==9.15.1.9; fi; " ) + # CI_E2E_DEBUG: start sleep before training so the job stays alive after OOM. + debug_keepalive = "" + if os.environ.get("CI_E2E_DEBUG", "False").strip().lower() in ("1", "true", "yes"): + print("CI_E2E_DEBUG=true: start 'sleep inf &' before training (after install)") + debug_keepalive = "sleep inf & " + if train_type == "sft": command = ( f"cd {current_dir}; pwd; {pip_package}; {image_version_check}" f"export GITHUB_RUN_ID={config.get('run_id')}; export WORK_DIR={work_dir}; " + cudnn_patch + + debug_keepalive + f"torchrun --nproc-per-node {nproc_per_node} --master_addr=${{MASTER_ADDR}} --master_port=${{MASTER_PORT}} --nnodes=${{WORLD_SIZE}} --node_rank=${{RANK}} " + f"xtuner/v1/train/cli/{train_type}.py" ) @@ -65,6 +72,9 @@ def get_cmd(config): command += f" --dataset {dataset_path}" command += f" --work_dir {work_dir}" + if debug_keepalive: + command += "; wait" + return command, config elif train_type == "rl": infer_type = config.get("parameters", {}).get("infer_backend", "lmdeploy") @@ -73,8 +83,11 @@ def get_cmd(config): f"cd {current_dir}; pwd; {pip_package}; {image_version_check}" f"export GITHUB_RUN_ID={config.get('run_id')}; export WORK_DIR={work_dir}; " + cudnn_patch + + debug_keepalive + f"bash -x autotest/utils/ci_run_rl.sh {accelerator} {infer_type} {config_path} ${{MODEL_PATH}} ${{DATA_PATH}} ${{EVAL_DATA_PATH}}" ) + if debug_keepalive: + command += "; wait" return command, config else: return "", config