Skip to content

Commit 1c76749

Browse files
gujieyegujieye
committed
feat: update datalist validate
1 parent 5d1b7aa commit 1c76749

3 files changed

Lines changed: 53 additions & 58 deletions

File tree

‎packages/commands/src/commands/dataset/upload.ts‎

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ const UPLOAD_FLAGS = {
2828
type: "string",
2929
valueHint: "<s>",
3030
description:
31-
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record.',
31+
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record.',
3232
},
3333
noValidate: {
3434
type: "switch",
@@ -44,7 +44,7 @@ export default defineCommand({
4444
description: "Upload a dataset file (.jsonl or .zip) to Bailian",
4545
auth: "apiKey",
4646
usageArgs:
47-
"--file <path> [--purpose <name>] [--schema <chatml|dpo|cpt|tts|image>] [--no-validate] [--full-validate]",
47+
"--file <path> [--purpose <name>] [--schema <chatml|dpo|cpt|tts|image|video>] [--no-validate] [--full-validate]",
4848
flags: UPLOAD_FLAGS,
4949
exampleArgs: [
5050
"--file train.jsonl",
@@ -72,15 +72,8 @@ export default defineCommand({
7272
const filePath = flags.file;
7373
const purpose = flags.purpose || "fine-tune";
7474
const schema = parseDatasetSchemaFlag(flags.schema);
75-
if (schema === "video") {
76-
throw new BailianError(
77-
`--schema video is not supported.`,
78-
ExitCode.USAGE,
79-
`Supported schemas: chatml, dpo, cpt, tts, image.`,
80-
);
81-
}
82-
// Image schema allows larger ZIPs (1 GB vs 300 MB for text).
83-
const isMediaSchema = schema === "image";
75+
// Image and video schemas allow larger ZIPs (1 GB vs 300 MB for text).
76+
const isMediaSchema = schema === "image" || schema === "video";
8477

8578
if (!flags.noValidate) {
8679
const maxBytes = isMediaSchema ? MAX_MEDIA_ZIP_BYTES : MAX_DATASET_BYTES;

‎packages/commands/src/commands/dataset/validate.ts‎

Lines changed: 12 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -23,21 +23,22 @@ const VALIDATE_FLAGS = {
2323
type: "string",
2424
valueHint: "<s>",
2525
description:
26-
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record.',
26+
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record.',
2727
},
2828
} satisfies FlagsDef;
2929

3030
export default defineCommand({
3131
description: "Locally validate a dataset file (.jsonl or .zip) without uploading",
3232
// 纯本地校验，不触网、不需 API key（与 `pipeline validate` 一致）。
3333
auth: "none",
34-
usageArgs: "--file <path> [--full-validate] [--schema <chatml|dpo|cpt|tts|image>]",
34+
usageArgs: "--file <path> [--full-validate] [--schema <chatml|dpo|cpt|tts|image|video>]",
3535
flags: VALIDATE_FLAGS,
3636
exampleArgs: [
3737
"--file train.jsonl",
3838
"--file dpo.jsonl --schema dpo",
3939
"--file cpt.jsonl --schema cpt",
4040
"--file audio.zip --schema tts",
41+
"--file wan-i2v-training-dataset.zip --schema video",
4142
"--file eval.jsonl --full-validate",
4243
"--file train.jsonl --output json",
4344
],
@@ -47,25 +48,20 @@ export default defineCommand({
4748
"Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen,",
4849
'rejected}; cpt = {text:"..."} (continual pre-training, raw text);',
4950
'tts = {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning);',
50-
'image = {img_path:"..."} (image generation). With no --schema, a record',
51-
"carrying wav_fn is validated as TTS, img_path as image, chosen/rejected",
52-
"as DPO, text (no messages) as CPT, otherwise ChatML. Pass --schema to",
53-
"require a specific shape on every record. ZIP archives (.zip) are",
54-
"validated structurally (data.jsonl present, media references resolve) in",
55-
"addition to per-record content checks. Use --full-validate to JSON.parse",
56-
"every line.",
51+
'image = {img_path:"..."} (image generation);',
52+
'video = {first_frame_path:"...", video_path:"..."} (video generation,',
53+
"i2v first-frame or kf2v first+last-frame with last_frame_path). With no",
54+
"--schema, a record carrying wav_fn is validated as TTS, img_path as image,",
55+
"first_frame_path/video_path as video, chosen/rejected as DPO, text (no",
56+
"messages) as CPT, otherwise ChatML. Pass --schema to require a specific",
57+
"shape on every record. ZIP archives (.zip) are validated structurally",
58+
"(data.jsonl present, media references resolve) in addition to per-record",
59+
"content checks. Use --full-validate to JSON.parse every line.",
5760
],
5861
async run(ctx) {
5962
const { settings, flags } = ctx;
6063
const filePath = flags.file;
6164
const schema = parseDatasetSchemaFlag(flags.schema);
62-
if (schema === "video") {
63-
throw new BailianError(
64-
`--schema video is not supported.`,
65-
ExitCode.USAGE,
66-
`Supported schemas: chatml, dpo, cpt, tts, image.`,
67-
);
68-
}
6965
if (settings.dryRun) {
7066
emitResult(
7167
{

‎skills/bailian-finetune/reference/dataset.md‎

Lines changed: 37 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -107,23 +107,23 @@ bl dataset list --output json
107107

108108
### `bl dataset upload`
109109

110-
| Field | Value |
111-
| --------------- | -------------------------------------------------------------------------------------------------------------------------------- |
112-
| **Name** | `dataset upload` |
113-
| **Description** | Upload a dataset file (.jsonl or .zip) to Bailian |
114-
| **Usage** | `bl dataset upload --file <path> [--purpose <name>] [--schema <chatml\|dpo\|cpt\|tts\|image>] [--no-validate] [--full-validate]` |
110+
| Field | Value |
111+
| --------------- | --------------------------------------------------------------------------------------------------------------------------------------- |
112+
| **Name** | `dataset upload` |
113+
| **Description** | Upload a dataset file (.jsonl or .zip) to Bailian |
114+
| **Usage** | `bl dataset upload --file <path> [--purpose <name>] [--schema <chatml\|dpo\|cpt\|tts\|image\|video>] [--no-validate] [--full-validate]` |
115115

116116
#### Flags
117117

118-
| Flag | Type | Required | Description |
119-
| ------------------ | ------ | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- |
120-
| `--file <path>` | string | yes | Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image) |
121-
| `--purpose <name>` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") |
122-
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record. |
123-
| `--no-validate` | switch | no | Skip the local JSONL pre-flight check (not recommended) |
124-
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
125-
| `--api-key <key>` | string | no | API key |
126-
| `--base-url <url>` | string | no | API base URL |
118+
| Flag | Type | Required | Description |
119+
| ------------------ | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
120+
| `--file <path>` | string | yes | Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image) |
121+
| `--purpose <name>` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") |
122+
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record. |
123+
| `--no-validate` | switch | no | Skip the local JSONL pre-flight check (not recommended) |
124+
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
125+
| `--api-key <key>` | string | no | API key |
126+
| `--base-url <url>` | string | no | API base URL |
127127

128128
#### Notes
129129

@@ -170,19 +170,19 @@ bl dataset upload --file train.jsonl --no-validate
170170

171171
### `bl dataset validate`
172172

173-
| Field | Value |
174-
| --------------- | ----------------------------------------------------------------------------------------------- |
175-
| **Name** | `dataset validate` |
176-
| **Description** | Locally validate a dataset file (.jsonl or .zip) without uploading |
177-
| **Usage** | `bl dataset validate --file <path> [--full-validate] [--schema <chatml\|dpo\|cpt\|tts\|image>]` |
173+
| Field | Value |
174+
| --------------- | ------------------------------------------------------------------------------------------------------ |
175+
| **Name** | `dataset validate` |
176+
| **Description** | Locally validate a dataset file (.jsonl or .zip) without uploading |
177+
| **Usage** | `bl dataset validate --file <path> [--full-validate] [--schema <chatml\|dpo\|cpt\|tts\|image\|video>]` |
178178

179179
#### Flags
180180

181-
| Flag | Type | Required | Description |
182-
| ----------------- | ------ | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- |
183-
| `--file <path>` | string | yes | Local dataset file (.jsonl or .zip) |
184-
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
185-
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record. |
181+
| Flag | Type | Required | Description |
182+
| ----------------- | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
183+
| `--file <path>` | string | yes | Local dataset file (.jsonl or .zip) |
184+
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
185+
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record. |
186186

187187
#### Notes
188188

@@ -191,13 +191,15 @@ bl dataset upload --file train.jsonl --no-validate
191191
- Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen,
192192
- rejected}; cpt = {text:"..."} (continual pre-training, raw text);
193193
- tts = {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning);
194-
- image = {img_path:"..."} (image generation). With no --schema, a record
195-
- carrying wav_fn is validated as TTS, img_path as image, chosen/rejected
196-
- as DPO, text (no messages) as CPT, otherwise ChatML. Pass --schema to
197-
- require a specific shape on every record. ZIP archives (.zip) are
198-
- validated structurally (data.jsonl present, media references resolve) in
199-
- addition to per-record content checks. Use --full-validate to JSON.parse
200-
- every line.
194+
- image = {img_path:"..."} (image generation);
195+
- video = {first_frame_path:"...", video_path:"..."} (video generation,
196+
- i2v first-frame or kf2v first+last-frame with last_frame_path). With no
197+
- --schema, a record carrying wav_fn is validated as TTS, img_path as image,
198+
- first_frame_path/video_path as video, chosen/rejected as DPO, text (no
199+
- messages) as CPT, otherwise ChatML. Pass --schema to require a specific
200+
- shape on every record. ZIP archives (.zip) are validated structurally
201+
- (data.jsonl present, media references resolve) in addition to per-record
202+
- content checks. Use --full-validate to JSON.parse every line.
201203

202204
#### Examples
203205

@@ -217,6 +219,10 @@ bl dataset validate --file cpt.jsonl --schema cpt
217219
bl dataset validate --file audio.zip --schema tts
218220
```
219221

222+
```bash
223+
bl dataset validate --file wan-i2v-training-dataset.zip --schema video
224+
```
225+
220226
```bash
221227
bl dataset validate --file eval.jsonl --full-validate
222228
```

0 commit comments

Comments
 (0)