OpenSceneFlow/assets/slurm/data-process.sh at main · KTH-RPL/OpenSceneFlow · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
#!/bin/bash
#SBATCH -J dataprocess
#SBATCH -p berzelius-cpu
#SBATCH --cpus-per-task 64
#SBATCH --mem 128G
#SBATCH --mincpus=64
#SBATCH -t 2-00:00:00
#SBATCH --mail-type=END,FAIL
#SBATCH --mail-user=qingwen@kth.se
#SBATCH --output /proj/berzelius-2023-154/users/x_qinzh/OpenSceneFlow/logs/slurm/%J_data.out
#SBATCH --error  /proj/berzelius-2023-154/users/x_qinzh/OpenSceneFlow/logs/slurm/%J_data.err

PYTHON=/proj/berzelius-2023-154/users/x_qinzh/mambaforge/envs/sftool/bin/python
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/proj/berzelius-2023-154/users/x_qinzh/mambaforge/lib
cd /proj/berzelius-2023-154/users/x_qinzh/OpenSceneFlow/
# export HYDRA_FULL_ERROR=1


# =============== Argoverse2
# $PYTHON dataprocess/extract_av2.py --nproc 64 \
#     --av2_type sensor \
#     --data_mode train \
#     --argo_dir /proj/berzelius-2023-154/users/x_qinzh/av2 \
#     --output_dir /proj/berzelius-2023-364/users/x_qinzh/data/av2/h5py

# $PYTHON dataprocess/extract_av2.py --nproc 64 \
#     --av2_type sensor \
#     --data_mode val \
#     --argo_dir /proj/berzelius-2023-154/users/x_qinzh/av2 \
#     --output_dir /proj/berzelius-2023-364/users/x_qinzh/data/av2/h5py \
#     --mask_dir /proj/berzelius-2023-154/users/x_qinzh/av2/3d_scene_flow

# $PYTHON dataprocess/extract_av2.py --nproc 64 \
#     --av2_type sensor \
#     --data_mode test \
#     --argo_dir /proj/berzelius-2023-154/users/x_qinzh/av2 \
#     --output_dir /proj/berzelius-2023-364/users/x_qinzh/data/av2/h5py \
#     --mask_dir /proj/berzelius-2023-154/users/x_qinzh/av2/3d_scene_flow

# =============== Waymo Open Dataset
# $PYTHON dataprocess/extract_waymo.py --nproc 12 \
#     --map_dir /proj/berzelius-2023-154/users/x_qinzh/dataset/waymo/flowlabel/map \
#     --flow_data_dir /proj/berzelius-2023-154/users/x_qinzh/dataset/waymo/flowlabel \
#     --output_dir /proj/berzelius-2023-154/users/x_qinzh/dataset/waymo/h5py \
#     --mode train

$PYTHON dataprocess/extract_waymo.py --nproc 12 \
    --map_dir /proj/berzelius-2023-154/users/x_qinzh/dataset/waymo/flowlabel/map \
    --flow_data_dir /proj/berzelius-2023-154/users/x_qinzh/dataset/waymo/flowlabel \
    --output_dir /proj/berzelius-2023-154/users/x_qinzh/dataset/waymo/h5py \
    --mode valid

# =============== MAN-Truckscenes
# $PYTHON dataprocess/extract_truckscenes.py --nproc 12 \
#     --data_dir /proj/common-datasets/MAN-TruckScenes/v1.0 \
#     --mode v1.0-trainval \
#     --output_dir /proj/berzelius-2023-364/data/truckscenes/h5py \
#     --split_name val

# $PYTHON dataprocess/extract_truckscenes.py --nproc 12 \
#     --data_dir /proj/common-datasets/MAN-TruckScenes/v1.0 \
#     --mode v1.0-trainval \
#     --output_dir /proj/berzelius-2023-364/data/truckscenes/h5py \
#     --split_name train