Skip to content

Commit 6f7a6bf

Browse files
Nakshatra SharmaNakshatra Sharma
authored andcommitted
fix(jindo): use dataset mounts when dataload target empty
1 parent e807509 commit 6f7a6bf

2 files changed

Lines changed: 101 additions & 7 deletions

File tree

pkg/ddc/jindo/load_data.go

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -160,13 +160,27 @@ func (e *JindoEngine) genDataLoadValue(image string, runtime *datav1alpha1.Jindo
160160
}
161161

162162
targetPaths := []cdataload.TargetPath{}
163-
for _, target := range dataload.Spec.Target {
164-
fluidNative := utils.IsTargetPathUnderFluidNativeMounts(target.Path, *targetDataset)
165-
targetPaths = append(targetPaths, cdataload.TargetPath{
166-
Path: target.Path,
167-
Replicas: target.Replicas,
168-
FluidNative: fluidNative,
169-
})
163+
if len(dataload.Spec.Target) > 0 {
164+
for _, target := range dataload.Spec.Target {
165+
fluidNative := utils.IsTargetPathUnderFluidNativeMounts(target.Path, *targetDataset)
166+
targetPaths = append(targetPaths, cdataload.TargetPath{
167+
Path: target.Path,
168+
Replicas: target.Replicas,
169+
FluidNative: fluidNative,
170+
})
171+
}
172+
} else {
173+
for _, m := range targetDataset.Spec.Mounts {
174+
if m.Path == "" {
175+
continue
176+
}
177+
fluidNative := utils.IsTargetPathUnderFluidNativeMounts(m.Path, *targetDataset)
178+
targetPaths = append(targetPaths, cdataload.TargetPath{
179+
Path: m.Path,
180+
Replicas: 1,
181+
FluidNative: fluidNative,
182+
})
183+
}
170184
}
171185
dataloadInfo.TargetPaths = targetPaths
172186
options := map[string]string{}

pkg/ddc/jindo/load_data_test.go

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,86 @@ func Test_genDataLoadValue(t *testing.T) {
143143
runtime *datav1alpha1.JindoRuntime
144144
want *cdataload.DataLoadValue
145145
}{
146+
"dataset with multiple mounts and no explicit target": {
147+
image: "fluid:v0.0.1",
148+
targetDataset: &datav1alpha1.Dataset{
149+
ObjectMeta: metav1.ObjectMeta{
150+
Name: "test-dataset",
151+
Namespace: "fluid",
152+
},
153+
Spec: datav1alpha1.DatasetSpec{
154+
Mounts: []datav1alpha1.Mount{
155+
{
156+
Name: "spark-0",
157+
MountPoint: "local://mnt/data0",
158+
Path: "/mnt0",
159+
},
160+
{
161+
Name: "spark-1",
162+
MountPoint: "local://mnt/data1",
163+
Path: "/mnt1",
164+
},
165+
},
166+
},
167+
},
168+
dataload: &datav1alpha1.DataLoad{
169+
ObjectMeta: metav1.ObjectMeta{
170+
Name: "test-dataload",
171+
Namespace: "fluid",
172+
},
173+
Spec: datav1alpha1.DataLoadSpec{
174+
Dataset: datav1alpha1.TargetDataset{
175+
Name: "test-dataset",
176+
Namespace: "fluid",
177+
},
178+
},
179+
},
180+
runtime: &datav1alpha1.JindoRuntime{
181+
Spec: datav1alpha1.JindoRuntimeSpec{
182+
TieredStore: datav1alpha1.TieredStore{
183+
Levels: []datav1alpha1.Level{
184+
{
185+
MediumType: "MEM",
186+
},
187+
},
188+
},
189+
HadoopConfig: "principal=root",
190+
},
191+
},
192+
want: &cdataload.DataLoadValue{
193+
Name: "test-dataload",
194+
OwnerDatasetId: "fluid-test-dataset",
195+
Owner: &common.OwnerReference{
196+
APIVersion: "/",
197+
Enabled: true,
198+
Name: "test-dataload",
199+
BlockOwnerDeletion: false,
200+
Controller: true,
201+
},
202+
DataLoadInfo: cdataload.DataLoadInfo{
203+
BackoffLimit: 3,
204+
Image: "fluid:v0.0.1",
205+
TargetDataset: "test-dataset",
206+
TargetPaths: []cdataload.TargetPath{
207+
{
208+
Path: "/mnt0",
209+
Replicas: 1,
210+
FluidNative: true,
211+
},
212+
{
213+
Path: "/mnt1",
214+
Replicas: 1,
215+
FluidNative: true,
216+
},
217+
},
218+
ImagePullSecrets: []corev1.LocalObjectReference{},
219+
Options: map[string]string{
220+
"loadMemorydata": "true",
221+
"hdfsConfig": "principal=root",
222+
},
223+
},
224+
},
225+
},
146226
"test case with scheduler name": {
147227
image: "fluid:v0.0.1",
148228
targetDataset: &datav1alpha1.Dataset{

0 commit comments

Comments
 (0)