Skip to content

Commit 07544c7

Browse files
author
Glover, Rene (rg9975)
committed
initial updates for fiberchannel livemigrate
1 parent 71f47d6 commit 07544c7

4 files changed

Lines changed: 263 additions & 4 deletions

File tree

‎engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2566,7 +2566,10 @@ protected void verifyLiveMigrationForKVM(Map<VolumeInfo, DataStore> volumeDataSt
25662566
}
25672567

25682568
boolean isSrcAndDestPoolPowerFlexStorage = srcStoragePoolVO.getPoolType().equals(Storage.StoragePoolType.PowerFlex) && destStoragePoolVO.getPoolType().equals(Storage.StoragePoolType.PowerFlex);
2569-
if (srcStoragePoolVO.isManaged() && !isSrcAndDestPoolPowerFlexStorage && srcStoragePoolVO.getId() != destStoragePoolVO.getId()) {
2569+
boolean isSrcAndDestPoolFiberChannelStorage = srcStoragePoolVO.getPoolType().equals(Storage.StoragePoolType.FiberChannel) && destStoragePoolVO.getPoolType().equals(Storage.StoragePoolType.FiberChannel);
2570+
boolean fiberChannelVmOnline = isSrcAndDestPoolFiberChannelStorage && volumeInfo.getAttachedVM() != null && volumeInfo.getAttachedVM().getState() == VirtualMachine.State.Running;
2571+
2572+
if (srcStoragePoolVO.isManaged() && !isSrcAndDestPoolPowerFlexStorage && !fiberChannelVmOnline && srcStoragePoolVO.getId() != destStoragePoolVO.getId()) {
25702573
throw new CloudRuntimeException("Migrating a volume online with KVM from managed storage is not currently supported.");
25712574
}
25722575

@@ -2766,6 +2769,27 @@ private Map<String, String> getVolumeDetails(VolumeInfo volumeInfo) {
27662769
return volumeDetails;
27672770
}
27682771

2772+
private boolean shouldAttemptLiveFiberChannelMigration(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo) {
2773+
if (srcVolumeInfo == null || destVolumeInfo == null) {
2774+
return false;
2775+
}
2776+
2777+
StoragePoolVO srcPool = _storagePoolDao.findById(srcVolumeInfo.getPoolId());
2778+
StoragePoolVO destPool = _storagePoolDao.findById(destVolumeInfo.getPoolId());
2779+
2780+
if (srcPool == null || destPool == null) {
2781+
return false;
2782+
}
2783+
2784+
if (srcPool.getPoolType() != StoragePoolType.FiberChannel || destPool.getPoolType() != StoragePoolType.FiberChannel) {
2785+
return false;
2786+
}
2787+
2788+
VirtualMachine attachedVm = srcVolumeInfo.getAttachedVM();
2789+
2790+
return attachedVm != null && attachedVm.getState() == VirtualMachine.State.Running;
2791+
}
2792+
27692793
private Map<String, String> getSnapshotDetails(SnapshotInfo snapshotInfo) {
27702794
Map<String, String> snapshotDetails = new HashMap<>();
27712795

@@ -3021,8 +3045,11 @@ private String migrateVolumeForKVM(VolumeInfo srcVolumeInfo, VolumeInfo destVolu
30213045

30223046
_volumeService.grantAccess(srcVolumeInfo, hostVO, srcVolumeInfo.getDataStore());
30233047

3048+
boolean fiberChannelOnline = shouldAttemptLiveFiberChannelMigration(srcVolumeInfo, destVolumeInfo);
3049+
int waitTimeout = fiberChannelOnline ? StorageManager.KvmStorageOnlineMigrationWait.value() : StorageManager.KvmStorageOfflineMigrationWait.value();
3050+
30243051
MigrateVolumeCommand migrateVolumeCommand = new MigrateVolumeCommand(srcVolumeInfo.getTO(), destVolumeInfo.getTO(),
3025-
srcDetails, destDetails, StorageManager.KvmStorageOfflineMigrationWait.value());
3052+
srcDetails, destDetails, waitTimeout);
30263053

30273054
_volumeService.grantAccess(srcVolumeInfo, hostVO, srcVolumeInfo.getDataStore());
30283055
handleQualityOfServiceForVolumeMigration(destVolumeInfo, PrimaryDataStoreDriver.QualityOfServiceState.MIGRATION);

‎engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,8 @@ public class KvmNonManagedStorageSystemDataMotionTest {
124124
Host host1;
125125
@Mock
126126
Host host2;
127+
@Mock
128+
com.cloud.vm.VirtualMachine attachedVm;
127129

128130
Map<VolumeInfo, DataStore> migrationMap;
129131

@@ -478,6 +480,21 @@ public void testVerifyLiveMigrationMapForKVM() {
478480
kvmNonManagedStorageDataMotionStrategy.verifyLiveMigrationForKVM(migrationMap);
479481
}
480482

483+
@Test
484+
public void testVerifyLiveMigrationMapForKVMManagedFiberChannelAllowed() {
485+
when(pool1.isManaged()).thenReturn(true);
486+
when(pool2.isManaged()).thenReturn(true);
487+
when(pool1.getPoolType()).thenReturn(Storage.StoragePoolType.FiberChannel);
488+
when(pool2.getPoolType()).thenReturn(Storage.StoragePoolType.FiberChannel);
489+
when(pool1.getId()).thenReturn(POOL_1_ID);
490+
when(pool2.getId()).thenReturn(POOL_2_ID);
491+
when(volumeInfo1.getAttachedVM()).thenReturn(attachedVm);
492+
when(volumeInfo2.getAttachedVM()).thenReturn(attachedVm);
493+
when(attachedVm.getState()).thenReturn(com.cloud.vm.VirtualMachine.State.Running);
494+
495+
kvmNonManagedStorageDataMotionStrategy.verifyLiveMigrationForKVM(migrationMap);
496+
}
497+
481498
@Test(expected = CloudRuntimeException.class)
482499
public void testVerifyLiveMigrationMapForKVMNotExistingSource() {
483500
when(primaryDataStoreDao.findById(POOL_1_ID)).thenReturn(null);

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1132,6 +1132,10 @@ public void setSerial(String serial) {
11321132
this._serial = serial;
11331133
}
11341134

1135+
public String getSerial() {
1136+
return _serial;
1137+
}
1138+
11351139
public void setLibvirtDiskEncryptDetails(LibvirtDiskEncryptDetails details)
11361140
{
11371141
this.encryptDetails = details;

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java‎

Lines changed: 213 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,9 @@
2525
import com.cloud.agent.api.storage.MigrateVolumeCommand;
2626
import com.cloud.agent.api.to.DiskTO;
2727
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
28+
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef;
2829
import com.cloud.hypervisor.kvm.resource.disconnecthook.VolumeMigrationCancelHook;
30+
import com.cloud.hypervisor.kvm.resource.wrapper.LibvirtUtilitiesHelper;
2931
import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk;
3032
import com.cloud.hypervisor.kvm.storage.KVMStoragePool;
3133
import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
@@ -37,6 +39,7 @@
3739
import java.io.File;
3840
import java.io.IOException;
3941
import java.io.StringWriter;
42+
import java.util.List;
4043
import java.util.Map;
4144
import java.util.UUID;
4245

@@ -74,18 +77,80 @@ public class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper<MigrateVo
7477
@Override
7578
public Answer execute(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) {
7679
VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData();
77-
PrimaryDataStoreTO srcPrimaryDataStore = (PrimaryDataStoreTO)srcVolumeObjectTO.getDataStore();
80+
PrimaryDataStoreTO srcPrimaryDataStore = srcVolumeObjectTO != null ? (PrimaryDataStoreTO)srcVolumeObjectTO.getDataStore() : null;
81+
VolumeObjectTO destVolumeObjectTO = (VolumeObjectTO)command.getDestData();
82+
PrimaryDataStoreTO destPrimaryDataStore = destVolumeObjectTO != null ? (PrimaryDataStoreTO)destVolumeObjectTO.getDataStore() : null;
7883

7984
MigrateVolumeAnswer answer;
80-
if (srcPrimaryDataStore.getPoolType().equals(Storage.StoragePoolType.PowerFlex)) {
85+
if (srcPrimaryDataStore != null && srcPrimaryDataStore.getPoolType().equals(Storage.StoragePoolType.PowerFlex)) {
8186
answer = migratePowerFlexVolume(command, libvirtComputingResource);
87+
} else if (shouldAttemptFiberChannelLiveMigration(command, srcPrimaryDataStore, destPrimaryDataStore)) {
88+
answer = migrateFiberChannelVolume(command, libvirtComputingResource);
8289
} else {
8390
answer = migrateRegularVolume(command, libvirtComputingResource);
8491
}
8592

8693
return answer;
8794
}
8895

96+
private String resolveVolumeIdentifier(VolumeObjectTO volumeObjectTO, Map<String, String> details) {
97+
if (details != null && details.get(DiskTO.IQN) != null) {
98+
return details.get(DiskTO.IQN);
99+
}
100+
return volumeObjectTO != null ? volumeObjectTO.getPath() : null;
101+
}
102+
103+
private DiskDef locateSourceDiskDefinition(List<DiskDef> disks, KVMPhysicalDisk srcPhysicalDisk, Map<String, String> srcDetails) {
104+
String expectedPath = srcPhysicalDisk != null ? srcPhysicalDisk.getPath() : null;
105+
String expectedSerial = srcDetails != null ? srcDetails.get(DiskTO.SCSI_NAA_DEVICE_ID) : null;
106+
107+
if (StringUtils.isNotBlank(expectedPath)) {
108+
for (DiskDef disk : disks) {
109+
if (StringUtils.isNotBlank(disk.getDiskPath()) && pathsReferToSameDevice(expectedPath, disk.getDiskPath())) {
110+
return disk;
111+
}
112+
}
113+
}
114+
115+
if (StringUtils.isNotBlank(expectedSerial)) {
116+
for (DiskDef disk : disks) {
117+
if (expectedSerial.equalsIgnoreCase(disk.getSerial())) {
118+
return disk;
119+
}
120+
}
121+
}
122+
123+
return null;
124+
}
125+
126+
private boolean pathsReferToSameDevice(String expectedPath, String candidatePath) {
127+
if (StringUtils.equals(expectedPath, candidatePath)) {
128+
return true;
129+
}
130+
String expectedToken = extractLastToken(expectedPath);
131+
String candidateToken = extractLastToken(candidatePath);
132+
return StringUtils.isNotBlank(expectedToken) && StringUtils.equalsIgnoreCase(expectedToken, candidateToken);
133+
}
134+
135+
private String extractLastToken(String path) {
136+
if (StringUtils.isBlank(path)) {
137+
return null;
138+
}
139+
int idx = path.lastIndexOf('/');
140+
return idx >= 0 ? path.substring(idx + 1) : path;
141+
}
142+
143+
private String buildFiberChannelDestinationDiskXml(DiskDef sourceDiskDef, String destDevicePath) {
144+
StringBuilder diskXml = new StringBuilder();
145+
diskXml.append("<disk type='block'>");
146+
diskXml.append("<driver name='qemu' type='raw' cache='none'/>");
147+
diskXml.append("<source dev='").append(destDevicePath).append("'/>");
148+
diskXml.append("<target dev='").append(sourceDiskDef.getDiskLabel()).append("' bus='")
149+
.append(sourceDiskDef.getBusType().toString()).append("'/>");
150+
diskXml.append("</disk>");
151+
return diskXml.toString();
152+
}
153+
89154
protected MigrateVolumeAnswer migratePowerFlexVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) {
90155

91156
// Source Details
@@ -183,6 +248,152 @@ protected MigrateVolumeAnswer migratePowerFlexVolume(final MigrateVolumeCommand
183248
}
184249
}
185250

251+
protected boolean shouldAttemptFiberChannelLiveMigration(MigrateVolumeCommand command, PrimaryDataStoreTO srcPrimaryDataStore, PrimaryDataStoreTO destPrimaryDataStore) {
252+
if (srcPrimaryDataStore == null || destPrimaryDataStore == null) {
253+
return false;
254+
}
255+
256+
if (!Storage.StoragePoolType.FiberChannel.equals(srcPrimaryDataStore.getPoolType()) ||
257+
!Storage.StoragePoolType.FiberChannel.equals(destPrimaryDataStore.getPoolType())) {
258+
return false;
259+
}
260+
261+
VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData();
262+
263+
return srcVolumeObjectTO != null && StringUtils.isNotBlank(srcVolumeObjectTO.getVmName());
264+
}
265+
266+
protected MigrateVolumeAnswer migrateFiberChannelVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) {
267+
VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData();
268+
VolumeObjectTO destVolumeObjectTO = (VolumeObjectTO)command.getDestData();
269+
if (srcVolumeObjectTO == null || destVolumeObjectTO == null || StringUtils.isBlank(srcVolumeObjectTO.getVmName())) {
270+
return migrateRegularVolume(command, libvirtComputingResource);
271+
}
272+
273+
String vmName = srcVolumeObjectTO.getVmName();
274+
Map<String, String> srcDetails = command.getSrcDetails();
275+
Map<String, String> destDetails = command.getDestDetails();
276+
277+
PrimaryDataStoreTO srcPrimaryDataStore = (PrimaryDataStoreTO)srcVolumeObjectTO.getDataStore();
278+
PrimaryDataStoreTO destPrimaryDataStore = (PrimaryDataStoreTO)destVolumeObjectTO.getDataStore();
279+
280+
KVMStoragePoolManager storagePoolManager = libvirtComputingResource.getStoragePoolMgr();
281+
KVMStoragePool sourceStoragePool = null;
282+
KVMStoragePool destStoragePool = null;
283+
Domain dm = null;
284+
VolumeMigrationCancelHook cancelHook = null;
285+
boolean migrationSucceeded = false;
286+
287+
String srcIdentifier = resolveVolumeIdentifier(srcVolumeObjectTO, srcDetails);
288+
String destIdentifier = resolveVolumeIdentifier(destVolumeObjectTO, destDetails);
289+
290+
if (StringUtils.isBlank(srcIdentifier) || StringUtils.isBlank(destIdentifier)) {
291+
return migrateRegularVolume(command, libvirtComputingResource);
292+
}
293+
294+
try {
295+
sourceStoragePool = storagePoolManager.getStoragePool(srcPrimaryDataStore.getPoolType(), srcPrimaryDataStore.getUuid());
296+
if (!sourceStoragePool.connectPhysicalDisk(srcIdentifier, srcDetails)) {
297+
return new MigrateVolumeAnswer(command, false, "Unable to connect source Fibre Channel volume on hypervisor", srcIdentifier);
298+
}
299+
KVMPhysicalDisk srcPhysicalDisk = storagePoolManager.getPhysicalDisk(srcPrimaryDataStore.getPoolType(), srcPrimaryDataStore.getUuid(), srcIdentifier);
300+
if (srcPhysicalDisk == null) {
301+
return new MigrateVolumeAnswer(command, false, "Unable to obtain source Fibre Channel disk handle", srcIdentifier);
302+
}
303+
304+
destStoragePool = storagePoolManager.getStoragePool(destPrimaryDataStore.getPoolType(), destPrimaryDataStore.getUuid());
305+
if (!destStoragePool.connectPhysicalDisk(destIdentifier, destDetails)) {
306+
return new MigrateVolumeAnswer(command, false, "Unable to connect destination Fibre Channel volume on hypervisor", destIdentifier);
307+
}
308+
309+
if (destVolumeObjectTO.getPath() == null) {
310+
destVolumeObjectTO.setPath(destIdentifier);
311+
}
312+
313+
KVMPhysicalDisk destPhysicalDisk = storagePoolManager.getPhysicalDisk(destPrimaryDataStore.getPoolType(), destPrimaryDataStore.getUuid(), destIdentifier);
314+
if (destPhysicalDisk == null) {
315+
return new MigrateVolumeAnswer(command, false, "Unable to obtain destination Fibre Channel disk handle", destIdentifier);
316+
}
317+
String destDevicePath = destPhysicalDisk.getPath();
318+
319+
LibvirtUtilitiesHelper helper = libvirtComputingResource.getLibvirtUtilitiesHelper();
320+
Connect conn = helper.getConnection();
321+
dm = libvirtComputingResource.getDomain(conn, vmName);
322+
if (dm == null) {
323+
return new MigrateVolumeAnswer(command, false, "Unable to locate libvirt domain for VM " + vmName, null);
324+
}
325+
326+
DomainInfo.DomainState domainState = dm.getInfo().state;
327+
if (domainState != DomainInfo.DomainState.VIR_DOMAIN_RUNNING) {
328+
return migrateRegularVolume(command, libvirtComputingResource);
329+
}
330+
331+
List<DiskDef> disks = libvirtComputingResource.getDisks(conn, vmName);
332+
DiskDef srcDiskDef = locateSourceDiskDefinition(disks, srcPhysicalDisk, srcDetails);
333+
if (srcDiskDef == null) {
334+
return new MigrateVolumeAnswer(command, false, "Unable to match Fibre Channel disk within VM definition", null);
335+
}
336+
337+
String diskLabel = srcDiskDef.getDiskLabel();
338+
String destinationDiskXml = buildFiberChannelDestinationDiskXml(srcDiskDef, destDevicePath);
339+
340+
TypedUlongParameter parameter = new TypedUlongParameter("bandwidth", 0);
341+
TypedParameter[] parameters = new TypedParameter[] { parameter };
342+
343+
cancelHook = new VolumeMigrationCancelHook(dm, diskLabel);
344+
libvirtComputingResource.addDisconnectHook(cancelHook);
345+
346+
libvirtComputingResource.createOrUpdateLogFileForCommand(command, Command.State.PROCESSING_IN_BACKEND);
347+
348+
dm.blockCopy(diskLabel, destinationDiskXml, parameters, Domain.BlockCopyFlags.REUSE_EXT);
349+
350+
MigrateVolumeAnswer answer = checkBlockJobStatus(command, dm, diskLabel, srcPhysicalDisk.getPath(), destDevicePath, libvirtComputingResource, conn, null);
351+
migrationSucceeded = answer != null && answer.getResult();
352+
353+
if (answer != null) {
354+
libvirtComputingResource.createOrUpdateLogFileForCommand(command, migrationSucceeded ? Command.State.COMPLETED : Command.State.FAILED);
355+
return answer;
356+
}
357+
358+
migrationSucceeded = true;
359+
libvirtComputingResource.createOrUpdateLogFileForCommand(command, Command.State.COMPLETED);
360+
return new MigrateVolumeAnswer(command, true, null, destIdentifier);
361+
} catch (LibvirtException e) {
362+
logger.warn("Fibre Channel live volume migration failed due to libvirt error", e);
363+
libvirtComputingResource.createOrUpdateLogFileForCommand(command, Command.State.FAILED);
364+
return new MigrateVolumeAnswer(command, false, "Libvirt error during Fibre Channel migration: " + e.getMessage(), null);
365+
} catch (Exception e) {
366+
logger.warn("Fibre Channel live volume migration failed", e);
367+
libvirtComputingResource.createOrUpdateLogFileForCommand(command, Command.State.FAILED);
368+
return new MigrateVolumeAnswer(command, false, "Fibre Channel migration failed: " + e.getMessage(), null);
369+
} finally {
370+
if (cancelHook != null) {
371+
libvirtComputingResource.removeDisconnectHook(cancelHook);
372+
}
373+
if (dm != null) {
374+
try {
375+
dm.free();
376+
} catch (LibvirtException e) {
377+
logger.trace("Ignoring libvirt error while freeing domain", e);
378+
}
379+
}
380+
if (!migrationSucceeded && destStoragePool != null) {
381+
try {
382+
destStoragePool.disconnectPhysicalDisk(destIdentifier);
383+
} catch (Exception e) {
384+
logger.warn("Unable to disconnect destination Fibre Channel disk after failed migration", e);
385+
}
386+
}
387+
if (sourceStoragePool != null) {
388+
try {
389+
sourceStoragePool.disconnectPhysicalDisk(srcIdentifier);
390+
} catch (Exception e) {
391+
logger.warn("Unable to disconnect source Fibre Channel disk after migration", e);
392+
}
393+
}
394+
}
395+
}
396+
186397
protected MigrateVolumeAnswer checkBlockJobStatus(MigrateVolumeCommand command, Domain dm, String diskLabel, String srcPath, String destPath, LibvirtComputingResource libvirtComputingResource, Connect conn, String srcSecretUUID) throws LibvirtException {
187398
int timeBetweenTries = 1000; // Try more frequently (every sec) and return early if disk is found
188399
int waitTimeInSec = command.getWait();

0 commit comments

Comments
 (0)