Skip to content

Fix mount/unmount error handling and timeout in LibvirtRestoreBackupCommandWrapper #14006

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
abh1sar wants to merge 2 commits into
base: 4.22
Choose a base branch
from
+143 −16
Open

Fix mount/unmount error handling and timeout in LibvirtRestoreBackupCommandWrapper #14006

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ad63ada
kvm: detect mount failures and honour the configured timeouts on back…
abh1sar Aug 29, 2026
30b30f2
kvm: clean up after a failed mount and bound the unmount of a backup …
abh1sar Aug 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 33 additions & 13 deletions ...in/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,11 +98,11 @@ public Answer execute(RestoreBackupCommand command, LibvirtComputingResource ser
newVolumeId = getVolumeUuidFromPath(volumePath, volumePool);
Long size = command.getRestoreVolumeSizes().get(0);
restoreVolume(storagePoolMgr, backupPath, volumePool, volumePath, diskType, backupFile, size,
new Pair<>(vmName, command.getVmState()), mountDirectory, timeout);
new Pair<>(vmName, command.getVmState()), mountDirectory, timeout, mountTimeout);
} else if (Boolean.TRUE.equals(vmExists)) {
restoreVolumesOfExistingVM(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backedVolumeUUIDs, backupPath, backupFiles, mountDirectory, timeout);
restoreVolumesOfExistingVM(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backedVolumeUUIDs, backupPath, backupFiles, mountDirectory, timeout, mountTimeout);
} else {
restoreVolumesOfDestroyedVMs(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backupPath, backupFiles, mountDirectory, timeout);
restoreVolumesOfDestroyedVMs(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backupPath, backupFiles, mountDirectory, timeout, mountTimeout);
}
} catch (CloudRuntimeException e) {
String errorMessage = e.getMessage() != null ? e.getMessage() : "";
Expand All @@ -123,7 +123,7 @@ private void verifyBackupFile(String backupPath, String volUuid) {

private void restoreVolumesOfExistingVM(KVMStoragePoolManager storagePoolMgr, List<PrimaryDataStoreTO> restoreVolumePools,
List<String> restoreVolumePaths, List<String> backedVolumesUUIDs,
String backupPath, List<String> backupFiles, String mountDirectory, int timeout) {
String backupPath, List<String> backupFiles, String mountDirectory, int timeout, Integer mountTimeout) {
String diskType = "root";
try {
for (int idx = 0; idx < restoreVolumePaths.size(); idx++) {
Expand All @@ -140,13 +140,13 @@ private void restoreVolumesOfExistingVM(KVMStoragePoolManager storagePoolMgr, Li
}
}
} finally {
unmountBackupDirectory(mountDirectory);
unmountBackupDirectory(mountDirectory, mountTimeout);
deleteTemporaryDirectory(mountDirectory);
}
}

private void restoreVolumesOfDestroyedVMs(KVMStoragePoolManager storagePoolMgr, List<PrimaryDataStoreTO> volumePools,
List<String> volumePaths, String backupPath, List<String> backupFiles, String mountDirectory, int timeout) {
List<String> volumePaths, String backupPath, List<String> backupFiles, String mountDirectory, int timeout, Integer mountTimeout) {
String diskType = "root";
try {
for (int i = 0; i < volumePaths.size(); i++) {
Expand All @@ -162,13 +162,13 @@ private void restoreVolumesOfDestroyedVMs(KVMStoragePoolManager storagePoolMgr,
}
}
} finally {
unmountBackupDirectory(mountDirectory);
unmountBackupDirectory(mountDirectory, mountTimeout);
deleteTemporaryDirectory(mountDirectory);
}
}

private void restoreVolume(KVMStoragePoolManager storagePoolMgr, String backupPath, PrimaryDataStoreTO volumePool, String volumePath, String diskType, String backupFile,
Long size, Pair<String, VirtualMachine.State> vmNameAndState, String mountDirectory, int timeout) {
Long size, Pair<String, VirtualMachine.State> vmNameAndState, String mountDirectory, int timeout, Integer mountTimeout) {
String bkpPath;
String volumeUuid;
try {
Expand All @@ -185,7 +185,7 @@ private void restoreVolume(KVMStoragePoolManager storagePoolMgr, String backupPa
}
}
} finally {
unmountBackupDirectory(mountDirectory);
unmountBackupDirectory(mountDirectory, mountTimeout);
deleteTemporaryDirectory(mountDirectory);
}
}
Expand All @@ -201,6 +201,7 @@ private String mountBackupDirectory(String backupRepoAddress, String backupRepoT
logger.error("Failed to create the tmp mount directory {} for restore", mountDirectory, e);
throw new CloudRuntimeException("Failed to create the tmp mount directory for restore on the KVM host");
}
int exitValue;
try {
String mountPath = Script.getExecutableAbsolutePath("mount");
List<String> mountCmd = new ArrayList<>();
Expand All @@ -221,23 +222,42 @@ private String mountBackupDirectory(String backupRepoAddress, String backupRepoT
mountCmd.add("-o");
mountCmd.add(mountOptions);
}
Script.executeCommand(mountCmd.toArray(new String[0]));
exitValue = Script.executeCommandForExitValue(mountTimeout, mountCmd.toArray(new String[0]));
} catch (Exception e) {
logger.error("Failed to mount repository {} of type {} to the directory {}", backupRepoAddress, backupRepoType, mountDirectory, e);
throw new CloudRuntimeException("Failed to mount the backup repository on the KVM host");
}
if (exitValue != 0) {
logger.error("Failed to mount repository {} of type {} to the directory {}, mount exited with {}", backupRepoAddress,
backupRepoType, mountDirectory, exitValue);
removeTemporaryDirectoryQuietly(mountDirectory);
throw new CloudRuntimeException("Failed to mount the backup repository on the KVM host");
}
return mountDirectory;
}

private void unmountBackupDirectory(String backupDirectory) {
private void unmountBackupDirectory(String backupDirectory, Integer mountTimeout) {
int exitValue;
try {
String umountPath = Script.getExecutableAbsolutePath("umount");
String[] umountCmd = new String[] { "sudo", umountPath, backupDirectory };
Script.executeCommand(umountCmd);
exitValue = Script.executeCommandForExitValue(mountTimeout, umountCmd);
} catch (Exception e) {
Comment thread
abh1sar marked this conversation as resolved.
logger.error("Failed to unmount backup directory {}", backupDirectory, e);
throw new CloudRuntimeException("Failed to unmount the backup directory");
}
if (exitValue != 0) {
logger.error("Failed to unmount backup directory {}, umount exited with {}", backupDirectory, exitValue);
throw new CloudRuntimeException("Failed to unmount the backup directory");
}
}

private void removeTemporaryDirectoryQuietly(String backupDirectory) {
try {
Files.deleteIfExists(Paths.get(backupDirectory));
} catch (IOException e) {
logger.warn("Failed to remove the temporary mount directory {} after the mount failed.", backupDirectory, e);
}
}

private void deleteTemporaryDirectory(String backupDirectory) {
Expand Down Expand Up @@ -276,7 +296,7 @@ private boolean replaceVolumeWithBackup(KVMStoragePoolManager storagePoolMgr, Pr
}

String[] rsyncCmd = new String[] { Script.getExecutableAbsolutePath("rsync"), "-az", backupPath, volumePath };
int exitValue = Script.executeCommandForExitValue(rsyncCmd);
int exitValue = Script.executeCommandForExitValue(timeout, rsyncCmd);
return exitValue == 0;
}

Expand Down
113 changes: 110 additions & 3 deletions ...ava/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -261,8 +261,8 @@ public void testExecuteWithMountFailure() throws Exception {
filesMock.when(() -> Files.createTempDirectory(anyString())).thenReturn(tempPath);

try (MockedStatic<Script> scriptMock = mockStatic(Script.class)) {
scriptMock.when(() -> Script.executeCommand(any(String[].class)))
.thenThrow(new RuntimeException("failure")); // Mount failure
scriptMock.when(() -> Script.executeCommandForExitValue(anyLong(), any(String[].class)))
.thenReturn(1); // Mount failure

Answer result = wrapper.execute(command, libvirtComputingResource);

Expand Down Expand Up @@ -404,7 +404,7 @@ public void testExecuteWithRsyncFailure() throws Exception {
.thenAnswer(invocation -> invocation.getArgument(0));
scriptMock.when(() -> Script.executeCommand(any(String[].class)))
.thenReturn(null);
scriptMock.when(() -> Script.executeCommandForExitValue(any(String[].class)))
scriptMock.when(() -> Script.executeCommandForExitValue(anyLong(), any(String[].class)))
.thenAnswer(invocation -> {
if (Arrays.stream(invocation.getArguments()).map(String::valueOf).anyMatch("rsync"::equals)) {
return 1; // Rsync failure
Comment thread
abh1sar marked this conversation as resolved.
Expand Down Expand Up @@ -579,4 +579,111 @@ public void testExecuteWithMultipleVolumes() throws Exception {
}
}
}

@Test
public void testMountUsesTheConfiguredTimeout() throws Exception {
when(command.getVmName()).thenReturn("test-vm");
when(command.getBackupPath()).thenReturn("backup/path");
when(command.getBackupRepoAddress()).thenReturn("192.168.1.100:/backup");
when(command.getBackupRepoType()).thenReturn("nfs");
when(command.getMountOptions()).thenReturn("rw");
when(command.getMountTimeout()).thenReturn(30);

try (MockedStatic<Files> filesMock = mockStatic(Files.class)) {
Path tempPath = Mockito.mock(Path.class);
when(tempPath.toString()).thenReturn("/tmp/csbackup.abc123");
filesMock.when(() -> Files.createTempDirectory(anyString())).thenReturn(tempPath);

try (MockedStatic<Script> scriptMock = mockStatic(Script.class)) {
scriptMock.when(() -> Script.getExecutableAbsolutePath(anyString()))
.thenAnswer(invocation -> invocation.getArgument(0));
final long[] mountTimeout = new long[1];
scriptMock.when(() -> Script.executeCommandForExitValue(anyLong(), any(String[].class)))
.thenAnswer(invocation -> {
if (Arrays.stream(invocation.getArguments()).map(String::valueOf).anyMatch("mount"::equals)) {
mountTimeout[0] = invocation.getArgument(0);
return 1; // stop the restore right after the mount
}
return 0;
});

wrapper.execute(command, libvirtComputingResource);

Assert.assertEquals(30 * 1000L, mountTimeout[0]);
}
}
}

@Test
public void testMountFailureRemovesTheTemporaryDirectory() throws Exception {
when(command.getVmName()).thenReturn("test-vm");
when(command.getBackupPath()).thenReturn("backup/path");
when(command.getBackupRepoAddress()).thenReturn("192.168.1.100:/backup");
when(command.getBackupRepoType()).thenReturn("nfs");
when(command.getMountOptions()).thenReturn("rw");
when(command.getMountTimeout()).thenReturn(30);

try (MockedStatic<Files> filesMock = mockStatic(Files.class)) {
Path tempPath = Mockito.mock(Path.class);
when(tempPath.toString()).thenReturn("/tmp/csbackup.abc123");
filesMock.when(() -> Files.createTempDirectory(anyString())).thenReturn(tempPath);
filesMock.when(() -> Files.deleteIfExists(any(Path.class))).thenReturn(true);

try (MockedStatic<Script> scriptMock = mockStatic(Script.class)) {
scriptMock.when(() -> Script.getExecutableAbsolutePath(anyString()))
.thenAnswer(invocation -> invocation.getArgument(0));
scriptMock.when(() -> Script.executeCommandForExitValue(anyLong(), any(String[].class)))
.thenReturn(1); // the mount fails

wrapper.execute(command, libvirtComputingResource);

// the directory created for the mount must not be left behind
filesMock.verify(() -> Files.deleteIfExists(any(Path.class)));
}
}
}

@Test
public void testUnmountIsBoundedByTheConfiguredTimeout() throws Exception {
when(command.getVmName()).thenReturn("test-vm");
when(command.getBackupPath()).thenReturn("backup/path");
when(command.getBackupRepoAddress()).thenReturn("192.168.1.100:/backup");
when(command.getBackupRepoType()).thenReturn("nfs");
when(command.getMountOptions()).thenReturn("rw");
when(command.isVmExists()).thenReturn(true);
when(command.getDiskType()).thenReturn("root");
PrimaryDataStoreTO primaryDataStore = Mockito.mock(PrimaryDataStoreTO.class);
when(primaryDataStore.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem);
when(command.getRestoreVolumePools()).thenReturn(Arrays.asList(primaryDataStore));
when(command.getRestoreVolumePaths()).thenReturn(Arrays.asList("/var/lib/libvirt/images/volume-123"));
when(command.getBackupVolumesUUIDs()).thenReturn(Arrays.asList("volume-123"));
when(command.getBackupFiles()).thenReturn(Arrays.asList("volume-123"));
when(command.getMountTimeout()).thenReturn(30);

try (MockedStatic<Files> filesMock = mockStatic(Files.class)) {
Path tempPath = Mockito.mock(Path.class);
when(tempPath.toString()).thenReturn("/tmp/csbackup.abc123");
filesMock.when(() -> Files.createTempDirectory(anyString())).thenReturn(tempPath);
filesMock.when(() -> Files.deleteIfExists(any(Path.class))).thenReturn(true);

try (MockedStatic<Script> scriptMock = mockStatic(Script.class)) {
scriptMock.when(() -> Script.getExecutableAbsolutePath(anyString()))
.thenAnswer(invocation -> invocation.getArgument(0));
final long[] umountTimeout = new long[] { -1 };
scriptMock.when(() -> Script.executeCommandForExitValue(anyLong(), any(String[].class)))
.thenAnswer(invocation -> {
if (Arrays.stream(invocation.getArguments()).map(String::valueOf).anyMatch("umount"::equals)) {
umountTimeout[0] = invocation.getArgument(0);
}
return 0;
});
scriptMock.when(() -> Script.runSimpleBashScriptForExitValue(anyString())).thenReturn(0);

wrapper.execute(command, libvirtComputingResource);

// an unreachable repository blocks umount just as it blocks mount
Assert.assertEquals(30 * 1000L, umountTimeout[0]);
}
}
}
}
Loading