From 69ca70e27f462b0fde00a976c54355ab67d5fa0a Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 18:57:41 +0000 Subject: [PATCH 001/107] Support vendor VFIO vGPU devices Linux 6.8 hosts with NVIDIA R580 drop the mdev interface: vGPUs are assigned by writing a type ID to a VF's nvidia/current_vgpu_type and passed to QEMU as a plain VFIO PCI device. Add a vendor VFIO backend behind the existing framework dispatch: profile discovery from the capacity-dependent creatable catalogs, least-loaded VF placement, create/verify/rollback, and release. Because the same VF path is reused across assignments (unlike mdev UUIDs), release is guarded: an in-process owner map covers the window before QEMU opens the device, and an open-VFIO-handle scan refuses to clear a VF a running VM still holds. Reconciliation clears orphaned assignments on startup, skipping VFs protected by the caller and failing closed when the protected set is unavailable. Branch the vGPU integration test by discovered framework and extend it to cover release on stop and reacquisition on start. --- integration/vgpu_test.go | 110 ++++-- lib/devices/GPU.md | 46 +-- lib/devices/gpu_mode.go | 30 -- lib/devices/mdev_darwin.go | 13 +- lib/devices/mdev_linux.go | 32 +- lib/devices/types.go | 15 +- lib/devices/vendor_vfio_linux.go | 476 ++++++++++++++++++++++++++ lib/devices/vendor_vfio_linux_test.go | 374 ++++++++++++++++++++ lib/devices/vgpu_linux.go | 116 ++++++- lib/devices/vgpu_linux_test.go | 50 +++ lib/resources/gpu.go | 35 +- lib/resources/monitoring_test.go | 2 +- lib/resources/resource.go | 8 +- 13 files changed, 1159 insertions(+), 148 deletions(-) delete mode 100644 lib/devices/gpu_mode.go create mode 100644 lib/devices/vendor_vfio_linux.go create mode 100644 lib/devices/vendor_vfio_linux_test.go create mode 100644 lib/devices/vgpu_linux_test.go diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 12861a6a7..3f8fdfd65 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -4,6 +4,8 @@ import ( "bytes" "context" "os" + "path/filepath" + "strings" "testing" "time" @@ -21,21 +23,23 @@ import ( "github.com/stretchr/testify/require" ) -// TestVGPU is an integration test that verifies vGPU (SR-IOV mdev) support works. +// TestVGPU is an integration test that verifies vGPU (SR-IOV) support works +// on the host's framework: mdev or NVIDIA's vendor-specific VFIO. // // This test automatically detects vGPU availability and skips if: -// - No SR-IOV VFs are found in /sys/class/mdev_bus/ +// - No vGPU framework (mdev or vendor VFIO) is discovered // - No vGPU profiles are available -// - Not running as root (required for mdev creation) +// - Not running as root (required for sysfs vGPU assignment) // - KVM is not available // // To run manually: // // sudo go test -v -run TestVGPU -timeout 5m ./integration/... // -// Note: This test verifies mdev creation and PCI device visibility inside the VM. -// It does NOT test nvidia-smi or CUDA functionality since that requires NVIDIA -// guest drivers pre-installed in the image. +// Note: This test verifies vGPU assignment, release on stop, reacquisition on +// start, and PCI device visibility inside the VM. It does NOT test nvidia-smi +// or CUDA functionality since that requires NVIDIA guest drivers pre-installed +// in the image. func TestVGPU(t *testing.T) { t.Parallel() if testing.Short() { @@ -159,9 +163,18 @@ func TestVGPU(t *testing.T) { instanceID = inst.Id t.Logf("Instance created: %s", inst.Id) - // Verify mdev UUID was assigned - require.NotEmpty(t, inst.GPUMdevUUID, "Instance should have mdev UUID assigned") - t.Logf("mdev UUID: %s", inst.GPUMdevUUID) + // Verify the assignment matches the host's framework + require.NotEmpty(t, inst.GPUDevicePath, "Instance should have a vGPU device path assigned") + switch inst.GPUFramework { + case devices.VGPUFrameworkMdev: + require.NotEmpty(t, inst.GPUMdevUUID, "mdev instance should have a UUID assigned") + t.Logf("mdev UUID: %s", inst.GPUMdevUUID) + case devices.VGPUFrameworkVendorVFIO: + require.Empty(t, inst.GPUMdevUUID, "vendor VFIO instance should not have an mdev UUID") + t.Logf("vendor VFIO VF: %s", inst.GPUDevicePath) + default: + t.Fatalf("unexpected vGPU framework %q", inst.GPUFramework) + } // Step 5: Check GPU resources AFTER creating instance t.Run("ResourcesDecrementedAfterCreation", func(t *testing.T) { @@ -180,12 +193,9 @@ func TestVGPU(t *testing.T) { assert.Less(t, availableAfter, availableBefore, "available instances should decrease after creating VM") }) - // Step 6: Verify mdev was created in sysfs - t.Run("MdevCreated", func(t *testing.T) { - mdevPath := "/sys/bus/mdev/devices/" + inst.GPUMdevUUID - _, err := os.Stat(mdevPath) - assert.NoError(t, err, "mdev device should exist at %s", mdevPath) - t.Logf("mdev exists at: %s", mdevPath) + // Step 6: Verify the assignment exists in sysfs + t.Run("VGPUAssignedInSysfs", func(t *testing.T) { + assertVGPUAssigned(t, inst.GPUFramework, inst.GPUDevicePath) }) // Step 7: Wait for guest agent to be ready @@ -225,13 +235,68 @@ func TestVGPU(t *testing.T) { require.NoError(t, err) assert.Equal(t, profile, actualInst.GPUProfile, "GPU profile should match") - assert.NotEmpty(t, actualInst.GPUMdevUUID, "mdev UUID should be set") - t.Logf("Instance GPU: profile=%s, mdev=%s", actualInst.GPUProfile, actualInst.GPUMdevUUID) + assert.Equal(t, inst.GPUFramework, actualInst.GPUFramework, "framework should match") + assert.NotEmpty(t, actualInst.GPUDevicePath, "device path should be set") + if inst.GPUFramework == devices.VGPUFrameworkMdev { + assert.NotEmpty(t, actualInst.GPUMdevUUID, "mdev UUID should be set") + } + t.Logf("Instance GPU: profile=%s, framework=%s, device=%s", actualInst.GPUProfile, actualInst.GPUFramework, actualInst.GPUDevicePath) + }) + + t.Log("Step 10: Stopping instance to release the vGPU...") + _, err = instanceManager.StopInstance(ctx, inst.Id) + require.NoError(t, err, "stop should succeed") + + t.Run("VGPUReleasedOnStop", func(t *testing.T) { + stopped, err := instanceManager.GetInstance(ctx, inst.Id) + require.NoError(t, err) + assert.Empty(t, stopped.GPUDevicePath, "assignment metadata should be cleared on stop") + assertVGPUReleased(t, inst.GPUFramework, inst.GPUDevicePath) + }) + + t.Log("Step 11: Starting instance to reacquire a vGPU...") + started, err := instanceManager.StartInstance(ctx, inst.Id, instances.StartInstanceRequest{}) + require.NoError(t, err, "start should succeed") + + t.Run("VGPUReacquiredOnStart", func(t *testing.T) { + require.NotEmpty(t, started.GPUDevicePath, "start should assign a vGPU") + assert.Equal(t, inst.GPUFramework, started.GPUFramework, "framework should match") + assertVGPUAssigned(t, started.GPUFramework, started.GPUDevicePath) }) t.Log("✅ vGPU test PASSED!") } +func assertVGPUAssigned(t *testing.T, framework devices.VGPUFramework, devicePath string) { + t.Helper() + switch framework { + case devices.VGPUFrameworkMdev: + _, err := os.Stat(devicePath) + assert.NoError(t, err, "mdev device should exist at %s", devicePath) + case devices.VGPUFrameworkVendorVFIO: + data, err := os.ReadFile(filepath.Join(devicePath, "nvidia", "current_vgpu_type")) + require.NoError(t, err, "VF should expose current_vgpu_type") + assert.NotEqual(t, "0", strings.TrimSpace(string(data)), "VF should have a vGPU type assigned") + default: + t.Fatalf("unexpected vGPU framework %q", framework) + } +} + +func assertVGPUReleased(t *testing.T, framework devices.VGPUFramework, devicePath string) { + t.Helper() + switch framework { + case devices.VGPUFrameworkMdev: + _, err := os.Stat(devicePath) + assert.True(t, os.IsNotExist(err), "mdev device should be gone from %s", devicePath) + case devices.VGPUFrameworkVendorVFIO: + data, err := os.ReadFile(filepath.Join(devicePath, "nvidia", "current_vgpu_type")) + require.NoError(t, err, "VF should expose current_vgpu_type") + assert.Equal(t, "0", strings.TrimSpace(string(data)), "VF assignment should be released") + default: + t.Fatalf("unexpected vGPU framework %q", framework) + } +} + // checkVGPUTestPrerequisites checks if vGPU test can run. // Returns (skipReason, profileName) - skipReason is empty if all prerequisites are met. func checkVGPUTestPrerequisites() (string, string) { @@ -245,10 +310,13 @@ func checkVGPUTestPrerequisites() (string, string) { return "vGPU test requires root (sudo) for mdev creation", "" } - // Check for vGPU mode (SR-IOV VFs present) - mode := devices.DetectHostGPUMode() - if mode != devices.GPUModeVGPU { - return "vGPU test requires SR-IOV VFs in /sys/class/mdev_bus/", "" + // Check for a vGPU framework (mdev or vendor VFIO) + framework, _, err := devices.DiscoverVGPU() + if err != nil { + return "vGPU test failed to discover vGPU framework: " + err.Error(), "" + } + if framework == devices.VGPUFrameworkNone { + return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", "" } // Check for available profiles diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 54a19c472..cdd269dc8 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -8,16 +8,17 @@ hypeman supports two GPU modes, automatically detected based on host configurati | Mode | Description | Use Case | |------|-------------|----------| -| **vGPU (SR-IOV)** | Virtual GPUs via mdev on SR-IOV VFs | Multi-tenant, shared GPU resources | +| **vGPU (SR-IOV)** | Virtual GPUs on SR-IOV VFs via mdev or vendor VFIO | Multi-tenant, shared GPU resources | | **Passthrough** | Whole GPU VFIO passthrough | Dedicated GPU per instance | The host's GPU mode is determined by the host driver configuration: -- If `/sys/class/mdev_bus/` contains VFs → vGPU mode -- If NVIDIA GPUs are available for VFIO → passthrough mode +- If `/sys/class/mdev_bus/` contains VFs → mdev vGPU mode +- If VFs expose `/sys/bus/pci/devices//nvidia/current_vgpu_type` → vendor VFIO vGPU mode +- If NVIDIA GPUs are available for whole-device VFIO → passthrough mode ## vGPU Mode (Recommended) -vGPU mode uses NVIDIA's SR-IOV technology to create Virtual Functions (VFs), each capable of hosting an mdev (mediated device) representing a vGPU. +vGPU mode uses NVIDIA's SR-IOV technology to create Virtual Functions (VFs). Hosts on older kernels represent each vGPU as an mdev. Hosts using NVIDIA's vendor VFIO framework assign the profile directly to the VF through `current_vgpu_type`. ### How It Works @@ -74,7 +75,7 @@ curl -X POST http://localhost:4973/instances \ }' ``` -The response includes the assigned mdev UUID: +On an mdev host, the response also includes the assigned mdev UUID: ```json { @@ -87,19 +88,16 @@ The response includes the assigned mdev UUID: } ``` -### Ephemeral mdev Lifecycle +### Ephemeral vGPU Lifecycle -mdev devices are **ephemeral**: created on instance start, destroyed on instance delete. +vGPU assignments are created on instance start and released on stop or delete. Hypeman creates/removes an mdev on mdev hosts and writes the profile ID/`0` to `current_vgpu_type` on vendor VFIO hosts. ``` -Instance Create → Create mdev → Attach to VM → Instance Running -Instance Delete → Stop VM → Destroy mdev → VF available again +Instance Create → Assign profile to VF → Attach VF to VM → Instance Running +Instance Stop/Delete → Release profile → VF available again ``` -This ensures: -- **Security**: No VRAM data leakage between instances -- **Clean state**: Fresh vGPU for each instance -- **Automatic cleanup**: Orphaned mdevs cleaned up on server restart +Hypeman reconciles orphaned assignments on server restart while preserving devices held open by a running VMM. ## Passthrough Mode @@ -241,7 +239,8 @@ To upgrade the NVIDIA driver version: 1. Check host GPU mode detection: ```bash - ls /sys/class/mdev_bus/ # Should show VFs for vGPU mode + ls /sys/class/mdev_bus/ + find /sys/bus/pci/devices -path '*/nvidia/current_vgpu_type' ``` 2. Verify NVIDIA drivers are loaded on host: @@ -265,17 +264,18 @@ curl -s http://localhost:4973/resources | jq '.gpu.profiles' curl http://localhost:4973/instances//logs?source=app ``` -### mdev creation fails +### vGPU assignment fails -1. Check if VFs are available: - ```bash - ls /sys/class/mdev_bus/ - ``` +Check the files for the framework detected on the host: -2. Verify mdev types: - ```bash - cat /sys/class/mdev_bus/*/mdev_supported_types/*/available_instances - ``` +```bash +# mdev +cat /sys/class/mdev_bus/*/mdev_supported_types/*/available_instances + +# vendor VFIO +cat /sys/bus/pci/devices/*/nvidia/creatable_vgpu_types +cat /sys/bus/pci/devices/*/nvidia/current_vgpu_type +``` ## Performance Tuning diff --git a/lib/devices/gpu_mode.go b/lib/devices/gpu_mode.go deleted file mode 100644 index 40b3b2ba0..000000000 --- a/lib/devices/gpu_mode.go +++ /dev/null @@ -1,30 +0,0 @@ -package devices - -import ( - "os" -) - -// DetectHostGPUMode determines the host's GPU configuration mode. -// -// Returns: -// - GPUModeVGPU if /sys/class/mdev_bus has entries (SR-IOV VFs present) -// - GPUModePassthrough if NVIDIA GPUs are available for VFIO passthrough -// - GPUModeNone if no GPUs are available -// -// Note: A host is configured for either vGPU or passthrough, not both, -// because the host driver determines which mode is available. -func DetectHostGPUMode() GPUMode { - // Check for vGPU mode first (SR-IOV VFs present) - entries, err := os.ReadDir("/sys/class/mdev_bus") - if err == nil && len(entries) > 0 { - return GPUModeVGPU - } - - // Check for passthrough mode (physical GPUs available) - gpus, err := DiscoverAvailableDevices() - if err == nil && len(gpus) > 0 { - return GPUModePassthrough - } - - return GPUModeNone -} diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index 1427a5095..4274063ed 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -12,10 +12,9 @@ func SetGPUProfileCacheTTL(ttl string) { // No-op on macOS } -// DiscoverVFs returns an empty list on macOS. -// SR-IOV Virtual Functions are not available on macOS. -func DiscoverVFs() ([]VirtualFunction, error) { - return []VirtualFunction{}, nil +// DiscoverVGPU reports no vGPU framework on macOS. +func DiscoverVGPU() (VGPUFramework, []VirtualFunction, error) { + return VGPUFrameworkNone, nil, nil } // ListGPUProfiles returns an empty list on macOS. @@ -24,7 +23,7 @@ func ListGPUProfiles() ([]GPUProfile, error) { } // ListGPUProfilesWithVFs returns an empty list on macOS. -func ListGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { +func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction) ([]GPUProfile, error) { return []GPUProfile{}, nil } @@ -59,6 +58,10 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { return nil } +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}) error { + return nil +} + // ReconcileMdevs is a no-op on macOS. func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) error { return nil diff --git a/lib/devices/mdev_linux.go b/lib/devices/mdev_linux.go index 1a398a418..e2891efc9 100644 --- a/lib/devices/mdev_linux.go +++ b/lib/devices/mdev_linux.go @@ -89,14 +89,13 @@ func getCachedProfiles(firstVF string) []profileMetadata { return cachedProfiles } -// DiscoverVFs returns all SR-IOV Virtual Functions available for vGPU. -// These are discovered by scanning /sys/class/mdev_bus/ which contains -// VFs that can host mdev devices. -func DiscoverVFs() ([]VirtualFunction, error) { +// discoverMdevVFs returns all SR-IOV Virtual Functions available for vGPU, +// discovered by scanning /sys/class/mdev_bus/. +func discoverMdevVFs() ([]VirtualFunction, error) { entries, err := os.ReadDir(mdevBusPath) if err != nil { if os.IsNotExist(err) { - return nil, nil // No mdev_bus means no vGPU support + return nil, nil // No mdev_bus means no mdev vGPU support } return nil, fmt.Errorf("read mdev_bus: %w", err) } @@ -133,20 +132,9 @@ func DiscoverVFs() ([]VirtualFunction, error) { return vfs, nil } -// ListGPUProfiles returns available vGPU profiles with availability counts. -// Profiles are discovered from the first VF's mdev_supported_types directory. -func ListGPUProfiles() ([]GPUProfile, error) { - vfs, err := DiscoverVFs() - if err != nil { - return nil, err - } - return ListGPUProfilesWithVFs(vfs) -} - -// ListGPUProfilesWithVFs returns available vGPU profiles using pre-discovered VFs. -// This avoids redundant VF discovery when the caller already has the list. -// Uses parallel sysfs reads for fast availability counting. -func ListGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { +// listMdevGPUProfilesWithVFs returns available vGPU profiles using +// pre-discovered VFs. Uses parallel sysfs reads for fast availability counting. +func listMdevGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { if len(vfs) == 0 { return nil, nil } @@ -305,7 +293,7 @@ func countAvailableForSingleProfile(freeVFsByParent map[string][]VirtualFunction // findProfileType finds the internal type name (e.g., "nvidia-556") for a profile name (e.g., "L40S-1Q") func findProfileType(profileName string) (string, error) { - vfs, err := DiscoverVFs() + vfs, err := discoverMdevVFs() if err != nil || len(vfs) == 0 { return "", fmt.Errorf("no VFs available") } @@ -531,7 +519,7 @@ func CreateMdev(ctx context.Context, profileName, instanceID string) (*MdevDevic } // Discover all VFs - vfs, err := DiscoverVFs() + vfs, err := discoverMdevVFs() if err != nil { return nil, fmt.Errorf("discover VFs: %w", err) } @@ -697,7 +685,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro log := logger.FromContext(ctx) _ = instanceInfos - vfs, err := DiscoverVFs() + vfs, err := discoverMdevVFs() if err != nil { return fmt.Errorf("discover managed VFs: %w", err) } diff --git a/lib/devices/types.go b/lib/devices/types.go index 809d669fe..c76d239ed 100644 --- a/lib/devices/types.go +++ b/lib/devices/types.go @@ -63,12 +63,13 @@ type GPUMode string type VGPUFramework string const ( - VGPUFrameworkNone VGPUFramework = "" - VGPUFrameworkMdev VGPUFramework = "mdev" + VGPUFrameworkNone VGPUFramework = "" + VGPUFrameworkMdev VGPUFramework = "mdev" + VGPUFrameworkVendorVFIO VGPUFramework = "vendor-vfio" // GPUModePassthrough indicates whole GPU VFIO passthrough GPUModePassthrough GPUMode = "passthrough" - // GPUModeVGPU indicates SR-IOV + mdev based vGPU + // GPUModeVGPU indicates vGPU mode GPUModeVGPU GPUMode = "vgpu" // GPUModeNone indicates no GPU available GPUModeNone GPUMode = "none" @@ -76,9 +77,10 @@ const ( // VirtualFunction represents an SR-IOV Virtual Function for vGPU type VirtualFunction struct { - PCIAddress string `json:"pci_address"` // e.g., "0000:82:00.4" - ParentGPU string `json:"parent_gpu"` // e.g., "0000:82:00.0" - Allocated bool `json:"allocated"` // true if a vGPU is assigned to this VF + PCIAddress string `json:"pci_address"` // e.g., "0000:82:00.4" + ParentGPU string `json:"parent_gpu"` // e.g., "0000:82:00.0" + Allocated bool `json:"allocated"` // true if a vGPU is assigned to this VF + ProfileType string `json:"profile_type,omitempty"` } // VGPUAssignment identifies an existing vGPU assignment to release. @@ -86,6 +88,7 @@ type VGPUAssignment struct { Framework VGPUFramework DevicePath string MdevUUID string + InstanceID string } type VGPUDevice struct { diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go new file mode 100644 index 000000000..23ab1e89a --- /dev/null +++ b/lib/devices/vendor_vfio_linux.go @@ -0,0 +1,476 @@ +//go:build linux + +package devices + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + + "github.com/kernel/hypeman/lib/logger" +) + +const ( + pciDevicesPath = "/sys/bus/pci/devices" + vfioDevicesPath = "/dev/vfio/devices" +) + +type vendorVFIOSysfs struct { + pciDevicesPath string + procPath string + vfioDevicesPath string + owners map[string]string +} + +var ( + hostVendorVFIO = vendorVFIOSysfs{ + pciDevicesPath: pciDevicesPath, + procPath: procPath, + vfioDevicesPath: vfioDevicesPath, + owners: make(map[string]string), + } + vendorVFIOMu sync.Mutex +) + +func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { + entries, err := os.ReadDir(s.pciDevicesPath) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read PCI devices: %w", err) + } + + vfs := make([]VirtualFunction, 0) + for _, entry := range entries { + vfPath := filepath.Join(s.pciDevicesPath, entry.Name()) + nvidiaPath := filepath.Join(vfPath, "nvidia") + if _, err := os.Stat(filepath.Join(nvidiaPath, "creatable_vgpu_types")); err != nil { + if os.IsNotExist(err) { + continue + } + return nil, fmt.Errorf("stat creatable vGPU types for VF %s: %w", entry.Name(), err) + } + + currentType, err := readCurrentVGPUType(filepath.Join(nvidiaPath, "current_vgpu_type")) + if err != nil { + return nil, fmt.Errorf("read current vGPU type for VF %s: %w", entry.Name(), err) + } + + parentGPU := "" + if target, err := os.Readlink(filepath.Join(vfPath, "physfn")); err == nil { + parentGPU = filepath.Base(target) + } + vfs = append(vfs, VirtualFunction{ + PCIAddress: entry.Name(), + ParentGPU: parentGPU, + Allocated: currentType != "0", + ProfileType: currentType, + }) + } + + sort.Slice(vfs, func(i, j int) bool { return vfs[i].PCIAddress < vfs[j].PCIAddress }) + return vfs, nil +} + +func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { + profilesByType := make(map[string]profileMetadata) + availability := make(map[string]int) + for _, vf := range vfs { + creatable, err := s.readCreatableProfiles(vf.PCIAddress) + if err != nil { + return nil, err + } + for _, profile := range creatable { + profilesByType[profile.TypeName] = profile + if !vf.Allocated { + availability[profile.TypeName]++ + } + } + } + + metadata := make([]profileMetadata, 0, len(profilesByType)) + for _, profile := range profilesByType { + metadata = append(metadata, profile) + } + sort.Slice(metadata, func(i, j int) bool { return metadata[i].Name < metadata[j].Name }) + + profiles := make([]GPUProfile, 0, len(metadata)) + for _, profile := range metadata { + profiles = append(profiles, GPUProfile{ + Name: profile.Name, + FramebufferMB: profile.FramebufferMB, + Available: availability[profile.TypeName], + }) + } + return profiles, nil +} + +func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) { + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + + vfs, err := s.discoverVFs() + if err != nil { + return nil, err + } + metadata, err := s.profileMetadata(vfs) + if err != nil { + return nil, err + } + + var requested profileMetadata + found := false + for _, profile := range metadata { + if profile.Name == profileName { + requested = profile + found = true + break + } + } + if !found { + if len(metadata) == 0 && len(vfs) > 0 { + return nil, fmt.Errorf("no creatable vGPU profiles on any VF, GPUs may be at capacity: profile %q", profileName) + } + return nil, fmt.Errorf("profile %q is not creatable on any VF (unknown profile or insufficient capacity)", profileName) + } + + targetVF, err := s.selectLeastLoadedVF(vfs, metadata, requested.TypeName) + if err != nil { + return nil, err + } + if targetVF == "" { + return nil, fmt.Errorf("no available VF for profile %q", profileName) + } + + currentTypePath := filepath.Join(s.pciDevicesPath, targetVF, "nvidia", "current_vgpu_type") + if err := os.WriteFile(currentTypePath, []byte(requested.TypeName), 0200); err != nil { + return nil, fmt.Errorf("create vGPU on VF %s: %w", targetVF, err) + } + currentType, err := readCurrentVGPUType(currentTypePath) + if err != nil { + verifyErr := fmt.Errorf("verify vGPU on VF %s: %w", targetVF, err) + return nil, rollbackVendorVFIOCreate(currentTypePath, targetVF, verifyErr) + } + if currentType != requested.TypeName { + verifyErr := fmt.Errorf("verify vGPU on VF %s: type is %s, want %s", targetVF, currentType, requested.TypeName) + return nil, rollbackVendorVFIOCreate(currentTypePath, targetVF, verifyErr) + } + s.owners[targetVF] = instanceID + + logger.FromContext(ctx).InfoContext(ctx, "created vendor VFIO vGPU", + "profile", profileName, + "vf", targetVF, + "instance_id", instanceID, + ) + return &VGPUDevice{ + Framework: VGPUFrameworkVendorVFIO, + VFAddress: targetVF, + ProfileType: requested.TypeName, + ProfileName: profileName, + SysfsPath: filepath.Join(s.pciDevicesPath, targetVF), + }, nil +} + +func (s vendorVFIOSysfs) destroy(ctx context.Context, vfAddress, instanceID string) error { + return s.destroyWithOpenPaths(ctx, vfAddress, instanceID, nil) +} + +func (s vendorVFIOSysfs) destroyWithOpenPaths(ctx context.Context, vfAddress, instanceID string, openPaths map[string]struct{}) error { + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + + log := logger.FromContext(ctx) + currentTypePath := filepath.Join(s.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type") + currentType, err := readCurrentVGPUType(currentTypePath) + if err != nil { + if os.IsNotExist(err) { + delete(s.owners, vfAddress) + return nil + } + return fmt.Errorf("read current vGPU type for VF %s: %w", vfAddress, err) + } + if currentType == "0" { + delete(s.owners, vfAddress) + return nil + } + + if owner, ok := s.owners[vfAddress]; ok && (instanceID == "" || owner != instanceID) { + log.WarnContext(ctx, "skipping vendor VFIO vGPU release owned by another instance", + "vf", vfAddress, + "owner_instance_id", owner, + "requesting_instance_id", instanceID, + ) + return nil + } + + if openPaths == nil { + if openPaths, err = s.openVFIOPaths(); err != nil { + return fmt.Errorf("scan open VFIO handles: %w", err) + } + } + inUse, err := s.vfioDeviceInUse(vfAddress, openPaths) + if err != nil { + return fmt.Errorf("check vendor VFIO vGPU usage for VF %s: %w", vfAddress, err) + } + if inUse { + return fmt.Errorf("vendor VFIO vGPU on VF %s is still in use", vfAddress) + } + + if err := os.WriteFile(currentTypePath, []byte("0"), 0200); err != nil { + return fmt.Errorf("destroy vGPU on VF %s: %w", vfAddress, err) + } + delete(s.owners, vfAddress) + log.InfoContext(ctx, "destroyed vendor VFIO vGPU", "vf", vfAddress) + return nil +} + +func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map[string]struct{}) error { + vfs, err := s.discoverVFs() + if err != nil { + return err + } + log := logger.FromContext(ctx) + protectedVFs := make(map[string]struct{}, len(protectedDevicePaths)) + for path := range protectedDevicePaths { + protectedVFs[filepath.Base(path)] = struct{}{} + } + var openPaths map[string]struct{} + for _, vf := range vfs { + if !vf.Allocated { + continue + } + if _, ok := protectedVFs[vf.PCIAddress]; ok { + log.DebugContext(ctx, "skipping vendor VFIO vGPU held by a live instance", "vf", vf.PCIAddress) + continue + } + if openPaths == nil { + if openPaths, err = s.openVFIOPaths(); err != nil { + return fmt.Errorf("scan open VFIO handles: %w", err) + } + } + inUse, err := s.vfioDeviceInUse(vf.PCIAddress, openPaths) + if err != nil { + log.WarnContext(ctx, "failed to check vendor VFIO vGPU usage", "vf", vf.PCIAddress, "error", err) + continue + } + if inUse { + continue + } + if err := s.destroyWithOpenPaths(ctx, vf.PCIAddress, "", openPaths); err != nil { + log.WarnContext(ctx, "failed to destroy orphaned vendor VFIO vGPU", "vf", vf.PCIAddress, "error", err) + } + } + return nil +} + +func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, metadata []profileMetadata, profileType string) (string, error) { + framebufferByType := make(map[string]int, len(metadata)) + for _, profile := range metadata { + framebufferByType[profile.TypeName] = profile.FramebufferMB + } + + usageByGPU := make(map[string]int) + freeByGPU := make(map[string][]VirtualFunction) + for _, vf := range vfs { + if vf.Allocated { + usageByGPU[vf.ParentGPU] += framebufferByType[vf.ProfileType] + continue + } + profiles, err := s.readCreatableProfiles(vf.PCIAddress) + if err != nil { + return "", err + } + for _, profile := range profiles { + if profile.TypeName == profileType { + freeByGPU[vf.ParentGPU] = append(freeByGPU[vf.ParentGPU], vf) + break + } + } + } + + gpus := make([]string, 0, len(freeByGPU)) + for gpu := range freeByGPU { + gpus = append(gpus, gpu) + } + sort.Slice(gpus, func(i, j int) bool { + if usageByGPU[gpus[i]] == usageByGPU[gpus[j]] { + return gpus[i] < gpus[j] + } + return usageByGPU[gpus[i]] < usageByGPU[gpus[j]] + }) + if len(gpus) == 0 { + return "", nil + } + return freeByGPU[gpus[0]][0].PCIAddress, nil +} + +func (s vendorVFIOSysfs) profileMetadata(vfs []VirtualFunction) ([]profileMetadata, error) { + profilesByType := make(map[string]profileMetadata) + for _, vf := range vfs { + profiles, err := s.readCreatableProfiles(vf.PCIAddress) + if err != nil { + return nil, err + } + for _, profile := range profiles { + profilesByType[profile.TypeName] = profile + } + } + profiles := make([]profileMetadata, 0, len(profilesByType)) + for _, profile := range profilesByType { + profiles = append(profiles, profile) + } + sort.Slice(profiles, func(i, j int) bool { return profiles[i].Name < profiles[j].Name }) + return profiles, nil +} + +func (s vendorVFIOSysfs) readCreatableProfiles(vfAddress string) ([]profileMetadata, error) { + path := filepath.Join(s.pciDevicesPath, vfAddress, "nvidia", "creatable_vgpu_types") + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read creatable vGPU types for VF %s: %w", vfAddress, err) + } + return parseCreatableVGPUTypes(string(data)) +} + +func (s vendorVFIOSysfs) vfioDeviceInUse(vfAddress string, openPaths map[string]struct{}) (bool, error) { + devicePaths := make([]string, 0, 2) + probeErrs := make([]error, 0, 2) + + vfioDevices, err := os.ReadDir(filepath.Join(s.pciDevicesPath, vfAddress, "vfio-dev")) + if os.IsNotExist(err) { + return false, nil + } + if err != nil { + probeErrs = append(probeErrs, fmt.Errorf("read VFIO devices for VF %s: %w", vfAddress, err)) + } else { + for _, device := range vfioDevices { + devicePaths = append(devicePaths, filepath.Join(s.vfioDevicesPath, device.Name())) + } + } + + target, err := os.Readlink(filepath.Join(s.pciDevicesPath, vfAddress, "iommu_group")) + if os.IsNotExist(err) { + return false, nil + } + if err != nil { + probeErrs = append(probeErrs, fmt.Errorf("read IOMMU group for VF %s: %w", vfAddress, err)) + } else { + devicePaths = append(devicePaths, filepath.Join(filepath.Dir(s.vfioDevicesPath), filepath.Base(target))) + } + + for _, path := range devicePaths { + if _, ok := openPaths[path]; ok { + return true, nil + } + } + if len(probeErrs) > 0 { + return false, errors.Join(probeErrs...) + } + return false, nil +} + +func (s vendorVFIOSysfs) openVFIOPaths() (map[string]struct{}, error) { + processes, err := os.ReadDir(s.procPath) + if err != nil { + return nil, err + } + prefix := filepath.Dir(s.vfioDevicesPath) + string(filepath.Separator) + open := make(map[string]struct{}) + for _, process := range processes { + if _, err := strconv.Atoi(process.Name()); err != nil { + continue + } + fdPath := filepath.Join(s.procPath, process.Name(), "fd") + fds, err := os.ReadDir(fdPath) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, fmt.Errorf("read process %s file descriptors: %w", process.Name(), err) + } + for _, fd := range fds { + target, err := os.Readlink(filepath.Join(fdPath, fd.Name())) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, fmt.Errorf("read process %s file descriptor %s: %w", process.Name(), fd.Name(), err) + } + if strings.HasPrefix(target, prefix) { + open[target] = struct{}{} + } + } + } + return open, nil +} + +func parseCreatableVGPUTypes(value string) ([]profileMetadata, error) { + profiles := make([]profileMetadata, 0) + for lineNumber, line := range strings.Split(value, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + typeID, name, found := strings.Cut(line, ":") + typeID = strings.TrimSpace(typeID) + name = strings.TrimSpace(name) + if typeID == "ID" { + continue + } + if !found || name == "" { + return nil, fmt.Errorf("parse creatable vGPU types line %d: %q", lineNumber+1, line) + } + if _, err := strconv.Atoi(typeID); err != nil { + return nil, fmt.Errorf("parse vGPU type ID %q: %w", typeID, err) + } + profiles = append(profiles, profileMetadata{ + TypeName: typeID, + Name: name, + FramebufferMB: framebufferFromProfileName(name), + }) + } + return profiles, nil +} + +func framebufferFromProfileName(name string) int { + series := strings.LastIndexAny(name, "ABCQ") + if series <= 0 { + return 0 + } + dash := strings.LastIndex(name[:series], "-") + if dash < 0 { + return 0 + } + gb, err := strconv.Atoi(name[dash+1 : series]) + if err != nil { + return 0 + } + return gb * 1024 +} + +func rollbackVendorVFIOCreate(currentTypePath, vfAddress string, verifyErr error) error { + if err := os.WriteFile(currentTypePath, []byte("0"), 0200); err != nil { + return errors.Join(verifyErr, fmt.Errorf("roll back vGPU on VF %s: %w", vfAddress, err)) + } + return verifyErr +} + +func readCurrentVGPUType(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + value := strings.TrimSpace(string(data)) + if _, err := strconv.Atoi(value); err != nil { + return "", fmt.Errorf("invalid current vGPU type %q", value) + } + return value, nil +} diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go new file mode 100644 index 000000000..8e79a7750 --- /dev/null +++ b/lib/devices/vendor_vfio_linux_test.go @@ -0,0 +1,374 @@ +//go:build linux + +package devices + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testCreatableTypes = `ID : vGPU Name +1147 : NVIDIA L40S-1Q +1148 : NVIDIA L40S-2Q +1159 : NVIDIA L40S-48Q +` + +func TestParseCreatableVGPUTypes(t *testing.T) { + t.Parallel() + + profiles, err := parseCreatableVGPUTypes(testCreatableTypes) + require.NoError(t, err) + require.Len(t, profiles, 3) + assert.Equal(t, profileMetadata{TypeName: "1147", Name: "NVIDIA L40S-1Q", FramebufferMB: 1024}, profiles[0]) + assert.Equal(t, profileMetadata{TypeName: "1159", Name: "NVIDIA L40S-48Q", FramebufferMB: 48 * 1024}, profiles[2]) +} + +func TestVendorVFIOCreateAndDestroy(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + require.Len(t, vfs, 1) + assert.False(t, vfs[0].Allocated) + assert.Equal(t, "0000:82:00.0", vfs[0].ParentGPU) + + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-2Q")) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-2Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, VGPUFrameworkVendorVFIO, device.Framework) + assert.Equal(t, "0000:82:00.4", device.VFAddress) + assert.Equal(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4"), device.SysfsPath) + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "1148") + + require.NoError(t, sysfs.destroy(context.Background(), device.VFAddress, "instance-1")) + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "0") +} + +func TestVendorVFIODestroySkipsAssignmentOwnedByAnotherInstance(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-2Q", "instance-1") + require.NoError(t, err) + + require.NoError(t, sysfs.destroy(context.Background(), device.VFAddress, "stale-instance")) + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "1148") + + require.NoError(t, sysfs.destroy(context.Background(), device.VFAddress, "instance-1")) + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "0") +} + +func TestVendorVFIODestroyRetainsAssignmentInUse(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + + activeDevice := filepath.Join(sysfs.vfioDevicesPath, "vfio42") + fdDir := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdDir, 0755)) + require.NoError(t, os.Symlink(activeDevice, filepath.Join(fdDir, "5"))) + + err := sysfs.destroy(context.Background(), vfAddress, "instance-1") + require.Error(t, err) + assert.ErrorContains(t, err, "still in use") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIODestroyReleasesUnboundVF(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + unbind func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) + }{ + { + name: "missing vfio device directory", + unbind: func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) { + require.NoError(t, os.RemoveAll(filepath.Join(sysfs.pciDevicesPath, vfAddress, "vfio-dev"))) + }, + }, + { + name: "missing iommu group symlink", + unbind: func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) { + require.NoError(t, os.Remove(filepath.Join(sysfs.pciDevicesPath, vfAddress, "iommu_group"))) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + tt.unbind(t, sysfs, vfAddress) + + require.NoError(t, sysfs.destroy(context.Background(), vfAddress, "instance-1")) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "0") + }) + } +} + +func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "ID : vGPU Name\n") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "1159", "") + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Empty(t, profiles) + + _, err = sysfs.create(context.Background(), "NVIDIA L40S-2Q", "instance-1") + require.Error(t, err) + assert.ErrorContains(t, err, "GPUs may be at capacity") +} + +func TestVendorVFIOCreateReportsAmbiguousMissingProfile(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "ID : vGPU Name\n1148 : NVIDIA L40S-2Q\n") + + _, err := sysfs.create(context.Background(), "NVIDIA L40S-48Q", "instance-1") + require.Error(t, err) + assert.ErrorContains(t, err, "not creatable on any VF") + assert.ErrorContains(t, err, "unknown profile or insufficient capacity") +} + +func TestVendorVFIOSelectsLeastLoadedGPU(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress) +} + +func TestVendorVFIOReconcile(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "43", "1148", "") + + activeDevice := filepath.Join(sysfs.vfioDevicesPath, "vfio43") + fdDir := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdDir, 0755)) + require.NoError(t, os.Symlink(activeDevice, filepath.Join(fdDir, "5"))) + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "0") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:e3:00.4", "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcileSkipsProtectedVF(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "43", "1148", "") + + protected := map[string]struct{}{ + filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4"): {}, + } + require.NoError(t, sysfs.reconcile(context.Background(), protected)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "1148") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:e3:00.4", "nvidia", "current_vgpu_type"), "0") +} + +func TestVendorVFIOReconcilePreservesLegacyGroupFD(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "43", "1148", "") + + legacyGroup := filepath.Join(filepath.Dir(sysfs.vfioDevicesPath), "43") + fdDir := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdDir, 0755)) + require.NoError(t, os.Symlink(legacyGroup, filepath.Join(fdDir, "5"))) + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "0") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:e3:00.4", "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcilePreservesVFWhenVFIODeviceProbeFails(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + vfioDevPath := filepath.Join(sysfs.pciDevicesPath, vfAddress, "vfio-dev") + require.NoError(t, os.RemoveAll(vfioDevPath)) + require.NoError(t, os.WriteFile(vfioDevPath, nil, 0644)) + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcilePreservesVFWhenIOMMUGroupProbeFails(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + iommuGroupPath := filepath.Join(sysfs.pciDevicesPath, vfAddress, "iommu_group") + require.NoError(t, os.Remove(iommuGroupPath)) + require.NoError(t, os.WriteFile(iommuGroupPath, nil, 0644)) + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcilePreservesVFWhenProcFDDirectoryScanFails(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + + processPath := filepath.Join(sysfs.procPath, "123") + require.NoError(t, os.MkdirAll(processPath, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(processPath, "fd"), nil, 0644)) + + err := sysfs.reconcile(context.Background(), nil) + require.Error(t, err) + assert.ErrorContains(t, err, "read process 123 file descriptors") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcilePreservesVFWhenProcFDLinkScanFails(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + + fdPath := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdPath, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(fdPath, "5"), nil, 0644)) + + err := sysfs.reconcile(context.Background(), nil) + require.Error(t, err) + assert.ErrorContains(t, err, "read process 123 file descriptor 5") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestParseCreatableVGPUTypesHeaderOnly(t *testing.T) { + t.Parallel() + + profiles, err := parseCreatableVGPUTypes("ID : vGPU Name\n") + require.NoError(t, err) + assert.Empty(t, profiles) +} + +func TestParseCreatableVGPUTypesRejectsMalformedLine(t *testing.T) { + t.Parallel() + + _, err := parseCreatableVGPUTypes("NVIDIA") + require.Error(t, err) + + _, err = parseCreatableVGPUTypes("not-an-id : NVIDIA L40S-1Q") + require.Error(t, err) +} + +func TestRollbackVendorVFIOCreate(t *testing.T) { + t.Parallel() + + verifyErr := errors.New("verification failed") + t.Run("preserves verification error", func(t *testing.T) { + currentTypePath := filepath.Join(t.TempDir(), "current_vgpu_type") + require.NoError(t, os.WriteFile(currentTypePath, []byte("1148"), 0644)) + + err := rollbackVendorVFIOCreate(currentTypePath, "0000:82:00.4", verifyErr) + require.ErrorIs(t, err, verifyErr) + assertFileValue(t, currentTypePath, "0") + }) + + t.Run("surfaces rollback error", func(t *testing.T) { + currentTypePath := filepath.Join(t.TempDir(), "missing", "current_vgpu_type") + + err := rollbackVendorVFIOCreate(currentTypePath, "0000:82:00.4", verifyErr) + require.ErrorIs(t, err, verifyErr) + assert.ErrorContains(t, err, "roll back vGPU on VF 0000:82:00.4") + }) +} + +func profileAvailability(profiles []GPUProfile, name string) int { + for _, profile := range profiles { + if profile.Name == name { + return profile.Available + } + } + return -1 +} + +type testVendorVFIOSysfs struct { + vendorVFIOSysfs +} + +func newTestVendorVFIOSysfs(t *testing.T) testVendorVFIOSysfs { + t.Helper() + root := t.TempDir() + pci := filepath.Join(root, "sys", "bus", "pci", "devices") + proc := filepath.Join(root, "proc") + vfio := filepath.Join(root, "dev", "vfio", "devices") + require.NoError(t, os.MkdirAll(pci, 0755)) + require.NoError(t, os.MkdirAll(proc, 0755)) + require.NoError(t, os.MkdirAll(vfio, 0755)) + return testVendorVFIOSysfs{vendorVFIOSysfs{ + pciDevicesPath: pci, + procPath: proc, + vfioDevicesPath: vfio, + owners: make(map[string]string), + }} +} + +func (s testVendorVFIOSysfs) addVF(t *testing.T, parent, address, vfioID, currentType, creatableTypes string) { + t.Helper() + parentPath := filepath.Join(s.pciDevicesPath, parent) + vfPath := filepath.Join(s.pciDevicesPath, address) + nvidiaPath := filepath.Join(vfPath, "nvidia") + require.NoError(t, os.MkdirAll(parentPath, 0755)) + require.NoError(t, os.MkdirAll(nvidiaPath, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "current_vgpu_type"), []byte(currentType), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "creatable_vgpu_types"), []byte(creatableTypes), 0444)) + require.NoError(t, os.Symlink(parentPath, filepath.Join(vfPath, "physfn"))) + vfioName := "vfio" + vfioID + require.NoError(t, os.MkdirAll(filepath.Join(vfPath, "vfio-dev", vfioName), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(s.vfioDevicesPath, vfioName), nil, 0600)) + require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "kernel", "iommu_groups", vfioID), filepath.Join(vfPath, "iommu_group"))) +} + +func assertFileValue(t *testing.T, path, expected string) { + t.Helper() + value, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, expected, string(value)) +} diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index eaf210b42..a4ccd2db6 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -8,31 +8,115 @@ import ( "path/filepath" ) +// DiscoverVGPU returns the host's active vGPU framework and virtual functions. +func DiscoverVGPU() (VGPUFramework, []VirtualFunction, error) { + return discoverVGPUWith(discoverMdevVFs, hostVendorVFIO.discoverVFs) +} + +func discoverVGPUWith(discoverMdev, discoverVendorVFIO func() ([]VirtualFunction, error)) (VGPUFramework, []VirtualFunction, error) { + vfs, err := discoverMdev() + if err != nil { + return VGPUFrameworkNone, nil, fmt.Errorf("discover mdev VFs: %w", err) + } + if len(vfs) > 0 { + return VGPUFrameworkMdev, vfs, nil + } + + vfs, err = discoverVendorVFIO() + if err != nil { + return VGPUFrameworkNone, nil, fmt.Errorf("discover vendor VFIO VFs: %w", err) + } + if len(vfs) == 0 { + return VGPUFrameworkNone, nil, nil + } + return VGPUFrameworkVendorVFIO, vfs, nil +} + +// ListGPUProfiles returns available vGPU profiles with availability counts. +func ListGPUProfiles() ([]GPUProfile, error) { + framework, vfs, err := DiscoverVGPU() + if err != nil { + return nil, err + } + return ListGPUProfilesWithVFs(framework, vfs) +} + +// ListGPUProfilesWithVFs returns available profiles for discovered VFs. +func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction) ([]GPUProfile, error) { + switch framework { + case VGPUFrameworkMdev: + return listMdevGPUProfilesWithVFs(vfs) + case VGPUFrameworkVendorVFIO: + return hostVendorVFIO.listProfiles(vfs) + default: + return nil, nil + } +} + func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) { - mdev, err := CreateMdev(ctx, profileName, instanceID) + framework, _, err := DiscoverVGPU() if err != nil { return nil, err } - return &VGPUDevice{ - Framework: VGPUFrameworkMdev, - VFAddress: mdev.VFAddress, - ProfileType: mdev.ProfileType, - ProfileName: mdev.ProfileName, - SysfsPath: mdev.SysfsPath, - MdevUUID: mdev.UUID, - }, nil + switch framework { + case VGPUFrameworkMdev: + mdev, err := CreateMdev(ctx, profileName, instanceID) + if err != nil { + return nil, err + } + return &VGPUDevice{ + Framework: VGPUFrameworkMdev, + VFAddress: mdev.VFAddress, + ProfileType: mdev.ProfileType, + ProfileName: mdev.ProfileName, + SysfsPath: mdev.SysfsPath, + MdevUUID: mdev.UUID, + }, nil + case VGPUFrameworkVendorVFIO: + return hostVendorVFIO.create(ctx, profileName, instanceID) + default: + return nil, fmt.Errorf("vGPU framework not available") + } } func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { - if assignment.Framework != VGPUFrameworkNone && assignment.Framework != VGPUFrameworkMdev { - return fmt.Errorf("unknown vGPU framework %q", assignment.Framework) + framework := assignment.Framework + if framework == VGPUFrameworkNone && assignment.MdevUUID != "" { + framework = VGPUFrameworkMdev + } + + switch framework { + case VGPUFrameworkMdev: + mdevUUID := assignment.MdevUUID + if mdevUUID == "" { + mdevUUID = filepath.Base(assignment.DevicePath) + } + return DestroyMdev(ctx, mdevUUID) + case VGPUFrameworkVendorVFIO: + return hostVendorVFIO.destroy(ctx, filepath.Base(assignment.DevicePath), assignment.InstanceID) + case VGPUFrameworkNone: + return nil + default: + return fmt.Errorf("unknown vGPU framework %q", framework) } - mdevUUID := assignment.MdevUUID - if mdevUUID == "" { - if assignment.DevicePath == "" { +} + +// ReconcileVGPUs releases orphaned vGPU assignments. +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}) error { + framework, _, err := DiscoverVGPU() + if err != nil { + return err + } + + switch framework { + case VGPUFrameworkMdev: + return ReconcileMdevs(ctx, nil) + case VGPUFrameworkVendorVFIO: + if protectedDevicePaths == nil { return nil } - mdevUUID = filepath.Base(assignment.DevicePath) + return hostVendorVFIO.reconcile(ctx, protectedDevicePaths) + default: + return nil } - return DestroyMdev(ctx, mdevUUID) } diff --git a/lib/devices/vgpu_linux_test.go b/lib/devices/vgpu_linux_test.go new file mode 100644 index 000000000..805f8da95 --- /dev/null +++ b/lib/devices/vgpu_linux_test.go @@ -0,0 +1,50 @@ +//go:build linux + +package devices + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDiscoverVGPUWithPropagatesMdevError(t *testing.T) { + t.Parallel() + + discoveryErr := errors.New("mdev discovery failed") + vendorCalled := false + framework, vfs, err := discoverVGPUWith( + func() ([]VirtualFunction, error) { + return nil, discoveryErr + }, + func() ([]VirtualFunction, error) { + vendorCalled = true + return []VirtualFunction{{PCIAddress: "0000:82:00.4"}}, nil + }, + ) + + require.ErrorIs(t, err, discoveryErr) + assert.Equal(t, VGPUFrameworkNone, framework) + assert.Nil(t, vfs) + assert.False(t, vendorCalled) +} + +func TestDiscoverVGPUWithPropagatesVendorVFIOError(t *testing.T) { + t.Parallel() + + discoveryErr := errors.New("vendor VFIO discovery failed") + framework, vfs, err := discoverVGPUWith( + func() ([]VirtualFunction, error) { + return nil, nil + }, + func() ([]VirtualFunction, error) { + return nil, discoveryErr + }, + ) + + require.ErrorIs(t, err, discoveryErr) + assert.Equal(t, VGPUFrameworkNone, framework) + assert.Nil(t, vfs) +} diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 78788412e..4069692c9 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -1,7 +1,10 @@ package resources import ( + "context" + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/logger" ) // GPUResourceStatus represents the GPU resource status for the API response. @@ -16,31 +19,22 @@ type GPUResourceStatus struct { // GetGPUStatus returns the current GPU resource status. // Returns nil if no GPU is available or the mode is "none". -func GetGPUStatus() *GPUResourceStatus { - mode := devices.DetectHostGPUMode() - if mode == devices.GPUModeNone { +func GetGPUStatus(ctx context.Context) *GPUResourceStatus { + framework, vfs, err := devices.DiscoverVGPU() + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU state", "error", err) return nil } - - switch mode { - case devices.GPUModeVGPU: - return getVGPUStatus() - case devices.GPUModePassthrough: - return getPassthroughStatus() - default: - return nil + if framework != devices.VGPUFrameworkNone { + return getVGPUStatus(ctx, framework, vfs) } + return getPassthroughStatus() } -// getVGPUStatus returns GPU status for vGPU mode (SR-IOV + mdev). -func getVGPUStatus() *GPUResourceStatus { - vfs, err := devices.DiscoverVFs() - if err != nil || len(vfs) == 0 { - return nil - } - - // Count used VFs (those with mdevs) +// getVGPUStatus returns GPU status for vGPU mode (SR-IOV). +func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []devices.VirtualFunction) *GPUResourceStatus { usedSlots := 0 + // Count used VFs (those with a vGPU assigned) for _, vf := range vfs { if vf.Allocated { usedSlots++ @@ -48,8 +42,9 @@ func getVGPUStatus() *GPUResourceStatus { } // Get available profiles (reuse VFs to avoid redundant discovery) - profiles, err := devices.ListGPUProfilesWithVFs(vfs) + profiles, err := devices.ListGPUProfilesWithVFs(framework, vfs) if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to list vGPU profiles; reporting none", "framework", framework, "error", err) profiles = nil } diff --git a/lib/resources/monitoring_test.go b/lib/resources/monitoring_test.go index ab6c3e4dd..bef0740dc 100644 --- a/lib/resources/monitoring_test.go +++ b/lib/resources/monitoring_test.go @@ -198,7 +198,7 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { mgr, _, _ := monitoringTestManager(t) originalProvider := currentGPUStatusProvider() - setGPUStatusProvider(func() *GPUResourceStatus { + setGPUStatusProvider(func(context.Context) *GPUResourceStatus { return &GPUResourceStatus{ Mode: "vgpu", TotalSlots: 8, diff --git a/lib/resources/resource.go b/lib/resources/resource.go index caaf5ba50..86f644bda 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -37,13 +37,13 @@ var ( gpuStatusProvider = GetGPUStatus ) -func currentGPUStatusProvider() func() *GPUResourceStatus { +func currentGPUStatusProvider() func(context.Context) *GPUResourceStatus { gpuStatusProviderMu.RLock() defer gpuStatusProviderMu.RUnlock() return gpuStatusProvider } -func setGPUStatusProvider(fn func() *GPUResourceStatus) { +func setGPUStatusProvider(fn func(context.Context) *GPUResourceStatus) { if fn == nil { fn = GetGPUStatus } @@ -427,7 +427,7 @@ func (m *Manager) GetFullStatus(ctx context.Context) (*FullResourceStatus, error } // Get GPU status - gpuStatus := currentGPUStatusProvider()() + gpuStatus := currentGPUStatusProvider()(ctx) return &FullResourceStatus{ CPU: *cpuStatus, @@ -691,7 +691,7 @@ func (m *Manager) validateAllocationLocked(ctx context.Context, excludeID string // Check GPU if needed if req.GPUSlots > 0 { - gpuStatus := currentGPUStatusProvider()() + gpuStatus := currentGPUStatusProvider()(ctx) if gpuStatus == nil { return fmt.Errorf("insufficient GPU: no GPU available on this host") } From d5f75621d4dd013d4da9e062708bb1e6754b2921 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:25:31 +0000 Subject: [PATCH 002/107] Account for consumed vGPU profiles --- lib/devices/vendor_vfio_linux.go | 34 ++++++++++++++------------- lib/devices/vendor_vfio_linux_test.go | 32 +++++++++++++++++++++---- 2 files changed, 46 insertions(+), 20 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 23ab1e89a..4d056db9e 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -22,18 +22,20 @@ const ( ) type vendorVFIOSysfs struct { - pciDevicesPath string - procPath string - vfioDevicesPath string - owners map[string]string + pciDevicesPath string + procPath string + vfioDevicesPath string + owners map[string]string + framebufferByType map[string]int } var ( hostVendorVFIO = vendorVFIOSysfs{ - pciDevicesPath: pciDevicesPath, - procPath: procPath, - vfioDevicesPath: vfioDevicesPath, - owners: make(map[string]string), + pciDevicesPath: pciDevicesPath, + procPath: procPath, + vfioDevicesPath: vfioDevicesPath, + owners: make(map[string]string), + framebufferByType: make(map[string]int), } vendorVFIOMu sync.Mutex ) @@ -141,7 +143,7 @@ func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID str return nil, fmt.Errorf("profile %q is not creatable on any VF (unknown profile or insufficient capacity)", profileName) } - targetVF, err := s.selectLeastLoadedVF(vfs, metadata, requested.TypeName) + targetVF, err := s.selectLeastLoadedVF(vfs, requested.TypeName) if err != nil { return nil, err } @@ -270,17 +272,16 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map return nil } -func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, metadata []profileMetadata, profileType string) (string, error) { - framebufferByType := make(map[string]int, len(metadata)) - for _, profile := range metadata { - framebufferByType[profile.TypeName] = profile.FramebufferMB - } - +func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType string) (string, error) { usageByGPU := make(map[string]int) freeByGPU := make(map[string][]VirtualFunction) for _, vf := range vfs { if vf.Allocated { - usageByGPU[vf.ParentGPU] += framebufferByType[vf.ProfileType] + framebuffer, ok := s.framebufferByType[vf.ProfileType] + if !ok { + return "", fmt.Errorf("framebuffer size for allocated vGPU type %s is unknown", vf.ProfileType) + } + usageByGPU[vf.ParentGPU] += framebuffer continue } profiles, err := s.readCreatableProfiles(vf.PCIAddress) @@ -320,6 +321,7 @@ func (s vendorVFIOSysfs) profileMetadata(vfs []VirtualFunction) ([]profileMetada } for _, profile := range profiles { profilesByType[profile.TypeName] = profile + s.framebufferByType[profile.TypeName] = profile.FramebufferMB } } profiles := make([]profileMetadata, 0, len(profilesByType)) diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 8e79a7750..9afd56292 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -169,6 +169,29 @@ func TestVendorVFIOSelectsLeastLoadedGPU(t *testing.T) { assert.Equal(t, "0000:e3:00.4", device.VFAddress) } +func TestVendorVFIOSelectsLeastLoadedGPUWithConsumedType(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + _, err = sysfs.profileMetadata(vfs) + require.NoError(t, err) + for _, vfAddress := range []string{"0000:82:00.5", "0000:e3:00.4"} { + creatableTypesPath := filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "creatable_vgpu_types") + require.NoError(t, os.Chmod(creatableTypesPath, 0644)) + require.NoError(t, os.WriteFile(creatableTypesPath, []byte("1147 : NVIDIA L40S-1Q\n"), 0444)) + } + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress) +} + func TestVendorVFIOReconcile(t *testing.T) { t.Parallel() @@ -343,10 +366,11 @@ func newTestVendorVFIOSysfs(t *testing.T) testVendorVFIOSysfs { require.NoError(t, os.MkdirAll(proc, 0755)) require.NoError(t, os.MkdirAll(vfio, 0755)) return testVendorVFIOSysfs{vendorVFIOSysfs{ - pciDevicesPath: pci, - procPath: proc, - vfioDevicesPath: vfio, - owners: make(map[string]string), + pciDevicesPath: pci, + procPath: proc, + vfioDevicesPath: vfio, + owners: make(map[string]string), + framebufferByType: make(map[string]int), }} } From 3827351fbec51e7aa3421910c04b9608a258b05e Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:25:31 +0000 Subject: [PATCH 003/107] Reject unowned vendor VFIO releases --- lib/devices/vendor_vfio_linux.go | 19 ++++++++++++------- lib/devices/vendor_vfio_linux_test.go | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 4d056db9e..f702f4591 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -203,13 +203,18 @@ func (s vendorVFIOSysfs) destroyWithOpenPaths(ctx context.Context, vfAddress, in return nil } - if owner, ok := s.owners[vfAddress]; ok && (instanceID == "" || owner != instanceID) { - log.WarnContext(ctx, "skipping vendor VFIO vGPU release owned by another instance", - "vf", vfAddress, - "owner_instance_id", owner, - "requesting_instance_id", instanceID, - ) - return nil + if owner, ok := s.owners[vfAddress]; ok { + if instanceID == "" { + return fmt.Errorf("cannot release vendor VFIO vGPU on VF %s without instance ID", vfAddress) + } + if owner != instanceID { + log.WarnContext(ctx, "skipping vendor VFIO vGPU release owned by another instance", + "vf", vfAddress, + "owner_instance_id", owner, + "requesting_instance_id", instanceID, + ) + return nil + } } if openPaths == nil { diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 9afd56292..10b87b081 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -72,6 +72,20 @@ func TestVendorVFIODestroySkipsAssignmentOwnedByAnotherInstance(t *testing.T) { assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "0") } +func TestVendorVFIODestroyRejectsMissingInstanceIDForOwnedVF(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-2Q", "instance-1") + require.NoError(t, err) + + err = sysfs.destroy(context.Background(), device.VFAddress, "") + require.ErrorContains(t, err, "without instance ID") + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "1148") +} + func TestVendorVFIODestroyRetainsAssignmentInUse(t *testing.T) { t.Parallel() From 36ce94f9c28e7b6eab0fead71bf8be8171026d6d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:25:37 +0000 Subject: [PATCH 004/107] Check all vendor VFIO device paths --- lib/devices/vendor_vfio_linux.go | 14 ++++---- lib/devices/vendor_vfio_linux_test.go | 46 +++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index f702f4591..6841c5294 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -351,11 +351,10 @@ func (s vendorVFIOSysfs) vfioDeviceInUse(vfAddress string, openPaths map[string] probeErrs := make([]error, 0, 2) vfioDevices, err := os.ReadDir(filepath.Join(s.pciDevicesPath, vfAddress, "vfio-dev")) - if os.IsNotExist(err) { - return false, nil - } if err != nil { - probeErrs = append(probeErrs, fmt.Errorf("read VFIO devices for VF %s: %w", vfAddress, err)) + if !os.IsNotExist(err) { + probeErrs = append(probeErrs, fmt.Errorf("read VFIO devices for VF %s: %w", vfAddress, err)) + } } else { for _, device := range vfioDevices { devicePaths = append(devicePaths, filepath.Join(s.vfioDevicesPath, device.Name())) @@ -363,11 +362,10 @@ func (s vendorVFIOSysfs) vfioDeviceInUse(vfAddress string, openPaths map[string] } target, err := os.Readlink(filepath.Join(s.pciDevicesPath, vfAddress, "iommu_group")) - if os.IsNotExist(err) { - return false, nil - } if err != nil { - probeErrs = append(probeErrs, fmt.Errorf("read IOMMU group for VF %s: %w", vfAddress, err)) + if !os.IsNotExist(err) { + probeErrs = append(probeErrs, fmt.Errorf("read IOMMU group for VF %s: %w", vfAddress, err)) + } } else { devicePaths = append(devicePaths, filepath.Join(filepath.Dir(s.vfioDevicesPath), filepath.Base(target))) } diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 10b87b081..8dc6e6558 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -138,6 +138,52 @@ func TestVendorVFIODestroyReleasesUnboundVF(t *testing.T) { } } +func TestVendorVFIODestroyRetainsAssignmentWhenOneVFIOPathIsMissing(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + remove func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) + open func(sysfs testVendorVFIOSysfs) string + }{ + { + name: "missing iommu group", + remove: func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) { + require.NoError(t, os.Remove(filepath.Join(sysfs.pciDevicesPath, vfAddress, "iommu_group"))) + }, + open: func(sysfs testVendorVFIOSysfs) string { + return filepath.Join(sysfs.vfioDevicesPath, "vfio42") + }, + }, + { + name: "missing vfio device directory", + remove: func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) { + require.NoError(t, os.RemoveAll(filepath.Join(sysfs.pciDevicesPath, vfAddress, "vfio-dev"))) + }, + open: func(sysfs testVendorVFIOSysfs) string { + return filepath.Join(filepath.Dir(sysfs.vfioDevicesPath), "42") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + tt.remove(t, sysfs, vfAddress) + + fdDir := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdDir, 0755)) + require.NoError(t, os.Symlink(tt.open(sysfs), filepath.Join(fdDir, "5"))) + + err := sysfs.destroy(context.Background(), vfAddress, "instance-1") + require.ErrorContains(t, err, "still in use") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") + }) + } +} + func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) { t.Parallel() From 4f446e9f49c107b40076e45a7704d9b2d31202a9 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:25:37 +0000 Subject: [PATCH 005/107] Require usable mdev types for discovery --- lib/devices/mdev_linux.go | 27 ++++++++++++++++++++++++--- lib/devices/vgpu_linux_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/lib/devices/mdev_linux.go b/lib/devices/mdev_linux.go index e2891efc9..61e55b599 100644 --- a/lib/devices/mdev_linux.go +++ b/lib/devices/mdev_linux.go @@ -92,7 +92,11 @@ func getCachedProfiles(firstVF string) []profileMetadata { // discoverMdevVFs returns all SR-IOV Virtual Functions available for vGPU, // discovered by scanning /sys/class/mdev_bus/. func discoverMdevVFs() ([]VirtualFunction, error) { - entries, err := os.ReadDir(mdevBusPath) + return discoverMdevVFsWith(mdevBusPath, pciDevicesPath, ListMdevDevices) +} + +func discoverMdevVFsWith(busPath, pciPath string, listMdevs func() ([]MdevDevice, error)) ([]VirtualFunction, error) { + entries, err := os.ReadDir(busPath) if err != nil { if os.IsNotExist(err) { return nil, nil // No mdev_bus means no mdev vGPU support @@ -101,7 +105,7 @@ func discoverMdevVFs() ([]VirtualFunction, error) { } // List mdevs once and build a lookup map to avoid O(n*m) performance - mdevs, _ := ListMdevDevices() + mdevs, _ := listMdevs() mdevByVF := make(map[string]bool, len(mdevs)) for _, mdev := range mdevs { mdevByVF[mdev.VFAddress] = true @@ -110,10 +114,27 @@ func discoverMdevVFs() ([]VirtualFunction, error) { var vfs []VirtualFunction for _, entry := range entries { vfAddr := entry.Name() + types, err := os.ReadDir(filepath.Join(busPath, vfAddr, "mdev_supported_types")) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, fmt.Errorf("read mdev supported types for VF %s: %w", vfAddr, err) + } + usable := false + for _, typ := range types { + if typ.IsDir() { + usable = true + break + } + } + if !usable { + continue + } // Find parent GPU by checking physfn symlink // VFs have a physfn symlink pointing to their parent Physical Function - physfnPath := filepath.Join("/sys/bus/pci/devices", vfAddr, "physfn") + physfnPath := filepath.Join(pciPath, vfAddr, "physfn") parentGPU := "" if target, err := os.Readlink(physfnPath); err == nil { parentGPU = filepath.Base(target) diff --git a/lib/devices/vgpu_linux_test.go b/lib/devices/vgpu_linux_test.go index 805f8da95..40b987394 100644 --- a/lib/devices/vgpu_linux_test.go +++ b/lib/devices/vgpu_linux_test.go @@ -4,6 +4,8 @@ package devices import ( "errors" + "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -31,6 +33,29 @@ func TestDiscoverVGPUWithPropagatesMdevError(t *testing.T) { assert.False(t, vendorCalled) } +func TestDiscoverVGPUWithFallsBackFromTypelessMdevBus(t *testing.T) { + t.Parallel() + + root := t.TempDir() + busPath := filepath.Join(root, "sys", "class", "mdev_bus") + require.NoError(t, os.MkdirAll(filepath.Join(busPath, "0000:82:00.4", "mdev_supported_types"), 0755)) + + framework, vfs, err := discoverVGPUWith( + func() ([]VirtualFunction, error) { + return discoverMdevVFsWith(busPath, filepath.Join(root, "sys", "bus", "pci", "devices"), func() ([]MdevDevice, error) { + return nil, nil + }) + }, + func() ([]VirtualFunction, error) { + return []VirtualFunction{{PCIAddress: "0000:82:00.4"}}, nil + }, + ) + + require.NoError(t, err) + assert.Equal(t, VGPUFrameworkVendorVFIO, framework) + assert.Equal(t, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}, vfs) +} + func TestDiscoverVGPUWithPropagatesVendorVFIOError(t *testing.T) { t.Parallel() From 063d06bb11178aa6335ad9efc5be3440ba0a8981 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:50:10 +0000 Subject: [PATCH 006/107] Test retained vendor VFIO assignments --- integration/vgpu_test.go | 50 ++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 3f8fdfd65..ce0da7fb2 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -89,6 +89,14 @@ func TestVGPU(t *testing.T) { // Cleanup any orphaned instances and mdevs t.Cleanup(func() { if instanceID != "" { + if inst, err := instanceManager.GetInstance(ctx, instanceID); err == nil && inst.GPUFramework == devices.VGPUFrameworkVendorVFIO && inst.GPUDevicePath != "" { + err := devices.DestroyVGPU(ctx, devices.VGPUAssignment{ + Framework: inst.GPUFramework, + DevicePath: inst.GPUDevicePath, + InstanceID: instanceID, + }) + require.NoError(t, err, "cleanup should release vendor VFIO vGPU") + } t.Log("Cleanup: Deleting instance...") instanceManager.DeleteInstance(ctx, instanceID) } @@ -243,26 +251,38 @@ func TestVGPU(t *testing.T) { t.Logf("Instance GPU: profile=%s, framework=%s, device=%s", actualInst.GPUProfile, actualInst.GPUFramework, actualInst.GPUDevicePath) }) - t.Log("Step 10: Stopping instance to release the vGPU...") + t.Log("Step 10: Stopping instance...") _, err = instanceManager.StopInstance(ctx, inst.Id) require.NoError(t, err, "stop should succeed") - t.Run("VGPUReleasedOnStop", func(t *testing.T) { - stopped, err := instanceManager.GetInstance(ctx, inst.Id) - require.NoError(t, err) - assert.Empty(t, stopped.GPUDevicePath, "assignment metadata should be cleared on stop") - assertVGPUReleased(t, inst.GPUFramework, inst.GPUDevicePath) - }) + switch inst.GPUFramework { + case devices.VGPUFrameworkMdev: + t.Run("VGPUReleasedOnStop", func(t *testing.T) { + stopped, err := instanceManager.GetInstance(ctx, inst.Id) + require.NoError(t, err) + assert.Empty(t, stopped.GPUDevicePath, "assignment metadata should be cleared on stop") + assertVGPUReleased(t, inst.GPUFramework, inst.GPUDevicePath) + }) - t.Log("Step 11: Starting instance to reacquire a vGPU...") - started, err := instanceManager.StartInstance(ctx, inst.Id, instances.StartInstanceRequest{}) - require.NoError(t, err, "start should succeed") + t.Log("Step 11: Starting instance to reacquire a vGPU...") + started, err := instanceManager.StartInstance(ctx, inst.Id, instances.StartInstanceRequest{}) + require.NoError(t, err, "start should succeed") - t.Run("VGPUReacquiredOnStart", func(t *testing.T) { - require.NotEmpty(t, started.GPUDevicePath, "start should assign a vGPU") - assert.Equal(t, inst.GPUFramework, started.GPUFramework, "framework should match") - assertVGPUAssigned(t, started.GPUFramework, started.GPUDevicePath) - }) + t.Run("VGPUReacquiredOnStart", func(t *testing.T) { + require.NotEmpty(t, started.GPUDevicePath, "start should assign a vGPU") + assert.Equal(t, inst.GPUFramework, started.GPUFramework, "framework should match") + assertVGPUAssigned(t, started.GPUFramework, started.GPUDevicePath) + }) + case devices.VGPUFrameworkVendorVFIO: + t.Run("VGPUAssignmentRetainedOnStop", func(t *testing.T) { + stopped, err := instanceManager.GetInstance(ctx, inst.Id) + require.NoError(t, err) + // Release requires an instance-owned assignment, so stop retains it. + assert.Equal(t, inst.GPUFramework, stopped.GPUFramework, "assignment framework should be retained on stop") + assert.Equal(t, inst.GPUDevicePath, stopped.GPUDevicePath, "assignment metadata should be retained on stop") + assertVGPUAssigned(t, stopped.GPUFramework, stopped.GPUDevicePath) + }) + } t.Log("✅ vGPU test PASSED!") } From 4fa5bb3f58326564bb74ddb139b216b419e3f9ea Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 21:08:51 +0000 Subject: [PATCH 007/107] Always delete the test instance during vGPU cleanup --- integration/vgpu_test.go | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index ce0da7fb2..059513d6e 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -36,10 +36,10 @@ import ( // // sudo go test -v -run TestVGPU -timeout 5m ./integration/... // -// Note: This test verifies vGPU assignment, release on stop, reacquisition on -// start, and PCI device visibility inside the VM. It does NOT test nvidia-smi -// or CUDA functionality since that requires NVIDIA guest drivers pre-installed -// in the image. +// Note: This test verifies vGPU assignment, stop behavior (mdev releases and +// reacquires on start; vendor VFIO retains the assignment), and PCI device +// visibility inside the VM. It does NOT test nvidia-smi or CUDA functionality +// since that requires NVIDIA guest drivers pre-installed in the image. func TestVGPU(t *testing.T) { t.Parallel() if testing.Short() { @@ -88,18 +88,23 @@ func TestVGPU(t *testing.T) { // Cleanup any orphaned instances and mdevs t.Cleanup(func() { - if instanceID != "" { - if inst, err := instanceManager.GetInstance(ctx, instanceID); err == nil && inst.GPUFramework == devices.VGPUFrameworkVendorVFIO && inst.GPUDevicePath != "" { - err := devices.DestroyVGPU(ctx, devices.VGPUAssignment{ - Framework: inst.GPUFramework, - DevicePath: inst.GPUDevicePath, - InstanceID: instanceID, - }) - require.NoError(t, err, "cleanup should release vendor VFIO vGPU") + if instanceID == "" { + return + } + if _, err := instanceManager.StopInstance(ctx, instanceID); err != nil { + t.Logf("Cleanup: stop instance: %v", err) + } + if inst, err := instanceManager.GetInstance(ctx, instanceID); err == nil && inst.GPUFramework == devices.VGPUFrameworkVendorVFIO && inst.GPUDevicePath != "" { + if err := devices.DestroyVGPU(ctx, devices.VGPUAssignment{ + Framework: inst.GPUFramework, + DevicePath: inst.GPUDevicePath, + InstanceID: instanceID, + }); err != nil { + t.Errorf("cleanup: release vendor VFIO vGPU: %v", err) } - t.Log("Cleanup: Deleting instance...") - instanceManager.DeleteInstance(ctx, instanceID) } + t.Log("Cleanup: Deleting instance...") + instanceManager.DeleteInstance(ctx, instanceID) }) // Step 1: Ensure system files (kernel, initrd) From 4e8b937184a179f437a5f52c4ab04394a60ab52f Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 21:27:38 +0000 Subject: [PATCH 008/107] Fall back to passthrough status when vGPU discovery fails --- lib/resources/gpu.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 4069692c9..6a34de5cc 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -22,8 +22,9 @@ type GPUResourceStatus struct { func GetGPUStatus(ctx context.Context) *GPUResourceStatus { framework, vfs, err := devices.DiscoverVGPU() if err != nil { + // A failed vGPU probe must not hide passthrough GPUs from status reporting. logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU state", "error", err) - return nil + return getPassthroughStatus() } if framework != devices.VGPUFrameworkNone { return getVGPUStatus(ctx, framework, vfs) From 98584f9bea5cf7ddb8aeac3849ee35b7ff40f0af Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 7 Aug 2026 20:39:31 +0000 Subject: [PATCH 009/107] Keep vGPU placement available when an allocated type is unknown Sort GPUs with unaccountable load last instead of rejecting placement, and stop reporting passthrough capacity when vGPU discovery fails. --- lib/devices/vendor_vfio_linux.go | 12 +++++++++++- lib/devices/vendor_vfio_linux_test.go | 27 +++++++++++++++++++++++++++ lib/resources/gpu.go | 7 +++++-- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 6841c5294..78526a45e 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -279,12 +279,19 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType string) (string, error) { usageByGPU := make(map[string]int) + unknownUsageByGPU := make(map[string]bool) freeByGPU := make(map[string][]VirtualFunction) for _, vf := range vfs { if vf.Allocated { + // framebufferByType only covers currently creatable profiles, so + // after a restart an allocated type can be missing when its + // capacity is exhausted. Prefer GPUs whose load is fully known + // instead of rejecting placement outright; the kernel driver + // still enforces real capacity through creatable_vgpu_types. framebuffer, ok := s.framebufferByType[vf.ProfileType] if !ok { - return "", fmt.Errorf("framebuffer size for allocated vGPU type %s is unknown", vf.ProfileType) + unknownUsageByGPU[vf.ParentGPU] = true + continue } usageByGPU[vf.ParentGPU] += framebuffer continue @@ -306,6 +313,9 @@ func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType gpus = append(gpus, gpu) } sort.Slice(gpus, func(i, j int) bool { + if unknownUsageByGPU[gpus[i]] != unknownUsageByGPU[gpus[j]] { + return !unknownUsageByGPU[gpus[i]] + } if usageByGPU[gpus[i]] == usageByGPU[gpus[j]] { return gpus[i] < gpus[j] } diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 8dc6e6558..ad2fc2a35 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -252,6 +252,33 @@ func TestVendorVFIOSelectsLeastLoadedGPUWithConsumedType(t *testing.T) { assert.Equal(t, "0000:e3:00.4", device.VFAddress) } +func TestVendorVFIOPlacementPrefersKnownLoadWhenAllocatedTypeIsUnknown(t *testing.T) { + t.Parallel() + + // Simulates a restart: type 1159 is allocated but no longer creatable + // anywhere, so its framebuffer size is unknown. + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "1147 : NVIDIA L40S-1Q\n") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", "1147 : NVIDIA L40S-1Q\n") + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress, "the GPU with unknown load should be picked last") +} + +func TestVendorVFIOPlacesOnGPUWithUnknownLoadWhenItHasTheOnlyCapacity(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "1147 : NVIDIA L40S-1Q\n") + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", device.VFAddress) +} + func TestVendorVFIOReconcile(t *testing.T) { t.Parallel() diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 6a34de5cc..054e3744e 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -22,9 +22,12 @@ type GPUResourceStatus struct { func GetGPUStatus(ctx context.Context) *GPUResourceStatus { framework, vfs, err := devices.DiscoverVGPU() if err != nil { - // A failed vGPU probe must not hide passthrough GPUs from status reporting. + // Only report passthrough once vGPU discovery confirms no vGPU + // framework. On a vGPU host a transient probe failure would otherwise + // expose the PFs/VFs as available passthrough slots while active vGPU + // assignments exist. logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU state", "error", err) - return getPassthroughStatus() + return nil } if framework != devices.VGPUFrameworkNone { return getVGPUStatus(ctx, framework, vfs) From 85ec1eb26ac7b21a1109f0f14f41a20419fa753b Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sat, 8 Aug 2026 00:53:16 +0000 Subject: [PATCH 010/107] Report vendor VFIO availability per parent GPU and thread instance ownership through releases --- lib/devices/vendor_vfio_linux.go | 17 ++++++++++++++--- lib/devices/vendor_vfio_linux_test.go | 16 ++++++++++++++++ lib/instances/create.go | 1 + lib/instances/start.go | 1 + lib/instances/vgpu.go | 1 + 5 files changed, 33 insertions(+), 3 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 78526a45e..6f59bed3d 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -81,9 +81,13 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { return vfs, nil } +// listProfiles aggregates creatable profiles per parent GPU. Free VFs on the +// same GPU share its framebuffer, so counting each advertising VF overreports +// availability. The driver only guarantees that a GPU still advertising a +// type can fit one more instance of it, so report that per-GPU lower bound. func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { profilesByType := make(map[string]profileMetadata) - availability := make(map[string]int) + creatableGPUs := make(map[string]map[string]struct{}) for _, vf := range vfs { creatable, err := s.readCreatableProfiles(vf.PCIAddress) if err != nil { @@ -92,7 +96,14 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro for _, profile := range creatable { profilesByType[profile.TypeName] = profile if !vf.Allocated { - availability[profile.TypeName]++ + gpu := vf.ParentGPU + if gpu == "" { + gpu = vf.PCIAddress + } + if creatableGPUs[profile.TypeName] == nil { + creatableGPUs[profile.TypeName] = make(map[string]struct{}) + } + creatableGPUs[profile.TypeName][gpu] = struct{}{} } } } @@ -108,7 +119,7 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro profiles = append(profiles, GPUProfile{ Name: profile.Name, FramebufferMB: profile.FramebufferMB, - Available: availability[profile.TypeName], + Available: len(creatableGPUs[profile.TypeName]), }) } return profiles, nil diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index ad2fc2a35..51668ed11 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -184,6 +184,22 @@ func TestVendorVFIODestroyRetainsAssignmentWhenOneVFIOPathIsMissing(t *testing.T } } +func TestVendorVFIOListProfilesCountsPerGPUNotPerVF(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-48Q"), + "free VFs share their parent GPU's capacity, so availability is per GPU") +} + func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) { t.Parallel() diff --git a/lib/instances/create.go b/lib/instances/create.go index add25e4e7..b4b0952da 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -316,6 +316,7 @@ func (m *manager) createInstance( Framework: gpuDevice.Framework, DevicePath: gpuDevice.SysfsPath, MdevUUID: gpuDevice.MdevUUID, + InstanceID: id, } if err := devices.DestroyVGPU(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) diff --git a/lib/instances/start.go b/lib/instances/start.go index 7e7855eac..a6c832450 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -177,6 +177,7 @@ func (m *manager) startInstance( Framework: device.Framework, DevicePath: device.SysfsPath, MdevUUID: device.MdevUUID, + InstanceID: id, } if err := devices.DestroyVGPU(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID, "error", err) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index cffe2ac1d..a8ca6aceb 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -27,6 +27,7 @@ func releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { Framework: stored.GPUFramework, DevicePath: path, MdevUUID: stored.GPUMdevUUID, + InstanceID: stored.Id, } if err := devices.DestroyVGPU(ctx, assignment); err != nil { return err From 6801fe376d4205b0a9cd2409a8414b13ba47bd83 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sun, 9 Aug 2026 06:18:53 +0000 Subject: [PATCH 011/107] Keep vendor VFIO out of the create path until lifecycle integration The instance lifecycle already routes create/start/stop/delete through CreateVGPU/DestroyVGPU, so dispatching vendor VFIO creates here would activate the backend before assignment durability and release guards exist. Reject vendor VFIO creates for now; destroy stays wired so existing assignments remain releasable. The integration test skips on vendor VFIO hosts at this layer and no longer asserts the transitional stop-retention behavior. --- integration/vgpu_test.go | 66 +++++++++++++++------------------------ lib/devices/vgpu_linux.go | 5 ++- 2 files changed, 30 insertions(+), 41 deletions(-) diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 059513d6e..7873aa35c 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -36,10 +36,10 @@ import ( // // sudo go test -v -run TestVGPU -timeout 5m ./integration/... // -// Note: This test verifies vGPU assignment, stop behavior (mdev releases and -// reacquires on start; vendor VFIO retains the assignment), and PCI device -// visibility inside the VM. It does NOT test nvidia-smi or CUDA functionality -// since that requires NVIDIA guest drivers pre-installed in the image. +// Note: This test verifies vGPU assignment, release on stop, reacquisition on +// start, and PCI device visibility inside the VM. It does NOT test nvidia-smi +// or CUDA functionality since that requires NVIDIA guest drivers pre-installed +// in the image. func TestVGPU(t *testing.T) { t.Parallel() if testing.Short() { @@ -94,17 +94,10 @@ func TestVGPU(t *testing.T) { if _, err := instanceManager.StopInstance(ctx, instanceID); err != nil { t.Logf("Cleanup: stop instance: %v", err) } - if inst, err := instanceManager.GetInstance(ctx, instanceID); err == nil && inst.GPUFramework == devices.VGPUFrameworkVendorVFIO && inst.GPUDevicePath != "" { - if err := devices.DestroyVGPU(ctx, devices.VGPUAssignment{ - Framework: inst.GPUFramework, - DevicePath: inst.GPUDevicePath, - InstanceID: instanceID, - }); err != nil { - t.Errorf("cleanup: release vendor VFIO vGPU: %v", err) - } - } t.Log("Cleanup: Deleting instance...") - instanceManager.DeleteInstance(ctx, instanceID) + if err := instanceManager.DeleteInstance(ctx, instanceID); err != nil { + t.Errorf("cleanup: delete instance: %v", err) + } }) // Step 1: Ensure system files (kernel, initrd) @@ -260,34 +253,22 @@ func TestVGPU(t *testing.T) { _, err = instanceManager.StopInstance(ctx, inst.Id) require.NoError(t, err, "stop should succeed") - switch inst.GPUFramework { - case devices.VGPUFrameworkMdev: - t.Run("VGPUReleasedOnStop", func(t *testing.T) { - stopped, err := instanceManager.GetInstance(ctx, inst.Id) - require.NoError(t, err) - assert.Empty(t, stopped.GPUDevicePath, "assignment metadata should be cleared on stop") - assertVGPUReleased(t, inst.GPUFramework, inst.GPUDevicePath) - }) + t.Run("VGPUReleasedOnStop", func(t *testing.T) { + stopped, err := instanceManager.GetInstance(ctx, inst.Id) + require.NoError(t, err) + assert.Empty(t, stopped.GPUDevicePath, "assignment metadata should be cleared on stop") + assertVGPUReleased(t, inst.GPUFramework, inst.GPUDevicePath) + }) - t.Log("Step 11: Starting instance to reacquire a vGPU...") - started, err := instanceManager.StartInstance(ctx, inst.Id, instances.StartInstanceRequest{}) - require.NoError(t, err, "start should succeed") + t.Log("Step 11: Starting instance to reacquire a vGPU...") + started, err := instanceManager.StartInstance(ctx, inst.Id, instances.StartInstanceRequest{}) + require.NoError(t, err, "start should succeed") - t.Run("VGPUReacquiredOnStart", func(t *testing.T) { - require.NotEmpty(t, started.GPUDevicePath, "start should assign a vGPU") - assert.Equal(t, inst.GPUFramework, started.GPUFramework, "framework should match") - assertVGPUAssigned(t, started.GPUFramework, started.GPUDevicePath) - }) - case devices.VGPUFrameworkVendorVFIO: - t.Run("VGPUAssignmentRetainedOnStop", func(t *testing.T) { - stopped, err := instanceManager.GetInstance(ctx, inst.Id) - require.NoError(t, err) - // Release requires an instance-owned assignment, so stop retains it. - assert.Equal(t, inst.GPUFramework, stopped.GPUFramework, "assignment framework should be retained on stop") - assert.Equal(t, inst.GPUDevicePath, stopped.GPUDevicePath, "assignment metadata should be retained on stop") - assertVGPUAssigned(t, stopped.GPUFramework, stopped.GPUDevicePath) - }) - } + t.Run("VGPUReacquiredOnStart", func(t *testing.T) { + require.NotEmpty(t, started.GPUDevicePath, "start should assign a vGPU") + assert.Equal(t, inst.GPUFramework, started.GPUFramework, "framework should match") + assertVGPUAssigned(t, started.GPUFramework, started.GPUDevicePath) + }) t.Log("✅ vGPU test PASSED!") } @@ -343,6 +324,11 @@ func checkVGPUTestPrerequisites() (string, string) { if framework == devices.VGPUFrameworkNone { return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", "" } + if framework == devices.VGPUFrameworkVendorVFIO { + // CreateVGPU rejects vendor VFIO until the instance lifecycle + // integration lands. + return "vGPU test requires the vendor VFIO instance lifecycle integration", "" + } // Check for available profiles profiles, err := devices.ListGPUProfiles() diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index a4ccd2db6..72fe3b944 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -73,7 +73,10 @@ func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevic MdevUUID: mdev.UUID, }, nil case VGPUFrameworkVendorVFIO: - return hostVendorVFIO.create(ctx, profileName, instanceID) + // The instance lifecycle does not yet persist vendor VFIO assignments + // durably or guard their release against live claims, so keep the + // backend out of the create path until that integration lands. + return nil, fmt.Errorf("vendor VFIO vGPU support is not yet integrated with the instance lifecycle") default: return nil, fmt.Errorf("vGPU framework not available") } From 7b93e822ba0cec5f9a6516da52a2541365340367 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 04:53:56 +0000 Subject: [PATCH 012/107] Report vendor VFIO profile availability per free VF --- lib/devices/vendor_vfio_linux.go | 21 +++++++-------------- lib/devices/vendor_vfio_linux_test.go | 6 +++--- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 6f59bed3d..ceab85ec5 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -81,13 +81,13 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { return vfs, nil } -// listProfiles aggregates creatable profiles per parent GPU. Free VFs on the -// same GPU share its framebuffer, so counting each advertising VF overreports -// availability. The driver only guarantees that a GPU still advertising a -// type can fit one more instance of it, so report that per-GPU lower bound. +// listProfiles counts each free VF advertising a type as one creatable +// instance, matching the driver-reported units that mdev sums through +// available_instances. This is a best-effort snapshot because creating on one +// VF may revoke the type from siblings that share its GPU framebuffer. func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { profilesByType := make(map[string]profileMetadata) - creatableGPUs := make(map[string]map[string]struct{}) + creatableVFs := make(map[string]int) for _, vf := range vfs { creatable, err := s.readCreatableProfiles(vf.PCIAddress) if err != nil { @@ -96,14 +96,7 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro for _, profile := range creatable { profilesByType[profile.TypeName] = profile if !vf.Allocated { - gpu := vf.ParentGPU - if gpu == "" { - gpu = vf.PCIAddress - } - if creatableGPUs[profile.TypeName] == nil { - creatableGPUs[profile.TypeName] = make(map[string]struct{}) - } - creatableGPUs[profile.TypeName][gpu] = struct{}{} + creatableVFs[profile.TypeName]++ } } } @@ -119,7 +112,7 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro profiles = append(profiles, GPUProfile{ Name: profile.Name, FramebufferMB: profile.FramebufferMB, - Available: len(creatableGPUs[profile.TypeName]), + Available: creatableVFs[profile.TypeName], }) } return profiles, nil diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 51668ed11..4555f5c2a 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -184,7 +184,7 @@ func TestVendorVFIODestroyRetainsAssignmentWhenOneVFIOPathIsMissing(t *testing.T } } -func TestVendorVFIOListProfilesCountsPerGPUNotPerVF(t *testing.T) { +func TestVendorVFIOListProfilesCountsFreeVFs(t *testing.T) { t.Parallel() sysfs := newTestVendorVFIOSysfs(t) @@ -196,8 +196,8 @@ func TestVendorVFIOListProfilesCountsPerGPUNotPerVF(t *testing.T) { require.NoError(t, err) profiles, err := sysfs.listProfiles(vfs) require.NoError(t, err) - assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-48Q"), - "free VFs share their parent GPU's capacity, so availability is per GPU") + assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-48Q"), + "each free VF advertising the type counts as one creatable instance") } func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) { From c9db2f97b7594d364214ffabda2fc26311c6e751 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 07:18:33 +0000 Subject: [PATCH 013/107] Clarify vGPU availability semantics --- lib/oapi/oapi.go | 493 ++++++++++++++++++------------------ lib/resources/monitoring.go | 2 +- openapi.yaml | 2 +- 3 files changed, 249 insertions(+), 248 deletions(-) diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index eb43eb45c..fa568c18b 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1006,7 +1006,7 @@ type GPUConfig struct { // GPUProfile Available vGPU profile type GPUProfile struct { - // Available Number of instances that can be created with this profile + // Available Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer. Available int `json:"available"` // FramebufferMb Frame buffer size in MB @@ -19026,251 +19026,252 @@ var swaggerSpec = []string{ "JLVWYtGUYuKRzuH11nlS+sO1hpQW687I1X0sOmT69TTZ9iTD2d2s+KqI4cLW4OXpcxtn30dwuiDIzyXo", "107auYOasEveHzKbaOThFYAjDbAETJj4jFCBuKBTWu24ame9x5jdm1CmI65bU6f/YVMMnizcgCEeYJCW", "JyYE1hQA8YFe7Z50up3zomiUZUzVpXlTFN5aWpEysn4Zc+fs3U0jbzPBJzQEhAORQPapVdRcTOqrvcF5", - "b/uvJmpekx9IbJSZ6KGUx7UaT/b9dhfRy7N3Z01jKqouIX90S3NaAe9TRrsUKK7G1Wh91VahcadB3zNF", - "J6Uo/jwk2gKKxzifTIgYpQFb20/6OTIvFCgEpz9WxVstRrdVos8qmwNa9ARHlE03W69+wD5Xm8Y6mL+X", - "Z+/e2Co+TQDIequKSj8G+7iPfi3qXKGXZ+9kCXLUDxju6kB6DXldZ7OFpBFOTIumSAVlvr0NiLO1wHxW", - "fmgtkwGxOVwezR0EtDGfZjkcw/M3vZPX77fSmMy7lTFBXN2MJ0SPe9PjFnMHLlAmoVWYxLzJ8GEIQ7Y9", - "QN5aFSe49SJ55zWwOoornIxkwkMhPG/1QwQP0cb7n0x2sB5BF2WVrdS/+8g/Pn0/CZ4YAJRv6PYcOqxb", - "UCsHPKhK1hHJOtXpVToNHRWT2bQs8izXJuSX1Y3ml+vr4ZlGmvs9cslXNRu3lSuRydEy8IkuKgGZT40R", - "21pKJMmwwIoki1p4ZBXfnCw7mMk1iW6Q/fVCv/7J1LTIBRmpmSByxpNqWMJud7kuqoQQ4DmxpaDMnDw7", - "vOIoxeISLkYnV6OcmRWoRpDvroN7mSmV3WBSP799e2aUbUXEHCf1HAS55HA/JgleoDFRV4QwNxUsEfZD", - "UOs5nLKhHI9Qo4wIyqtr2NkN9HtuwpLRVOCIIPOVq6pst0TCrdl2KW0vAQDBKCJSNuzv9qr9tZ9O8qTd", - "HoeGtb22CHl0kw1+e3TmysoUZXvdMu8sr/IZET1z5Fz93tVbuyNXFzhyXTGDC7skMIwJVDyyiR9+XqYL", - "N4KKTvrzSi6kxxykn71n+4FTYJaqa875h1bCXv24h/zqKWZxqPyxifc3mfJTgM+CoGORgyuJxiYWySTv", - "G7HcT1sQBMeUESlracZRLpJOt9Ob2FkdbG0lPMIJIAju7W4/21od1bkynNdGL41iukrdczFOJgrGJbUa", - "+DWYdJUktnCWtTCImXVccz8Ae1oOH4SSx/pu8yQ8F08+GCxlTF/jSLkibWAhq3hAsX9sAee3Mh9oMDV1", - "V7So/dw/n4OgSzrDalYl/60l2ocwFQgx1DRirA+1dTRE/jEoUXERqsTFhbIJeGPiYhCL+9BF+DlE24qv", - "bPDMn+WT/f3d/XV8CJhN7ZjbcxeYqnm7mmJMvOW259c2ANFSVZnDHenVxUT1uqyhKc0Rl0hq9YLyjLAb", - "ref+3u7Ozdaz7UROXJRWjS+FEFqOTo+NTBRxpjBlRKCUKBxjhatMBkxLmstAqRdMUsjSmXy/mrU0hDP4", - "kCu3rVP1pZzhDSXr3jiE5hQzOtEM2b7p9yxneGf/yYEprBmTyd7+k36/f1Mgihcl8kSrrdgyMXMeJkVf", - "zj5vH+4Ab6LNXP7onB2+/VkzslwKc2ltyTFlB96/i3+WD+AP888xZWGcija1WOlkqQZrNTwst0DAJD5A", - "ZbltJ/e0qtcftg1DsDCA4wRR3ypBk3cH71bQOK2i+N8gVXdF6qoWV16zZNG4Nrcuq1oWHFdeOVU/+atF", - "aVX6cbW725m74B3bp8GDLqrOLju6b1U3WK4srbhUeSsjrCimmCTmr4gzwNkNVVasXJHuWYvCXHCN2Apc", - "RZf+j0Xv3o9H/kC8311hL+8nW2Lxww0jVFYKpH9blkPXcyEnjq45zGHbY3ErtC1na2HhgqHZD3wX3iaK", - "q9r76+l//f5/5NnTv2///ur9+/+ev/yv41/pf79Pzl5/FuTIakTAB4X1+2JIfqZEuw/n15aUTrGKAjY6", - "rf41rLB9YiwOKppBAU40JgdD1kOvqCLClHOr5SIOO2iDgKYEX2lxF2rVmDSwTf3xmXEw6o//cGLwp3ob", - "sc0RF3ZDCugPmY9jnmLKNodsyGxbyE1Egl6g/4pRhDNTk40ypPXfBRoLKKBnPT5l5130B86yT5tDZovm", - "G3DrDEMNskmRhMWcv9aOykSl2tdJgQJhnC5DVtzWBSSecfv1S/x8SpJ6Ck/DoqzW36zm9GwQAg+E9BK9", - "kVAjBlSQgrI1GRV5L+jZYHNZn1ujYxQ0tIL8rF/c5B8e5iFzcVPO4jGJaQR8xaXtzWxiZ5ExaSjNGvEy", - "wa8XsDdvTC5ZjHCuZpoXRTbPPeL8kpIubGkXTQQUeOiaL417fcaz3njRm/GswDzAwgSfYOOgrirZ/6dn", - "J9p7TwSd2J6CqeuaRAJCJxwZOzOTAVhYF5Ym9taU12Ba9JkT+7qp0yJNaQ4TWa1ywVz1CAI1JAFco6A+", - "EpLJv0dRQsHqJGc8T2I0A/w7pZsJQdh1BkUGCx5HMZnU/12NMNjZfwIarPv37k7rBFKzdKuoLE8COm3q", - "WF8Ljm3YJAzAiAcjZwhfExOkb0BbyArsFIrDf8+Ra6g8cQUjMd4pk9ImbTGCRHrJbJvBrCN7DCyCxwjb", - "09TmPlo6hZUkphYtmGAB+CxpgR3ywiRPvn11jhQRqUtn34j07sApMUARPSplbutiHR6dvtjsd4K4RxVX", - "FWzVyiSn6qAD0Ac2eKApJqK00eCUdNHJMSSv2mul1MUg2+AnLlBibsXyMjoA7IyquQebinonx1YATRZl", - "BIIRW4adTddiVr/eDtCbQgXExVCKNMSStlyT5WUCzdp4NJMKsdR6LWsV/GNW/bP3MSQ+QLlBw4sBu7Hx", - "/mpvc3RIUPqiqlnIbnwh+UEhjfYvb++/NMrxl5fRd28mo1sf8CibYRmi7pnv1YSXlvbddyNX2b1oDuGp", - "9DuSNHi2/uaKrnjXkCL6nqt8HoL13O9tb7/d3ru5+e6mALVVZCoPta7AqG0PLnsXIK0ByFWqRo2x3kg/", - "tpHdzi7y/hTNsGTfKXhYs45s7z5tY5SAXttGSfvx0XxihlRwKQdzVUT3GsCvS5okRoCRdMpwgp6jjfOT", - "l7+cvHq1iXro9evT+las+iK4P7fAqoVbANbRJHwFkI4qGfqoSOV7+/YVHK6EQDaEkcMvb49gu9a02ALR", - "1g3u5dk7cPxjOXJxlM2pg7hMvyXXVCq5DHLWKhz5cxB0zaftau67SZo2ytL7q2F4f67gxAZR6zbvAD/X", - "xZIvLecDQMs+ZM7e1wdruxKI9nPRZK2d4Y7AZBuvtBAQaw2gYL/pdrs9LOydDKeC7xJiW76E4xKqb43D", - "2u3QQDLpodQXD4nRyVlZeKl0Rrjma3N6vtPffvIMaoduD9ow9hRHK/o+PTxq3/lgx9wyB3h8EMUHoLDf", - "1mdlCduoIDi5wguovWeWdtgxF6an3XrH1iqSreJrluFub4duWxfjGvBrQZx1gUtylK4s/dEiW7COYZbm", - "FqExpUlCJYk4i2VVRp5hiWRmgElNCYxCgh8yGGAXFZWIQUpBOIpEXpoerXRt5f08s3QPZTgzzrQOADj8", - "v5CFRCkFJ2jRPYQ+SlQkpcRDtiFcAlORqQQVOGP9A6QDdF3YuR4aVVDqQ38wZHKWK83ENvvoiDOZp0RY", - "qywaU/AYbSKZG5UWxgursdAMU9KYiCHTrwWgT/8o1JODJ4PBYNDtFJrcrv73IERNd+r87FtoX5OCC+B7", - "zIL8AqqfyBnKWUxEUY6bGHKoh8jd0HH6mZi+7vN24pX9vJSrwgdzHQRwO2zfzwVUhaE26OcQHXoL5Xz/", - "9iJ6qxQgJ7/a5B/71egmEQwERTxPYq3xjfVtZwxyJLZmSEmU4c7mXSrRO1MKszp1G3qsOPo9J2KB3p+e", - "VsIeBJloHtBu4sAlGvaBZzfahp01NpK1o7mJe9mDn70PyNm6pOJJiF8cYNb3MLqcZEOhFcNWRXFeZV/T", - "WmUwi4Qys0+aaFZMsGbKj8l8lOchrUQ/crAk796dHFcoBeMn288Gz573no23n/T24sF2D2/vPunt7OPB", - "ZDd6uru9s7si46RFFtntE8OCqmkgYLgIDx+5MPVQ9HBTkkBNCLCBz1eUxfyqcs8EI1H93m2U67rul2PY", - "Ww8hmPmSYKmMnaCBZZzCbUoi3baJ/LaZikWJobBF8cnbwfbnmllgcA3M+K3ImfFfmtz6wlafegP2N6s6", - "ztvxVhiQyzBZt1p+5+0XbXCw//xg/3MXzWVJrBtjnZzucXObQq8c9nAtDcOlAnoGG2cJ7Fjhw5jPbdZG", - "p9spEkvgb7h1a0HLxeNW2VJNB7YbZiOr+HdD0vBJRTGAkBCDPRcfaJGgyDcd5woVqela1jhKeB4jz+hl", - "oLjA4XXiKQm6GfA/WVuYgdY0WQ9amQAsZ6iYQJlmxODo043YDOMD9BLehUc4NfqTHYSp2+H7uHC8MIEp", - "+ny5ro02s3rI51aRgW+0VoP0v2DaehmsbXR1E0YMOkC/cvimUKsYrxtZzeugzyy/XjfIbtjUX4cYAZ1Z", - "me4A/VTIcYUkaCW/DUnsnyPLsEqgls1Kurzd8Y6mlnLnvNTvbsesaKfbcQsFKeLLyeLvSqpfOn8+KYYi", - "tghO4CyXybi5oomFp4aZUKloJG2Wht7cJvnClhQi8choKU3BnybD02oyxUdOfHl/ijYAgfAvyFqQ9b82", - "i0DRyl2383zv+ZOnO8+ftMIZKge4XgY9gvzj5cGtFUijLB9ZI0TT1I/O3hkjQ2TU9yLI5P2pD+uQCa5Z", - "j565a9Dv/Hn/uQ+vFPN8nHgePIvFZtBcYcOCCGIFL2oIOPydJnM6mbDfP0aXO38XNN2+fiJ3xtsNsLGm", - "o7B968T34i8Zg8m4Z8oDhRFwgKCEbASJekMkzACdE4WAfnoIR6BHFGnDluQclJRd8SBh7e3u7j57ur/T", - "iq7s6LyDMwJrV+BStiPwjhi8iTbenJ+jLY/gTJsOSwGQvZnVMcPnDNnavoOqQNrfHuyGqKTh4i6pxrY9", - "TxuX/L3V0+yk7KJD9nOhwy2d8uBq7+4Onu7tP9tvd4ytHXYkrldzGJcbZJbHAtD7O78B0uTbwzMEmbcT", - "HFWNKC4U60ajUjcaFRRPMKDnNxjYs6dP9vd2d7bboZ2Fojssjl/lwFZ5V+DQBYgisBuBpVhmvd2m2yIk", - "ThkCe0OiBNP0MHK5DLXbx4Cbj4R5rdyENheDNf0vXVwtvm1lRSpsQyYTxogGXKCcFSU1+ut9n1/EhdnM", - "tc31sJ6rh/JfmF49C8tjSofdYikzQeaU5/ILNMSVSU6dJJyLG33bpLC8ITJPlPEzUonen34HPEXTGpKK", - "ZFUdylLjCvCiW07uRue5QiJhIm9arFa70WbrV02423Bqu6uQKyrcoBEyLNacK2froyyPcBLlUEQGF/up", - "ZwVoP5B7n2XJwgTRJwnnDEUzzMAbIWzJKjZFGM14EveDIaf6yWgSDF/gVyjhBuz4kpDM1lcxg9CfaRGG", - "zgna8CuLGVKq1fvcTw2TsRU0qtS4n4YLF2IZygorcs71emLFPRxe80nF5JjwqQSlUEF6QL8O/55hYaL+", - "MTP1guap0SUDkc2BIdaYeehGNTcpn1gF14ockNFtVhJHgkuJSEKnUJvm/WktUXhFclmRLrw+crI62Bak", - "azyHgasMrjjZuqxY6H4MJM58zg0JNAzJeStiEp1xMsUsh4orHiFbi3e/ddzhjEs1KgCgbjhYqUZQRiEX", - "pESJK9LbC3uQeyd4LzrWdpvlsgG+t/p6iarCTTUNsJmnBlc0vFrdggZDZLwMgbUSdauE8apjNt0EpK3E", - "3acSWqUePhjagOQSjy152PGbbaJRwiqr7mdJW7VVMV/tDc7b4qethks7w2p2wiY8ALJxAxels0TbsNCM", - "iJRCIREUE0ZJ7HTJwldpTV2QmZ1IguKc2JUz8qnAdsGxOd4AlMGcjYyyaY3X1ztsYx42Y1hdZQH6tS+2", - "iSuS4czVtyKHtTKBgRLhMoe1VbQllaOwO2u5YUGmeYIFqmMErhiyXKQJZZdtWpeLdMwTGiH9Qd0BPeFJ", - "wq9G+pH8Aeay2Wp2+oNRU0meczM4m4BnNqTWbzmFH/QsN2vpv2CJ2TLfbwE0S5swrWBI9k80IRZG7x2j", - "1x6hV2HI93YGTWnpDY1WEtKXIRhvyrktyQZPfC4DSXwrpRxXTIjEFhveiD1ZLk1dkxa3kgMfdS7A23l0", - "qhkan4cBcmT4dQ0BBI0JJNi4qS1zjRZssc1UgiUVcjlDf+fjqkG0bXxtoFDXBiuxKASZBAPpYUdXGqTN", - "G0tr4u3uTcAegK3qicJHN8RQWFfSrAxkauInb5aqe82IXTLq5mgqfbWoXOECLQqcANtre8CAeh22QGAw", - "wMFItYCKplBGZuEVF5RozIUA5GUt4XDmZgP4Jlrm0WvtAKbQ2xlZIEFSTNmQUVYYSQG1jCBG5kR46ahc", - "aCVrSuI++pun4gFWdZqphQVBB+P5dxLxK1aMccj8QerGc6nbOWTGsihyKJJfvqSbBa1PEwqkB4MTTAnq", - "8G8ngsiZP/dQrUgt411xETcW4Vkg9wrUdgEfK1L8kjCflRXNBFVD09DIfLUcLmcKvcJTq3+iSu1VVK+t", - "urq/XBIRFhKLKRWvtApd8Y6Kp5wYtBWAHoGaevYvw+ILuJEW4CJl8391TZY/nRWNV3+rveYBiDg830Nj", - "tg2aYCOTL1ML9ql60taGqkC+2SrYmGVfAtpwscquMElVEvAKhLS6J9ulvNWj8t1otiSJqr3vPdt/+qRl", - "hZbPctYZmKwv7Zqbpytccg07ddrG7/Ns/9nz57t7+893buRhcQkcDfvTlMTh7w/aINdKH9bkX//45/vT", - "mtdnH4KdBzcalEnhCA+pIY2jOqD3p//6xz/dqG49oBCjWYbibvDbN0bpJP5OukCBqguvnZNshX5/WDES", - "4ILNoA0ymRAwg47MuvXKwdTwNtpJwTjDEVWLACPHVyasvHilBindxh1UHWxI5DVtW/hRzblkPi6zOzdc", - "5+g/jW+4RgvPWhd6kvm4yQ/9ut6r8UKXXgs/xqFFiIEsaowvG7iL+VxhWYmc1n9HkODgUrmW01rMG6vh", - "bes5BxDFYuuZeaGAIVj0mjxpP/K3v7adnt+yYtapr/iHFeew+QjeyOobuJEDRt9ofQ5rjT/YC/B2X43G", - "fgm2lTXuKvXaylv35v22SNNdLkhQ3GA378/LTLzJh3XwXaBHOwa75GXb3QpJNFCTl3QSMKDxhPSKQD2b", - "kYJkbjyC+sxbPPdAqmR0ySeTKqjsfjMIOeDrQFaV6wUrpTWTLiLXzmZRR7A2YDrDzr4cdrQKMOxsp8NO", - "zW0VzFNM8fXIdlAFURmsQgUv88xrg5RuBuOER5emuhgUre6jAUoJZhLlDA5/zau2PVjtHep2Mm9vCgxu", - "YkKcltgWjGlMZnhOofSD9alMK4GY5JoqCQGj0M4BirmBVaqUVrUz1K+ZLMKDctJw6WC2sA3rBvV7nLmI", - "1vJdMPBNoKAr+0gE71pUAM2xX78+7ZoABgg9NAOrxDe6iZoRaAZZdFGrY1D+Ho4fHidkBOOu4+Kny+vo", - "J3+DZ1UQSZS0QNklOdSIAEU8Z6oOmJ+2U+Sq+VvLV1LOINjPhn8AAJrt3dblj0kEJ1Iun8Uqod+CuGt5", - "A3alQ4kDuyEShkMBvqSwr/iNdQjXB2CMDV5VZNOOH9dtvIQjqbgto1Wc6hG5jgiJ68ia4VfaxsrbL4Ox", - "8q+wBeMpChbbtyHeeXl2/bvLpIKxNq22H9PPOOsBDIjbUgvZYTD4LChMldAqGN8edsQohGMaeqFNajO5", - "Xr3Wv5JrBUDkcZ4YdLkw6VpWZS+jdSt+6xTCpgPNxfr6/ndQrs3Em9+qYJsNVX+Imm32rTup07a0O+dE", - "uXfPLRk1V+uvVFSpuLRcwL97pRpjY0ipi+wFj7bTzRoJ7s3CVhGLftsyGZLhlIwyQSb0egXxmBeMYlzF", - "DykPUpHBYIA8N1J8jfaeomiGhayNndHpTCWLagDOXgC06LOKGQqiCHOGwjY7X+6m+3A52s1up996SDg+", - "9zB4lmqHWJF0tAqg+qj0tlnrfIYXYMVpdBI+3d0bDHZ3BrdCqHbDusFyHZWf2NJ/1XaaUuq876yjvxKl", - "6rdQZDMv15O9EhSSootlkkoQnB5A4k2GI4ISMgE0uqKO93rPYr3r1YO3ApVFfino322U3Tfng6/Wpim6", - "suDebhod51ysgv34z9c4RBvYTLSEXRfIudvtDZ683d492H9ysL19F6jSxSI1ZXs8/bh99TTZwZO95Nni", - "6e/bs6fTnXQ3qIddUlOCpw2t/qLfbYyyKS/JKmhQhaWhDTuHjIh6oeB6gW1JEspITxYZUuvTFFfwAuN/", - "X3v+b2bnNzNYKTucVyfpixBYlYtToayHAbqyk1npu6jP5uR49SxulYFUH0iY3upDAfJqNxgoBbHd+UwI", - "hJy1vIbeeS+2vohWZsWtu4pCHnY46cFdbljxEHnXEBC8Wa+6wJcvuYDtdMoFVbN09W1RvFbgdUPc9Eep", - "4iqwUh+dTBlUCfd/LsLkfCVKf9zpdpKPe9UzY39vD7FloX4LArRb7UsFLcLIoAj96lWAV0rFQ5hIdq2r", - "6zH/sN3bfg5xCMnHvR8GvefViIOuWS1/+bbd25VfB23W0K+152o0bT+/UcS1W89VFPQLDVWKK+9lCwJs", - "abysyeyuDpdwW9ng8vHSHtcgcxoF0M+V9OzlNvKFppgkeBECgfcMtbKmPfpEhsZkSplsY7fdHRSG2/10", - "2OmjQ4vEDbpsWYG/0jzUXvfohKYpiamWMY3q35zBsNPSFlfXJW5WBMR9FZDW+mFx7fl6iIR1CVfrrsn+", - "Z+Tjfpb2207jXYXeAXY1p6ICWBe82EV0gjCrVQKlbI4TGttEekiMhHi1A4eIVpKs5QGylAOdnaSLplyh", - "MoW+pb0tZ812wWL85BrsrSswMwxB7HwRQJQCqYuuYl8nxygTPM6jMn80gUGXiB8ir2GhrRDy14fk3qV9", - "AxKzJ1yg9faNJoNGO/tk037XbJOaYJu3enuwfqvvxCjS7eRZvJ6HmZfacbAbQaSvSUEMmGiqy16TBL3J", - "fGjB0d/4K7is8xpbcqRFojxzDhZNU8uUFHC3gIshFNd7TBKir6nlRhBP4jJLgsqSi65nqdtPns2aXJzg", - "kVoeyC+EZFpXAfwj6C/FbBEcmKvvWdwlGwMHqy2Nw6tn6gLZ1aoO7ulaSaxxq3wTblOtAsPlazZvg5dy", - "6Zm/CzBtXzRbRkBxDL8ipL1pxtq3X7qwt0b78V2Y5R5SSHttXQ81fFSH3lvAkLv+y1hgLdZViXcv5J4P", - "kcVbqxk3Yb7Ws0B9q/Nh73+MlRmN+gdbP/zl/+59+M+gtbmmN0siejGZQKDRJVn0TJUfraP3q4inUGJA", - "C9NTSyoEp2BDAjRxexj98e4PCqax+BWnS1OACC2vRM/22gn95T+a45u8ZXwHfHItyX52BY67qFSquLuO", - "NlIipi6W3CWSbfaHDHLTLslCIq/wlxVpHKF+J4tPvAh0dGHEwD5h8ws0plBJUQ6Z1mpxFJFMaxO2lgw1", - "5cA5cB9BcOK3YwuQucRv65A08QQEvT9dgst9/e7tj6/f/Xo8en324tfDk9EvL/4bQjyueqaHuKdpb2//", - "iS0C7q/kdrAQxc3rKfTRqQ3Tt67+SQ4KLeB0SZTmKoegEHIdJbmkc+cgVMntKycsJ+vevhLBZ0LtKpWE", - "ohIsJHRCJwT8+nCd2KAaKh0xUgnV061xgzK0fGMbwhl2gJN6xe9DdSv0VoRXu9zY6qI/mbVjoQaENHDY", - "IeMVytwHtBcqAa/CxX54L6MNyBxxJV5d4uzmzUBRD4sGg5GHX7iSz+D5l6i2+W5lec05T3pavWkoSRC0", - "Jpu1CEbOQ1MmI6HT5HSYjgMyvDXtTukUB/wMIX/CF6mK6Qa0NmNqaf8by4OF8xiO6/UazLE0S1WrL1Az", - "EkjVa05zSLVUOypL/1eDZ3Jmc1epF1tXTVRNmdqy1WtDeBkxB9TwVdnK5Slz6Ig9+Gh9Eu5KvcqbmTeS", - "5r05depDTcFZsUBnemmuZkQQbyPggxIH/4ZLZvNyWqCwmOp/GRFlzKpL6tFSKbibJdooLD9uCYps42Vz", - "+Oo6B6f4uugBXClYLvkfYR5lnaXtlz8CJv0bV1uSTlwTMIyachdGYK9S0ao1cVS1vBk+VS3P27wfPHiW", - "V63gfk1nq0acZR8V0gzR498wVT9xAepgM+bJnQO5w+UfEwEYcHWY9lYY5zQl8YjnavX5t6Xr7ZVf1B8t", - "69c61RcDEUeVdN4mXuBQOcoxLK+0Xg4S5YKqxbleLxvMDWmQrmgsLCR0BD+XHUOhzk+fwGg8CSSMvCSM", - "CBpBGVR9HlPMQGNC70+9animMOISXiuIQK+PTqy5wUH+gvpIFZCei7s8PDvpdDtzIozK3Rn0d/sDOMwZ", - "YTijnYPObn+7P+iAVjWDKW5B6XqbP23zjQvF9SS2ktCP7iX9pcApUfDFbwEkAIg7tK+DCoKnnhKZYSqs", - "FpklgFBgCIbqrwHX312oB+ZW7pplb20zhTRjyH4h2Wu7uR9AUIazA9PcGQwssLmy1y/k7piEga2/2+jR", - "st9WUp1dogDM/ZKa52TLYuk/dTt7g+0bjWnVUODshjp+x7BN4iWgne/fcCFu1ekJM2l5NsnahkP5Jw4I", - "yT9rv33QeybzNMVi4RbMX62MyybBmEiE3btGj1MSRZpVQDGePnrNiHmOsELYRC6LnEENY/ehptDqKTBt", - "u00uQIp+5PHiiy1hpQ9no/hUZWf6uHxaoucvRzsFGS9vpH3kELYN1d4DAf2IiwLcD3ZS9gbP777TI84m", - "CY0U6hUEbOORqYSQnwTwwh32EBfo95wrjIpw/kd0pK3MOi7IrVteRVt/0PiTOd4JCZnBz4hIMTPJEead", - "NYd+6Tgbl0R5nFfeao7wobQH3FQOhMdcVCDIVY+of23VhcHl62gvgMBg+zTTix+Q8Pfu4YTbyRY1WB/y", - "yEHlS5RL8piOk3WxjUshJCjLvSTqa6H5wX1eWbaIwJ/wFD0WAn5JCgmv3K2lS2ErEzkzCnBQAnxTJiza", - "776rCn9vyydelAz4NXTTUM5CGb8qjhd95NbUKP1qARBLgsA84+Vr5UwP72s5YTv3ccJgxoWn6Ns19e2a", - "WnXKDbW4KcDB9E55CxvEjSwQfz77w42tD99sD+1tD60sD4xcWevC3/m4j2xEasRjguSM50mMxgQZvCMX", - "e6Kw6E8/IiyiGZ0TALWDIm15omiGBUSWpCjGChsfeqNhYqVZomhuSzfXc3GI5QLXcSwkGQEO36gJf7KM", - "QKSMkRjpTyx0XwknuFS325z9oIG9aLC8GtHVjEtS4Pkx5d3mkN4sjXYMzfaH7K0FetULCMHUjtdIkgBc", - "7Qr7D2cID5n94HvHQlwgmMRpybmwAMxAapApzbYsp7bpkY5kxENYO28Jw0z1ZEYiOqGRndYlWdh4zmCD", - "reou6QG7cb4/LRI20M5mGK8N4BnD4LzHxTNkKanqv2EQBB0leVw6uRyEEBZjnCTBwhzThI9xMjLrc0kC", - "PsGX8IZdFL++v/MmMR4TU6s9W6gZZ+bvfJwzlZu/x4JfSSKGnc3+kEEihl1rEndLARFdQSG3NOP6nAme", - "mj63zBC3/rgki0/9ITuMU8ocRcAnOJEckWv4DupbAWaG4V4N9GBOU9gPfpRLxVMf+dTRnRkmz1WWK5tR", - "IonqhlA/h0xx9IfDdvy09UfZ4ydwFhMcazrxXjFTAtm6adRyhPXsR/BqwN1OYAGGHX2RmjCPqcBMGdjO", - "ApwSTf0t3SiqI+hDullf4QgzlPHMVJYAopphTXKVNgCrAScJUnCU3LdacIedbJiPhd5Lx424ewYorXaM", - "KEOnP3qHabD3LHyeJIkECUWU/Nf5618R3Mp6D8xrZbiWSelgWmBAcQ6uU8fTXuBohoyjCooJDjs0HnYK", - "d268CWPNpQ2X6fXAp/iDHtoPppsujX/o93VTxl15gH77w7RyoM9Slhoc0GHnUxd5D6ZUzfJx8exDeEGb", - "4MvOK4wAbZhrbhM4CaaANOPd+OaKxCxG3N4CyQJhVHIgP3BlTBkWi1WJhIGltyvIJyaS0VuMP4YQuTjs", - "HAxd7OKw0x12CJvDbzbAcdj5FF4B67VsrlwH91nh3CyI6MlgsLkeCduub8Bn2cIx8IV1wEatqCi7qXfQ", - "wrD+ufwD/9b6Z+H6wUx3XkITGcXfGd8foQPCk9h9TTTggqiJ3ZhFJHFi93pDz/07D/RmRSRJ7ptAH4o8", - "C/dYgdT/qMgRNqs8RivN9w9McYP7ulQqZvuHod9HZz8PWM+t7ZzMXahzuE4JYNBYVRqZlxGW6BzG1DvX", - "yvcL+LVv/+t0P8BUvEj49OLAqO4o4VOUUGbzAbxAZS0e2LWEjwwMTfGdRaVxReI2jCTxr3/8EwZF2fRf", - "//inxXb/1z/+Ccd9y8CrQY3pixnBQo0JVhcH6BdCsh5O6Jy4yUAVWDInYoF2B9bmD4+QV+reSmlyyIbs", - "DVG5YF7ehKnXJm2D1lWg50NZTqSF8dEv0oktJmNiGwN2G3eWzVLe64nuBuAQYQbeBPSt6GgAsOSoKbRt", - "NdFO2GRq5lwxmtbDNJeC9dbzF0WulaHenhngDRkMLHHo3MEDO2m0cX7+YrOPQNsyVAEFg0B3KJuxakT/", - "G09az5MMR6kyFFhlw5sinOExTagzOTZUOzFHMMXRjDJSxhcXWOOuiQM3Us1jDs9OkA2E7MKrQ/b6fAtM", - "rIpEKhekazmBsAijZTk0bvNcoAfgX1RBdFjPvjtkE4IhT+jk2DABD4S7yAcsGmYA5AExrlRVKq91h8wg", - "yVrkYn3wUh6TBD6C/qdYkSu86KKi1q2rjpJgpRVi2dUvD5nBerVr0AOoEuQNsw/8zAyp5yJ5bc6WIJNE", - "q8YQgW/KfkPfGxMukI1w9qr8u+5MkqUZll60FEevz/X8pqAJcmMPhJZen7vd2OwiyVGUUKCGCLMhm0Ig", - "kAPv5ayyq0VC2QyLuBdxfQn4YE6XjF8lJJ428dgjn8juUJKp9BM4Tj/XyfWxCRez5QnoQ2wA6lZ77o7t", - "O+1cd7bFP5PvzhaCvIHzzlhwieE3ZnW/OfJaOPLC6+aceiHP2rFDYLy7iF/TxQMF/DraW15z88Rbsoew", - "6KENB20DXhEu0NnRCcJxLIiUm//e9j49U0Olpfyn70fNih8i9MSOhQsL+mftLVUCeSzs4I0dNcJuXvX6", - "uv79tlUpvtN40xV1eMor7+5vj1qnN7lGSqG3pLVvN8naYFsqIw5lBktq6YFolJBCfCnOqU9F66zKJoy3", - "uHJWikuWPZ8cuwN5f/Zl23XO6nfDPTDF4xpDfEBGWE219qtmPyZqflfsokObXmF+/rpIc3B/UtB9m6JD", - "ZP6Y1MW4tmyaCxqgk8YL9CVRBt7kLvV020Ng4udEuFNtBrowsy6mZT5FBqcFJgSWmNW674l5pZ3qa9r7", - "M2m+sDw3kVjskn8TUVoou+VarVJwT2wJ6LvTb6GHG6m3Xy5sxRJYYJHBijp2biewrG5guWDR5rfIlS9O", - "0SausVRihZs3iQtLtkFTKvSs+5LrDplfb1zLdFavpQxNEjqdWSdATCcQq6f8+t0wyp17GGVRJ1tgRWyI", - "4mPM+z3Ti2y9wHMiFHp9dGLW379St/6AoNX1qpJjXitv13dvXvUIi3hcOE+aZVL75AsrTIb+K7m893/q", - "HmE+K3XiQZPA+Bn7b4LJkYl/71P+v3Z+SuhYYLH4Xzs/4SSjjPyv3cMEKyLV5p0Ry+C+brr7VmAeMfFp", - "/YVWFw1YE5sCZOwagb94q6XM797/U4n9ZtI3EvyLdf0m+7eR/f3lWin+2624UwXA9PFAHq6C2EKrDY++", - "Qdrcg9HUUqQHaVPxIpWgNjMuFTx6fPnNNqicFhTnXxstrf/lgVx5fTjSPTnuwkJCRWmoaGHTB+/JF+DG", - "ce/Cre33/h0Bh+mYTnOeSz8zMcUqmhFps3YTUmXAj03sLq/nRsH7K6bSwX1eHfcuV3+j+zuS+Osbapi3", - "ceitk/ndW21lfvu+lvkNoqnNbLZlN7quJNNmQ6C1wzRtS8YV6NflAPDQuEK6CHqnFZVSXUCgQRwM2f/W", - "+sdviuD0ww8uhTIfDHaewO+EzT/84LIo2akjFcKUoLaC3uGvx+BFnUKgLBTZKxO26+MwNbuB9FxZgX87", - "Bal0JLfXkBwVftOQWmlI3nKt1pDsXtytilQtTXLvOpKjt9CCW0zxP6eW9Cd3j1Q0OJlPJjSihEGBF0hM", - "l0vxgEaT++YZuWVCMrP+SC+YqCKJtFYjC661RkIva0p/yWidbiPOO0dYKZJmCk0FjsgkT0xlBCRnuYr5", - "FXOw7zBBV0GIlvMJXe+uqZFrJJyEFq7+21bTLSp+3beq62ptP84sMJ7Z4rVWuSxFm2bt8mGJ9251yhZX", - "7f1rlY+ZxIz6trx0mdYQAmWMTAGrNDcpc8WXJQJaH719+8qlx2n1RLiiWIq7SliuSOiQ+ZWw+uhFWWLM", - "vOBa0OoDiW06LSQN2tpSMcFxQhmBeGIiQ5ls1fp1D3osvrwEHC7O10oCvudjacutPpwE/GCs4F5kzZNK", - "FWteGiT8un3FaXHyJpyaR8WvLAMKMJ6QrLeFc8V7NuF2a8YNClsYiPIswRHgUOrXDESaxTgwmIh+UwBc", - "IHiSEGGg77JcOXFryIrBUeYVpLeS2YVufpQzRZOLrgnnAfwSiTBbWPynIat0ZmU+yEOGHHsYoSCZGXGt", - "UqUeNOW5hLcgZdjvEuHkCi/kkNnMZfM5VPUVJDIokUnSRz9zAI1AeIop8xivKZf4nRyyCxonZGQxHy4Q", - "lUjOuFCEkRilfE5ktV+CRUKJgEkcYb1yEqV4AeBrBofSrA/PiAE4qyBLcP1vzGIKhfd0z8WUD4YMo53B", - "AKUEM2nzxCWewIVj20AwiMqAvkcY7Q2e269q+wYAwW75N/RpEoLMeYTHyQIRTcWAVKE2YQNTWwjTFBTW", - "2zehQpr9KuybtsJZZWOpdHUd4y7KWZkJD7b+nBWJ63q7VC4YzNN6AQkVxTVowT/GJMJ6PRmv9gOwizyK", - "chG6IPVWexVZ/x0FR29657BU4TzzBEwGEYlhzxlXMzjTHI7S5vcNVFUS1Z/jogkeEi4QRh5dlxYNEuXA", - "GjcApvCiLC/IXLngi83v3dnRx9cyAnf8DVDgY7mfgIj4ZFI5gOuvJnOAV+V3LJPwn/WcHrm6sj6Liyme", - "Mi4VjRwzrJeh/6YQtlYIV69skJonXFz6slWVfn/i4rKtBmbBT+njUsT8GX6Fjgg9PACafnh/BFjDjbKi", - "iebelbQ6fRWnFIQuqqQLdOYo4WyqT1Fplb93t4Gv1W0Y0Dh9mQrj7C4gfrQSMrI/mtK0ejK28Ce4GCLb", - "6kPzIt37PTijfuUK0TRLSEqgdG3PEJve7BIOCsr8U+mBIt2MV+pT5ecuG11QmviDrhOHgK7chm2A9L68", - "XUGmmvDpetDBonOHsBdAHRyyd9LAgV8Y19MFKniwFmgNxD+6mtFoBgiEoLfq9g1AIc6yiwJ8efMAvYSD", - "7GNQQ+cbBthf05rkCTHAgvM0vThYLs76/vQUPjLgg6YM68UBcgVZi/tD6rd8REE9iwRLhX61OIkbhTIO", - "O3qhsNY3i/ltWqzBEhx7yEK4g4xc2QbpBF14EIQXDfhYjt++4lP51biKypIGZi6KI6s6Am0SFneagjxo", - "Enb8bA8GIaTtlkiIZhh3DIS4NJhXfFqUU6iQMs6ytuRrhwlUPE/TFTSMNjxYNalinqu/SBUTIeBjS91N", - "xI02cGRLaeFLTagWRM8d7E0gv2Aok8E3Dy6VZqqdboewPO0c/Gb/NU/TTrdjx+Phot9AuF+DKFlvcDnk", - "Ru+MBxv5TSy/CSBkldl7iJC1m8Oq080S+Rvzwp/eW+hsdg9IhiAf1Iy4X5MI6o23avBhvEC2hJE9v4+R", - "AfwlihIuScXB83jAs6yhqyYzNhuK3Br39PDi3FUbahPBcm4/PXdffgW697pYETdm5KZ770EjyyN4zInA", - "cmk2Ey7qiEvrokm+ekL6cluyNNU2FPKNNm9uZWxFmFpPWGYR9oPYVJ/DueIpVjSCykfRjHPpkX0Bj2xq", - "lFnjcUGZYFoxWq7NILjQpHphzdAXVo04sCYzhP1Hto8+fG7zDsJfuEflFz95VoGC43ed6A/VAaA0u6Bk", - "gjKcS6KlujwlKFpEmiuaUlcERzMU4UzlgkAVP4JSymiapz7utd6xOQaMjovt9KKLxrlCCRZT0MrMQxds", - "E/E0JSwmYJ8bshnBc6pVSoESrAiLFj1JoPrvnKArLi4TjmMwMWQxBk8PVA8URFMggIinROEYKwyCzoU+", - "8SOTxHRRFAQ2aj0j1yU1xEMmcva9qWigm71wA71ABCC7qZwVhSMjHBMWBaGsz79uNvblbdHnRNUn+kCR", - "QbfipQ8ZKuTbXN1wvo4ookcWi82F3cY2bH6F0CubVdhq9ocjo3/PI23m6ub4QA6mYolXneKvw7NUEN1X", - "4116ePcRFyjOTXfeqQQy/7P6hAqG4gdbQWap2cbbOoaKCnnFMt+I52394f48uYUt7yvhhN1Gxb6pFlM5", - "6a+B5dpVvRXPfSAjprUl+Ta5h2PBLqLrwcQnLjwu91iMrZZhm6NZ8G2fOymBQfvi7BvbrrNtG/BwW7bt", - "bLNLLn2PkVPWgxjRMAe3ZtxGVm1NB/+m2Si12Xks88FZZOm5uDe2eFIwQsMaM7xIOI7/DEHCK/xHERfC", - "wF8AoMZjgl/1rIZ+egDY5soib12Xrfn+9HSziUsItZJHCPWIOYSXkqM/S+NlA+7rORGCxhalFB2dHttw", - "XSqRyFkfvU6pQoqjS0KyMqMFsgr7en4OCGS5oHwF8aPbIUyJRcYpU2tHUb56N4P5dKsy9PfMJy2e9zd3", - "eGt3OFj2Hx87Ay4DORtmAqs1U4XV2jqjlE24SI1chsc8161rHqSXSe+nQSqY0ITIhVQkNVGJkzyB4wa1", - "IWz9X/ud2eUuxOTqk2PS5TIiUiol5UwOmc0VyYjQfevPdftegFXQIaBwwV/PDJP8OoL39GBMvBpWTasG", - "kE1QV7Rz0NnCWbYVY4UbAsTs8D5jSD9BNB6Si3TMExqhhLJLiTYSemnUEzSXKNF/bK4M5xvBd1+6uvHt", - "T5Ze6RM24cHacYZmC2L+U2V1WbbmHJOPjq29JP5hcfwHNjrM1tbXTxYEJz2oR+yAe1CuaEI/GlanG6FS", - "0cikHOFi7d6fFky1P2SnRAn9DobUtiQxiAagXW5lgkdbw3ww2I0yCuhvuwQGBwyv+XEKPR6dvTNpqCTl", - "YtEdMv0PaPjt4Znx7k6wtSZ4A7WFk9HJ1us1Ac7nsEz/xhGCZoIr0QuCG/7NJXhzjJHGMyQbjijPVqlK", - "PPvTh7BaCe6bXeFx2hUA5KmYzUYB7OXQuMI2hDlP8lT/w/xxsg7XTOFo9h5e/WqkXTOctd24CT6KQ2nn", - "FBNT2/JBnB5mwR5rzKpeODcFEGIq0YDBW+BQ/Rmp+8ub7/11/ArdnXZFXd3Yr+Zs3ffNZ8fgEDb89Xgs", - "x9xQmpuJ4qutT1eYNluffkx4dCktFItvNtR6G+Cr6x9LPGzrIgQxATJDkYUwMkBZRHaHrGaANIg/EmGk", - "iEgpw8kWzNk0AsjezoqF55xCgnYEeSo9SWPATEoAvhvg7/RswFDlGvA8utJW1vLf8Z2RiqMxiXhKHNr5", - "Zkh1+xum6icuqtDlXwtffOutP0ACYgr29jVo7c09fhZ6+ym+hlDpOLcOZTeijZe8/NGYgroI9mbY2R3I", - "YaeLhp2ddNjRO3CEwYSKFdpHKWW5IrKPjo19C1JwnwyQJBFnsXSg686CtzuQTQm5hiwbsjufwHf3KfZY", - "qoKlfGM7CbEH/R7S30PSDtrwD5w9k3EXDl2MeK6Mud+eK/tWTBSYRzbv3VfrnZFvun0bTv43e3wrPAp2", - "WbNLb+sNZ89yOSPNJrdXppBRrsYA5u2Ki8oZ+jsfyy5i5MpYw4VU/SW+p78+Mx3cR6EB3dVNigzYuX+r", - "MNCiwkC5VmGwRhNgqa9kRx0GsZFcZ1woQHG0ufaGhkCTAOQIHuEEvT46GbJIsyIDLShIyoE7WTx0cwsf", - "/u0cvTh600XHUOgS/ZyPN/voNUsWrty48dEMmZHEDPOKMENjQ7UkDl3PZuxAPXcZLK47eKDK0eZkBDwr", - "bq9ckHi3MyM4Bonkj84rbjoLoA6/eaUPEAD/mi+Lbe+sFD46b4gSi97hRBGx3OypzZNiBWaGvaQdBJ0V", - "3Azwpe5QOuS1sk8jGxhojN2dTgAp49O3og93XyD1frxkJk7ElNsb54A0yiDJAMeLxxXLJGeoYI4hFuhf", - "10XZhKYsYcvLVioY0GVT5PdXZHJfybsq2PL/rqcLZvpoHU1ZZZ80ERflVtZ6el1y8MzAIVtHVYQzHFG1", - "6CKcJPaOsjdBEZHSK8TfsSD4MuZXrD9kb4pCLzahFx2dves6Ry2Kqbw0LVhfbB+9nhMh83ExOAQHzXiN", - "Yc1JPGSKowgnUZ5ocYNMJiSCXFyo3yIbfLnFUDp3eHbKToLFZryo9vzR1bgL0wTsXkkWdYrbMlu9JUiU", - "YJo2g49bQQ0CDiHUYKwb5QxRNklsSFUkuJTINtUjCZ3ScWIDhGQfvZ0RJHFKhixLMGNEoFyaqHg99F4m", - "iJS5SfDWDQBIr6GoLiqBBTPBlQ1NSDgX0kQTaAp/f4qkItkKMntjWj6FOd+RbGsatz09kJG6NoZmU4h9", - "BekNMZRiFlzTUZ64AMZ7DUU3A3poKfGxHPy3gk6nROhTgQ2TNeF45li75TSHvpKx3Fjv8rx4q129y6JV", - "LyvRy9hbCQw3KrG2487Nov4CnV/SRuxA++hmWcS/6I9a9l3NVg0Pwj76zFmGSnf+O1bJPPeSBNsasEoK", - "f2zmJG/klaNaSbRdD6vVOrP2LjNdW+NnPRhs1mNGy8KV9NkmhffrI4TB/aI83HeRtcdNWxW0q4pu2pDy", - "vx5N/6ugwLuB0X9glJNbwOh/VXn3gHP+cPgnwYP6UHn0Fd+zK7b7p0fCv6v0eQOHD3BsTenzhuvZ4NWV", - "itJ7+047Ncm2+GeS4G284w3kd7fs37T+FiqDt1jrXNCa4EmaqYULaLO+yjLoTNKPpN/gCC7iVu/OFXyL", - "kM4vRx6OThsDOv+ctfEfJGbUlg6kEp0cB4rOPzKMQf/MVS6WLX3r9LCIZnROmo3u1RNslygTpJfxDJwr", - "sVkwux7uLlNY9KcfkW3eYq7af0HtSYDqJzGKqSCRShamDqjmCKaP7yQSXGsC8JyLRXOUiDkiPwmeHtrZ", - "rLkP7ZmyxrAyzjBd9GKscG/uuM0KE9pnRHe6eErN8BBl6OWPaINcK2EqXKCJ1nwQnRRLSq4jQmIJNLnp", - "D3h70GDZpB/JaDpuM8oVtUpe21owKMql4qnb+5NjtAG1z6aE6b3Qov4EJNlM8DmNSVwZY2fOE7Oq2w0L", - "elO7qxYqisJ1Trkwg3sQGabNhTT9SLMqWyhCYsaUYRjc2qog1TNlkvh1f5gyF4Bj98iN4tsVZjW/Dafs", - "aEqEOpx2ERXnBuJ589s195ivOT8Zyt1pldvOheesNl63y49qmbZ0F4Ufity5+zVbv/96UnqofJTZPNZ0", - "Pi8U0iaz+ddFgoP7ux/u21z+/hGngL4kTvn2TOXQgG4xRDCvIKY7JnOS8CyFeujwbqfbyUXSOejMlMoO", - "trYg9nvGpTrYe/50t/Ppw6f/PwAA//91ZvGSme8BAA==", + "b/uvJmpekx9IbJSZ6KGUx7UaT/b9dhfRy7N3Z01jKqouIX90S3NaAe+zHO0yyZmp++TZVLCtx2gR1wwe", + "j+msj34kUvXIZMKFOrCBLOCQ8mq+4gUSJM41PfjQZJwhSccJnNGiVy3w61/0hAAIZJxPJkRUUQeeh2Rp", + "7+1RGjDu/aSfI/NCAXtw+mNVntZye1ut/axCDaC2T3BE2XSz9XYHDIK1aazDFXx59u6NLRvUhLisl7Io", + "LWTAlvvo16Kwll5qWaIq9QOWwjpyX0Mi2dlsIWmEE9OiqYpBmW/gg9PQWkI/Kz+0ptCAnB6ux+ZOHtqY", + "T7Mczv35m97J6/dbaUzm3cqYIJBvxhOix73psae5QzMos94qXGneZGkxhCHbnlhvrQqW0XqRPAYRWB3F", + "FU5GMuGhmKG3+iGCh2jj/U8mHVmPoIuyylbq332oIZ++nwRPDCDYN3R7Dh3WTbaVAx7UXesQaJ3q9Cqd", + "ho6KSaValrGWiyHyy+pG88v1BfhMI839Hrlsr5pR3QqyyCSFGbxGFwaBzKfGam5NM5JkWGBFkkUtHrMK", + "qE6WPdrkmkQ3SDd7oV//ZIpo5IKM1EwQOeNJNQ5it7tciFVCzPGc2NpTZk6e4V9xlGJxCTexE+RRzswK", + "VEPWd9fhy8yUym4wqZ/fvj0z2r0iYo6TetKDXPLwH5MEL9CYqCtCmJsKlgj7Ma/1pFHZUP9HqFFGBOXV", + "NezsBvo9N3HQaCpwRJD5ypVxtlsiIb6o7VLaXgKIhVFEpGzY3+1V+2s/neRJuz0ODWt7bdXz6CYb/Pbo", + "zNWxKeoEu2XeWV7lMyJ65si5gsGrt3ZHrq6o5LpiBoh2SWAYEyixZDNN/ERQF98EJaT055XkS485SD9d", + "0PYDp8AsVdec8w+tpMv6cQ858lPM4lC9ZZNgYFLzp4DXBVHOIgfRj8Ym+MmgBRg9wM+TEATHlBEpa3nN", + "US6STrfTm9hZHWxtJTzCCUAW7u1uP9taHUa6Mn7YhkuNYrpKv3RBVSbsxmXRGrw3mHSVJLZwlrWwwJl1", + "XHM/AHtajleEGsv6bvMkPBfAPhgspWhf40i5qnBgkqu4XLF/bAFYuDIfaDA1hV60qP3cP5+DoA88w2pW", + "Jf+tJdqHuBiIadQ0YswdtXU0RP4xKFFxESr9xYWyGX9j4oIei/vQhRQ6CN2Kc27wzJ/lk/393f11fAiY", + "Te2Y23MXmKp5u5rTTLzltufXNgDhWVWZwx3p1dVL9bqsoSnNEZdIavWC8oywG63n/t7uzs3Ws+1ETlxY", + "WI0vhSBhjk6PjUwUcaYwZUSglCgcY4WrTAZsWZrLQG0ZTFJIC5p8v5q1NMRP+Bgvty2M9aW87w018t44", + "SOgUMzrRDNm+6fcsZ3hn/8mBqeQZk8ne/pN+v39T5IsXJdRFq63YMkF6HghGX84+bx/uAOCizVz+6Jwd", + "vv1ZM7JcCnNpbckxZQfev4t/lg/gD/PPMWVhYIw2xV/pZKnoazUeLbfIwyQ+QGV9byf3tIkPajBGQ3Qy", + "oPEEYeYqUZp3hydX0Ditlg24QW7wilxZLa68ZsmicW1uXce1rHCuvPqtfrZZi1qu9ONq/7ozd8E7tk8D", + "QF2UuV32rN+qULFcWctxqdRXRlhRvTFJzF8RZwDsGyrlWLki3bMWlcDgGrElv4ou/R+L3r0fj/yBeL+7", + "SmLeT7am44cbhsSsFEj/tiyHrudCThxdc5jDtsfiVmhbP9fi0AVjwR/4LrxN2Fi199fT//r9/8izp3/f", + "/v3V+/f/PX/5X8e/0v9+n5y9/iyMk9UQhA+KI/jFoANNTXgfP7AtKZ1iFQVsdFr9a1hh+8RYHFQ0g4qf", + "aEwOhqyHXlFFhKkfV0t+HHbQBgFNCb7S4i4UxzF5Z5v64zPj0dQf/+HE4E/1NmKblC7shhRYIzIfxzzF", + "lG0O2ZDZtpCbiAS9QP8VowhnpggcZUjrvws0FlCxz7qYys676A+cZZ82h8xW6Tdo2hmGomeTIuuLOQex", + "HZUJg7WvkwJ2wmQkDllxWxcYfMbP2C8B+ylJ6jlDDYuyWn+zmtOzQQitEPJZ9EZCURpQQQrK1mRUJNqg", + "Z4PNZX1ujY5R0NAK8rOOeJPweJiHzMVNSZLHJKYR8BWXJzizmaRFiqahNGvEywS/XsDevDHJazHCuZpp", + "XhTZxPqI80tKurClXXCHQeQHfGn8+TOe9caL3oxnBcgCFibaBRuPeFXJ/j89O9HeeyLoxPYUzJXXJBIQ", + "OuHI2JmZlMPCurA0sbemngfTos+c2NdNYRhpaoGYUG6VC+bKVRAoWgloHgX1kZBM/j2KEgpWJznjeRKj", + "GQDuKd1MCDOvMyhSZvA4ismk/u9qSMPO/hPQYN2/d3daZ6yapVtFZXkS0GlTx/pacGzDJmEARjwYOUP4", + "miAkfQNaPy7YKRSH/54j11B54gpGYrxTJodO2uoHifSy5zaDaU72GFjIkBG2p6nNfbR0CitZUy1aMNEJ", + "8FnSAqzkhcnWfPvqHCkiUpc/vxHp3YFTYpApelTK3BbiOjw6fbHZ7wSBliquKtiqlVlV1UEHsBZstEJT", + "EEZpo8Ep6aKTY8iWtddKqYtBesNPXKDE3IrlZXQAYB1Vcw82JfxOjq0AmizKkAcjtgw7m67FrH69HaA3", + "hQqIi6EUeY8lbbkmy8sEmrUBcCb3Yqn1Wpos+Mes+mfvY8i0gPqGhhcDWGTj/dXe5uigp/RFVbOQ3fhC", + "8qNQGu1f3t5/aVjlLy+j795MRrc+4FE2wzJE3TPfqwkvLe2770ausnvRHDNU6XckafBs/c1VefGuIUX0", + "PVf5PIQjut/b3n67vXdz891NEXGrUFgeTF4BitsezfYuUGEDGK9UjRqDy5F+bEPJnV3k/SmaYcm+U/Cw", + "Zh3Z3n3axigBvbYNy/YDsvnEDKngUg5XqwgnNghjlzRJjAAj6ZThBD1HG+cnL385efVqE/XQ69en9a1Y", + "9UVwf24Bjgu3AKyjyTALQCtVIAFQkTv49u0rOFwJgfQLI4df3h4yd61psQWErhvcy7N34PjHcuQCN5tz", + "FXGZ70uuqVRyGVWtVfzz50D2mk/bFfl3kzRtlLX+V+P+/lwBpg3C5G3eAWCvC15fWs4HwLJ9yCTBrw9H", + "dyXy7efC11o7wx2h1zZeaSHk1xoiwn7T7XZ7HNo7GU4FUCbEtnwJx2Vw3xr4tduhgezVQ6kvHhKjk7Oy", + "0lPpjHDN1+b0fKe//eQZFCvdHrRh7CmOVvR9enjUvvPBjrllDvD4IIoPQGG/rc/KErZRQXByhRdQ7M8s", + "7bBjLkxPu/WOrVUkW8XXLOPr3g5Oty7GNQDmgjjrApfkKF1Za6RFemIdNC3NLSRkSpOEShJxFsuqjDzD", + "EsnMIKGamhuFBD9kMMAuKkofg5SCcBSJvDQ9Wunayvt5Zuke6n5mnGkdAID/fyELiVIKTtCiewh9lKjI", + "gomHbEO4jKkiNQpKfsb6B8g/6NrI9lgPjSqoLaI/GDI5y5VmYpt9dMSZzFMirFUWjSl4jDaRzI1KC+OF", + "1VhohilpTMSQ6dcCWKt/FOrJwZPBYDDodgpNblf/exCipjt1fvYtlrDJ+QW0P2ZRhQFGUOQM5Swmoqj/", + "TQw51EPkbug4/UwQYfd5O/HKfl7KVeGDuQ5zuB2Y8OciuMJQG/RziA69hXK+f3sRvVXOkZNfbbaR/Wp0", + "kwgGgiKeJ7HW+Mb6tjMGORJbM6QkynBn8y6V6J2pvVmdug09Vhz9nhOxQO9PTythD4JMNA9oN3HgEg37", + "wLMbbcPOGhvJ2tHcxL3s4d3eB8ZtXVLxJMQvjmjrexhdErSh0Iphq6I4r7Kvaa0ymEVCmdknTTQrJlgz", + "5cdkPsrzkFaiHzkclHfvTo4rlILxk+1ng2fPe8/G2096e/Fgu4e3d5/0dvbxYLIbPd3d3tldkXHSIm3t", + "9ploQdU0EDBchIePXJh6KHq4KUmgJgTYwOcrymJ+VblngpGofu82ynVd98sx7K2HEMx8SbBUxk7QwDJO", + "4TYlkW7bRH7b1MiiplHYovjk7WD7c80sMLgGZvxW5Mz4L00yf2GrT70B+5tVHefteCsMyGWYrFstv/P2", + "izY42H9+sP+5i+ayJNaNsU5O97i5TaFXDuy4lobhUgE9g42zBHas8GHM5zZro9PtFIkl8DfcurWg5eJx", + "q2yppgPbDbORVfy7IUv5pKIYQEiIAbuLD7RI4AR/KJ1Q5MJrWeMo4XmMPKOXwf4Ch9eJpyToZsD/ZG1h", + "BsvTZD1oZQLAo6FEA2WaEYOjTzdiU5oP0Et4Fx7h1OhPdhCmUIjv48LxwgSm6PPlujbazOohn1tFBr7R", + "Wg3S/4Jp62WwttHVTRgx6AD9yuGbQq1ivG5kNa+DPrP8et0gu2FxsR1EBXRmZboD9FMhxxWSoJX8NiSx", + "f44swyqRYTYr+fl2xzuaWsqd83LNux2zop1uxy0U5KQvZ6e/K6l+6fz5pBiK2CI4gbNcJuPmiiYWDxtm", + "QqWikbRZGnpzm+QLW8OIxCOjpTQFf5oMT6vJFB858eX9KdoAyMO/IGtB1v/aLAJFK3fdzvO950+e7jx/", + "0grYqBzgehn0CPKPlwe3ViCNsnxkjRBNUz86e2eMDJFR34sgk/enPo5EJrhmPXrmrkG/8+f95z6eU8zz", + "ceJ58Cz4m4GPhQ0LQpYVvKgh4PB3mszpZMJ+/xhd7vxd0HT7+oncGW834NSajsL2rRPfi79kDCbjnqlH", + "FIbcAYISshGV6g2RMAN0ThQC+ukhHIEeUaQNW5Jz2FV2xYOEtbe7u/vs6f5OK7qyo/MOzgisXYFL2Y7A", + "O2LwJtp4c36OtjyCM2068AaAEmdWxwyfM2SLCQ+qAml/e7AbopKGi7ukGtv2PG1c8vdWT7OTsosO2c+F", + "Drd0yoOrvbs7eLq3/2y/3TG2dtiRuF7NYVxukFkei3jv7/wGSJNvD88QZN5OcFQ1orhQrBuNSt1oVFCt", + "waCs32Bgz54+2d/b3dluB68Wiu6wwIGVA1vlXYFDFyCKwG4ElmKZ9XabbouQOGUI7A2JEkzTw8jlMtRu", + "H4OmPhLmtXIT2lwM1vS/dHG1+LaVFamwDZlMGCMacIFyVtTw6K/3fX4RF2Yz1zbXw3quHsp/YXr1LA6Q", + "qVV2i6XMBJlTnssv0BBXJjl1knAubvRtk8Lyhsg8UcbPSCV6f/od8BRNa0gqklV1KEuNK9CSbjm5G53n", + "ComEibxpsVrtRputXzXhbsOp7a5Crqhwg0aMslhzrpytj7I8wkmUQ9UaXOynnhWAbUHufZYlCxNEnySc", + "MxTNMANvhPCghdCMJ3E/GHKqn4wmwfAFfoUSbtCVLwnJbEEXMwj9mRZh6JygDb+UmSGlWoHR/dQwGVuy", + "o0qN+2m4UiKWoaywIudcrydW3AP+NZ9UTI4Jn0pQChWkB/TrePMZFibqHzNToGieGl0yENkcGGKNmYdu", + "VHOT8olVcK3IARndZiVxJLiUiCR0CsVw3p/WEoVXJJcV6cLrIyerg21BusZzGLjKDOxU6zpmofsxkDjz", + "OTck0DAk562ISXTGyRSzHEq8eIRsLd791nGHMy7VqACAuuFgpRpB3YZckBKWrkhvL+xB7p3gvehY222W", + "ywb43urrJaoKN9U0wGaeGlzR8Gp1CxoMkfEyBNZK1K0SxquO2XQTVLgS6J9KaJV6+GBoA5JLPLbkYb1t", + "tolGCausup8lbdWW4Xy1Nzhvi5+2Gi7tDKvZCZvwAMjGDVyUzhJtw0IzIlIKlUtQTBglsdMlC1+lNXVB", + "ZnYiCYpzYlfOyKcC2wXH5ngDUAZzNjLKpjVeX++wjXnYjGF1WQfo177YJq5IhjNX34oc1soEBkqEyxzW", + "VtGWVI7C7qzlhgWZ5gkWyCIfthmyXKQJZZdtWpeLdMwTGiH9Qd0BPeFJwq9G+pH8Aeay2Wp2+oNRUw2g", + "czM4m4BnNqTWbzmFH/QsN2vpv2CJ2TLfbwE0S5swrWBI9k80IRZG7x2j1x6hV3HP93YGTWnpDY1WEtKX", + "IRhvyrktyQZPfC4DSXwrpRxXvYjEFozeiD1ZLk0hlRa3kkM7dS7A23l0qhkan4cBcmT4dQ0BBI0JJNi4", + "qS1zjRZssc1UgjUccjlDf+fjqkG0bXxtoDLYBiuxKASZBAPpYUdXGqTNG0tr4u3uTcAegK3qicJHN8RQ", + "WFdDrQxkauInb5bKic2IXTLq5mhKi7UoleECLQqcANtre8CAeuG3QGAwwMFItYASqlC3ZuFVM5RozIUA", + "qGct4XDmZgP4Jlrm0WvtAKbQ2xlZIEFSTNmQUVYYSQG1jCBG5kR46ahcaCVrSuI++pun4gE4dpqphUVd", + "B+P5dxLxK1aMccj8QerGc6nbOWTGsihyqMpfvqSbBa1PEwqkB4MTTAkoVEjVDE0EkTN/7qHilFrGu+Ii", + "bqz6s0DuFSgmAz5WpPglYT4rK5oJqoamoZH5ajlczlSWhadW/0SVYq+oXsx1dX+5JCIsJBZTKl5pFbri", + "HRVPOTFoKwA9AkX87F+GxRdwIy3ARcrm/+qaLH86Kxqv/lZ7zQMQcXi+h8ZsGzTBRiZfphbsU/WkrQ1V", + "gXyzVbAxy74EtOFilV0llKok4FUkaXVPtkt5q0flu9FsSRJVe997tv/0ScuSMJ/lrDMwWV/aNTdPV7jk", + "GnbqtI3f59n+s+fPd/f2n+/cyMPiEjga9qcpicPfH7RBrpU+rMm//vHP96c1r88+BDsPbjQok8IRHlJD", + "Gkd1QO9P//WPf7pR3XpAIUazDMXd4LdvjNJJ/J10gQJVF147J9kK/f6wYiTABZtBG2QyIWAGHZl165WD", + "qeFttJOCcYYjqhYBRo6vTFh58UoNUrqNO6g62JDIa9q28KOac8l8XGZ3brjO0X8a33CNFp61riwl83GT", + "H/p1vVfjhS69Fn6MQ4sQA1kUNV82cBfzucKyEjmt/44gwcGlci2ntZg3VsPb1nMOIIrFFlDzQgFDsOg1", + "edJ+5G9/bTs9v2XFrFNf8Q8rzmHzEbyR1TdwIweMvtH6HNYaf7AX4O2+Go39mm8ri+pVCsSVt+7N+22R", + "prtckKC4wW7en5eZeJMP6+C7QI92DHbJy7a7FZJooCYv6SRgQOMJ6RWBejYjBcnceAT1mbd47oFUyeiS", + "TyZVUNn9ZhBywNeBrCrXC1ZKayZdRK6dzaKOYG3AdIadfTnsaBVg2NlOh52a2yqYp5ji65HtoAqiMliF", + "Cl7mmdcGKd0MxgmPLk05M6iS3UcDlBLMJMoZHP6aV217sNo71O1k3t4UGNzEhDgtsS0Y05jM8JxC6Qfr", + "U5lWAjHJNVUSAkahnQMUcwOrVKnlameoXzNZhAflpOHSwWxhG9YN6vc4cxGt5btg4JtABVn2kQjetagA", + "mmO/fn3aNQEMEHpoBlaJb3QTNSPQDLLoolbHoPw9HD88TsgIxl3HxU+X19FP/gbPqiCSKGmBsktyqBEB", + "injOVB0wP22nyFXzt5avpJxBsJ8N/wAANNu7IRAUkwhOpFw+i1VCvwVx1/IG7EqHEgd2QyQMhwJ8SWFf", + "8RvrEK4PwBgbvDLMph0/rtt4CUdScVu3qzjVI3IdERLXkTXDr7SNlbdfBmPlX2ELxlNUSLZvQ7zz8uz6", + "d5dJBWNtWm0/pp9x1gMYELelFrLDYPBZUJgqoVUwvj3siFEIxzT0QpvUZnK9eq1/JdcKgMjjPDHocmHS", + "tazKXkbrVvzWKYRNB5oLsrYO3h3UhzPx5reqEGdD1R+iSJx9604Kwy3tzjlR7t1zS0aNO1StqFJxabmA", + "f/dKNcbGkFIX2QsebaebNRLcm4WtIhb9tmUyJMMpGWWCTOj1CuIxLxjFuIofUh6kIoPBAHlupPga7T1F", + "0QwLWRs7o9OZShbVAJy9AGjRZ1VPFEQR5gyFbXa+3E334XK0m91Ov/WQcHzuYfAs1Q6xIuloFUD1Uelt", + "s9b5DC/AitPoJHy6uzcY7O4MboVQ7YZ1g+U6Kj+xtQar7TSl1HnfWUd/JUrVb6HIZl4uYHslKCRFF8sk", + "lSA4PYDEmwxHBCVkAmh0ReHw9Z7FeterB28FKov8UtC/2yi7b84HX61NU3Rlwb3dNDrOuVgF+/Gfr3GI", + "NrCZaAm7LpBzt9sbPHm7vXuw/+Rge/suUKWLRWrK9nj6cfvqabKDJ3vJs8XT37dnT6c76W5QD7ukpgRP", + "G1r9Rb/bGGVTXpJV0KAKS0Mbdg4ZEfXKxPWK3pIklJGeLDKk1qcpruAFxv++9vzfzM5vZrBSdjivTtIX", + "IbAqF6dCWQ8DdGUns9J3UZ/NyfHqWdwqA6k+kDC91YcC5NVuMFAKYrvzmRAIOWt5Db3zXmx9Ea3Milt3", + "FYU87HDSg7vcsOIh8q4hIHizXnWBL19yAdvplAuqZunq26J4rcDrhrjpj1LFVWClPjqZMihL7v9chMn5", + "SpT+uNPtJB/3qmfG/t4eYstC/RYEaLfalwpahJFB1fvVqwCvlIqHMJHsWlfXY/5hu7f9HOIQko97Pwx6", + "z6sRB12zWv7ybbu3K78O2qyhX2vP1Wjafn6jiGu3nqso6BcaqhRX3ssWBNjSeFkE2l0dLuG2ssHl46U9", + "rkHmNAqgnyvp2ctt5AtNMUnwIgQC7xlqZU179IkMjcmUMtnGbrs7KAy3++mw00eHFokbdNmy5H+leSj2", + "7tEJTVMSUy1jGtW/OYNhp6Utrq5L3KwIiPsqIK31w+La8/UQCesSrtZdk/3PyMf9LO23nca7Cr0D7GpO", + "RQWwLnixi+gEYVarBErZHCc0ton0kBgJ8WoHDhGtJFnLA2QpBzo7SRdNuUJlCn1Le1vOmu2CxfjJNdhb", + "V2BmGILY+SKAKAVSF13Fvk6OUSZ4nEdl/mgCgy4RP0Rew0JbIeSvD8m9S/sGJGZPuEDr7RtNBo129smm", + "/a7ZJjXBNm/19mD9Vt+JUaTbybN4PQ8zL7XjYDeCSF+Tghgw0VSXvSYJepP50IKjv/FXcFnnNbbkSItE", + "eeYcLJqmlikp4G4BF0MorveYJERfU8uNIJ7EZZYElSUXXc9St588mzW5OMEjtTyQXwjJtK4C+EfQX4rZ", + "IjgwV9+zuEs2Bg5WWxqHV8/UBbKrVR3c07WSWONW+SbcploFhsvXbN4GL+XSM38XYNq+aLaMgOIYfkVI", + "e9OMtW+/dGFvjfbjuzDLPaSQ9tq6Hmr4qA69t4Ahd/2XscBarKsS717IPR8ii7dWM27CfK1ngfpW58Pe", + "/xgrMxr1D7Z++Mv/3fvwn0Frc01vlkT0YjKBQKNLsuiZKj9aR+9XEU+hxIAWpqeWVAhOwYYEaOL2MPrj", + "3R8UTGPxK06XpgARWl6Jnu21E/rLfzTHN3nL+A745FqS/ewKHHdRqVRxdx1tpERMXSy5SyTb7A8Z5KZd", + "koVEXuEvK9I4Qv1OFp94EejowoiBfcLmF2hMoZKiHDKt1eIoIpnWJmwtGWrKgXPgPoLgxG/HFiBzid/W", + "IWniCQh6f7oEl/v63dsfX7/79Xj0+uzFr4cno19e/DeEeFz1TA9xT9Pe3v4TWwTcX8ntYCGKm9dT6KNT", + "G6ZvXf2THBRawOmSKM1VDkEh5DpKcknnzkGokttXTlhO1r19JYLPhNpVKglFJVhI6IROCPj14TqxQTVU", + "OmKkEqqnW+MGZWj5xjaEM+wAJ/WK34fqVuitCK92ubHVRX8ya8dCDQhp4LBDxiuUuQ9oL1QCXoWL/fBe", + "RhuQOeJKvLrE2c2bgaIeFg0GIw+/cCWfwfMvUW3z3crymnOe9LR601CSIGhNNmsRjJyHpkxGQqfJ6TAd", + "B2R4a9qd0ikO+BlC/oQvUhXTDWhtxtTS/jeWBwvnMRzX6zWYY2mWqlZfoGYkkKrXnOaQaql2VJb+rwbP", + "5MzmrlIvtq6aqJoytWWr14bwMmIOqOGrspXLU+bQEXvw0fok3JV6lTczbyTNe3Pq1IeagrNigc700lzN", + "iCDeRsAHJQ7+DZfM5uW0QGEx1f8yIsqYVZfUo6VScDdLtFFYftwSFNnGy+bw1XUOTvF10QO4UrBc8j/C", + "PMo6S9svfwRM+jeutiSduCZgGDXlLozAXqWiVWviqGp5M3yqWp63eT948CyvWsH9ms5WjTjLPiqkGaLH", + "v2GqfuIC1MFmzJM7B3KHyz8mAjDg6jDtrTDOaUriEc/V6vNvS9fbK7+oP1rWr3WqLwYijirpvE28wKFy", + "lGNYXmm9HCTKBVWLc71eNpgb0iBd0VhYSOgIfi47hkKdnz6B0XgSSBh5SRgRNIIyqPo8ppiBxoTen3rV", + "8ExhxCW8VhCBXh+dWHODg/wF9ZEqID0Xd3l4dtLpduZEGJW7M+jv9gdwmDPCcEY7B53d/nZ/0AGtagZT", + "3ILS9TZ/2uYbF4rrSWwloR/dS/pLgVOi4IvfAkgAEHdoXwcVBE89JTLDVFgtMksAocAQDNVfA66/u1AP", + "zK3cNcve2mYKacaQ/UKy13ZzP4CgDGcHprkzGFhgc2WvX8jdMQkDW3+30aNlv62kOrtEAZj7JTXPyZbF", + "0n/qdvYG2zca06qhwNkNdfyOYZvES0A737/hQtyq0xNm0vJskrUNh/JPHBCSf9Z++6D3TOZpisXCLZi/", + "WhmXTYIxkQi7d40epySKNKuAYjx99JoR8xxhhbCJXBY5gxrG7kNNodVTYNp2m1yAFP3I48UXW8JKH85G", + "8anKzvRx+bREz1+OdgoyXt5I+8ghbBuqvQcC+hEXBbgf7KTsDZ7ffadHnE0SGinUKwjYxiNTCSE/CeCF", + "O+whLtDvOVcYFeH8j+hIW5l1XJBbt7yKtv6g8SdzvBMSMoOfEZFiZpIjzDtrDv3ScTYuifI4r7zVHOFD", + "aQ+4qRwIj7moQJCrHlH/2qoLg8vX0V4AgcH2aaYXPyDh793DCbeTLWqwPuSRg8qXKJfkMR0n62Ibl0JI", + "UJZ7SdTXQvOD+7yybBGBP+EpeiwE/JIUEl65W0uXwlYmcmYU4KAE+KZMWLTffVcV/t6WT7woGfBr6Kah", + "nIUyflUcL/rIralR+tUCIJYEgXnGy9fKmR7e13LCdu7jhMGMC0/Rt2vq2zW16pQbanFTgIPpnfIWNogb", + "WSD+fPaHG1sfvtke2tseWlkeGLmy1oW/83Ef2YjUiMcEyRnPkxiNCTJ4Ry72RGHRn35EWEQzOicAagdF", + "2vJE0QwLiCxJUYwVNj70RsPESrNE0dyWbq7n4hDLBa7jWEgyAhy+URP+ZBmBSBkjMdKfWOi+Ek5wqW63", + "OftBA3vRYHk1oqsZl6TA82PKu80hvVka7Ria7Q/ZWwv0qhcQgqkdr5EkAbjaFfYfzhAeMvvB946FuEAw", + "idOSc2EBmIHUIFOabVlObdMjHcmIh7B23hKGmerJjER0QiM7rUuysPGcwQZb1V3SA3bjfH9aJGygnc0w", + "XhvAM4bBeY+LZ8hSUtV/wyAIOkryuHRyOQghLMY4SYKFOaYJH+NkZNbnkgR8gi/hDbsofn1/501iPCam", + "Vnu2UDPOzN/5OGcqN3+PBb+SRAw7m/0hg0QMu9Yk7pYCIrqCQm5pxvU5Ezw1fW6ZIW79cUkWn/pDdhin", + "lDmKgE9wIjki1/Ad1LcCzAzDvRrowZymsB/8KJeKpz7yqaM7M0yeqyxXNqNEEtUNoX4OmeLoD4ft+Gnr", + "j7LHT+AsJjjWdOK9YqYEsnXTqOUI69mP4NWAu53AAgw7+iI1YR5TgZkysJ0FOCWa+lu6UVRH0Id0s77C", + "EWYo45mpLAFENcOa5CptAFYDThKk4Ci5b7XgDjvZMB8LvZeOG3H3DFBa7RhRhk5/9A7TYO9Z+DxJEgkS", + "iij5r/PXvyK4lfUemNfKcC2T0sG0wIDiHFynjqe9wNEMGUcVFBMcdmg87BTu3HgTxppLGy7T64FP8Qc9", + "tB9MN10a/9Dv66aMu/IA/faHaeVAn6UsNTigw86nLvIeTKma5ePi2YfwgjbBl51XGAHaMNfcJnASTAFp", + "xrvxzRWJWYy4vQWSBcKo5EB+4MqYMiwWqxIJA0tvV5BPTCSjtxh/DCFycdg5GLrYxWGnO+wQNoffbIDj", + "sPMpvALWa9lcuQ7us8K5WRDRk8Fgcz0Stl3fgM+yhWPgC+uAjVpRUXZT76CFYf1z+Qf+rfXPwvWDme68", + "hCYyir8zvj9CB4QnsfuaaMAFURO7MYtI4sTu9Yae+3ce6M2KSJLcN4E+FHkW7rECqf9RkSNsVnmMVprv", + "H5jiBvd1qVTM9g9Dv4/Ofh6wnlvbOZm7UOdwnRLAoLGqNDIvIyzROYypd66V7xfwa9/+1+l+gKl4kfDp", + "xYFR3VHCpyihzOYDeIHKWjywawkfGRia4juLSuOKxG0YSeJf//gnDIqy6b/+8U+L7f6vf/wTjvuWgVeD", + "GtMXM4KFGhOsLg7QL4RkPZzQOXGTgSqwZE7EAu0OrM0fHiGv1L2V0uSQDdkbonLBvLwJU69N2gatq0DP", + "h7KcSAvjo1+kE1tMxsQ2Buw27iybpbzXE90NwCHCDLwJ6FvR0QBgyVFTaNtqop2wydTMuWI0rYdpLgXr", + "recvilwrQ709M8AbMhhY4tC5gwd20mjj/PzFZh+BtmWoAgoGge5QNmPViP43nrSeJxmOUmUosMqGN0U4", + "w2OaUGdybKh2Yo5giqMZZaSMLy6wxl0TB26kmsccnp0gGwjZhVeH7PX5FphYFYlULkjXcgJhEUbLcmjc", + "5rlAD8C/qILosJ59d8gmBEOe0MmxYQIeCHeRD1g0zADIA2JcqapUXusOmUGStcjF+uClPCYJfAT9T7Ei", + "V3jRRUWtW1cdJcFKK8Syq18eMoP1ategB1AlyBtmH/iZGVLPRfLanC1BJolWjSEC35T9hr43JlwgG+Hs", + "Vfl33ZkkSzMsvWgpjl6f6/lNQRPkxh4ILb0+d7ux2UWSoyihQA0RZkM2hUAgB97LWWVXi4SyGRZxL+L6", + "EvDBnC4Zv0pIPG3isUc+kd2hJFPpJ3Ccfq6T62MTLmbLE9CH2ADUrfbcHdt32rnubIt/Jt+dLQR5A+ed", + "seASw2/M6n5z5LVw5IXXzTn1Qp61Y4fAeHcRv6aLBwr4dbS3vObmibdkD2HRQxsO2ga8Ilygs6MThONY", + "ECk3/73tfXqmhkpL+U/fj5oVP0ToiR0LFxb0z9pbqgTyWNjBGztqhN286vV1/fttq1J8p/GmK+rwlFfe", + "3d8etU5vco2UQm9Ja99ukrXBtlRGHMoMltTSA9EoIYX4UpxTn4rWWZVNGG9x5awUlyx7Pjl2B/L+7Mu2", + "65zV74Z7YIrHNYb4gIywmmrtV81+TNT8rthFhza9wvz8dZHm4P6koPs2RYfI/DGpi3Ft2TQXNEAnjRfo", + "S6IMvMld6um2h8DEz4lwp9oMdGFmXUzLfIoMTgtMCCwxq3XfE/NKO9XXtPdn0nxheW4isdgl/yaitFB2", + "y7VapeCe2BLQd6ffQg83Um+/XNiKJbDAIoMVdezcTmBZ3cBywaLNb5ErX5yiTVxjqcQKN28SF5Zsg6ZU", + "6Fn3JdcdMr/euJbprF5LGZokdDqzToCYTiBWT/n1u2GUO/cwyqJOtsCK2BDFx5j3e6YX2XqB50Qo9Pro", + "xKy/f6Vu/QFBq+tVJce8Vt6u79686hEW8bhwnjTLpPbJF1aYDP1Xcnnv/9Q9wnxW6sSDJoHxM/bfBJMj", + "E//ep/x/7fyU0LHAYvG/dn7CSUYZ+V+7hwlWRKrNOyOWwX3ddPetwDxi4tP6C60uGrAmNgXI2DUCf/FW", + "S5nfvf+nEvvNpG8k+Bfr+k32byP7+8u1Uvy3W3GnCoDp44E8XAWxhVYbHn2DtLkHo6mlSA/SpuJFKkFt", + "ZlwqePT48pttUDktKM6/Nlpa/8sDufL6cKR7ctyFhYSK0lDRwqYP3pMvwI3j3oVb2+/9OwIO0zGd5jyX", + "fmZiilU0I9Jm7SakyoAfm9hdXs+NgvdXTKWD+7w67l2u/kb3dyTx1zfUMG/j0Fsn87u32sr89n0t8xtE", + "U5vZbMtudF1Jps2GQGuHadqWjCvQr8sB4KFxhXQR9E4rKqW6gECDOBiy/631j98UwemHH1wKZT4Y7DyB", + "3wmbf/jBZVGyU0cqhClBbQW9w1+PwYs6hUBZKLJXJmzXx2FqdgPpubIC/3YKUulIbq8hOSr8piG10pC8", + "5VqtIdm9uFsVqVqa5N51JEdvoQW3mOJ/Ti3pT+4eqWhwMp9MaEQJgwIvkJgul+IBjSb3zTNyy4RkZv2R", + "XjBRRRJprUYWXGuNhF7WlP6S0TrdRpx3jrBSJM0UmgockUmemMoISM5yFfMr5mDfYYKughAt5xO63l1T", + "I9dIOAktXP23raZbVPy6b1XX1dp+nFlgPLPFa61yWYo2zdrlwxLv3eqULa7a+9cqHzOJGfVteekyrSEE", + "yhiZAlZpblLmii9LBLQ+evv2lUuP0+qJcEWxFHeVsFyR0CHzK2H10YuyxJh5wbWg1QcS23RaSBq0taVi", + "guOEMgLxxESGMtmq9ese9Fh8eQk4XJyvlQR8z8fSllt9OAn4wVjBvciaJ5Uq1rw0SPh1+4rT4uRNODWP", + "il9ZBhRgPCFZbwvnivdswu3WjBsUtjAQ5VmCI8Ch1K8ZiDSLcWAwEf2mALhA8CQhwkDfZbly4taQFYOj", + "zCtIbyWzC938KGeKJhddE84D+CUSYbaw+E9DVunMynyQhww59jBCQTIz4lqlSj1oynMJb0HKsN8lwskV", + "Xsghs5nL5nOo6itIZFAik6SPfuYAGoHwFFPmMV5TLvE7OWQXNE7IyGI+XCAqkZxxoQgjMUr5nMhqvwSL", + "hBIBkzjCeuUkSvECwNcMDqVZH54RA3BWQZbg+t+YxRQK7+meiykfDBlGO4MBSglm0uaJSzyBC8e2gWAQ", + "lQF9jzDaGzy3X9X2DQCC3fJv6NMkBJnzCI+TBSKaigGpQm3CBqa2EKYpKKy3b0KFNPtV2DdthbPKxlLp", + "6jrGXZSzMhMebP05KxLX9XapXDCYp/UCEiqKa9CCf4xJhPV6Ml7tB2AXeRTlInRB6q32KrL+OwqO3vTO", + "YanCeeYJmAwiEsOeM65mcKY5HKXN7xuoqiSqP8dFEzwkXCCMPLouLRokyoE1bgBM4UVZXpC5csEXm9+7", + "s6OPr2UE7vgboMDHcj8BEfHJpHIA119N5gCvyu9YJuE/6zk9cnVlfRYXUzxlXCoaOWZYL0P/TSFsrRCu", + "XtkgNU+4uPRlqyr9/sTFZVsNzIKf0seliPkz/AodEXp4ADT98P4IsIYbZUUTzb0raXX6Kk4pCF1USRfo", + "zFHC2VSfotIqf+9uA1+r2zCgcfoyFcbZXUD8aCVkZH80pWn1ZGzhT3AxRLbVh+ZFuvd7cEb9yhWiaZaQ", + "lEDp2p4hNr3ZJRwUlPmn0gNFuhmv1KfKz102uqA08QddJw4BXbkN2wDpfXm7gkw14dP1oINF5w5hL4A6", + "OGTvpIEDvzCupwtU8GAt0BqIf3Q1o9EMEAhBb9XtG4BCnGUXBfjy5gF6CQfZx6CGzjcMsL+mNckTYoAF", + "52l6cbBcnPX96Sl8ZMAHTRnWiwPkCrIW94fUb/mIgnoWCZYK/WpxEjcKZRx29EJhrW8W89u0WIMlOPaQ", + "hXAHGbmyDdIJuvAgCC8a8LEcv33Fp/KrcRWVJQ3MXBRHVnUE2iQs7jQFedAk7PjZHgxCSNstkRDNMO4Y", + "CHFpMK/4tCinUCFlnGVtydcOE6h4nqYraBhteLBqUsU8V3+RKiZCwMeWupuIG23gyJbSwpeaUC2InjvY", + "m0B+wVAmg28eXCrNVDvdDmF52jn4zf5rnqadbseOx8NFv4FwvwZRst7gcsiN3hkPNvKbWH4TQMgqs/cQ", + "IWs3h1WnmyXyN+aFP7230NnsHpAMQT6oGXG/JhHUG2/V4MN4gWwJI3t+HyMD+EsUJVySioPn8YBnWUNX", + "TWZsNhS5Ne7p4cW5qzbUJoLl3H567r78CnTvdbEibszITffeg0aWR/CYE4Hl0mwmXNQRl9ZFk3z1hPTl", + "tmRpqm0o5Btt3tzK2IowtZ6wzCLsB7GpPodzxVOsaASVj6IZ59Ij+wIe2dQos8bjgjLBtGK0XJtBcKFJ", + "9cKaoS+sGnFgTWYI+49sH3343OYdhL9wj8ovfvKsAgXH7zrRH6oDQGl2QckEZTiXREt1eUpQtIg0VzSl", + "rgiOZijCmcoFgSp+BKWU0TRPfdxrvWNzDBgdF9vpRReNc4USLKaglZmHLtgm4mlKWEzAPjdkM4LnVKuU", + "AiVYERYtepJA9d85QVdcXCYcx2BiyGIMnh6oHiiIpkAAEU+JwjFWGASdC33iRyaJ6aIoCGzUekauS2qI", + "h0zk7HtT0UA3e+EGeoEIQHZTOSsKR0Y4JiwKQlmff91s7Mvbos+Jqk/0gSKDbsVLHzJUyLe5uuF8HVFE", + "jywWmwu7jW3Y/AqhVzarsNXsD0dG/55H2szVzfGBHEzFEq86xV+HZ6kguq/Gu/Tw7iMuUJyb7rxTCWT+", + "Z/UJFQzFD7aCzFKzjbd1DBUV8oplvhHP2/rD/XlyC1veV8IJu42KfVMtpnLSXwPLtat6K577QEZMa0vy", + "bXIPx4JdRNeDiU9ceFzusRhbLcM2R7Pg2z53UgKD9sXZN7ZdZ9s24OG2bNvZZpdc+h4jp6wHMaJhDm7N", + "uI2s2poO/k2zUWqz81jmg7PI0nNxb2zxpGCEhjVmeJFwHP8ZgoRX+I8iLoSBvwBAjccEv+pZDf30ALDN", + "lUXeui5b8/3p6WYTlxBqJY8Q6hFzCC8lR3+WxssG3NdzIgSNLUopOjo9tuG6VCKRsz56nVKFFEeXhGRl", + "RgtkFfb1/BwQyHJB+QriR7dDmBKLjFOm1o6ifPVuBvPpVmXo75lPWjzvb+7w1u5wsOw/PnYGXAZyNswE", + "VmumCqu1dUYpm3CRGrkMj3muW9c8SC+T3k+DVDChCZELqUhqohIneQLHDWpD2Pq/9juzy12IydUnx6TL", + "ZUSkVErKmRwymyuSEaH71p/r9r0Aq6BDQOGCv54ZJvl1BO/pwZh4NayaVg0gm6CuaOegs4WzbCvGCjcE", + "iNnhfcaQfoJoPCQX6ZgnNEIJZZcSbST00qgnaC5Rov/YXBnON4LvvnR149ufLL3SJ2zCg7XjDM0WxPyn", + "yuqybM05Jh8dW3tJ/MPi+A9sdJitra+fLAhOelCP2AH3oFzRhH40rE43QqWikUk5wsXavT8tmGp/yE6J", + "EvodDKltSWIQDUC73MoEj7aG+WCwG2UU0N92CQwOGF7z4xR6PDp7Z9JQScrFojtk+h/Q8NvDM+PdnWBr", + "TfAGagsno5Ot12sCnM9hmf6NIwTNBFeiFwQ3/JtL8OYYI41nSDYcUZ6tUpV49qcPYbUS3De7wuO0KwDI", + "UzGbjQLYy6FxhW0Ic57kqf6H+eNkHa6ZwtHsPbz61Ui7Zjhru3ETfBSH0s4pJqa25YM4PcyCPdaYVb1w", + "bgogxFSiAYO3wKH6M1L3lzff++v4Fbo77Yq6urFfzdm675vPjsEhbPjr8ViOuaE0NxPFV1ufrjBttj79", + "mPDoUlooFt9sqPU2wFfXP5Z42NZFCGICZIYiC2FkgLKI7A5ZzQBpEH8kwkgRkVKGky2Ys2kEkL2dFQvP", + "OYUE7QjyVHqSxoCZlAB8N8Df6dmAoco14Hl0pa2s5b/jOyMVR2MS8ZQ4tPPNkOr2N0zVT1xUocu/Fr74", + "1lt/gATEFOzta9Dam3v8LPT2U3wNodJxbh3KbkQbL3n5ozEFdRHszbCzO5DDThcNOzvpsKN34AiDCRUr", + "tI9SynJFZB8dG/sWpOA+GSBJIs5i6UDXnQVvdyCbEnINWTZkdz6B7+5T7LFUBUv5xnYSYg/6PaS/h6Qd", + "tOEfOHsm4y4cuhjxXBlzvz1X9q2YKDCPbN67r9Y7I990+zac/G/2+FZ4FOyyZpfe1hvOnuVyRppNbq9M", + "IaNcjQHM2xUXlTP0dz6WXcTIlbGGC6n6S3xPf31mOriPQgO6q5sUGbBz/1ZhoEWFgXKtwmCNJsBSX8mO", + "OgxiI7nOuFCA4mhz7Q0NgSYByBE8wgl6fXQyZJFmRQZaUJCUA3eyeOjmFj782zl6cfSmi46h0CX6OR9v", + "9tFrlixcuXHjoxkyI4kZ5hVhhsaGakkcup7N2IF67jJYXHfwQJWjzckIeFbcXrkg8W5nRnAMEskfnVfc", + "dBZAHX7zSh8gAP41Xxbb3lkpfHTeECUWvcOJImK52VObJ8UKzAx7STsIOiu4GeBL3aF0yGtln0Y2MNAY", + "uzudAFLGp29FH+6+QOr9eMlMnIgptzfOAWmUQZIBjhePK5ZJzlDBHEMs0L+ui7IJTVnClpetVDCgy6bI", + "76/I5L6Sd1Ww5f9dTxfM9NE6mrLKPmkiLsqtrPX0uuTgmYFDto6qCGc4omrRRThJ7B1lb4IiIqVXiL9j", + "QfBlzK9Yf8jeFIVebEIvOjp713WOWhRTeWlasL7YPno9J0Lm42JwCA6a8RrDmpN4yBRHEU6iPNHiBplM", + "SAS5uFC/RTb4couhdO7w7JSdBIvNeFHt+aOrcRemCdi9kizqFLdltnpLkCjBNG0GH7eCGgQcQqjBWDfK", + "GaJsktiQqkhwKZFtqkcSOqXjxAYIyT56OyNI4pQMWZZgxohAuTRR8XrovUwQKXOT4K0bAJBeQ1FdVAIL", + "ZoIrG5qQcC6kiSbQFP7+FElFshVk9sa0fApzviPZ1jRue3ogI3VtDM2mEPsK0htiKMUsuKajPHEBjPca", + "im4G9NBS4mM5+G8FnU6J0KcCGyZrwvHMsXbLaQ59JWO5sd7lefFWu3qXRateVqKXsbcSGG5UYm3HnZtF", + "/QU6v6SN2IH20c2yiH/RH7Xsu5qtGh6EffSZswyV7vx3rJJ57iUJtjVglRT+2MxJ3sgrR7WSaLseVqt1", + "Zu1dZrq2xs96MNisx4yWhSvps00K79dHCIP7RXm47yJrj5u2KmhXFd20IeV/PZr+V0GBdwOj/8AoJ7eA", + "0f+q8u4B5/zh8E+CB/Wh8ugrvmdXbPdPj4R/V+nzBg4f4Nia0ucN17PBqysVpff2nXZqkm3xzyTB23jH", + "G8jvbtm/af0tVAZvsda5oDXBkzRTCxfQZn2VZdCZpB9Jv8ERXMSt3p0r+BYhnV+OPBydNgZ0/jlr4z9I", + "zKgtHUglOjkOFJ1/ZBiD/pmrXCxb+tbpYRHN6Jw0G92rJ9guUSZIL+MZOFdis2B2PdxdprDoTz8i27zF", + "XLX/gtqTANVPYhRTQSKVLEwdUM0RTB/fSSS41gTgOReL5igRc0R+Ejw9tLNZcx/aM2WNYWWcYbroxVjh", + "3txxmxUmtM+I7nTxlJrhIcrQyx/RBrlWwlS4QBOt+SA6KZaUXEeExBJoctMf8PagwbJJP5LRdNxmlCtq", + "lby2tWBQlEvFU7f3J8doA2qfTQnTe6FF/QlIspngcxqTuDLGzpwnZlW3Gxb0pnZXLVQUheuccmEG9yAy", + "TJsLafqRZlW2UITEjCnDMLi1VUGqZ8ok8ev+MGUuAMfukRvFtyvMan4bTtnRlAh1OO0iKs4NxPPmt2vu", + "MV9zfjKUu9Mqt50Lz1ltvG6XH9UybekuCj8UuXP3a7Z+//Wk9FD5KLN5rOl8XiikTWbzr4sEB/d3P9y3", + "ufz9I04BfUmc8u2ZyqEB3WKIYF5BTHdM5iThWQr10OHdTreTi6Rz0JkplR1sbUHs94xLdbD3/Olu59OH", + "T/9/AAAA//9gew1xCvABAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/resources/monitoring.go b/lib/resources/monitoring.go index 32f584193..d69b1e2a4 100644 --- a/lib/resources/monitoring.go +++ b/lib/resources/monitoring.go @@ -199,7 +199,7 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { gpuProfileSlots, err := meter.Int64ObservableGauge( "hypeman_resources_gpu_profile_slots", - metric.WithDescription("Available GPU slots by vGPU profile"), + metric.WithDescription("Virtual functions able to create each vGPU profile (best-effort snapshot)"), ) if err != nil { return err diff --git a/openapi.yaml b/openapi.yaml index a0b640ef4..fa0e1381f 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1775,7 +1775,7 @@ components: example: 1024 available: type: integer - description: Number of instances that can be created with this profile + description: "Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer." example: 59 PassthroughDevice: From 1e1c3fb1559314cc9a7c944aea39e0272113c7da Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:12:47 +0000 Subject: [PATCH 014/107] Report conservative per-GPU vGPU profile capacity Counting every free VF advertising a type overreports concurrent capacity: sibling VFs share their parent GPU's framebuffer, so one 48Q assignment revokes the type from every other VF on that GPU. Bound each GPU's contribution by both its free VFs and how many times the profile framebuffer fits into the GPU's remaining framebuffer, using the largest still-creatable profile as a lower bound on what remains. --- lib/devices/vendor_vfio_linux.go | 51 ++- lib/devices/vendor_vfio_linux_test.go | 28 +- lib/oapi/oapi.go | 494 +++++++++++++------------- lib/resources/monitoring.go | 2 +- openapi.yaml | 2 +- 5 files changed, 317 insertions(+), 260 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index ceab85ec5..d67ddbc00 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -81,22 +81,37 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { return vfs, nil } -// listProfiles counts each free VF advertising a type as one creatable -// instance, matching the driver-reported units that mdev sums through -// available_instances. This is a best-effort snapshot because creating on one -// VF may revoke the type from siblings that share its GPU framebuffer. +// listProfiles reports a conservative estimate of how many instances of each +// profile are concurrently creatable. Free VFs on the same GPU share its +// framebuffer, so counting every advertising VF overreports capacity: one 48Q +// assignment can revoke the type from all sibling VFs. Each GPU instead +// contributes min(free VFs advertising the type, remaining framebuffer / +// profile framebuffer), where the largest profile still creatable on the GPU +// is a lower bound on its remaining framebuffer. func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { profilesByType := make(map[string]profileMetadata) - creatableVFs := make(map[string]int) + freeVFsByGPU := make(map[string]map[string]int) + remainingFBByGPU := make(map[string]int) for _, vf := range vfs { creatable, err := s.readCreatableProfiles(vf.PCIAddress) if err != nil { return nil, err } + gpu := vf.ParentGPU + if gpu == "" { + gpu = vf.PCIAddress + } for _, profile := range creatable { profilesByType[profile.TypeName] = profile - if !vf.Allocated { - creatableVFs[profile.TypeName]++ + if vf.Allocated { + continue + } + if freeVFsByGPU[gpu] == nil { + freeVFsByGPU[gpu] = make(map[string]int) + } + freeVFsByGPU[gpu][profile.TypeName]++ + if profile.FramebufferMB > remainingFBByGPU[gpu] { + remainingFBByGPU[gpu] = profile.FramebufferMB } } } @@ -109,15 +124,35 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro profiles := make([]GPUProfile, 0, len(metadata)) for _, profile := range metadata { + available := 0 + for gpu, freeVFs := range freeVFsByGPU { + available += gpuProfileCapacity(freeVFs[profile.TypeName], remainingFBByGPU[gpu], profile.FramebufferMB) + } profiles = append(profiles, GPUProfile{ Name: profile.Name, FramebufferMB: profile.FramebufferMB, - Available: creatableVFs[profile.TypeName], + Available: available, }) } return profiles, nil } +// gpuProfileCapacity estimates how many instances of a profile one GPU can +// still create concurrently. When a profile's framebuffer is unknown (0), the +// free VF count is the only signal available. +func gpuProfileCapacity(freeVFs, remainingFB, profileFB int) int { + if freeVFs == 0 { + return 0 + } + if profileFB <= 0 || remainingFB <= 0 { + return freeVFs + } + if byFB := remainingFB / profileFB; byFB < freeVFs { + return byFB + } + return freeVFs +} + func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) { vendorVFIOMu.Lock() defer vendorVFIOMu.Unlock() diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 4555f5c2a..b68a9f912 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -184,7 +184,7 @@ func TestVendorVFIODestroyRetainsAssignmentWhenOneVFIOPathIsMissing(t *testing.T } } -func TestVendorVFIOListProfilesCountsFreeVFs(t *testing.T) { +func TestVendorVFIOListProfilesReportsPerGPUCapacity(t *testing.T) { t.Parallel() sysfs := newTestVendorVFIOSysfs(t) @@ -196,8 +196,30 @@ func TestVendorVFIOListProfilesCountsFreeVFs(t *testing.T) { require.NoError(t, err) profiles, err := sysfs.listProfiles(vfs) require.NoError(t, err) - assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-48Q"), - "each free VF advertising the type counts as one creatable instance") + assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-48Q"), + "one 48Q consumes a whole GPU, so two GPUs mean two creatable instances despite three free VFs") + assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-2Q"), + "small profiles stay capped by the free VF count") +} + +func TestVendorVFIOListProfilesCapsCapacityByRemainingFramebuffer(t *testing.T) { + t.Parallel() + + // One 48G GPU with a 24Q already assigned: siblings only advertise up to + // 24Q, so at most one more 24Q fits despite two free VFs. + remaining := "ID : vGPU Name\n1147 : NVIDIA L40S-1Q\n1153 : NVIDIA L40S-24Q\n" + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1153", remaining) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", remaining) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.6", "44", "0", remaining) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-24Q")) + assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-1Q"), + "framebuffer allows more 1Q instances than free VFs, so the VF count caps it") } func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) { diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index fa568c18b..e4d26040d 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1006,7 +1006,7 @@ type GPUConfig struct { // GPUProfile Available vGPU profile type GPUProfile struct { - // Available Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer. + // Available Conservative estimate of instances concurrently creatable with this profile across all GPUs, bounded per GPU by free virtual functions and remaining framebuffer. Best-effort: recomputed from driver state on each query. Available int `json:"available"` // FramebufferMb Frame buffer size in MB @@ -19026,252 +19026,252 @@ var swaggerSpec = []string{ "JLVWYtGUYuKRzuH11nlS+sO1hpQW687I1X0sOmT69TTZ9iTD2d2s+KqI4cLW4OXpcxtn30dwuiDIzyXo", "107auYOasEveHzKbaOThFYAjDbAETJj4jFCBuKBTWu24ame9x5jdm1CmI65bU6f/YVMMnizcgCEeYJCW", "JyYE1hQA8YFe7Z50up3zomiUZUzVpXlTFN5aWpEysn4Zc+fs3U0jbzPBJzQEhAORQPapVdRcTOqrvcF5", - "b/uvJmpekx9IbJSZ6KGUx7UaT/b9dhfRy7N3Z01jKqouIX90S3NaAe+zHO0yyZmp++TZVLCtx2gR1wwe", - "j+msj34kUvXIZMKFOrCBLOCQ8mq+4gUSJM41PfjQZJwhSccJnNGiVy3w61/0hAAIZJxPJkRUUQeeh2Rp", - "7+1RGjDu/aSfI/NCAXtw+mNVntZye1ut/axCDaC2T3BE2XSz9XYHDIK1aazDFXx59u6NLRvUhLisl7Io", - "LWTAlvvo16Kwll5qWaIq9QOWwjpyX0Mi2dlsIWmEE9OiqYpBmW/gg9PQWkI/Kz+0ptCAnB6ux+ZOHtqY", - "T7Mczv35m97J6/dbaUzm3cqYIJBvxhOix73psae5QzMos94qXGneZGkxhCHbnlhvrQqW0XqRPAYRWB3F", - "FU5GMuGhmKG3+iGCh2jj/U8mHVmPoIuyylbq332oIZ++nwRPDCDYN3R7Dh3WTbaVAx7UXesQaJ3q9Cqd", - "ho6KSaValrGWiyHyy+pG88v1BfhMI839Hrlsr5pR3QqyyCSFGbxGFwaBzKfGam5NM5JkWGBFkkUtHrMK", - "qE6WPdrkmkQ3SDd7oV//ZIpo5IKM1EwQOeNJNQ5it7tciFVCzPGc2NpTZk6e4V9xlGJxCTexE+RRzswK", - "VEPWd9fhy8yUym4wqZ/fvj0z2r0iYo6TetKDXPLwH5MEL9CYqCtCmJsKlgj7Ma/1pFHZUP9HqFFGBOXV", - "NezsBvo9N3HQaCpwRJD5ypVxtlsiIb6o7VLaXgKIhVFEpGzY3+1V+2s/neRJuz0ODWt7bdXz6CYb/Pbo", - "zNWxKeoEu2XeWV7lMyJ65si5gsGrt3ZHrq6o5LpiBoh2SWAYEyixZDNN/ERQF98EJaT055XkS485SD9d", - "0PYDp8AsVdec8w+tpMv6cQ858lPM4lC9ZZNgYFLzp4DXBVHOIgfRj8Ym+MmgBRg9wM+TEATHlBEpa3nN", - "US6STrfTm9hZHWxtJTzCCUAW7u1uP9taHUa6Mn7YhkuNYrpKv3RBVSbsxmXRGrw3mHSVJLZwlrWwwJl1", - "XHM/AHtajleEGsv6bvMkPBfAPhgspWhf40i5qnBgkqu4XLF/bAFYuDIfaDA1hV60qP3cP5+DoA88w2pW", - "Jf+tJdqHuBiIadQ0YswdtXU0RP4xKFFxESr9xYWyGX9j4oIei/vQhRQ6CN2Kc27wzJ/lk/393f11fAiY", - "Te2Y23MXmKp5u5rTTLzltufXNgDhWVWZwx3p1dVL9bqsoSnNEZdIavWC8oywG63n/t7uzs3Ws+1ETlxY", - "WI0vhSBhjk6PjUwUcaYwZUSglCgcY4WrTAZsWZrLQG0ZTFJIC5p8v5q1NMRP+Bgvty2M9aW87w018t44", - "SOgUMzrRDNm+6fcsZ3hn/8mBqeQZk8ne/pN+v39T5IsXJdRFq63YMkF6HghGX84+bx/uAOCizVz+6Jwd", - "vv1ZM7JcCnNpbckxZQfev4t/lg/gD/PPMWVhYIw2xV/pZKnoazUeLbfIwyQ+QGV9byf3tIkPajBGQ3Qy", - "oPEEYeYqUZp3hydX0Ditlg24QW7wilxZLa68ZsmicW1uXce1rHCuvPqtfrZZi1qu9ONq/7ozd8E7tk8D", - "QF2UuV32rN+qULFcWctxqdRXRlhRvTFJzF8RZwDsGyrlWLki3bMWlcDgGrElv4ou/R+L3r0fj/yBeL+7", - "SmLeT7am44cbhsSsFEj/tiyHrudCThxdc5jDtsfiVmhbP9fi0AVjwR/4LrxN2Fi199fT//r9/8izp3/f", - "/v3V+/f/PX/5X8e/0v9+n5y9/iyMk9UQhA+KI/jFoANNTXgfP7AtKZ1iFQVsdFr9a1hh+8RYHFQ0g4qf", - "aEwOhqyHXlFFhKkfV0t+HHbQBgFNCb7S4i4UxzF5Z5v64zPj0dQf/+HE4E/1NmKblC7shhRYIzIfxzzF", - "lG0O2ZDZtpCbiAS9QP8VowhnpggcZUjrvws0FlCxz7qYys676A+cZZ82h8xW6Tdo2hmGomeTIuuLOQex", - "HZUJg7WvkwJ2wmQkDllxWxcYfMbP2C8B+ylJ6jlDDYuyWn+zmtOzQQitEPJZ9EZCURpQQQrK1mRUJNqg", - "Z4PNZX1ujY5R0NAK8rOOeJPweJiHzMVNSZLHJKYR8BWXJzizmaRFiqahNGvEywS/XsDevDHJazHCuZpp", - "XhTZxPqI80tKurClXXCHQeQHfGn8+TOe9caL3oxnBcgCFibaBRuPeFXJ/j89O9HeeyLoxPYUzJXXJBIQ", - "OuHI2JmZlMPCurA0sbemngfTos+c2NdNYRhpaoGYUG6VC+bKVRAoWgloHgX1kZBM/j2KEgpWJznjeRKj", - "GQDuKd1MCDOvMyhSZvA4ismk/u9qSMPO/hPQYN2/d3daZ6yapVtFZXkS0GlTx/pacGzDJmEARjwYOUP4", - "miAkfQNaPy7YKRSH/54j11B54gpGYrxTJodO2uoHifSy5zaDaU72GFjIkBG2p6nNfbR0CitZUy1aMNEJ", - "8FnSAqzkhcnWfPvqHCkiUpc/vxHp3YFTYpApelTK3BbiOjw6fbHZ7wSBliquKtiqlVlV1UEHsBZstEJT", - "EEZpo8Ep6aKTY8iWtddKqYtBesNPXKDE3IrlZXQAYB1Vcw82JfxOjq0AmizKkAcjtgw7m67FrH69HaA3", - "hQqIi6EUeY8lbbkmy8sEmrUBcCb3Yqn1Wpos+Mes+mfvY8i0gPqGhhcDWGTj/dXe5uigp/RFVbOQ3fhC", - "8qNQGu1f3t5/aVjlLy+j795MRrc+4FE2wzJE3TPfqwkvLe2770ausnvRHDNU6XckafBs/c1VefGuIUX0", - "PVf5PIQjut/b3n67vXdz891NEXGrUFgeTF4BitsezfYuUGEDGK9UjRqDy5F+bEPJnV3k/SmaYcm+U/Cw", - "Zh3Z3n3axigBvbYNy/YDsvnEDKngUg5XqwgnNghjlzRJjAAj6ZThBD1HG+cnL385efVqE/XQ69en9a1Y", - "9UVwf24Bjgu3AKyjyTALQCtVIAFQkTv49u0rOFwJgfQLI4df3h4yd61psQWErhvcy7N34PjHcuQCN5tz", - "FXGZ70uuqVRyGVWtVfzz50D2mk/bFfl3kzRtlLX+V+P+/lwBpg3C5G3eAWCvC15fWs4HwLJ9yCTBrw9H", - "dyXy7efC11o7wx2h1zZeaSHk1xoiwn7T7XZ7HNo7GU4FUCbEtnwJx2Vw3xr4tduhgezVQ6kvHhKjk7Oy", - "0lPpjHDN1+b0fKe//eQZFCvdHrRh7CmOVvR9enjUvvPBjrllDvD4IIoPQGG/rc/KErZRQXByhRdQ7M8s", - "7bBjLkxPu/WOrVUkW8XXLOPr3g5Oty7GNQDmgjjrApfkKF1Za6RFemIdNC3NLSRkSpOEShJxFsuqjDzD", - "EsnMIKGamhuFBD9kMMAuKkofg5SCcBSJvDQ9Wunayvt5Zuke6n5mnGkdAID/fyELiVIKTtCiewh9lKjI", - "gomHbEO4jKkiNQpKfsb6B8g/6NrI9lgPjSqoLaI/GDI5y5VmYpt9dMSZzFMirFUWjSl4jDaRzI1KC+OF", - "1VhohilpTMSQ6dcCWKt/FOrJwZPBYDDodgpNblf/exCipjt1fvYtlrDJ+QW0P2ZRhQFGUOQM5Swmoqj/", - "TQw51EPkbug4/UwQYfd5O/HKfl7KVeGDuQ5zuB2Y8OciuMJQG/RziA69hXK+f3sRvVXOkZNfbbaR/Wp0", - "kwgGgiKeJ7HW+Mb6tjMGORJbM6QkynBn8y6V6J2pvVmdug09Vhz9nhOxQO9PTythD4JMNA9oN3HgEg37", - "wLMbbcPOGhvJ2tHcxL3s4d3eB8ZtXVLxJMQvjmjrexhdErSh0Iphq6I4r7Kvaa0ymEVCmdknTTQrJlgz", - "5cdkPsrzkFaiHzkclHfvTo4rlILxk+1ng2fPe8/G2096e/Fgu4e3d5/0dvbxYLIbPd3d3tldkXHSIm3t", - "9ploQdU0EDBchIePXJh6KHq4KUmgJgTYwOcrymJ+VblngpGofu82ynVd98sx7K2HEMx8SbBUxk7QwDJO", - "4TYlkW7bRH7b1MiiplHYovjk7WD7c80sMLgGZvxW5Mz4L00yf2GrT70B+5tVHefteCsMyGWYrFstv/P2", - "izY42H9+sP+5i+ayJNaNsU5O97i5TaFXDuy4lobhUgE9g42zBHas8GHM5zZro9PtFIkl8DfcurWg5eJx", - "q2yppgPbDbORVfy7IUv5pKIYQEiIAbuLD7RI4AR/KJ1Q5MJrWeMo4XmMPKOXwf4Ch9eJpyToZsD/ZG1h", - "BsvTZD1oZQLAo6FEA2WaEYOjTzdiU5oP0Et4Fx7h1OhPdhCmUIjv48LxwgSm6PPlujbazOohn1tFBr7R", - "Wg3S/4Jp62WwttHVTRgx6AD9yuGbQq1ivG5kNa+DPrP8et0gu2FxsR1EBXRmZboD9FMhxxWSoJX8NiSx", - "f44swyqRYTYr+fl2xzuaWsqd83LNux2zop1uxy0U5KQvZ6e/K6l+6fz5pBiK2CI4gbNcJuPmiiYWDxtm", - "QqWikbRZGnpzm+QLW8OIxCOjpTQFf5oMT6vJFB858eX9KdoAyMO/IGtB1v/aLAJFK3fdzvO950+e7jx/", - "0grYqBzgehn0CPKPlwe3ViCNsnxkjRBNUz86e2eMDJFR34sgk/enPo5EJrhmPXrmrkG/8+f95z6eU8zz", - "ceJ58Cz4m4GPhQ0LQpYVvKgh4PB3mszpZMJ+/xhd7vxd0HT7+oncGW834NSajsL2rRPfi79kDCbjnqlH", - "FIbcAYISshGV6g2RMAN0ThQC+ukhHIEeUaQNW5Jz2FV2xYOEtbe7u/vs6f5OK7qyo/MOzgisXYFL2Y7A", - "O2LwJtp4c36OtjyCM2068AaAEmdWxwyfM2SLCQ+qAml/e7AbopKGi7ukGtv2PG1c8vdWT7OTsosO2c+F", - "Drd0yoOrvbs7eLq3/2y/3TG2dtiRuF7NYVxukFkei3jv7/wGSJNvD88QZN5OcFQ1orhQrBuNSt1oVFCt", - "waCs32Bgz54+2d/b3dluB68Wiu6wwIGVA1vlXYFDFyCKwG4ElmKZ9XabbouQOGUI7A2JEkzTw8jlMtRu", - "H4OmPhLmtXIT2lwM1vS/dHG1+LaVFamwDZlMGCMacIFyVtTw6K/3fX4RF2Yz1zbXw3quHsp/YXr1LA6Q", - "qVV2i6XMBJlTnssv0BBXJjl1knAubvRtk8Lyhsg8UcbPSCV6f/od8BRNa0gqklV1KEuNK9CSbjm5G53n", - "ComEibxpsVrtRputXzXhbsOp7a5Crqhwg0aMslhzrpytj7I8wkmUQ9UaXOynnhWAbUHufZYlCxNEnySc", - "MxTNMANvhPCghdCMJ3E/GHKqn4wmwfAFfoUSbtCVLwnJbEEXMwj9mRZh6JygDb+UmSGlWoHR/dQwGVuy", - "o0qN+2m4UiKWoaywIudcrydW3AP+NZ9UTI4Jn0pQChWkB/TrePMZFibqHzNToGieGl0yENkcGGKNmYdu", - "VHOT8olVcK3IARndZiVxJLiUiCR0CsVw3p/WEoVXJJcV6cLrIyerg21BusZzGLjKDOxU6zpmofsxkDjz", - "OTck0DAk562ISXTGyRSzHEq8eIRsLd791nGHMy7VqACAuuFgpRpB3YZckBKWrkhvL+xB7p3gvehY222W", - "ywb43urrJaoKN9U0wGaeGlzR8Gp1CxoMkfEyBNZK1K0SxquO2XQTVLgS6J9KaJV6+GBoA5JLPLbkYb1t", - "tolGCausup8lbdWW4Xy1Nzhvi5+2Gi7tDKvZCZvwAMjGDVyUzhJtw0IzIlIKlUtQTBglsdMlC1+lNXVB", - "ZnYiCYpzYlfOyKcC2wXH5ngDUAZzNjLKpjVeX++wjXnYjGF1WQfo177YJq5IhjNX34oc1soEBkqEyxzW", - "VtGWVI7C7qzlhgWZ5gkWyCIfthmyXKQJZZdtWpeLdMwTGiH9Qd0BPeFJwq9G+pH8Aeay2Wp2+oNRUw2g", - "czM4m4BnNqTWbzmFH/QsN2vpv2CJ2TLfbwE0S5swrWBI9k80IRZG7x2j1x6hV3HP93YGTWnpDY1WEtKX", - "IRhvyrktyQZPfC4DSXwrpRxXvYjEFozeiD1ZLk0hlRa3kkM7dS7A23l0qhkan4cBcmT4dQ0BBI0JJNi4", - "qS1zjRZssc1UgjUccjlDf+fjqkG0bXxtoDLYBiuxKASZBAPpYUdXGqTNG0tr4u3uTcAegK3qicJHN8RQ", - "WFdDrQxkauInb5bKic2IXTLq5mhKi7UoleECLQqcANtre8CAeuG3QGAwwMFItYASqlC3ZuFVM5RozIUA", - "qGct4XDmZgP4Jlrm0WvtAKbQ2xlZIEFSTNmQUVYYSQG1jCBG5kR46ahcaCVrSuI++pun4gE4dpqphUVd", - "B+P5dxLxK1aMccj8QerGc6nbOWTGsihyqMpfvqSbBa1PEwqkB4MTTAkoVEjVDE0EkTN/7qHilFrGu+Ii", - "bqz6s0DuFSgmAz5WpPglYT4rK5oJqoamoZH5ajlczlSWhadW/0SVYq+oXsx1dX+5JCIsJBZTKl5pFbri", - "HRVPOTFoKwA9AkX87F+GxRdwIy3ARcrm/+qaLH86Kxqv/lZ7zQMQcXi+h8ZsGzTBRiZfphbsU/WkrQ1V", - "gXyzVbAxy74EtOFilV0llKok4FUkaXVPtkt5q0flu9FsSRJVe997tv/0ScuSMJ/lrDMwWV/aNTdPV7jk", - "GnbqtI3f59n+s+fPd/f2n+/cyMPiEjga9qcpicPfH7RBrpU+rMm//vHP96c1r88+BDsPbjQok8IRHlJD", - "Gkd1QO9P//WPf7pR3XpAIUazDMXd4LdvjNJJ/J10gQJVF147J9kK/f6wYiTABZtBG2QyIWAGHZl165WD", - "qeFttJOCcYYjqhYBRo6vTFh58UoNUrqNO6g62JDIa9q28KOac8l8XGZ3brjO0X8a33CNFp61riwl83GT", - "H/p1vVfjhS69Fn6MQ4sQA1kUNV82cBfzucKyEjmt/44gwcGlci2ntZg3VsPb1nMOIIrFFlDzQgFDsOg1", - "edJ+5G9/bTs9v2XFrFNf8Q8rzmHzEbyR1TdwIweMvtH6HNYaf7AX4O2+Go39mm8ri+pVCsSVt+7N+22R", - "prtckKC4wW7en5eZeJMP6+C7QI92DHbJy7a7FZJooCYv6SRgQOMJ6RWBejYjBcnceAT1mbd47oFUyeiS", - "TyZVUNn9ZhBywNeBrCrXC1ZKayZdRK6dzaKOYG3AdIadfTnsaBVg2NlOh52a2yqYp5ji65HtoAqiMliF", - "Cl7mmdcGKd0MxgmPLk05M6iS3UcDlBLMJMoZHP6aV217sNo71O1k3t4UGNzEhDgtsS0Y05jM8JxC6Qfr", - "U5lWAjHJNVUSAkahnQMUcwOrVKnlameoXzNZhAflpOHSwWxhG9YN6vc4cxGt5btg4JtABVn2kQjetagA", - "mmO/fn3aNQEMEHpoBlaJb3QTNSPQDLLoolbHoPw9HD88TsgIxl3HxU+X19FP/gbPqiCSKGmBsktyqBEB", - "injOVB0wP22nyFXzt5avpJxBsJ8N/wAANNu7IRAUkwhOpFw+i1VCvwVx1/IG7EqHEgd2QyQMhwJ8SWFf", - "8RvrEK4PwBgbvDLMph0/rtt4CUdScVu3qzjVI3IdERLXkTXDr7SNlbdfBmPlX2ELxlNUSLZvQ7zz8uz6", - "d5dJBWNtWm0/pp9x1gMYELelFrLDYPBZUJgqoVUwvj3siFEIxzT0QpvUZnK9eq1/JdcKgMjjPDHocmHS", - "tazKXkbrVvzWKYRNB5oLsrYO3h3UhzPx5reqEGdD1R+iSJx9604Kwy3tzjlR7t1zS0aNO1StqFJxabmA", - "f/dKNcbGkFIX2QsebaebNRLcm4WtIhb9tmUyJMMpGWWCTOj1CuIxLxjFuIofUh6kIoPBAHlupPga7T1F", - "0QwLWRs7o9OZShbVAJy9AGjRZ1VPFEQR5gyFbXa+3E334XK0m91Ov/WQcHzuYfAs1Q6xIuloFUD1Uelt", - "s9b5DC/AitPoJHy6uzcY7O4MboVQ7YZ1g+U6Kj+xtQar7TSl1HnfWUd/JUrVb6HIZl4uYHslKCRFF8sk", - "lSA4PYDEmwxHBCVkAmh0ReHw9Z7FeterB28FKov8UtC/2yi7b84HX61NU3Rlwb3dNDrOuVgF+/Gfr3GI", - "NrCZaAm7LpBzt9sbPHm7vXuw/+Rge/suUKWLRWrK9nj6cfvqabKDJ3vJs8XT37dnT6c76W5QD7ukpgRP", - "G1r9Rb/bGGVTXpJV0KAKS0Mbdg4ZEfXKxPWK3pIklJGeLDKk1qcpruAFxv++9vzfzM5vZrBSdjivTtIX", - "IbAqF6dCWQ8DdGUns9J3UZ/NyfHqWdwqA6k+kDC91YcC5NVuMFAKYrvzmRAIOWt5Db3zXmx9Ea3Milt3", - "FYU87HDSg7vcsOIh8q4hIHizXnWBL19yAdvplAuqZunq26J4rcDrhrjpj1LFVWClPjqZMihL7v9chMn5", - "SpT+uNPtJB/3qmfG/t4eYstC/RYEaLfalwpahJFB1fvVqwCvlIqHMJHsWlfXY/5hu7f9HOIQko97Pwx6", - "z6sRB12zWv7ybbu3K78O2qyhX2vP1Wjafn6jiGu3nqso6BcaqhRX3ssWBNjSeFkE2l0dLuG2ssHl46U9", - "rkHmNAqgnyvp2ctt5AtNMUnwIgQC7xlqZU179IkMjcmUMtnGbrs7KAy3++mw00eHFokbdNmy5H+leSj2", - "7tEJTVMSUy1jGtW/OYNhp6Utrq5L3KwIiPsqIK31w+La8/UQCesSrtZdk/3PyMf9LO23nca7Cr0D7GpO", - "RQWwLnixi+gEYVarBErZHCc0ton0kBgJ8WoHDhGtJFnLA2QpBzo7SRdNuUJlCn1Le1vOmu2CxfjJNdhb", - "V2BmGILY+SKAKAVSF13Fvk6OUSZ4nEdl/mgCgy4RP0Rew0JbIeSvD8m9S/sGJGZPuEDr7RtNBo129smm", - "/a7ZJjXBNm/19mD9Vt+JUaTbybN4PQ8zL7XjYDeCSF+Tghgw0VSXvSYJepP50IKjv/FXcFnnNbbkSItE", - "eeYcLJqmlikp4G4BF0MorveYJERfU8uNIJ7EZZYElSUXXc9St588mzW5OMEjtTyQXwjJtK4C+EfQX4rZ", - "IjgwV9+zuEs2Bg5WWxqHV8/UBbKrVR3c07WSWONW+SbcploFhsvXbN4GL+XSM38XYNq+aLaMgOIYfkVI", - "e9OMtW+/dGFvjfbjuzDLPaSQ9tq6Hmr4qA69t4Ahd/2XscBarKsS717IPR8ii7dWM27CfK1ngfpW58Pe", - "/xgrMxr1D7Z++Mv/3fvwn0Frc01vlkT0YjKBQKNLsuiZKj9aR+9XEU+hxIAWpqeWVAhOwYYEaOL2MPrj", - "3R8UTGPxK06XpgARWl6Jnu21E/rLfzTHN3nL+A745FqS/ewKHHdRqVRxdx1tpERMXSy5SyTb7A8Z5KZd", - "koVEXuEvK9I4Qv1OFp94EejowoiBfcLmF2hMoZKiHDKt1eIoIpnWJmwtGWrKgXPgPoLgxG/HFiBzid/W", - "IWniCQh6f7oEl/v63dsfX7/79Xj0+uzFr4cno19e/DeEeFz1TA9xT9Pe3v4TWwTcX8ntYCGKm9dT6KNT", - "G6ZvXf2THBRawOmSKM1VDkEh5DpKcknnzkGokttXTlhO1r19JYLPhNpVKglFJVhI6IROCPj14TqxQTVU", - "OmKkEqqnW+MGZWj5xjaEM+wAJ/WK34fqVuitCK92ubHVRX8ya8dCDQhp4LBDxiuUuQ9oL1QCXoWL/fBe", - "RhuQOeJKvLrE2c2bgaIeFg0GIw+/cCWfwfMvUW3z3crymnOe9LR601CSIGhNNmsRjJyHpkxGQqfJ6TAd", - "B2R4a9qd0ikO+BlC/oQvUhXTDWhtxtTS/jeWBwvnMRzX6zWYY2mWqlZfoGYkkKrXnOaQaql2VJb+rwbP", - "5MzmrlIvtq6aqJoytWWr14bwMmIOqOGrspXLU+bQEXvw0fok3JV6lTczbyTNe3Pq1IeagrNigc700lzN", - "iCDeRsAHJQ7+DZfM5uW0QGEx1f8yIsqYVZfUo6VScDdLtFFYftwSFNnGy+bw1XUOTvF10QO4UrBc8j/C", - "PMo6S9svfwRM+jeutiSduCZgGDXlLozAXqWiVWviqGp5M3yqWp63eT948CyvWsH9ms5WjTjLPiqkGaLH", - "v2GqfuIC1MFmzJM7B3KHyz8mAjDg6jDtrTDOaUriEc/V6vNvS9fbK7+oP1rWr3WqLwYijirpvE28wKFy", - "lGNYXmm9HCTKBVWLc71eNpgb0iBd0VhYSOgIfi47hkKdnz6B0XgSSBh5SRgRNIIyqPo8ppiBxoTen3rV", - "8ExhxCW8VhCBXh+dWHODg/wF9ZEqID0Xd3l4dtLpduZEGJW7M+jv9gdwmDPCcEY7B53d/nZ/0AGtagZT", - "3ILS9TZ/2uYbF4rrSWwloR/dS/pLgVOi4IvfAkgAEHdoXwcVBE89JTLDVFgtMksAocAQDNVfA66/u1AP", - "zK3cNcve2mYKacaQ/UKy13ZzP4CgDGcHprkzGFhgc2WvX8jdMQkDW3+30aNlv62kOrtEAZj7JTXPyZbF", - "0n/qdvYG2zca06qhwNkNdfyOYZvES0A737/hQtyq0xNm0vJskrUNh/JPHBCSf9Z++6D3TOZpisXCLZi/", - "WhmXTYIxkQi7d40epySKNKuAYjx99JoR8xxhhbCJXBY5gxrG7kNNodVTYNp2m1yAFP3I48UXW8JKH85G", - "8anKzvRx+bREz1+OdgoyXt5I+8ghbBuqvQcC+hEXBbgf7KTsDZ7ffadHnE0SGinUKwjYxiNTCSE/CeCF", - "O+whLtDvOVcYFeH8j+hIW5l1XJBbt7yKtv6g8SdzvBMSMoOfEZFiZpIjzDtrDv3ScTYuifI4r7zVHOFD", - "aQ+4qRwIj7moQJCrHlH/2qoLg8vX0V4AgcH2aaYXPyDh793DCbeTLWqwPuSRg8qXKJfkMR0n62Ibl0JI", - "UJZ7SdTXQvOD+7yybBGBP+EpeiwE/JIUEl65W0uXwlYmcmYU4KAE+KZMWLTffVcV/t6WT7woGfBr6Kah", - "nIUyflUcL/rIralR+tUCIJYEgXnGy9fKmR7e13LCdu7jhMGMC0/Rt2vq2zW16pQbanFTgIPpnfIWNogb", - "WSD+fPaHG1sfvtke2tseWlkeGLmy1oW/83Ef2YjUiMcEyRnPkxiNCTJ4Ry72RGHRn35EWEQzOicAagdF", - "2vJE0QwLiCxJUYwVNj70RsPESrNE0dyWbq7n4hDLBa7jWEgyAhy+URP+ZBmBSBkjMdKfWOi+Ek5wqW63", - "OftBA3vRYHk1oqsZl6TA82PKu80hvVka7Ria7Q/ZWwv0qhcQgqkdr5EkAbjaFfYfzhAeMvvB946FuEAw", - "idOSc2EBmIHUIFOabVlObdMjHcmIh7B23hKGmerJjER0QiM7rUuysPGcwQZb1V3SA3bjfH9aJGygnc0w", - "XhvAM4bBeY+LZ8hSUtV/wyAIOkryuHRyOQghLMY4SYKFOaYJH+NkZNbnkgR8gi/hDbsofn1/501iPCam", - "Vnu2UDPOzN/5OGcqN3+PBb+SRAw7m/0hg0QMu9Yk7pYCIrqCQm5pxvU5Ezw1fW6ZIW79cUkWn/pDdhin", - "lDmKgE9wIjki1/Ad1LcCzAzDvRrowZymsB/8KJeKpz7yqaM7M0yeqyxXNqNEEtUNoX4OmeLoD4ft+Gnr", - "j7LHT+AsJjjWdOK9YqYEsnXTqOUI69mP4NWAu53AAgw7+iI1YR5TgZkysJ0FOCWa+lu6UVRH0Id0s77C", - "EWYo45mpLAFENcOa5CptAFYDThKk4Ci5b7XgDjvZMB8LvZeOG3H3DFBa7RhRhk5/9A7TYO9Z+DxJEgkS", - "iij5r/PXvyK4lfUemNfKcC2T0sG0wIDiHFynjqe9wNEMGUcVFBMcdmg87BTu3HgTxppLGy7T64FP8Qc9", - "tB9MN10a/9Dv66aMu/IA/faHaeVAn6UsNTigw86nLvIeTKma5ePi2YfwgjbBl51XGAHaMNfcJnASTAFp", - "xrvxzRWJWYy4vQWSBcKo5EB+4MqYMiwWqxIJA0tvV5BPTCSjtxh/DCFycdg5GLrYxWGnO+wQNoffbIDj", - "sPMpvALWa9lcuQ7us8K5WRDRk8Fgcz0Stl3fgM+yhWPgC+uAjVpRUXZT76CFYf1z+Qf+rfXPwvWDme68", - "hCYyir8zvj9CB4QnsfuaaMAFURO7MYtI4sTu9Yae+3ce6M2KSJLcN4E+FHkW7rECqf9RkSNsVnmMVprv", - "H5jiBvd1qVTM9g9Dv4/Ofh6wnlvbOZm7UOdwnRLAoLGqNDIvIyzROYypd66V7xfwa9/+1+l+gKl4kfDp", - "xYFR3VHCpyihzOYDeIHKWjywawkfGRia4juLSuOKxG0YSeJf//gnDIqy6b/+8U+L7f6vf/wTjvuWgVeD", - "GtMXM4KFGhOsLg7QL4RkPZzQOXGTgSqwZE7EAu0OrM0fHiGv1L2V0uSQDdkbonLBvLwJU69N2gatq0DP", - "h7KcSAvjo1+kE1tMxsQ2Buw27iybpbzXE90NwCHCDLwJ6FvR0QBgyVFTaNtqop2wydTMuWI0rYdpLgXr", - "recvilwrQ709M8AbMhhY4tC5gwd20mjj/PzFZh+BtmWoAgoGge5QNmPViP43nrSeJxmOUmUosMqGN0U4", - "w2OaUGdybKh2Yo5giqMZZaSMLy6wxl0TB26kmsccnp0gGwjZhVeH7PX5FphYFYlULkjXcgJhEUbLcmjc", - "5rlAD8C/qILosJ59d8gmBEOe0MmxYQIeCHeRD1g0zADIA2JcqapUXusOmUGStcjF+uClPCYJfAT9T7Ei", - "V3jRRUWtW1cdJcFKK8Syq18eMoP1ategB1AlyBtmH/iZGVLPRfLanC1BJolWjSEC35T9hr43JlwgG+Hs", - "Vfl33ZkkSzMsvWgpjl6f6/lNQRPkxh4ILb0+d7ux2UWSoyihQA0RZkM2hUAgB97LWWVXi4SyGRZxL+L6", - "EvDBnC4Zv0pIPG3isUc+kd2hJFPpJ3Ccfq6T62MTLmbLE9CH2ADUrfbcHdt32rnubIt/Jt+dLQR5A+ed", - "seASw2/M6n5z5LVw5IXXzTn1Qp61Y4fAeHcRv6aLBwr4dbS3vObmibdkD2HRQxsO2ga8Ilygs6MThONY", - "ECk3/73tfXqmhkpL+U/fj5oVP0ToiR0LFxb0z9pbqgTyWNjBGztqhN286vV1/fttq1J8p/GmK+rwlFfe", - "3d8etU5vco2UQm9Ja99ukrXBtlRGHMoMltTSA9EoIYX4UpxTn4rWWZVNGG9x5awUlyx7Pjl2B/L+7Mu2", - "65zV74Z7YIrHNYb4gIywmmrtV81+TNT8rthFhza9wvz8dZHm4P6koPs2RYfI/DGpi3Ft2TQXNEAnjRfo", - "S6IMvMld6um2h8DEz4lwp9oMdGFmXUzLfIoMTgtMCCwxq3XfE/NKO9XXtPdn0nxheW4isdgl/yaitFB2", - "y7VapeCe2BLQd6ffQg83Um+/XNiKJbDAIoMVdezcTmBZ3cBywaLNb5ErX5yiTVxjqcQKN28SF5Zsg6ZU", - "6Fn3JdcdMr/euJbprF5LGZokdDqzToCYTiBWT/n1u2GUO/cwyqJOtsCK2BDFx5j3e6YX2XqB50Qo9Pro", - "xKy/f6Vu/QFBq+tVJce8Vt6u79686hEW8bhwnjTLpPbJF1aYDP1Xcnnv/9Q9wnxW6sSDJoHxM/bfBJMj", - "E//ep/x/7fyU0LHAYvG/dn7CSUYZ+V+7hwlWRKrNOyOWwX3ddPetwDxi4tP6C60uGrAmNgXI2DUCf/FW", - "S5nfvf+nEvvNpG8k+Bfr+k32byP7+8u1Uvy3W3GnCoDp44E8XAWxhVYbHn2DtLkHo6mlSA/SpuJFKkFt", - "ZlwqePT48pttUDktKM6/Nlpa/8sDufL6cKR7ctyFhYSK0lDRwqYP3pMvwI3j3oVb2+/9OwIO0zGd5jyX", - "fmZiilU0I9Jm7SakyoAfm9hdXs+NgvdXTKWD+7w67l2u/kb3dyTx1zfUMG/j0Fsn87u32sr89n0t8xtE", - "U5vZbMtudF1Jps2GQGuHadqWjCvQr8sB4KFxhXQR9E4rKqW6gECDOBiy/631j98UwemHH1wKZT4Y7DyB", - "3wmbf/jBZVGyU0cqhClBbQW9w1+PwYs6hUBZKLJXJmzXx2FqdgPpubIC/3YKUulIbq8hOSr8piG10pC8", - "5VqtIdm9uFsVqVqa5N51JEdvoQW3mOJ/Ti3pT+4eqWhwMp9MaEQJgwIvkJgul+IBjSb3zTNyy4RkZv2R", - "XjBRRRJprUYWXGuNhF7WlP6S0TrdRpx3jrBSJM0UmgockUmemMoISM5yFfMr5mDfYYKughAt5xO63l1T", - "I9dIOAktXP23raZbVPy6b1XX1dp+nFlgPLPFa61yWYo2zdrlwxLv3eqULa7a+9cqHzOJGfVteekyrSEE", - "yhiZAlZpblLmii9LBLQ+evv2lUuP0+qJcEWxFHeVsFyR0CHzK2H10YuyxJh5wbWg1QcS23RaSBq0taVi", - "guOEMgLxxESGMtmq9ese9Fh8eQk4XJyvlQR8z8fSllt9OAn4wVjBvciaJ5Uq1rw0SPh1+4rT4uRNODWP", - "il9ZBhRgPCFZbwvnivdswu3WjBsUtjAQ5VmCI8Ch1K8ZiDSLcWAwEf2mALhA8CQhwkDfZbly4taQFYOj", - "zCtIbyWzC938KGeKJhddE84D+CUSYbaw+E9DVunMynyQhww59jBCQTIz4lqlSj1oynMJb0HKsN8lwskV", - "Xsghs5nL5nOo6itIZFAik6SPfuYAGoHwFFPmMV5TLvE7OWQXNE7IyGI+XCAqkZxxoQgjMUr5nMhqvwSL", - "hBIBkzjCeuUkSvECwNcMDqVZH54RA3BWQZbg+t+YxRQK7+meiykfDBlGO4MBSglm0uaJSzyBC8e2gWAQ", - "lQF9jzDaGzy3X9X2DQCC3fJv6NMkBJnzCI+TBSKaigGpQm3CBqa2EKYpKKy3b0KFNPtV2DdthbPKxlLp", - "6jrGXZSzMhMebP05KxLX9XapXDCYp/UCEiqKa9CCf4xJhPV6Ml7tB2AXeRTlInRB6q32KrL+OwqO3vTO", - "YanCeeYJmAwiEsOeM65mcKY5HKXN7xuoqiSqP8dFEzwkXCCMPLouLRokyoE1bgBM4UVZXpC5csEXm9+7", - "s6OPr2UE7vgboMDHcj8BEfHJpHIA119N5gCvyu9YJuE/6zk9cnVlfRYXUzxlXCoaOWZYL0P/TSFsrRCu", - "XtkgNU+4uPRlqyr9/sTFZVsNzIKf0seliPkz/AodEXp4ADT98P4IsIYbZUUTzb0raXX6Kk4pCF1USRfo", - "zFHC2VSfotIqf+9uA1+r2zCgcfoyFcbZXUD8aCVkZH80pWn1ZGzhT3AxRLbVh+ZFuvd7cEb9yhWiaZaQ", - "lEDp2p4hNr3ZJRwUlPmn0gNFuhmv1KfKz102uqA08QddJw4BXbkN2wDpfXm7gkw14dP1oINF5w5hL4A6", - "OGTvpIEDvzCupwtU8GAt0BqIf3Q1o9EMEAhBb9XtG4BCnGUXBfjy5gF6CQfZx6CGzjcMsL+mNckTYoAF", - "52l6cbBcnPX96Sl8ZMAHTRnWiwPkCrIW94fUb/mIgnoWCZYK/WpxEjcKZRx29EJhrW8W89u0WIMlOPaQ", - "hXAHGbmyDdIJuvAgCC8a8LEcv33Fp/KrcRWVJQ3MXBRHVnUE2iQs7jQFedAk7PjZHgxCSNstkRDNMO4Y", - "CHFpMK/4tCinUCFlnGVtydcOE6h4nqYraBhteLBqUsU8V3+RKiZCwMeWupuIG23gyJbSwpeaUC2InjvY", - "m0B+wVAmg28eXCrNVDvdDmF52jn4zf5rnqadbseOx8NFv4FwvwZRst7gcsiN3hkPNvKbWH4TQMgqs/cQ", - "IWs3h1WnmyXyN+aFP7230NnsHpAMQT6oGXG/JhHUG2/V4MN4gWwJI3t+HyMD+EsUJVySioPn8YBnWUNX", - "TWZsNhS5Ne7p4cW5qzbUJoLl3H567r78CnTvdbEibszITffeg0aWR/CYE4Hl0mwmXNQRl9ZFk3z1hPTl", - "tmRpqm0o5Btt3tzK2IowtZ6wzCLsB7GpPodzxVOsaASVj6IZ59Ij+wIe2dQos8bjgjLBtGK0XJtBcKFJ", - "9cKaoS+sGnFgTWYI+49sH3343OYdhL9wj8ovfvKsAgXH7zrRH6oDQGl2QckEZTiXREt1eUpQtIg0VzSl", - "rgiOZijCmcoFgSp+BKWU0TRPfdxrvWNzDBgdF9vpRReNc4USLKaglZmHLtgm4mlKWEzAPjdkM4LnVKuU", - "AiVYERYtepJA9d85QVdcXCYcx2BiyGIMnh6oHiiIpkAAEU+JwjFWGASdC33iRyaJ6aIoCGzUekauS2qI", - "h0zk7HtT0UA3e+EGeoEIQHZTOSsKR0Y4JiwKQlmff91s7Mvbos+Jqk/0gSKDbsVLHzJUyLe5uuF8HVFE", - "jywWmwu7jW3Y/AqhVzarsNXsD0dG/55H2szVzfGBHEzFEq86xV+HZ6kguq/Gu/Tw7iMuUJyb7rxTCWT+", - "Z/UJFQzFD7aCzFKzjbd1DBUV8oplvhHP2/rD/XlyC1veV8IJu42KfVMtpnLSXwPLtat6K577QEZMa0vy", - "bXIPx4JdRNeDiU9ceFzusRhbLcM2R7Pg2z53UgKD9sXZN7ZdZ9s24OG2bNvZZpdc+h4jp6wHMaJhDm7N", - "uI2s2poO/k2zUWqz81jmg7PI0nNxb2zxpGCEhjVmeJFwHP8ZgoRX+I8iLoSBvwBAjccEv+pZDf30ALDN", - "lUXeui5b8/3p6WYTlxBqJY8Q6hFzCC8lR3+WxssG3NdzIgSNLUopOjo9tuG6VCKRsz56nVKFFEeXhGRl", - "RgtkFfb1/BwQyHJB+QriR7dDmBKLjFOm1o6ifPVuBvPpVmXo75lPWjzvb+7w1u5wsOw/PnYGXAZyNswE", - "VmumCqu1dUYpm3CRGrkMj3muW9c8SC+T3k+DVDChCZELqUhqohIneQLHDWpD2Pq/9juzy12IydUnx6TL", - "ZUSkVErKmRwymyuSEaH71p/r9r0Aq6BDQOGCv54ZJvl1BO/pwZh4NayaVg0gm6CuaOegs4WzbCvGCjcE", - "iNnhfcaQfoJoPCQX6ZgnNEIJZZcSbST00qgnaC5Rov/YXBnON4LvvnR149ufLL3SJ2zCg7XjDM0WxPyn", - "yuqybM05Jh8dW3tJ/MPi+A9sdJitra+fLAhOelCP2AH3oFzRhH40rE43QqWikUk5wsXavT8tmGp/yE6J", - "EvodDKltSWIQDUC73MoEj7aG+WCwG2UU0N92CQwOGF7z4xR6PDp7Z9JQScrFojtk+h/Q8NvDM+PdnWBr", - "TfAGagsno5Ot12sCnM9hmf6NIwTNBFeiFwQ3/JtL8OYYI41nSDYcUZ6tUpV49qcPYbUS3De7wuO0KwDI", - "UzGbjQLYy6FxhW0Ic57kqf6H+eNkHa6ZwtHsPbz61Ui7Zjhru3ETfBSH0s4pJqa25YM4PcyCPdaYVb1w", - "bgogxFSiAYO3wKH6M1L3lzff++v4Fbo77Yq6urFfzdm675vPjsEhbPjr8ViOuaE0NxPFV1ufrjBttj79", - "mPDoUlooFt9sqPU2wFfXP5Z42NZFCGICZIYiC2FkgLKI7A5ZzQBpEH8kwkgRkVKGky2Ys2kEkL2dFQvP", - "OYUE7QjyVHqSxoCZlAB8N8Df6dmAoco14Hl0pa2s5b/jOyMVR2MS8ZQ4tPPNkOr2N0zVT1xUocu/Fr74", - "1lt/gATEFOzta9Dam3v8LPT2U3wNodJxbh3KbkQbL3n5ozEFdRHszbCzO5DDThcNOzvpsKN34AiDCRUr", - "tI9SynJFZB8dG/sWpOA+GSBJIs5i6UDXnQVvdyCbEnINWTZkdz6B7+5T7LFUBUv5xnYSYg/6PaS/h6Qd", - "tOEfOHsm4y4cuhjxXBlzvz1X9q2YKDCPbN67r9Y7I990+zac/G/2+FZ4FOyyZpfe1hvOnuVyRppNbq9M", - "IaNcjQHM2xUXlTP0dz6WXcTIlbGGC6n6S3xPf31mOriPQgO6q5sUGbBz/1ZhoEWFgXKtwmCNJsBSX8mO", - "OgxiI7nOuFCA4mhz7Q0NgSYByBE8wgl6fXQyZJFmRQZaUJCUA3eyeOjmFj782zl6cfSmi46h0CX6OR9v", - "9tFrlixcuXHjoxkyI4kZ5hVhhsaGakkcup7N2IF67jJYXHfwQJWjzckIeFbcXrkg8W5nRnAMEskfnVfc", - "dBZAHX7zSh8gAP41Xxbb3lkpfHTeECUWvcOJImK52VObJ8UKzAx7STsIOiu4GeBL3aF0yGtln0Y2MNAY", - "uzudAFLGp29FH+6+QOr9eMlMnIgptzfOAWmUQZIBjhePK5ZJzlDBHEMs0L+ui7IJTVnClpetVDCgy6bI", - "76/I5L6Sd1Ww5f9dTxfM9NE6mrLKPmkiLsqtrPX0uuTgmYFDto6qCGc4omrRRThJ7B1lb4IiIqVXiL9j", - "QfBlzK9Yf8jeFIVebEIvOjp713WOWhRTeWlasL7YPno9J0Lm42JwCA6a8RrDmpN4yBRHEU6iPNHiBplM", - "SAS5uFC/RTb4couhdO7w7JSdBIvNeFHt+aOrcRemCdi9kizqFLdltnpLkCjBNG0GH7eCGgQcQqjBWDfK", - "GaJsktiQqkhwKZFtqkcSOqXjxAYIyT56OyNI4pQMWZZgxohAuTRR8XrovUwQKXOT4K0bAJBeQ1FdVAIL", - "ZoIrG5qQcC6kiSbQFP7+FElFshVk9sa0fApzviPZ1jRue3ogI3VtDM2mEPsK0htiKMUsuKajPHEBjPca", - "im4G9NBS4mM5+G8FnU6J0KcCGyZrwvHMsXbLaQ59JWO5sd7lefFWu3qXRateVqKXsbcSGG5UYm3HnZtF", - "/QU6v6SN2IH20c2yiH/RH7Xsu5qtGh6EffSZswyV7vx3rJJ57iUJtjVglRT+2MxJ3sgrR7WSaLseVqt1", - "Zu1dZrq2xs96MNisx4yWhSvps00K79dHCIP7RXm47yJrj5u2KmhXFd20IeV/PZr+V0GBdwOj/8AoJ7eA", - "0f+q8u4B5/zh8E+CB/Wh8ugrvmdXbPdPj4R/V+nzBg4f4Nia0ucN17PBqysVpff2nXZqkm3xzyTB23jH", - "G8jvbtm/af0tVAZvsda5oDXBkzRTCxfQZn2VZdCZpB9Jv8ERXMSt3p0r+BYhnV+OPBydNgZ0/jlr4z9I", - "zKgtHUglOjkOFJ1/ZBiD/pmrXCxb+tbpYRHN6Jw0G92rJ9guUSZIL+MZOFdis2B2PdxdprDoTz8i27zF", - "XLX/gtqTANVPYhRTQSKVLEwdUM0RTB/fSSS41gTgOReL5igRc0R+Ejw9tLNZcx/aM2WNYWWcYbroxVjh", - "3txxmxUmtM+I7nTxlJrhIcrQyx/RBrlWwlS4QBOt+SA6KZaUXEeExBJoctMf8PagwbJJP5LRdNxmlCtq", - "lby2tWBQlEvFU7f3J8doA2qfTQnTe6FF/QlIspngcxqTuDLGzpwnZlW3Gxb0pnZXLVQUheuccmEG9yAy", - "TJsLafqRZlW2UITEjCnDMLi1VUGqZ8ok8ev+MGUuAMfukRvFtyvMan4bTtnRlAh1OO0iKs4NxPPmt2vu", - "MV9zfjKUu9Mqt50Lz1ltvG6XH9UybekuCj8UuXP3a7Z+//Wk9FD5KLN5rOl8XiikTWbzr4sEB/d3P9y3", - "ufz9I04BfUmc8u2ZyqEB3WKIYF5BTHdM5iThWQr10OHdTreTi6Rz0JkplR1sbUHs94xLdbD3/Olu59OH", - "T/9/AAAA//9gew1xCvABAA==", + "b/uvJmpekx9IbJSZ6KGUx7UaT/b9dhfRy7N3Z01jKqouIX90S3NaAe9zxJkkYm7MGkQqmurd5JNigaSH", + "9ZwsTKCKQZo050PfPHZRcCS4lBAj8PLsnewaww2JASNVj2+8QBNBSBFYM8mZKTFlDTquYCXgf4zzyYSI", + "PvqRSNUjkwkX6gCQB9MsL7Rga4gxNMiZyfH6PSe1/IT95yHZ2+tmlAaMgT/p58i8UMAknP5Ylb+1nN9W", + "yz+rUA+o+RMcUTbdbE0eAQNibRrrcAhfnr17Y8sMNSE0670qShEZcOY++rUoxAW7W6Iw9QOWxTrSX0Pi", + "2dlsIWmEDb3YKhqU+QZBOD2tJfqz8kNrOg3I9eH6be6koo35NMuBT5y/6Z28fr+VxmTerYwJAv9mPCF6", + "3JseO5s79IMyS67CxeZNlhlDGLLtCffWqmAxrRfJYyiB1VFc4WQkEx6KMXqrHyJ4iDbe/2TSl/UIuiir", + "bKX+3Ycm8un7SfDEAOJ9Q7fn0GHdxFs54EFdtw6Z1qlOr9Jp6KiY1KtlmWy5eCK/rG40v1xfsM800tzv", + "kcsOqxnhreCLTBKZwXd0YRPIfGqs7NaUI0mGBVYkWdTiN6sA7GTZA06uSXSD9LQX+vVPpuhGLshIzQSR", + "M55U4yZ2u6ErKMrhBjL1OsycPEeB4ijF4hJubif4o5yZFaiy+t11eDQzpbIbTOrnt2/PjDVA6XsyqSdJ", + "yKWIgGOS4AUaE3VFCHNTwRJhP0a2nmQqG+oFCTXKiKC8uoad3UC/5yZuGk0FjggyX7myz3ZLJMQjtV1K", + "20sA4TCKiJQN+7u9an/tp5M8abfHoWFtr62SHt1kg98enbm6N0VdYbfMO8urfEZEzxw5V2B49dbuyNUV", + "mFxXzADXLgkMYwIlmWxmip846uKhoOSU/rySrOkxB+mnF9p+4BSYpeqac/6hlTRaP+4hx3+KWRySNE1C", + "gknlnwK+F0RFixx8XTQ2wVIGXcDoDX5ehSA4poxIWcuDjnKRdLqd3sTO6mBrK+ERTgDicG93+9nW6rDT", + "lfHGNrxqFNNV+qgLwjJhOi7r1uDDwaSrJLGFs6yFxc6s45r7AdjTcnwj1GTWd5sn4bmA98FgKaX7GkfK", + "VZEDE17FRYv9YwtAxJX5QIOpKQyjRe3n/vkcBH3mGVazKvlvLdE+xNFADKSmEWMeqa2jIfKPQYmKi1Cp", + "MC6UzRAcExckWdyHLgTRQe5WnHmDZ/4sn+zv7+6v40PAbGrH3J67wFTN29UcaOIttz2/tgEI56rKHO5I", + "r652qtdlDU1pjrhEUqsXlGeE3Wg99/d2d262nm0ncuLCyGp8KQQhc3R6bGSiiDOFKSMCpUThGCtcZTJg", + "+9JcBmrRYJJCGtHk+9WspSHewseEuW0hrS/lrW+oqffGQUinmNGJZsj2Tb9nOcM7+08OTOXPmEz29p/0", + "+/2bImW8KKExWm3Flgnq80Az+nL2eftwB4AYbebyR+fs8O3PmpHlUphLa0uOKTvw/l38s3wAf5h/jikL", + "A2m0KRZLJ0tFYqvxa7lFKibxASrrgTu5p008UYPxGqKZAb0nCEtXieq8O/y5gsZptczADXKJV+TWanHl", + "NUsWjWtz67qvZUV05dV79bPTWtR+pR9X++OduQvesX0awOqiLO6yJ/5WhY3lytqPS6XBMsKKao9JYv6K", + "OAMg4FDpx8oV6Z61qBwG14gtEVZ06f9Y9O79eOQPxPvdVR7zfrI1ID/cMIRmpUD6t2U5dD0XcuLomsMc", + "tj0Wt0LbersWty4YO/7Ad+Ftwsyqvb+e/tfv/0eePf379u+v3r//7/nL/zr+lf73++Ts9WdhoqyGLHxQ", + "3MEvBjVoasj7eINtSekUqyhgo9PqX8MK2yfG4qCiGVQIRWNyMGQ99IoqIky9uVqy5LCDNghoSvCVFneh", + "mI7JU9vUH58ZD6j++A8nBn+qtxHbJHZhN6TAJpH5OOYppmxzyIbMtoXcRCToBfqvGEU4M0XjKENa/12g", + "sYAKf9YlVXbeRX/gLPu0OWS2qr9B384wFEmbFFlizDmU7ahM2Kx9nRQwFSaDcciK27rA7DN+yX4J8E9J", + "Us8xaliU1fqb1ZyeDULohpD/ojcSitiAClJQtiajIjEHPRtsLutza3SMgoZWkJ913JsEycM8ZC5uSqo8", + "JjGNgK+4vMKZzTwtUjoNpVkjXib49QL25o1JdosRztVM86LIJuJHnF9S0oUt7YIfDSJF4Evj/5/xrDde", + "9GY8K0AZsDDRMdh40KtK9v/p2Yn23hNBJ7anYG69JpGA0AlHxs7MpCgW1oWlib019T+YFn3mxL5uCslI", + "UzvEeApVLpgrb0GgyCWgfxTUR0Iy+fcoSihYneSM50mMZgDQp3QzIYy9zqBIscHjKCaT+r+rIRA7+09A", + "g3X/3t1pneFqlm4VleVJQKdNHetrwbENm4QBGPFg5Azha4KW9A1oK22BnUJx+O85cg2VJ65gJMY7ZXLu", + "pK2WkEgv224zmBZlj4GFGBlhe5ra3EdLp7CSZdWiBRPNAJ8lLcBNXpjszrevzpEiInX59huR3h04JQbJ", + "okelzG3hrsOj0xeb/U4QmKniqoKtWpmFVR10AJvBRjc0BW2UNhqcki46OYbsWnutlLoYpEP8xAVKzK1Y", + "XkYHAO5RNfdgU/Lv5NgKoMmiDJEwYsuws+lazOrX2wF6U6iAuBhKkSdZ0pZrsrxMoFkbMGdyNZZar6XV", + "gn/Mqn/2PobMDKiHaHgxgEs23l/tbY4OqkpfVDUL2Y0vJD9qpdH+5e39l4Zh/vIy+u7NZHTrAx5lMyxD", + "1D3zvZrw0tK++27kKrsXzTFGlX5HkgbP1t9cVRjvGlJE33OVz0O4o/u97e2323s3N9/dFEG3Cp3lweoV", + "ILrt0W/vAkU2gAlL1agxGB3pxzb03NlF3p+iGZbsOwUPa9aR7d2nbYwS0GvbMG4/gJtPzJAKLuVwuIrw", + "Y4NIdkmTxAgwkk4ZTtBztHF+8vKXk1evNlEPvX59Wt+KVV8E9+cWYLpwC8A6moy0ABRTBUIAFbmGb9++", + "gsOVEEjXMHL45e0hdteaFltA7rrBvTx7B45/LEcu0LM5txGX+cHkmkoll1HYWsVLfw7Er/m0NIy1maRp", + "w0Z0rcUJ/rkCZBuE1du8A4BfF+y+tJwPgH37kEmFXx/u7kqk3M+Fu7V2hjtCu2280kJIsTUEhf2m2+32", + "uLV3MpwKAE2IbfkSjsv4vjVQbLdDA9muh1JfPCRGJ2dlZajSGeGar83p+U5/+8kzKG66PWjD2FMcrej7", + "9PCofeeDHXPLHODxQRQfgMJ+W5+VJWyjguDkCi+gOKBZ2mHHXJiedusdW6tItoqvWcbjvR38bl2MawDY", + "BXHWBS7JUbqyNkmLdMY6yFqaWwjJlCYJlSTiLJZVGXmGJZKZQU418duFBD9kMMAuKkolg5SCcBSJvDQ9", + "Wunayvt5Zuke6oRmnGkdAAoF/EIWEqUUnKBF9xD6KFGRNRMP2YZwGVZFKhWUCI31D5Cv0LW1x2M9NKqg", + "Fon+YMjkLFeaiW320RFnMk+JsFZZNKbgMdpEMjcqLYwXVmOhGaakMRFDpl8LYLP+UagnB08Gg8Gg2yk0", + "uV3970GImu7U+dm32MMmRxjQAZlFIQbYQZEzlLOYiKJeODHkUA+Ru6Hj9DNBh93n7cQr+3kpV4UP5jqM", + "4nbgw5+L+ApDbdDPITr0Fsr5/u1F9FY5Sk5+tdlJ9qvRTSIYCIp4nsRa4xvr284Y5EhszZCSKMOdzbtU", + "onemVmd16jb0WHGTP4Len55Wwh4EmWge0G7iwCUa9oFnN9qGnTU2krWjuYl72cPHvQ9M3Lqk4kmIXxwB", + "1/cwuqRpQ6EVw1ZFcV5lX9NaZTCLhDKzT5poVkywZsqPyXyU5yGtRD9yuCnv3p0cVygF4yfbzwbPnvee", + "jbef9PbiwXYPb+8+6e3s48FkN3q6u72zuyLjpEWa2+0z14KqaSBguAgPH7kw9YY8tWCSQE0IsIHPV5TF", + "/KpyzwQjUf3ebZTruu6XY9hbDyGY+ZJgqYydoIFlnMJtSiLdton8tqmURQ2ksEXxydvB9ueaWWBwDcz4", + "rciZ8V+a5P/CVp96A/Y3qzrO2/FWGJDLMFm3Wn7n7RdtcLD//GD/cxfNZUmsG2OdnO5xc5tCrxw4ci0N", + "w6UCegYbZwnsWOHDmM9t1kan2ykSS+BvuHVrQcvF41bZUk0HthtmI6v4d0NW80lFMYCQEAOOFx9okcAJ", + "/lBqocid17LGUcLzGHlGL4MVBg6vE09J0M2A/8nawgz2p8l60MoEgE1DSQfKNCMGR59uxKZAH6CX8C48", + "wqnRn+wgTGER38eF44UJTNHny3VttJnVQz63igx8o7UapP8F09bLYG2jq5swYtAB+pXDN4VaxXjdyGpe", + "B31m+fW6QXbD4mg7SAvozMp0B+inQo4rJEEr+W1IYv8cWYZVIslsVvL57Y53NLWUO+flpnc7ZkU73Y5b", + "KMhhX85mf1dS/dL580kxFLFFcAJnuUzGzRVNLH42zIRKRSNpszT05jbJF7bmEYlHRktpCv40GZ5Wkyk+", + "cuLL+1O0ARCJf0HWgqz/tVkEilbuup3ne8+fPN15/qQVEFI5wPUy6BHkHy8Pbq1AGmX5yBohmqZ+dPbO", + "GBkio74XQSbvT33ciUxwzXr0zF2DfufP+899/KeY5+PE8+BZsDgDNwsbFoQ4K3hRQ8Dh7zSZ08mE/f4x", + "utz5u6Dp9vUTuTPebsC1NR2F7Vsnvhd/yRhMxj1TvygM0QMEJWQjitUbImEG6JwoBPTTQzgCPaJIG7Yk", + "57Cu7IoHCWtvd3f32dP9nVZ0ZUfnHZwRWLsCl7IdgXfE4E208eb8HG15BGfadGAPAD3OrI4ZPmfIFh8e", + "VAXS/vZgN0QlDRd3STW27XnauOTvrZ5mJ2UXHbKfCx1u6ZQHV3t3d/B0b//ZfrtjbO2wI3G9msO43CCz", + "PBYh39/5DZAm3x6eIci8neCoakRxoVg3GpW60aiguoNBZb/BwJ49fbK/t7uz3Q6OLRTdYYEGKwe2yrsC", + "hy5AFIHdCCzFMuvtNt0WIXHKENgbEiWYpoeRy2Wo3T4GfX0kzGvlJrS5GKzpf+niavFtKytSYRsymTBG", + "NOAC5ayo+dFf7/v8Ii7MZq5trof1XD2U/8L06lncIFPb7BZLmQkypzyXX6Ahrkxy6iThXNzo2yaF5Q2R", + "eaKMn5FK9P70O+ApmtaQVCSr6lCWGlegK91ycjc6zxUSCRN502K12o02W79qwt2GU9tdhVxR4QaNmGax", + "5lw5Wx9leYSTKIcqN7jYTz0rAB+C3PssSxYmiD5JOGcommEG3ghha2qxKcJoxpO4Hww51U9Gk2D4Ar9C", + "CTdozJeEZLYAjBmE/kyLMHRO0IZf+syQUq0g6X5qmIwt8VGlxv00XFkRy1BWWJFzrtcTK+4BBZtPKibH", + "hE8NtpKC9IB+HZ8+w8JE/WNmChrNU6NLBiKbA0OsMfPQjWpuUj6xCq4VOSCj26ykBY0iCZ1C8Zz3p7VE", + "4RXJZUW68PrIyepgW5Cu8RwGrjKDWNW67lnofgwkznzODQk0DMl5K2ISnXEyxSyHkjAeIVuLd7913OGM", + "SzUqAKBuOFipRlDnIRekhLEr0tsLe5B7J3gvOtZ2m+WyAb63+nqJqsJNNQ2wmacGVzS8Wt2CBkNkvAyB", + "tRJ1q4TxqmM23QRFriwMQCW0Sj18MLQBySUeW/LA7TfbRKOEVVbdz5K2ast2vtobnLfFT1sNl3aG1eyE", + "TXgAZOMGLkpnibZhoRkRKYVKJygmjJLY6ZKFr9KauiAzO5EExTmxK2fkU4HtgmNzvAEogzkbGWXTGq+v", + "d9jGPGzGsLoMBPRrX2wTVyTDmatvRQ5rZQIDJcJlDmuraEsqR2F31nLDgkzzBAtkkRLbDFku0oSyyzat", + "y0U65gmNkP6g7oCe8CThVyP9SP4Ac9lsNTv9waipZtC5GZxNwDMbUuu3nMIPepabtfRfsMRsme+3AJql", + "TZhWMCT7J5oQC6P3jtFrj9CrOOl7O4OmtPSGRisJ6csQjDfl3JZkgyc+l4EkvpVSjqt2RGILXm/EniyX", + "pvBKi1vJoaM6F+DtPDrVDI3PwwA5Mvy6hgCCxgQSbNzUlrlGC7bYZirBmg+5nKG/83HVINo2vjZQSWyD", + "lVgUgkyCgfSwoysN0uaNpTXxdvcmYA/AVvVE4aMbYiisq7lWBjI18ZM3S+XHZsQuGXVzNKXIWpTWcIEW", + "BU6A7bU9YEC9UFwgMBjgYKRaQMlVqHOz8KofSjTmQgA0tJZwOHOzAXwTLfPotXYAU+jtjCwsFO2QUVYY", + "SQG1jCBG5kR46ahcaCVrSuI++pun4gGYdpqphUVpB+P5dxLxK1aMccj8QerGc6nbOWTGsihyqOJfvqSb", + "Ba1PEwqkB4MTTAkobEjVDE0EkTN/7qFillrGu+IibqwStEDuFSg+Az5WpPglYT4rK5oJqoamoZH5ajlc", + "zlSihadW/0SV4rCoXvx1dX+5JCIsJBZTKl5pFbriHRVPOTFoKwA9AkX/7F+GxRdwIy3ARcrm/+qaLH86", + "Kxqv/lZ7zQMQcXi+h8ZsGzTBRiZfphbsU/WkrQ1VgXyzVbAxy74EtOFilV3llKok4FUwaXVPtkt5q0fl", + "u9FsSRJVe997tv/0ScsSMp/lrDMwWV/aNTdPV7jkGnbqtI3f59n+s+fPd/f2n+/cyMPiEjga9qcpicPf", + "H7RBrpU+rMm//vHP96c1r88+BDsPbjQok8IRHlJDGkd1QO9P//WPf7pR3XpAIUazDMXd4LdvjNJJ/J10", + "gQJVF147J9kK/f6wYiTABZtBG2QyIWAGHZl165WDqeFttJOCcYYjqhYBRo6vTFh58UoNUrqNO6g62JDI", + "a9q28KOac8l8XGZ3brjO0X8a33CNFp61rkQl83GTH/p1vVfjhS69Fn6MQ4sQA1kUQV82cBfzucKyEjmt", + "/44gwcGlci2ntZg3VsPb1nMOIIrFFlzzQgFDsOg1edJ+5G9/bTs9v2XFrFNf8Q8rzmHzEbyR1TdwIweM", + "vtH6HNYaf7AX4O2+Go39GnEri/BVCsqVt+7N+22RprtckKC4wW7en5eZeJMP6+C7QI92DHbJy7a7FZJo", + "oCYv6SRgQOMJ6RWBejYjBcnceAT1mbd47oFUyeiSTyZVUNn9ZhBywNeBrCrXC1ZKayZdRK6dzaKOYG3A", + "dIadfTnsaBVg2NlOh52a2yqYp5ji65HtoAqiMliFCl7mmdcGKd0MxgmPLk35M6iq3UcDlBLMJMoZHP6a", + "V217sNo71O1k3t4UGNzEhDgtsS0Y05jM8JxC6QfrU5lWAjHJNVUSAkahnQMUcwOrVKn9ameoXzNZhAfl", + "pOHSwWxhG9YN6vc4cxGt5btg4JtAxVn2kQjetagAmmO/fn3aNQEMEHpoBlaJb3QTNSPQDLLoolbHoPw9", + "HD88TsgIxl3HxU+X19FP/gbPqiCSKGmBsktyqBEBinjOVB0wP22nyFXzt5avpJxBsJ8N/wAANNu7IRAU", + "kwhOpFw+i1VCvwVx1/IG7EqHEgd2QyQMhwJ8SWFf8RvrEK4PwBgbvLLNph0/rtt4CUdScVvnqzjVI3Id", + "ERLXkTXDr7SNlbdfBmPlX2ELxlNUVLZvQ7zz8uz6d5dJBWNtWm0/pp9x1gMYELelFrLDYPBZUJgqoVUw", + "vj3siFEIxzT0QpvUZnK9eq1/JdcKgMjjPDHocmHStazKXkbrVvzWKYRNB5oLsrZu3h3UkzPx5reqKGdD", + "1R+iqJx9604KyS3tzjlR7t1zS0aNO1StqFJxabmAf/dKNcbGkFIX2QsebaebNRLcm4WtIhb9tmUyJMMp", + "GWWCTOj1CuIxLxjFuIofUh6kIoPBAHlupPga7T1F0QwLWRs7o9OZShbVAJy9AGjRZ1VbFEQR5gyFbXa+", + "3E334XK0m91Ov/WQcHzuYfAs1Q6xIuloFUD1Uelts9b5DC/AitPoJHy6uzcY7O4MboVQ7YZ1g+U6Kj+x", + "tQmr7TSl1HnfWUd/JUrVb6HIZl4ueHslKCRFF8sklSA4PYDEmwxHBCVkAmh0RaHx9Z7FeterB28FKov8", + "UtC/2yi7b84HX61NU3Rlwb3dNDrOuVgF+/Gfr3GINrCZaAm7LpBzt9sbPHm7vXuw/+Rge/suUKWLRWrK", + "9nj6cfvqabKDJ3vJs8XT37dnT6c76W5QD7ukpgRPG1r9Rb/bGGVTXpJV0KAKS0Mbdg4ZEfVKxvUK4JIk", + "lJGeLDKk1qcpruAFxv++9vzfzM5vZrBSdjivTtIXIbAqF6dCWQ8DdGUns9J3UZ/NyfHqWdwqA6k+kDC9", + "1YcC5NVuMFAKYrvzmRAIOWt5Db3zXmx9Ea3Milt3FYU87HDSg7vcsOIh8q4hIHizXnWBL19yAdvplAuq", + "Zunq26J4rcDrhrjpj1LFVWClPjqZMihj7v9chMn5SpT+uNPtJB/3qmfG/t4eYstC/RYEaLfalwpahJFB", + "lfzVqwCvlIqHMJHsWlfXY/5hu7f9HOIQko97Pwx6z6sRB12zWv7ybbu3K78O2qyhX2vP1Wjafn6jiGu3", + "nqso6BcaqhRX3ssWBNjSeFk02l0dLuG2ssHl46U9rkHmNAqgnyvp2ctt5AtNMUnwIgQC7xlqZU179IkM", + "jcmUMtnGbrs7KAy3++mw00eHFokbdFnFi3785qE4vEcnNE1JTLWMaVT/5gyGnZa2uLoucbMiIO6rgLTW", + "D4trz9dDJKxLuFp3TfY/Ix/3s7TfdhrvKvQOsKs5FRXAuuDFLqIThFmtEihlc5zQ2CbSQ2IkxKsdOES0", + "kmQtD5ClHOjsJF005QqVKfQt7W05a7YLFuMn12BvXYGZYQhi54sAohRIXXQV+zo5RpngcR6V+aMJDLpE", + "/BB5DQtthZC/PiT3Lu0bkJg94QKtt280GTTa2Seb9rtmm9QE27zV24P1W30nRpFuJ8/i9TzMvNSOg90I", + "In1NCmLARFNd9pok6E3mQwuO/sZfwWWd19iSIy0S5ZlzsGiaWqakgLsFXAyhuN5jkhB9TS03gngSl1kS", + "VJZcdD1L3X7ybNbk4gSP1PJAfiEk07oK4B9Bfylmi+DAXH3P4i7ZGDhYbWkcXj1TF8iuVnVwT9dKYo1b", + "5Ztwm2oVGC5fs3kbvJRLz/xdgGn7otkyAopj+BUh7U0z1r790oW9NdqP78Is95BC2mvreqjhozr03gKG", + "3PVfxgJrsa5KvHsh93yILN5azbgJ87WeBepbnQ97/2OszGjUP9j64S//d+/DfwatzTW9WRLRi8kEAo0u", + "yaJnqvxoHb1fRTyFEgNamJ5aUiE4BRsSoInbw+iPd39QMI3FrzhdmgJEaHklerbXTugv/9Ec3+Qt4zvg", + "k2tJ9rMrcNxFpVLF3XW0kRIxdbHkLpFssz9kkJt2SRYSeYW/rEjjCPU7WXziRaCjCyMG9gmbX6AxhUqK", + "csi0VoujiGRam7C1ZKgpB86B+wiCE78dW4DMJX5bh6SJJyDo/ekSXO7rd29/fP3u1+PR67MXvx6ejH55", + "8d8Q4nHVMz3EPU17e/tPbBFwfyW3g4Uobl5PoY9ObZi+dfVPclBoAadLojRXOQSFkOsoySWdOwehSm5f", + "OWE5Wff2lQg+E2pXqSQUlWAhoRM6IeDXh+vEBtVQ6YiRSqiebo0blKHlG9sQzrADnNQrfh+qW6G3Irza", + "5cZWF/3JrB0LNSCkgcMOGa9Q5j6gvVAJeBUu9sN7GW1A5ogr8eoSZzdvBop6WDQYjDz8wpV8Bs+/RLXN", + "dyvLa8550tPqTUNJgqA12axFMHIemjIZCZ0mp8N0HJDhrWl3Sqc44GcI+RO+SFVMN6C1GVNL+99YHiyc", + "x3Bcr9dgjqVZqlp9gZqRQKpec5pDqqXaUVn6vxo8kzObu0q92LpqomrK1JatXhvCy4g5oIavylYuT5lD", + "R+zBR+uTcFfqVd7MvJE0782pUx9qCs6KBTrTS3M1I4J4GwEflDj4N1wym5fTAoXFVP/LiChjVl1Sj5ZK", + "wd0s0UZh+XFLUGQbL5vDV9c5OMXXRQ/gSsFyyf8I8yjrLG2//BEw6d+42pJ04pqAYdSUuzACe5WKVq2J", + "o6rlzfCpanne5v3gwbO8agX3azpbNeIs+6iQZoge/4ap+okLUAebMU/uHMgdLv+YCMCAq8O0t8I4pymJ", + "RzxXq8+/LV1vr/yi/mhZv9apvhiIOKqk8zbxAofKUY5heaX1cpAoF1QtzvV62WBuSIN0RWNhIaEj+Lns", + "GAp1fvoERuNJIGHkJWFE0AjKoOrzmGIGGhN6f+pVwzOFEZfwWkEEen10Ys0NDvIX1EeqgPRc3OXh2Umn", + "25kTYVTuzqC/2x/AYc4IwxntHHR2+9v9QQe0qhlMcQtK19v8aZtvXCiuJ7GVhH50L+kvBU6Jgi9+CyAB", + "QNyhfR1UEDz1lMgMU2G1yCwBhAJDMFR/Dbj+7kI9MLdy1yx7a5sppBlD9gvJXtvN/QCCMpwdmObOYGCB", + "zZW9fiF3xyQMbP3dRo+W/baS6uwSBWDul9Q8J1sWS/+p29kbbN9oTKuGAmc31PE7hm0SLwHtfP+GC3Gr", + "Tk+YScuzSdY2HMo/cUBI/ln77YPeM5mnKRYLt2D+amVcNgnGRCLs3jV6nJIo0qwCivH00WtGzHOEFcIm", + "clnkDGoYuw81hVZPgWnbbXIBUvQjjxdfbAkrfTgbxacqO9PH5dMSPX852inIeHkj7SOHsG2o9h4I6Edc", + "FOB+sJOyN3h+950ecTZJaKRQryBgG49MJYT8JIAX7rCHuEC/51xhVITzP6IjbWXWcUFu3fIq2vqDxp/M", + "8U5IyAx+RkSKmUmOMO+sOfRLx9m4JMrjvPJWc4QPpT3gpnIgPOaiAkGuekT9a6suDC5fR3sBBAbbp5le", + "/ICEv3cPJ9xOtqjB+pBHDipfolySx3ScrIttXAohQVnuJVFfC80P7vPKskUE/oSn6LEQ8EtSSHjlbi1d", + "CluZyJlRgIMS4JsyYdF+911V+HtbPvGiZMCvoZuGchbK+FVxvOgjt6ZG6VcLgFgSBOYZL18rZ3p4X8sJ", + "27mPEwYzLjxF366pb9fUqlNuqMVNAQ6md8pb2CBuZIH489kfbmx9+GZ7aG97aGV5YOTKWhf+zsd9ZCNS", + "Ix4TJGc8T2I0JsjgHbnYE4VFf/oRYRHN6JwAqB0UacsTRTMsILIkRTFW2PjQGw0TK80SRXNburmei0Ms", + "F7iOYyHJCHD4Rk34k2UEImWMxEh/YqH7SjjBpbrd5uwHDexFg+XViK5mXJICz48p7zaH9GZptGNotj9k", + "by3Qq15ACKZ2vEaSBOBqV9h/OEN4yOwH3zsW4gLBJE5LzoUFYAZSg0xptmU5tU2PdCQjHsLaeUsYZqon", + "MxLRCY3stC7JwsZzBhtsVXdJD9iN8/1pkbCBdjbDeG0AzxgG5z0uniFLSVX/DYMg6CjJ49LJ5SCEsBjj", + "JAkW5pgmfIyTkVmfSxLwCb6EN+yi+PX9nTeJ8ZiYWu3ZQs04M3/n45yp3Pw9FvxKEjHsbPaHDBIx7FqT", + "uFsKiOgKCrmlGdfnTPDU9Lllhrj1xyVZfOoP2WGcUuYoAj7BieSIXMN3UN8KMDMM92qgB3Oawn7wo1wq", + "nvrIp47uzDB5rrJc2YwSSVQ3hPo5ZIqjPxy246etP8oeP4GzmOBY04n3ipkSyNZNo5YjrGc/glcD7nYC", + "CzDs6IvUhHlMBWbKwHYW4JRo6m/pRlEdQR/SzfoKR5ihjGemsgQQ1Qxrkqu0AVgNOEmQgqPkvtWCO+xk", + "w3ws9F46bsTdM0BptWNEGTr90TtMg71n4fMkSSRIKKLkv85f/4rgVtZ7YF4rw7VMSgfTAgOKc3CdOp72", + "AkczZBxVUExw2KHxsFO4c+NNGGsubbhMrwc+xR/00H4w3XRp/EO/r5sy7soD9NsfppUDfZay1OCADjuf", + "ush7MKVqlo+LZx/CC9oEX3ZeYQRow1xzm8BJMAWkGe/GN1ckZjHi9hZIFgijkgP5gStjyrBYrEokDCy9", + "XUE+MZGM3mL8MYTIxWHnYOhiF4ed7rBD2Bx+swGOw86n8ApYr2Vz5Tq4zwrnZkFETwaDzfVI2HZ9Az7L", + "Fo6BL6wDNmpFRdlNvYMWhvXP5R/4t9Y/C9cPZrrzEprIKP7O+P4IHRCexO5rogEXRE3sxiwiiRO71xt6", + "7t95oDcrIkly3wT6UORZuMcKpP5HRY6wWeUxWmm+f2CKG9zXpVIx2z8M/T46+3nAem5t52TuQp3DdUoA", + "g8aq0si8jLBE5zCm3rlWvl/Ar337X6f7AabiRcKnFwdGdUcJn6KEMpsP4AUqa/HAriV8ZGBoiu8sKo0r", + "ErdhJIl//eOfMCjKpv/6xz8ttvu//vFPOO5bBl4NakxfzAgWakywujhAvxCS9XBC58RNBqrAkjkRC7Q7", + "sDZ/eIS8UvdWSpNDNmRviMoF8/ImTL02aRu0rgI9H8pyIi2Mj36RTmwxGRPbGLDbuLNslvJeT3Q3AIcI", + "M/AmoG9FRwOAJUdNoW2riXbCJlMz54rRtB6muRSst56/KHKtDPX2zABvyGBgiUPnDh7YSaON8/MXm30E", + "2pahCigYBLpD2YxVI/rfeNJ6nmQ4SpWhwCob3hThDI9pQp3JsaHaiTmCKY5mlJEyvrjAGndNHLiRah5z", + "eHaCbCBkF14dstfnW2BiVSRSuSBdywmERRgty6Fxm+cCPQD/ogqiw3r23SGbEAx5QifHhgl4INxFPmDR", + "MAMgD4hxpapSea07ZAZJ1iIX64OX8pgk8BH0P8WKXOFFFxW1bl11lAQrrRDLrn55yAzWq12DHkCVIG+Y", + "feBnZkg9F8lrc7YEmSRaNYYIfFP2G/remHCBbISzV+XfdWeSLM2w9KKlOHp9ruc3BU2QG3sgtPT63O3G", + "ZhdJjqKEAjVEmA3ZFAKBHHgvZ5VdLRLKZljEvYjrS8AHc7pk/Coh8bSJxx75RHaHkkyln8Bx+rlOro9N", + "uJgtT0AfYgNQt9pzd2zfaee6sy3+mXx3thDkDZx3xoJLDL8xq/vNkdfCkRdeN+fUC3nWjh0C491F/Jou", + "Hijg19He8pqbJ96SPYRFD204aBvwinCBzo5OEI5jQaTc/Pe29+mZGiot5T99P2pW/BChJ3YsXFjQP2tv", + "qRLIY2EHb+yoEXbzqtfX9e+3rUrxncabrqjDU155d3971Dq9yTVSCr0lrX27SdYG21IZcSgzWFJLD0Sj", + "hBTiS3FOfSpaZ1U2YbzFlbNSXLLs+eTYHcj7sy/brnNWvxvugSke1xjiAzLCaqq1XzX7MVHzu2IXHdr0", + "CvPz10Wag/uTgu7bFB0i88ekLsa1ZdNc0ACdNF6gL4ky8CZ3qafbHgITPyfCnWoz0IWZdTEt8ykyOC0w", + "IbDErNZ9T8wr7VRf096fSfOF5bmJxGKX/JuI0kLZLddqlYJ7YktA351+Cz3cSL39cmErlsACiwxW1LFz", + "O4FldQPLBYs2v0WufHGKNnGNpRIr3LxJXFiyDZpSoWfdl1x3yPx641qms3otZWiS0OnMOgFiOoFYPeXX", + "74ZR7tzDKIs62QIrYkMUH2Pe75leZOsFnhOh0OujE7P+/pW69QcEra5XlRzzWnm7vnvzqkdYxOPCedIs", + "k9onX1hhMvRfyeW9/1P3CPNZqRMPmgTGz9h/E0yOTPx7n/L/tfNTQscCi8X/2vkJJxll5H/tHiZYEak2", + "74xYBvd10923AvOIiU/rL7S6aMCa2BQgY9cI/MVbLWV+9/6fSuw3k76R4F+s6zfZv43s7y/XSvHfbsWd", + "KgCmjwfycBXEFlptePQN0uYejKaWIj1Im4oXqQS1mXGp4NHjy2+2QeW0oDj/2mhp/S8P5Mrrw5HuyXEX", + "FhIqSkNFC5s+eE++ADeOexdubb/37wg4TMd0mvNc+pmJKVbRjEibtZuQKgN+bGJ3eT03Ct5fMZUO7vPq", + "uHe5+hvd35HEX99Qw7yNQ2+dzO/eaivz2/e1zG8QTW1msy270XUlmTYbAq0dpmlbMq5Avy4HgIfGFdJF", + "0DutqJTqAgIN4mDI/rfWP35TBKcffnAplPlgsPMEfids/uEHl0XJTh2pEKYEtRX0Dn89Bi/qFAJloche", + "mbBdH4ep2Q2k58oK/NspSKUjub2G5Kjwm4bUSkPylmu1hmT34m5VpGppknvXkRy9hRbcYor/ObWkP7l7", + "pKLByXwyoRElDAq8QGK6XIoHNJrcN8/ILROSmfVHesFEFUmktRpZcK01EnpZU/pLRut0G3HeOcJKkTRT", + "aCpwRCZ5YiojIDnLVcyvmIN9hwm6CkK0nE/oendNjVwj4SS0cPXftppuUfHrvlVdV2v7cWaB8cwWr7XK", + "ZSnaNGuXD0u8d6tTtrhq71+rfMwkZtS35aXLtIYQKGNkCliluUmZK74sEdD66O3bVy49TqsnwhXFUtxV", + "wnJFQofMr4TVRy/KEmPmBdeCVh9IbNNpIWnQ1paKCY4TygjEExMZymSr1q970GPx5SXgcHG+VhLwPR9L", + "W2714STgB2MF9yJrnlSqWPPSIOHX7StOi5M34dQ8Kn5lGVCA8YRkvS2cK96zCbdbM25Q2MJAlGcJjgCH", + "Ur9mINIsxoHBRPSbAuACwZOECAN9l+XKiVtDVgyOMq8gvZXMLnTzo5wpmlx0TTgP4JdIhNnC4j8NWaUz", + "K/NBHjLk2MMIBcnMiGuVKvWgKc8lvAUpw36XCCdXeCGHzGYum8+hqq8gkUGJTJI++pkDaATCU0yZx3hN", + "ucTv5JBd0DghI4v5cIGoRHLGhSKMxCjlcyKr/RIsEkoETOII65WTKMULAF8zOJRmfXhGDMBZBVmC639j", + "FlMovKd7LqZ8MGQY7QwGKCWYSZsnLvEELhzbBoJBVAb0PcJob/DcflXbNwAIdsu/oU+TEGTOIzxOFoho", + "KgakCrUJG5jaQpimoLDevgkV0uxXYd+0Fc4qG0ulq+sYd1HOykx4sPXnrEhc19ulcsFgntYLSKgorkEL", + "/jEmEdbryXi1H4Bd5FGUi9AFqbfaq8j67yg4etM7h6UK55knYDKISAx7zriawZnmcJQ2v2+gqpKo/hwX", + "TfCQcIEw8ui6tGiQKAfWuAEwhRdleUHmygVfbH7vzo4+vpYRuONvgAIfy/0ERMQnk8oBXH81mQO8Kr9j", + "mYT/rOf0yNWV9VlcTPGUcalo5JhhvQz9N4WwtUK4emWD1Dzh4tKXrar0+xMXl201MAt+Sh+XIubP8Ct0", + "ROjhAdD0w/sjwBpulBVNNPeupNXpqzilIHRRJV2gM0cJZ1N9ikqr/L27DXytbsOAxunLVBhndwHxo5WQ", + "kf3RlKbVk7GFP8HFENlWH5oX6d7vwRn1K1eIpllCUgKla3uG2PRml3BQUOafSg8U6Wa8Up8qP3fZ6ILS", + "xB90nTgEdOU2bAOk9+XtCjLVhE/Xgw4WnTuEvQDq4JC9kwYO/MK4ni5QwYO1QGsg/tHVjEYzQCAEvVW3", + "bwAKcZZdFODLmwfoJRxkH4MaOt8wwP6a1iRPiAEWnKfpxcFycdb3p6fwkQEfNGVYLw6QK8ha3B9Sv+Uj", + "CupZJFgq9KvFSdwolHHY0QuFtb5ZzG/TYg2W4NhDFsIdZOTKNkgn6MKDILxowMdy/PYVn8qvxlVUljQw", + "c1EcWdURaJOwuNMU5EGTsONnezAIIW23REI0w7hjIMSlwbzi06KcQoWUcZa1JV87TKDieZquoGG04cGq", + "SRXzXP1FqpgIAR9b6m4ibrSBI1tKC19qQrUgeu5gbwL5BUOZDL55cKk0U+10O4TlaefgN/uveZp2uh07", + "Hg8X/QbC/RpEyXqDyyE3emc82MhvYvlNACGrzN5DhKzdHFadbpbI35gX/vTeQmeze0AyBPmgZsT9mkRQ", + "b7xVgw/jBbIljOz5fYwM4C9RlHBJKg6exwOeZQ1dNZmx2VDk1rinhxfnrtpQmwiWc/vpufvyK9C918WK", + "uDEjN917DxpZHsFjTgSWS7OZcFFHXFoXTfLVE9KX25KlqbahkG+0eXMrYyvC1HrCMouwH8Sm+hzOFU+x", + "ohFUPopmnEuP7At4ZFOjzBqPC8oE04rRcm0GwYUm1Qtrhr6wasSBNZkh7D+yffThc5t3EP7CPSq/+Mmz", + "ChQcv+tEf6gOAKXZBSUTlOFcEi3V5SlB0SLSXNGUuiI4mqEIZyoXBKr4EZRSRtM89XGv9Y7NMWB0XGyn", + "F100zhVKsJiCVmYeumCbiKcpYTEB+9yQzQieU61SCpRgRVi06EkC1X/nBF1xcZlwHIOJIYsxeHqgeqAg", + "mgIBRDwlCsdYYRB0LvSJH5kkpouiILBR6xm5LqkhHjKRs+9NRQPd7IUb6AUiANlN5awoHBnhmLAoCGV9", + "/nWzsS9viz4nqj7RB4oMuhUvfchQId/m6obzdUQRPbJYbC7sNrZh8yuEXtmswlazPxwZ/XseaTNXN8cH", + "cjAVS7zqFH8dnqWC6L4a79LDu4+4QHFuuvNOJZD5n9UnVDAUP9gKMkvNNt7WMVRUyCuW+UY8b+sP9+fJ", + "LWx5Xwkn7DYq9k21mMpJfw0s167qrXjuAxkxrS3Jt8k9HAt2EV0PJj5x4XG5x2JstQzbHM2Cb/vcSQkM", + "2hdn39h2nW3bgIfbsm1nm11y6XuMnLIexIiGObg14zayams6+DfNRqnNzmOZD84iS8/FvbHFk4IRGtaY", + "4UXCcfxnCBJe4T+KuBAG/gIANR4T/KpnNfTTA8A2VxZ567pszfenp5tNXEKolTxCqEfMIbyUHP1ZGi8b", + "cF/PiRA0tiil6Oj02IbrUolEzvrodUoVUhxdEpKVGS2QVdjX83NAIMsF5SuIH90OYUosMk6ZWjuK8tW7", + "GcynW5Whv2c+afG8v7nDW7vDwbL/+NgZcBnI2TATWK2ZKqzW1hmlbMJFauQyPOa5bl3zIL1Mej8NUsGE", + "JkQupCKpiUqc5AkcN6gNYev/2u/MLnchJlefHJMulxGRUikpZ3LIbK5IRoTuW3+u2/cCrIIOAYUL/npm", + "mOTXEbynB2Pi1bBqWjWAbIK6op2DzhbOsq0YK9wQIGaH9xlD+gmi8ZBcpGOe0AgllF1KtJHQS6OeoLlE", + "if5jc2U43wi++9LVjW9/svRKn7AJD9aOMzRbEPOfKqvLsjXnmHx0bO0l8Q+L4z+w0WG2tr5+siA46UE9", + "Ygfcg3JFE/rRsDrdCJWKRiblCBdr9/60YKr9ITslSuh3MKS2JYlBNADtcisTPNoa5oPBbpRRQH/bJTA4", + "YHjNj1Po8ejsnUlDJSkXi+6Q6X9Aw28Pz4x3d4KtNcEbqC2cjE62Xq8JcD6HZfo3jhA0E1yJXhDc8G8u", + "wZtjjDSeIdlwRHm2SlXi2Z8+hNVKcN/sCo/TrgAgT8VsNgpgL4fGFbYhzHmSp/of5o+TdbhmCkez9/Dq", + "VyPtmuGs7cZN8FEcSjunmJjalg/i9DAL9lhjVvXCuSmAEFOJBgzeAofqz0jdX95876/jV+jutCvq6sZ+", + "NWfrvm8+OwaHsOGvx2M55obS3EwUX219usK02fr0Y8KjS2mhWHyzodbbAF9d/1jiYVsXIYgJkBmKLISR", + "AcoisjtkNQOkQfyRCCNFREoZTrZgzqYRQPZ2Viw85xQStCPIU+lJGgNmUgLw3QB/p2cDhirXgOfRlbay", + "lv+O74xUHI1JxFPi0M43Q6rb3zBVP3FRhS7/WvjiW2/9ARIQU7C3r0Frb+7xs9DbT/E1hErHuXUouxFt", + "vOTlj8YU1EWwN8PO7kAOO1007Oykw47egSMMJlSs0D5KKcsVkX10bOxbkIL7ZIAkiTiLpQNddxa83YFs", + "Ssg1ZNmQ3fkEvrtPscdSFSzlG9tJiD3o95D+HpJ20IZ/4OyZjLtw6GLEc2XM/fZc2bdiosA8snnvvlrv", + "jHzT7dtw8r/Z41vhUbDLml16W284e5bLGWk2ub0yhYxyNQYwb1dcVM7Q3/lYdhEjV8YaLqTqL/E9/fWZ", + "6eA+Cg3orm5SZMDO/VuFgRYVBsq1CoM1mgBLfSU76jCIjeQ640IBiqPNtTc0BJoEIEfwCCfo9dHJkEWa", + "FRloQUFSDtzJ4qGbW/jwb+foxdGbLjqGQpfo53y82UevWbJw5caNj2bIjCRmmFeEGRobqiVx6Ho2Ywfq", + "uctgcd3BA1WONicj4Flxe+WCxLudGcExSCR/dF5x01kAdfjNK32AAPjXfFlse2el8NF5Q5RY9A4niojl", + "Zk9tnhQrMDPsJe0g6KzgZoAvdYfSIa+VfRrZwEBj7O50AkgZn74Vfbj7Aqn34yUzcSKm3N44B6RRBkkG", + "OF48rlgmOUMFcwyxQP+6LsomNGUJW162UsGALpsiv78ik/tK3lXBlv93PV0w00fraMoq+6SJuCi3stbT", + "65KDZwYO2TqqIpzhiKpFF+EksXeUvQmKiJReIf6OBcGXMb9i/SF7UxR6sQm96OjsXdc5alFM5aVpwfpi", + "++j1nAiZj4vBIThoxmsMa07iIVMcRTiJ8kSLG2QyIRHk4kL9Ftngyy2G0rnDs1N2Eiw240W154+uxl2Y", + "JmD3SrKoU9yW2eotQaIE07QZfNwKahBwCKEGY90oZ4iySWJDqiLBpUS2qR5J6JSOExsgJPvo7YwgiVMy", + "ZFmCGSMC5dJExeuh9zJBpMxNgrduAEB6DUV1UQksmAmubGhCwrmQJppAU/j7UyQVyVaQ2RvT8inM+Y5k", + "W9O47emBjNS1MTSbQuwrSG+IoRSz4JqO8sQFMN5rKLoZ0ENLiY/l4L8VdDolQp8KbJisCcczx9otpzn0", + "lYzlxnqX58Vb7epdFq16WYlext5KYLhRibUdd24W9Rfo/JI2YgfaRzfLIv5Ff9Sy72q2angQ9tFnzjJU", + "uvPfsUrmuZck2NaAVVL4YzMneSOvHNVKou16WK3WmbV3menaGj/rwWCzHjNaFq6kzzYpvF8fIQzuF+Xh", + "vousPW7aqqBdVXTThpT/9Wj6XwUF3g2M/gOjnNwCRv+ryrsHnPOHwz8JHtSHyqOv+J5dsd0/PRL+XaXP", + "Gzh8gGNrSp83XM8Gr65UlN7bd9qpSbbFP5MEb+MdbyC/u2X/pvW3UBm8xVrngtYET9JMLVxAm/VVlkFn", + "kn4k/QZHcBG3eneu4FuEdH458nB02hjQ+eesjf8gMaO2dCCV6OQ4UHT+kWEM+meucrFs6Vunh0U0o3PS", + "bHSvnmC7RJkgvYxn4FyJzYLZ9XB3mcKiP/2IbPMWc9X+C2pPAlQ/iVFMBYlUsjB1QDVHMH18J5HgWhOA", + "51wsmqNEzBH5SfD00M5mzX1oz5Q1hpVxhumiF2OFe3PHbVaY0D4jutPFU2qGhyhDL39EG+RaCVPhAk20", + "5oPopFhSch0REkugyU1/wNuDBssm/UhG03GbUa6oVfLa1oJBUS4VT93enxyjDah9NiVM74UW9ScgyWaC", + "z2lM4soYO3OemFXdbljQm9pdtVBRFK5zyoUZ3IPIMG0upOlHmlXZQhESM6YMw+DWVgWpnimTxK/7w5S5", + "ABy7R24U364wq/ltOGVHUyLU4bSLqDg3EM+b3665x3zN+clQ7k6r3HYuPGe18bpdflTLtKW7KPxQ5M7d", + "r9n6/deT0kPlo8zmsabzeaGQNpnNvy4SHNzf/XDf5vL3jzgF9CVxyrdnKocGdIshgnkFMd0xmZOEZynU", + "Q4d3O91OLpLOQWemVHawtQWx3zMu1cHe86e7nU8fPv3/AQAA//8swBTDOvABAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/resources/monitoring.go b/lib/resources/monitoring.go index d69b1e2a4..2e7eeea4e 100644 --- a/lib/resources/monitoring.go +++ b/lib/resources/monitoring.go @@ -199,7 +199,7 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { gpuProfileSlots, err := meter.Int64ObservableGauge( "hypeman_resources_gpu_profile_slots", - metric.WithDescription("Virtual functions able to create each vGPU profile (best-effort snapshot)"), + metric.WithDescription("Estimated concurrently creatable instances per vGPU profile (best-effort snapshot)"), ) if err != nil { return err diff --git a/openapi.yaml b/openapi.yaml index fa0e1381f..5ebb29556 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1775,7 +1775,7 @@ components: example: 1024 available: type: integer - description: "Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer." + description: "Conservative estimate of instances concurrently creatable with this profile across all GPUs, bounded per GPU by free virtual functions and remaining framebuffer. Best-effort: recomputed from driver state on each query." example: 59 PassthroughDevice: From 3a36785a38323ef614ddd559ef3caf524bd69f73 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:29:25 +0000 Subject: [PATCH 015/107] Revert "Report conservative per-GPU vGPU profile capacity" This reverts commit d6adc4c41df2aff189c966df5dfdd73619f6b5d9. --- lib/devices/vendor_vfio_linux.go | 51 +-- lib/devices/vendor_vfio_linux_test.go | 28 +- lib/oapi/oapi.go | 494 +++++++++++++------------- lib/resources/monitoring.go | 2 +- openapi.yaml | 2 +- 5 files changed, 260 insertions(+), 317 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index d67ddbc00..ceab85ec5 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -81,37 +81,22 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { return vfs, nil } -// listProfiles reports a conservative estimate of how many instances of each -// profile are concurrently creatable. Free VFs on the same GPU share its -// framebuffer, so counting every advertising VF overreports capacity: one 48Q -// assignment can revoke the type from all sibling VFs. Each GPU instead -// contributes min(free VFs advertising the type, remaining framebuffer / -// profile framebuffer), where the largest profile still creatable on the GPU -// is a lower bound on its remaining framebuffer. +// listProfiles counts each free VF advertising a type as one creatable +// instance, matching the driver-reported units that mdev sums through +// available_instances. This is a best-effort snapshot because creating on one +// VF may revoke the type from siblings that share its GPU framebuffer. func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { profilesByType := make(map[string]profileMetadata) - freeVFsByGPU := make(map[string]map[string]int) - remainingFBByGPU := make(map[string]int) + creatableVFs := make(map[string]int) for _, vf := range vfs { creatable, err := s.readCreatableProfiles(vf.PCIAddress) if err != nil { return nil, err } - gpu := vf.ParentGPU - if gpu == "" { - gpu = vf.PCIAddress - } for _, profile := range creatable { profilesByType[profile.TypeName] = profile - if vf.Allocated { - continue - } - if freeVFsByGPU[gpu] == nil { - freeVFsByGPU[gpu] = make(map[string]int) - } - freeVFsByGPU[gpu][profile.TypeName]++ - if profile.FramebufferMB > remainingFBByGPU[gpu] { - remainingFBByGPU[gpu] = profile.FramebufferMB + if !vf.Allocated { + creatableVFs[profile.TypeName]++ } } } @@ -124,35 +109,15 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro profiles := make([]GPUProfile, 0, len(metadata)) for _, profile := range metadata { - available := 0 - for gpu, freeVFs := range freeVFsByGPU { - available += gpuProfileCapacity(freeVFs[profile.TypeName], remainingFBByGPU[gpu], profile.FramebufferMB) - } profiles = append(profiles, GPUProfile{ Name: profile.Name, FramebufferMB: profile.FramebufferMB, - Available: available, + Available: creatableVFs[profile.TypeName], }) } return profiles, nil } -// gpuProfileCapacity estimates how many instances of a profile one GPU can -// still create concurrently. When a profile's framebuffer is unknown (0), the -// free VF count is the only signal available. -func gpuProfileCapacity(freeVFs, remainingFB, profileFB int) int { - if freeVFs == 0 { - return 0 - } - if profileFB <= 0 || remainingFB <= 0 { - return freeVFs - } - if byFB := remainingFB / profileFB; byFB < freeVFs { - return byFB - } - return freeVFs -} - func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) { vendorVFIOMu.Lock() defer vendorVFIOMu.Unlock() diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index b68a9f912..4555f5c2a 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -184,7 +184,7 @@ func TestVendorVFIODestroyRetainsAssignmentWhenOneVFIOPathIsMissing(t *testing.T } } -func TestVendorVFIOListProfilesReportsPerGPUCapacity(t *testing.T) { +func TestVendorVFIOListProfilesCountsFreeVFs(t *testing.T) { t.Parallel() sysfs := newTestVendorVFIOSysfs(t) @@ -196,30 +196,8 @@ func TestVendorVFIOListProfilesReportsPerGPUCapacity(t *testing.T) { require.NoError(t, err) profiles, err := sysfs.listProfiles(vfs) require.NoError(t, err) - assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-48Q"), - "one 48Q consumes a whole GPU, so two GPUs mean two creatable instances despite three free VFs") - assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-2Q"), - "small profiles stay capped by the free VF count") -} - -func TestVendorVFIOListProfilesCapsCapacityByRemainingFramebuffer(t *testing.T) { - t.Parallel() - - // One 48G GPU with a 24Q already assigned: siblings only advertise up to - // 24Q, so at most one more 24Q fits despite two free VFs. - remaining := "ID : vGPU Name\n1147 : NVIDIA L40S-1Q\n1153 : NVIDIA L40S-24Q\n" - sysfs := newTestVendorVFIOSysfs(t) - sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1153", remaining) - sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", remaining) - sysfs.addVF(t, "0000:82:00.0", "0000:82:00.6", "44", "0", remaining) - - vfs, err := sysfs.discoverVFs() - require.NoError(t, err) - profiles, err := sysfs.listProfiles(vfs) - require.NoError(t, err) - assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-24Q")) - assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-1Q"), - "framebuffer allows more 1Q instances than free VFs, so the VF count caps it") + assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-48Q"), + "each free VF advertising the type counts as one creatable instance") } func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) { diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index e4d26040d..fa568c18b 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1006,7 +1006,7 @@ type GPUConfig struct { // GPUProfile Available vGPU profile type GPUProfile struct { - // Available Conservative estimate of instances concurrently creatable with this profile across all GPUs, bounded per GPU by free virtual functions and remaining framebuffer. Best-effort: recomputed from driver state on each query. + // Available Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer. Available int `json:"available"` // FramebufferMb Frame buffer size in MB @@ -19026,252 +19026,252 @@ var swaggerSpec = []string{ "JLVWYtGUYuKRzuH11nlS+sO1hpQW687I1X0sOmT69TTZ9iTD2d2s+KqI4cLW4OXpcxtn30dwuiDIzyXo", "107auYOasEveHzKbaOThFYAjDbAETJj4jFCBuKBTWu24ame9x5jdm1CmI65bU6f/YVMMnizcgCEeYJCW", "JyYE1hQA8YFe7Z50up3zomiUZUzVpXlTFN5aWpEysn4Zc+fs3U0jbzPBJzQEhAORQPapVdRcTOqrvcF5", - "b/uvJmpekx9IbJSZ6KGUx7UaT/b9dhfRy7N3Z01jKqouIX90S3NaAe9zxJkkYm7MGkQqmurd5JNigaSH", - "9ZwsTKCKQZo050PfPHZRcCS4lBAj8PLsnewaww2JASNVj2+8QBNBSBFYM8mZKTFlDTquYCXgf4zzyYSI", - "PvqRSNUjkwkX6gCQB9MsL7Rga4gxNMiZyfH6PSe1/IT95yHZ2+tmlAaMgT/p58i8UMAknP5Ylb+1nN9W", - "yz+rUA+o+RMcUTbdbE0eAQNibRrrcAhfnr17Y8sMNSE0670qShEZcOY++rUoxAW7W6Iw9QOWxTrSX0Pi", - "2dlsIWmEDb3YKhqU+QZBOD2tJfqz8kNrOg3I9eH6be6koo35NMuBT5y/6Z28fr+VxmTerYwJAv9mPCF6", - "3JseO5s79IMyS67CxeZNlhlDGLLtCffWqmAxrRfJYyiB1VFc4WQkEx6KMXqrHyJ4iDbe/2TSl/UIuiir", - "bKX+3Ycm8un7SfDEAOJ9Q7fn0GHdxFs54EFdtw6Z1qlOr9Jp6KiY1KtlmWy5eCK/rG40v1xfsM800tzv", - "kcsOqxnhreCLTBKZwXd0YRPIfGqs7NaUI0mGBVYkWdTiN6sA7GTZA06uSXSD9LQX+vVPpuhGLshIzQSR", - "M55U4yZ2u6ErKMrhBjL1OsycPEeB4ijF4hJubif4o5yZFaiy+t11eDQzpbIbTOrnt2/PjDVA6XsyqSdJ", - "yKWIgGOS4AUaE3VFCHNTwRJhP0a2nmQqG+oFCTXKiKC8uoad3UC/5yZuGk0FjggyX7myz3ZLJMQjtV1K", - "20sA4TCKiJQN+7u9an/tp5M8abfHoWFtr62SHt1kg98enbm6N0VdYbfMO8urfEZEzxw5V2B49dbuyNUV", - "mFxXzADXLgkMYwIlmWxmip846uKhoOSU/rySrOkxB+mnF9p+4BSYpeqac/6hlTRaP+4hx3+KWRySNE1C", - "gknlnwK+F0RFixx8XTQ2wVIGXcDoDX5ehSA4poxIWcuDjnKRdLqd3sTO6mBrK+ERTgDicG93+9nW6rDT", - "lfHGNrxqFNNV+qgLwjJhOi7r1uDDwaSrJLGFs6yFxc6s45r7AdjTcnwj1GTWd5sn4bmA98FgKaX7GkfK", - "VZEDE17FRYv9YwtAxJX5QIOpKQyjRe3n/vkcBH3mGVazKvlvLdE+xNFADKSmEWMeqa2jIfKPQYmKi1Cp", - "MC6UzRAcExckWdyHLgTRQe5WnHmDZ/4sn+zv7+6v40PAbGrH3J67wFTN29UcaOIttz2/tgEI56rKHO5I", - "r652qtdlDU1pjrhEUqsXlGeE3Wg99/d2d262nm0ncuLCyGp8KQQhc3R6bGSiiDOFKSMCpUThGCtcZTJg", - "+9JcBmrRYJJCGtHk+9WspSHewseEuW0hrS/lrW+oqffGQUinmNGJZsj2Tb9nOcM7+08OTOXPmEz29p/0", - "+/2bImW8KKExWm3Flgnq80Az+nL2eftwB4AYbebyR+fs8O3PmpHlUphLa0uOKTvw/l38s3wAf5h/jikL", - "A2m0KRZLJ0tFYqvxa7lFKibxASrrgTu5p008UYPxGqKZAb0nCEtXieq8O/y5gsZptczADXKJV+TWanHl", - "NUsWjWtz67qvZUV05dV79bPTWtR+pR9X++OduQvesX0awOqiLO6yJ/5WhY3lytqPS6XBMsKKao9JYv6K", - "OAMg4FDpx8oV6Z61qBwG14gtEVZ06f9Y9O79eOQPxPvdVR7zfrI1ID/cMIRmpUD6t2U5dD0XcuLomsMc", - "tj0Wt0LbersWty4YO/7Ad+Ftwsyqvb+e/tfv/0eePf379u+v3r//7/nL/zr+lf73++Ts9WdhoqyGLHxQ", - "3MEvBjVoasj7eINtSekUqyhgo9PqX8MK2yfG4qCiGVQIRWNyMGQ99IoqIky9uVqy5LCDNghoSvCVFneh", - "mI7JU9vUH58ZD6j++A8nBn+qtxHbJHZhN6TAJpH5OOYppmxzyIbMtoXcRCToBfqvGEU4M0XjKENa/12g", - "sYAKf9YlVXbeRX/gLPu0OWS2qr9B384wFEmbFFlizDmU7ahM2Kx9nRQwFSaDcciK27rA7DN+yX4J8E9J", - "Us8xaliU1fqb1ZyeDULohpD/ojcSitiAClJQtiajIjEHPRtsLutza3SMgoZWkJ913JsEycM8ZC5uSqo8", - "JjGNgK+4vMKZzTwtUjoNpVkjXib49QL25o1JdosRztVM86LIJuJHnF9S0oUt7YIfDSJF4Evj/5/xrDde", - "9GY8K0AZsDDRMdh40KtK9v/p2Yn23hNBJ7anYG69JpGA0AlHxs7MpCgW1oWlib019T+YFn3mxL5uCslI", - "UzvEeApVLpgrb0GgyCWgfxTUR0Iy+fcoSihYneSM50mMZgDQp3QzIYy9zqBIscHjKCaT+r+rIRA7+09A", - "g3X/3t1pneFqlm4VleVJQKdNHetrwbENm4QBGPFg5Azha4KW9A1oK22BnUJx+O85cg2VJ65gJMY7ZXLu", - "pK2WkEgv224zmBZlj4GFGBlhe5ra3EdLp7CSZdWiBRPNAJ8lLcBNXpjszrevzpEiInX59huR3h04JQbJ", - "okelzG3hrsOj0xeb/U4QmKniqoKtWpmFVR10AJvBRjc0BW2UNhqcki46OYbsWnutlLoYpEP8xAVKzK1Y", - "XkYHAO5RNfdgU/Lv5NgKoMmiDJEwYsuws+lazOrX2wF6U6iAuBhKkSdZ0pZrsrxMoFkbMGdyNZZar6XV", - "gn/Mqn/2PobMDKiHaHgxgEs23l/tbY4OqkpfVDUL2Y0vJD9qpdH+5e39l4Zh/vIy+u7NZHTrAx5lMyxD", - "1D3zvZrw0tK++27kKrsXzTFGlX5HkgbP1t9cVRjvGlJE33OVz0O4o/u97e2323s3N9/dFEG3Cp3lweoV", - "ILrt0W/vAkU2gAlL1agxGB3pxzb03NlF3p+iGZbsOwUPa9aR7d2nbYwS0GvbMG4/gJtPzJAKLuVwuIrw", - "Y4NIdkmTxAgwkk4ZTtBztHF+8vKXk1evNlEPvX59Wt+KVV8E9+cWYLpwC8A6moy0ABRTBUIAFbmGb9++", - "gsOVEEjXMHL45e0hdteaFltA7rrBvTx7B45/LEcu0LM5txGX+cHkmkoll1HYWsVLfw7Er/m0NIy1maRp", - "w0Z0rcUJ/rkCZBuE1du8A4BfF+y+tJwPgH37kEmFXx/u7kqk3M+Fu7V2hjtCu2280kJIsTUEhf2m2+32", - "uLV3MpwKAE2IbfkSjsv4vjVQbLdDA9muh1JfPCRGJ2dlZajSGeGar83p+U5/+8kzKG66PWjD2FMcrej7", - "9PCofeeDHXPLHODxQRQfgMJ+W5+VJWyjguDkCi+gOKBZ2mHHXJiedusdW6tItoqvWcbjvR38bl2MawDY", - "BXHWBS7JUbqyNkmLdMY6yFqaWwjJlCYJlSTiLJZVGXmGJZKZQU418duFBD9kMMAuKkolg5SCcBSJvDQ9", - "Wunayvt5Zuke6oRmnGkdAAoF/EIWEqUUnKBF9xD6KFGRNRMP2YZwGVZFKhWUCI31D5Cv0LW1x2M9NKqg", - "Fon+YMjkLFeaiW320RFnMk+JsFZZNKbgMdpEMjcqLYwXVmOhGaakMRFDpl8LYLP+UagnB08Gg8Gg2yk0", - "uV3970GImu7U+dm32MMmRxjQAZlFIQbYQZEzlLOYiKJeODHkUA+Ru6Hj9DNBh93n7cQr+3kpV4UP5jqM", - "4nbgw5+L+ApDbdDPITr0Fsr5/u1F9FY5Sk5+tdlJ9qvRTSIYCIp4nsRa4xvr284Y5EhszZCSKMOdzbtU", - "onemVmd16jb0WHGTP4Len55Wwh4EmWge0G7iwCUa9oFnN9qGnTU2krWjuYl72cPHvQ9M3Lqk4kmIXxwB", - "1/cwuqRpQ6EVw1ZFcV5lX9NaZTCLhDKzT5poVkywZsqPyXyU5yGtRD9yuCnv3p0cVygF4yfbzwbPnvee", - "jbef9PbiwXYPb+8+6e3s48FkN3q6u72zuyLjpEWa2+0z14KqaSBguAgPH7kw9YY8tWCSQE0IsIHPV5TF", - "/KpyzwQjUf3ebZTruu6XY9hbDyGY+ZJgqYydoIFlnMJtSiLdton8tqmURQ2ksEXxydvB9ueaWWBwDcz4", - "rciZ8V+a5P/CVp96A/Y3qzrO2/FWGJDLMFm3Wn7n7RdtcLD//GD/cxfNZUmsG2OdnO5xc5tCrxw4ci0N", - "w6UCegYbZwnsWOHDmM9t1kan2ykSS+BvuHVrQcvF41bZUk0HthtmI6v4d0NW80lFMYCQEAOOFx9okcAJ", - "/lBqocid17LGUcLzGHlGL4MVBg6vE09J0M2A/8nawgz2p8l60MoEgE1DSQfKNCMGR59uxKZAH6CX8C48", - "wqnRn+wgTGER38eF44UJTNHny3VttJnVQz63igx8o7UapP8F09bLYG2jq5swYtAB+pXDN4VaxXjdyGpe", - "B31m+fW6QXbD4mg7SAvozMp0B+inQo4rJEEr+W1IYv8cWYZVIslsVvL57Y53NLWUO+flpnc7ZkU73Y5b", - "KMhhX85mf1dS/dL580kxFLFFcAJnuUzGzRVNLH42zIRKRSNpszT05jbJF7bmEYlHRktpCv40GZ5Wkyk+", - "cuLL+1O0ARCJf0HWgqz/tVkEilbuup3ne8+fPN15/qQVEFI5wPUy6BHkHy8Pbq1AGmX5yBohmqZ+dPbO", - "GBkio74XQSbvT33ciUxwzXr0zF2DfufP+899/KeY5+PE8+BZsDgDNwsbFoQ4K3hRQ8Dh7zSZ08mE/f4x", - "utz5u6Dp9vUTuTPebsC1NR2F7Vsnvhd/yRhMxj1TvygM0QMEJWQjitUbImEG6JwoBPTTQzgCPaJIG7Yk", - "57Cu7IoHCWtvd3f32dP9nVZ0ZUfnHZwRWLsCl7IdgXfE4E208eb8HG15BGfadGAPAD3OrI4ZPmfIFh8e", - "VAXS/vZgN0QlDRd3STW27XnauOTvrZ5mJ2UXHbKfCx1u6ZQHV3t3d/B0b//ZfrtjbO2wI3G9msO43CCz", - "PBYh39/5DZAm3x6eIci8neCoakRxoVg3GpW60aiguoNBZb/BwJ49fbK/t7uz3Q6OLRTdYYEGKwe2yrsC", - "hy5AFIHdCCzFMuvtNt0WIXHKENgbEiWYpoeRy2Wo3T4GfX0kzGvlJrS5GKzpf+niavFtKytSYRsymTBG", - "NOAC5ayo+dFf7/v8Ii7MZq5trof1XD2U/8L06lncIFPb7BZLmQkypzyXX6Ahrkxy6iThXNzo2yaF5Q2R", - "eaKMn5FK9P70O+ApmtaQVCSr6lCWGlegK91ycjc6zxUSCRN502K12o02W79qwt2GU9tdhVxR4QaNmGax", - "5lw5Wx9leYSTKIcqN7jYTz0rAB+C3PssSxYmiD5JOGcommEG3ghha2qxKcJoxpO4Hww51U9Gk2D4Ar9C", - "CTdozJeEZLYAjBmE/kyLMHRO0IZf+syQUq0g6X5qmIwt8VGlxv00XFkRy1BWWJFzrtcTK+4BBZtPKibH", - "hE8NtpKC9IB+HZ8+w8JE/WNmChrNU6NLBiKbA0OsMfPQjWpuUj6xCq4VOSCj26ykBY0iCZ1C8Zz3p7VE", - "4RXJZUW68PrIyepgW5Cu8RwGrjKDWNW67lnofgwkznzODQk0DMl5K2ISnXEyxSyHkjAeIVuLd7913OGM", - "SzUqAKBuOFipRlDnIRekhLEr0tsLe5B7J3gvOtZ2m+WyAb63+nqJqsJNNQ2wmacGVzS8Wt2CBkNkvAyB", - "tRJ1q4TxqmM23QRFriwMQCW0Sj18MLQBySUeW/LA7TfbRKOEVVbdz5K2ast2vtobnLfFT1sNl3aG1eyE", - "TXgAZOMGLkpnibZhoRkRKYVKJygmjJLY6ZKFr9KauiAzO5EExTmxK2fkU4HtgmNzvAEogzkbGWXTGq+v", - "d9jGPGzGsLoMBPRrX2wTVyTDmatvRQ5rZQIDJcJlDmuraEsqR2F31nLDgkzzBAtkkRLbDFku0oSyyzat", - "y0U65gmNkP6g7oCe8CThVyP9SP4Ac9lsNTv9waipZtC5GZxNwDMbUuu3nMIPepabtfRfsMRsme+3AJql", - "TZhWMCT7J5oQC6P3jtFrj9CrOOl7O4OmtPSGRisJ6csQjDfl3JZkgyc+l4EkvpVSjqt2RGILXm/EniyX", - "pvBKi1vJoaM6F+DtPDrVDI3PwwA5Mvy6hgCCxgQSbNzUlrlGC7bYZirBmg+5nKG/83HVINo2vjZQSWyD", - "lVgUgkyCgfSwoysN0uaNpTXxdvcmYA/AVvVE4aMbYiisq7lWBjI18ZM3S+XHZsQuGXVzNKXIWpTWcIEW", - "BU6A7bU9YEC9UFwgMBjgYKRaQMlVqHOz8KofSjTmQgA0tJZwOHOzAXwTLfPotXYAU+jtjCwsFO2QUVYY", - "SQG1jCBG5kR46ahcaCVrSuI++pun4gGYdpqphUVpB+P5dxLxK1aMccj8QerGc6nbOWTGsihyqOJfvqSb", - "Ba1PEwqkB4MTTAkobEjVDE0EkTN/7qFillrGu+IibqwStEDuFSg+Az5WpPglYT4rK5oJqoamoZH5ajlc", - "zlSihadW/0SV4rCoXvx1dX+5JCIsJBZTKl5pFbriHRVPOTFoKwA9AkX/7F+GxRdwIy3ARcrm/+qaLH86", - "Kxqv/lZ7zQMQcXi+h8ZsGzTBRiZfphbsU/WkrQ1VgXyzVbAxy74EtOFilV3llKok4FUwaXVPtkt5q0fl", - "u9FsSRJVe997tv/0ScsSMp/lrDMwWV/aNTdPV7jkGnbqtI3f59n+s+fPd/f2n+/cyMPiEjga9qcpicPf", - "H7RBrpU+rMm//vHP96c1r88+BDsPbjQok8IRHlJDGkd1QO9P//WPf7pR3XpAIUazDMXd4LdvjNJJ/J10", - "gQJVF147J9kK/f6wYiTABZtBG2QyIWAGHZl165WDqeFttJOCcYYjqhYBRo6vTFh58UoNUrqNO6g62JDI", - "a9q28KOac8l8XGZ3brjO0X8a33CNFp61rkQl83GTH/p1vVfjhS69Fn6MQ4sQA1kUQV82cBfzucKyEjmt", - "/44gwcGlci2ntZg3VsPb1nMOIIrFFlzzQgFDsOg1edJ+5G9/bTs9v2XFrFNf8Q8rzmHzEbyR1TdwIweM", - "vtH6HNYaf7AX4O2+Go39GnEri/BVCsqVt+7N+22RprtckKC4wW7en5eZeJMP6+C7QI92DHbJy7a7FZJo", - "oCYv6SRgQOMJ6RWBejYjBcnceAT1mbd47oFUyeiSTyZVUNn9ZhBywNeBrCrXC1ZKayZdRK6dzaKOYG3A", - "dIadfTnsaBVg2NlOh52a2yqYp5ji65HtoAqiMliFCl7mmdcGKd0MxgmPLk35M6iq3UcDlBLMJMoZHP6a", - "V217sNo71O1k3t4UGNzEhDgtsS0Y05jM8JxC6QfrU5lWAjHJNVUSAkahnQMUcwOrVKn9ameoXzNZhAfl", - "pOHSwWxhG9YN6vc4cxGt5btg4JtAxVn2kQjetagAmmO/fn3aNQEMEHpoBlaJb3QTNSPQDLLoolbHoPw9", - "HD88TsgIxl3HxU+X19FP/gbPqiCSKGmBsktyqBEBinjOVB0wP22nyFXzt5avpJxBsJ8N/wAANNu7IRAU", - "kwhOpFw+i1VCvwVx1/IG7EqHEgd2QyQMhwJ8SWFf8RvrEK4PwBgbvLLNph0/rtt4CUdScVvnqzjVI3Id", - "ERLXkTXDr7SNlbdfBmPlX2ELxlNUVLZvQ7zz8uz6d5dJBWNtWm0/pp9x1gMYELelFrLDYPBZUJgqoVUw", - "vj3siFEIxzT0QpvUZnK9eq1/JdcKgMjjPDHocmHStazKXkbrVvzWKYRNB5oLsrZu3h3UkzPx5reqKGdD", - "1R+iqJx9604KyS3tzjlR7t1zS0aNO1StqFJxabmAf/dKNcbGkFIX2QsebaebNRLcm4WtIhb9tmUyJMMp", - "GWWCTOj1CuIxLxjFuIofUh6kIoPBAHlupPga7T1F0QwLWRs7o9OZShbVAJy9AGjRZ1VbFEQR5gyFbXa+", - "3E334XK0m91Ov/WQcHzuYfAs1Q6xIuloFUD1Uelts9b5DC/AitPoJHy6uzcY7O4MboVQ7YZ1g+U6Kj+x", - "tQmr7TSl1HnfWUd/JUrVb6HIZl4ueHslKCRFF8sklSA4PYDEmwxHBCVkAmh0RaHx9Z7FeterB28FKov8", - "UtC/2yi7b84HX61NU3Rlwb3dNDrOuVgF+/Gfr3GINrCZaAm7LpBzt9sbPHm7vXuw/+Rge/suUKWLRWrK", - "9nj6cfvqabKDJ3vJs8XT37dnT6c76W5QD7ukpgRPG1r9Rb/bGGVTXpJV0KAKS0Mbdg4ZEfVKxvUK4JIk", - "lJGeLDKk1qcpruAFxv++9vzfzM5vZrBSdjivTtIXIbAqF6dCWQ8DdGUns9J3UZ/NyfHqWdwqA6k+kDC9", - "1YcC5NVuMFAKYrvzmRAIOWt5Db3zXmx9Ea3Milt3FYU87HDSg7vcsOIh8q4hIHizXnWBL19yAdvplAuq", - "Zunq26J4rcDrhrjpj1LFVWClPjqZMihj7v9chMn5SpT+uNPtJB/3qmfG/t4eYstC/RYEaLfalwpahJFB", - "lfzVqwCvlIqHMJHsWlfXY/5hu7f9HOIQko97Pwx6z6sRB12zWv7ybbu3K78O2qyhX2vP1Wjafn6jiGu3", - "nqso6BcaqhRX3ssWBNjSeFk02l0dLuG2ssHl46U9rkHmNAqgnyvp2ctt5AtNMUnwIgQC7xlqZU179IkM", - "jcmUMtnGbrs7KAy3++mw00eHFokbdFnFi3785qE4vEcnNE1JTLWMaVT/5gyGnZa2uLoucbMiIO6rgLTW", - "D4trz9dDJKxLuFp3TfY/Ix/3s7TfdhrvKvQOsKs5FRXAuuDFLqIThFmtEihlc5zQ2CbSQ2IkxKsdOES0", - "kmQtD5ClHOjsJF005QqVKfQt7W05a7YLFuMn12BvXYGZYQhi54sAohRIXXQV+zo5RpngcR6V+aMJDLpE", - "/BB5DQtthZC/PiT3Lu0bkJg94QKtt280GTTa2Seb9rtmm9QE27zV24P1W30nRpFuJ8/i9TzMvNSOg90I", - "In1NCmLARFNd9pok6E3mQwuO/sZfwWWd19iSIy0S5ZlzsGiaWqakgLsFXAyhuN5jkhB9TS03gngSl1kS", - "VJZcdD1L3X7ybNbk4gSP1PJAfiEk07oK4B9Bfylmi+DAXH3P4i7ZGDhYbWkcXj1TF8iuVnVwT9dKYo1b", - "5Ztwm2oVGC5fs3kbvJRLz/xdgGn7otkyAopj+BUh7U0z1r790oW9NdqP78Is95BC2mvreqjhozr03gKG", - "3PVfxgJrsa5KvHsh93yILN5azbgJ87WeBepbnQ97/2OszGjUP9j64S//d+/DfwatzTW9WRLRi8kEAo0u", - "yaJnqvxoHb1fRTyFEgNamJ5aUiE4BRsSoInbw+iPd39QMI3FrzhdmgJEaHklerbXTugv/9Ec3+Qt4zvg", - "k2tJ9rMrcNxFpVLF3XW0kRIxdbHkLpFssz9kkJt2SRYSeYW/rEjjCPU7WXziRaCjCyMG9gmbX6AxhUqK", - "csi0VoujiGRam7C1ZKgpB86B+wiCE78dW4DMJX5bh6SJJyDo/ekSXO7rd29/fP3u1+PR67MXvx6ejH55", - "8d8Q4nHVMz3EPU17e/tPbBFwfyW3g4Uobl5PoY9ObZi+dfVPclBoAadLojRXOQSFkOsoySWdOwehSm5f", - "OWE5Wff2lQg+E2pXqSQUlWAhoRM6IeDXh+vEBtVQ6YiRSqiebo0blKHlG9sQzrADnNQrfh+qW6G3Irza", - "5cZWF/3JrB0LNSCkgcMOGa9Q5j6gvVAJeBUu9sN7GW1A5ogr8eoSZzdvBop6WDQYjDz8wpV8Bs+/RLXN", - "dyvLa8550tPqTUNJgqA12axFMHIemjIZCZ0mp8N0HJDhrWl3Sqc44GcI+RO+SFVMN6C1GVNL+99YHiyc", - "x3Bcr9dgjqVZqlp9gZqRQKpec5pDqqXaUVn6vxo8kzObu0q92LpqomrK1JatXhvCy4g5oIavylYuT5lD", - "R+zBR+uTcFfqVd7MvJE0782pUx9qCs6KBTrTS3M1I4J4GwEflDj4N1wym5fTAoXFVP/LiChjVl1Sj5ZK", - "wd0s0UZh+XFLUGQbL5vDV9c5OMXXRQ/gSsFyyf8I8yjrLG2//BEw6d+42pJ04pqAYdSUuzACe5WKVq2J", - "o6rlzfCpanne5v3gwbO8agX3azpbNeIs+6iQZoge/4ap+okLUAebMU/uHMgdLv+YCMCAq8O0t8I4pymJ", - "RzxXq8+/LV1vr/yi/mhZv9apvhiIOKqk8zbxAofKUY5heaX1cpAoF1QtzvV62WBuSIN0RWNhIaEj+Lns", - "GAp1fvoERuNJIGHkJWFE0AjKoOrzmGIGGhN6f+pVwzOFEZfwWkEEen10Ys0NDvIX1EeqgPRc3OXh2Umn", - "25kTYVTuzqC/2x/AYc4IwxntHHR2+9v9QQe0qhlMcQtK19v8aZtvXCiuJ7GVhH50L+kvBU6Jgi9+CyAB", - "QNyhfR1UEDz1lMgMU2G1yCwBhAJDMFR/Dbj+7kI9MLdy1yx7a5sppBlD9gvJXtvN/QCCMpwdmObOYGCB", - "zZW9fiF3xyQMbP3dRo+W/baS6uwSBWDul9Q8J1sWS/+p29kbbN9oTKuGAmc31PE7hm0SLwHtfP+GC3Gr", - "Tk+YScuzSdY2HMo/cUBI/ln77YPeM5mnKRYLt2D+amVcNgnGRCLs3jV6nJIo0qwCivH00WtGzHOEFcIm", - "clnkDGoYuw81hVZPgWnbbXIBUvQjjxdfbAkrfTgbxacqO9PH5dMSPX852inIeHkj7SOHsG2o9h4I6Edc", - "FOB+sJOyN3h+950ecTZJaKRQryBgG49MJYT8JIAX7rCHuEC/51xhVITzP6IjbWXWcUFu3fIq2vqDxp/M", - "8U5IyAx+RkSKmUmOMO+sOfRLx9m4JMrjvPJWc4QPpT3gpnIgPOaiAkGuekT9a6suDC5fR3sBBAbbp5le", - "/ICEv3cPJ9xOtqjB+pBHDipfolySx3ScrIttXAohQVnuJVFfC80P7vPKskUE/oSn6LEQ8EtSSHjlbi1d", - "CluZyJlRgIMS4JsyYdF+911V+HtbPvGiZMCvoZuGchbK+FVxvOgjt6ZG6VcLgFgSBOYZL18rZ3p4X8sJ", - "27mPEwYzLjxF366pb9fUqlNuqMVNAQ6md8pb2CBuZIH489kfbmx9+GZ7aG97aGV5YOTKWhf+zsd9ZCNS", - "Ix4TJGc8T2I0JsjgHbnYE4VFf/oRYRHN6JwAqB0UacsTRTMsILIkRTFW2PjQGw0TK80SRXNburmei0Ms", - "F7iOYyHJCHD4Rk34k2UEImWMxEh/YqH7SjjBpbrd5uwHDexFg+XViK5mXJICz48p7zaH9GZptGNotj9k", - "by3Qq15ACKZ2vEaSBOBqV9h/OEN4yOwH3zsW4gLBJE5LzoUFYAZSg0xptmU5tU2PdCQjHsLaeUsYZqon", - "MxLRCY3stC7JwsZzBhtsVXdJD9iN8/1pkbCBdjbDeG0AzxgG5z0uniFLSVX/DYMg6CjJ49LJ5SCEsBjj", - "JAkW5pgmfIyTkVmfSxLwCb6EN+yi+PX9nTeJ8ZiYWu3ZQs04M3/n45yp3Pw9FvxKEjHsbPaHDBIx7FqT", - "uFsKiOgKCrmlGdfnTPDU9Lllhrj1xyVZfOoP2WGcUuYoAj7BieSIXMN3UN8KMDMM92qgB3Oawn7wo1wq", - "nvrIp47uzDB5rrJc2YwSSVQ3hPo5ZIqjPxy246etP8oeP4GzmOBY04n3ipkSyNZNo5YjrGc/glcD7nYC", - "CzDs6IvUhHlMBWbKwHYW4JRo6m/pRlEdQR/SzfoKR5ihjGemsgQQ1Qxrkqu0AVgNOEmQgqPkvtWCO+xk", - "w3ws9F46bsTdM0BptWNEGTr90TtMg71n4fMkSSRIKKLkv85f/4rgVtZ7YF4rw7VMSgfTAgOKc3CdOp72", - "AkczZBxVUExw2KHxsFO4c+NNGGsubbhMrwc+xR/00H4w3XRp/EO/r5sy7soD9NsfppUDfZay1OCADjuf", - "ush7MKVqlo+LZx/CC9oEX3ZeYQRow1xzm8BJMAWkGe/GN1ckZjHi9hZIFgijkgP5gStjyrBYrEokDCy9", - "XUE+MZGM3mL8MYTIxWHnYOhiF4ed7rBD2Bx+swGOw86n8ApYr2Vz5Tq4zwrnZkFETwaDzfVI2HZ9Az7L", - "Fo6BL6wDNmpFRdlNvYMWhvXP5R/4t9Y/C9cPZrrzEprIKP7O+P4IHRCexO5rogEXRE3sxiwiiRO71xt6", - "7t95oDcrIkly3wT6UORZuMcKpP5HRY6wWeUxWmm+f2CKG9zXpVIx2z8M/T46+3nAem5t52TuQp3DdUoA", - "g8aq0si8jLBE5zCm3rlWvl/Ar337X6f7AabiRcKnFwdGdUcJn6KEMpsP4AUqa/HAriV8ZGBoiu8sKo0r", - "ErdhJIl//eOfMCjKpv/6xz8ttvu//vFPOO5bBl4NakxfzAgWakywujhAvxCS9XBC58RNBqrAkjkRC7Q7", - "sDZ/eIS8UvdWSpNDNmRviMoF8/ImTL02aRu0rgI9H8pyIi2Mj36RTmwxGRPbGLDbuLNslvJeT3Q3AIcI", - "M/AmoG9FRwOAJUdNoW2riXbCJlMz54rRtB6muRSst56/KHKtDPX2zABvyGBgiUPnDh7YSaON8/MXm30E", - "2pahCigYBLpD2YxVI/rfeNJ6nmQ4SpWhwCob3hThDI9pQp3JsaHaiTmCKY5mlJEyvrjAGndNHLiRah5z", - "eHaCbCBkF14dstfnW2BiVSRSuSBdywmERRgty6Fxm+cCPQD/ogqiw3r23SGbEAx5QifHhgl4INxFPmDR", - "MAMgD4hxpapSea07ZAZJ1iIX64OX8pgk8BH0P8WKXOFFFxW1bl11lAQrrRDLrn55yAzWq12DHkCVIG+Y", - "feBnZkg9F8lrc7YEmSRaNYYIfFP2G/remHCBbISzV+XfdWeSLM2w9KKlOHp9ruc3BU2QG3sgtPT63O3G", - "ZhdJjqKEAjVEmA3ZFAKBHHgvZ5VdLRLKZljEvYjrS8AHc7pk/Coh8bSJxx75RHaHkkyln8Bx+rlOro9N", - "uJgtT0AfYgNQt9pzd2zfaee6sy3+mXx3thDkDZx3xoJLDL8xq/vNkdfCkRdeN+fUC3nWjh0C491F/Jou", - "Hijg19He8pqbJ96SPYRFD204aBvwinCBzo5OEI5jQaTc/Pe29+mZGiot5T99P2pW/BChJ3YsXFjQP2tv", - "qRLIY2EHb+yoEXbzqtfX9e+3rUrxncabrqjDU155d3971Dq9yTVSCr0lrX27SdYG21IZcSgzWFJLD0Sj", - "hBTiS3FOfSpaZ1U2YbzFlbNSXLLs+eTYHcj7sy/brnNWvxvugSke1xjiAzLCaqq1XzX7MVHzu2IXHdr0", - "CvPz10Wag/uTgu7bFB0i88ekLsa1ZdNc0ACdNF6gL4ky8CZ3qafbHgITPyfCnWoz0IWZdTEt8ykyOC0w", - "IbDErNZ9T8wr7VRf096fSfOF5bmJxGKX/JuI0kLZLddqlYJ7YktA351+Cz3cSL39cmErlsACiwxW1LFz", - "O4FldQPLBYs2v0WufHGKNnGNpRIr3LxJXFiyDZpSoWfdl1x3yPx641qms3otZWiS0OnMOgFiOoFYPeXX", - "74ZR7tzDKIs62QIrYkMUH2Pe75leZOsFnhOh0OujE7P+/pW69QcEra5XlRzzWnm7vnvzqkdYxOPCedIs", - "k9onX1hhMvRfyeW9/1P3CPNZqRMPmgTGz9h/E0yOTPx7n/L/tfNTQscCi8X/2vkJJxll5H/tHiZYEak2", - "74xYBvd10923AvOIiU/rL7S6aMCa2BQgY9cI/MVbLWV+9/6fSuw3k76R4F+s6zfZv43s7y/XSvHfbsWd", - "KgCmjwfycBXEFlptePQN0uYejKaWIj1Im4oXqQS1mXGp4NHjy2+2QeW0oDj/2mhp/S8P5Mrrw5HuyXEX", - "FhIqSkNFC5s+eE++ADeOexdubb/37wg4TMd0mvNc+pmJKVbRjEibtZuQKgN+bGJ3eT03Ct5fMZUO7vPq", - "uHe5+hvd35HEX99Qw7yNQ2+dzO/eaivz2/e1zG8QTW1msy270XUlmTYbAq0dpmlbMq5Avy4HgIfGFdJF", - "0DutqJTqAgIN4mDI/rfWP35TBKcffnAplPlgsPMEfids/uEHl0XJTh2pEKYEtRX0Dn89Bi/qFAJloche", - "mbBdH4ep2Q2k58oK/NspSKUjub2G5Kjwm4bUSkPylmu1hmT34m5VpGppknvXkRy9hRbcYor/ObWkP7l7", - "pKLByXwyoRElDAq8QGK6XIoHNJrcN8/ILROSmfVHesFEFUmktRpZcK01EnpZU/pLRut0G3HeOcJKkTRT", - "aCpwRCZ5YiojIDnLVcyvmIN9hwm6CkK0nE/oendNjVwj4SS0cPXftppuUfHrvlVdV2v7cWaB8cwWr7XK", - "ZSnaNGuXD0u8d6tTtrhq71+rfMwkZtS35aXLtIYQKGNkCliluUmZK74sEdD66O3bVy49TqsnwhXFUtxV", - "wnJFQofMr4TVRy/KEmPmBdeCVh9IbNNpIWnQ1paKCY4TygjEExMZymSr1q970GPx5SXgcHG+VhLwPR9L", - "W2714STgB2MF9yJrnlSqWPPSIOHX7StOi5M34dQ8Kn5lGVCA8YRkvS2cK96zCbdbM25Q2MJAlGcJjgCH", - "Ur9mINIsxoHBRPSbAuACwZOECAN9l+XKiVtDVgyOMq8gvZXMLnTzo5wpmlx0TTgP4JdIhNnC4j8NWaUz", - "K/NBHjLk2MMIBcnMiGuVKvWgKc8lvAUpw36XCCdXeCGHzGYum8+hqq8gkUGJTJI++pkDaATCU0yZx3hN", - "ucTv5JBd0DghI4v5cIGoRHLGhSKMxCjlcyKr/RIsEkoETOII65WTKMULAF8zOJRmfXhGDMBZBVmC639j", - "FlMovKd7LqZ8MGQY7QwGKCWYSZsnLvEELhzbBoJBVAb0PcJob/DcflXbNwAIdsu/oU+TEGTOIzxOFoho", - "KgakCrUJG5jaQpimoLDevgkV0uxXYd+0Fc4qG0ulq+sYd1HOykx4sPXnrEhc19ulcsFgntYLSKgorkEL", - "/jEmEdbryXi1H4Bd5FGUi9AFqbfaq8j67yg4etM7h6UK55knYDKISAx7zriawZnmcJQ2v2+gqpKo/hwX", - "TfCQcIEw8ui6tGiQKAfWuAEwhRdleUHmygVfbH7vzo4+vpYRuONvgAIfy/0ERMQnk8oBXH81mQO8Kr9j", - "mYT/rOf0yNWV9VlcTPGUcalo5JhhvQz9N4WwtUK4emWD1Dzh4tKXrar0+xMXl201MAt+Sh+XIubP8Ct0", - "ROjhAdD0w/sjwBpulBVNNPeupNXpqzilIHRRJV2gM0cJZ1N9ikqr/L27DXytbsOAxunLVBhndwHxo5WQ", - "kf3RlKbVk7GFP8HFENlWH5oX6d7vwRn1K1eIpllCUgKla3uG2PRml3BQUOafSg8U6Wa8Up8qP3fZ6ILS", - "xB90nTgEdOU2bAOk9+XtCjLVhE/Xgw4WnTuEvQDq4JC9kwYO/MK4ni5QwYO1QGsg/tHVjEYzQCAEvVW3", - "bwAKcZZdFODLmwfoJRxkH4MaOt8wwP6a1iRPiAEWnKfpxcFycdb3p6fwkQEfNGVYLw6QK8ha3B9Sv+Uj", - "CupZJFgq9KvFSdwolHHY0QuFtb5ZzG/TYg2W4NhDFsIdZOTKNkgn6MKDILxowMdy/PYVn8qvxlVUljQw", - "c1EcWdURaJOwuNMU5EGTsONnezAIIW23REI0w7hjIMSlwbzi06KcQoWUcZa1JV87TKDieZquoGG04cGq", - "SRXzXP1FqpgIAR9b6m4ibrSBI1tKC19qQrUgeu5gbwL5BUOZDL55cKk0U+10O4TlaefgN/uveZp2uh07", - "Hg8X/QbC/RpEyXqDyyE3emc82MhvYvlNACGrzN5DhKzdHFadbpbI35gX/vTeQmeze0AyBPmgZsT9mkRQ", - "b7xVgw/jBbIljOz5fYwM4C9RlHBJKg6exwOeZQ1dNZmx2VDk1rinhxfnrtpQmwiWc/vpufvyK9C918WK", - "uDEjN917DxpZHsFjTgSWS7OZcFFHXFoXTfLVE9KX25KlqbahkG+0eXMrYyvC1HrCMouwH8Sm+hzOFU+x", - "ohFUPopmnEuP7At4ZFOjzBqPC8oE04rRcm0GwYUm1Qtrhr6wasSBNZkh7D+yffThc5t3EP7CPSq/+Mmz", - "ChQcv+tEf6gOAKXZBSUTlOFcEi3V5SlB0SLSXNGUuiI4mqEIZyoXBKr4EZRSRtM89XGv9Y7NMWB0XGyn", - "F100zhVKsJiCVmYeumCbiKcpYTEB+9yQzQieU61SCpRgRVi06EkC1X/nBF1xcZlwHIOJIYsxeHqgeqAg", - "mgIBRDwlCsdYYRB0LvSJH5kkpouiILBR6xm5LqkhHjKRs+9NRQPd7IUb6AUiANlN5awoHBnhmLAoCGV9", - "/nWzsS9viz4nqj7RB4oMuhUvfchQId/m6obzdUQRPbJYbC7sNrZh8yuEXtmswlazPxwZ/XseaTNXN8cH", - "cjAVS7zqFH8dnqWC6L4a79LDu4+4QHFuuvNOJZD5n9UnVDAUP9gKMkvNNt7WMVRUyCuW+UY8b+sP9+fJ", - "LWx5Xwkn7DYq9k21mMpJfw0s167qrXjuAxkxrS3Jt8k9HAt2EV0PJj5x4XG5x2JstQzbHM2Cb/vcSQkM", - "2hdn39h2nW3bgIfbsm1nm11y6XuMnLIexIiGObg14zayams6+DfNRqnNzmOZD84iS8/FvbHFk4IRGtaY", - "4UXCcfxnCBJe4T+KuBAG/gIANR4T/KpnNfTTA8A2VxZ567pszfenp5tNXEKolTxCqEfMIbyUHP1ZGi8b", - "cF/PiRA0tiil6Oj02IbrUolEzvrodUoVUhxdEpKVGS2QVdjX83NAIMsF5SuIH90OYUosMk6ZWjuK8tW7", - "GcynW5Whv2c+afG8v7nDW7vDwbL/+NgZcBnI2TATWK2ZKqzW1hmlbMJFauQyPOa5bl3zIL1Mej8NUsGE", - "JkQupCKpiUqc5AkcN6gNYev/2u/MLnchJlefHJMulxGRUikpZ3LIbK5IRoTuW3+u2/cCrIIOAYUL/npm", - "mOTXEbynB2Pi1bBqWjWAbIK6op2DzhbOsq0YK9wQIGaH9xlD+gmi8ZBcpGOe0AgllF1KtJHQS6OeoLlE", - "if5jc2U43wi++9LVjW9/svRKn7AJD9aOMzRbEPOfKqvLsjXnmHx0bO0l8Q+L4z+w0WG2tr5+siA46UE9", - "Ygfcg3JFE/rRsDrdCJWKRiblCBdr9/60YKr9ITslSuh3MKS2JYlBNADtcisTPNoa5oPBbpRRQH/bJTA4", - "YHjNj1Po8ejsnUlDJSkXi+6Q6X9Aw28Pz4x3d4KtNcEbqC2cjE62Xq8JcD6HZfo3jhA0E1yJXhDc8G8u", - "wZtjjDSeIdlwRHm2SlXi2Z8+hNVKcN/sCo/TrgAgT8VsNgpgL4fGFbYhzHmSp/of5o+TdbhmCkez9/Dq", - "VyPtmuGs7cZN8FEcSjunmJjalg/i9DAL9lhjVvXCuSmAEFOJBgzeAofqz0jdX95876/jV+jutCvq6sZ+", - "NWfrvm8+OwaHsOGvx2M55obS3EwUX219usK02fr0Y8KjS2mhWHyzodbbAF9d/1jiYVsXIYgJkBmKLISR", - "AcoisjtkNQOkQfyRCCNFREoZTrZgzqYRQPZ2Viw85xQStCPIU+lJGgNmUgLw3QB/p2cDhirXgOfRlbay", - "lv+O74xUHI1JxFPi0M43Q6rb3zBVP3FRhS7/WvjiW2/9ARIQU7C3r0Frb+7xs9DbT/E1hErHuXUouxFt", - "vOTlj8YU1EWwN8PO7kAOO1007Oykw47egSMMJlSs0D5KKcsVkX10bOxbkIL7ZIAkiTiLpQNddxa83YFs", - "Ssg1ZNmQ3fkEvrtPscdSFSzlG9tJiD3o95D+HpJ20IZ/4OyZjLtw6GLEc2XM/fZc2bdiosA8snnvvlrv", - "jHzT7dtw8r/Z41vhUbDLml16W284e5bLGWk2ub0yhYxyNQYwb1dcVM7Q3/lYdhEjV8YaLqTqL/E9/fWZ", - "6eA+Cg3orm5SZMDO/VuFgRYVBsq1CoM1mgBLfSU76jCIjeQ640IBiqPNtTc0BJoEIEfwCCfo9dHJkEWa", - "FRloQUFSDtzJ4qGbW/jwb+foxdGbLjqGQpfo53y82UevWbJw5caNj2bIjCRmmFeEGRobqiVx6Ho2Ywfq", - "uctgcd3BA1WONicj4Flxe+WCxLudGcExSCR/dF5x01kAdfjNK32AAPjXfFlse2el8NF5Q5RY9A4niojl", - "Zk9tnhQrMDPsJe0g6KzgZoAvdYfSIa+VfRrZwEBj7O50AkgZn74Vfbj7Aqn34yUzcSKm3N44B6RRBkkG", - "OF48rlgmOUMFcwyxQP+6LsomNGUJW162UsGALpsiv78ik/tK3lXBlv93PV0w00fraMoq+6SJuCi3stbT", - "65KDZwYO2TqqIpzhiKpFF+EksXeUvQmKiJReIf6OBcGXMb9i/SF7UxR6sQm96OjsXdc5alFM5aVpwfpi", - "++j1nAiZj4vBIThoxmsMa07iIVMcRTiJ8kSLG2QyIRHk4kL9Ftngyy2G0rnDs1N2Eiw240W154+uxl2Y", - "JmD3SrKoU9yW2eotQaIE07QZfNwKahBwCKEGY90oZ4iySWJDqiLBpUS2qR5J6JSOExsgJPvo7YwgiVMy", - "ZFmCGSMC5dJExeuh9zJBpMxNgrduAEB6DUV1UQksmAmubGhCwrmQJppAU/j7UyQVyVaQ2RvT8inM+Y5k", - "W9O47emBjNS1MTSbQuwrSG+IoRSz4JqO8sQFMN5rKLoZ0ENLiY/l4L8VdDolQp8KbJisCcczx9otpzn0", - "lYzlxnqX58Vb7epdFq16WYlext5KYLhRibUdd24W9Rfo/JI2YgfaRzfLIv5Ff9Sy72q2angQ9tFnzjJU", - "uvPfsUrmuZck2NaAVVL4YzMneSOvHNVKou16WK3WmbV3menaGj/rwWCzHjNaFq6kzzYpvF8fIQzuF+Xh", - "vousPW7aqqBdVXTThpT/9Wj6XwUF3g2M/gOjnNwCRv+ryrsHnPOHwz8JHtSHyqOv+J5dsd0/PRL+XaXP", - "Gzh8gGNrSp83XM8Gr65UlN7bd9qpSbbFP5MEb+MdbyC/u2X/pvW3UBm8xVrngtYET9JMLVxAm/VVlkFn", - "kn4k/QZHcBG3eneu4FuEdH458nB02hjQ+eesjf8gMaO2dCCV6OQ4UHT+kWEM+meucrFs6Vunh0U0o3PS", - "bHSvnmC7RJkgvYxn4FyJzYLZ9XB3mcKiP/2IbPMWc9X+C2pPAlQ/iVFMBYlUsjB1QDVHMH18J5HgWhOA", - "51wsmqNEzBH5SfD00M5mzX1oz5Q1hpVxhumiF2OFe3PHbVaY0D4jutPFU2qGhyhDL39EG+RaCVPhAk20", - "5oPopFhSch0REkugyU1/wNuDBssm/UhG03GbUa6oVfLa1oJBUS4VT93enxyjDah9NiVM74UW9ScgyWaC", - "z2lM4soYO3OemFXdbljQm9pdtVBRFK5zyoUZ3IPIMG0upOlHmlXZQhESM6YMw+DWVgWpnimTxK/7w5S5", - "ABy7R24U364wq/ltOGVHUyLU4bSLqDg3EM+b3665x3zN+clQ7k6r3HYuPGe18bpdflTLtKW7KPxQ5M7d", - "r9n6/deT0kPlo8zmsabzeaGQNpnNvy4SHNzf/XDf5vL3jzgF9CVxyrdnKocGdIshgnkFMd0xmZOEZynU", - "Q4d3O91OLpLOQWemVHawtQWx3zMu1cHe86e7nU8fPv3/AQAA//8swBTDOvABAA==", + "b/uvJmpekx9IbJSZ6KGUx7UaT/b9dhfRy7N3Z01jKqouIX90S3NaAe+zHO0yyZmp++TZVLCtx2gR1wwe", + "j+msj34kUvXIZMKFOrCBLOCQ8mq+4gUSJM41PfjQZJwhSccJnNGiVy3w61/0hAAIZJxPJkRUUQeeh2Rp", + "7+1RGjDu/aSfI/NCAXtw+mNVntZye1ut/axCDaC2T3BE2XSz9XYHDIK1aazDFXx59u6NLRvUhLisl7Io", + "LWTAlvvo16Kwll5qWaIq9QOWwjpyX0Mi2dlsIWmEE9OiqYpBmW/gg9PQWkI/Kz+0ptCAnB6ux+ZOHtqY", + "T7Mczv35m97J6/dbaUzm3cqYIJBvxhOix73psae5QzMos94qXGneZGkxhCHbnlhvrQqW0XqRPAYRWB3F", + "FU5GMuGhmKG3+iGCh2jj/U8mHVmPoIuyylbq332oIZ++nwRPDCDYN3R7Dh3WTbaVAx7UXesQaJ3q9Cqd", + "ho6KSaValrGWiyHyy+pG88v1BfhMI839Hrlsr5pR3QqyyCSFGbxGFwaBzKfGam5NM5JkWGBFkkUtHrMK", + "qE6WPdrkmkQ3SDd7oV//ZIpo5IKM1EwQOeNJNQ5it7tciFVCzPGc2NpTZk6e4V9xlGJxCTexE+RRzswK", + "VEPWd9fhy8yUym4wqZ/fvj0z2r0iYo6TetKDXPLwH5MEL9CYqCtCmJsKlgj7Ma/1pFHZUP9HqFFGBOXV", + "NezsBvo9N3HQaCpwRJD5ypVxtlsiIb6o7VLaXgKIhVFEpGzY3+1V+2s/neRJuz0ODWt7bdXz6CYb/Pbo", + "zNWxKeoEu2XeWV7lMyJ65si5gsGrt3ZHrq6o5LpiBoh2SWAYEyixZDNN/ERQF98EJaT055XkS485SD9d", + "0PYDp8AsVdec8w+tpMv6cQ858lPM4lC9ZZNgYFLzp4DXBVHOIgfRj8Ym+MmgBRg9wM+TEATHlBEpa3nN", + "US6STrfTm9hZHWxtJTzCCUAW7u1uP9taHUa6Mn7YhkuNYrpKv3RBVSbsxmXRGrw3mHSVJLZwlrWwwJl1", + "XHM/AHtajleEGsv6bvMkPBfAPhgspWhf40i5qnBgkqu4XLF/bAFYuDIfaDA1hV60qP3cP5+DoA88w2pW", + "Jf+tJdqHuBiIadQ0YswdtXU0RP4xKFFxESr9xYWyGX9j4oIei/vQhRQ6CN2Kc27wzJ/lk/393f11fAiY", + "Te2Y23MXmKp5u5rTTLzltufXNgDhWVWZwx3p1dVL9bqsoSnNEZdIavWC8oywG63n/t7uzs3Ws+1ETlxY", + "WI0vhSBhjk6PjUwUcaYwZUSglCgcY4WrTAZsWZrLQG0ZTFJIC5p8v5q1NMRP+Bgvty2M9aW87w018t44", + "SOgUMzrRDNm+6fcsZ3hn/8mBqeQZk8ne/pN+v39T5IsXJdRFq63YMkF6HghGX84+bx/uAOCizVz+6Jwd", + "vv1ZM7JcCnNpbckxZQfev4t/lg/gD/PPMWVhYIw2xV/pZKnoazUeLbfIwyQ+QGV9byf3tIkPajBGQ3Qy", + "oPEEYeYqUZp3hydX0Ditlg24QW7wilxZLa68ZsmicW1uXce1rHCuvPqtfrZZi1qu9ONq/7ozd8E7tk8D", + "QF2UuV32rN+qULFcWctxqdRXRlhRvTFJzF8RZwDsGyrlWLki3bMWlcDgGrElv4ou/R+L3r0fj/yBeL+7", + "SmLeT7am44cbhsSsFEj/tiyHrudCThxdc5jDtsfiVmhbP9fi0AVjwR/4LrxN2Fi199fT//r9/8izp3/f", + "/v3V+/f/PX/5X8e/0v9+n5y9/iyMk9UQhA+KI/jFoANNTXgfP7AtKZ1iFQVsdFr9a1hh+8RYHFQ0g4qf", + "aEwOhqyHXlFFhKkfV0t+HHbQBgFNCb7S4i4UxzF5Z5v64zPj0dQf/+HE4E/1NmKblC7shhRYIzIfxzzF", + "lG0O2ZDZtpCbiAS9QP8VowhnpggcZUjrvws0FlCxz7qYys676A+cZZ82h8xW6Tdo2hmGomeTIuuLOQex", + "HZUJg7WvkwJ2wmQkDllxWxcYfMbP2C8B+ylJ6jlDDYuyWn+zmtOzQQitEPJZ9EZCURpQQQrK1mRUJNqg", + "Z4PNZX1ujY5R0NAK8rOOeJPweJiHzMVNSZLHJKYR8BWXJzizmaRFiqahNGvEywS/XsDevDHJazHCuZpp", + "XhTZxPqI80tKurClXXCHQeQHfGn8+TOe9caL3oxnBcgCFibaBRuPeFXJ/j89O9HeeyLoxPYUzJXXJBIQ", + "OuHI2JmZlMPCurA0sbemngfTos+c2NdNYRhpaoGYUG6VC+bKVRAoWgloHgX1kZBM/j2KEgpWJznjeRKj", + "GQDuKd1MCDOvMyhSZvA4ismk/u9qSMPO/hPQYN2/d3daZ6yapVtFZXkS0GlTx/pacGzDJmEARjwYOUP4", + "miAkfQNaPy7YKRSH/54j11B54gpGYrxTJodO2uoHifSy5zaDaU72GFjIkBG2p6nNfbR0CitZUy1aMNEJ", + "8FnSAqzkhcnWfPvqHCkiUpc/vxHp3YFTYpApelTK3BbiOjw6fbHZ7wSBliquKtiqlVlV1UEHsBZstEJT", + "EEZpo8Ep6aKTY8iWtddKqYtBesNPXKDE3IrlZXQAYB1Vcw82JfxOjq0AmizKkAcjtgw7m67FrH69HaA3", + "hQqIi6EUeY8lbbkmy8sEmrUBcCb3Yqn1Wpos+Mes+mfvY8i0gPqGhhcDWGTj/dXe5uigp/RFVbOQ3fhC", + "8qNQGu1f3t5/aVjlLy+j795MRrc+4FE2wzJE3TPfqwkvLe2770ausnvRHDNU6XckafBs/c1VefGuIUX0", + "PVf5PIQjut/b3n67vXdz891NEXGrUFgeTF4BitsezfYuUGEDGK9UjRqDy5F+bEPJnV3k/SmaYcm+U/Cw", + "Zh3Z3n3axigBvbYNy/YDsvnEDKngUg5XqwgnNghjlzRJjAAj6ZThBD1HG+cnL385efVqE/XQ69en9a1Y", + "9UVwf24Bjgu3AKyjyTALQCtVIAFQkTv49u0rOFwJgfQLI4df3h4yd61psQWErhvcy7N34PjHcuQCN5tz", + "FXGZ70uuqVRyGVWtVfzz50D2mk/bFfl3kzRtlLX+V+P+/lwBpg3C5G3eAWCvC15fWs4HwLJ9yCTBrw9H", + "dyXy7efC11o7wx2h1zZeaSHk1xoiwn7T7XZ7HNo7GU4FUCbEtnwJx2Vw3xr4tduhgezVQ6kvHhKjk7Oy", + "0lPpjHDN1+b0fKe//eQZFCvdHrRh7CmOVvR9enjUvvPBjrllDvD4IIoPQGG/rc/KErZRQXByhRdQ7M8s", + "7bBjLkxPu/WOrVUkW8XXLOPr3g5Oty7GNQDmgjjrApfkKF1Za6RFemIdNC3NLSRkSpOEShJxFsuqjDzD", + "EsnMIKGamhuFBD9kMMAuKkofg5SCcBSJvDQ9Wunayvt5Zuke6n5mnGkdAID/fyELiVIKTtCiewh9lKjI", + "gomHbEO4jKkiNQpKfsb6B8g/6NrI9lgPjSqoLaI/GDI5y5VmYpt9dMSZzFMirFUWjSl4jDaRzI1KC+OF", + "1VhohilpTMSQ6dcCWKt/FOrJwZPBYDDodgpNblf/exCipjt1fvYtlrDJ+QW0P2ZRhQFGUOQM5Swmoqj/", + "TQw51EPkbug4/UwQYfd5O/HKfl7KVeGDuQ5zuB2Y8OciuMJQG/RziA69hXK+f3sRvVXOkZNfbbaR/Wp0", + "kwgGgiKeJ7HW+Mb6tjMGORJbM6QkynBn8y6V6J2pvVmdug09Vhz9nhOxQO9PTythD4JMNA9oN3HgEg37", + "wLMbbcPOGhvJ2tHcxL3s4d3eB8ZtXVLxJMQvjmjrexhdErSh0Iphq6I4r7Kvaa0ymEVCmdknTTQrJlgz", + "5cdkPsrzkFaiHzkclHfvTo4rlILxk+1ng2fPe8/G2096e/Fgu4e3d5/0dvbxYLIbPd3d3tldkXHSIm3t", + "9ploQdU0EDBchIePXJh6KHq4KUmgJgTYwOcrymJ+VblngpGofu82ynVd98sx7K2HEMx8SbBUxk7QwDJO", + "4TYlkW7bRH7b1MiiplHYovjk7WD7c80sMLgGZvxW5Mz4L00yf2GrT70B+5tVHefteCsMyGWYrFstv/P2", + "izY42H9+sP+5i+ayJNaNsU5O97i5TaFXDuy4lobhUgE9g42zBHas8GHM5zZro9PtFIkl8DfcurWg5eJx", + "q2yppgPbDbORVfy7IUv5pKIYQEiIAbuLD7RI4AR/KJ1Q5MJrWeMo4XmMPKOXwf4Ch9eJpyToZsD/ZG1h", + "BsvTZD1oZQLAo6FEA2WaEYOjTzdiU5oP0Et4Fx7h1OhPdhCmUIjv48LxwgSm6PPlujbazOohn1tFBr7R", + "Wg3S/4Jp62WwttHVTRgx6AD9yuGbQq1ivG5kNa+DPrP8et0gu2FxsR1EBXRmZboD9FMhxxWSoJX8NiSx", + "f44swyqRYTYr+fl2xzuaWsqd83LNux2zop1uxy0U5KQvZ6e/K6l+6fz5pBiK2CI4gbNcJuPmiiYWDxtm", + "QqWikbRZGnpzm+QLW8OIxCOjpTQFf5oMT6vJFB858eX9KdoAyMO/IGtB1v/aLAJFK3fdzvO950+e7jx/", + "0grYqBzgehn0CPKPlwe3ViCNsnxkjRBNUz86e2eMDJFR34sgk/enPo5EJrhmPXrmrkG/8+f95z6eU8zz", + "ceJ58Cz4m4GPhQ0LQpYVvKgh4PB3mszpZMJ+/xhd7vxd0HT7+oncGW834NSajsL2rRPfi79kDCbjnqlH", + "FIbcAYISshGV6g2RMAN0ThQC+ukhHIEeUaQNW5Jz2FV2xYOEtbe7u/vs6f5OK7qyo/MOzgisXYFL2Y7A", + "O2LwJtp4c36OtjyCM2068AaAEmdWxwyfM2SLCQ+qAml/e7AbopKGi7ukGtv2PG1c8vdWT7OTsosO2c+F", + "Drd0yoOrvbs7eLq3/2y/3TG2dtiRuF7NYVxukFkei3jv7/wGSJNvD88QZN5OcFQ1orhQrBuNSt1oVFCt", + "waCs32Bgz54+2d/b3dluB68Wiu6wwIGVA1vlXYFDFyCKwG4ElmKZ9XabbouQOGUI7A2JEkzTw8jlMtRu", + "H4OmPhLmtXIT2lwM1vS/dHG1+LaVFamwDZlMGCMacIFyVtTw6K/3fX4RF2Yz1zbXw3quHsp/YXr1LA6Q", + "qVV2i6XMBJlTnssv0BBXJjl1knAubvRtk8Lyhsg8UcbPSCV6f/od8BRNa0gqklV1KEuNK9CSbjm5G53n", + "ComEibxpsVrtRputXzXhbsOp7a5Crqhwg0aMslhzrpytj7I8wkmUQ9UaXOynnhWAbUHufZYlCxNEnySc", + "MxTNMANvhPCghdCMJ3E/GHKqn4wmwfAFfoUSbtCVLwnJbEEXMwj9mRZh6JygDb+UmSGlWoHR/dQwGVuy", + "o0qN+2m4UiKWoaywIudcrydW3AP+NZ9UTI4Jn0pQChWkB/TrePMZFibqHzNToGieGl0yENkcGGKNmYdu", + "VHOT8olVcK3IARndZiVxJLiUiCR0CsVw3p/WEoVXJJcV6cLrIyerg21BusZzGLjKDOxU6zpmofsxkDjz", + "OTck0DAk562ISXTGyRSzHEq8eIRsLd791nGHMy7VqACAuuFgpRpB3YZckBKWrkhvL+xB7p3gvehY222W", + "ywb43urrJaoKN9U0wGaeGlzR8Gp1CxoMkfEyBNZK1K0SxquO2XQTVLgS6J9KaJV6+GBoA5JLPLbkYb1t", + "tolGCausup8lbdWW4Xy1Nzhvi5+2Gi7tDKvZCZvwAMjGDVyUzhJtw0IzIlIKlUtQTBglsdMlC1+lNXVB", + "ZnYiCYpzYlfOyKcC2wXH5ngDUAZzNjLKpjVeX++wjXnYjGF1WQfo177YJq5IhjNX34oc1soEBkqEyxzW", + "VtGWVI7C7qzlhgWZ5gkWyCIfthmyXKQJZZdtWpeLdMwTGiH9Qd0BPeFJwq9G+pH8Aeay2Wp2+oNRUw2g", + "czM4m4BnNqTWbzmFH/QsN2vpv2CJ2TLfbwE0S5swrWBI9k80IRZG7x2j1x6hV3HP93YGTWnpDY1WEtKX", + "IRhvyrktyQZPfC4DSXwrpRxXvYjEFozeiD1ZLk0hlRa3kkM7dS7A23l0qhkan4cBcmT4dQ0BBI0JJNi4", + "qS1zjRZssc1UgjUccjlDf+fjqkG0bXxtoDLYBiuxKASZBAPpYUdXGqTNG0tr4u3uTcAegK3qicJHN8RQ", + "WFdDrQxkauInb5bKic2IXTLq5mhKi7UoleECLQqcANtre8CAeuG3QGAwwMFItYASqlC3ZuFVM5RozIUA", + "qGct4XDmZgP4Jlrm0WvtAKbQ2xlZIEFSTNmQUVYYSQG1jCBG5kR46ahcaCVrSuI++pun4gE4dpqphUVd", + "B+P5dxLxK1aMccj8QerGc6nbOWTGsihyqMpfvqSbBa1PEwqkB4MTTAkoVEjVDE0EkTN/7qHilFrGu+Ii", + "bqz6s0DuFSgmAz5WpPglYT4rK5oJqoamoZH5ajlczlSWhadW/0SVYq+oXsx1dX+5JCIsJBZTKl5pFbri", + "HRVPOTFoKwA9AkX87F+GxRdwIy3ARcrm/+qaLH86Kxqv/lZ7zQMQcXi+h8ZsGzTBRiZfphbsU/WkrQ1V", + "gXyzVbAxy74EtOFilV0llKok4FUkaXVPtkt5q0flu9FsSRJVe997tv/0ScuSMJ/lrDMwWV/aNTdPV7jk", + "GnbqtI3f59n+s+fPd/f2n+/cyMPiEjga9qcpicPfH7RBrpU+rMm//vHP96c1r88+BDsPbjQok8IRHlJD", + "Gkd1QO9P//WPf7pR3XpAIUazDMXd4LdvjNJJ/J10gQJVF147J9kK/f6wYiTABZtBG2QyIWAGHZl165WD", + "qeFttJOCcYYjqhYBRo6vTFh58UoNUrqNO6g62JDIa9q28KOac8l8XGZ3brjO0X8a33CNFp61riwl83GT", + "H/p1vVfjhS69Fn6MQ4sQA1kUNV82cBfzucKyEjmt/44gwcGlci2ntZg3VsPb1nMOIIrFFlDzQgFDsOg1", + "edJ+5G9/bTs9v2XFrFNf8Q8rzmHzEbyR1TdwIweMvtH6HNYaf7AX4O2+Go39mm8ri+pVCsSVt+7N+22R", + "prtckKC4wW7en5eZeJMP6+C7QI92DHbJy7a7FZJooCYv6SRgQOMJ6RWBejYjBcnceAT1mbd47oFUyeiS", + "TyZVUNn9ZhBywNeBrCrXC1ZKayZdRK6dzaKOYG3AdIadfTnsaBVg2NlOh52a2yqYp5ji65HtoAqiMliF", + "Cl7mmdcGKd0MxgmPLk05M6iS3UcDlBLMJMoZHP6aV217sNo71O1k3t4UGNzEhDgtsS0Y05jM8JxC6Qfr", + "U5lWAjHJNVUSAkahnQMUcwOrVKnlameoXzNZhAflpOHSwWxhG9YN6vc4cxGt5btg4JtABVn2kQjetagA", + "mmO/fn3aNQEMEHpoBlaJb3QTNSPQDLLoolbHoPw9HD88TsgIxl3HxU+X19FP/gbPqiCSKGmBsktyqBEB", + "injOVB0wP22nyFXzt5avpJxBsJ8N/wAANNu7IRAUkwhOpFw+i1VCvwVx1/IG7EqHEgd2QyQMhwJ8SWFf", + "8RvrEK4PwBgbvDLMph0/rtt4CUdScVu3qzjVI3IdERLXkTXDr7SNlbdfBmPlX2ELxlNUSLZvQ7zz8uz6", + "d5dJBWNtWm0/pp9x1gMYELelFrLDYPBZUJgqoVUwvj3siFEIxzT0QpvUZnK9eq1/JdcKgMjjPDHocmHS", + "tazKXkbrVvzWKYRNB5oLsrYO3h3UhzPx5reqEGdD1R+iSJx9604Kwy3tzjlR7t1zS0aNO1StqFJxabmA", + "f/dKNcbGkFIX2QsebaebNRLcm4WtIhb9tmUyJMMpGWWCTOj1CuIxLxjFuIofUh6kIoPBAHlupPga7T1F", + "0QwLWRs7o9OZShbVAJy9AGjRZ1VPFEQR5gyFbXa+3E334XK0m91Ov/WQcHzuYfAs1Q6xIuloFUD1Uelt", + "s9b5DC/AitPoJHy6uzcY7O4MboVQ7YZ1g+U6Kj+xtQar7TSl1HnfWUd/JUrVb6HIZl4uYHslKCRFF8sk", + "lSA4PYDEmwxHBCVkAmh0ReHw9Z7FeterB28FKov8UtC/2yi7b84HX61NU3Rlwb3dNDrOuVgF+/Gfr3GI", + "NrCZaAm7LpBzt9sbPHm7vXuw/+Rge/suUKWLRWrK9nj6cfvqabKDJ3vJs8XT37dnT6c76W5QD7ukpgRP", + "G1r9Rb/bGGVTXpJV0KAKS0Mbdg4ZEfXKxPWK3pIklJGeLDKk1qcpruAFxv++9vzfzM5vZrBSdjivTtIX", + "IbAqF6dCWQ8DdGUns9J3UZ/NyfHqWdwqA6k+kDC91YcC5NVuMFAKYrvzmRAIOWt5Db3zXmx9Ea3Milt3", + "FYU87HDSg7vcsOIh8q4hIHizXnWBL19yAdvplAuqZunq26J4rcDrhrjpj1LFVWClPjqZMihL7v9chMn5", + "SpT+uNPtJB/3qmfG/t4eYstC/RYEaLfalwpahJFB1fvVqwCvlIqHMJHsWlfXY/5hu7f9HOIQko97Pwx6", + "z6sRB12zWv7ybbu3K78O2qyhX2vP1Wjafn6jiGu3nqso6BcaqhRX3ssWBNjSeFkE2l0dLuG2ssHl46U9", + "rkHmNAqgnyvp2ctt5AtNMUnwIgQC7xlqZU179IkMjcmUMtnGbrs7KAy3++mw00eHFokbdNmy5H+leSj2", + "7tEJTVMSUy1jGtW/OYNhp6Utrq5L3KwIiPsqIK31w+La8/UQCesSrtZdk/3PyMf9LO23nca7Cr0D7GpO", + "RQWwLnixi+gEYVarBErZHCc0ton0kBgJ8WoHDhGtJFnLA2QpBzo7SRdNuUJlCn1Le1vOmu2CxfjJNdhb", + "V2BmGILY+SKAKAVSF13Fvk6OUSZ4nEdl/mgCgy4RP0Rew0JbIeSvD8m9S/sGJGZPuEDr7RtNBo129smm", + "/a7ZJjXBNm/19mD9Vt+JUaTbybN4PQ8zL7XjYDeCSF+Tghgw0VSXvSYJepP50IKjv/FXcFnnNbbkSItE", + "eeYcLJqmlikp4G4BF0MorveYJERfU8uNIJ7EZZYElSUXXc9St588mzW5OMEjtTyQXwjJtK4C+EfQX4rZ", + "IjgwV9+zuEs2Bg5WWxqHV8/UBbKrVR3c07WSWONW+SbcploFhsvXbN4GL+XSM38XYNq+aLaMgOIYfkVI", + "e9OMtW+/dGFvjfbjuzDLPaSQ9tq6Hmr4qA69t4Ahd/2XscBarKsS717IPR8ii7dWM27CfK1ngfpW58Pe", + "/xgrMxr1D7Z++Mv/3fvwn0Frc01vlkT0YjKBQKNLsuiZKj9aR+9XEU+hxIAWpqeWVAhOwYYEaOL2MPrj", + "3R8UTGPxK06XpgARWl6Jnu21E/rLfzTHN3nL+A745FqS/ewKHHdRqVRxdx1tpERMXSy5SyTb7A8Z5KZd", + "koVEXuEvK9I4Qv1OFp94EejowoiBfcLmF2hMoZKiHDKt1eIoIpnWJmwtGWrKgXPgPoLgxG/HFiBzid/W", + "IWniCQh6f7oEl/v63dsfX7/79Xj0+uzFr4cno19e/DeEeFz1TA9xT9Pe3v4TWwTcX8ntYCGKm9dT6KNT", + "G6ZvXf2THBRawOmSKM1VDkEh5DpKcknnzkGokttXTlhO1r19JYLPhNpVKglFJVhI6IROCPj14TqxQTVU", + "OmKkEqqnW+MGZWj5xjaEM+wAJ/WK34fqVuitCK92ubHVRX8ya8dCDQhp4LBDxiuUuQ9oL1QCXoWL/fBe", + "RhuQOeJKvLrE2c2bgaIeFg0GIw+/cCWfwfMvUW3z3crymnOe9LR601CSIGhNNmsRjJyHpkxGQqfJ6TAd", + "B2R4a9qd0ikO+BlC/oQvUhXTDWhtxtTS/jeWBwvnMRzX6zWYY2mWqlZfoGYkkKrXnOaQaql2VJb+rwbP", + "5MzmrlIvtq6aqJoytWWr14bwMmIOqOGrspXLU+bQEXvw0fok3JV6lTczbyTNe3Pq1IeagrNigc700lzN", + "iCDeRsAHJQ7+DZfM5uW0QGEx1f8yIsqYVZfUo6VScDdLtFFYftwSFNnGy+bw1XUOTvF10QO4UrBc8j/C", + "PMo6S9svfwRM+jeutiSduCZgGDXlLozAXqWiVWviqGp5M3yqWp63eT948CyvWsH9ms5WjTjLPiqkGaLH", + "v2GqfuIC1MFmzJM7B3KHyz8mAjDg6jDtrTDOaUriEc/V6vNvS9fbK7+oP1rWr3WqLwYijirpvE28wKFy", + "lGNYXmm9HCTKBVWLc71eNpgb0iBd0VhYSOgIfi47hkKdnz6B0XgSSBh5SRgRNIIyqPo8ppiBxoTen3rV", + "8ExhxCW8VhCBXh+dWHODg/wF9ZEqID0Xd3l4dtLpduZEGJW7M+jv9gdwmDPCcEY7B53d/nZ/0AGtagZT", + "3ILS9TZ/2uYbF4rrSWwloR/dS/pLgVOi4IvfAkgAEHdoXwcVBE89JTLDVFgtMksAocAQDNVfA66/u1AP", + "zK3cNcve2mYKacaQ/UKy13ZzP4CgDGcHprkzGFhgc2WvX8jdMQkDW3+30aNlv62kOrtEAZj7JTXPyZbF", + "0n/qdvYG2zca06qhwNkNdfyOYZvES0A737/hQtyq0xNm0vJskrUNh/JPHBCSf9Z++6D3TOZpisXCLZi/", + "WhmXTYIxkQi7d40epySKNKuAYjx99JoR8xxhhbCJXBY5gxrG7kNNodVTYNp2m1yAFP3I48UXW8JKH85G", + "8anKzvRx+bREz1+OdgoyXt5I+8ghbBuqvQcC+hEXBbgf7KTsDZ7ffadHnE0SGinUKwjYxiNTCSE/CeCF", + "O+whLtDvOVcYFeH8j+hIW5l1XJBbt7yKtv6g8SdzvBMSMoOfEZFiZpIjzDtrDv3ScTYuifI4r7zVHOFD", + "aQ+4qRwIj7moQJCrHlH/2qoLg8vX0V4AgcH2aaYXPyDh793DCbeTLWqwPuSRg8qXKJfkMR0n62Ibl0JI", + "UJZ7SdTXQvOD+7yybBGBP+EpeiwE/JIUEl65W0uXwlYmcmYU4KAE+KZMWLTffVcV/t6WT7woGfBr6Kah", + "nIUyflUcL/rIralR+tUCIJYEgXnGy9fKmR7e13LCdu7jhMGMC0/Rt2vq2zW16pQbanFTgIPpnfIWNogb", + "WSD+fPaHG1sfvtke2tseWlkeGLmy1oW/83Ef2YjUiMcEyRnPkxiNCTJ4Ry72RGHRn35EWEQzOicAagdF", + "2vJE0QwLiCxJUYwVNj70RsPESrNE0dyWbq7n4hDLBa7jWEgyAhy+URP+ZBmBSBkjMdKfWOi+Ek5wqW63", + "OftBA3vRYHk1oqsZl6TA82PKu80hvVka7Ria7Q/ZWwv0qhcQgqkdr5EkAbjaFfYfzhAeMvvB946FuEAw", + "idOSc2EBmIHUIFOabVlObdMjHcmIh7B23hKGmerJjER0QiM7rUuysPGcwQZb1V3SA3bjfH9aJGygnc0w", + "XhvAM4bBeY+LZ8hSUtV/wyAIOkryuHRyOQghLMY4SYKFOaYJH+NkZNbnkgR8gi/hDbsofn1/501iPCam", + "Vnu2UDPOzN/5OGcqN3+PBb+SRAw7m/0hg0QMu9Yk7pYCIrqCQm5pxvU5Ezw1fW6ZIW79cUkWn/pDdhin", + "lDmKgE9wIjki1/Ad1LcCzAzDvRrowZymsB/8KJeKpz7yqaM7M0yeqyxXNqNEEtUNoX4OmeLoD4ft+Gnr", + "j7LHT+AsJjjWdOK9YqYEsnXTqOUI69mP4NWAu53AAgw7+iI1YR5TgZkysJ0FOCWa+lu6UVRH0Id0s77C", + "EWYo45mpLAFENcOa5CptAFYDThKk4Ci5b7XgDjvZMB8LvZeOG3H3DFBa7RhRhk5/9A7TYO9Z+DxJEgkS", + "iij5r/PXvyK4lfUemNfKcC2T0sG0wIDiHFynjqe9wNEMGUcVFBMcdmg87BTu3HgTxppLGy7T64FP8Qc9", + "tB9MN10a/9Dv66aMu/IA/faHaeVAn6UsNTigw86nLvIeTKma5ePi2YfwgjbBl51XGAHaMNfcJnASTAFp", + "xrvxzRWJWYy4vQWSBcKo5EB+4MqYMiwWqxIJA0tvV5BPTCSjtxh/DCFycdg5GLrYxWGnO+wQNoffbIDj", + "sPMpvALWa9lcuQ7us8K5WRDRk8Fgcz0Stl3fgM+yhWPgC+uAjVpRUXZT76CFYf1z+Qf+rfXPwvWDme68", + "hCYyir8zvj9CB4QnsfuaaMAFURO7MYtI4sTu9Yae+3ce6M2KSJLcN4E+FHkW7rECqf9RkSNsVnmMVprv", + "H5jiBvd1qVTM9g9Dv4/Ofh6wnlvbOZm7UOdwnRLAoLGqNDIvIyzROYypd66V7xfwa9/+1+l+gKl4kfDp", + "xYFR3VHCpyihzOYDeIHKWjywawkfGRia4juLSuOKxG0YSeJf//gnDIqy6b/+8U+L7f6vf/wTjvuWgVeD", + "GtMXM4KFGhOsLg7QL4RkPZzQOXGTgSqwZE7EAu0OrM0fHiGv1L2V0uSQDdkbonLBvLwJU69N2gatq0DP", + "h7KcSAvjo1+kE1tMxsQ2Buw27iybpbzXE90NwCHCDLwJ6FvR0QBgyVFTaNtqop2wydTMuWI0rYdpLgXr", + "recvilwrQ709M8AbMhhY4tC5gwd20mjj/PzFZh+BtmWoAgoGge5QNmPViP43nrSeJxmOUmUosMqGN0U4", + "w2OaUGdybKh2Yo5giqMZZaSMLy6wxl0TB26kmsccnp0gGwjZhVeH7PX5FphYFYlULkjXcgJhEUbLcmjc", + "5rlAD8C/qILosJ59d8gmBEOe0MmxYQIeCHeRD1g0zADIA2JcqapUXusOmUGStcjF+uClPCYJfAT9T7Ei", + "V3jRRUWtW1cdJcFKK8Syq18eMoP1ategB1AlyBtmH/iZGVLPRfLanC1BJolWjSEC35T9hr43JlwgG+Hs", + "Vfl33ZkkSzMsvWgpjl6f6/lNQRPkxh4ILb0+d7ux2UWSoyihQA0RZkM2hUAgB97LWWVXi4SyGRZxL+L6", + "EvDBnC4Zv0pIPG3isUc+kd2hJFPpJ3Ccfq6T62MTLmbLE9CH2ADUrfbcHdt32rnubIt/Jt+dLQR5A+ed", + "seASw2/M6n5z5LVw5IXXzTn1Qp61Y4fAeHcRv6aLBwr4dbS3vObmibdkD2HRQxsO2ga8Ilygs6MThONY", + "ECk3/73tfXqmhkpL+U/fj5oVP0ToiR0LFxb0z9pbqgTyWNjBGztqhN286vV1/fttq1J8p/GmK+rwlFfe", + "3d8etU5vco2UQm9Ja99ukrXBtlRGHMoMltTSA9EoIYX4UpxTn4rWWZVNGG9x5awUlyx7Pjl2B/L+7Mu2", + "65zV74Z7YIrHNYb4gIywmmrtV81+TNT8rthFhza9wvz8dZHm4P6koPs2RYfI/DGpi3Ft2TQXNEAnjRfo", + "S6IMvMld6um2h8DEz4lwp9oMdGFmXUzLfIoMTgtMCCwxq3XfE/NKO9XXtPdn0nxheW4isdgl/yaitFB2", + "y7VapeCe2BLQd6ffQg83Um+/XNiKJbDAIoMVdezcTmBZ3cBywaLNb5ErX5yiTVxjqcQKN28SF5Zsg6ZU", + "6Fn3JdcdMr/euJbprF5LGZokdDqzToCYTiBWT/n1u2GUO/cwyqJOtsCK2BDFx5j3e6YX2XqB50Qo9Pro", + "xKy/f6Vu/QFBq+tVJce8Vt6u79686hEW8bhwnjTLpPbJF1aYDP1Xcnnv/9Q9wnxW6sSDJoHxM/bfBJMj", + "E//ep/x/7fyU0LHAYvG/dn7CSUYZ+V+7hwlWRKrNOyOWwX3ddPetwDxi4tP6C60uGrAmNgXI2DUCf/FW", + "S5nfvf+nEvvNpG8k+Bfr+k32byP7+8u1Uvy3W3GnCoDp44E8XAWxhVYbHn2DtLkHo6mlSA/SpuJFKkFt", + "ZlwqePT48pttUDktKM6/Nlpa/8sDufL6cKR7ctyFhYSK0lDRwqYP3pMvwI3j3oVb2+/9OwIO0zGd5jyX", + "fmZiilU0I9Jm7SakyoAfm9hdXs+NgvdXTKWD+7w67l2u/kb3dyTx1zfUMG/j0Fsn87u32sr89n0t8xtE", + "U5vZbMtudF1Jps2GQGuHadqWjCvQr8sB4KFxhXQR9E4rKqW6gECDOBiy/631j98UwemHH1wKZT4Y7DyB", + "3wmbf/jBZVGyU0cqhClBbQW9w1+PwYs6hUBZKLJXJmzXx2FqdgPpubIC/3YKUulIbq8hOSr8piG10pC8", + "5VqtIdm9uFsVqVqa5N51JEdvoQW3mOJ/Ti3pT+4eqWhwMp9MaEQJgwIvkJgul+IBjSb3zTNyy4RkZv2R", + "XjBRRRJprUYWXGuNhF7WlP6S0TrdRpx3jrBSJM0UmgockUmemMoISM5yFfMr5mDfYYKughAt5xO63l1T", + "I9dIOAktXP23raZbVPy6b1XX1dp+nFlgPLPFa61yWYo2zdrlwxLv3eqULa7a+9cqHzOJGfVteekyrSEE", + "yhiZAlZpblLmii9LBLQ+evv2lUuP0+qJcEWxFHeVsFyR0CHzK2H10YuyxJh5wbWg1QcS23RaSBq0taVi", + "guOEMgLxxESGMtmq9ese9Fh8eQk4XJyvlQR8z8fSllt9OAn4wVjBvciaJ5Uq1rw0SPh1+4rT4uRNODWP", + "il9ZBhRgPCFZbwvnivdswu3WjBsUtjAQ5VmCI8Ch1K8ZiDSLcWAwEf2mALhA8CQhwkDfZbly4taQFYOj", + "zCtIbyWzC938KGeKJhddE84D+CUSYbaw+E9DVunMynyQhww59jBCQTIz4lqlSj1oynMJb0HKsN8lwskV", + "Xsghs5nL5nOo6itIZFAik6SPfuYAGoHwFFPmMV5TLvE7OWQXNE7IyGI+XCAqkZxxoQgjMUr5nMhqvwSL", + "hBIBkzjCeuUkSvECwNcMDqVZH54RA3BWQZbg+t+YxRQK7+meiykfDBlGO4MBSglm0uaJSzyBC8e2gWAQ", + "lQF9jzDaGzy3X9X2DQCC3fJv6NMkBJnzCI+TBSKaigGpQm3CBqa2EKYpKKy3b0KFNPtV2DdthbPKxlLp", + "6jrGXZSzMhMebP05KxLX9XapXDCYp/UCEiqKa9CCf4xJhPV6Ml7tB2AXeRTlInRB6q32KrL+OwqO3vTO", + "YanCeeYJmAwiEsOeM65mcKY5HKXN7xuoqiSqP8dFEzwkXCCMPLouLRokyoE1bgBM4UVZXpC5csEXm9+7", + "s6OPr2UE7vgboMDHcj8BEfHJpHIA119N5gCvyu9YJuE/6zk9cnVlfRYXUzxlXCoaOWZYL0P/TSFsrRCu", + "XtkgNU+4uPRlqyr9/sTFZVsNzIKf0seliPkz/AodEXp4ADT98P4IsIYbZUUTzb0raXX6Kk4pCF1USRfo", + "zFHC2VSfotIqf+9uA1+r2zCgcfoyFcbZXUD8aCVkZH80pWn1ZGzhT3AxRLbVh+ZFuvd7cEb9yhWiaZaQ", + "lEDp2p4hNr3ZJRwUlPmn0gNFuhmv1KfKz102uqA08QddJw4BXbkN2wDpfXm7gkw14dP1oINF5w5hL4A6", + "OGTvpIEDvzCupwtU8GAt0BqIf3Q1o9EMEAhBb9XtG4BCnGUXBfjy5gF6CQfZx6CGzjcMsL+mNckTYoAF", + "52l6cbBcnPX96Sl8ZMAHTRnWiwPkCrIW94fUb/mIgnoWCZYK/WpxEjcKZRx29EJhrW8W89u0WIMlOPaQ", + "hXAHGbmyDdIJuvAgCC8a8LEcv33Fp/KrcRWVJQ3MXBRHVnUE2iQs7jQFedAk7PjZHgxCSNstkRDNMO4Y", + "CHFpMK/4tCinUCFlnGVtydcOE6h4nqYraBhteLBqUsU8V3+RKiZCwMeWupuIG23gyJbSwpeaUC2InjvY", + "m0B+wVAmg28eXCrNVDvdDmF52jn4zf5rnqadbseOx8NFv4FwvwZRst7gcsiN3hkPNvKbWH4TQMgqs/cQ", + "IWs3h1WnmyXyN+aFP7230NnsHpAMQT6oGXG/JhHUG2/V4MN4gWwJI3t+HyMD+EsUJVySioPn8YBnWUNX", + "TWZsNhS5Ne7p4cW5qzbUJoLl3H567r78CnTvdbEibszITffeg0aWR/CYE4Hl0mwmXNQRl9ZFk3z1hPTl", + "tmRpqm0o5Btt3tzK2IowtZ6wzCLsB7GpPodzxVOsaASVj6IZ59Ij+wIe2dQos8bjgjLBtGK0XJtBcKFJ", + "9cKaoS+sGnFgTWYI+49sH3343OYdhL9wj8ovfvKsAgXH7zrRH6oDQGl2QckEZTiXREt1eUpQtIg0VzSl", + "rgiOZijCmcoFgSp+BKWU0TRPfdxrvWNzDBgdF9vpRReNc4USLKaglZmHLtgm4mlKWEzAPjdkM4LnVKuU", + "AiVYERYtepJA9d85QVdcXCYcx2BiyGIMnh6oHiiIpkAAEU+JwjFWGASdC33iRyaJ6aIoCGzUekauS2qI", + "h0zk7HtT0UA3e+EGeoEIQHZTOSsKR0Y4JiwKQlmff91s7Mvbos+Jqk/0gSKDbsVLHzJUyLe5uuF8HVFE", + "jywWmwu7jW3Y/AqhVzarsNXsD0dG/55H2szVzfGBHEzFEq86xV+HZ6kguq/Gu/Tw7iMuUJyb7rxTCWT+", + "Z/UJFQzFD7aCzFKzjbd1DBUV8oplvhHP2/rD/XlyC1veV8IJu42KfVMtpnLSXwPLtat6K577QEZMa0vy", + "bXIPx4JdRNeDiU9ceFzusRhbLcM2R7Pg2z53UgKD9sXZN7ZdZ9s24OG2bNvZZpdc+h4jp6wHMaJhDm7N", + "uI2s2poO/k2zUWqz81jmg7PI0nNxb2zxpGCEhjVmeJFwHP8ZgoRX+I8iLoSBvwBAjccEv+pZDf30ALDN", + "lUXeui5b8/3p6WYTlxBqJY8Q6hFzCC8lR3+WxssG3NdzIgSNLUopOjo9tuG6VCKRsz56nVKFFEeXhGRl", + "RgtkFfb1/BwQyHJB+QriR7dDmBKLjFOm1o6ifPVuBvPpVmXo75lPWjzvb+7w1u5wsOw/PnYGXAZyNswE", + "VmumCqu1dUYpm3CRGrkMj3muW9c8SC+T3k+DVDChCZELqUhqohIneQLHDWpD2Pq/9juzy12IydUnx6TL", + "ZUSkVErKmRwymyuSEaH71p/r9r0Aq6BDQOGCv54ZJvl1BO/pwZh4NayaVg0gm6CuaOegs4WzbCvGCjcE", + "iNnhfcaQfoJoPCQX6ZgnNEIJZZcSbST00qgnaC5Rov/YXBnON4LvvnR149ufLL3SJ2zCg7XjDM0WxPyn", + "yuqybM05Jh8dW3tJ/MPi+A9sdJitra+fLAhOelCP2AH3oFzRhH40rE43QqWikUk5wsXavT8tmGp/yE6J", + "EvodDKltSWIQDUC73MoEj7aG+WCwG2UU0N92CQwOGF7z4xR6PDp7Z9JQScrFojtk+h/Q8NvDM+PdnWBr", + "TfAGagsno5Ot12sCnM9hmf6NIwTNBFeiFwQ3/JtL8OYYI41nSDYcUZ6tUpV49qcPYbUS3De7wuO0KwDI", + "UzGbjQLYy6FxhW0Ic57kqf6H+eNkHa6ZwtHsPbz61Ui7Zjhru3ETfBSH0s4pJqa25YM4PcyCPdaYVb1w", + "bgogxFSiAYO3wKH6M1L3lzff++v4Fbo77Yq6urFfzdm675vPjsEhbPjr8ViOuaE0NxPFV1ufrjBttj79", + "mPDoUlooFt9sqPU2wFfXP5Z42NZFCGICZIYiC2FkgLKI7A5ZzQBpEH8kwkgRkVKGky2Ys2kEkL2dFQvP", + "OYUE7QjyVHqSxoCZlAB8N8Df6dmAoco14Hl0pa2s5b/jOyMVR2MS8ZQ4tPPNkOr2N0zVT1xUocu/Fr74", + "1lt/gATEFOzta9Dam3v8LPT2U3wNodJxbh3KbkQbL3n5ozEFdRHszbCzO5DDThcNOzvpsKN34AiDCRUr", + "tI9SynJFZB8dG/sWpOA+GSBJIs5i6UDXnQVvdyCbEnINWTZkdz6B7+5T7LFUBUv5xnYSYg/6PaS/h6Qd", + "tOEfOHsm4y4cuhjxXBlzvz1X9q2YKDCPbN67r9Y7I990+zac/G/2+FZ4FOyyZpfe1hvOnuVyRppNbq9M", + "IaNcjQHM2xUXlTP0dz6WXcTIlbGGC6n6S3xPf31mOriPQgO6q5sUGbBz/1ZhoEWFgXKtwmCNJsBSX8mO", + "OgxiI7nOuFCA4mhz7Q0NgSYByBE8wgl6fXQyZJFmRQZaUJCUA3eyeOjmFj782zl6cfSmi46h0CX6OR9v", + "9tFrlixcuXHjoxkyI4kZ5hVhhsaGakkcup7N2IF67jJYXHfwQJWjzckIeFbcXrkg8W5nRnAMEskfnVfc", + "dBZAHX7zSh8gAP41Xxbb3lkpfHTeECUWvcOJImK52VObJ8UKzAx7STsIOiu4GeBL3aF0yGtln0Y2MNAY", + "uzudAFLGp29FH+6+QOr9eMlMnIgptzfOAWmUQZIBjhePK5ZJzlDBHEMs0L+ui7IJTVnClpetVDCgy6bI", + "76/I5L6Sd1Ww5f9dTxfM9NE6mrLKPmkiLsqtrPX0uuTgmYFDto6qCGc4omrRRThJ7B1lb4IiIqVXiL9j", + "QfBlzK9Yf8jeFIVebEIvOjp713WOWhRTeWlasL7YPno9J0Lm42JwCA6a8RrDmpN4yBRHEU6iPNHiBplM", + "SAS5uFC/RTb4couhdO7w7JSdBIvNeFHt+aOrcRemCdi9kizqFLdltnpLkCjBNG0GH7eCGgQcQqjBWDfK", + "GaJsktiQqkhwKZFtqkcSOqXjxAYIyT56OyNI4pQMWZZgxohAuTRR8XrovUwQKXOT4K0bAJBeQ1FdVAIL", + "ZoIrG5qQcC6kiSbQFP7+FElFshVk9sa0fApzviPZ1jRue3ogI3VtDM2mEPsK0htiKMUsuKajPHEBjPca", + "im4G9NBS4mM5+G8FnU6J0KcCGyZrwvHMsXbLaQ59JWO5sd7lefFWu3qXRateVqKXsbcSGG5UYm3HnZtF", + "/QU6v6SN2IH20c2yiH/RH7Xsu5qtGh6EffSZswyV7vx3rJJ57iUJtjVglRT+2MxJ3sgrR7WSaLseVqt1", + "Zu1dZrq2xs96MNisx4yWhSvps00K79dHCIP7RXm47yJrj5u2KmhXFd20IeV/PZr+V0GBdwOj/8AoJ7eA", + "0f+q8u4B5/zh8E+CB/Wh8ugrvmdXbPdPj4R/V+nzBg4f4Nia0ucN17PBqysVpff2nXZqkm3xzyTB23jH", + "G8jvbtm/af0tVAZvsda5oDXBkzRTCxfQZn2VZdCZpB9Jv8ERXMSt3p0r+BYhnV+OPBydNgZ0/jlr4z9I", + "zKgtHUglOjkOFJ1/ZBiD/pmrXCxb+tbpYRHN6Jw0G92rJ9guUSZIL+MZOFdis2B2PdxdprDoTz8i27zF", + "XLX/gtqTANVPYhRTQSKVLEwdUM0RTB/fSSS41gTgOReL5igRc0R+Ejw9tLNZcx/aM2WNYWWcYbroxVjh", + "3txxmxUmtM+I7nTxlJrhIcrQyx/RBrlWwlS4QBOt+SA6KZaUXEeExBJoctMf8PagwbJJP5LRdNxmlCtq", + "lby2tWBQlEvFU7f3J8doA2qfTQnTe6FF/QlIspngcxqTuDLGzpwnZlW3Gxb0pnZXLVQUheuccmEG9yAy", + "TJsLafqRZlW2UITEjCnDMLi1VUGqZ8ok8ev+MGUuAMfukRvFtyvMan4bTtnRlAh1OO0iKs4NxPPmt2vu", + "MV9zfjKUu9Mqt50Lz1ltvG6XH9UybekuCj8UuXP3a7Z+//Wk9FD5KLN5rOl8XiikTWbzr4sEB/d3P9y3", + "ufz9I04BfUmc8u2ZyqEB3WKIYF5BTHdM5iThWQr10OHdTreTi6Rz0JkplR1sbUHs94xLdbD3/Olu59OH", + "T/9/AAAA//9gew1xCvABAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/resources/monitoring.go b/lib/resources/monitoring.go index 2e7eeea4e..d69b1e2a4 100644 --- a/lib/resources/monitoring.go +++ b/lib/resources/monitoring.go @@ -199,7 +199,7 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { gpuProfileSlots, err := meter.Int64ObservableGauge( "hypeman_resources_gpu_profile_slots", - metric.WithDescription("Estimated concurrently creatable instances per vGPU profile (best-effort snapshot)"), + metric.WithDescription("Virtual functions able to create each vGPU profile (best-effort snapshot)"), ) if err != nil { return err diff --git a/openapi.yaml b/openapi.yaml index 5ebb29556..fa0e1381f 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1775,7 +1775,7 @@ components: example: 1024 available: type: integer - description: "Conservative estimate of instances concurrently creatable with this profile across all GPUs, bounded per GPU by free virtual functions and remaining framebuffer. Best-effort: recomputed from driver state on each query." + description: "Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer." example: 59 PassthroughDevice: From 34b2ad81170f8e6c568218ee5590ed837da4205c Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:23:13 +0000 Subject: [PATCH 016/107] Retain vendor vGPU assignments after rollback failure --- lib/devices/types.go | 10 ++++++++++ lib/devices/vendor_vfio_linux.go | 27 ++++++++++++++++----------- lib/devices/vendor_vfio_linux_test.go | 18 +++++++++++++++--- 3 files changed, 41 insertions(+), 14 deletions(-) diff --git a/lib/devices/types.go b/lib/devices/types.go index c76d239ed..31ebd90ef 100644 --- a/lib/devices/types.go +++ b/lib/devices/types.go @@ -100,6 +100,16 @@ type VGPUDevice struct { MdevUUID string } +// VGPUCreateCleanupPendingError reports a failed create whose assignment could +// not be rolled back. Device identifies the assignment that still needs release. +type VGPUCreateCleanupPendingError struct { + Device VGPUDevice + Err error +} + +func (e *VGPUCreateCleanupPendingError) Error() string { return e.Err.Error() } +func (e *VGPUCreateCleanupPendingError) Unwrap() error { return e.Err } + // MdevDevice represents an active mediated device (vGPU instance) type MdevDevice struct { UUID string `json:"uuid"` // e.g., "aa618089-8b16-4d01-a136-25a0f3c73123" diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index ceab85ec5..6615e035c 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -159,14 +159,21 @@ func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID str if err := os.WriteFile(currentTypePath, []byte(requested.TypeName), 0200); err != nil { return nil, fmt.Errorf("create vGPU on VF %s: %w", targetVF, err) } + device := VGPUDevice{ + Framework: VGPUFrameworkVendorVFIO, + VFAddress: targetVF, + ProfileType: requested.TypeName, + ProfileName: profileName, + SysfsPath: filepath.Join(s.pciDevicesPath, targetVF), + } currentType, err := readCurrentVGPUType(currentTypePath) if err != nil { verifyErr := fmt.Errorf("verify vGPU on VF %s: %w", targetVF, err) - return nil, rollbackVendorVFIOCreate(currentTypePath, targetVF, verifyErr) + return nil, s.rollbackCreate(currentTypePath, targetVF, instanceID, device, verifyErr) } if currentType != requested.TypeName { verifyErr := fmt.Errorf("verify vGPU on VF %s: type is %s, want %s", targetVF, currentType, requested.TypeName) - return nil, rollbackVendorVFIOCreate(currentTypePath, targetVF, verifyErr) + return nil, s.rollbackCreate(currentTypePath, targetVF, instanceID, device, verifyErr) } s.owners[targetVF] = instanceID @@ -175,13 +182,7 @@ func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID str "vf", targetVF, "instance_id", instanceID, ) - return &VGPUDevice{ - Framework: VGPUFrameworkVendorVFIO, - VFAddress: targetVF, - ProfileType: requested.TypeName, - ProfileName: profileName, - SysfsPath: filepath.Join(s.pciDevicesPath, targetVF), - }, nil + return &device, nil } func (s vendorVFIOSysfs) destroy(ctx context.Context, vfAddress, instanceID string) error { @@ -475,9 +476,13 @@ func framebufferFromProfileName(name string) int { return gb * 1024 } -func rollbackVendorVFIOCreate(currentTypePath, vfAddress string, verifyErr error) error { +func (s vendorVFIOSysfs) rollbackCreate(currentTypePath, vfAddress, instanceID string, device VGPUDevice, verifyErr error) error { if err := os.WriteFile(currentTypePath, []byte("0"), 0200); err != nil { - return errors.Join(verifyErr, fmt.Errorf("roll back vGPU on VF %s: %w", vfAddress, err)) + s.owners[vfAddress] = instanceID + return &VGPUCreateCleanupPendingError{ + Device: device, + Err: errors.Join(verifyErr, fmt.Errorf("roll back vGPU on VF %s: %w", vfAddress, err)), + } } return verifyErr } diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 4555f5c2a..b45884640 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -428,21 +428,33 @@ func TestRollbackVendorVFIOCreate(t *testing.T) { t.Parallel() verifyErr := errors.New("verification failed") + device := VGPUDevice{ + Framework: VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } t.Run("preserves verification error", func(t *testing.T) { currentTypePath := filepath.Join(t.TempDir(), "current_vgpu_type") require.NoError(t, os.WriteFile(currentTypePath, []byte("1148"), 0644)) + sysfs := vendorVFIOSysfs{owners: make(map[string]string)} - err := rollbackVendorVFIOCreate(currentTypePath, "0000:82:00.4", verifyErr) + err := sysfs.rollbackCreate(currentTypePath, device.VFAddress, "instance-1", device, verifyErr) require.ErrorIs(t, err, verifyErr) assertFileValue(t, currentTypePath, "0") + assert.Empty(t, sysfs.owners) }) - t.Run("surfaces rollback error", func(t *testing.T) { + t.Run("retains assignment when rollback fails", func(t *testing.T) { currentTypePath := filepath.Join(t.TempDir(), "missing", "current_vgpu_type") + sysfs := vendorVFIOSysfs{owners: make(map[string]string)} - err := rollbackVendorVFIOCreate(currentTypePath, "0000:82:00.4", verifyErr) + err := sysfs.rollbackCreate(currentTypePath, device.VFAddress, "instance-1", device, verifyErr) require.ErrorIs(t, err, verifyErr) assert.ErrorContains(t, err, "roll back vGPU on VF 0000:82:00.4") + var pending *VGPUCreateCleanupPendingError + require.ErrorAs(t, err, &pending) + assert.Equal(t, device, pending.Device) + assert.Equal(t, "instance-1", sysfs.owners[device.VFAddress]) }) } From 75020cd1f223859910b4bdd71459b8bb1375ac9f Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:39:33 +0000 Subject: [PATCH 017/107] Degrade vendor VFIO discovery per VF instead of failing the host A single unreadable current_vgpu_type failed discoverVFs wholesale, and GetGPUStatus turns a discovery error into a host with no GPU, so one flaky sysfs read blanked out the host's entire GPU capacity for admission and monitoring. Skip unreadable VFs with a warning and keep the readable inventory: a skipped VF is never selected for placement and never reconciled, both safe directions. When no VF is readable, discovery still fails so a wholesale sysfs outage cannot demote a vGPU host to passthrough while assignments exist. Also document that vendor VFIO vGPUs are known broken on Cloud Hypervisor upstream and QEMU is the required hypervisor for GPU instances. --- lib/devices/GPU.md | 7 +++++++ lib/devices/vendor_vfio_linux.go | 19 ++++++++++++++++-- lib/devices/vendor_vfio_linux_test.go | 28 +++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index cdd269dc8..0cba075fa 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -99,6 +99,13 @@ Instance Stop/Delete → Release profile → VF available again Hypeman reconciles orphaned assignments on server restart while preserving devices held open by a running VMM. +### Hypervisor Support + +Hypervisor selection for vGPU instances is caller policy; hypeman does not enforce it. In practice **QEMU is the only hypervisor with working vGPU support**: + +- **QEMU**: fully supported and validated on both mdev and vendor VFIO hosts. +- **Cloud Hypervisor**: vendor VFIO vGPUs are known broken upstream ([cloud-hypervisor#7572](https://github.com/cloud-hypervisor/cloud-hypervisor/issues/7572)) — the VM boots and the VF attaches, but VFIO region reads fail, the guest driver cannot initialize, and the vGPU is non-functional. Do not place vGPU instances on Cloud Hypervisor. + ## Passthrough Mode Passthrough mode assigns entire physical GPUs to instances via VFIO. diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 6615e035c..bdd026ea8 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -6,6 +6,7 @@ import ( "context" "errors" "fmt" + "log/slog" "os" "path/filepath" "sort" @@ -50,6 +51,7 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { } vfs := make([]VirtualFunction, 0) + var vfErrs []error for _, entry := range entries { vfPath := filepath.Join(s.pciDevicesPath, entry.Name()) nvidiaPath := filepath.Join(vfPath, "nvidia") @@ -57,12 +59,14 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { if os.IsNotExist(err) { continue } - return nil, fmt.Errorf("stat creatable vGPU types for VF %s: %w", entry.Name(), err) + vfErrs = append(vfErrs, fmt.Errorf("stat creatable vGPU types for VF %s: %w", entry.Name(), err)) + continue } currentType, err := readCurrentVGPUType(filepath.Join(nvidiaPath, "current_vgpu_type")) if err != nil { - return nil, fmt.Errorf("read current vGPU type for VF %s: %w", entry.Name(), err) + vfErrs = append(vfErrs, fmt.Errorf("read current vGPU type for VF %s: %w", entry.Name(), err)) + continue } parentGPU := "" @@ -76,6 +80,17 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { ProfileType: currentType, }) } + if len(vfErrs) > 0 { + // One unreadable VF must not blank out the host's GPU capacity: skip + // it and keep the readable inventory. A skipped VF is never selected + // for placement and never reconciled, both safe directions. Only when + // no VF is readable does discovery fail, so a wholesale sysfs outage + // cannot demote a vGPU host to passthrough while assignments exist. + if len(vfs) == 0 { + return nil, errors.Join(vfErrs...) + } + slog.Default().Warn("skipping unreadable vendor VFIO VFs", "error", errors.Join(vfErrs...)) + } sort.Slice(vfs, func(i, j int) bool { return vfs[i].PCIAddress < vfs[j].PCIAddress }) return vfs, nil diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index b45884640..0879af3e7 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -29,6 +29,34 @@ func TestParseCreatableVGPUTypes(t *testing.T) { assert.Equal(t, profileMetadata{TypeName: "1159", Name: "NVIDIA L40S-48Q", FramebufferMB: 48 * 1024}, profiles[2]) } +func TestVendorVFIODiscoverSkipsUnreadableVF(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + // Replace one VF's current_vgpu_type with a directory so reading it fails. + badType := filepath.Join(sysfs.pciDevicesPath, "0000:82:00.5", "nvidia", "current_vgpu_type") + require.NoError(t, os.Remove(badType)) + require.NoError(t, os.Mkdir(badType, 0755)) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + require.Len(t, vfs, 1) + assert.Equal(t, "0000:82:00.4", vfs[0].PCIAddress) +} + +func TestVendorVFIODiscoverFailsWhenNoVFIsReadable(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + badType := filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type") + require.NoError(t, os.Remove(badType)) + require.NoError(t, os.Mkdir(badType, 0755)) + + // With every VF unreadable, discovery must fail rather than report an + // empty inventory that would demote the host to passthrough. + _, err := sysfs.discoverVFs() + require.Error(t, err) +} + func TestVendorVFIOCreateAndDestroy(t *testing.T) { t.Parallel() From b366bb721e3b79199276e3eb396c6dc9051b5167 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:10:26 +0000 Subject: [PATCH 018/107] Degrade listProfiles per VF and document deliberate probe strictness listProfiles failed wholesale when one VF's creatable_vgpu_types read failed, blanking every advertised profile while discoverVFs directly above it already skips unreadable VFs for exactly that reason. Skip and warn instead; underreporting is the safe direction for status and admission. Also document why openVFIOPaths stays strict where mdev's scan is lax (it authorizes clearing a reused VF path), and the 0Q/0B parsing caveat in framebufferFromProfileName. --- lib/devices/vendor_vfio_linux.go | 15 ++++++++++++++- lib/devices/vendor_vfio_linux_test.go | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index bdd026ea8..dbb9ed1f7 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -106,7 +106,11 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro for _, vf := range vfs { creatable, err := s.readCreatableProfiles(vf.PCIAddress) if err != nil { - return nil, err + // Mirror discoverVFs: one unreadable VF must not blank the host's + // advertised capacity. Skipping only underreports availability, + // the safe direction for status and admission. + slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) + continue } for _, profile := range creatable { profilesByType[profile.TypeName] = profile @@ -411,6 +415,11 @@ func (s vendorVFIOSysfs) vfioDeviceInUse(vfAddress string, openPaths map[string] return false, nil } +// openVFIOPaths hard-fails on any unreadable /proc entry, unlike mdev's +// isVFIOGroupInUse which skips them. That is deliberate: this scan authorizes +// clearing current_vgpu_type on a VF path that is reused across assignments, +// so an incomplete scan must fail the release (which retains metadata for a +// later retry) rather than risk a false "not in use" answer. func (s vendorVFIOSysfs) openVFIOPaths() (map[string]struct{}, error) { processes, err := os.ReadDir(s.procPath) if err != nil { @@ -475,6 +484,10 @@ func parseCreatableVGPUTypes(value string) ([]profileMetadata, error) { return profiles, nil } +// framebufferFromProfileName parses the framebuffer size from names like +// "NVIDIA L40S-12Q". NVIDIA's 512 MB 0Q/0B profiles parse as 0 and would read +// as free VRAM in least-loaded placement; no supported GPU exposes them today, +// so placement only skews if such a profile ever appears. func framebufferFromProfileName(name string) int { series := strings.LastIndexAny(name, "ABCQ") if series <= 0 { diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 0879af3e7..50e4d9f00 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -57,6 +57,24 @@ func TestVendorVFIODiscoverFailsWhenNoVFIsReadable(t *testing.T) { require.Error(t, err) } +func TestVendorVFIOListProfilesSkipsUnreadableVF(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + badCreatable := filepath.Join(sysfs.pciDevicesPath, "0000:82:00.5", "nvidia", "creatable_vgpu_types") + require.NoError(t, os.Remove(badCreatable)) + require.NoError(t, os.Mkdir(badCreatable, 0755)) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + require.NotEmpty(t, profiles, "readable VFs must keep advertising capacity") + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-1Q"), + "only the readable VF counts toward availability") +} + func TestVendorVFIOCreateAndDestroy(t *testing.T) { t.Parallel() From fe5d9b543d1c65326f2bc4d5c4cba3e3abd6bf24 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:20:06 +0000 Subject: [PATCH 019/107] Skip unreadable VFs in create placement like listProfiles does listProfiles skips an unreadable VF but create still failed placement wholesale when profileMetadata or selectLeastLoadedVF hit the same VF, so /resources could advertise capacity a create then failed to use. Skip the VF in both loops; it simply stops being a placement candidate. --- lib/devices/vendor_vfio_linux.go | 9 +++++++-- lib/devices/vendor_vfio_linux_test.go | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index dbb9ed1f7..566a0d772 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -322,7 +322,9 @@ func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType } profiles, err := s.readCreatableProfiles(vf.PCIAddress) if err != nil { - return "", err + // An unreadable free VF is just not a placement candidate. + slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) + continue } for _, profile := range profiles { if profile.TypeName == profileType { @@ -356,7 +358,10 @@ func (s vendorVFIOSysfs) profileMetadata(vfs []VirtualFunction) ([]profileMetada for _, vf := range vfs { profiles, err := s.readCreatableProfiles(vf.PCIAddress) if err != nil { - return nil, err + // Mirror listProfiles: an unreadable VF must not fail placement + // while /resources still advertises the remaining capacity. + slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) + continue } for _, profile := range profiles { profilesByType[profile.TypeName] = profile diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 50e4d9f00..4f2e40844 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -75,6 +75,20 @@ func TestVendorVFIOListProfilesSkipsUnreadableVF(t *testing.T) { "only the readable VF counts toward availability") } +func TestVendorVFIOCreateSkipsUnreadableVF(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + badCreatable := filepath.Join(sysfs.pciDevicesPath, "0000:82:00.5", "nvidia", "creatable_vgpu_types") + require.NoError(t, os.Remove(badCreatable)) + require.NoError(t, os.Mkdir(badCreatable, 0755)) + + // Create places on the readable VF instead of failing the placement. + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "inst-1") + require.NoError(t, err) + assert.Equal(t, "0000:82:00.4", device.VFAddress) +} + func TestVendorVFIOCreateAndDestroy(t *testing.T) { t.Parallel() From 2088a231046f40d8f84bb3e01e2d3d4adc3f8415 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 17 Aug 2026 19:42:26 +0000 Subject: [PATCH 020/107] Degrade mdev discovery per VF instead of failing the host --- lib/devices/mdev_linux.go | 16 ++++++++++++++- lib/devices/vgpu_linux_test.go | 37 ++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/lib/devices/mdev_linux.go b/lib/devices/mdev_linux.go index 61e55b599..6ef9f6c60 100644 --- a/lib/devices/mdev_linux.go +++ b/lib/devices/mdev_linux.go @@ -5,7 +5,9 @@ package devices import ( "bufio" "context" + "errors" "fmt" + "log/slog" "os" "os/exec" "path/filepath" @@ -112,6 +114,7 @@ func discoverMdevVFsWith(busPath, pciPath string, listMdevs func() ([]MdevDevice } var vfs []VirtualFunction + var vfErrs []error for _, entry := range entries { vfAddr := entry.Name() types, err := os.ReadDir(filepath.Join(busPath, vfAddr, "mdev_supported_types")) @@ -119,7 +122,8 @@ func discoverMdevVFsWith(busPath, pciPath string, listMdevs func() ([]MdevDevice if os.IsNotExist(err) { continue } - return nil, fmt.Errorf("read mdev supported types for VF %s: %w", vfAddr, err) + vfErrs = append(vfErrs, fmt.Errorf("read mdev supported types for VF %s: %w", vfAddr, err)) + continue } usable := false for _, typ := range types { @@ -149,6 +153,16 @@ func discoverMdevVFsWith(busPath, pciPath string, listMdevs func() ([]MdevDevice Allocated: hasMdev, }) } + if len(vfErrs) > 0 { + // Mirror vendor VFIO discovery: one unreadable VF must not blank out + // the host's GPU capacity. Only when no VF is readable does discovery + // fail, so a wholesale sysfs outage cannot demote an mdev host to + // vendor VFIO or passthrough while assignments exist. + if len(vfs) == 0 { + return nil, errors.Join(vfErrs...) + } + slog.Default().Warn("skipping unreadable mdev VFs", "error", errors.Join(vfErrs...)) + } return vfs, nil } diff --git a/lib/devices/vgpu_linux_test.go b/lib/devices/vgpu_linux_test.go index 40b987394..7b03d9c26 100644 --- a/lib/devices/vgpu_linux_test.go +++ b/lib/devices/vgpu_linux_test.go @@ -73,3 +73,40 @@ func TestDiscoverVGPUWithPropagatesVendorVFIOError(t *testing.T) { assert.Equal(t, VGPUFrameworkNone, framework) assert.Nil(t, vfs) } + +func TestDiscoverMdevVFsSkipsUnreadableVF(t *testing.T) { + t.Parallel() + + root := t.TempDir() + busPath := filepath.Join(root, "sys", "class", "mdev_bus") + pciPath := filepath.Join(root, "sys", "bus", "pci", "devices") + require.NoError(t, os.MkdirAll(filepath.Join(busPath, "0000:82:00.4", "mdev_supported_types", "nvidia-556"), 0755)) + // A regular file makes the supported-types read fail without IsNotExist. + require.NoError(t, os.MkdirAll(filepath.Join(busPath, "0000:82:00.5"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(busPath, "0000:82:00.5", "mdev_supported_types"), nil, 0644)) + + vfs, err := discoverMdevVFsWith(busPath, pciPath, func() ([]MdevDevice, error) { + return nil, nil + }) + require.NoError(t, err) + require.Len(t, vfs, 1) + assert.Equal(t, "0000:82:00.4", vfs[0].PCIAddress) +} + +func TestDiscoverMdevVFsFailsWhenNoVFIsReadable(t *testing.T) { + t.Parallel() + + root := t.TempDir() + busPath := filepath.Join(root, "sys", "class", "mdev_bus") + pciPath := filepath.Join(root, "sys", "bus", "pci", "devices") + require.NoError(t, os.MkdirAll(filepath.Join(busPath, "0000:82:00.4"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(busPath, "0000:82:00.4", "mdev_supported_types"), nil, 0644)) + + // With every VF unreadable, discovery must fail rather than report an + // empty inventory that would demote the host to vendor VFIO or + // passthrough. + _, err := discoverMdevVFsWith(busPath, pciPath, func() ([]MdevDevice, error) { + return nil, nil + }) + require.Error(t, err) +} From 0a23f19e627a08c6f74f050e5a4fb2e5d0ea0f50 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 17 Aug 2026 19:42:26 +0000 Subject: [PATCH 021/107] Warn when reconcile preserves an unclaimed in-use vGPU --- lib/devices/vendor_vfio_linux.go | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 566a0d772..112030896 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -292,6 +292,7 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map continue } if inUse { + log.WarnContext(ctx, "preserving vendor VFIO vGPU held open without a live instance claim", "vf", vf.PCIAddress) continue } if err := s.destroyWithOpenPaths(ctx, vf.PCIAddress, "", openPaths); err != nil { From 7c60709f14f10a21623f25e35e4e5867cbe132b9 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 17 Aug 2026 21:10:15 +0000 Subject: [PATCH 022/107] Document wedged-VF forensics and SR-IOV cycle remediation --- lib/devices/GPU.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 0cba075fa..3a1504428 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -271,6 +271,44 @@ curl -s http://localhost:4973/resources | jq '.gpu.profiles' curl http://localhost:4973/instances//logs?source=app ``` +### Guest driver init times out on one VF (vendor VFIO) + +A VF can be wedged inside the NVIDIA stack while its sysfs interface stays +healthy: assignment succeeds, the host plugin logs `display_init inst: 0 +successful`, but the guest driver loops on + +``` +NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884) +``` + +(0x65 = timeout; the guest's init requests are never answered, and +`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle). Because +placement is deterministic least-loaded, an idle host re-picks the same VF for +every request, so one wedged VF presents as all vGPU instances failing while +`/resources` reports full capacity. + +The wedge itself leaves no host-side log: no kernel error, no XID, no plugin +crash. In the observed case it followed a period of heavy attach/teardown +churn on the VF, including QEMU processes that exited within seconds of +opening the VFIO device (failed start attempts that were then retried), so +suspect any workload that repeatedly kills the VMM mid-device-init. + +Confirm by assigning the same profile on a different VF: if that guest +initializes, the VF is wedged, not the driver stack. Remediate by cycling +SR-IOV on the parent GPU (this destroys and recreates all of its VFs, so it +requires no vGPU assignments on that GPU): + +```bash +/usr/lib/nvidia/sriov-manage -d +/usr/lib/nvidia/sriov-manage -e +``` + +Do not unbind/rebind the VF from the nvidia driver — it breaks the +nvidia-vgpu-vfio core-device registration (`vfio_pci_core_device not found`) +and the VF stops accepting assignments entirely until the SR-IOV cycle. +Services holding the GPU (DCGM, persistenced) must be stopped for the cycle +to obtain the unbind lock. + ### vGPU assignment fails Check the files for the framework detected on the host: From 797bf5cd04b831d0a1b8387757b78728d98386c9 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:10:10 +0000 Subject: [PATCH 023/107] Tolerate processes exiting mid-scan in the open-VFIO-handle scan A process that exits between the /proc listing and its fd walk surfaces ENOENT or ESRCH; it holds nothing open, so skipping it cannot produce a false "not in use" answer. Everything else still fails the scan closed. --- lib/devices/vendor_vfio_linux.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 112030896..a6d37e80d 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -13,6 +13,7 @@ import ( "strconv" "strings" "sync" + "syscall" "github.com/kernel/hypeman/lib/logger" ) @@ -425,7 +426,9 @@ func (s vendorVFIOSysfs) vfioDeviceInUse(vfAddress string, openPaths map[string] // isVFIOGroupInUse which skips them. That is deliberate: this scan authorizes // clearing current_vgpu_type on a VF path that is reused across assignments, // so an incomplete scan must fail the release (which retains metadata for a -// later retry) rather than risk a false "not in use" answer. +// later retry) rather than risk a false "not in use" answer. The exception +// is a process that exits mid-scan (ENOENT/ESRCH): a dead process holds +// nothing open, so skipping it cannot produce that false answer. func (s vendorVFIOSysfs) openVFIOPaths() (map[string]struct{}, error) { processes, err := os.ReadDir(s.procPath) if err != nil { @@ -440,7 +443,7 @@ func (s vendorVFIOSysfs) openVFIOPaths() (map[string]struct{}, error) { fdPath := filepath.Join(s.procPath, process.Name(), "fd") fds, err := os.ReadDir(fdPath) if err != nil { - if os.IsNotExist(err) { + if os.IsNotExist(err) || errors.Is(err, syscall.ESRCH) { continue } return nil, fmt.Errorf("read process %s file descriptors: %w", process.Name(), err) @@ -448,7 +451,7 @@ func (s vendorVFIOSysfs) openVFIOPaths() (map[string]struct{}, error) { for _, fd := range fds { target, err := os.Readlink(filepath.Join(fdPath, fd.Name())) if err != nil { - if os.IsNotExist(err) { + if os.IsNotExist(err) || errors.Is(err, syscall.ESRCH) { continue } return nil, fmt.Errorf("read process %s file descriptor %s: %w", process.Name(), fd.Name(), err) From c179e59346df809b6d6ef4dbfad37a4b17602369 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:10:10 +0000 Subject: [PATCH 024/107] Skip mdev release for an assignment with no device An mdev assignment carrying neither a UUID nor a device path would resolve to DestroyMdev("."); release nothing instead. --- lib/devices/vgpu_linux.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index 72fe3b944..b3c497899 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -92,6 +92,9 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { case VGPUFrameworkMdev: mdevUUID := assignment.MdevUUID if mdevUUID == "" { + if assignment.DevicePath == "" { + return nil + } mdevUUID = filepath.Base(assignment.DevicePath) } return DestroyMdev(ctx, mdevUUID) From 20a1cde6db0e7b4e3fbbdcd9119a02771aec2436 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:18:00 +0000 Subject: [PATCH 025/107] Recheck VFIO handles before orphan cleanup --- lib/devices/vendor_vfio_linux.go | 17 ++++++++--------- lib/devices/vendor_vfio_linux_test.go | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index a6d37e80d..2c576b4d2 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -29,6 +29,7 @@ type vendorVFIOSysfs struct { vfioDevicesPath string owners map[string]string framebufferByType map[string]int + openVFIOPathsFunc func() (map[string]struct{}, error) } var ( @@ -206,10 +207,6 @@ func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID str } func (s vendorVFIOSysfs) destroy(ctx context.Context, vfAddress, instanceID string) error { - return s.destroyWithOpenPaths(ctx, vfAddress, instanceID, nil) -} - -func (s vendorVFIOSysfs) destroyWithOpenPaths(ctx context.Context, vfAddress, instanceID string, openPaths map[string]struct{}) error { vendorVFIOMu.Lock() defer vendorVFIOMu.Unlock() @@ -242,10 +239,9 @@ func (s vendorVFIOSysfs) destroyWithOpenPaths(ctx context.Context, vfAddress, in } } - if openPaths == nil { - if openPaths, err = s.openVFIOPaths(); err != nil { - return fmt.Errorf("scan open VFIO handles: %w", err) - } + openPaths, err := s.openVFIOPaths() + if err != nil { + return fmt.Errorf("scan open VFIO handles: %w", err) } inUse, err := s.vfioDeviceInUse(vfAddress, openPaths) if err != nil { @@ -296,7 +292,7 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map log.WarnContext(ctx, "preserving vendor VFIO vGPU held open without a live instance claim", "vf", vf.PCIAddress) continue } - if err := s.destroyWithOpenPaths(ctx, vf.PCIAddress, "", openPaths); err != nil { + if err := s.destroy(ctx, vf.PCIAddress, ""); err != nil { log.WarnContext(ctx, "failed to destroy orphaned vendor VFIO vGPU", "vf", vf.PCIAddress, "error", err) } } @@ -430,6 +426,9 @@ func (s vendorVFIOSysfs) vfioDeviceInUse(vfAddress string, openPaths map[string] // is a process that exits mid-scan (ENOENT/ESRCH): a dead process holds // nothing open, so skipping it cannot produce that false answer. func (s vendorVFIOSysfs) openVFIOPaths() (map[string]struct{}, error) { + if s.openVFIOPathsFunc != nil { + return s.openVFIOPathsFunc() + } processes, err := os.ReadDir(s.procPath) if err != nil { return nil, err diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 4f2e40844..bd7293922 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -372,6 +372,28 @@ func TestVendorVFIOReconcile(t *testing.T) { assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:e3:00.4", "nvidia", "current_vgpu_type"), "1148") } +func TestVendorVFIOReconcileRechecksOpenHandlesBeforeDestroy(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + + activeDevice := filepath.Join(sysfs.vfioDevicesPath, "vfio42") + scans := 0 + sysfs.openVFIOPathsFunc = func() (map[string]struct{}, error) { + scans++ + if scans == 1 { + return map[string]struct{}{}, nil + } + return map[string]struct{}{activeDevice: {}}, nil + } + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assert.Equal(t, 2, scans) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + func TestVendorVFIOReconcileSkipsProtectedVF(t *testing.T) { t.Parallel() From e6034291a57046ce9fa67c3f42219ce9fdf20779 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 19:05:18 +0000 Subject: [PATCH 026/107] Guard vGPU releases with live-instance claims A vGPU assignment goes stale when its release succeeds but the metadata save does not (or start fails between the release and its first save). The backend's owner map only covers assignments created since the last restart and the VFIO handle scan only covers VMs that have opened the device, so after a restart a stale release could still clear a VF during another live instance's pre-open boot window. Consult live instance metadata on every release: when another instance with a live hypervisor process claims the same device path, drop the stale metadata without touching the device. Tag assignments with the owning instance ID, persist the assignment before booting a started instance, and retain assignment metadata when rollback release fails in create and start so later release paths can still find the device. --- lib/instances/create.go | 41 ++++++++++++++++++++- lib/instances/delete.go | 2 +- lib/instances/lifecycle_noop_test.go | 55 ++++++++++++++++++++++++++++ lib/instances/start.go | 9 ++++- lib/instances/stop.go | 2 +- lib/instances/vgpu.go | 44 +++++++++++++++++----- lib/instances/vgpu_test.go | 50 ++++++++++++++++++++----- 7 files changed, 180 insertions(+), 23 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index b4b0952da..2608b2bd2 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -277,11 +277,13 @@ func (m *manager) createInstance( var gpuFramework devices.VGPUFramework var gpuDevicePath string var gpuMdevUUID string + var stored *StoredMetadata + var retainedVGPU *StoredMetadata // Setup cleanup stack early so device attachment errors trigger cleanup cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) - m.deleteInstanceData(id) + m.cleanupFailedCreate(ctx, id, retainedVGPU) }) defer cu.Clean() @@ -320,6 +322,25 @@ func (m *manager) createInstance( } if err := devices.DestroyVGPU(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) + retainedVGPU = stored + if retainedVGPU == nil { + retainedVGPU = &StoredMetadata{ + Id: id, + Name: req.Name, + Image: req.Image, + ResolvedImage: resolvedImageRef, + Platform: imageInfo.Platform, + CreatedAt: time.Now(), + HypervisorType: hvType, + HypervisorVersion: hvVersion, + SocketPath: m.paths.InstanceSocket(id, starter.SocketName()), + DataDir: m.paths.InstanceDir(id), + GPUProfile: gpuDevice.ProfileName, + GPUFramework: gpuDevice.Framework, + GPUDevicePath: gpuDevice.SysfsPath, + GPUMdevUUID: gpuDevice.MdevUUID, + } + } } }) } @@ -360,7 +381,7 @@ func (m *manager) createInstance( if err != nil { return nil, err } - stored := &StoredMetadata{ + stored = &StoredMetadata{ Id: id, Name: req.Name, Image: req.Image, @@ -610,6 +631,22 @@ func resolveCreateExpiration(req CreateInstanceRequest, now time.Time) (*time.Ti return &expiresAt, nil } +func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) { + if retainedVGPU == nil { + m.deleteInstanceData(id) + return + } + + log := logger.FromContext(ctx) + if err := m.ensureDirectories(id); err != nil { + log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) + return + } + if err := m.saveMetadata(&metadata{StoredMetadata: *retainedVGPU}); err != nil { + log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) + } +} + // validateCreateRequest validates the create instance request. // The request is mutated in-place to persist normalized egress/credential policy fields. func validateCreateRequest(req *CreateInstanceRequest) error { diff --git a/lib/instances/delete.go b/lib/instances/delete.go index e897ff049..08781b977 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -150,7 +150,7 @@ func (m *manager) deleteInstanceWithOptions( if hadVGPUAssignment { log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID) } - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { // Log error but continue with cleanup. log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) } else if hadVGPUAssignment { diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index a9b918dfb..89468d970 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -3,6 +3,7 @@ package instances import ( "context" "errors" + "net" "os" "path/filepath" "sync" @@ -195,6 +196,46 @@ func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) { assert.Equal(t, restartpolicy.BlockedReasonManualStop, persisted.RestartStatus.BlockedReason) } +func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { + now := time.Now().UTC() + m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, now) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + require.NoError(t, m.saveMetadata(meta)) + + claimantID := "inst-live-claimant" + require.NoError(t, m.ensureDirectories(claimantID)) + pid := os.Getpid() + socketPath := m.paths.InstanceSocket(claimantID, "noop.sock") + listener, err := net.Listen("unix", socketPath) + require.NoError(t, err) + defer listener.Close() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: claimantID, + Name: claimantID, + Image: "test-image", + CreatedAt: now, + HypervisorType: lifecycleNoopHypervisorType, + HypervisorPID: &pid, + SocketPath: socketPath, + DataDir: m.paths.InstanceDir(claimantID), + GPUProfile: "NVIDIA L40S-2Q", + GPUFramework: devices.VGPUFramework("future-framework"), + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }})) + + require.NoError(t, m.DeleteInstance(context.Background(), id)) + + _, err = m.loadMetadata(id) + require.Error(t, err, "deleted instance metadata should be gone") + claimant, err := m.loadMetadata(claimantID) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", claimant.GPUDevicePath, "live claimant keeps its assignment") +} + func TestDeleteContinuesTeardownAfterFailedVGPURelease(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) deviceManager := &recordingDeviceManager{} @@ -300,6 +341,20 @@ func (m *recordingDeviceManager) UnbindFromVFIO(ctx context.Context, id string) return nil } +func TestLifecycleNoopStandbyRejectsVendorVFIOVGPU(t *testing.T) { + m, id := newLifecycleNoopManagerWithInstance(t, StateRunning, time.Now().UTC()) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFrameworkVendorVFIO + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + require.NoError(t, m.saveMetadata(meta)) + + _, err = m.StandbyInstance(context.Background(), id, StandbyInstanceRequest{}) + require.ErrorIs(t, err, ErrInvalidState) + assert.ErrorContains(t, err, "standby is not supported for instances with vGPU attached") +} + func newLifecycleNoopManagerWithInstance(t *testing.T, state State, now time.Time) (*manager, string) { t.Helper() diff --git a/lib/instances/start.go b/lib/instances/start.go index a6c832450..ff8e94ea3 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -54,7 +54,7 @@ func (m *manager) startInstance( // cannot leave on-disk metadata pointing at a device that is already // gone (matching releaseRetainedVGPULocked). if storedVGPUDevicePath(stored) != "" { - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { log.ErrorContext(ctx, "failed to release stale vGPU before start", "instance_id", id, "error", err) return nil, fmt.Errorf("release stale vGPU before start: %w", err) } @@ -181,8 +181,15 @@ func (m *manager) startInstance( } if err := devices.DestroyVGPU(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID, "error", err) + if saveErr := m.saveMetadata(meta); saveErr != nil { + log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", saveErr) + } } }) + if err := m.saveMetadata(meta); err != nil { + log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err) + return nil, fmt.Errorf("save metadata after vGPU creation: %w", err) + } } // 5. Regenerate config disk with new network configuration diff --git a/lib/instances/stop.go b/lib/instances/stop.go index 7eddeb034..ebcf58708 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -232,7 +232,7 @@ func (m *manager) stopInstance( // 7. Release the vGPU assignment if present (frees the vGPU slot for other VMs). if path := storedVGPUDevicePath(stored); path != "" { log.InfoContext(ctx, "destroying vGPU on stop", "instance_id", id, "device_path", path) - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on stop; retaining assignment metadata", "instance_id", id, "device_path", path, "error", err) } } diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index a8ca6aceb..b7d275428 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -20,23 +20,49 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUMdevUUID = "" } -func releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { +func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { path := storedVGPUDevicePath(stored) if path != "" { - assignment := devices.VGPUAssignment{ - Framework: stored.GPUFramework, - DevicePath: path, - MdevUUID: stored.GPUMdevUUID, - InstanceID: stored.Id, - } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path) + if err != nil { return err } + if claimed { + logger.FromContext(ctx).WarnContext(ctx, "dropping stale vGPU assignment claimed by another live instance", + "instance_id", stored.Id, "device_path", path) + } else { + assignment := devices.VGPUAssignment{ + Framework: stored.GPUFramework, + DevicePath: path, + MdevUUID: stored.GPUMdevUUID, + InstanceID: stored.Id, + } + if err := devices.DestroyVGPU(ctx, assignment); err != nil { + return err + } + } } clearStoredVGPUDevice(stored) return nil } +func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { + instances, err := m.listInstances(ctx) + if err != nil { + return false, fmt.Errorf("list instances for vGPU release check: %w", err) + } + for i := range instances { + inst := &instances[i] + if inst.Id == excludeID || inst.GPUDevicePath != devicePath || inst.HypervisorPID == nil { + continue + } + if HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) { + return true, nil + } + } + return false, nil +} + // releaseRetainedVGPULocked releases a vGPU assignment retained on a stopped // instance after a failed release during the original stop. It is a no-op // when no assignment is retained, and a failed retry only logs so the @@ -52,7 +78,7 @@ func (m *manager) releaseRetainedVGPULocked(ctx context.Context, id string) { if storedVGPUDevicePath(stored) == "" { return } - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { log.WarnContext(ctx, "failed to destroy retained vGPU; retaining assignment metadata", "instance_id", id, "error", err) return } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 6f2c46819..2b7d84a94 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -5,14 +5,47 @@ import ( "testing" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func TestValidateVGPUHypervisor(t *testing.T) { + t.Parallel() + + assert.NoError(t, validateVGPUHypervisor(hypervisor.TypeQEMU)) + assert.EqualError(t, validateVGPUHypervisor(hypervisor.TypeCloudHypervisor), "vGPU is only supported with qemu, got cloud-hypervisor") +} + +func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + stored := &StoredMetadata{ + Id: "failed-create", + Name: "failed-create", + GPUProfile: "NVIDIA L40S-2Q", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + HypervisorType: "qemu", + DataDir: m.paths.InstanceDir("failed-create"), + } + + m.cleanupFailedCreate(context.Background(), stored.Id, stored) + + retained, err := m.loadMetadata(stored.Id) + require.NoError(t, err) + assert.Equal(t, stored.GPUProfile, retained.GPUProfile) + assert.Equal(t, stored.GPUFramework, retained.GPUFramework) + assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) +} + func TestStoredVGPUDevicePath(t *testing.T) { t.Parallel() - assert.Equal(t, "/sys/bus/mdev/devices/new-uuid", storedVGPUDevicePath(&StoredMetadata{ - GPUDevicePath: "/sys/bus/mdev/devices/new-uuid", + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", storedVGPUDevicePath(&StoredMetadata{ + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUMdevUUID: "legacy-uuid", })) assert.Equal(t, "/sys/bus/mdev/devices/legacy-uuid", storedVGPUDevicePath(&StoredMetadata{ @@ -24,11 +57,12 @@ func TestStoredVGPUDevicePath(t *testing.T) { func TestReleaseStoredVGPURetainsMetadataOnFailure(t *testing.T) { t.Parallel() + m := &manager{paths: paths.New(t.TempDir())} stored := &StoredMetadata{ GPUFramework: devices.VGPUFramework("future-framework"), GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", } - err := releaseStoredVGPU(context.Background(), stored) + err := m.releaseStoredVGPU(context.Background(), stored) assert.Error(t, err) assert.Equal(t, devices.VGPUFramework("future-framework"), stored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) @@ -39,13 +73,11 @@ func TestSetAndClearStoredVGPUDevice(t *testing.T) { stored := &StoredMetadata{} setStoredVGPUDevice(stored, &devices.VGPUDevice{ - Framework: devices.VGPUFrameworkMdev, - SysfsPath: "/sys/bus/mdev/devices/new-uuid", - MdevUUID: "new-uuid", + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", }) - assert.Equal(t, devices.VGPUFrameworkMdev, stored.GPUFramework) - assert.Equal(t, "/sys/bus/mdev/devices/new-uuid", stored.GPUDevicePath) - assert.Equal(t, "new-uuid", stored.GPUMdevUUID) + assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) clearStoredVGPUDevice(stored) assert.Empty(t, stored.GPUFramework) From 5e7955a5cca4cce96280a640cdf66c2595152971 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 19:05:42 +0000 Subject: [PATCH 027/107] Reconcile vendor VFIO vGPUs against a fail-closed instance inventory Startup reconciliation protects the VFs of instances whose hypervisor survived the restart, verified by socket ownership so a reused PID cannot hold a VF. The inventory behind that protected set must not silently skip unreadable metadata: a skipped live claimant would leave its VF unprotected during the pre-VFIO-open boot window. Add ListInstancesForReconcile, which fails on any unreadable metadata, and skip vendor VFIO reconciliation when the inventory is unavailable while keeping mdev reconciliation running. --- cmd/api/main.go | 33 ++++++++++++++++++++++++++++----- lib/builds/manager_test.go | 4 ++++ lib/instances/manager.go | 6 ++++++ lib/instances/query.go | 13 +++++++++++-- lib/instances/query_test.go | 22 ++++++++++++++++++++++ lib/instances/storage.go | 8 +++++++- lib/instances/wait_test.go | 3 +++ 7 files changed, 81 insertions(+), 8 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index f1bbfcf3d..0479f1583 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -185,6 +185,24 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta }, logger), nil } +func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, error) { + allInstances, err := instanceManager.ListInstancesForReconcile(ctx) + if err != nil { + return nil, err + } + protected := make(map[string]struct{}) + for _, inst := range allInstances { + if inst.GPUDevicePath == "" || inst.HypervisorPID == nil { + continue + } + if !instances.HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) { + continue + } + protected[inst.GPUDevicePath] = struct{}{} + } + return protected, nil +} + func run() error { startupStarted := time.Now() slog.Info("starting hypeman initialization") @@ -384,11 +402,16 @@ func run() error { return fmt.Errorf("reconcile device state: %w", err) } - // Reconcile mdev devices (clears orphaned vGPUs from previous runs) - logger.Info("Reconciling mdev devices...") - if err := devices.ReconcileMdevs(app.Ctx, nil); err != nil { - // Log but don't fail - mdev cleanup is best-effort - logger.Warn("failed to reconcile mdev devices", "error", err) + // Reconcile vGPU devices (clears orphaned vGPUs from previous runs) + logger.Info("Reconciling vGPU devices...") + protected, err := liveInstanceVGPUDevicePaths(app.Ctx, app.InstanceManager) + if err != nil { + logger.Warn("failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconciliation", "error", err) + protected = nil + } + if err := devices.ReconcileVGPUs(app.Ctx, protected); err != nil { + // Log but don't fail - vGPU cleanup is best-effort + logger.Warn("failed to reconcile vGPU devices", "error", err) } // Wire up resource validator for aggregate limit checking diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index a137edc66..44596bf68 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -51,6 +51,10 @@ func (m *mockInstanceManager) ListInstances(ctx context.Context, filter *instanc return result, nil } +func (m *mockInstanceManager) ListInstancesForReconcile(ctx context.Context) ([]instances.Instance, error) { + return m.ListInstances(ctx, nil) +} + func (m *mockInstanceManager) ListSnapshots(ctx context.Context, filter *instances.ListSnapshotsFilter) ([]instances.Snapshot, error) { return nil, nil } diff --git a/lib/instances/manager.go b/lib/instances/manager.go index bc23fdf74..dd4404972 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -27,6 +27,7 @@ import ( type Manager interface { ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) + ListInstancesForReconcile(ctx context.Context) ([]Instance, error) ListSnapshots(ctx context.Context, filter *ListSnapshotsFilter) ([]Snapshot, error) GetSnapshot(ctx context.Context, snapshotID string) (*Snapshot, error) CreateInstance(ctx context.Context, req CreateInstanceRequest) (*Instance, error) @@ -732,6 +733,11 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { return m.defaultHypervisor } +// ListInstancesForReconcile returns every instance or an invalid metadata error. +func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, error) { + return m.loadInstances(ctx, false) +} + // ListInstances returns instances, optionally filtered by the given criteria. // Pass nil to return all instances. func (m *manager) ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) { diff --git a/lib/instances/query.go b/lib/instances/query.go index 8e3ed61f1..eea66ab13 100644 --- a/lib/instances/query.go +++ b/lib/instances/query.go @@ -784,14 +784,18 @@ func parseSentinelTimestamp(line, sentinelPrefix string) (time.Time, bool) { return time.Time{}, false } -// listInstances returns all instances +// listInstances returns all instances, skipping metadata files that cannot be loaded. func (m *manager) listInstances(ctx context.Context) ([]Instance, error) { + return m.loadInstances(ctx, true) +} + +func (m *manager) loadInstances(ctx context.Context, skipInvalid bool) ([]Instance, error) { ctx, span := m.tracerOrDefault().Start(ctx, "instances.list_metadata") defer span.End() log := logger.FromContext(ctx) log.DebugContext(ctx, "listing all instances") - files, err := m.listMetadataFiles() + files, err := m.listMetadataFilesWithStatErrors(!skipInvalid) if err != nil { log.ErrorContext(ctx, "failed to list metadata files", "error", err) return nil, err @@ -809,6 +813,11 @@ func (m *manager) listInstances(ctx context.Context) ([]Instance, error) { ) meta, err := m.loadMetadata(id) if err != nil { + if !skipInvalid { + hydrateSpan.RecordError(err) + hydrateSpan.End() + return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) + } // Skip instances with invalid metadata log.WarnContext(hydrateCtx, "skipping instance with invalid metadata", "instance_id", id, "error", err) hydrateSpan.End() diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index 8bb0bb464..41aba54e8 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -14,6 +14,28 @@ import ( "github.com/stretchr/testify/require" ) +func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { + m := &manager{paths: paths.New(t.TempDir())} + + require.NoError(t, m.ensureDirectories("valid")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "valid", + Name: "valid", + CreatedAt: time.Now(), + DataDir: m.paths.InstanceDir("valid"), + }})) + require.NoError(t, m.ensureDirectories("invalid")) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid"), []byte("{"), 0644)) + + listed, err := m.ListInstances(context.Background(), nil) + require.NoError(t, err) + require.Len(t, listed, 1) + + _, err = m.ListInstancesForReconcile(context.Background()) + require.Error(t, err) + assert.ErrorContains(t, err, "load metadata for instance invalid") +} + func TestParseExitSentinelLine(t *testing.T) { t.Parallel() tests := []struct { diff --git a/lib/instances/storage.go b/lib/instances/storage.go index a293fc6e1..dd932d41a 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -187,8 +187,12 @@ func removeAllWithRetry(path string, removeAll func(string) error, sleep func(ti } } -// listMetadataFiles returns paths to all instance metadata files +// listMetadataFiles returns paths to all instance metadata files. func (m *manager) listMetadataFiles() ([]string, error) { + return m.listMetadataFilesWithStatErrors(false) +} + +func (m *manager) listMetadataFilesWithStatErrors(failOnStatError bool) ([]string, error) { guestsDir := m.paths.GuestsDir() // Ensure guests directory exists @@ -210,6 +214,8 @@ func (m *manager) listMetadataFiles() ([]string, error) { metaPath := filepath.Join(guestsDir, entry.Name(), "metadata.json") if _, err := os.Stat(metaPath); err == nil { metaFiles = append(metaFiles, metaPath) + } else if failOnStatError && !os.IsNotExist(err) { + return nil, fmt.Errorf("stat metadata for instance %s: %w", entry.Name(), err) } } diff --git a/lib/instances/wait_test.go b/lib/instances/wait_test.go index dbb630185..415003594 100644 --- a/lib/instances/wait_test.go +++ b/lib/instances/wait_test.go @@ -32,6 +32,9 @@ func (s *stubManager) GetInstance(ctx context.Context, id string) (*Instance, er func (s *stubManager) ListInstances(context.Context, *ListInstancesFilter) ([]Instance, error) { return nil, nil } +func (s *stubManager) ListInstancesForReconcile(context.Context) ([]Instance, error) { + return nil, nil +} func (s *stubManager) ListSnapshots(context.Context, *ListSnapshotsFilter) ([]Snapshot, error) { return nil, nil } From d7eadf8db0e0ac8dd6a7949191909d0b013c120c Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:19:26 +0000 Subject: [PATCH 028/107] Fail closed on vGPU claim checks --- lib/instances/vgpu.go | 2 +- lib/instances/vgpu_test.go | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index b7d275428..29edfeab7 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -47,7 +47,7 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) } func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { - instances, err := m.listInstances(ctx) + instances, err := m.ListInstancesForReconcile(ctx) if err != nil { return false, fmt.Errorf("list instances for vGPU release check: %w", err) } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 2b7d84a94..29341d230 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -2,6 +2,7 @@ package instances import ( "context" + "os" "testing" "github.com/kernel/hypeman/lib/devices" @@ -41,6 +42,17 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) } +func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("invalid-instance")) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid-instance"), []byte("{"), 0o644)) + + _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.Error(t, err) +} + func TestStoredVGPUDevicePath(t *testing.T) { t.Parallel() From 506b42117e168ac014f3b98fd92c88ec721c0d4d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:19:50 +0000 Subject: [PATCH 029/107] Retain only vGPU assignment after failed create --- lib/instances/create.go | 8 +++++++- lib/instances/vgpu_test.go | 13 ++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index 2608b2bd2..d25bd91ed 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -642,7 +642,13 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) return } - if err := m.saveMetadata(&metadata{StoredMetadata: *retainedVGPU}); err != nil { + retained := StoredMetadata{ + Id: id, + GPUFramework: retainedVGPU.GPUFramework, + GPUDevicePath: retainedVGPU.GPUDevicePath, + GPUMdevUUID: retainedVGPU.GPUMdevUUID, + } + if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) } } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 29341d230..b408ccafe 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -29,6 +29,10 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { GPUProfile: "NVIDIA L40S-2Q", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUMdevUUID: "mdev-uuid", + NetworkEnabled: true, + IP: "192.0.2.1", + Volumes: []VolumeAttachment{{VolumeID: "volume"}}, HypervisorType: "qemu", DataDir: m.paths.InstanceDir("failed-create"), } @@ -37,9 +41,16 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { retained, err := m.loadMetadata(stored.Id) require.NoError(t, err) - assert.Equal(t, stored.GPUProfile, retained.GPUProfile) + assert.Equal(t, stored.Id, retained.Id) assert.Equal(t, stored.GPUFramework, retained.GPUFramework) assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) + assert.Equal(t, stored.GPUMdevUUID, retained.GPUMdevUUID) + assert.Empty(t, retained.Name) + assert.Empty(t, retained.GPUProfile) + assert.False(t, retained.NetworkEnabled) + assert.Empty(t, retained.IP) + assert.Empty(t, retained.Volumes) + assert.Empty(t, retained.DataDir) } func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { From bd1fd3898180d48f0757d91a24a5f87bfe65c30e Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:20:05 +0000 Subject: [PATCH 030/107] Clear released vGPU assignment on start rollback --- lib/instances/start.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/instances/start.go b/lib/instances/start.go index ff8e94ea3..d9c8c9aaa 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -184,6 +184,11 @@ func (m *manager) startInstance( if saveErr := m.saveMetadata(meta); saveErr != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", saveErr) } + } else { + clearStoredVGPUDevice(stored) + if saveErr := m.saveMetadata(meta); saveErr != nil { + log.ErrorContext(ctx, "failed to save metadata after vGPU cleanup", "instance_id", id, "error", saveErr) + } } }) if err := m.saveMetadata(meta); err != nil { From 5ca85a2a6fb65f97fac6b17a028ed840239eead9 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:50:00 +0000 Subject: [PATCH 031/107] Test start rollback vGPU cleanup --- lib/instances/vgpu_test.go | 70 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index b408ccafe..d9aa02f02 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -3,7 +3,10 @@ package instances import ( "context" "os" + "path/filepath" + "sync" "testing" + _ "unsafe" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" @@ -53,6 +56,73 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { assert.Empty(t, retained.DataDir) } +//go:linkname hostVendorVFIO github.com/kernel/hypeman/lib/devices.hostVendorVFIO +var hostVendorVFIO vendorVFIOSysfs + +type vendorVFIOSysfs struct { + pciDevicesPath string + procPath string + vfioDevicesPath string + owners map[string]string +} + +func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { + root := t.TempDir() + pciDevicesPath := filepath.Join(root, "sys", "bus", "pci", "devices") + vfAddress := "0000:82:00.4" + nvidiaPath := filepath.Join(pciDevicesPath, vfAddress, "nvidia") + require.NoError(t, os.MkdirAll(nvidiaPath, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "current_vgpu_type"), []byte("0"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "creatable_vgpu_types"), []byte("ID : vGPU Name\n1148 : NVIDIA L40S-2Q\n"), 0o644)) + + originalVendorVFIO := hostVendorVFIO + hostVendorVFIO = vendorVFIOSysfs{ + pciDevicesPath: pciDevicesPath, + procPath: filepath.Join(root, "proc"), + vfioDevicesPath: filepath.Join(root, "dev", "vfio", "devices"), + owners: make(map[string]string), + } + t.Cleanup(func() { hostVendorVFIO = originalVendorVFIO }) + require.NoError(t, os.MkdirAll(hostVendorVFIO.procPath, 0o755)) + + m := &manager{ + paths: paths.New(t.TempDir()), + imageManager: readyFixtureImageManager{name: "test-image"}, + instanceLocks: sync.Map{}, + bootMarkerScans: sync.Map{}, + } + const id = "start-rollback" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + Image: "test-image", + GPUProfile: "NVIDIA L40S-2Q", + HypervisorType: lifecycleNoopHypervisorType, + SocketPath: m.paths.InstanceSocket(id, "noop.sock"), + DataDir: m.paths.InstanceDir(id), + }})) + + t.Setenv("TMPDIR", filepath.Join(root, "missing")) + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.Error(t, err) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile) + assert.Empty(t, stored.GPUFramework) + assert.Empty(t, stored.GPUDevicePath) + assert.Empty(t, stored.GPUMdevUUID) + assertFileContents(t, filepath.Join(nvidiaPath, "current_vgpu_type"), "0") +} + +func assertFileContents(t *testing.T, path, want string) { + t.Helper() + got, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, want, string(got)) +} + func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { t.Parallel() From afc8dade88d6c3f9d5cb58a3c23e996a2dd93c38 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 7 Aug 2026 15:02:16 +0000 Subject: [PATCH 032/107] Normalize legacy mdev paths in live-claim check The claim guard compared raw GPUDevicePath, which is empty on records persisted before the framework migration; a live claimant with only a legacy GPUMdevUUID was invisible to the check. Normalize the inventory side with storedVGPUDevicePath, matching the release subject. --- lib/instances/vgpu.go | 2 +- lib/instances/vgpu_test.go | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 29edfeab7..23ea2d825 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -53,7 +53,7 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu } for i := range instances { inst := &instances[i] - if inst.Id == excludeID || inst.GPUDevicePath != devicePath || inst.HypervisorPID == nil { + if inst.Id == excludeID || storedVGPUDevicePath(&inst.StoredMetadata) != devicePath || inst.HypervisorPID == nil { continue } if HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index d9aa02f02..79e6fe0c0 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -134,6 +134,24 @@ func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) require.Error(t, err) } +func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("legacy-claimant")) + pid := os.Getpid() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "legacy-claimant", + Name: "legacy-claimant", + GPUMdevUUID: "legacy-uuid", + HypervisorPID: &pid, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/mdev/devices/legacy-uuid") + require.NoError(t, err) + assert.True(t, claimed) +} + func TestStoredVGPUDevicePath(t *testing.T) { t.Parallel() From df1a288748c7141f9f1a4f00100fa99b727faab5 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 7 Aug 2026 20:45:49 +0000 Subject: [PATCH 033/107] Bind the live-claimant test socket under /tmp for macOS --- lib/instances/lifecycle_noop_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 89468d970..f3baa9dae 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -209,7 +209,14 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { claimantID := "inst-live-claimant" require.NoError(t, m.ensureDirectories(claimantID)) pid := os.Getpid() - socketPath := m.paths.InstanceSocket(claimantID, "noop.sock") + // Bind under /tmp: a t.TempDir()-derived path exceeds the macOS AF_UNIX + // path limit. + socketDir, err := os.MkdirTemp("/tmp", "hypeman-claimant-socket-") + require.NoError(t, err) + t.Cleanup(func() { + _ = os.RemoveAll(socketDir) + }) + socketPath := filepath.Join(socketDir, "noop.sock") listener, err := net.Listen("unix", socketPath) require.NoError(t, err) defer listener.Close() From 44536513a33b084c32558a4b9409f68c18f32e18 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 7 Aug 2026 20:45:49 +0000 Subject: [PATCH 034/107] Surface retained vGPU cleanup through a typed create error and manager seam Replace the go:linkname shadow of devices.hostVendorVFIO with createVGPU/destroyVGPU manager fields, and wrap failed creates whose rollback release also failed in VGPUCleanupPendingError so the API can point callers at the retained instance record. --- cmd/api/api/instances.go | 7 +++ lib/instances/create.go | 28 +++++++++--- lib/instances/manager.go | 4 ++ lib/instances/start.go | 8 ++-- lib/instances/vgpu.go | 40 ++++++++++++++++- lib/instances/vgpu_test.go | 92 ++++++++++++++++++++++++-------------- 6 files changed, 134 insertions(+), 45 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 669d38633..e9cee6ed9 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -362,6 +362,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst inst, err := s.InstanceManager.CreateInstance(ctx, domainReq) if err != nil { + var vgpuPending *instances.VGPUCleanupPendingError switch { case errors.Is(err, instances.ErrImageNotReady): return oapi.CreateInstance400JSONResponse{ @@ -413,6 +414,12 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Code: "not_found", Message: err.Error(), }, nil + case errors.As(err, &vgpuPending): + log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) + return oapi.CreateInstance500JSONResponse{ + Code: "vgpu_cleanup_pending", + Message: fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID), + }, nil default: log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) return oapi.CreateInstance500JSONResponse{ diff --git a/lib/instances/create.go b/lib/instances/create.go index d25bd91ed..dc70639fd 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -280,10 +280,20 @@ func (m *manager) createInstance( var stored *StoredMetadata var retainedVGPU *StoredMetadata - // Setup cleanup stack early so device attachment errors trigger cleanup + // Setup cleanup stack early so device attachment errors trigger cleanup. + // When rollback retains a vGPU assignment, surface the retained instance + // ID to the caller so the record is discoverable and can be deleted to + // retry the release. The wrapping defer is registered first so it runs + // after cu.Clean has decided whether metadata was retained. + vgpuRetained := false + defer func() { + if retErr != nil && vgpuRetained { + retErr = &VGPUCleanupPendingError{InstanceID: id, Err: retErr} + } + }() cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) - m.cleanupFailedCreate(ctx, id, retainedVGPU) + vgpuRetained = m.cleanupFailedCreate(ctx, id, retainedVGPU) }) defer cu.Clean() @@ -300,7 +310,7 @@ func (m *manager) createInstance( // Handle vGPU profile request if req.GPU != nil && req.GPU.Profile != "" { log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) - gpuDevice, err = devices.CreateVGPU(ctx, req.GPU.Profile, id) + gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) if err != nil { log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) return nil, wrapCreateVGPUErr(req.GPU.Profile, err) @@ -320,7 +330,7 @@ func (m *manager) createInstance( MdevUUID: gpuDevice.MdevUUID, InstanceID: id, } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { + if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) retainedVGPU = stored if retainedVGPU == nil { @@ -631,16 +641,18 @@ func resolveCreateExpiration(req CreateInstanceRequest, now time.Time) (*time.Ti return &expiresAt, nil } -func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) { +// cleanupFailedCreate reports whether it retained instance metadata for a +// vGPU assignment whose release failed during rollback. +func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) bool { if retainedVGPU == nil { m.deleteInstanceData(id) - return + return false } log := logger.FromContext(ctx) if err := m.ensureDirectories(id); err != nil { log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) - return + return false } retained := StoredMetadata{ Id: id, @@ -650,7 +662,9 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) + return false } + return true } // validateCreateRequest validates the create instance request. diff --git a/lib/instances/manager.go b/lib/instances/manager.go index dd4404972..458275bb5 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -182,6 +182,8 @@ type manager struct { now func() time.Time writeFile func(string, []byte, os.FileMode) error deleteInstanceFn func(context.Context, string) error + createVGPU func(context.Context, string, string) (*devices.VGPUDevice, error) + destroyVGPU func(context.Context, devices.VGPUAssignment) error deleteSnapshotFn func(context.Context, string) error ttlReaperDeleteTimeout time.Duration egressProxy *egressproxy.Service @@ -282,6 +284,8 @@ func NewManagerWithConfigE(p *paths.Paths, imageManager images.Manager, systemMa defaultHypervisor: defaultHypervisor, now: time.Now, writeFile: os.WriteFile, + createVGPU: devices.CreateVGPU, + destroyVGPU: devices.DestroyVGPU, meter: meter, tracer: tracer, guestMemoryPolicy: policy, diff --git a/lib/instances/start.go b/lib/instances/start.go index d9c8c9aaa..b830bafb2 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -162,11 +162,11 @@ func (m *manager) startInstance( // 4b. Recreate the vGPU if this instance had a GPU profile // Note: GPU availability was already validated in step 2b if stored.GPUProfile != "" { - log.InfoContext(ctx, "creating vGPU mdev for start", "instance_id", id, "profile", stored.GPUProfile) - device, err := devices.CreateVGPU(ctx, stored.GPUProfile, id) + log.InfoContext(ctx, "creating vGPU for start", "instance_id", id, "profile", stored.GPUProfile) + device, err := m.createVGPUDevice(ctx, stored.GPUProfile, id) if err != nil { log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) - return nil, fmt.Errorf("create vGPU mdev for profile %s: %w", stored.GPUProfile, err) + return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) } setStoredVGPUDevice(stored, device) log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", stored.GPUProfile, "uuid", device.MdevUUID) @@ -179,7 +179,7 @@ func (m *manager) startInstance( MdevUUID: device.MdevUUID, InstanceID: id, } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { + if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID, "error", err) if saveErr := m.saveMetadata(meta); saveErr != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", saveErr) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 23ea2d825..ae100a39e 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -5,9 +5,47 @@ import ( "path/filepath" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) +func validateVGPUHypervisor(hvType hypervisor.Type) error { + if hvType != hypervisor.TypeQEMU { + return fmt.Errorf("vGPU is only supported with qemu, got %s", hvType) + } + return nil +} + +// VGPUCleanupPendingError reports a failed create whose vGPU release also +// failed during rollback. The instance record identified by InstanceID is +// retained so the release can be retried; deleting the instance retries it. +type VGPUCleanupPendingError struct { + InstanceID string + Err error +} + +func (e *VGPUCleanupPendingError) Error() string { + return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID) +} + +func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } + +func (m *manager) createVGPUDevice(ctx context.Context, profileName, instanceID string) (*devices.VGPUDevice, error) { + create := m.createVGPU + if create == nil { + create = devices.CreateVGPU + } + return create(ctx, profileName, instanceID) +} + +func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { + destroy := m.destroyVGPU + if destroy == nil { + destroy = devices.DestroyVGPU + } + return destroy(ctx, assignment) +} + func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice) { stored.GPUFramework = device.Framework stored.GPUDevicePath = device.SysfsPath @@ -37,7 +75,7 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) MdevUUID: stored.GPUMdevUUID, InstanceID: stored.Id, } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { + if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { return err } } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 79e6fe0c0..da8426efe 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -2,11 +2,11 @@ package instances import ( "context" + "errors" "os" "path/filepath" "sync" "testing" - _ "unsafe" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" @@ -40,7 +40,7 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { DataDir: m.paths.InstanceDir("failed-create"), } - m.cleanupFailedCreate(context.Background(), stored.Id, stored) + assert.True(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored)) retained, err := m.loadMetadata(stored.Id) require.NoError(t, err) @@ -56,40 +56,43 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { assert.Empty(t, retained.DataDir) } -//go:linkname hostVendorVFIO github.com/kernel/hypeman/lib/devices.hostVendorVFIO -var hostVendorVFIO vendorVFIOSysfs +func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("failed-create")) -type vendorVFIOSysfs struct { - pciDevicesPath string - procPath string - vfioDevicesPath string - owners map[string]string + assert.False(t, m.cleanupFailedCreate(context.Background(), "failed-create", nil)) + _, err := m.loadMetadata("failed-create") + require.Error(t, err) } -func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { - root := t.TempDir() - pciDevicesPath := filepath.Join(root, "sys", "bus", "pci", "devices") - vfAddress := "0000:82:00.4" - nvidiaPath := filepath.Join(pciDevicesPath, vfAddress, "nvidia") - require.NoError(t, os.MkdirAll(nvidiaPath, 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "current_vgpu_type"), []byte("0"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "creatable_vgpu_types"), []byte("ID : vGPU Name\n1148 : NVIDIA L40S-2Q\n"), 0o644)) - - originalVendorVFIO := hostVendorVFIO - hostVendorVFIO = vendorVFIOSysfs{ - pciDevicesPath: pciDevicesPath, - procPath: filepath.Join(root, "proc"), - vfioDevicesPath: filepath.Join(root, "dev", "vfio", "devices"), - owners: make(map[string]string), - } - t.Cleanup(func() { hostVendorVFIO = originalVendorVFIO }) - require.NoError(t, os.MkdirAll(hostVendorVFIO.procPath, 0o755)) +func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { + t.Parallel() + + cause := errors.New("boot failed") + err := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause} + assert.ErrorIs(t, err, cause) + assert.Contains(t, err.Error(), "inst-1") +} +func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) { + t.Helper() m := &manager{ paths: paths.New(t.TempDir()), imageManager: readyFixtureImageManager{name: "test-image"}, instanceLocks: sync.Map{}, bootMarkerScans: sync.Map{}, + createVGPU: func(_ context.Context, profileName, _ string) (*devices.VGPUDevice, error) { + return &devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + ProfileType: "1148", + ProfileName: profileName, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + }, nil + }, + destroyVGPU: destroy, } const id = "start-rollback" require.NoError(t, m.ensureDirectories(id)) @@ -102,25 +105,48 @@ func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { SocketPath: m.paths.InstanceSocket(id, "noop.sock"), DataDir: m.paths.InstanceDir(id), }})) + return m, id +} + +func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { + var destroyed []devices.VGPUAssignment + m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }) - t.Setenv("TMPDIR", filepath.Join(root, "missing")) + t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) require.Error(t, err) + require.Len(t, destroyed, 1) + assert.Equal(t, devices.VGPUAssignment{ + Framework: devices.VGPUFrameworkVendorVFIO, + DevicePath: "/sys/bus/pci/devices/0000:82:00.4", + InstanceID: id, + }, destroyed[0]) + stored, err := m.loadMetadata(id) require.NoError(t, err) assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile) assert.Empty(t, stored.GPUFramework) assert.Empty(t, stored.GPUDevicePath) assert.Empty(t, stored.GPUMdevUUID) - assertFileContents(t, filepath.Join(nvidiaPath, "current_vgpu_type"), "0") } -func assertFileContents(t *testing.T, path, want string) { - t.Helper() - got, err := os.ReadFile(path) +func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return errors.New("destroy failed") + }) + + t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.Error(t, err) + + stored, err := m.loadMetadata(id) require.NoError(t, err) - assert.Equal(t, want, string(got)) + assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) } func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { From 38f14b1ef03b5094d229d191e9674f0d20d91ac8 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 7 Aug 2026 20:45:49 +0000 Subject: [PATCH 035/107] Generalize the create vGPU error text --- lib/instances/create.go | 2 +- lib/instances/create_mdev_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index dc70639fd..2cf34b5d9 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -56,7 +56,7 @@ func wrapCreateVGPUErr(profile string, err error) error { if errors.Is(err, devices.ErrVGPUNotSupportedOnMacOS) { return fmt.Errorf("%w: %w", ErrInvalidRequest, err) } - return fmt.Errorf("create vGPU mdev for profile %s: %w", profile, err) + return fmt.Errorf("create vGPU for profile %s: %w", profile, err) } // generateVsockCID converts first 8 chars of instance ID to a unique CID diff --git a/lib/instances/create_mdev_test.go b/lib/instances/create_mdev_test.go index e6e4f55c5..05b3e9d8c 100644 --- a/lib/instances/create_mdev_test.go +++ b/lib/instances/create_mdev_test.go @@ -63,7 +63,7 @@ func TestWrapCreateVGPUErr(t *testing.T) { { name: "other vGPU error", err: errors.New("boom"), - wantMessage: "create vGPU mdev for profile profile: boom", + wantMessage: "create vGPU for profile profile: boom", }, } { t.Run(tc.name, func(t *testing.T) { From 1411cbd43ef984833acc9233138b25b8a188701e Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sat, 8 Aug 2026 00:58:24 +0000 Subject: [PATCH 036/107] Scope vGPU claim scan to vendor VFIO and close reconcile gaps --- cmd/api/api/instances.go | 14 +++++++------ cmd/api/api/instances_test.go | 31 ++++++++++++++++++++++++++++ cmd/api/main.go | 8 +++++-- cmd/api/main_test.go | 30 +++++++++++++++++++++++++++ lib/instances/lifecycle_noop_test.go | 4 ++-- lib/instances/vgpu.go | 15 +++++++++++--- lib/instances/vgpu_test.go | 21 +++++++++++++++++++ 7 files changed, 110 insertions(+), 13 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index e9cee6ed9..2e2ef615a 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -364,6 +364,14 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst if err != nil { var vgpuPending *instances.VGPUCleanupPendingError switch { + // Checked first: it wraps the original create error, so a later + // errors.Is case would match the cause and hide the retained instance. + case errors.As(err, &vgpuPending): + log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) + return oapi.CreateInstance500JSONResponse{ + Code: "vgpu_cleanup_pending", + Message: fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID), + }, nil case errors.Is(err, instances.ErrImageNotReady): return oapi.CreateInstance400JSONResponse{ Code: "image_not_ready", @@ -414,12 +422,6 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Code: "not_found", Message: err.Error(), }, nil - case errors.As(err, &vgpuPending): - log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) - return oapi.CreateInstance500JSONResponse{ - Code: "vgpu_cleanup_pending", - Message: fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID), - }, nil default: log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) return oapi.CreateInstance500JSONResponse{ diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index cc94c572d..b2bc46871 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -17,6 +17,7 @@ import ( "github.com/kernel/hypeman/lib/instances" "github.com/kernel/hypeman/lib/instances/phasetracking" mw "github.com/kernel/hypeman/lib/middleware" + "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" "github.com/kernel/hypeman/lib/paths" restartpolicy "github.com/kernel/hypeman/lib/restart-policy" @@ -47,6 +48,36 @@ func TestGetInstance_NotFound(t *testing.T) { require.Error(t, err) } +type createErrorInstanceManager struct { + instances.Manager + err error +} + +func (m createErrorInstanceManager) CreateInstance(context.Context, instances.CreateInstanceRequest) (*instances.Instance, error) { + return nil, m.err +} + +// A retained-assignment error must win over the mapping of the create error +// it wraps, or the response omits the instance the caller has to delete. +func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { + t.Parallel() + svc := newTestService(t) + svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Err: network.ErrNameExists, + }} + + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{Image: "test-image"}, + }) + require.NoError(t, err) + + pending, ok := resp.(oapi.CreateInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "inst-1") +} + func TestCreateInstance_AutoPullImage(t *testing.T) { t.Parallel() if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { diff --git a/cmd/api/main.go b/cmd/api/main.go index 0479f1583..3374d9682 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -192,10 +192,14 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances. } protected := make(map[string]struct{}) for _, inst := range allInstances { - if inst.GPUDevicePath == "" || inst.HypervisorPID == nil { + if inst.GPUDevicePath == "" { continue } - if !instances.HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) { + // A nil PID does not mean the assignment is orphaned: the PID is + // persisted only after the hypervisor starts, so a crash during boot + // leaves the device path without one. Only skip protection when the + // recorded hypervisor is known to be gone. + if inst.HypervisorPID != nil && !instances.HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) { continue } protected[inst.GPUDevicePath] = struct{}{} diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 34dbba428..573a404c1 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -2,15 +2,18 @@ package main import ( "bytes" + "context" "net/http" "net/http/httptest" "net/url" + "os/exec" "testing" "time" "github.com/getkin/kin-openapi/openapi3filter" "github.com/go-chi/chi/v5" "github.com/golang-jwt/jwt/v5" + "github.com/kernel/hypeman/lib/instances" mw "github.com/kernel/hypeman/lib/middleware" "github.com/kernel/hypeman/lib/oapi" nethttpmiddleware "github.com/oapi-codegen/nethttp-middleware" @@ -338,3 +341,30 @@ func TestImageNameWithSlashes_URLEncoding(t *testing.T) { }) } } + +type vgpuReconcileManagerStub struct { + instances.Manager + list []instances.Instance +} + +func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([]instances.Instance, error) { + return s.list, nil +} + +// The hypervisor PID is persisted only after boot, so an assignment without +// one may belong to a VM that is still starting and must stay protected. +func TestLiveInstanceVGPUDevicePathsProtectsAssignmentsWithoutPID(t *testing.T) { + dead := exec.Command("true") + require.NoError(t, dead.Run()) + deadPID := dead.Process.Pid + + manager := vgpuReconcileManagerStub{list: []instances.Instance{ + {StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4"}}, + {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", HypervisorPID: &deadPID}}, + }} + + protected, err := liveInstanceVGPUDevicePaths(context.Background(), manager) + require.NoError(t, err) + assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") +} diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index f3baa9dae..15f632b3a 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -202,7 +202,7 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" - meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUFramework = devices.VGPUFrameworkVendorVFIO meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" require.NoError(t, m.saveMetadata(meta)) @@ -230,7 +230,7 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { SocketPath: socketPath, DataDir: m.paths.InstanceDir(claimantID), GPUProfile: "NVIDIA L40S-2Q", - GPUFramework: devices.VGPUFramework("future-framework"), + GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", }})) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index ae100a39e..9b97fddb7 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -61,9 +61,18 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { path := storedVGPUDevicePath(stored) if path != "" { - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path) - if err != nil { - return err + // Vendor VFIO VFs are reused across instances, so stale metadata can + // point at a path claimed by a live instance and the release must fail + // closed on an incomplete inventory. mdev UUIDs are unique and never + // reused, so skip the scan there — it would let one unreadable + // metadata file block every mdev release on the host. + claimed := false + if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { + var err error + claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path) + if err != nil { + return err + } } if claimed { logger.FromContext(ctx).WarnContext(ctx, "dropping stale vGPU assignment claimed by another live instance", diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index da8426efe..6f9a20073 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -178,6 +178,27 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. assert.True(t, claimed) } +func TestReleaseStoredVGPUSkipsClaimScanForMdev(t *testing.T) { + t.Parallel() + + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, + } + require.NoError(t, m.ensureDirectories("invalid-instance")) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid-instance"), []byte("{"), 0o644)) + + stored := &StoredMetadata{ + Id: "mdev-instance", + GPUFramework: devices.VGPUFrameworkMdev, + GPUMdevUUID: "uuid-1", + GPUDevicePath: "/sys/bus/mdev/devices/uuid-1", + } + require.NoError(t, m.releaseStoredVGPU(context.Background(), stored), + "an unreadable metadata file must not block mdev releases") + assert.Empty(t, stored.GPUDevicePath) +} + func TestStoredVGPUDevicePath(t *testing.T) { t.Parallel() From 520533a92dd5f28afc9500a2f0050fccd9eb88fe Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sun, 9 Aug 2026 06:25:10 +0000 Subject: [PATCH 037/107] Harden the vendor VFIO release path Enable vendor VFIO dispatch in CreateVGPU now that the lifecycle persists assignments durably and guards releases. Protect nil-PID claims in the release guard: the hypervisor PID is only persisted after the claimant boots, so a matching assignment without a PID must be treated as live, matching the startup reconcile protection. Scan raw metadata instead of hydrating instances for the claim check. Hydration derives state through hypervisor queries for every instance on the host, which every vendor VFIO release would pay; the guard only needs the stored assignment, PID, and socket. Unreadable metadata still fails the release closed. Report pending vGPU cleanup even when retaining the rollback record fails: the destroy already failed, so the caller must learn about the outstanding assignment either way. --- integration/vgpu_test.go | 5 ---- lib/devices/vgpu_linux.go | 5 +--- lib/instances/create.go | 11 +++++--- lib/instances/vgpu.go | 35 ++++++++++++++++++++++---- lib/instances/vgpu_test.go | 51 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 89 insertions(+), 18 deletions(-) diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 7873aa35c..1f1a82776 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -324,11 +324,6 @@ func checkVGPUTestPrerequisites() (string, string) { if framework == devices.VGPUFrameworkNone { return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", "" } - if framework == devices.VGPUFrameworkVendorVFIO { - // CreateVGPU rejects vendor VFIO until the instance lifecycle - // integration lands. - return "vGPU test requires the vendor VFIO instance lifecycle integration", "" - } // Check for available profiles profiles, err := devices.ListGPUProfiles() diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index b3c497899..be92e8ee6 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -73,10 +73,7 @@ func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevic MdevUUID: mdev.UUID, }, nil case VGPUFrameworkVendorVFIO: - // The instance lifecycle does not yet persist vendor VFIO assignments - // durably or guard their release against live claims, so keep the - // backend out of the create path until that integration lands. - return nil, fmt.Errorf("vendor VFIO vGPU support is not yet integrated with the instance lifecycle") + return hostVendorVFIO.create(ctx, profileName, instanceID) default: return nil, fmt.Errorf("vGPU framework not available") } diff --git a/lib/instances/create.go b/lib/instances/create.go index 2cf34b5d9..f1edb8aaa 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -641,8 +641,11 @@ func resolveCreateExpiration(req CreateInstanceRequest, now time.Time) (*time.Ti return &expiresAt, nil } -// cleanupFailedCreate reports whether it retained instance metadata for a -// vGPU assignment whose release failed during rollback. +// cleanupFailedCreate reports whether a vGPU assignment is still outstanding +// after a failed create. The vGPU destroy already failed when retainedVGPU is +// set, so the pending cleanup is reported even when the retention record +// cannot be persisted — in that case the assignment is orphaned until the +// next startup reconcile, and the caller must still surface it. func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) bool { if retainedVGPU == nil { m.deleteInstanceData(id) @@ -652,7 +655,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG log := logger.FromContext(ctx) if err := m.ensureDirectories(id); err != nil { log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) - return false + return true } retained := StoredMetadata{ Id: id, @@ -662,7 +665,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) - return false + return true } return true } diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 9b97fddb7..d62fc0d52 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -93,19 +93,44 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) return nil } +// vgpuAssignmentClaimedByLiveInstance reports whether another instance's +// stored metadata claims devicePath. It reads raw metadata instead of +// hydrating full instances: the scan runs on every vendor VFIO release, and +// deriving state would query the hypervisor of every instance on the host. +// It fails closed: unreadable metadata is an error, and a matching claim +// without a persisted PID counts as live because the PID is only persisted +// after the claimant's hypervisor starts. func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { - instances, err := m.ListInstancesForReconcile(ctx) + files, err := m.listMetadataFilesWithStatErrors(true) if err != nil { return false, fmt.Errorf("list instances for vGPU release check: %w", err) } - for i := range instances { - inst := &instances[i] - if inst.Id == excludeID || storedVGPUDevicePath(&inst.StoredMetadata) != devicePath || inst.HypervisorPID == nil { + for _, file := range files { + id := filepath.Base(filepath.Dir(file)) + if id == excludeID { continue } - if HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) { + meta, err := m.loadMetadata(id) + if err != nil { + return false, fmt.Errorf("load metadata for vGPU release check: instance %s: %w", id, err) + } + stored := &meta.StoredMetadata + if storedVGPUDevicePath(stored) != devicePath { + continue + } + if stored.HypervisorPID == nil { return true, nil } + if HypervisorProcessExists(*stored.HypervisorPID, stored.SocketPath) { + return true, nil + } + // The stored PID can be stale after a hypeman restart; a live owner + // of the claimant's socket still marks the claim as live. + if stored.SocketPath != "" && !ProcessExists(*stored.HypervisorPID) { + if owner, _, err := hypervisor.ResolveProcessPID(stored.SocketPath); err == nil && ProcessExists(owner) { + return true, nil + } + } } return false, nil } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 6f9a20073..0253d83da 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -67,6 +67,23 @@ func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { require.Error(t, err) } +func TestCleanupFailedCreateReportsPendingWhenRetentionFails(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + // A file at the guests directory path makes ensureDirectories fail even + // when running as root. + require.NoError(t, os.WriteFile(m.paths.GuestsDir(), nil, 0o644)) + + stored := &StoredMetadata{ + Id: "failed-create", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + } + assert.True(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored), + "a failed retention must still report the outstanding vGPU assignment") +} + func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { t.Parallel() @@ -178,6 +195,40 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. assert.True(t, claimed) } +func TestVGPUAssignmentClaimedByLiveInstanceProtectsNilPIDClaim(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("booting-claimant")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "booting-claimant", + Name: "booting-claimant", + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.NoError(t, err) + assert.True(t, claimed, "a matching claim without a persisted PID must be treated as live: the PID is only persisted after the claimant boots") +} + +func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("dead-claimant")) + deadPID := 1 << 30 + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "dead-claimant", + Name: "dead-claimant", + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + HypervisorPID: &deadPID, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.NoError(t, err) + assert.False(t, claimed, "a claim whose hypervisor is gone must not block the release") +} + func TestReleaseStoredVGPUSkipsClaimScanForMdev(t *testing.T) { t.Parallel() From 7e1737c3bb78ab73f71998307b5fb639670d659b Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sun, 9 Aug 2026 07:40:01 +0000 Subject: [PATCH 038/107] Fail closed on retained vGPU cleanup --- lib/instances/create.go | 26 +++++------- lib/instances/process_identity_linux_test.go | 43 ++++++++++++++++++++ lib/instances/vgpu.go | 21 ++++------ lib/instances/vgpu_test.go | 23 ++++++----- 4 files changed, 76 insertions(+), 37 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index f1edb8aaa..de00bdcf3 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -281,19 +281,18 @@ func (m *manager) createInstance( var retainedVGPU *StoredMetadata // Setup cleanup stack early so device attachment errors trigger cleanup. - // When rollback retains a vGPU assignment, surface the retained instance - // ID to the caller so the record is discoverable and can be deleted to - // retry the release. The wrapping defer is registered first so it runs - // after cu.Clean has decided whether metadata was retained. - vgpuRetained := false + // When rollback cannot release a vGPU assignment, report whether its + // retention record was persisted. The wrapping defer is registered first + // so it runs after cu.Clean has attempted to retain the metadata. + vgpuPersisted := false defer func() { - if retErr != nil && vgpuRetained { - retErr = &VGPUCleanupPendingError{InstanceID: id, Err: retErr} + if retErr != nil && retainedVGPU != nil { + retErr = &VGPUCleanupPendingError{InstanceID: id, Retained: vgpuPersisted, Err: retErr} } }() cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) - vgpuRetained = m.cleanupFailedCreate(ctx, id, retainedVGPU) + vgpuPersisted = m.cleanupFailedCreate(ctx, id, retainedVGPU) }) defer cu.Clean() @@ -641,11 +640,8 @@ func resolveCreateExpiration(req CreateInstanceRequest, now time.Time) (*time.Ti return &expiresAt, nil } -// cleanupFailedCreate reports whether a vGPU assignment is still outstanding -// after a failed create. The vGPU destroy already failed when retainedVGPU is -// set, so the pending cleanup is reported even when the retention record -// cannot be persisted — in that case the assignment is orphaned until the -// next startup reconcile, and the caller must still surface it. +// cleanupFailedCreate reports whether the retention record for a vGPU +// assignment whose release failed during rollback was persisted. func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) bool { if retainedVGPU == nil { m.deleteInstanceData(id) @@ -655,7 +651,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG log := logger.FromContext(ctx) if err := m.ensureDirectories(id); err != nil { log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) - return true + return false } retained := StoredMetadata{ Id: id, @@ -665,7 +661,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) - return true + return false } return true } diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index 7450ed112..d5bd973d9 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -17,7 +17,9 @@ import ( "testing" "time" + "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -508,6 +510,47 @@ func TestRefreshHypervisorPIDResolvesSocketOwnerWhenStoredPIDIsDead(t *testing.T assert.Equal(t, hostBootID(), stored.HypervisorBootID) } +func TestVGPUAssignmentClaimedByLiveInstanceProtectsReusedPIDClaim(t *testing.T) { + socketPath := filepath.Join(t.TempDir(), "test.sock") + owner := exec.Command(os.Args[0], "-test.run=^TestHypervisorProcessExistsWithReboundSocketPathHelper$") + owner.Env = append(os.Environ(), "HYPERVISOR_SOCKET_HELPER=1", "HYPERVISOR_SOCKET_PATH="+socketPath) + stdin, err := owner.StdinPipe() + require.NoError(t, err) + stdout, err := owner.StdoutPipe() + require.NoError(t, err) + require.NoError(t, owner.Start()) + t.Cleanup(func() { + _ = stdin.Close() + _ = owner.Process.Kill() + _ = owner.Wait() + }) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + + stale := exec.Command("sleep", "30") + require.NoError(t, stale.Start()) + t.Cleanup(func() { + _ = stale.Process.Kill() + _ = stale.Wait() + }) + + m := &manager{paths: paths.New(t.TempDir())} + const devicePath = "/sys/bus/pci/devices/0000:82:00.4" + stalePID := stale.Process.Pid + require.NoError(t, m.ensureDirectories("live-claimant")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "live-claimant", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + HypervisorPID: &stalePID, + SocketPath: socketPath, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", devicePath) + require.NoError(t, err) + assert.True(t, claimed) +} + func TestKillHypervisorSurvivesConcurrentReaper(t *testing.T) { socketPath := filepath.Join(t.TempDir(), "test.sock") process := exec.Command(os.Args[0], "-test.run=^TestSocketListenerHelper$") diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index d62fc0d52..e27cf7a54 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -5,7 +5,6 @@ import ( "path/filepath" "github.com/kernel/hypeman/lib/devices" - "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) @@ -17,15 +16,19 @@ func validateVGPUHypervisor(hvType hypervisor.Type) error { } // VGPUCleanupPendingError reports a failed create whose vGPU release also -// failed during rollback. The instance record identified by InstanceID is -// retained so the release can be retried; deleting the instance retries it. +// failed during rollback. When Retained is true, deleting the retained instance +// retries the release; otherwise startup reconciliation recovers the assignment. type VGPUCleanupPendingError struct { InstanceID string + Retained bool Err error } func (e *VGPUCleanupPendingError) Error() string { - return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID) + if e.Retained { + return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID) + } + return fmt.Sprintf("%v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", e.Err, e.InstanceID) } func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } @@ -121,16 +124,10 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu if stored.HypervisorPID == nil { return true, nil } - if HypervisorProcessExists(*stored.HypervisorPID, stored.SocketPath) { + pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.SocketPath) + if err != nil || pid > 0 { return true, nil } - // The stored PID can be stale after a hypeman restart; a live owner - // of the claimant's socket still marks the claim as live. - if stored.SocketPath != "" && !ProcessExists(*stored.HypervisorPID) { - if owner, _, err := hypervisor.ResolveProcessPID(stored.SocketPath); err == nil && ProcessExists(owner) { - return true, nil - } - } } return false, nil } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 0253d83da..a0d3ae9da 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -67,30 +67,33 @@ func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { require.Error(t, err) } -func TestCleanupFailedCreateReportsPendingWhenRetentionFails(t *testing.T) { +func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { t.Parallel() m := &manager{paths: paths.New(t.TempDir())} - // A file at the guests directory path makes ensureDirectories fail even - // when running as root. - require.NoError(t, os.WriteFile(m.paths.GuestsDir(), nil, 0o644)) + const id = "failed-create" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, os.Mkdir(m.paths.InstanceMetadata(id), 0o755)) stored := &StoredMetadata{ - Id: "failed-create", + Id: id, GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", } - assert.True(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored), - "a failed retention must still report the outstanding vGPU assignment") + assert.False(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored)) } func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { t.Parallel() cause := errors.New("boot failed") - err := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause} - assert.ErrorIs(t, err, cause) - assert.Contains(t, err.Error(), "inst-1") + retained := &VGPUCleanupPendingError{InstanceID: "inst-1", Retained: true, Err: cause} + assert.ErrorIs(t, retained, cause) + assert.Equal(t, "boot failed; vGPU release failed during rollback, instance inst-1 retains the assignment", retained.Error()) + + unpersisted := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause} + assert.ErrorIs(t, unpersisted, cause) + assert.Equal(t, "boot failed; vGPU release failed during rollback and the retention record for instance inst-1 could not be saved; the assignment is recovered on the next startup reconcile", unpersisted.Error()) } func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) { From 928f8d2226f504633d377ca578ddd37613d4dbe0 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sun, 9 Aug 2026 22:02:02 +0000 Subject: [PATCH 039/107] Report surviving vGPU retention metadata --- lib/instances/create.go | 14 ++++++++++++-- lib/instances/vgpu_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index de00bdcf3..8af298b66 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -649,9 +649,19 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } log := logger.FromContext(ctx) + retentionFailed := func() bool { + meta, err := m.loadMetadata(id) + if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { + return true + } + if err := m.deleteInstanceData(id); err != nil { + log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) + } + return false + } if err := m.ensureDirectories(id); err != nil { log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) - return false + return retentionFailed() } retained := StoredMetadata{ Id: id, @@ -661,7 +671,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) - return false + return retentionFailed() } return true } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index a0d3ae9da..b6831109c 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -81,6 +81,34 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", } assert.False(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored)) + _, err := m.loadMetadata(id) + require.Error(t, err) +} + +func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m := &manager{paths: paths.New(t.TempDir())} + const id = "failed-create" + require.NoError(t, m.ensureDirectories(id)) + stored := &StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + } + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: *stored})) + + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + assert.True(t, m.cleanupFailedCreate(context.Background(), id, stored)) + retained, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, stored.GPUFramework, retained.GPUFramework) + assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) } func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { From 3f80948246a1ed30f2e5c24d63a06b0d6adeaa0b Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sun, 9 Aug 2026 22:02:38 +0000 Subject: [PATCH 040/107] Return accurate vGPU cleanup guidance --- cmd/api/api/instances.go | 8 ++++++-- cmd/api/api/instances_test.go | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 2e2ef615a..d62f1d7d2 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -365,12 +365,16 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst var vgpuPending *instances.VGPUCleanupPendingError switch { // Checked first: it wraps the original create error, so a later - // errors.Is case would match the cause and hide the retained instance. + // errors.Is case would match the cause and hide the pending vGPU cleanup. case errors.As(err, &vgpuPending): log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) + message := fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID) + if !vgpuPending.Retained { + message = fmt.Sprintf("failed to create instance; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.InstanceID) + } return oapi.CreateInstance500JSONResponse{ Code: "vgpu_cleanup_pending", - Message: fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID), + Message: message, }, nil case errors.Is(err, instances.ErrImageNotReady): return oapi.CreateInstance400JSONResponse{ diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index b2bc46871..eec8e6dbb 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -64,6 +64,7 @@ func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) svc := newTestService(t) svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{ InstanceID: "inst-1", + Retained: true, Err: network.ErrNameExists, }} @@ -76,6 +77,28 @@ func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) assert.Contains(t, pending.Message, "inst-1") + assert.Contains(t, pending.Message, "delete it to retry") +} + +func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance(t *testing.T) { + t.Parallel() + svc := newTestService(t) + svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Err: network.ErrNameExists, + }} + + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{Image: "test-image"}, + }) + require.NoError(t, err) + + pending, ok := resp.(oapi.CreateInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") + assert.Contains(t, pending.Message, "startup reconcile") + assert.NotContains(t, pending.Message, "delete") } func TestCreateInstance_AutoPullImage(t *testing.T) { From 59601959569b1ea30913c86ca304d684b12bec99 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sun, 9 Aug 2026 22:26:00 +0000 Subject: [PATCH 041/107] Clarify vGPU retention fallback --- lib/instances/create.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index 8af298b66..42fdcac39 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -649,7 +649,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } log := logger.FromContext(ctx) - retentionFailed := func() bool { + retentionSurvives := func() bool { meta, err := m.loadMetadata(id) if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { return true @@ -661,7 +661,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } if err := m.ensureDirectories(id); err != nil { log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) - return retentionFailed() + return retentionSurvives() } retained := StoredMetadata{ Id: id, @@ -671,7 +671,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) - return retentionFailed() + return retentionSurvives() } return true } From f677f8bfcd265b634550b3fc7fed8c67ec004658 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 13:43:17 +0000 Subject: [PATCH 042/107] Pass hypervisor identity token to vGPU claim check --- lib/instances/vgpu.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index e27cf7a54..acc5eb749 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -124,7 +124,7 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu if stored.HypervisorPID == nil { return true, nil } - pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.SocketPath) + pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.SocketPath) if err != nil || pid > 0 { return true, nil } From 51508f0e3a336d5ba88e2e55414c44a3a6b022f2 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:12:07 +0000 Subject: [PATCH 043/107] Fail safely on ambiguous vGPU claims --- lib/instances/vgpu.go | 15 +++++++++------ lib/instances/vgpu_test.go | 38 ++++++++++++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index acc5eb749..9bca44dbc 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -96,13 +96,13 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) return nil } -// vgpuAssignmentClaimedByLiveInstance reports whether another instance's +// vgpuAssignmentClaimedByLiveInstance reports whether another live instance's // stored metadata claims devicePath. It reads raw metadata instead of // hydrating full instances: the scan runs on every vendor VFIO release, and // deriving state would query the hypervisor of every instance on the host. -// It fails closed: unreadable metadata is an error, and a matching claim -// without a persisted PID counts as live because the PID is only persisted -// after the claimant's hypervisor starts. +// A confirmed live claimant returns true. Unreadable metadata, a missing PID, +// or unverifiable process ownership returns an error so the requester retains +// its assignment for a later retry. func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { files, err := m.listMetadataFilesWithStatErrors(true) if err != nil { @@ -122,10 +122,13 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu continue } if stored.HypervisorPID == nil { - return true, nil + return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) } pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.SocketPath) - if err != nil || pid > 0 { + if err != nil { + return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) + } + if pid > 0 { return true, nil } } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index b6831109c..d4e0c6489 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -226,7 +226,7 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. assert.True(t, claimed) } -func TestVGPUAssignmentClaimedByLiveInstanceProtectsNilPIDClaim(t *testing.T) { +func TestVGPUAssignmentClaimedByLiveInstanceErrorsOnNilPIDClaim(t *testing.T) { t.Parallel() m := &manager{paths: paths.New(t.TempDir())} @@ -237,9 +237,9 @@ func TestVGPUAssignmentClaimedByLiveInstanceProtectsNilPIDClaim(t *testing.T) { GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", }})) - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") - require.NoError(t, err) - assert.True(t, claimed, "a matching claim without a persisted PID must be treated as live: the PID is only persisted after the claimant boots") + _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.Error(t, err) + assert.Contains(t, err.Error(), "booting-claimant") } func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) { @@ -281,6 +281,36 @@ func TestReleaseStoredVGPUSkipsClaimScanForMdev(t *testing.T) { assert.Empty(t, stored.GPUDevicePath) } +func TestReleaseStoredVGPURetainsRequesterOnAmbiguousClaim(t *testing.T) { + t.Parallel() + + const devicePath = "/sys/bus/pci/devices/0000:82:00.4" + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + t.Fatal("destroyVGPU must not be called for an ambiguous claim") + return nil + }, + } + require.NoError(t, m.ensureDirectories("ambiguous-claimant")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "ambiguous-claimant", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + }})) + + stored := &StoredMetadata{ + Id: "requester", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + } + err := m.releaseStoredVGPU(context.Background(), stored) + require.Error(t, err) + assert.Contains(t, err.Error(), "ambiguous-claimant") + assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) + assert.Equal(t, devicePath, stored.GPUDevicePath) +} + func TestStoredVGPUDevicePath(t *testing.T) { t.Parallel() From 3af8301c0c9e098769a6cf5f4434cd886a1290ba Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:14:21 +0000 Subject: [PATCH 044/107] Expose retained vGPU instance IDs --- cmd/api/api/instances.go | 6 ++++++ cmd/api/api/instances_test.go | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index d62f1d7d2..8cfe72728 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -369,12 +369,18 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst case errors.As(err, &vgpuPending): log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) message := fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID) + innerCode := "vgpu_retained_instance" if !vgpuPending.Retained { message = fmt.Sprintf("failed to create instance; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.InstanceID) + innerCode = "vgpu_unretained_instance" } return oapi.CreateInstance500JSONResponse{ Code: "vgpu_cleanup_pending", Message: message, + InnerError: &oapi.ErrorDetail{ + Code: lo.ToPtr(innerCode), + Message: lo.ToPtr(vgpuPending.InstanceID), + }, }, nil case errors.Is(err, instances.ErrImageNotReady): return oapi.CreateInstance400JSONResponse{ diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index eec8e6dbb..a27334849 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -78,6 +78,11 @@ func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) assert.Contains(t, pending.Message, "inst-1") assert.Contains(t, pending.Message, "delete it to retry") + require.NotNil(t, pending.InnerError) + require.NotNil(t, pending.InnerError.Code) + assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code) + require.NotNil(t, pending.InnerError.Message) + assert.Equal(t, "inst-1", *pending.InnerError.Message) } func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance(t *testing.T) { @@ -99,6 +104,11 @@ func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance( assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") assert.Contains(t, pending.Message, "startup reconcile") assert.NotContains(t, pending.Message, "delete") + require.NotNil(t, pending.InnerError) + require.NotNil(t, pending.InnerError.Code) + assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code) + require.NotNil(t, pending.InnerError.Message) + assert.Equal(t, "inst-1", *pending.InnerError.Message) } func TestCreateInstance_AutoPullImage(t *testing.T) { From ad4c71388cc21547db92497412b60bab2b74eadf Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 15:31:22 +0000 Subject: [PATCH 045/107] Harden vGPU startup rollback recovery --- cmd/api/main.go | 62 +++++++++++++++++++++++++++---------- cmd/api/main_test.go | 18 +++++++---- lib/instances/create.go | 4 +++ lib/instances/start.go | 25 +++------------ lib/instances/types.go | 3 +- lib/instances/vgpu.go | 27 +++++++++++++++- lib/instances/vgpu_test.go | 63 ++++++++++++++++++++++++++++++++++++-- 7 files changed, 155 insertions(+), 47 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 3374d9682..46308e201 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -185,26 +185,62 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta }, logger), nil } -func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, error) { +const vgpuAssignmentStartupGracePeriod = 5 * time.Minute + +func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, time.Duration, error) { allInstances, err := instanceManager.ListInstancesForReconcile(ctx) if err != nil { - return nil, err + return nil, 0, err } protected := make(map[string]struct{}) + var retryAfter time.Duration for _, inst := range allInstances { if inst.GPUDevicePath == "" { continue } - // A nil PID does not mean the assignment is orphaned: the PID is - // persisted only after the hypervisor starts, so a crash during boot - // leaves the device path without one. Only skip protection when the - // recorded hypervisor is known to be gone. - if inst.HypervisorPID != nil && !instances.HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) { + if inst.HypervisorPID != nil { + if !instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.SocketPath) { + continue + } + protected[inst.GPUDevicePath] = struct{}{} + continue + } + if inst.GPUAssignedAt == nil { + continue + } + remaining := vgpuAssignmentStartupGracePeriod - time.Since(*inst.GPUAssignedAt) + if remaining <= 0 { continue } protected[inst.GPUDevicePath] = struct{}{} + if retryAfter == 0 || remaining < retryAfter { + retryAfter = remaining + } } - return protected, nil + return protected, retryAfter, nil +} + +func reconcileVGPUs(ctx context.Context, instanceManager instances.Manager, logger *slog.Logger) { + protected, retryAfter, err := liveInstanceVGPUDevicePaths(ctx, instanceManager) + if err != nil { + logger.Warn("failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconciliation", "error", err) + return + } + if err := devices.ReconcileVGPUs(ctx, protected); err != nil { + logger.Warn("failed to reconcile vGPU devices", "error", err) + } + if retryAfter <= 0 { + return + } + go func() { + timer := time.NewTimer(retryAfter) + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + reconcileVGPUs(ctx, instanceManager, logger) + } + }() } func run() error { @@ -408,15 +444,7 @@ func run() error { // Reconcile vGPU devices (clears orphaned vGPUs from previous runs) logger.Info("Reconciling vGPU devices...") - protected, err := liveInstanceVGPUDevicePaths(app.Ctx, app.InstanceManager) - if err != nil { - logger.Warn("failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconciliation", "error", err) - protected = nil - } - if err := devices.ReconcileVGPUs(app.Ctx, protected); err != nil { - // Log but don't fail - vGPU cleanup is best-effort - logger.Warn("failed to reconcile vGPU devices", "error", err) - } + reconcileVGPUs(ctx, app.InstanceManager, logger) // Wire up resource validator for aggregate limit checking // This enables the instance manager to validate CPU, memory, network, and GPU diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 573a404c1..02909e6ad 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -351,20 +351,26 @@ func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([] return s.list, nil } -// The hypervisor PID is persisted only after boot, so an assignment without -// one may belong to a VM that is still starting and must stay protected. -func TestLiveInstanceVGPUDevicePathsProtectsAssignmentsWithoutPID(t *testing.T) { +func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) { dead := exec.Command("true") require.NoError(t, dead.Run()) deadPID := dead.Process.Pid + recent := time.Now().Add(-time.Minute) + stale := time.Now().Add(-vgpuAssignmentStartupGracePeriod - time.Minute) manager := vgpuReconcileManagerStub{list: []instances.Instance{ - {StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4"}}, - {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", HypervisorPID: &deadPID}}, + {StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}}, + {StoredMetadata: instances.StoredMetadata{Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}}, + {StoredMetadata: instances.StoredMetadata{Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}}, + {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorPID: &deadPID}}, }} - protected, err := liveInstanceVGPUDevicePaths(context.Background(), manager) + protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager) require.NoError(t, err) + require.Positive(t, retryAfter) + require.LessOrEqual(t, retryAfter, vgpuAssignmentStartupGracePeriod) assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") } diff --git a/lib/instances/create.go b/lib/instances/create.go index 42fdcac39..98e1a6f29 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -277,6 +277,7 @@ func (m *manager) createInstance( var gpuFramework devices.VGPUFramework var gpuDevicePath string var gpuMdevUUID string + var gpuAssignedAt *time.Time var stored *StoredMetadata var retainedVGPU *StoredMetadata @@ -318,6 +319,8 @@ func (m *manager) createInstance( gpuFramework = gpuDevice.Framework gpuDevicePath = gpuDevice.SysfsPath gpuMdevUUID = gpuDevice.MdevUUID + assignedAt := m.nowUTC() + gpuAssignedAt = &assignedAt log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", gpuProfile, "uuid", gpuMdevUUID) // Add vGPU cleanup to stack @@ -426,6 +429,7 @@ func (m *manager) createInstance( GPUFramework: gpuFramework, GPUDevicePath: gpuDevicePath, GPUMdevUUID: gpuMdevUUID, + GPUAssignedAt: gpuAssignedAt, Entrypoint: req.Entrypoint, Cmd: req.Cmd, SkipKernelHeaders: req.SkipKernelHeaders, diff --git a/lib/instances/start.go b/lib/instances/start.go index b830bafb2..775038559 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/egressproxy" "github.com/kernel/hypeman/lib/instances/phasetracking" "github.com/kernel/hypeman/lib/logger" @@ -64,6 +63,8 @@ func (m *manager) startInstance( } } + rollbackMeta := *meta + // 2a. Clear stale exit info from previous run and apply command overrides stored.ExitCode = nil stored.ExitMessage = "" @@ -168,28 +169,12 @@ func (m *manager) startInstance( log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) } - setStoredVGPUDevice(stored, device) + assignedAt := m.nowUTC() + setStoredVGPUDevice(stored, device, assignedAt) log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", stored.GPUProfile, "uuid", device.MdevUUID) // Add vGPU cleanup to stack cu.Add(func() { - log.DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID) - assignment := devices.VGPUAssignment{ - Framework: device.Framework, - DevicePath: device.SysfsPath, - MdevUUID: device.MdevUUID, - InstanceID: id, - } - if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { - log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID, "error", err) - if saveErr := m.saveMetadata(meta); saveErr != nil { - log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", saveErr) - } - } else { - clearStoredVGPUDevice(stored) - if saveErr := m.saveMetadata(meta); saveErr != nil { - log.ErrorContext(ctx, "failed to save metadata after vGPU cleanup", "instance_id", id, "error", saveErr) - } - } + m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) }) if err := m.saveMetadata(meta); err != nil { log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err) diff --git a/lib/instances/types.go b/lib/instances/types.go index 6aac15985..a264498fd 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -154,7 +154,8 @@ type StoredMetadata struct { GPUProfile string // vGPU profile name (e.g., "L40S-1Q") GPUFramework devices.VGPUFramework GPUDevicePath string - GPUMdevUUID string // populated for mdev-backed vGPUs + GPUMdevUUID string // populated for mdev-backed vGPUs + GPUAssignedAt *time.Time // set before hypervisor startup to bound crash recovery protection // Command overrides (like docker run ) Entrypoint []string // Override image entrypoint (nil = use image default) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 9bca44dbc..3e7dc350c 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -3,6 +3,7 @@ package instances import ( "context" "path/filepath" + "time" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/logger" @@ -49,16 +50,40 @@ func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices. return destroy(ctx, assignment) } -func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice) { +func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice, assignedAt time.Time) { stored.GPUFramework = device.Framework stored.GPUDevicePath = device.SysfsPath stored.GPUMdevUUID = device.MdevUUID + stored.GPUAssignedAt = &assignedAt } func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUFramework = devices.VGPUFrameworkNone stored.GPUDevicePath = "" stored.GPUMdevUUID = "" + stored.GPUAssignedAt = nil +} + +func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) { + assignment := devices.VGPUAssignment{ + Framework: device.Framework, + DevicePath: device.SysfsPath, + MdevUUID: device.MdevUUID, + InstanceID: instanceID, + } + cleanupMeta := rollbackMeta + releaseErr := m.destroyVGPUAssignment(ctx, assignment) + if releaseErr != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "error", releaseErr) + setStoredVGPUDevice(&cleanupMeta.StoredMetadata, device, assignedAt) + } + if err := m.saveMetadata(&cleanupMeta); err != nil { + message := "failed to save metadata after vGPU cleanup" + if releaseErr != nil { + message = "failed to retain vGPU assignment metadata after cleanup failure" + } + logger.FromContext(ctx).ErrorContext(ctx, message, "instance_id", instanceID, "error", err) + } } func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index d4e0c6489..366e343f2 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "sync" "testing" + "time" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" @@ -188,13 +189,68 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { }) t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) - _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{Entrypoint: []string{"new-entrypoint"}}) require.Error(t, err) stored, err := m.loadMetadata(id) require.NoError(t, err) assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) + assert.NotNil(t, stored.GPUAssignedAt) + assert.Empty(t, stored.Entrypoint) +} + +func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + return nil + }, + } + const id = "failed-start" + require.NoError(t, m.ensureDirectories(id)) + + previousStart := time.Now().Add(-time.Hour).UTC() + previousProgramStart := previousStart.Add(time.Second) + exitCode := 1 + rollbackMeta := metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUProfile: "NVIDIA L40S-2Q", + Entrypoint: []string{"old-entrypoint"}, + Cmd: []string{"old-command"}, + StartedAt: &previousStart, + ProgramStartedAt: &previousProgramStart, + ExitCode: &exitCode, + ExitMessage: "previous exit", + }} + + partial := rollbackMeta + partial.Entrypoint = []string{"new-entrypoint"} + partial.Cmd = []string{"new-command"} + partial.StartedAt = ptr(time.Now().UTC()) + partial.ProgramStartedAt = nil + partial.ExitCode = nil + partial.ExitMessage = "" + assignedAt := time.Now().UTC() + device := &devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + setStoredVGPUDevice(&partial.StoredMetadata, device, assignedAt) + require.NoError(t, m.saveMetadata(&partial)) + + m.cleanupStartVGPU(context.Background(), id, device, assignedAt, rollbackMeta) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, rollbackMeta.Entrypoint, stored.Entrypoint) + assert.Equal(t, rollbackMeta.Cmd, stored.Cmd) + assert.Equal(t, rollbackMeta.StartedAt, stored.StartedAt) + assert.Equal(t, rollbackMeta.ProgramStartedAt, stored.ProgramStartedAt) + assert.Equal(t, rollbackMeta.ExitCode, stored.ExitCode) + assert.Equal(t, rollbackMeta.ExitMessage, stored.ExitMessage) + assert.Empty(t, stored.GPUDevicePath) + assert.Nil(t, stored.GPUAssignedAt) } func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { @@ -341,16 +397,19 @@ func TestReleaseStoredVGPURetainsMetadataOnFailure(t *testing.T) { func TestSetAndClearStoredVGPUDevice(t *testing.T) { t.Parallel() + assignedAt := time.Now().UTC() stored := &StoredMetadata{} setStoredVGPUDevice(stored, &devices.VGPUDevice{ Framework: devices.VGPUFrameworkVendorVFIO, SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", - }) + }, assignedAt) assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) + assert.Equal(t, assignedAt, *stored.GPUAssignedAt) clearStoredVGPUDevice(stored) assert.Empty(t, stored.GPUFramework) assert.Empty(t, stored.GPUDevicePath) assert.Empty(t, stored.GPUMdevUUID) + assert.Nil(t, stored.GPUAssignedAt) } From df3dcf8d64ae9cc22e96cb6a3d7bd1ed0572c8ef Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:12:47 +0000 Subject: [PATCH 046/107] Preserve the create failure cause in vGPU cleanup errors The vgpu_cleanup_pending response replaced the original create error with cleanup guidance, leaving the cause only in server logs. Prefix the message with the wrapped error so callers see why creation failed as well as how to recover. --- cmd/api/api/instances.go | 4 ++-- cmd/api/api/instances_test.go | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 8cfe72728..681be6ab1 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -368,10 +368,10 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst // errors.Is case would match the cause and hide the pending vGPU cleanup. case errors.As(err, &vgpuPending): log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) - message := fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID) + message := fmt.Sprintf("failed to create instance: %v; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.Err, vgpuPending.InstanceID) innerCode := "vgpu_retained_instance" if !vgpuPending.Retained { - message = fmt.Sprintf("failed to create instance; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.InstanceID) + message = fmt.Sprintf("failed to create instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.Err, vgpuPending.InstanceID) innerCode = "vgpu_unretained_instance" } return oapi.CreateInstance500JSONResponse{ diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index a27334849..fdd444f8e 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -77,6 +77,8 @@ func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) assert.Contains(t, pending.Message, "inst-1") + assert.Contains(t, pending.Message, network.ErrNameExists.Error(), + "the underlying create failure must survive the cleanup guidance") assert.Contains(t, pending.Message, "delete it to retry") require.NotNil(t, pending.InnerError) require.NotNil(t, pending.InnerError.Code) @@ -102,6 +104,8 @@ func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance( require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") + assert.Contains(t, pending.Message, network.ErrNameExists.Error(), + "the underlying create failure must survive the cleanup guidance") assert.Contains(t, pending.Message, "startup reconcile") assert.NotContains(t, pending.Message, "delete") require.NotNil(t, pending.InnerError) From d60193ed3a5ac051ba66b187ef3763d5b326d10b Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:33:44 +0000 Subject: [PATCH 047/107] Fix vGPU reconciliation edge cases --- cmd/api/main.go | 9 ++++----- cmd/api/main_test.go | 4 ++-- lib/instances/create.go | 2 ++ lib/instances/vgpu.go | 16 ++++++++++++---- lib/instances/vgpu_test.go | 27 ++++++++++++++++++++++++++- 5 files changed, 46 insertions(+), 12 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 46308e201..eeb6e9fae 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -185,8 +185,6 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta }, logger), nil } -const vgpuAssignmentStartupGracePeriod = 5 * time.Minute - func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, time.Duration, error) { allInstances, err := instanceManager.ListInstancesForReconcile(ctx) if err != nil { @@ -208,7 +206,7 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances. if inst.GPUAssignedAt == nil { continue } - remaining := vgpuAssignmentStartupGracePeriod - time.Since(*inst.GPUAssignedAt) + remaining := instances.VGPUAssignmentStartupGracePeriod - time.Since(*inst.GPUAssignedAt) if remaining <= 0 { continue } @@ -223,8 +221,9 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances. func reconcileVGPUs(ctx context.Context, instanceManager instances.Manager, logger *slog.Logger) { protected, retryAfter, err := liveInstanceVGPUDevicePaths(ctx, instanceManager) if err != nil { - logger.Warn("failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconciliation", "error", err) - return + logger.Warn("failed to list instances for vGPU reconcile protection; reconciling mdev only", "error", err) + protected = nil + retryAfter = 0 } if err := devices.ReconcileVGPUs(ctx, protected); err != nil { logger.Warn("failed to reconcile vGPU devices", "error", err) diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 02909e6ad..09a4a2419 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -356,7 +356,7 @@ func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) { require.NoError(t, dead.Run()) deadPID := dead.Process.Pid recent := time.Now().Add(-time.Minute) - stale := time.Now().Add(-vgpuAssignmentStartupGracePeriod - time.Minute) + stale := time.Now().Add(-instances.VGPUAssignmentStartupGracePeriod - time.Minute) manager := vgpuReconcileManagerStub{list: []instances.Instance{ {StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}}, @@ -368,7 +368,7 @@ func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) { protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager) require.NoError(t, err) require.Positive(t, retryAfter) - require.LessOrEqual(t, retryAfter, vgpuAssignmentStartupGracePeriod) + require.LessOrEqual(t, retryAfter, instances.VGPUAssignmentStartupGracePeriod) assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") diff --git a/lib/instances/create.go b/lib/instances/create.go index 98e1a6f29..6ea250e55 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -351,6 +351,7 @@ func (m *manager) createInstance( GPUFramework: gpuDevice.Framework, GPUDevicePath: gpuDevice.SysfsPath, GPUMdevUUID: gpuDevice.MdevUUID, + GPUAssignedAt: gpuAssignedAt, } } } @@ -672,6 +673,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG GPUFramework: retainedVGPU.GPUFramework, GPUDevicePath: retainedVGPU.GPUDevicePath, GPUMdevUUID: retainedVGPU.GPUMdevUUID, + GPUAssignedAt: retainedVGPU.GPUAssignedAt, } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 3e7dc350c..0d12e5323 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -6,6 +6,7 @@ import ( "time" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) @@ -16,6 +17,10 @@ func validateVGPUHypervisor(hvType hypervisor.Type) error { return nil } +// VGPUAssignmentStartupGracePeriod bounds how long an assignment without a +// persisted hypervisor PID is treated as potentially live. +const VGPUAssignmentStartupGracePeriod = 5 * time.Minute + // VGPUCleanupPendingError reports a failed create whose vGPU release also // failed during rollback. When Retained is true, deleting the retained instance // retries the release; otherwise startup reconciliation recovers the assignment. @@ -125,9 +130,9 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) // stored metadata claims devicePath. It reads raw metadata instead of // hydrating full instances: the scan runs on every vendor VFIO release, and // deriving state would query the hypervisor of every instance on the host. -// A confirmed live claimant returns true. Unreadable metadata, a missing PID, -// or unverifiable process ownership returns an error so the requester retains -// its assignment for a later retry. +// A confirmed live claimant returns true. Unreadable metadata, a recent +// assignment without a PID, or unverifiable process ownership returns an error +// so the requester retains its assignment for a later retry. func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { files, err := m.listMetadataFilesWithStatErrors(true) if err != nil { @@ -147,7 +152,10 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu continue } if stored.HypervisorPID == nil { - return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) + if stored.GPUAssignedAt == nil || time.Since(*stored.GPUAssignedAt) >= VGPUAssignmentStartupGracePeriod { + continue + } + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) } pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.SocketPath) if err != nil { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 366e343f2..7eed17444 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -27,6 +27,7 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { t.Parallel() m := &manager{paths: paths.New(t.TempDir())} + assignedAt := time.Now().UTC() stored := &StoredMetadata{ Id: "failed-create", Name: "failed-create", @@ -34,6 +35,7 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUMdevUUID: "mdev-uuid", + GPUAssignedAt: &assignedAt, NetworkEnabled: true, IP: "192.0.2.1", Volumes: []VolumeAttachment{{VolumeID: "volume"}}, @@ -49,6 +51,7 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { assert.Equal(t, stored.GPUFramework, retained.GPUFramework) assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) assert.Equal(t, stored.GPUMdevUUID, retained.GPUMdevUUID) + assert.Equal(t, stored.GPUAssignedAt, retained.GPUAssignedAt) assert.Empty(t, retained.Name) assert.Empty(t, retained.GPUProfile) assert.False(t, retained.NetworkEnabled) @@ -282,15 +285,17 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. assert.True(t, claimed) } -func TestVGPUAssignmentClaimedByLiveInstanceErrorsOnNilPIDClaim(t *testing.T) { +func TestVGPUAssignmentClaimedByLiveInstanceErrorsOnRecentNilPIDClaim(t *testing.T) { t.Parallel() m := &manager{paths: paths.New(t.TempDir())} require.NoError(t, m.ensureDirectories("booting-claimant")) + assignedAt := time.Now().UTC() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ Id: "booting-claimant", Name: "booting-claimant", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, }})) _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") @@ -298,6 +303,24 @@ func TestVGPUAssignmentClaimedByLiveInstanceErrorsOnNilPIDClaim(t *testing.T) { assert.Contains(t, err.Error(), "booting-claimant") } +func TestVGPUAssignmentClaimedByLiveInstanceIgnoresStaleNilPIDClaim(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("stale-claimant")) + assignedAt := time.Now().Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "stale-claimant", + Name: "stale-claimant", + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.NoError(t, err) + assert.False(t, claimed) +} + func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) { t.Parallel() @@ -349,10 +372,12 @@ func TestReleaseStoredVGPURetainsRequesterOnAmbiguousClaim(t *testing.T) { }, } require.NoError(t, m.ensureDirectories("ambiguous-claimant")) + assignedAt := time.Now().UTC() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ Id: "ambiguous-claimant", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: devicePath, + GPUAssignedAt: &assignedAt, }})) stored := &StoredMetadata{ From d0de0ccbc3e89460c7b52415fa7e46cc25ee5006 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:26:01 +0000 Subject: [PATCH 048/107] Use boot-scoped hypervisor identities for vGPUs --- cmd/api/main.go | 2 +- lib/instances/vgpu.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index eeb6e9fae..026267a05 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -197,7 +197,7 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances. continue } if inst.HypervisorPID != nil { - if !instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.SocketPath) { + if !instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.HypervisorBootID, inst.SocketPath) { continue } protected[inst.GPUDevicePath] = struct{}{} diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 0d12e5323..772d93e17 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -157,7 +157,7 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu } return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) } - pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.SocketPath) + pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.HypervisorBootID, stored.SocketPath) if err != nil { return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) } From 5b1b190eb83d2167706629ab897e942d585d3a29 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:34:19 +0000 Subject: [PATCH 049/107] Run vGPU rollback tests with QEMU --- lib/instances/vgpu_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 7eed17444..56c6f8574 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -153,7 +153,7 @@ func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, dev Name: id, Image: "test-image", GPUProfile: "NVIDIA L40S-2Q", - HypervisorType: lifecycleNoopHypervisorType, + HypervisorType: hypervisor.TypeQEMU, SocketPath: m.paths.InstanceSocket(id, "noop.sock"), DataDir: m.paths.InstanceDir(id), }})) From 5420fb9be464d4a7f76d00d1628e20326950b362 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:07:05 +0000 Subject: [PATCH 050/107] Protect new vGPU assignments from stale PIDs --- cmd/api/main.go | 5 +---- cmd/api/main_test.go | 4 +++- lib/instances/create.go | 6 ++++++ lib/instances/start.go | 9 +++++++++ lib/instances/vgpu.go | 1 + lib/instances/vgpu_test.go | 12 +++++++++++- 6 files changed, 31 insertions(+), 6 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 026267a05..982e15c3f 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -196,10 +196,7 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances. if inst.GPUDevicePath == "" { continue } - if inst.HypervisorPID != nil { - if !instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.HypervisorBootID, inst.SocketPath) { - continue - } + if inst.HypervisorPID != nil && instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.HypervisorBootID, inst.SocketPath) { protected[inst.GPUDevicePath] = struct{}{} continue } diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 09a4a2419..217e9db8d 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -351,7 +351,7 @@ func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([] return s.list, nil } -func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) { +func TestLiveInstanceVGPUDevicePathsBoundsStartupProtection(t *testing.T) { dead := exec.Command("true") require.NoError(t, dead.Run()) deadPID := dead.Process.Pid @@ -363,6 +363,7 @@ func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) { {StoredMetadata: instances.StoredMetadata{Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}}, {StoredMetadata: instances.StoredMetadata{Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}}, {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorPID: &deadPID}}, + {StoredMetadata: instances.StoredMetadata{Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorPID: &deadPID, GPUAssignedAt: &recent}}, }} protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager) @@ -373,4 +374,5 @@ func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) { assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") + assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") } diff --git a/lib/instances/create.go b/lib/instances/create.go index 6ea250e55..0e8ef4ce7 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -106,6 +106,11 @@ func (m *manager) createInstance( if hvType == "" { hvType = m.defaultHypervisor } + if req.GPU != nil && req.GPU.Profile != "" { + if err := validateVGPUHypervisor(hvType); err != nil { + return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err) + } + } starter, starterErr := m.getVMStarter(hvType) if starterErr == nil { if err := m.validateCreateVMConfig(starter, req, hvType); err != nil { @@ -113,6 +118,7 @@ func (m *manager) createInstance( } } + // 2. Validate image exists and is ready; auto-pull if not found log.DebugContext(ctx, "validating image", "image", req.Image) imageCtx, imageSpanEnd := m.startLifecycleStep(ctx, "resolve_image", diff --git a/lib/instances/start.go b/lib/instances/start.go index 775038559..ecb33f11f 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -48,6 +48,11 @@ func (m *manager) startInstance( return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } + if stored.GPUProfile != "" { + if err := validateVGPUHypervisor(stored.HypervisorType); err != nil { + return nil, fmt.Errorf("%w: %w", ErrInvalidState, err) + } + } // Release any assignment retained by an earlier failed release and // persist the cleared fields immediately, so a failure later in start // cannot leave on-disk metadata pointing at a device that is already @@ -63,6 +68,10 @@ func (m *manager) startInstance( } } + // Do not persist the previous VMM's identity with a new vGPU assignment. + stored.HypervisorPID = nil + stored.HypervisorStartTime = 0 + stored.HypervisorBootID = "" rollbackMeta := *meta // 2a. Clear stale exit info from previous run and apply command overrides diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 772d93e17..6fd02352a 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -2,6 +2,7 @@ package instances import ( "context" + "fmt" "path/filepath" "time" diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 56c6f8574..4491bdd17 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -190,9 +190,16 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { return errors.New("destroy failed") }) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + stalePID := os.Getpid() + meta.HypervisorPID = &stalePID + meta.HypervisorStartTime = 1 + meta.HypervisorBootID = "previous-boot" + require.NoError(t, m.saveMetadata(meta)) t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) - _, err := m.startInstance(context.Background(), id, StartInstanceRequest{Entrypoint: []string{"new-entrypoint"}}) + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{Entrypoint: []string{"new-entrypoint"}}) require.Error(t, err) stored, err := m.loadMetadata(id) @@ -200,6 +207,9 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) assert.NotNil(t, stored.GPUAssignedAt) + assert.Nil(t, stored.HypervisorPID) + assert.Zero(t, stored.HypervisorStartTime) + assert.Empty(t, stored.HypervisorBootID) assert.Empty(t, stored.Entrypoint) } From 83c7f5167c0dc8542b5afeb6e3c9e275d428d379 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:30:28 +0000 Subject: [PATCH 051/107] Persist vGPU assignments after create rollback failure --- lib/instances/create.go | 1 + lib/instances/start.go | 8 ++++++ lib/instances/vgpu.go | 23 ++++++++++++++++ lib/instances/vgpu_test.go | 56 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+) diff --git a/lib/instances/create.go b/lib/instances/create.go index 0e8ef4ce7..c221f698a 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -318,6 +318,7 @@ func (m *manager) createInstance( log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) if err != nil { + retainedVGPU = retainedVGPUFromCreateError(id, m.nowUTC(), err) log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) return nil, wrapCreateVGPUErr(req.GPU.Profile, err) } diff --git a/lib/instances/start.go b/lib/instances/start.go index ecb33f11f..39418ad7d 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -175,6 +175,14 @@ func (m *manager) startInstance( log.InfoContext(ctx, "creating vGPU for start", "instance_id", id, "profile", stored.GPUProfile) device, err := m.createVGPUDevice(ctx, stored.GPUProfile, id) if err != nil { + if pendingDevice, ok := vgpuDevicePendingCleanup(err); ok { + assignedAt := m.nowUTC() + setStoredVGPUDevice(stored, pendingDevice, assignedAt) + if saveErr := m.saveMetadata(meta); saveErr != nil { + log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) + return nil, fmt.Errorf("create vGPU for profile %s: %w; retain assignment: %v", stored.GPUProfile, err, saveErr) + } + } log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) } diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 6fd02352a..a52cfa5d1 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -2,6 +2,7 @@ package instances import ( "context" + "errors" "fmt" "path/filepath" "time" @@ -48,6 +49,28 @@ func (m *manager) createVGPUDevice(ctx context.Context, profileName, instanceID return create(ctx, profileName, instanceID) } +func vgpuDevicePendingCleanup(err error) (*devices.VGPUDevice, bool) { + var pending *devices.VGPUCreateCleanupPendingError + if !errors.As(err, &pending) { + return nil, false + } + return &pending.Device, true +} + +func retainedVGPUFromCreateError(instanceID string, assignedAt time.Time, err error) *StoredMetadata { + device, ok := vgpuDevicePendingCleanup(err) + if !ok { + return nil + } + return &StoredMetadata{ + Id: instanceID, + GPUFramework: device.Framework, + GPUDevicePath: device.SysfsPath, + GPUMdevUUID: device.MdevUUID, + GPUAssignedAt: &assignedAt, + } +} + func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { destroy := m.destroyVGPU if destroy == nil { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 4491bdd17..3cfd04430 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -3,6 +3,7 @@ package instances import ( "context" "errors" + "fmt" "os" "path/filepath" "sync" @@ -128,6 +129,35 @@ func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { assert.Equal(t, "boot failed; vGPU release failed during rollback and the retention record for instance inst-1 could not be saved; the assignment is recovered on the next startup reconcile", unpersisted.Error()) } +func TestVGPUDevicePendingCleanup(t *testing.T) { + t.Parallel() + + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + cause := errors.New("rollback failed") + pending := &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} + + wrapped := fmt.Errorf("create failed: %w", pending) + actual, ok := vgpuDevicePendingCleanup(wrapped) + require.True(t, ok) + assert.Equal(t, device, *actual) + + assignedAt := time.Now().UTC() + retained := retainedVGPUFromCreateError("inst-1", assignedAt, wrapped) + require.NotNil(t, retained) + assert.Equal(t, "inst-1", retained.Id) + assert.Equal(t, device.Framework, retained.GPUFramework) + assert.Equal(t, device.SysfsPath, retained.GPUDevicePath) + assert.Equal(t, assignedAt, *retained.GPUAssignedAt) + + actual, ok = vgpuDevicePendingCleanup(cause) + assert.False(t, ok) + assert.Nil(t, actual) + assert.Nil(t, retainedVGPUFromCreateError("inst-1", assignedAt, cause)) +} + func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) { t.Helper() m := &manager{ @@ -160,6 +190,32 @@ func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, dev return m, id } +func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return nil + }) + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + ProfileType: "1148", + ProfileName: "NVIDIA L40S-2Q", + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + cause := errors.New("create verification and rollback failed") + m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { + return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} + } + + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.ErrorIs(t, err, cause) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, device.Framework, stored.GPUFramework) + assert.Equal(t, device.SysfsPath, stored.GPUDevicePath) + assert.NotNil(t, stored.GPUAssignedAt) +} + func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { var destroyed []devices.VGPUAssignment m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { From abf009bd424045f0bbc71c7073acd3ed9e71bb30 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:52:26 +0000 Subject: [PATCH 052/107] Preserve vGPU lifecycle compatibility --- lib/instances/create.go | 6 --- lib/instances/start.go | 10 ++--- lib/instances/vgpu.go | 8 ---- lib/instances/vgpu_test.go | 76 ++++++++++++++++++++++++++++++++++---- 4 files changed, 71 insertions(+), 29 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index c221f698a..3ffa322e6 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -106,11 +106,6 @@ func (m *manager) createInstance( if hvType == "" { hvType = m.defaultHypervisor } - if req.GPU != nil && req.GPU.Profile != "" { - if err := validateVGPUHypervisor(hvType); err != nil { - return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err) - } - } starter, starterErr := m.getVMStarter(hvType) if starterErr == nil { if err := m.validateCreateVMConfig(starter, req, hvType); err != nil { @@ -118,7 +113,6 @@ func (m *manager) createInstance( } } - // 2. Validate image exists and is ready; auto-pull if not found log.DebugContext(ctx, "validating image", "image", req.Image) imageCtx, imageSpanEnd := m.startLifecycleStep(ctx, "resolve_image", diff --git a/lib/instances/start.go b/lib/instances/start.go index 39418ad7d..f95967699 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -48,11 +48,6 @@ func (m *manager) startInstance( return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } - if stored.GPUProfile != "" { - if err := validateVGPUHypervisor(stored.HypervisorType); err != nil { - return nil, fmt.Errorf("%w: %w", ErrInvalidState, err) - } - } // Release any assignment retained by an earlier failed release and // persist the cleared fields immediately, so a failure later in start // cannot leave on-disk metadata pointing at a device that is already @@ -177,8 +172,9 @@ func (m *manager) startInstance( if err != nil { if pendingDevice, ok := vgpuDevicePendingCleanup(err); ok { assignedAt := m.nowUTC() - setStoredVGPUDevice(stored, pendingDevice, assignedAt) - if saveErr := m.saveMetadata(meta); saveErr != nil { + retentionMeta := rollbackMeta + setStoredVGPUDevice(&retentionMeta.StoredMetadata, pendingDevice, assignedAt) + if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) return nil, fmt.Errorf("create vGPU for profile %s: %w; retain assignment: %v", stored.GPUProfile, err, saveErr) } diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index a52cfa5d1..290c474bb 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -8,17 +8,9 @@ import ( "time" "github.com/kernel/hypeman/lib/devices" - "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) -func validateVGPUHypervisor(hvType hypervisor.Type) error { - if hvType != hypervisor.TypeQEMU { - return fmt.Errorf("vGPU is only supported with qemu, got %s", hvType) - } - return nil -} - // VGPUAssignmentStartupGracePeriod bounds how long an assignment without a // persisted hypervisor PID is treated as potentially live. const VGPUAssignmentStartupGracePeriod = 5 * time.Minute diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 3cfd04430..1ff759363 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -12,18 +12,12 @@ import ( "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestValidateVGPUHypervisor(t *testing.T) { - t.Parallel() - - assert.NoError(t, validateVGPUHypervisor(hypervisor.TypeQEMU)) - assert.EqualError(t, validateVGPUHypervisor(hypervisor.TypeCloudHypervisor), "vGPU is only supported with qemu, got cloud-hypervisor") -} - func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { t.Parallel() @@ -158,6 +152,22 @@ func TestVGPUDevicePendingCleanup(t *testing.T) { assert.Nil(t, retainedVGPUFromCreateError("inst-1", assignedAt, cause)) } +type startRetentionNetworkManager struct { + network.Manager + config network.NetworkConfig + releaseCalls int +} + +func (m *startRetentionNetworkManager) CreateAllocation(context.Context, network.AllocateRequest) (*network.NetworkConfig, error) { + config := m.config + return &config, nil +} + +func (m *startRetentionNetworkManager) ReleaseAllocation(context.Context, *network.Allocation) error { + m.releaseCalls++ + return nil +} + func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) { t.Helper() m := &manager{ @@ -194,6 +204,27 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { return nil }) + networkManager := &startRetentionNetworkManager{config: network.NetworkConfig{ + IP: "192.0.2.20", + MAC: "02:00:00:00:00:20", + TAPDevice: "tap-new", + }} + m.networkManager = networkManager + + previousProgramStart := time.Now().Add(-time.Hour).UTC() + previousExitCode := 23 + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.NetworkEnabled = true + meta.IP = "192.0.2.10" + meta.MAC = "02:00:00:00:00:10" + meta.Entrypoint = []string{"old-entrypoint"} + meta.Cmd = []string{"old-command"} + meta.ProgramStartedAt = &previousProgramStart + meta.ExitCode = &previousExitCode + meta.ExitMessage = "previous exit" + require.NoError(t, m.saveMetadata(meta)) + device := devices.VGPUDevice{ Framework: devices.VGPUFrameworkVendorVFIO, VFAddress: "0000:82:00.4", @@ -206,7 +237,10 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} } - _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{ + Entrypoint: []string{"new-entrypoint"}, + Cmd: []string{"new-command"}, + }) require.ErrorIs(t, err, cause) stored, err := m.loadMetadata(id) @@ -214,6 +248,32 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { assert.Equal(t, device.Framework, stored.GPUFramework) assert.Equal(t, device.SysfsPath, stored.GPUDevicePath) assert.NotNil(t, stored.GPUAssignedAt) + assert.Equal(t, []string{"old-entrypoint"}, stored.Entrypoint) + assert.Equal(t, []string{"old-command"}, stored.Cmd) + assert.Equal(t, previousProgramStart, *stored.ProgramStartedAt) + assert.Equal(t, previousExitCode, *stored.ExitCode) + assert.Equal(t, "previous exit", stored.ExitMessage) + assert.Equal(t, "192.0.2.10", stored.IP) + assert.Equal(t, "02:00:00:00:00:10", stored.MAC) + assert.Equal(t, 1, networkManager.releaseCalls) +} + +func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return nil + }) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.HypervisorType = hypervisor.TypeCloudHypervisor + require.NoError(t, m.saveMetadata(meta)) + + cause := errors.New("create failed") + m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { + return nil, cause + } + + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{}) + assert.ErrorIs(t, err, cause) } func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { From e5e35e2f508f6063ef8f4ffa250da4df7de1b0ff Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 21:11:08 +0000 Subject: [PATCH 053/107] Reconcile vGPU protection from raw metadata and restore GPUAssignedAt ListInstancesForReconcile hydrated every instance (socket stat, UFFD health, /vm.info per instance) before the API served and again on each grace retry, while the protected-set scan only reads stored metadata fields. List raw metadata fail-closed instead, matching the release claim scan, and drop the now-unused loadInstances parameterization. Snapshot restore preserved the source's vGPU assignment path fields but not GPUAssignedAt, so a retained assignment lost its crash-recovery grace timestamp across a restore. Carry the timestamp with the rest of the assignment. --- lib/instances/manager.go | 21 +++++++++++++++++++-- lib/instances/query.go | 11 +---------- lib/instances/query_test.go | 6 ++++++ lib/instances/snapshot.go | 1 + lib/instances/snapshot_test.go | 4 ++++ 5 files changed, 31 insertions(+), 12 deletions(-) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 458275bb5..1bb1e53ec 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "path/filepath" "strings" "sync" "time" @@ -737,9 +738,25 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { return m.defaultHypervisor } -// ListInstancesForReconcile returns every instance or an invalid metadata error. +// ListInstancesForReconcile returns every instance's stored metadata or an +// invalid metadata error. It does not derive state: reconcile protection only +// needs raw metadata fields, and hydration would query the hypervisor of +// every instance on the host before the API serves. func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, error) { - return m.loadInstances(ctx, false) + files, err := m.listMetadataFilesWithStatErrors(true) + if err != nil { + return nil, err + } + result := make([]Instance, 0, len(files)) + for _, file := range files { + id := filepath.Base(filepath.Dir(file)) + meta, err := m.loadMetadata(id) + if err != nil { + return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) + } + result = append(result, Instance{StoredMetadata: meta.StoredMetadata}) + } + return result, nil } // ListInstances returns instances, optionally filtered by the given criteria. diff --git a/lib/instances/query.go b/lib/instances/query.go index eea66ab13..0621460da 100644 --- a/lib/instances/query.go +++ b/lib/instances/query.go @@ -786,16 +786,12 @@ func parseSentinelTimestamp(line, sentinelPrefix string) (time.Time, bool) { // listInstances returns all instances, skipping metadata files that cannot be loaded. func (m *manager) listInstances(ctx context.Context) ([]Instance, error) { - return m.loadInstances(ctx, true) -} - -func (m *manager) loadInstances(ctx context.Context, skipInvalid bool) ([]Instance, error) { ctx, span := m.tracerOrDefault().Start(ctx, "instances.list_metadata") defer span.End() log := logger.FromContext(ctx) log.DebugContext(ctx, "listing all instances") - files, err := m.listMetadataFilesWithStatErrors(!skipInvalid) + files, err := m.listMetadataFiles() if err != nil { log.ErrorContext(ctx, "failed to list metadata files", "error", err) return nil, err @@ -813,11 +809,6 @@ func (m *manager) loadInstances(ctx context.Context, skipInvalid bool) ([]Instan ) meta, err := m.loadMetadata(id) if err != nil { - if !skipInvalid { - hydrateSpan.RecordError(err) - hydrateSpan.End() - return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) - } // Skip instances with invalid metadata log.WarnContext(hydrateCtx, "skipping instance with invalid metadata", "instance_id", id, "error", err) hydrateSpan.End() diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index 41aba54e8..ab3db29c3 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -34,6 +34,12 @@ func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { _, err = m.ListInstancesForReconcile(context.Background()) require.Error(t, err) assert.ErrorContains(t, err, "load metadata for instance invalid") + + require.NoError(t, os.Remove(m.paths.InstanceMetadata("invalid"))) + listed, err = m.ListInstancesForReconcile(context.Background()) + require.NoError(t, err) + require.Len(t, listed, 1) + assert.Equal(t, "valid", listed[0].Id) } func TestParseExitSentinelLine(t *testing.T) { diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 2d2676c72..48c51328a 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -312,6 +312,7 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str restored.GPUFramework = sourceMeta.GPUFramework restored.GPUDevicePath = sourceMeta.GPUDevicePath restored.GPUMdevUUID = sourceMeta.GPUMdevUUID + restored.GPUAssignedAt = sourceMeta.GPUAssignedAt restored.HypervisorType = targetHypervisor restored.HypervisorVersion = targetHypervisorVersion restored.SocketPath = m.paths.InstanceSocket(id, starter.SocketName()) diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index b23e364ee..a92763b28 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -113,6 +113,8 @@ func TestRestoreSnapshotKeepsCurrentVGPUAssignment(t *testing.T) { meta.GPUFramework = devices.VGPUFramework("future-framework") meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" meta.GPUMdevUUID = "retained-uuid" + assignedAt := time.Now().UTC().Truncate(time.Second) + meta.GPUAssignedAt = &assignedAt require.NoError(t, mgr.saveMetadata(meta)) _, err = mgr.RestoreSnapshot(ctx, sourceID, snapshot.Id, RestoreSnapshotRequest{ @@ -126,6 +128,8 @@ func TestRestoreSnapshotKeepsCurrentVGPUAssignment(t *testing.T) { assert.Equal(t, devices.VGPUFramework("future-framework"), restored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", restored.GPUDevicePath) assert.Equal(t, "retained-uuid", restored.GPUMdevUUID) + require.NotNil(t, restored.GPUAssignedAt) + assert.True(t, assignedAt.Equal(*restored.GPUAssignedAt)) } func TestStoppedSnapshotLifecycleAndForkAfterSourceDeletion(t *testing.T) { From 081ead159dec992fd30e4e6ed934a9778b27dce8 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:46:50 +0000 Subject: [PATCH 054/107] Surface pending vGPU cleanup from start as a typed error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When start's vGPU create fails with a pending device-layer cleanup, the error was returned untyped, so the API mapped it to a generic internal_error. Create already wraps the same condition in VGPUCleanupPendingError and surfaces vgpu_cleanup_pending with retained/unretained guidance. Wrap start's pending-cleanup error the same way — Retained reflects whether the retention record was persisted — and map it in the StartInstance handler ahead of the errors.Is cases so the wrapped cause cannot hide the pending cleanup. --- cmd/api/api/instances.go | 19 +++++++++++ cmd/api/api/instances_test.go | 62 +++++++++++++++++++++++++++++++++++ lib/instances/start.go | 6 ++-- lib/instances/vgpu_test.go | 39 ++++++++++++++++++++++ 4 files changed, 124 insertions(+), 2 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 681be6ab1..6d1589182 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -853,7 +853,26 @@ func (s *ApiService) StartInstance(ctx context.Context, request oapi.StartInstan result, err := s.InstanceManager.StartInstance(ctx, inst.Id, startReq) if err != nil { + var vgpuPending *instances.VGPUCleanupPendingError switch { + // Checked first: it wraps the original start error, so a later + // errors.Is case would match the cause and hide the pending vGPU cleanup. + case errors.As(err, &vgpuPending): + log.ErrorContext(ctx, "failed to start instance", "error", err) + message := fmt.Sprintf("failed to start instance: %v; vGPU release failed during rollback and instance %s retains the assignment, delete it or retry start to release it", vgpuPending.Err, vgpuPending.InstanceID) + innerCode := "vgpu_retained_instance" + if !vgpuPending.Retained { + message = fmt.Sprintf("failed to start instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.Err, vgpuPending.InstanceID) + innerCode = "vgpu_unretained_instance" + } + return oapi.StartInstance500JSONResponse{ + Code: "vgpu_cleanup_pending", + Message: message, + InnerError: &oapi.ErrorDetail{ + Code: lo.ToPtr(innerCode), + Message: lo.ToPtr(vgpuPending.InstanceID), + }, + }, nil case errors.Is(err, instances.ErrInvalidState): return oapi.StartInstance409JSONResponse{ Code: "invalid_state", diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index fdd444f8e..738cf547c 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -1082,6 +1082,68 @@ func TestRestoreInstance_ErrorMapping(t *testing.T) { } } +// A retained-assignment error must win over the mapping of the start error +// it wraps, or the response omits the pending vGPU cleanup the caller has to +// resolve. +func TestStartInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { + t.Parallel() + + resolved := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{Id: "inst-1", Name: "inst-1"}, + State: instances.StateStopped, + } + + t.Run("retained", func(t *testing.T) { + t.Parallel() + svc := newTestService(t) + svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Retained: true, + Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources), + }} + + resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id}) + require.NoError(t, rerr) + + pending, ok := resp.(oapi.StartInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "inst-1") + assert.Contains(t, pending.Message, instances.ErrInsufficientResources.Error(), + "the underlying start failure must survive the cleanup guidance") + assert.Contains(t, pending.Message, "delete it or retry start") + require.NotNil(t, pending.InnerError) + require.NotNil(t, pending.InnerError.Code) + assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code) + require.NotNil(t, pending.InnerError.Message) + assert.Equal(t, "inst-1", *pending.InnerError.Message) + }) + + t.Run("unretained", func(t *testing.T) { + t.Parallel() + svc := newTestService(t) + svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources), + }} + + resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id}) + require.NoError(t, rerr) + + pending, ok := resp.(oapi.StartInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") + assert.Contains(t, pending.Message, "startup reconcile") + assert.NotContains(t, pending.Message, "delete") + require.NotNil(t, pending.InnerError) + require.NotNil(t, pending.InnerError.Code) + assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code) + require.NotNil(t, pending.InnerError.Message) + assert.Equal(t, "inst-1", *pending.InnerError.Message) + }) +} + func TestInstanceActions_ImageNotFoundMapsTo404(t *testing.T) { t.Parallel() diff --git a/lib/instances/start.go b/lib/instances/start.go index f95967699..2b7712af7 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -170,16 +170,18 @@ func (m *manager) startInstance( log.InfoContext(ctx, "creating vGPU for start", "instance_id", id, "profile", stored.GPUProfile) device, err := m.createVGPUDevice(ctx, stored.GPUProfile, id) if err != nil { + log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) if pendingDevice, ok := vgpuDevicePendingCleanup(err); ok { assignedAt := m.nowUTC() retentionMeta := rollbackMeta setStoredVGPUDevice(&retentionMeta.StoredMetadata, pendingDevice, assignedAt) + wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) - return nil, fmt.Errorf("create vGPU for profile %s: %w; retain assignment: %v", stored.GPUProfile, err, saveErr) + return nil, &VGPUCleanupPendingError{InstanceID: id, Err: fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr)} } + return nil, &VGPUCleanupPendingError{InstanceID: id, Retained: true, Err: wrapped} } - log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) } assignedAt := m.nowUTC() diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 1ff759363..d9e599d9b 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -242,6 +242,10 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { Cmd: []string{"new-command"}, }) require.ErrorIs(t, err, cause) + var pending *VGPUCleanupPendingError + require.ErrorAs(t, err, &pending) + assert.Equal(t, id, pending.InstanceID) + assert.True(t, pending.Retained) stored, err := m.loadMetadata(id) require.NoError(t, err) @@ -258,6 +262,41 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { assert.Equal(t, 1, networkManager.releaseCalls) } +func TestStartReportsUnretainedVGPUWhenRetentionSaveFails(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return nil + }) + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + ProfileType: "1148", + ProfileName: "NVIDIA L40S-2Q", + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + cause := errors.New("create verification and rollback failed") + m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} + } + + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.ErrorIs(t, err, cause) + var pending *VGPUCleanupPendingError + require.ErrorAs(t, err, &pending) + assert.Equal(t, id, pending.InstanceID) + assert.False(t, pending.Retained) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "retention save failed, so no assignment should be recorded") +} + func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { return nil From 45fbedce7dea25c7a2b83a07095423d95eef337d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:41:14 +0000 Subject: [PATCH 055/107] Cover retained-stub delete recovery and flag reconcile inventory failures The vgpu_cleanup_pending guidance tells callers to delete the retained instance to retry a failed vGPU release, but no test exercised delete against the minimal GPU-fields-only stub cleanupFailedCreate writes. Add one. Losing the reconcile inventory disables vendor VFIO reconciliation host-wide while releases fail closed on the same inventory, so log it at error level instead of warn. Also document the wholesale-restore assumption in cleanupStartVGPU. --- cmd/api/main.go | 5 +++- lib/instances/lifecycle_noop_test.go | 37 ++++++++++++++++++++++++++++ lib/instances/vgpu.go | 4 +++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 982e15c3f..f38a6a7c3 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -218,7 +218,10 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances. func reconcileVGPUs(ctx context.Context, instanceManager instances.Manager, logger *slog.Logger) { protected, retryAfter, err := liveInstanceVGPUDevicePaths(ctx, instanceManager) if err != nil { - logger.Warn("failed to list instances for vGPU reconcile protection; reconciling mdev only", "error", err) + // Operator-actionable: vendor VFIO reconciliation stays disabled + // host-wide (and releases fail closed on the same inventory) until + // the unreadable instance metadata is repaired. + logger.Error("failed to list instances for vGPU reconcile protection; reconciling mdev only", "error", err) protected = nil retryAfter = 0 } diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 15f632b3a..c74d352fd 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -196,6 +196,43 @@ func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) { assert.Equal(t, restartpolicy.BlockedReasonManualStop, persisted.RestartStatus.BlockedReason) } +// A failed create whose vGPU release also failed retains a minimal +// GPU-fields-only stub, and the API tells the caller to delete it to retry +// the release. Exercise that recovery path against the exact stub shape +// cleanupFailedCreate writes. +func TestDeleteReleasesRetainedCreateStub(t *testing.T) { + p := paths.New(t.TempDir()) + var destroyed []devices.VGPUAssignment + m := &manager{ + paths: p, + instanceLocks: sync.Map{}, + bootMarkerScans: sync.Map{}, + now: time.Now, + lifecycleEvents: newLifecycleSubscribers(), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }, + } + const id = "retained-stub" + require.NoError(t, m.ensureDirectories(id)) + assignedAt := time.Now().UTC() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + }})) + + require.NoError(t, m.DeleteInstance(context.Background(), id)) + + require.Len(t, destroyed, 1) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", destroyed[0].DevicePath) + assert.Equal(t, id, destroyed[0].InstanceID) + _, err := m.loadMetadata(id) + require.Error(t, err, "retained stub must be fully deleted") +} + func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { now := time.Now().UTC() m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, now) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 290c474bb..f25c7c63e 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -85,6 +85,10 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUAssignedAt = nil } +// cleanupStartVGPU wholesale-restores the pre-start metadata snapshot. That +// is safe while the instance lock serializes start and no cleanup registered +// after the vGPU one persists metadata; a future cleanup that writes metadata +// must switch this to targeted field restores. func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) { assignment := devices.VGPUAssignment{ Framework: device.Framework, From c162c87c264810012d4c1335096a223c41594f2a Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:16:52 +0000 Subject: [PATCH 056/107] Reject vendor VFIO vGPUs on Cloud Hypervisor and improve wedge forensics Vendor VFIO vGPUs boot but are non-functional on Cloud Hypervisor (upstream cloud-hypervisor#7572), and the wedged VM then blocks the VF release until startup reconcile. Reject the combination at create and start after the rollback handler is registered, so the rejected device is released through the normal cleanup path. Hypervisor selection otherwise stays caller policy and mdev on Cloud Hypervisor keeps working. Retain identity fields (name, image, hypervisor, data dir) on the failed-create retention record so it lists as a recognizable, deletable instance instead of a nameless phantom; resource claims released by rollback stay dropped. Expose the assigned vGPU device_path in the instance API - on vendor VFIO hosts mdev_uuid is empty and the sysfs path is the identity an operator needs when a release wedges. --- cmd/api/api/instances.go | 3 + lib/instances/create.go | 29 ++- lib/instances/start.go | 6 + lib/instances/vgpu.go | 24 ++- lib/instances/vgpu_test.go | 39 +++- lib/oapi/oapi.go | 404 +++++++++++++++++++------------------ openapi.yaml | 6 +- 7 files changed, 298 insertions(+), 213 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 6d1589182..a277b5476 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -1307,6 +1307,9 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { if inst.GPUMdevUUID != "" { gpu.MdevUuid = lo.ToPtr(inst.GPUMdevUUID) } + if inst.GPUDevicePath != "" { + gpu.DevicePath = lo.ToPtr(inst.GPUDevicePath) + } oapiInst.Gpu = gpu } diff --git a/lib/instances/create.go b/lib/instances/create.go index 3ffa322e6..23fd534b1 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -357,6 +357,12 @@ func (m *manager) createInstance( } } }) + // Checked after the cleanup handler is registered so rejection + // releases the device through the normal rollback. + if err := validateVGPUHypervisorCompat(gpuDevice.Framework, hvType); err != nil { + log.ErrorContext(ctx, "unsupported vGPU hypervisor combination", "instance_id", id, "framework", gpuDevice.Framework, "hypervisor", hvType) + return nil, err + } } if len(req.Devices) > 0 && m.deviceManager != nil { @@ -669,12 +675,25 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) return retentionSurvives() } + // Retain identity fields so the instance lists as a recognizable, + // deletable record rather than a nameless phantom, but drop resource + // claims (network, volumes, devices) that rollback already released. retained := StoredMetadata{ - Id: id, - GPUFramework: retainedVGPU.GPUFramework, - GPUDevicePath: retainedVGPU.GPUDevicePath, - GPUMdevUUID: retainedVGPU.GPUMdevUUID, - GPUAssignedAt: retainedVGPU.GPUAssignedAt, + Id: id, + Name: retainedVGPU.Name, + Image: retainedVGPU.Image, + ResolvedImage: retainedVGPU.ResolvedImage, + Platform: retainedVGPU.Platform, + CreatedAt: retainedVGPU.CreatedAt, + HypervisorType: retainedVGPU.HypervisorType, + HypervisorVersion: retainedVGPU.HypervisorVersion, + SocketPath: retainedVGPU.SocketPath, + DataDir: retainedVGPU.DataDir, + GPUProfile: retainedVGPU.GPUProfile, + GPUFramework: retainedVGPU.GPUFramework, + GPUDevicePath: retainedVGPU.GPUDevicePath, + GPUMdevUUID: retainedVGPU.GPUMdevUUID, + GPUAssignedAt: retainedVGPU.GPUAssignedAt, } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) diff --git a/lib/instances/start.go b/lib/instances/start.go index 2b7712af7..b44a68d50 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -191,6 +191,12 @@ func (m *manager) startInstance( cu.Add(func() { m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) }) + // Checked after the cleanup handler is registered so rejection + // releases the device through the normal rollback. + if err := validateVGPUHypervisorCompat(device.Framework, stored.HypervisorType); err != nil { + log.ErrorContext(ctx, "unsupported vGPU hypervisor combination", "instance_id", id, "framework", device.Framework, "hypervisor", stored.HypervisorType) + return nil, err + } if err := m.saveMetadata(meta); err != nil { log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err) return nil, fmt.Errorf("save metadata after vGPU creation: %w", err) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index f25c7c63e..8c6351742 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -8,6 +8,7 @@ import ( "time" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) @@ -63,6 +64,18 @@ func retainedVGPUFromCreateError(instanceID string, assignedAt time.Time, err er } } +// validateVGPUHypervisorCompat rejects the one proven-broken combination: +// vendor VFIO vGPUs boot but are non-functional on Cloud Hypervisor (upstream +// cloud-hypervisor#7572), and the wedged VM then blocks the VF release until +// startup reconcile. Hypervisor selection otherwise remains caller policy; +// mdev on Cloud Hypervisor keeps working. See lib/devices/GPU.md. +func validateVGPUHypervisorCompat(framework devices.VGPUFramework, hvType hypervisor.Type) error { + if framework == devices.VGPUFrameworkVendorVFIO && hvType == hypervisor.TypeCloudHypervisor { + return fmt.Errorf("%w: vendor VFIO vGPUs are not functional on cloud-hypervisor, use qemu", ErrInvalidRequest) + } + return nil +} + func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { destroy := m.destroyVGPU if destroy == nil { @@ -85,10 +98,13 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUAssignedAt = nil } -// cleanupStartVGPU wholesale-restores the pre-start metadata snapshot. That -// is safe while the instance lock serializes start and no cleanup registered -// after the vGPU one persists metadata; a future cleanup that writes metadata -// must switch this to targeted field restores. +// cleanupStartVGPU wholesale-restores the pre-start metadata snapshot. The +// cleanup stack is LIFO, so cleanups registered after this one run before it +// and this restore would clobber anything they persisted; it is safe only +// while no such cleanup writes metadata and the instance lock serializes +// start. The snapshot is also a shallow copy (Phases shares its map), so it +// must be persisted before any Phases.Record on the live struct. Violating +// either invariant requires switching to targeted field restores. func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) { assignment := devices.VGPUAssignment{ Framework: device.Framework, diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index d9e599d9b..d466f1883 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -47,12 +47,16 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) assert.Equal(t, stored.GPUMdevUUID, retained.GPUMdevUUID) assert.Equal(t, stored.GPUAssignedAt, retained.GPUAssignedAt) - assert.Empty(t, retained.Name) - assert.Empty(t, retained.GPUProfile) + // Identity fields survive so the retained record lists as a + // recognizable, deletable instance instead of a nameless phantom. + assert.Equal(t, stored.Name, retained.Name) + assert.Equal(t, stored.GPUProfile, retained.GPUProfile) + assert.Equal(t, stored.HypervisorType, retained.HypervisorType) + assert.Equal(t, stored.DataDir, retained.DataDir) + // Resource claims released by rollback stay dropped. assert.False(t, retained.NetworkEnabled) assert.Empty(t, retained.IP) assert.Empty(t, retained.Volumes) - assert.Empty(t, retained.DataDir) } func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { @@ -315,6 +319,35 @@ func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { assert.ErrorIs(t, err, cause) } +func TestValidateVGPUHypervisorCompat(t *testing.T) { + t.Parallel() + + err := validateVGPUHypervisorCompat(devices.VGPUFrameworkVendorVFIO, hypervisor.TypeCloudHypervisor) + require.ErrorIs(t, err, ErrInvalidRequest) + assert.NoError(t, validateVGPUHypervisorCompat(devices.VGPUFrameworkVendorVFIO, hypervisor.TypeQEMU)) + assert.NoError(t, validateVGPUHypervisorCompat(devices.VGPUFrameworkMdev, hypervisor.TypeCloudHypervisor)) +} + +func TestStartRejectsVendorVFIOOnCloudHypervisor(t *testing.T) { + var destroyed []devices.VGPUAssignment + m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.HypervisorType = hypervisor.TypeCloudHypervisor + require.NoError(t, m.saveMetadata(meta)) + + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.ErrorIs(t, err, ErrInvalidRequest) + + require.Len(t, destroyed, 1, "the rejected vGPU must be released by rollback") + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "no assignment may be persisted for a rejected combination") +} + func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { var destroyed []devices.VGPUAssignment m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index fa568c18b..27d459268 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1354,7 +1354,10 @@ type InstanceHypervisor string // InstanceGPU GPU information attached to the instance type InstanceGPU struct { - // MdevUuid mdev device UUID + // DevicePath sysfs path of the assigned vGPU device + DevicePath *string `json:"device_path,omitempty"` + + // MdevUuid mdev device UUID (mdev hosts only) MdevUuid *string `json:"mdev_uuid,omitempty"` // Profile vGPU profile name @@ -19073,205 +19076,206 @@ var swaggerSpec = []string{ "1VhohilpTMSQ6dcCWKt/FOrJwZPBYDDodgpNblf/exCipjt1fvYtlrDJ+QW0P2ZRhQFGUOQM5Swmoqj/", "TQw51EPkbug4/UwQYfd5O/HKfl7KVeGDuQ5zuB2Y8OciuMJQG/RziA69hXK+f3sRvVXOkZNfbbaR/Wp0", "kwgGgiKeJ7HW+Mb6tjMGORJbM6QkynBn8y6V6J2pvVmdug09Vhz9nhOxQO9PTythD4JMNA9oN3HgEg37", - "wLMbbcPOGhvJ2tHcxL3s4d3eB8ZtXVLxJMQvjmjrexhdErSh0Iphq6I4r7Kvaa0ymEVCmdknTTQrJlgz", - "5cdkPsrzkFaiHzkclHfvTo4rlILxk+1ng2fPe8/G2096e/Fgu4e3d5/0dvbxYLIbPd3d3tldkXHSIm3t", - "9ploQdU0EDBchIePXJh6KHq4KUmgJgTYwOcrymJ+VblngpGofu82ynVd98sx7K2HEMx8SbBUxk7QwDJO", - "4TYlkW7bRH7b1MiiplHYovjk7WD7c80sMLgGZvxW5Mz4L00yf2GrT70B+5tVHefteCsMyGWYrFstv/P2", - "izY42H9+sP+5i+ayJNaNsU5O97i5TaFXDuy4lobhUgE9g42zBHas8GHM5zZro9PtFIkl8DfcurWg5eJx", - "q2yppgPbDbORVfy7IUv5pKIYQEiIAbuLD7RI4AR/KJ1Q5MJrWeMo4XmMPKOXwf4Ch9eJpyToZsD/ZG1h", - "BsvTZD1oZQLAo6FEA2WaEYOjTzdiU5oP0Et4Fx7h1OhPdhCmUIjv48LxwgSm6PPlujbazOohn1tFBr7R", - "Wg3S/4Jp62WwttHVTRgx6AD9yuGbQq1ivG5kNa+DPrP8et0gu2FxsR1EBXRmZboD9FMhxxWSoJX8NiSx", - "f44swyqRYTYr+fl2xzuaWsqd83LNux2zop1uxy0U5KQvZ6e/K6l+6fz5pBiK2CI4gbNcJuPmiiYWDxtm", - "QqWikbRZGnpzm+QLW8OIxCOjpTQFf5oMT6vJFB858eX9KdoAyMO/IGtB1v/aLAJFK3fdzvO950+e7jx/", - "0grYqBzgehn0CPKPlwe3ViCNsnxkjRBNUz86e2eMDJFR34sgk/enPo5EJrhmPXrmrkG/8+f95z6eU8zz", - "ceJ58Cz4m4GPhQ0LQpYVvKgh4PB3mszpZMJ+/xhd7vxd0HT7+oncGW834NSajsL2rRPfi79kDCbjnqlH", - "FIbcAYISshGV6g2RMAN0ThQC+ukhHIEeUaQNW5Jz2FV2xYOEtbe7u/vs6f5OK7qyo/MOzgisXYFL2Y7A", - "O2LwJtp4c36OtjyCM2068AaAEmdWxwyfM2SLCQ+qAml/e7AbopKGi7ukGtv2PG1c8vdWT7OTsosO2c+F", - "Drd0yoOrvbs7eLq3/2y/3TG2dtiRuF7NYVxukFkei3jv7/wGSJNvD88QZN5OcFQ1orhQrBuNSt1oVFCt", - "waCs32Bgz54+2d/b3dluB68Wiu6wwIGVA1vlXYFDFyCKwG4ElmKZ9XabbouQOGUI7A2JEkzTw8jlMtRu", - "H4OmPhLmtXIT2lwM1vS/dHG1+LaVFamwDZlMGCMacIFyVtTw6K/3fX4RF2Yz1zbXw3quHsp/YXr1LA6Q", - "qVV2i6XMBJlTnssv0BBXJjl1knAubvRtk8Lyhsg8UcbPSCV6f/od8BRNa0gqklV1KEuNK9CSbjm5G53n", - "ComEibxpsVrtRputXzXhbsOp7a5Crqhwg0aMslhzrpytj7I8wkmUQ9UaXOynnhWAbUHufZYlCxNEnySc", - "MxTNMANvhPCghdCMJ3E/GHKqn4wmwfAFfoUSbtCVLwnJbEEXMwj9mRZh6JygDb+UmSGlWoHR/dQwGVuy", - "o0qN+2m4UiKWoaywIudcrydW3AP+NZ9UTI4Jn0pQChWkB/TrePMZFibqHzNToGieGl0yENkcGGKNmYdu", - "VHOT8olVcK3IARndZiVxJLiUiCR0CsVw3p/WEoVXJJcV6cLrIyerg21BusZzGLjKDOxU6zpmofsxkDjz", - "OTck0DAk562ISXTGyRSzHEq8eIRsLd791nGHMy7VqACAuuFgpRpB3YZckBKWrkhvL+xB7p3gvehY222W", - "ywb43urrJaoKN9U0wGaeGlzR8Gp1CxoMkfEyBNZK1K0SxquO2XQTVLgS6J9KaJV6+GBoA5JLPLbkYb1t", - "tolGCausup8lbdWW4Xy1Nzhvi5+2Gi7tDKvZCZvwAMjGDVyUzhJtw0IzIlIKlUtQTBglsdMlC1+lNXVB", - "ZnYiCYpzYlfOyKcC2wXH5ngDUAZzNjLKpjVeX++wjXnYjGF1WQfo177YJq5IhjNX34oc1soEBkqEyxzW", - "VtGWVI7C7qzlhgWZ5gkWyCIfthmyXKQJZZdtWpeLdMwTGiH9Qd0BPeFJwq9G+pH8Aeay2Wp2+oNRUw2g", - "czM4m4BnNqTWbzmFH/QsN2vpv2CJ2TLfbwE0S5swrWBI9k80IRZG7x2j1x6hV3HP93YGTWnpDY1WEtKX", - "IRhvyrktyQZPfC4DSXwrpRxXvYjEFozeiD1ZLk0hlRa3kkM7dS7A23l0qhkan4cBcmT4dQ0BBI0JJNi4", - "qS1zjRZssc1UgjUccjlDf+fjqkG0bXxtoDLYBiuxKASZBAPpYUdXGqTNG0tr4u3uTcAegK3qicJHN8RQ", - "WFdDrQxkauInb5bKic2IXTLq5mhKi7UoleECLQqcANtre8CAeuG3QGAwwMFItYASqlC3ZuFVM5RozIUA", - "qGct4XDmZgP4Jlrm0WvtAKbQ2xlZIEFSTNmQUVYYSQG1jCBG5kR46ahcaCVrSuI++pun4gE4dpqphUVd", - "B+P5dxLxK1aMccj8QerGc6nbOWTGsihyqMpfvqSbBa1PEwqkB4MTTAkoVEjVDE0EkTN/7qHilFrGu+Ii", - "bqz6s0DuFSgmAz5WpPglYT4rK5oJqoamoZH5ajlczlSWhadW/0SVYq+oXsx1dX+5JCIsJBZTKl5pFbri", - "HRVPOTFoKwA9AkX87F+GxRdwIy3ARcrm/+qaLH86Kxqv/lZ7zQMQcXi+h8ZsGzTBRiZfphbsU/WkrQ1V", - "gXyzVbAxy74EtOFilV0llKok4FUkaXVPtkt5q0flu9FsSRJVe997tv/0ScuSMJ/lrDMwWV/aNTdPV7jk", - "GnbqtI3f59n+s+fPd/f2n+/cyMPiEjga9qcpicPfH7RBrpU+rMm//vHP96c1r88+BDsPbjQok8IRHlJD", - "Gkd1QO9P//WPf7pR3XpAIUazDMXd4LdvjNJJ/J10gQJVF147J9kK/f6wYiTABZtBG2QyIWAGHZl165WD", - "qeFttJOCcYYjqhYBRo6vTFh58UoNUrqNO6g62JDIa9q28KOac8l8XGZ3brjO0X8a33CNFp61riwl83GT", - "H/p1vVfjhS69Fn6MQ4sQA1kUNV82cBfzucKyEjmt/44gwcGlci2ntZg3VsPb1nMOIIrFFlDzQgFDsOg1", - "edJ+5G9/bTs9v2XFrFNf8Q8rzmHzEbyR1TdwIweMvtH6HNYaf7AX4O2+Go39mm8ri+pVCsSVt+7N+22R", - "prtckKC4wW7en5eZeJMP6+C7QI92DHbJy7a7FZJooCYv6SRgQOMJ6RWBejYjBcnceAT1mbd47oFUyeiS", - "TyZVUNn9ZhBywNeBrCrXC1ZKayZdRK6dzaKOYG3AdIadfTnsaBVg2NlOh52a2yqYp5ji65HtoAqiMliF", - "Cl7mmdcGKd0MxgmPLk05M6iS3UcDlBLMJMoZHP6aV217sNo71O1k3t4UGNzEhDgtsS0Y05jM8JxC6Qfr", - "U5lWAjHJNVUSAkahnQMUcwOrVKnlameoXzNZhAflpOHSwWxhG9YN6vc4cxGt5btg4JtABVn2kQjetagA", - "mmO/fn3aNQEMEHpoBlaJb3QTNSPQDLLoolbHoPw9HD88TsgIxl3HxU+X19FP/gbPqiCSKGmBsktyqBEB", - "injOVB0wP22nyFXzt5avpJxBsJ8N/wAANNu7IRAUkwhOpFw+i1VCvwVx1/IG7EqHEgd2QyQMhwJ8SWFf", - "8RvrEK4PwBgbvDLMph0/rtt4CUdScVu3qzjVI3IdERLXkTXDr7SNlbdfBmPlX2ELxlNUSLZvQ7zz8uz6", - "d5dJBWNtWm0/pp9x1gMYELelFrLDYPBZUJgqoVUwvj3siFEIxzT0QpvUZnK9eq1/JdcKgMjjPDHocmHS", - "tazKXkbrVvzWKYRNB5oLsrYO3h3UhzPx5reqEGdD1R+iSJx9604Kwy3tzjlR7t1zS0aNO1StqFJxabmA", - "f/dKNcbGkFIX2QsebaebNRLcm4WtIhb9tmUyJMMpGWWCTOj1CuIxLxjFuIofUh6kIoPBAHlupPga7T1F", - "0QwLWRs7o9OZShbVAJy9AGjRZ1VPFEQR5gyFbXa+3E334XK0m91Ov/WQcHzuYfAs1Q6xIuloFUD1Uelt", - "s9b5DC/AitPoJHy6uzcY7O4MboVQ7YZ1g+U6Kj+xtQar7TSl1HnfWUd/JUrVb6HIZl4uYHslKCRFF8sk", - "lSA4PYDEmwxHBCVkAmh0ReHw9Z7FeterB28FKov8UtC/2yi7b84HX61NU3Rlwb3dNDrOuVgF+/Gfr3GI", - "NrCZaAm7LpBzt9sbPHm7vXuw/+Rge/suUKWLRWrK9nj6cfvqabKDJ3vJs8XT37dnT6c76W5QD7ukpgRP", - "G1r9Rb/bGGVTXpJV0KAKS0Mbdg4ZEfXKxPWK3pIklJGeLDKk1qcpruAFxv++9vzfzM5vZrBSdjivTtIX", - "IbAqF6dCWQ8DdGUns9J3UZ/NyfHqWdwqA6k+kDC91YcC5NVuMFAKYrvzmRAIOWt5Db3zXmx9Ea3Milt3", - "FYU87HDSg7vcsOIh8q4hIHizXnWBL19yAdvplAuqZunq26J4rcDrhrjpj1LFVWClPjqZMihL7v9chMn5", - "SpT+uNPtJB/3qmfG/t4eYstC/RYEaLfalwpahJFB1fvVqwCvlIqHMJHsWlfXY/5hu7f9HOIQko97Pwx6", - "z6sRB12zWv7ybbu3K78O2qyhX2vP1Wjafn6jiGu3nqso6BcaqhRX3ssWBNjSeFkE2l0dLuG2ssHl46U9", - "rkHmNAqgnyvp2ctt5AtNMUnwIgQC7xlqZU179IkMjcmUMtnGbrs7KAy3++mw00eHFokbdNmy5H+leSj2", - "7tEJTVMSUy1jGtW/OYNhp6Utrq5L3KwIiPsqIK31w+La8/UQCesSrtZdk/3PyMf9LO23nca7Cr0D7GpO", - "RQWwLnixi+gEYVarBErZHCc0ton0kBgJ8WoHDhGtJFnLA2QpBzo7SRdNuUJlCn1Le1vOmu2CxfjJNdhb", - "V2BmGILY+SKAKAVSF13Fvk6OUSZ4nEdl/mgCgy4RP0Rew0JbIeSvD8m9S/sGJGZPuEDr7RtNBo129smm", - "/a7ZJjXBNm/19mD9Vt+JUaTbybN4PQ8zL7XjYDeCSF+Tghgw0VSXvSYJepP50IKjv/FXcFnnNbbkSItE", - "eeYcLJqmlikp4G4BF0MorveYJERfU8uNIJ7EZZYElSUXXc9St588mzW5OMEjtTyQXwjJtK4C+EfQX4rZ", - "IjgwV9+zuEs2Bg5WWxqHV8/UBbKrVR3c07WSWONW+SbcploFhsvXbN4GL+XSM38XYNq+aLaMgOIYfkVI", - "e9OMtW+/dGFvjfbjuzDLPaSQ9tq6Hmr4qA69t4Ahd/2XscBarKsS717IPR8ii7dWM27CfK1ngfpW58Pe", - "/xgrMxr1D7Z++Mv/3fvwn0Frc01vlkT0YjKBQKNLsuiZKj9aR+9XEU+hxIAWpqeWVAhOwYYEaOL2MPrj", - "3R8UTGPxK06XpgARWl6Jnu21E/rLfzTHN3nL+A745FqS/ewKHHdRqVRxdx1tpERMXSy5SyTb7A8Z5KZd", - "koVEXuEvK9I4Qv1OFp94EejowoiBfcLmF2hMoZKiHDKt1eIoIpnWJmwtGWrKgXPgPoLgxG/HFiBzid/W", - "IWniCQh6f7oEl/v63dsfX7/79Xj0+uzFr4cno19e/DeEeFz1TA9xT9Pe3v4TWwTcX8ntYCGKm9dT6KNT", - "G6ZvXf2THBRawOmSKM1VDkEh5DpKcknnzkGokttXTlhO1r19JYLPhNpVKglFJVhI6IROCPj14TqxQTVU", - "OmKkEqqnW+MGZWj5xjaEM+wAJ/WK34fqVuitCK92ubHVRX8ya8dCDQhp4LBDxiuUuQ9oL1QCXoWL/fBe", - "RhuQOeJKvLrE2c2bgaIeFg0GIw+/cCWfwfMvUW3z3crymnOe9LR601CSIGhNNmsRjJyHpkxGQqfJ6TAd", - "B2R4a9qd0ikO+BlC/oQvUhXTDWhtxtTS/jeWBwvnMRzX6zWYY2mWqlZfoGYkkKrXnOaQaql2VJb+rwbP", - "5MzmrlIvtq6aqJoytWWr14bwMmIOqOGrspXLU+bQEXvw0fok3JV6lTczbyTNe3Pq1IeagrNigc700lzN", - "iCDeRsAHJQ7+DZfM5uW0QGEx1f8yIsqYVZfUo6VScDdLtFFYftwSFNnGy+bw1XUOTvF10QO4UrBc8j/C", - "PMo6S9svfwRM+jeutiSduCZgGDXlLozAXqWiVWviqGp5M3yqWp63eT948CyvWsH9ms5WjTjLPiqkGaLH", - "v2GqfuIC1MFmzJM7B3KHyz8mAjDg6jDtrTDOaUriEc/V6vNvS9fbK7+oP1rWr3WqLwYijirpvE28wKFy", - "lGNYXmm9HCTKBVWLc71eNpgb0iBd0VhYSOgIfi47hkKdnz6B0XgSSBh5SRgRNIIyqPo8ppiBxoTen3rV", - "8ExhxCW8VhCBXh+dWHODg/wF9ZEqID0Xd3l4dtLpduZEGJW7M+jv9gdwmDPCcEY7B53d/nZ/0AGtagZT", - "3ILS9TZ/2uYbF4rrSWwloR/dS/pLgVOi4IvfAkgAEHdoXwcVBE89JTLDVFgtMksAocAQDNVfA66/u1AP", - "zK3cNcve2mYKacaQ/UKy13ZzP4CgDGcHprkzGFhgc2WvX8jdMQkDW3+30aNlv62kOrtEAZj7JTXPyZbF", - "0n/qdvYG2zca06qhwNkNdfyOYZvES0A737/hQtyq0xNm0vJskrUNh/JPHBCSf9Z++6D3TOZpisXCLZi/", - "WhmXTYIxkQi7d40epySKNKuAYjx99JoR8xxhhbCJXBY5gxrG7kNNodVTYNp2m1yAFP3I48UXW8JKH85G", - "8anKzvRx+bREz1+OdgoyXt5I+8ghbBuqvQcC+hEXBbgf7KTsDZ7ffadHnE0SGinUKwjYxiNTCSE/CeCF", - "O+whLtDvOVcYFeH8j+hIW5l1XJBbt7yKtv6g8SdzvBMSMoOfEZFiZpIjzDtrDv3ScTYuifI4r7zVHOFD", - "aQ+4qRwIj7moQJCrHlH/2qoLg8vX0V4AgcH2aaYXPyDh793DCbeTLWqwPuSRg8qXKJfkMR0n62Ibl0JI", - "UJZ7SdTXQvOD+7yybBGBP+EpeiwE/JIUEl65W0uXwlYmcmYU4KAE+KZMWLTffVcV/t6WT7woGfBr6Kah", - "nIUyflUcL/rIralR+tUCIJYEgXnGy9fKmR7e13LCdu7jhMGMC0/Rt2vq2zW16pQbanFTgIPpnfIWNogb", - "WSD+fPaHG1sfvtke2tseWlkeGLmy1oW/83Ef2YjUiMcEyRnPkxiNCTJ4Ry72RGHRn35EWEQzOicAagdF", - "2vJE0QwLiCxJUYwVNj70RsPESrNE0dyWbq7n4hDLBa7jWEgyAhy+URP+ZBmBSBkjMdKfWOi+Ek5wqW63", - "OftBA3vRYHk1oqsZl6TA82PKu80hvVka7Ria7Q/ZWwv0qhcQgqkdr5EkAbjaFfYfzhAeMvvB946FuEAw", - "idOSc2EBmIHUIFOabVlObdMjHcmIh7B23hKGmerJjER0QiM7rUuysPGcwQZb1V3SA3bjfH9aJGygnc0w", - "XhvAM4bBeY+LZ8hSUtV/wyAIOkryuHRyOQghLMY4SYKFOaYJH+NkZNbnkgR8gi/hDbsofn1/501iPCam", - "Vnu2UDPOzN/5OGcqN3+PBb+SRAw7m/0hg0QMu9Yk7pYCIrqCQm5pxvU5Ezw1fW6ZIW79cUkWn/pDdhin", - "lDmKgE9wIjki1/Ad1LcCzAzDvRrowZymsB/8KJeKpz7yqaM7M0yeqyxXNqNEEtUNoX4OmeLoD4ft+Gnr", - "j7LHT+AsJjjWdOK9YqYEsnXTqOUI69mP4NWAu53AAgw7+iI1YR5TgZkysJ0FOCWa+lu6UVRH0Id0s77C", - "EWYo45mpLAFENcOa5CptAFYDThKk4Ci5b7XgDjvZMB8LvZeOG3H3DFBa7RhRhk5/9A7TYO9Z+DxJEgkS", - "iij5r/PXvyK4lfUemNfKcC2T0sG0wIDiHFynjqe9wNEMGUcVFBMcdmg87BTu3HgTxppLGy7T64FP8Qc9", - "tB9MN10a/9Dv66aMu/IA/faHaeVAn6UsNTigw86nLvIeTKma5ePi2YfwgjbBl51XGAHaMNfcJnASTAFp", - "xrvxzRWJWYy4vQWSBcKo5EB+4MqYMiwWqxIJA0tvV5BPTCSjtxh/DCFycdg5GLrYxWGnO+wQNoffbIDj", - "sPMpvALWa9lcuQ7us8K5WRDRk8Fgcz0Stl3fgM+yhWPgC+uAjVpRUXZT76CFYf1z+Qf+rfXPwvWDme68", - "hCYyir8zvj9CB4QnsfuaaMAFURO7MYtI4sTu9Yae+3ce6M2KSJLcN4E+FHkW7rECqf9RkSNsVnmMVprv", - "H5jiBvd1qVTM9g9Dv4/Ofh6wnlvbOZm7UOdwnRLAoLGqNDIvIyzROYypd66V7xfwa9/+1+l+gKl4kfDp", - "xYFR3VHCpyihzOYDeIHKWjywawkfGRia4juLSuOKxG0YSeJf//gnDIqy6b/+8U+L7f6vf/wTjvuWgVeD", - "GtMXM4KFGhOsLg7QL4RkPZzQOXGTgSqwZE7EAu0OrM0fHiGv1L2V0uSQDdkbonLBvLwJU69N2gatq0DP", - "h7KcSAvjo1+kE1tMxsQ2Buw27iybpbzXE90NwCHCDLwJ6FvR0QBgyVFTaNtqop2wydTMuWI0rYdpLgXr", - "recvilwrQ709M8AbMhhY4tC5gwd20mjj/PzFZh+BtmWoAgoGge5QNmPViP43nrSeJxmOUmUosMqGN0U4", - "w2OaUGdybKh2Yo5giqMZZaSMLy6wxl0TB26kmsccnp0gGwjZhVeH7PX5FphYFYlULkjXcgJhEUbLcmjc", - "5rlAD8C/qILosJ59d8gmBEOe0MmxYQIeCHeRD1g0zADIA2JcqapUXusOmUGStcjF+uClPCYJfAT9T7Ei", - "V3jRRUWtW1cdJcFKK8Syq18eMoP1ategB1AlyBtmH/iZGVLPRfLanC1BJolWjSEC35T9hr43JlwgG+Hs", - "Vfl33ZkkSzMsvWgpjl6f6/lNQRPkxh4ILb0+d7ux2UWSoyihQA0RZkM2hUAgB97LWWVXi4SyGRZxL+L6", - "EvDBnC4Zv0pIPG3isUc+kd2hJFPpJ3Ccfq6T62MTLmbLE9CH2ADUrfbcHdt32rnubIt/Jt+dLQR5A+ed", - "seASw2/M6n5z5LVw5IXXzTn1Qp61Y4fAeHcRv6aLBwr4dbS3vObmibdkD2HRQxsO2ga8Ilygs6MThONY", - "ECk3/73tfXqmhkpL+U/fj5oVP0ToiR0LFxb0z9pbqgTyWNjBGztqhN286vV1/fttq1J8p/GmK+rwlFfe", - "3d8etU5vco2UQm9Ja99ukrXBtlRGHMoMltTSA9EoIYX4UpxTn4rWWZVNGG9x5awUlyx7Pjl2B/L+7Mu2", - "65zV74Z7YIrHNYb4gIywmmrtV81+TNT8rthFhza9wvz8dZHm4P6koPs2RYfI/DGpi3Ft2TQXNEAnjRfo", - "S6IMvMld6um2h8DEz4lwp9oMdGFmXUzLfIoMTgtMCCwxq3XfE/NKO9XXtPdn0nxheW4isdgl/yaitFB2", - "y7VapeCe2BLQd6ffQg83Um+/XNiKJbDAIoMVdezcTmBZ3cBywaLNb5ErX5yiTVxjqcQKN28SF5Zsg6ZU", - "6Fn3JdcdMr/euJbprF5LGZokdDqzToCYTiBWT/n1u2GUO/cwyqJOtsCK2BDFx5j3e6YX2XqB50Qo9Pro", - "xKy/f6Vu/QFBq+tVJce8Vt6u79686hEW8bhwnjTLpPbJF1aYDP1Xcnnv/9Q9wnxW6sSDJoHxM/bfBJMj", - "E//ep/x/7fyU0LHAYvG/dn7CSUYZ+V+7hwlWRKrNOyOWwX3ddPetwDxi4tP6C60uGrAmNgXI2DUCf/FW", - "S5nfvf+nEvvNpG8k+Bfr+k32byP7+8u1Uvy3W3GnCoDp44E8XAWxhVYbHn2DtLkHo6mlSA/SpuJFKkFt", - "ZlwqePT48pttUDktKM6/Nlpa/8sDufL6cKR7ctyFhYSK0lDRwqYP3pMvwI3j3oVb2+/9OwIO0zGd5jyX", - "fmZiilU0I9Jm7SakyoAfm9hdXs+NgvdXTKWD+7w67l2u/kb3dyTx1zfUMG/j0Fsn87u32sr89n0t8xtE", - "U5vZbMtudF1Jps2GQGuHadqWjCvQr8sB4KFxhXQR9E4rKqW6gECDOBiy/631j98UwemHH1wKZT4Y7DyB", - "3wmbf/jBZVGyU0cqhClBbQW9w1+PwYs6hUBZKLJXJmzXx2FqdgPpubIC/3YKUulIbq8hOSr8piG10pC8", - "5VqtIdm9uFsVqVqa5N51JEdvoQW3mOJ/Ti3pT+4eqWhwMp9MaEQJgwIvkJgul+IBjSb3zTNyy4RkZv2R", - "XjBRRRJprUYWXGuNhF7WlP6S0TrdRpx3jrBSJM0UmgockUmemMoISM5yFfMr5mDfYYKughAt5xO63l1T", - "I9dIOAktXP23raZbVPy6b1XX1dp+nFlgPLPFa61yWYo2zdrlwxLv3eqULa7a+9cqHzOJGfVteekyrSEE", - "yhiZAlZpblLmii9LBLQ+evv2lUuP0+qJcEWxFHeVsFyR0CHzK2H10YuyxJh5wbWg1QcS23RaSBq0taVi", - "guOEMgLxxESGMtmq9ese9Fh8eQk4XJyvlQR8z8fSllt9OAn4wVjBvciaJ5Uq1rw0SPh1+4rT4uRNODWP", - "il9ZBhRgPCFZbwvnivdswu3WjBsUtjAQ5VmCI8Ch1K8ZiDSLcWAwEf2mALhA8CQhwkDfZbly4taQFYOj", - "zCtIbyWzC938KGeKJhddE84D+CUSYbaw+E9DVunMynyQhww59jBCQTIz4lqlSj1oynMJb0HKsN8lwskV", - "Xsghs5nL5nOo6itIZFAik6SPfuYAGoHwFFPmMV5TLvE7OWQXNE7IyGI+XCAqkZxxoQgjMUr5nMhqvwSL", - "hBIBkzjCeuUkSvECwNcMDqVZH54RA3BWQZbg+t+YxRQK7+meiykfDBlGO4MBSglm0uaJSzyBC8e2gWAQ", - "lQF9jzDaGzy3X9X2DQCC3fJv6NMkBJnzCI+TBSKaigGpQm3CBqa2EKYpKKy3b0KFNPtV2DdthbPKxlLp", - "6jrGXZSzMhMebP05KxLX9XapXDCYp/UCEiqKa9CCf4xJhPV6Ml7tB2AXeRTlInRB6q32KrL+OwqO3vTO", - "YanCeeYJmAwiEsOeM65mcKY5HKXN7xuoqiSqP8dFEzwkXCCMPLouLRokyoE1bgBM4UVZXpC5csEXm9+7", - "s6OPr2UE7vgboMDHcj8BEfHJpHIA119N5gCvyu9YJuE/6zk9cnVlfRYXUzxlXCoaOWZYL0P/TSFsrRCu", - "XtkgNU+4uPRlqyr9/sTFZVsNzIKf0seliPkz/AodEXp4ADT98P4IsIYbZUUTzb0raXX6Kk4pCF1USRfo", - "zFHC2VSfotIqf+9uA1+r2zCgcfoyFcbZXUD8aCVkZH80pWn1ZGzhT3AxRLbVh+ZFuvd7cEb9yhWiaZaQ", - "lEDp2p4hNr3ZJRwUlPmn0gNFuhmv1KfKz102uqA08QddJw4BXbkN2wDpfXm7gkw14dP1oINF5w5hL4A6", - "OGTvpIEDvzCupwtU8GAt0BqIf3Q1o9EMEAhBb9XtG4BCnGUXBfjy5gF6CQfZx6CGzjcMsL+mNckTYoAF", - "52l6cbBcnPX96Sl8ZMAHTRnWiwPkCrIW94fUb/mIgnoWCZYK/WpxEjcKZRx29EJhrW8W89u0WIMlOPaQ", - "hXAHGbmyDdIJuvAgCC8a8LEcv33Fp/KrcRWVJQ3MXBRHVnUE2iQs7jQFedAk7PjZHgxCSNstkRDNMO4Y", - "CHFpMK/4tCinUCFlnGVtydcOE6h4nqYraBhteLBqUsU8V3+RKiZCwMeWupuIG23gyJbSwpeaUC2InjvY", - "m0B+wVAmg28eXCrNVDvdDmF52jn4zf5rnqadbseOx8NFv4FwvwZRst7gcsiN3hkPNvKbWH4TQMgqs/cQ", - "IWs3h1WnmyXyN+aFP7230NnsHpAMQT6oGXG/JhHUG2/V4MN4gWwJI3t+HyMD+EsUJVySioPn8YBnWUNX", - "TWZsNhS5Ne7p4cW5qzbUJoLl3H567r78CnTvdbEibszITffeg0aWR/CYE4Hl0mwmXNQRl9ZFk3z1hPTl", - "tmRpqm0o5Btt3tzK2IowtZ6wzCLsB7GpPodzxVOsaASVj6IZ59Ij+wIe2dQos8bjgjLBtGK0XJtBcKFJ", - "9cKaoS+sGnFgTWYI+49sH3343OYdhL9wj8ovfvKsAgXH7zrRH6oDQGl2QckEZTiXREt1eUpQtIg0VzSl", - "rgiOZijCmcoFgSp+BKWU0TRPfdxrvWNzDBgdF9vpRReNc4USLKaglZmHLtgm4mlKWEzAPjdkM4LnVKuU", - "AiVYERYtepJA9d85QVdcXCYcx2BiyGIMnh6oHiiIpkAAEU+JwjFWGASdC33iRyaJ6aIoCGzUekauS2qI", - "h0zk7HtT0UA3e+EGeoEIQHZTOSsKR0Y4JiwKQlmff91s7Mvbos+Jqk/0gSKDbsVLHzJUyLe5uuF8HVFE", - "jywWmwu7jW3Y/AqhVzarsNXsD0dG/55H2szVzfGBHEzFEq86xV+HZ6kguq/Gu/Tw7iMuUJyb7rxTCWT+", - "Z/UJFQzFD7aCzFKzjbd1DBUV8oplvhHP2/rD/XlyC1veV8IJu42KfVMtpnLSXwPLtat6K577QEZMa0vy", - "bXIPx4JdRNeDiU9ceFzusRhbLcM2R7Pg2z53UgKD9sXZN7ZdZ9s24OG2bNvZZpdc+h4jp6wHMaJhDm7N", - "uI2s2poO/k2zUWqz81jmg7PI0nNxb2zxpGCEhjVmeJFwHP8ZgoRX+I8iLoSBvwBAjccEv+pZDf30ALDN", - "lUXeui5b8/3p6WYTlxBqJY8Q6hFzCC8lR3+WxssG3NdzIgSNLUopOjo9tuG6VCKRsz56nVKFFEeXhGRl", - "RgtkFfb1/BwQyHJB+QriR7dDmBKLjFOm1o6ifPVuBvPpVmXo75lPWjzvb+7w1u5wsOw/PnYGXAZyNswE", - "VmumCqu1dUYpm3CRGrkMj3muW9c8SC+T3k+DVDChCZELqUhqohIneQLHDWpD2Pq/9juzy12IydUnx6TL", - "ZUSkVErKmRwymyuSEaH71p/r9r0Aq6BDQOGCv54ZJvl1BO/pwZh4NayaVg0gm6CuaOegs4WzbCvGCjcE", - "iNnhfcaQfoJoPCQX6ZgnNEIJZZcSbST00qgnaC5Rov/YXBnON4LvvnR149ufLL3SJ2zCg7XjDM0WxPyn", - "yuqybM05Jh8dW3tJ/MPi+A9sdJitra+fLAhOelCP2AH3oFzRhH40rE43QqWikUk5wsXavT8tmGp/yE6J", - "EvodDKltSWIQDUC73MoEj7aG+WCwG2UU0N92CQwOGF7z4xR6PDp7Z9JQScrFojtk+h/Q8NvDM+PdnWBr", - "TfAGagsno5Ot12sCnM9hmf6NIwTNBFeiFwQ3/JtL8OYYI41nSDYcUZ6tUpV49qcPYbUS3De7wuO0KwDI", - "UzGbjQLYy6FxhW0Ic57kqf6H+eNkHa6ZwtHsPbz61Ui7Zjhru3ETfBSH0s4pJqa25YM4PcyCPdaYVb1w", - "bgogxFSiAYO3wKH6M1L3lzff++v4Fbo77Yq6urFfzdm675vPjsEhbPjr8ViOuaE0NxPFV1ufrjBttj79", - "mPDoUlooFt9sqPU2wFfXP5Z42NZFCGICZIYiC2FkgLKI7A5ZzQBpEH8kwkgRkVKGky2Ys2kEkL2dFQvP", - "OYUE7QjyVHqSxoCZlAB8N8Df6dmAoco14Hl0pa2s5b/jOyMVR2MS8ZQ4tPPNkOr2N0zVT1xUocu/Fr74", - "1lt/gATEFOzta9Dam3v8LPT2U3wNodJxbh3KbkQbL3n5ozEFdRHszbCzO5DDThcNOzvpsKN34AiDCRUr", - "tI9SynJFZB8dG/sWpOA+GSBJIs5i6UDXnQVvdyCbEnINWTZkdz6B7+5T7LFUBUv5xnYSYg/6PaS/h6Qd", - "tOEfOHsm4y4cuhjxXBlzvz1X9q2YKDCPbN67r9Y7I990+zac/G/2+FZ4FOyyZpfe1hvOnuVyRppNbq9M", - "IaNcjQHM2xUXlTP0dz6WXcTIlbGGC6n6S3xPf31mOriPQgO6q5sUGbBz/1ZhoEWFgXKtwmCNJsBSX8mO", - "OgxiI7nOuFCA4mhz7Q0NgSYByBE8wgl6fXQyZJFmRQZaUJCUA3eyeOjmFj782zl6cfSmi46h0CX6OR9v", - "9tFrlixcuXHjoxkyI4kZ5hVhhsaGakkcup7N2IF67jJYXHfwQJWjzckIeFbcXrkg8W5nRnAMEskfnVfc", - "dBZAHX7zSh8gAP41Xxbb3lkpfHTeECUWvcOJImK52VObJ8UKzAx7STsIOiu4GeBL3aF0yGtln0Y2MNAY", - "uzudAFLGp29FH+6+QOr9eMlMnIgptzfOAWmUQZIBjhePK5ZJzlDBHEMs0L+ui7IJTVnClpetVDCgy6bI", - "76/I5L6Sd1Ww5f9dTxfM9NE6mrLKPmkiLsqtrPX0uuTgmYFDto6qCGc4omrRRThJ7B1lb4IiIqVXiL9j", - "QfBlzK9Yf8jeFIVebEIvOjp713WOWhRTeWlasL7YPno9J0Lm42JwCA6a8RrDmpN4yBRHEU6iPNHiBplM", - "SAS5uFC/RTb4couhdO7w7JSdBIvNeFHt+aOrcRemCdi9kizqFLdltnpLkCjBNG0GH7eCGgQcQqjBWDfK", - "GaJsktiQqkhwKZFtqkcSOqXjxAYIyT56OyNI4pQMWZZgxohAuTRR8XrovUwQKXOT4K0bAJBeQ1FdVAIL", - "ZoIrG5qQcC6kiSbQFP7+FElFshVk9sa0fApzviPZ1jRue3ogI3VtDM2mEPsK0htiKMUsuKajPHEBjPca", - "im4G9NBS4mM5+G8FnU6J0KcCGyZrwvHMsXbLaQ59JWO5sd7lefFWu3qXRateVqKXsbcSGG5UYm3HnZtF", - "/QU6v6SN2IH20c2yiH/RH7Xsu5qtGh6EffSZswyV7vx3rJJ57iUJtjVglRT+2MxJ3sgrR7WSaLseVqt1", - "Zu1dZrq2xs96MNisx4yWhSvps00K79dHCIP7RXm47yJrj5u2KmhXFd20IeV/PZr+V0GBdwOj/8AoJ7eA", - "0f+q8u4B5/zh8E+CB/Wh8ugrvmdXbPdPj4R/V+nzBg4f4Nia0ucN17PBqysVpff2nXZqkm3xzyTB23jH", - "G8jvbtm/af0tVAZvsda5oDXBkzRTCxfQZn2VZdCZpB9Jv8ERXMSt3p0r+BYhnV+OPBydNgZ0/jlr4z9I", - "zKgtHUglOjkOFJ1/ZBiD/pmrXCxb+tbpYRHN6Jw0G92rJ9guUSZIL+MZOFdis2B2PdxdprDoTz8i27zF", - "XLX/gtqTANVPYhRTQSKVLEwdUM0RTB/fSSS41gTgOReL5igRc0R+Ejw9tLNZcx/aM2WNYWWcYbroxVjh", - "3txxmxUmtM+I7nTxlJrhIcrQyx/RBrlWwlS4QBOt+SA6KZaUXEeExBJoctMf8PagwbJJP5LRdNxmlCtq", - "lby2tWBQlEvFU7f3J8doA2qfTQnTe6FF/QlIspngcxqTuDLGzpwnZlW3Gxb0pnZXLVQUheuccmEG9yAy", - "TJsLafqRZlW2UITEjCnDMLi1VUGqZ8ok8ev+MGUuAMfukRvFtyvMan4bTtnRlAh1OO0iKs4NxPPmt2vu", - "MV9zfjKUu9Mqt50Lz1ltvG6XH9UybekuCj8UuXP3a7Z+//Wk9FD5KLN5rOl8XiikTWbzr4sEB/d3P9y3", - "ufz9I04BfUmc8u2ZyqEB3WKIYF5BTHdM5iThWQr10OHdTreTi6Rz0JkplR1sbUHs94xLdbD3/Olu59OH", - "T/9/AAAA//9gew1xCvABAA==", + "wLMbbcPOGhvJ2tHcxL3s4d3eB8ZtXVLxJMQvjmjrexhdErSh0Iphq6I4r7Kvaa0ymEVCmdknTTQrJhhK", + "kRmVgZF+q3IhJxbf0YrS2MlDBm+6gH8onfpyIbfGudzKIrpl82+2AJvjGWBz7AWTp2MyH+V5SDXSjxwY", + "y7t3J8doA34BbFlIoawSMMZPtp8Nnj3vPRtvP+ntxYPtHt7efdLb2ceDyW70dHd7Z3dFIkyLbLrbJ8gF", + "NeZAHHMRtT5y0fOhoOam3IWabGLjsa8oi/lV5foLBsj6vdvg23XdL4fWtx5CMCEnwVIZ80UDJzuFS55E", + "um0TkG4zNotSS2FD55O3g+3Ptf7A4BruiLciZ8atajAGChdC6g3Y36zqOG/H8mFALvFl3Wr5nbdftMHB", + "/vOD/c9dNJe8sW6MdXK6x81tighzGMy17BCXoejZkZyBsmNlImPVt8kknW6nyHeBv0EYqMVSF49bJXE1", + "HdhumI2sulYakqdPKvoKRKoYDL74QEsqTh+Big5Fir4WgY4SnsfIs8UZSDLww514uotuBtxi1kRnIEZN", + "MobWcQDTGipHUKYZMfgfdSM20/oAvYR34RFOjVpnB2Hql/iuNxwvTLyMPl+ua6NkrR7yudWv4ButbCH9", + "L5i2XgZrsl3dhJHODtCvHL4ptD3G67Zf8zqoWcuv1+3EGxau2yFnQGdW1DxAPxXiZSGgWoF0QxL758gy", + "rBKwZrMCG2B3vKOppdw5LwW+2zEr2ul23EJBqvxy0vy7kuqXzp9PiqFAMoITOMtljnCuaGJhumEmVCoa", + "SZs8oje3SeyxpZVIPDLKU1NMqkk8tQpW8ZGTqt6fog1AYvwLsoZt/a/NIn61ctftPN97/uTpzvMnrfCW", + "ygGuF42PIC16eXBr5eQoy0fWNtI09aOzd8b2ERmrQhH78v7Uh7fIBNesR8/cNeh3/rz/3IeZink+TjzH", + "osWkM6i2sGFBJLWCFzXEQf5OkzmdTNjvH6PLnb8Lmm5fP5E74+0G+FzTUdjsduIHFyzZqMm4Z8okhZGA", + "gKCEbATLekMkzACdE4WAfnoIR6DeFNnMluQcpJZd8SBh7e3u7j57ur/Tiq7s6LyDMwIjXOBStiPwjhi8", + "iTbenJ+jLY/gTJsOUwIQzplVfcPnDNkax4OqQNrfHuyGqKTh4i6pxrY9TxuX/L1VH+2k7KJDUnahWi6d", + "8uBq7+4Onu7tP9tvd4yteXgkrldzGJeyZJbHAvH7O78B0uTbwzMECcETHFVtOy5C7EajUjcaFRSRMODv", + "NxjYs6dP9vd2d7bbob6Fgk4snmHlwFZ5V+DQBYgisBuBpVhmvd2m2yIkThkCe0OiBNP0MHIpFrXbx4C8", + "j4R5rdyENheD1cCXLq4W37YybhUmK5OgY0QDLlDOitIi/fUu2S/iWW3m2uZ6WM/VQ2k5TK+ehScyJdRu", + "sZSZIHPKc/kFGuLK5MxOEs7Fjb5tUljeEJknythsqETvT78DnqJpDUlFsqoOZalxBYjTLSd3o/NcIZEw", + "kTctVqvdaLP1qybcbTi13VWAGhVu0AidFmvOlbP1wZ9HOIlyKKaDi/3UswIMMIAEyLJkYWL7k4RzhqIZ", + "ZuAkER7iEZrxJO4HI2H1k9EkGFXBr1DCDejzJSGZrTNjBqE/0yIMnRO04VdYM6RUq3u6nxomYyuJVKlx", + "Pw0XcMQylKxWpMLr9cSKe3jE5pOKJTThUwlKoYKshX4dBj/DwiQjYGbqJs1To0sGAq4DQ6wx89CNam5S", + "PrEKrhU5INHcrCSOBJcSkYROoUbP+9Na/vKKnLcii3l9QGd1sC1I1zg0A1eZQcNqXV4tdD8G8nk+54YE", + "GoacwRWhks44mWKWQ+UZj5CtIb7fOhxyxqUaFbhUNxysVCMoJ5ELUqLlFVn3hT3IvRO8Fx1ru81y2bjj", + "W329RFXhppoG2MxTgysaXq1uQYMhMl5G5loJBlaii9WhpG4CVlfWH6ASWqUebBnagJwXjy15EHSbbYJk", + "wiqr7mdJW7XVQV/tDc7bwrqtRnE7w2p2wiY8gP1xA8+ps0TbaNWMiJRCQRUUE0ZJ7HTJwoVqTV2QMJ5I", + "guKc2JUz8qnAdsGxOd7gs2LORkbZtMbr6x22MQ+bMayuNgH92hfbhDvJcELtW5HDWpl4RYlwmVrbKgiU", + "ylHYnbXcsCDTPMECWUDGNkOWizSh7LJN63KRjnlCI6Q/qPvFJzxJ+NVIP5I/wFw2W81OfzBqKk10bgZn", + "8wLNhtT6Lafwg57lZi0rGSwxW+b7LXCMtokeC0aK/0QTYtH93jF67RF6FY59b2fQlC3f0GglT34ZGfKm", + "nNuSbPDE5zKQW7hSynFFlUhsMfKN2JPl0tR3aXErORBW5wK8nUenmjjyedAkR4Zf14BJ0JhA3o+b2jLX", + "aMEW20wlWFoilzP0dz6uGkTbhv0GCpZtsBIiQ5BJML4fdnSlQdq8sbQm3u7eBIMC2KqeKHx0Q2iHdaXd", + "yviqJn7yZqnK2YzYJaNujqbiWYsKHi7+o4AvsL22xzGo16MLxCsDSo1UC6jsCuV0Fl6RRYnGXAhAoNYS", + "DmduNgC7omUevdYO9wq9nZEFEiTFlA0ZZYWRFMDUCGJkToSXJcuFVrKmJO6jv3kqHmB2p5laWDB4MJ5/", + "JxG/YsUYh8wfpG48l7qdQ2YsiyLPVKVcpG4WtD5NKJC1DE4wJaB+IlUzNBFEzvy5h2pmahnviou4sRjR", + "ArlXoMYN+FiR4peE+aysaCaoGpqGRuar5Sg+U/AWnlr9E1Vq0KJ6jdnV/eWSiLCQWEypeKVV6Ip3VDzl", + "xIDAACIK1Ba0fxkWX6CgtMA8KZv/q2uy/OmsaLz6W+01D9fEwQwfGrNt0AQbmTSeWrBP1ZO2NlQF0uBW", + "odks+xLQhguhdgVaqpKAVyil1T3ZLhOvnizgRrMlSVTtfe/Z/tMnLSvVfJazzqB3fWnX3Dxd4ZJr2KnT", + "Nn6fZ/vPnj/f3dt/vnMjD4vLK2nYn6bcEn9/0Aa5VvqwJv/6xz/fn9a8PvsQgz240aBMZkl4SA3ZJdUB", + "vT/91z/+6UZ16wGFGM0yQniD374xSifxd9IFClRdeO2cZCv0+8OKkQAXbAZtkMmEgBl0ZNatVw6mBgPS", + "TgrGGY6oWgQYOb4y0e7FKzWk6zbuoOpgQyKvaduiomrOJfNxmXS64TpH/2l8wzVaeNa64JXMx01+6Nf1", + "Xo0XuvRa+DEOLUIMZFFrfdnAXcznCstKQLf+O4K8C5dhtpxtY95YjbpbT4WAKBZb180LBQyhtdfkSfuR", + "v/217fT8lhWzTn3FP6w4h81H8EZW38CNHDD6RutTa2v8wV6At/tqNPZL0a2s9VepW1feujfvt0X28HKd", + "hOIGu3l/XsLkTT6sYwIDPdox2CUv2+5WSKKBmrxcmIABjSekVwTq2UQZJHPjEdRn3sLMBzI4o0s+mVSx", + "bvebsdEB9geSvVwvWCmtmXQRuXY2izqwtsH4GXb25bCjVYBhZzsddmpuq2D6ZIqvR7aDKrbLYBVYeZn+", + "XhukdDMYJzy6NFXWoHh3Hw1QSjCTKGdw+Gtete3Bau9Qt5N5e1NAgxMT4rTEtmBMYzLDcwoVKaxPZVoJ", + "xCTXVEkIGIV2DlDMDdpTpcSsnaF+zSQ3HpSThksHs4VtWDeo3+PMRbSW74KBbwKFbdlHInjXghVojv36", + "9WnXBDBA6KEZWCW+0U3UjEAzyKKLWnmF8vdw/PA4ISMYdx2uP11eRz8nHTyrgkiipMXvLsmhRgQo4jlT", + "dRz/tJ0iV00rW76ScgbBfjb8A3DZbO+GQFBMIjiRcvksVgn9FsRdyxuwKx1KHNgNkTAcCvAlhX3Fb6xD", + "uD4AY2zwqkObdvy4buMlHEnFbTmx4lSPyHVESFwH/Ay/0jZW3n4ZjJV/hS1GUFG42b4N8c7Ls+vfXYIX", + "jLVptf2YfsZZD9BJ3JZaJBEDDWixaqqEVoEe9yAtRiF41dALbTKuyfXqtf6VXCvAR4/zxIDehUnXsip7", + "Ga1b8VtnNjYdaC7I2vJ8d1C2zsSb36pwnQ1Vf4jadfatO6lXt7Q750S5d88tGTXuULXQS8Wl5QL+3SvV", + "GBtDSl1kL3i0nW7WSHBvFraKWFDeljmaDKdklAkyodcriMe8YBTjKqxJeZCKDAaDL7qR4mu09xRFMyxk", + "beyMTmcqWVQDcPYCWEqfVdRREEWYMxS22flyN92Hy9Fudjv91kPC8bkHDbRU0sSKpKNVuNlHpbfNWucz", + "vAArTqOT8Onu3mCwuzO4FXC2G9YNluuo/MSWQKy205RS531nHf2VKFW/hSLJermu7pWgkKtdLJNUguD0", + "ABJvMhwRlJAJgOQVCa3rPYv1rlcP3gpUNou2oH+3UXbfnA++WjKn6MpijrtpdJxzsYpB5D9f4xBtYDPR", + "EqReIOdutzd48nZ792D/ycH29l2AXReL1JTt8fTj9tXTZAdP9pJni6e/b8+eTnfS3aAedklNZaA2tPqL", + "frcxyqa8JKtYRhWWhjbsHDIi6gWT64XGJUkoIz1ZZEitT1NcwQuM/33t+b+Znd/MYKXscF6dpC9CYFUu", + "ToWyHgZ/y05mpe+iPpuT49WzuFUGUn0gYXqrDwXIq91goELFduczkRly1vIaeue92PoiWpkVt+4qCnnY", + "4aQHd7lhxUPkXQNm8Ga96gJfvuQCttMpF1TN0tW3RfFaASMOcdMfpYqreE99dDJlUC3d/7kIk/OVKP1x", + "p9tJPu5Vz4z9vT3yl0UgLgjQbrUvFbQII4Ni/KtXAV4pFQ9hItm1rq7H/MN2b/s5xCEkH/d+GPSeVyMO", + "uma1/OXbdm9Xfh20WUO/BKArHbX9/EYR1249V1HQLzRUwK68ly02saXxsja1uzpcwm1lg8vHS3tcQ/Jp", + "FEA/V9Kzl9vIF5pikuBFCJveM9TKmvboExkakyllso3ddndQGG7302Gnjw4tQDjosooX/fjNQw16j05o", + "mpKYahnTqP7NGQw7LW1xdV3iZrVJ3FcBaa0fFteer4dIWJdwte6a7H9GPu5nab/tNN5V6B1gV3MqKmCI", + "wYtdRCcIs1qBUsrmOKGxTaSHxEiIVztwQG0lyVoeIEs50NlJumjKFSpT6Fva23LWbBcsxk+uwd66AjPD", + "EMTOFwFEKQDE6Cr2dXKMMsHjPCrzRxMYdIn4IfIaRNsKIX99SO5d2jcgMXvCBVpv32gyaLSzTzbtd802", + "qQm2eau3B+u3+k6MIt1OnsXreZh5qR0HuxFy+5oUxICJprrsNUnQm8yHFhz9jb+CyzqvsSVHWiTKM+dg", + "0TS1TEkBdwu4GEJxvcckIfqaWm4E8SQusySoLLnoepa6/eTZrMnFCR6p5YH8QkimdRXAP4L+UswWwYG5", + "sqPFXbIxcGjf0ji8eqZckV2t6uCerpXEGrfKN+E2lVAwXL5m8zZ4KZee+bvA+PZFs2UEFMfwK0Lam+YS", + "APZLF/bWaD++C7PcQwppr63roQbb6kCFC3R0138ZC6zFuirx7oXc8yGyeGs14yYo2noWqG91Puz9j7Ey", + "o1H/YOuHv/zfvQ//GbQ21/RmSUQvJhMINLoki54pPqR19H4ViBUqH2hhempJheAUbEgAcm4Poz/e/UHB", + "NBa/4nRpChCh5VUO2l47ob/8R3N8k7eM74BPriXZzy4MchcFVBV319FGSsTUxZK7RLLN/pBBbtolWUjk", + "1SOzIo0j1O9k8YkXgY4ujBjYJ2x+gcYUCjzKIdNaLY4ikmltwpa4oaZKOQfuIwhO/HZsXTSX+G0dkiae", + "gKD3p0sovq/fvf3x9btfj0evz178engy+uXFf0OIx1XP9BD3NO3t7T+xtcn9ldwO1se4eZmHPjq1YfrW", + "1T/JQaEFnC6J0lzlEBRCrqMkl3TuHIQquX1Bh+Vk3dsXSPhMBGClklBUgkWqTuiEgF8frhMbVEOlI0Yq", + "oai7NW5QhpZvbEM4ww5wUq8mf6icht6K8GqXG1td9CezdizUYKMGDjtkvEL1/YD2QiXgVbjYD+9ltAGZ", + "I67yrEuc3bwZVuth0WAw8vALFxgaPP8SRUDfraz6OedJT6s3DZUSgtZksxbByHloymQkdJqcDtNxQIa3", + "pt0pneKAnyHkT/gixTrdgNZmTC3tf2PVsnAew3G9jIQ5lmapamUPakYCqXrNaQ6plmobgHcBWdjkrlIv", + "tq6aqJoytWWL6obwMmIOYOarspXLU+bQEXvw0fok3JV6lTczbyTNe3Pq1IeagrNigc700lzNiCDeRsAH", + "JTz/DZfM5uW0QGExRQkzIsqYVZfUo6VScDdLtFFYftwSFNnGy+bw1eUXTvF10QO4UrBc8j/CPMryT9sv", + "fwSo/Deu5CWduCZgGDXlLgwMX6WiVWviqGp5M3yqWp63eT948CyvWsH9ms5WjTjLPiqkGaLHv2GqfuIC", + "1MFmzJM7x5eHyz8mAjDg6ujxraDXaUriEc/V6vNvK+rbK78oi1qW1XWqLwYijirpvE28wKFylGNYXmm9", + "HCTKBVWLc71eNpgb0iBdLVtYSOgIfi47hvqhnz6B0XgSSBh5SRgRNILqrPo8ppiBxoTen3pF+ky9xiW8", + "VhCBXh+dWHODg/wF9ZEqID0Xd3l4dtLpduZEGJW7M+jv9gdwmDPCcEY7B53d/nZ/0AGtagZT3IKK+jZ/", + "2uYbF4rrSWwloR/dS/pLgVOi4IvfAkgAEHdoXwcVBE89JTLDVFgtMksAocAQDNVfQ7kBd6EemFu5a5a9", + "tc0U0owh+4Vkr+3mfgBBGc4OTHNnMLDA5spev5C7YxIGtv5uo0fLfltJdXaJAuj7S2qeky2Lpf/U7ewN", + "tm80plVDgbMb6vgdwzaJl4B2vn/DhbhVpyfMpOXZJGsbDuWfOCAk/6z99kHvmczTFIuFWzB/tTIumwRj", + "IhF27xo9TkkUaVYBNYL66DUj5jnCCmETuSxyBqWV3YeaQqunwLTtNrkAKfqRx4svtoSVPpyN4lOVnenj", + "8mmJnr8c7RRkvLyR9pFD2DZUew8E9CMu6oI/2EnZGzy/+06POJskNFKoVxCwjUemEkJ+EsALd9hDXKDf", + "c64wKsL5H9GRtjLruCC3bnkVbf1B40/meCckZAY/IyLFzCRHmHfWHPql42xcEuVxXnmrOcI/Oe7Ym8qB", + "8JiLCgS56hH1r626MLh8He0FEBhsn2Z68QMS/t49nHA72aI07EMeOSjIiXJJHtNxsi62cSmEBGW5l0R9", + "LTQ/uM8ryxYR+BOeosdCwC9JIeGVu7V0KWxlImdGAQ5KgG/KhEX73XdV4e9t+cSLkgG/hm4aylko41fF", + "8aKP3JoapV8tAGJJEJhnvHytnOnhfS0nbOc+ThjMuPAUfbumvl1Tq065oRY3BTiY3ilvYYO4kQXiz2d/", + "uLH14Zvtob3toZXlgZEra134Ox/3kY1IjXhMkJzxPInRmCCDd+RiTxQW/elHhEU0o3MCoHZQpC1PFM2w", + "gMiSFMVYYeNDbzRMrDRLFM1t6eZ6Lg6xXOA6joUkI8DhGzXhT5YRiJQxEiP9iYXuK+EEl8qJm7MfNLAX", + "DZZXI7qacUkKPD+mvNsc0pul0Y6h2f6QvbVAr3oBIZja8RpJEoCrXWH/4QzhIbMffO9YiAsEkzgtORcW", + "gBlIDTKl2Zbl1DY90pGMeAhr5y1hmKmezEhEJzSy07okCxvPGWywVd0lPWA3zvenRcIG2tkM47UBPGMY", + "nPe4eIYsJVX9NwyCoKMkj0snl4MQwmKMkyRYmGOa8DFORmZ9LknAJ/gS3rCLUjpcSm8S4zExJeSzhZpx", + "Zv7OxzlTufl7LPiVJGLY2ewPGSRi2LUmcbcUENEVFHJLM67PmeCp6XPLDHHrj0uy+NQfssM4pcxRBHyC", + "E8kRuYbvoL4VYGYY7tVAD+Y0hf3gR7lUPPWRTx3dmWHyXGW5shklkqhuCPVzyBRHfzhsx09bf5Q9fgJn", + "McGxphPvFTMlkK2bRi1HWM9+BK8G3O0EFmDY0RepCfOYCsyUge0swCnR1N/SjaI6AlRMra9whBnKeGYq", + "SwBRzbAmuUobgNWAkwQpOEruWy24w042zMdC76XjRtw9A5RWO0aUodMfvcM02HsWPk+SRIKEIkr+6/z1", + "rwhuZb0H5rUyXMukdDAtMKA4B9ep42kvcDRDxlEFxQSHHRoPO4U7N96EsebShsv0euBT/EEP7QfTTZfG", + "P/T7uinjrjxAv/1hWjnQZylLDQ7osPOpi7wHU6pm+bh49iG8oE3wZecVRoA2zDW3CZwEU0Ca8W58c0Vi", + "FiNub4FkgTAqOZAfuDKmDIvFqkTCwNLbFeQTE8noLcYfQ4hcHHYOhi52cdjpDjuEzeE3G+A47HwKr4D1", + "WjZXroP7rHBuFkT0ZDDYXI+Ebdc34LNs4Rj4wjpgo1ZUlN3UO2hhWP9c/oF/a/2zcP1gpjsvoYmM4u+M", + "74/QAeFJ7L4mGnBB1MRuzCKSOLF7vaHn/p0HerMikiT3TaAPRZ6Fe6xA6n9U5AibVR6jleb7B6a4wX1d", + "KhWz/cPQ76Oznwes59Z2TuYu1DlcpwQwaKwqjczLCEt0DmPqnWvl+wX82rf/dbofYCpeJHx6cWBUd5Tw", + "KUoos/kAXqCyFg/sWsJHBoam+M6i0rgicRtGkvjXP/4Jg6Js+q9//NNiu//rH/+E475l4NWgxvTFjGCh", + "xgSriwP0CyFZDyd0TtxkoAosmROxQLsDa/OHR8grdW+lNDlkQ/aGqFwwL2/C1GuTtkHrKtDzoSwn0sL4", + "6BfpxBaTMbGNAbuNO8tmKe/1RHcDcIgwA28C+lZ0NABYctQU2raaaCdsMjVzrhhN62GaS8F66/mLItfK", + "UG/PDPCGDAaWOHTu4IGdNNo4P3+x2UegbRmqgIJBoDuUzVg1ov+NJ63nSYajVBkKrLLhTRHO8Jgm1Jkc", + "G6qdmCOY4mhGGSnjiwuscdfEgRup5jGHZyfIBkJ24dUhe32+BSZWRSKVC9K1nEBYhNGyHBq3eS7QA/Av", + "qiA6rGffHbIJwZAndHJsmIAHwl3kAxYNMwDygBhXqiqV17pDZpBkLXKxPngpj0kCH0H/U6zIFV50UVHr", + "1lVHSbDSCrHs6peHzGC92jXoAVQJ8obZB35mhtRzkbw2Z0uQSaJVY4jAN2W/oe+NCRfIRjh7Vf5ddybJ", + "0gxLL1qKo9fnen5T0AS5sQdCS6/P3W5sdpHkKEooUEOE2ZBNIRDIgfdyVtnVIqFshkXci7i+BHwwp0vG", + "rxIST5t47JFPZHcoyVT6CRynn+vk+tiEi9nyBPQhNgB1qz13x/addq472+KfyXdnC0HewHlnLLjE8Buz", + "ut8ceS0ceeF1c069kGft2CEw3l3Er+nigQJ+He0tr7l54i3ZQ1j00IaDtgGvCBfo7OgE4TgWRMrNf297", + "n56podJS/tP3o2bFDxF6YsfChQX9s/aWKoE8Fnbwxo4aYTeven1d/37bqhTfabzpijo85ZV397dHrdOb", + "XCOl0FvS2rebZG2wLZURhzKDJbX0QDRKSCG+FOfUp6J1VmUTxltcOSvFJcueT47dgbw/+7LtOmf1u+Ee", + "mOJxjSE+ICOsplr7VbMfEzW/K3bRoU2vMD9/XaQ5uD8p6L5N0SEyf0zqYlxbNs0FDdBJ4wX6kigDb3KX", + "errtITDxcyLcqTYDXZhZF9MynyKD0wITAkvMat33xLzSTvU17f2ZNF9YnptILHbJv4koLZTdcq1WKbgn", + "tgT03em30MON1NsvF7ZiCSywyGBFHTu3E1hWN7BcsGjzW+TKF6doE9dYKrHCzZvEhSXboCkVetZ9yXWH", + "zK83rmU6q9dShiYJnc6sEyCmE4jVU379bhjlzj2MsqiTLbAiNkTxMeb9nulFtl7gOREKvT46MevvX6lb", + "f0DQ6npVyTGvlbfruzeveoRFPC6cJ80yqX3yhRUmQ/+VXN77P3WPMJ+VOvGgSWD8jP03weTIxL/3Kf9f", + "Oz8ldCywWPyvnZ9wklFG/tfuYYIVkWrzzohlcF833X0rMI+Y+LT+QquLBqyJTQEydo3AX7zVUuZ37/+p", + "xH4z6RsJ/sW6fpP928j+/nKtFP/tVtypAmD6eCAPV0FsodWGR98gbe7BaGop0oO0qXiRSlCbGZcKHj2+", + "/GYbVE4LivOvjZbW//JArrw+HOmeHHdhIaGiNFS0sOmD9+QLcOO4d+HW9nv/joDDdEynOc+ln5mYYhXN", + "iLRZuwmpMuDHJnaX13Oj4P0VU+ngPq+Oe5erv9H9HUn89Q01zNs49NbJ/O6ttjK/fV/L/AbR1GY227Ib", + "XVeSabMh0NphmrYl4wr063IAeGhcIV0EvdOKSqkuINAgDobsf2v94zdFcPrhB5dCmQ8GO0/gd8LmH35w", + "WZTs1JEKYUpQW0Hv8Ndj8KJOIVAWiuyVCdv1cZia3UB6rqzAv52CVDqS22tIjgq/aUitNCRvuVZrSHYv", + "7lZFqpYmuXcdydFbaMEtpvifU0v6k7tHKhqczCcTGlHCoMALJKbLpXhAo8l984zcMiGZWX+kF0xUkURa", + "q5EF11ojoZc1pb9ktE63EeedI6wUSTOFpgJHZJInpjICkrNcxfyKOdh3mKCrIETL+YSud9fUyDUSTkIL", + "V/9tq+kWFb/uW9V1tbYfZxYYz2zxWqtclqJNs3b5sMR7tzpli6v2/rXKx0xiRn1bXrpMawiBMkamgFWa", + "m5S54ssSAa2P3r595dLjtHoiXFEsxV0lLFckdMj8Slh99KIsMWZecC1o9YHENp0WkgZtbamY4DihjEA8", + "MZGhTLZq/boHPRZfXgIOF+drJQHf87G05VYfTgJ+MFZwL7LmSaWKNS8NEn7dvuK0OHkTTs2j4leWAQUY", + "T0jW28K54j2bcLs14waFLQxEeZbgCHAo9WsGIs1iHBhMRL8pAC4QPEmIMNB3Wa6cuDVkxeAo8wrSW8ns", + "Qjc/ypmiyUXXhPMAfolEmC0s/tOQVTqzMh/kIUOOPYxQkMyMuFapUg+a8lzCW5Ay7HeJcHKFF3LIbOay", + "+Ryq+goSGZTIJOmjnzmARiA8xZR5jNeUS/xODtkFjRMyspgPF4hKJGdcKMJIjFI+J7LaL8EioUTAJI6w", + "XjmJUrwA8DWDQ2nWh2fEAJxVkCW4/jdmMYXCe7rnYsoHQ4bRzmCAUoKZtHniEk/gwrFtIBhEZUDfI4z2", + "Bs/tV7V9A4Bgt/wb+jQJQeY8wuNkgYimYkCqUJuwgakthGkKCuvtm1AhzX4V9k1b4ayysVS6uo5xF+Ws", + "zIQHW3/OisR1vV0qFwzmab2AhIriGrTgH2MSYb2ejFf7AdhFHkW5CF2Qequ9iqz/joKjN71zWKpwnnkC", + "JoOIxLDnjKsZnGkOR2nz+waqKonqz3HRBA8JFwgjj65LiwaJcmCNGwBTeFGWF2SuXPDF5vfu7OjjaxmB", + "O/4GKPCx3E9ARHwyqRzA9VeTOcCr8juWSfjPek6PXF1Zn8XFFE8Zl4pGjhnWy9B/UwhbK4SrVzZIzRMu", + "Ln3Zqkq/P3Fx2VYDs+Cn9HEpYv4Mv0JHhB4eAE0/vD8CrOFGWdFEc+9KWp2+ilMKQhdV0gU6c5RwNtWn", + "qLTK37vbwNfqNgxonL5MhXF2FxA/WgkZ2R9NaVo9GVv4E1wMkW31oXmR7v0enFG/coVomiUkJVC6tmeI", + "TW92CQcFZf6p9ECRbsYr9anyc5eNLihN/EHXiUNAV27DNkB6X96uIFNN+HQ96GDRuUPYC6AODtk7aeDA", + "L4zr6QIVPFgLtAbiH13NaDQDBELQW3X7BqAQZ9lFAb68eYBewkH2Maih8w0D7K9pTfKEGGDBeZpeHCwX", + "Z31/egofGfBBU4b14gC5gqzF/SH1Wz6ioJ5FgqVCv1qcxI1CGYcdvVBY65vF/DYt1mAJjj1kIdxBRq5s", + "g3SCLjwIwosGfCzHb1/xqfxqXEVlSQMzF8WRVR2BNgmLO01BHjQJO362B4MQ0nZLJEQzjDsGQlwazCs+", + "LcopVEgZZ1lb8rXDBCqep+kKGkYbHqyaVDHP1V+kiokQ8LGl7ibiRhs4sqW08KUmVAui5w72JpBfMJTJ", + "4JsHl0oz1U63Q1iedg5+s/+ap2mn27Hj8XDRbyDcr0GUrDe4HHKjd8aDjfwmlt8EELLK7D1EyNrNYdXp", + "Zon8jXnhT+8tdDa7ByRDkA9qRtyvSQT1xls1+DBeIFvCyJ7fx8gA/hJFCZek4uB5POBZ1tBVkxmbDUVu", + "jXt6eHHuqg21iWA5t5+euy+/At17XayIGzNy0733oJHlETzmRGC5NJsJF3XEpXXRJF89IX25LVmaahsK", + "+UabN7cytiJMrScsswj7QWyqz+Fc8RQrGkHlo2jGufTIvoBHNjXKrPG4oEwwrRgt12YQXGhSvbBm6Aur", + "RhxYkxnC/iPbRx8+t3kH4S/co/KLnzyrQMHxu070h+oAUJpdUDJBGc4l0VJdnhIULSLNFU2pK4KjGYpw", + "pnJBoIofQSllNM1TH/da79gcA0bHxXZ60UXjXKEEiyloZeahC7aJeJoSFhOwzw3ZjOA51SqlQAlWhEWL", + "niRQ/XdO0BUXlwnHMZgYshiDpweqBwqiKRBAxFOicIwVBkHnQp/4kUliuigKAhu1npHrkhriIRM5+95U", + "NNDNXriBXiACkN1UzorCkRGOCYuCUNbnXzcb+/K26HOi6hN9oMigW/HShwwV8m2ubjhfRxTRI4vF5sJu", + "Yxs2v0Lolc0qbDX7w5HRv+eRNnN1c3wgB1OxxKtO8dfhWSqI7qvxLj28+4gLFOemO+9UApn/WX1CBUPx", + "g60gs9Rs420dQ0WFvGKZb8Tztv5wf57cwpb3lXDCbqNi31SLqZz018By7areiuc+kBHT2pJ8m9zDsWAX", + "0fVg4hMXHpd7LMZWy7DN0Sz4ts+dlMCgfXH2jW3X2bYNeLgt23a22SWXvsfIKetBjGiYg1szbiOrtqaD", + "f9NslNrsPJb54Cyy9FzcG1s8KRihYY0ZXiQcx3+GIOEV/qOIC2HgLwBQ4zHBr3pWQz89AGxzZZG3rsvW", + "fH96utnEJYRaySOEesQcwkvJ0Z+l8bIB9/WcCEFji1KKjk6PbbgulUjkrI9ep1QhxdElIVmZ0QJZhX09", + "PwcEslxQvoL40e0QpsQi45SptaMoX72bwXy6VRn6e+aTFs/7mzu8tTscLPuPj50Bl4GcDTOB1Zqpwmpt", + "nVHKJlykRi7DY57r1jUP0suk99MgFUxoQuRCKpKaqMRJnsBxg9oQtv6v/c7schdicvXJMelyGREplZJy", + "JofM5opkROi+9ee6fS/AKugQULjgr2eGSX4dwXt6MCZeDaumVQPIJqgr2jnobOEs24qxwg0BYnZ4nzGk", + "nyAaD8lFOuYJjVBC2aVEGwm9NOoJmkuU6D82V4bzjeC7L13d+PYnS6/0CZvwYO04Q7MFMf+psrosW3OO", + "yUfH1l4S/7A4/gMbHWZr6+snC4KTHtQjdsA9KFc0oR8Nq9ONUKloZFKOcLF2708LptofslOihH4HQ2pb", + "khhEA9AutzLBo61hPhjsRhkF9LddAoMDhtf8OIUej87emTRUknKx6A6Z/gc0/PbwzHh3J9haE7yB2sLJ", + "6GTr9ZoA53NYpn/jCEEzwZXoBcEN/+YSvDnGSOMZkg1HlGerVCWe/elDWK0E982u8DjtCgDyVMxmowD2", + "cmhcYRvCnCd5qv9h/jhZh2umcDR7D69+NdKuGc7abtwEH8WhtHOKialt+SBOD7NgjzVmVS+cmwIIMZVo", + "wOAtcKj+jNT95c33/jp+he5Ou6KubuxXc7bu++azY3AIG/56PJZjbijNzUTx1danK0ybrU8/Jjy6lBaK", + "xTcbar0N8NX1jyUetnURgpgAmaHIQhgZoCwiu0NWM0AaxB+JMFJEpJThZAvmbBoBZG9nxcJzTiFBO4I8", + "lZ6kMWAmJQDfDfB3ejZgqHINeB5daStr+e/4zkjF0ZhEPCUO7XwzpLr9DVP1ExdV6PKvhS++9dYfIAEx", + "BXv7GrT25h4/C739FF9DqHScW4eyG9HGS17+aExBXQR7M+zsDuSw00XDzk467OgdOMJgQsUK7aOUslwR", + "2UfHxr4FKbhPBkiSiLNYOtB1Z8HbHcimhFxDlg3ZnU/gu/sUeyxVwVK+sZ2E2IN+D+nvIWkHbfgHzp7J", + "uAuHLkY8V8bcb8+VfSsmCswjm/fuq/XOyDfdvg0n/5s9vhUeBbus2aW39YazZ7mckWaT2ytTyChXYwDz", + "dsVF5Qz9nY9lFzFyZazhQqr+Et/TX5+ZDu6j0IDu6iZFBuzcv1UYaFFhoFyrMFijCbDUV7KjDoPYSK4z", + "LhSgONpce0NDoEkAcgSPcIJeH50MWaRZkYEWFCTlwJ0sHrq5hQ//do5eHL3pomModIl+zsebffSaJQtX", + "btz4aIbMSGKGeUWYobGhWhKHrmczdqCeuwwW1x08UOVoczICnhW3Vy5IvNuZERyDRPJH5xU3nQVQh9+8", + "0gcIgH/Nl8W2d1YKH503RIlF73CiiFhu9tTmSbECM8Ne0g6CzgpuBvhSdygd8lrZp5ENDDTG7k4ngJTx", + "6VvRh7svkHo/XjITJ2LK7Y1zQBplkGSA48XjimWSM1QwxxAL9K/romxCU5aw5WUrFQzosiny+ysyua/k", + "XRVs+X/X0wUzfbSOpqyyT5qIi3Iraz29Ljl4ZuCQraMqwhmOqFp0EU4Se0fZm6CISOkV4u9YEHwZ8yvW", + "H7I3RaEXm9CLjs7edZ2jFsVUXpoWrC+2j17PiZD5uBgcgoNmvMaw5iQeMsVRhJMoT7S4QSYTEkEuLtRv", + "kQ2+3GIonTs8O2UnwWIzXlR7/uhq3IVpAnavJIs6xW2Zrd4SJEowTZvBx62gBgGHEGow1o1yhiibJDak", + "KhJcSmSb6pGETuk4sQFCso/ezgiSOCVDliWYMSJQLk1UvB56LxNEytwkeOsGAKTXUFQXlcCCmeDKhiYk", + "nAtpogk0hb8/RVKRbAWZvTEtn8Kc70i2NY3bnh7ISF0bQ7MpxL6C9IYYSjELrukoT1wA472GopsBPbSU", + "+FgO/ltBp1Mi9KnAhsmacDxzrN1ymkNfyVhurHd5XrzVrt5l0aqXlehl7K0EhhuVWNtx52ZRf4HOL2kj", + "dqB9dLMs4l/0Ry37rmarhgdhH33mLEOlO/8dq2See0mCbQ1YJYU/NnOSN/LKUa0k2q6H1WqdWXuXma6t", + "8bMeDDbrMaNl4Ur6bJPC+/URwuB+UR7uu8ja46atCtpVRTdtSPlfj6b/VVDg3cDoPzDKyS1g9L+qvHvA", + "OX84/JPgQX2oPPqK79kV2/3TI+HfVfq8gcMHOLam9HnD9Wzw6kpF6b19p52aZFv8M0nwNt7xBvK7W/Zv", + "Wn8LlcFbrHUuaE3wJM3UwgW0WV9lGXQm6UfSb3AEF3Grd+cKvkVI55cjD0enjQGdf87a+A8SM2pLB1KJ", + "To4DRecfGcagf+YqF8uWvnV6WEQzOifNRvfqCbZLlAnSy3gGzpXYLJhdD3eXKSz604/INm8xV+2/oPYk", + "QPWTGMVUkEglC1MHVHME08d3EgmuNQF4zsWiOUrEHJGfBE8P7WzW3If2TFljWBlnmC56MVa4N3fcZoUJ", + "7TOiO108pWZ4iDL08ke0Qa6VMBUu0ERrPohOiiUl1xEhsQSa3PQHvD1osGzSj2Q0HbcZ5YpaJa9tLRgU", + "5VLx1O39yTHagNpnU8L0XmhRfwKSbCb4nMYkroyxM+eJWdXthgW9qd1VCxVF4TqnXJjBPYgM0+ZCmn6k", + "WZUtFCExY8owDG5tVZDqmTJJ/Lo/TJkLwLF75Ebx7Qqzmt+GU3Y0JUIdTruIinMD8bz57Zp7zNecnwzl", + "7rTKbefCc1Ybr9vlR7VMW7qLwg9F7tz9mq3ffz0pPVQ+ymweazqfFwppk9n86yLBwf3dD/dtLn//iFNA", + "XxKnfHumcmhAtxgimFcQ0x2TOUl4lkI9dHi30+3kIukcdGZKZQdbWxD7PeNSHew9f7rb+fTh0/8fAAD/", + "//uqpNOh8AEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/openapi.yaml b/openapi.yaml index fa0e1381f..c33c796d8 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1757,8 +1757,12 @@ components: example: "L40S-1Q" mdev_uuid: type: string - description: mdev device UUID + description: mdev device UUID (mdev hosts only) example: "aa618089-8b16-4d01-a136-25a0f3c73123" + device_path: + type: string + description: sysfs path of the assigned vGPU device + example: "/sys/bus/pci/devices/0000:82:00.4" GPUProfile: type: object From 4fe3ad926e51be92698b9ba4a7bbccd111ccfab0 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:53:21 +0000 Subject: [PATCH 057/107] Surface retained rollback assignments from start as vgpu_cleanup_pending When a later start step failed and rollback could not destroy the freshly created vGPU, cleanupStartVGPU retained the assignment on disk but startInstance returned the original failure untyped, so the API reported a generic error instead of vgpu_cleanup_pending with the retained-assignment guidance. Mirror create's named-return wrap: cleanupStartVGPU reports retention state and start wraps the returned error in VGPUCleanupPendingError. --- lib/instances/start.go | 12 +++++++++++- lib/instances/vgpu.go | 10 +++++++++- lib/instances/vgpu_test.go | 4 ++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/lib/instances/start.go b/lib/instances/start.go index b44a68d50..1045829a1 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -116,6 +116,16 @@ func (m *manager) startInstance( } // Setup cleanup stack for automatic rollback on errors + // Registered before cu.Clean so it runs after cleanup and can report a + // vGPU assignment that rollback failed to destroy, matching create's + // vgpu_cleanup_pending contract. + vgpuRetained := false + vgpuRetentionPersisted := false + defer func() { + if retErr != nil && vgpuRetained { + retErr = &VGPUCleanupPendingError{InstanceID: id, Retained: vgpuRetentionPersisted, Err: retErr} + } + }() cu := cleanup.Make(func() {}) defer cu.Clean() @@ -189,7 +199,7 @@ func (m *manager) startInstance( log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", stored.GPUProfile, "uuid", device.MdevUUID) // Add vGPU cleanup to stack cu.Add(func() { - m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) + vgpuRetained, vgpuRetentionPersisted = m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) }) // Checked after the cleanup handler is registered so rejection // releases the device through the normal rollback. diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 8c6351742..1e3dcc356 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -105,7 +105,12 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { // start. The snapshot is also a shallow copy (Phases shares its map), so it // must be persisted before any Phases.Record on the live struct. Violating // either invariant requires switching to targeted field restores. -func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) { +// +// It reports whether the assignment was retained after a failed destroy and +// whether that retention record was persisted, so start can surface the +// pending cleanup as a typed error like create does. +func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) (retained, persisted bool) { + logger.FromContext(ctx).DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID) assignment := devices.VGPUAssignment{ Framework: device.Framework, DevicePath: device.SysfsPath, @@ -117,6 +122,7 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic if releaseErr != nil { logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "error", releaseErr) setStoredVGPUDevice(&cleanupMeta.StoredMetadata, device, assignedAt) + retained = true } if err := m.saveMetadata(&cleanupMeta); err != nil { message := "failed to save metadata after vGPU cleanup" @@ -124,7 +130,9 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic message = "failed to retain vGPU assignment metadata after cleanup failure" } logger.FromContext(ctx).ErrorContext(ctx, message, "instance_id", instanceID, "error", err) + return retained, false } + return retained, retained } func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index d466f1883..7668ff90c 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -389,6 +389,10 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) _, err = m.startInstance(context.Background(), id, StartInstanceRequest{Entrypoint: []string{"new-entrypoint"}}) require.Error(t, err) + var pending *VGPUCleanupPendingError + require.ErrorAs(t, err, &pending, "a retained rollback assignment must surface as vgpu_cleanup_pending") + assert.Equal(t, id, pending.InstanceID) + assert.True(t, pending.Retained) stored, err := m.loadMetadata(id) require.NoError(t, err) From 01f754b9292f51306620c41eea3b8c404cfa3cfc Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 18:07:37 +0000 Subject: [PATCH 058/107] Report retention as persisted when the mid-start save survives When start rollback fails to destroy a vGPU and the cleanup metadata save also fails, the assignment may still be on disk from the mid-start save. Reporting Retained: false then misdirects callers to wait for startup reconcile when delete or a retried start can already release it. Check whether the surviving record still points at the device, matching create's retention-survives check. --- lib/instances/vgpu.go | 12 +++++++++++- lib/instances/vgpu_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 1e3dcc356..330e2d820 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -130,7 +130,17 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic message = "failed to retain vGPU assignment metadata after cleanup failure" } logger.FromContext(ctx).ErrorContext(ctx, message, "instance_id", instanceID, "error", err) - return retained, false + if !retained { + return false, false + } + // The mid-start save may already have persisted this assignment, in + // which case the on-disk record still points at the device and + // delete or a retried start can release it (matching create's + // retention-survives check). + if meta, loadErr := m.loadMetadata(instanceID); loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath { + return true, true + } + return true, false } return retained, retained } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 7668ff90c..b34fe242a 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -405,6 +405,38 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { assert.Empty(t, stored.Entrypoint) } +func TestCleanupStartVGPUReportsRetainedWhenMidStartSaveSurvives(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return errors.New("destroy failed") + }) + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + assignedAt := time.Now().UTC() + + // The mid-start save already persisted the assignment. + meta, err := m.loadMetadata(id) + require.NoError(t, err) + rollbackMeta := *meta + setStoredVGPUDevice(&meta.StoredMetadata, &device, assignedAt) + require.NoError(t, m.saveMetadata(meta)) + + // The cleanup save fails, but the surviving on-disk record still points + // at the device, so retention must be reported as persisted. + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + retained, persisted := m.cleanupStartVGPU(context.Background(), id, &device, assignedAt, rollbackMeta) + assert.True(t, retained) + assert.True(t, persisted, "a surviving mid-start save keeps the assignment recoverable via delete") +} + func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { m := &manager{ paths: paths.New(t.TempDir()), From 18388464f864ddd0afe5daaf57f0530ebb949b6d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:07:40 +0000 Subject: [PATCH 059/107] Leave vGPU hypervisor selection to callers Drop the vendor-VFIO-on-Cloud-Hypervisor rejection from create and start, restoring the phase-0 decision that hypervisor selection is caller policy: production callers pin vGPU instances to QEMU, and the Cloud Hypervisor limitation stays documented in lib/devices/GPU.md. --- lib/instances/create.go | 6 ------ lib/instances/start.go | 6 ------ lib/instances/vgpu.go | 13 ------------- lib/instances/vgpu_test.go | 29 ----------------------------- 4 files changed, 54 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index 23fd534b1..a6cce101a 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -357,12 +357,6 @@ func (m *manager) createInstance( } } }) - // Checked after the cleanup handler is registered so rejection - // releases the device through the normal rollback. - if err := validateVGPUHypervisorCompat(gpuDevice.Framework, hvType); err != nil { - log.ErrorContext(ctx, "unsupported vGPU hypervisor combination", "instance_id", id, "framework", gpuDevice.Framework, "hypervisor", hvType) - return nil, err - } } if len(req.Devices) > 0 && m.deviceManager != nil { diff --git a/lib/instances/start.go b/lib/instances/start.go index 1045829a1..3e164b398 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -201,12 +201,6 @@ func (m *manager) startInstance( cu.Add(func() { vgpuRetained, vgpuRetentionPersisted = m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) }) - // Checked after the cleanup handler is registered so rejection - // releases the device through the normal rollback. - if err := validateVGPUHypervisorCompat(device.Framework, stored.HypervisorType); err != nil { - log.ErrorContext(ctx, "unsupported vGPU hypervisor combination", "instance_id", id, "framework", device.Framework, "hypervisor", stored.HypervisorType) - return nil, err - } if err := m.saveMetadata(meta); err != nil { log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err) return nil, fmt.Errorf("save metadata after vGPU creation: %w", err) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 330e2d820..88ae217da 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -8,7 +8,6 @@ import ( "time" "github.com/kernel/hypeman/lib/devices" - "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) @@ -64,18 +63,6 @@ func retainedVGPUFromCreateError(instanceID string, assignedAt time.Time, err er } } -// validateVGPUHypervisorCompat rejects the one proven-broken combination: -// vendor VFIO vGPUs boot but are non-functional on Cloud Hypervisor (upstream -// cloud-hypervisor#7572), and the wedged VM then blocks the VF release until -// startup reconcile. Hypervisor selection otherwise remains caller policy; -// mdev on Cloud Hypervisor keeps working. See lib/devices/GPU.md. -func validateVGPUHypervisorCompat(framework devices.VGPUFramework, hvType hypervisor.Type) error { - if framework == devices.VGPUFrameworkVendorVFIO && hvType == hypervisor.TypeCloudHypervisor { - return fmt.Errorf("%w: vendor VFIO vGPUs are not functional on cloud-hypervisor, use qemu", ErrInvalidRequest) - } - return nil -} - func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { destroy := m.destroyVGPU if destroy == nil { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index b34fe242a..1bdb0fb97 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -319,35 +319,6 @@ func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { assert.ErrorIs(t, err, cause) } -func TestValidateVGPUHypervisorCompat(t *testing.T) { - t.Parallel() - - err := validateVGPUHypervisorCompat(devices.VGPUFrameworkVendorVFIO, hypervisor.TypeCloudHypervisor) - require.ErrorIs(t, err, ErrInvalidRequest) - assert.NoError(t, validateVGPUHypervisorCompat(devices.VGPUFrameworkVendorVFIO, hypervisor.TypeQEMU)) - assert.NoError(t, validateVGPUHypervisorCompat(devices.VGPUFrameworkMdev, hypervisor.TypeCloudHypervisor)) -} - -func TestStartRejectsVendorVFIOOnCloudHypervisor(t *testing.T) { - var destroyed []devices.VGPUAssignment - m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { - destroyed = append(destroyed, assignment) - return nil - }) - meta, err := m.loadMetadata(id) - require.NoError(t, err) - meta.HypervisorType = hypervisor.TypeCloudHypervisor - require.NoError(t, m.saveMetadata(meta)) - - _, err = m.startInstance(context.Background(), id, StartInstanceRequest{}) - require.ErrorIs(t, err, ErrInvalidRequest) - - require.Len(t, destroyed, 1, "the rejected vGPU must be released by rollback") - stored, err := m.loadMetadata(id) - require.NoError(t, err) - assert.Empty(t, stored.GPUDevicePath, "no assignment may be persisted for a rejected combination") -} - func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { var destroyed []devices.VGPUAssignment m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { From b79b13e26e55aabef14199915aa5402c0d80bcb6 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:21:39 +0000 Subject: [PATCH 060/107] Carry identity fields into the create-pending retention stub retainedVGPUFromCreateError built a GPU-fields-only stub, so the retained record from a failed device-layer cleanup listed nameless and, with GPUProfile empty, the API hid its gpu block including device_path. The caller now supplies the identity fields and the stub picks up the pending device's profile. --- lib/instances/create.go | 16 +++++++++++++++- lib/instances/vgpu.go | 19 +++++++++++-------- lib/instances/vgpu_test.go | 6 ++++-- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index a6cce101a..07abfd4b6 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -312,7 +312,21 @@ func (m *manager) createInstance( log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) if err != nil { - retainedVGPU = retainedVGPUFromCreateError(id, m.nowUTC(), err) + stub := StoredMetadata{ + Id: id, + Name: req.Name, + Image: req.Image, + ResolvedImage: resolvedImageRef, + Platform: imageInfo.Platform, + CreatedAt: time.Now(), + HypervisorType: hvType, + HypervisorVersion: hvVersion, + DataDir: m.paths.InstanceDir(id), + } + if starterErr == nil { + stub.SocketPath = m.paths.InstanceSocket(id, starter.SocketName()) + } + retainedVGPU = retainedVGPUFromCreateError(stub, m.nowUTC(), err) log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) return nil, wrapCreateVGPUErr(req.GPU.Profile, err) } diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 88ae217da..4caa365bb 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -49,18 +49,21 @@ func vgpuDevicePendingCleanup(err error) (*devices.VGPUDevice, bool) { return &pending.Device, true } -func retainedVGPUFromCreateError(instanceID string, assignedAt time.Time, err error) *StoredMetadata { +// retainedVGPUFromCreateError fills stub with the pending device's assignment +// fields when err carries a failed device-layer cleanup. The caller provides +// identity fields on stub so the retained record lists as a recognizable, +// deletable instance. +func retainedVGPUFromCreateError(stub StoredMetadata, assignedAt time.Time, err error) *StoredMetadata { device, ok := vgpuDevicePendingCleanup(err) if !ok { return nil } - return &StoredMetadata{ - Id: instanceID, - GPUFramework: device.Framework, - GPUDevicePath: device.SysfsPath, - GPUMdevUUID: device.MdevUUID, - GPUAssignedAt: &assignedAt, - } + stub.GPUProfile = device.ProfileName + stub.GPUFramework = device.Framework + stub.GPUDevicePath = device.SysfsPath + stub.GPUMdevUUID = device.MdevUUID + stub.GPUAssignedAt = &assignedAt + return &stub } func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 1bdb0fb97..e5694ef3f 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -143,9 +143,11 @@ func TestVGPUDevicePendingCleanup(t *testing.T) { assert.Equal(t, device, *actual) assignedAt := time.Now().UTC() - retained := retainedVGPUFromCreateError("inst-1", assignedAt, wrapped) + retained := retainedVGPUFromCreateError(StoredMetadata{Id: "inst-1", Name: "named", Image: "img"}, assignedAt, wrapped) require.NotNil(t, retained) assert.Equal(t, "inst-1", retained.Id) + assert.Equal(t, "named", retained.Name, "identity fields must survive into the retention stub") + assert.Equal(t, "img", retained.Image) assert.Equal(t, device.Framework, retained.GPUFramework) assert.Equal(t, device.SysfsPath, retained.GPUDevicePath) assert.Equal(t, assignedAt, *retained.GPUAssignedAt) @@ -153,7 +155,7 @@ func TestVGPUDevicePendingCleanup(t *testing.T) { actual, ok = vgpuDevicePendingCleanup(cause) assert.False(t, ok) assert.Nil(t, actual) - assert.Nil(t, retainedVGPUFromCreateError("inst-1", assignedAt, cause)) + assert.Nil(t, retainedVGPUFromCreateError(StoredMetadata{Id: "inst-1"}, assignedAt, cause)) } type startRetentionNetworkManager struct { From 84f821a671de69d2d2f1083683469617678cae3c Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:39:32 +0000 Subject: [PATCH 061/107] Grace recent dead-PID claims in the release scan like reconcile does Startup reconcile protects an assignment whose PID is absent or stale for a bounded grace window, but the release-side claim scan treated a dead PID as unclaimed immediately. Align the two guards: a recent assignment whose recorded hypervisor is not running fails the scan closed so the requester retains and retries, and past the grace window the dead claim no longer blocks the release. --- lib/instances/vgpu.go | 8 +++++++- lib/instances/vgpu_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 4caa365bb..2f9cfa7d1 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -110,7 +110,7 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic cleanupMeta := rollbackMeta releaseErr := m.destroyVGPUAssignment(ctx, assignment) if releaseErr != nil { - logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "error", releaseErr) + logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID, "error", releaseErr) setStoredVGPUDevice(&cleanupMeta.StoredMetadata, device, assignedAt) retained = true } @@ -208,6 +208,12 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu if pid > 0 { return true, nil } + // A dead PID with a recent assignment gets the same bounded grace as + // startup reconcile protection, so the two guards agree in the + // fail-closed direction while a mid-boot claimant hydrates. + if stored.GPUAssignedAt != nil && time.Since(*stored.GPUAssignedAt) < VGPUAssignmentStartupGracePeriod { + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: recorded hypervisor is not running", id, devicePath) + } } return false, nil } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index e5694ef3f..d5e988a43 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -528,6 +528,38 @@ func TestVGPUAssignmentClaimedByLiveInstanceIgnoresStaleNilPIDClaim(t *testing.T assert.False(t, claimed) } +func TestVGPUAssignmentClaimedByLiveInstanceGracesRecentDeadPIDClaim(t *testing.T) { + m := &manager{paths: paths.New(t.TempDir())} + claimantID := "claimant-dead-pid" + require.NoError(t, m.ensureDirectories(claimantID)) + deadPID := 1<<22 - 1 + require.False(t, ProcessExists(deadPID)) + assignedAt := time.Now().UTC() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: claimantID, + HypervisorPID: &deadPID, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + }})) + + // Same bounded grace as startup reconcile: a recent claim whose PID is + // dead fails closed instead of being treated as unclaimed. + _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", "/sys/bus/pci/devices/0000:82:00.4") + require.Error(t, err) + + // Past the grace period the dead claim no longer blocks the release. + stale := assignedAt.Add(-2 * VGPUAssignmentStartupGracePeriod) + meta, err := m.loadMetadata(claimantID) + require.NoError(t, err) + meta.GPUAssignedAt = &stale + require.NoError(t, m.saveMetadata(meta)) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", "/sys/bus/pci/devices/0000:82:00.4") + require.NoError(t, err) + assert.False(t, claimed) +} + func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) { t.Parallel() From af8d527a6399512681d68638d7136697220d6e0d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 13 Aug 2026 18:57:32 +0000 Subject: [PATCH 062/107] Reject start on vGPU retention records A failed create whose vGPU release also failed persists a delete-only retention stub with no boot configuration. The stub derives as Stopped, so start would release the retained VF and then try to boot the incomplete record. Mark the stub with GPURetainedForCleanup and reject start with invalid_state guidance pointing at delete, which retries the release. --- lib/instances/create.go | 31 ++++++++++++++-------------- lib/instances/lifecycle_noop_test.go | 20 ++++++++++++++++++ lib/instances/start.go | 7 +++++++ lib/instances/types.go | 4 ++++ lib/instances/vgpu_test.go | 2 ++ 5 files changed, 49 insertions(+), 15 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index 07abfd4b6..e7ab3b6ad 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -687,21 +687,22 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG // deletable record rather than a nameless phantom, but drop resource // claims (network, volumes, devices) that rollback already released. retained := StoredMetadata{ - Id: id, - Name: retainedVGPU.Name, - Image: retainedVGPU.Image, - ResolvedImage: retainedVGPU.ResolvedImage, - Platform: retainedVGPU.Platform, - CreatedAt: retainedVGPU.CreatedAt, - HypervisorType: retainedVGPU.HypervisorType, - HypervisorVersion: retainedVGPU.HypervisorVersion, - SocketPath: retainedVGPU.SocketPath, - DataDir: retainedVGPU.DataDir, - GPUProfile: retainedVGPU.GPUProfile, - GPUFramework: retainedVGPU.GPUFramework, - GPUDevicePath: retainedVGPU.GPUDevicePath, - GPUMdevUUID: retainedVGPU.GPUMdevUUID, - GPUAssignedAt: retainedVGPU.GPUAssignedAt, + Id: id, + Name: retainedVGPU.Name, + Image: retainedVGPU.Image, + ResolvedImage: retainedVGPU.ResolvedImage, + Platform: retainedVGPU.Platform, + CreatedAt: retainedVGPU.CreatedAt, + HypervisorType: retainedVGPU.HypervisorType, + HypervisorVersion: retainedVGPU.HypervisorVersion, + SocketPath: retainedVGPU.SocketPath, + DataDir: retainedVGPU.DataDir, + GPUProfile: retainedVGPU.GPUProfile, + GPUFramework: retainedVGPU.GPUFramework, + GPUDevicePath: retainedVGPU.GPUDevicePath, + GPUMdevUUID: retainedVGPU.GPUMdevUUID, + GPUAssignedAt: retainedVGPU.GPUAssignedAt, + GPURetainedForCleanup: true, } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index c74d352fd..e440e7e4a 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -324,6 +324,26 @@ func TestStartPersistsStaleVGPUReleaseImmediately(t *testing.T) { assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile, "profile is kept for the next start") } +func TestStartRejectsVGPURetentionRecord(t *testing.T) { + m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFrameworkNone + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, m.saveMetadata(meta)) + + _, err = m.StartInstance(context.Background(), id, StartInstanceRequest{}) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") + + // The retained assignment must survive the rejected start for delete. + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) +} + func TestStopStoppedInstanceReleasesRetainedVGPU(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) meta, err := m.loadMetadata(id) diff --git a/lib/instances/start.go b/lib/instances/start.go index 3e164b398..8914f2ae1 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -47,6 +47,13 @@ func (m *manager) startInstance( log.ErrorContext(ctx, "invalid state for start", "instance_id", id, "state", inst.State) return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } + if stored.GPURetainedForCleanup { + // A delete-only retention stub from a failed create: it carries no + // boot configuration, so starting it would release the retained VF + // and then boot an incomplete record. Delete retries the release. + log.ErrorContext(ctx, "refusing to start vGPU retention record", "instance_id", id) + return nil, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + } // Release any assignment retained by an earlier failed release and // persist the cleared fields immediately, so a failure later in start diff --git a/lib/instances/types.go b/lib/instances/types.go index a264498fd..ab41178cd 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -156,6 +156,10 @@ type StoredMetadata struct { GPUDevicePath string GPUMdevUUID string // populated for mdev-backed vGPUs GPUAssignedAt *time.Time // set before hypervisor startup to bound crash recovery protection + // GPURetainedForCleanup marks a delete-only retention stub written when a + // failed create could not release its vGPU: the record has no boot + // configuration, so only delete (which retries the release) may act on it. + GPURetainedForCleanup bool // Command overrides (like docker run ) Entrypoint []string // Override image entrypoint (nil = use image default) diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index d5e988a43..a6fe4e49a 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -57,6 +57,8 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { assert.False(t, retained.NetworkEnabled) assert.Empty(t, retained.IP) assert.Empty(t, retained.Volumes) + // The stub has no boot configuration, so it is marked delete-only. + assert.True(t, retained.GPURetainedForCleanup) } func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { From ca6f98d2aadf6915053681ceb55e6369a5ad7c7f Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 13 Aug 2026 19:05:38 +0000 Subject: [PATCH 063/107] Make vGPU retention records fully delete-only A fork or snapshot of a failed-create retention stub could never boot: the stub has no boot configuration, and clearing the delete-only marker on the child would only produce a startable-but-broken record that recreates a vGPU from GPUProfile with incomplete metadata. Reject fork and snapshot of retention stubs with the same invalid_state guidance as start, so delete (which retries the release) is the only action on them. --- lib/instances/fork.go | 6 ++++++ lib/instances/fork_test.go | 25 +++++++++++++++++++++++++ lib/instances/snapshot.go | 7 +++++++ lib/instances/snapshot_test.go | 25 +++++++++++++++++++++++++ 4 files changed, 63 insertions(+) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index e0c778860..266299e96 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -219,6 +219,12 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin default: return nil, false, fmt.Errorf("%w: cannot fork from state %s (must be Stopped or Standby)", ErrInvalidState, source.State) } + if stored.GPURetainedForCleanup { + // A delete-only retention stub from a failed create has no boot + // configuration, so a fork of it could never boot. Delete the stub to + // release its retained vGPU assignment. + return nil, false, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + } if !supportValidated { if err := m.validateForkSupport(ctx, stored.HypervisorType); err != nil { diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index 26bc6cfcb..ef802d1ad 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -63,6 +63,31 @@ func TestForkInstanceClearsVGPUAssignment(t *testing.T) { assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", source.GPUDevicePath) } +func TestForkInstanceRejectsVGPURetentionRecord(t *testing.T) { + manager, _ := setupTestManager(t) + ctx := context.Background() + hvType := hypervisor.Type("fork-vgpu-retention-test") + hypervisor.RegisterCapabilities(hvType, hypervisor.Capabilities{SupportsConcurrentForkPrepare: true}) + manager.vmStarters[hvType] = concurrentForkPrepareTestStarter{} + + sourceID := "fork-vgpu-retention-source" + createStoppedSnapshotSourceFixture(t, manager, sourceID, sourceID, hvType) + + meta, err := manager.loadMetadata(sourceID) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, manager.saveMetadata(meta)) + + // The delete-only retention stub has no boot configuration, so a fork of + // it could never boot; only delete may act on it. + _, err = manager.ForkInstance(ctx, sourceID, ForkInstanceRequest{Name: "fork-vgpu-retention-copy"}) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") +} + func TestForkInstance_VZStoppedSourceSupported(t *testing.T) { t.Parallel() manager, _ := setupTestManager(t) diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 48c51328a..297c7ef03 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -66,6 +66,13 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata + if stored.GPURetainedForCleanup { + // A delete-only retention stub from a failed create has no boot + // configuration, so a snapshot of it could never be restored or + // forked into a bootable instance. Delete the stub to release its + // retained vGPU assignment. + return nil, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + } if err := validateForkVolumeSafety(stored.Volumes); err != nil { return nil, fmt.Errorf("%w: snapshot requires readonly volume attachments: %v", ErrNotSupported, err) } diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index a92763b28..f8c022fe9 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -52,6 +52,31 @@ func TestForkSnapshotClearsVGPUAssignment(t *testing.T) { assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", source.GPUDevicePath) } +func TestCreateSnapshotRejectsVGPURetentionRecord(t *testing.T) { + mgr, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "snapshot-vgpu-retention" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, sourceID, mgr.defaultHypervisor) + + meta, err := mgr.loadMetadata(sourceID) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, mgr.saveMetadata(meta)) + + // The delete-only retention stub has no boot configuration, so a snapshot + // of it could never be restored or forked into a bootable instance. + _, err = mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Name: "snapshot-vgpu-retention", + }) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") +} + func TestRestoreSnapshotDoesNotResurrectStaleVGPUAssignment(t *testing.T) { mgr, _ := setupTestManager(t) ctx := context.Background() From ab20441d7b3bb0fc65c7f6e24e130d5019fe5d3f Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 14 Aug 2026 20:56:18 +0000 Subject: [PATCH 064/107] Adapt vGPU liveness guards to the identity struct resolver The claim guard and startup reconcile protection predate the HypervisorProcessIdentity struct and the removal of the standalone identity-exists helpers. Route both through resolveLiveHypervisorPID: the claim guard keeps failing closed on unresolvable ownership, and reconcile protection gets a fail-open HypervisorMayBeAlive wrapper so unresolvable ownership still protects the device. --- cmd/api/main.go | 2 +- cmd/api/main_test.go | 4 +-- lib/instances/lifecycle_noop_test.go | 22 ++++++++--------- lib/instances/process_identity.go | 10 ++++++++ lib/instances/process_identity_linux_test.go | 12 ++++----- lib/instances/vgpu.go | 2 +- lib/instances/vgpu_test.go | 26 ++++++++++---------- 7 files changed, 44 insertions(+), 34 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index f38a6a7c3..71e147ecf 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -196,7 +196,7 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances. if inst.GPUDevicePath == "" { continue } - if inst.HypervisorPID != nil && instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.HypervisorBootID, inst.SocketPath) { + if inst.HypervisorPID != nil && instances.HypervisorMayBeAlive(inst.HypervisorProcessIdentity, inst.SocketPath) { protected[inst.GPUDevicePath] = struct{}{} continue } diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 217e9db8d..85c132814 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -362,8 +362,8 @@ func TestLiveInstanceVGPUDevicePathsBoundsStartupProtection(t *testing.T) { {StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}}, {StoredMetadata: instances.StoredMetadata{Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}}, {StoredMetadata: instances.StoredMetadata{Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}}, - {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorPID: &deadPID}}, - {StoredMetadata: instances.StoredMetadata{Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorPID: &deadPID, GPUAssignedAt: &recent}}, + {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorProcessIdentity: instances.HypervisorProcessIdentity{HypervisorPID: &deadPID}}}, + {StoredMetadata: instances.StoredMetadata{Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorProcessIdentity: instances.HypervisorProcessIdentity{HypervisorPID: &deadPID}, GPUAssignedAt: &recent}}, }} protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager) diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index e440e7e4a..f5f495988 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -258,17 +258,17 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { require.NoError(t, err) defer listener.Close() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: claimantID, - Name: claimantID, - Image: "test-image", - CreatedAt: now, - HypervisorType: lifecycleNoopHypervisorType, - HypervisorPID: &pid, - SocketPath: socketPath, - DataDir: m.paths.InstanceDir(claimantID), - GPUProfile: "NVIDIA L40S-2Q", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + Id: claimantID, + Name: claimantID, + Image: "test-image", + CreatedAt: now, + HypervisorType: lifecycleNoopHypervisorType, + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, + SocketPath: socketPath, + DataDir: m.paths.InstanceDir(claimantID), + GPUProfile: "NVIDIA L40S-2Q", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", }})) require.NoError(t, m.DeleteInstance(context.Background(), id)) diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 1d885d0b5..be645e111 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -180,6 +180,16 @@ func classifyResolvedHypervisorOwner(socketPath string, stored, resolved int, er return 0, fmt.Errorf("cannot confirm ownership of socket %s: %w", socketPath, err) } +// HypervisorMayBeAlive reports whether the recorded hypervisor process may +// still be running. It fails open: when ownership cannot be resolved it +// returns true, which is the safe direction for its callers (reconcile +// protection and claim checks, where true means "protect"). Do not use it to +// authorize teardown. +func HypervisorMayBeAlive(id HypervisorProcessIdentity, socketPath string) bool { + pid, err := resolveLiveHypervisorPID(id, socketPath) + return err != nil || pid > 0 +} + // ProcessExists reports whether pid belongs to a live, non-zombie process. func ProcessExists(pid int) bool { if pid <= 0 { diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index d5bd973d9..7ad259c41 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -512,7 +512,7 @@ func TestRefreshHypervisorPIDResolvesSocketOwnerWhenStoredPIDIsDead(t *testing.T func TestVGPUAssignmentClaimedByLiveInstanceProtectsReusedPIDClaim(t *testing.T) { socketPath := filepath.Join(t.TempDir(), "test.sock") - owner := exec.Command(os.Args[0], "-test.run=^TestHypervisorProcessExistsWithReboundSocketPathHelper$") + owner := exec.Command(os.Args[0], "-test.run=^TestSocketListenerHelper$") owner.Env = append(os.Environ(), "HYPERVISOR_SOCKET_HELPER=1", "HYPERVISOR_SOCKET_PATH="+socketPath) stdin, err := owner.StdinPipe() require.NoError(t, err) @@ -539,11 +539,11 @@ func TestVGPUAssignmentClaimedByLiveInstanceProtectsReusedPIDClaim(t *testing.T) stalePID := stale.Process.Pid require.NoError(t, m.ensureDirectories("live-claimant")) require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "live-claimant", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: devicePath, - HypervisorPID: &stalePID, - SocketPath: socketPath, + Id: "live-claimant", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &stalePID}, + SocketPath: socketPath, }})) claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", devicePath) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 2f9cfa7d1..54a0bf9c6 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -201,7 +201,7 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu } return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) } - pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.HypervisorBootID, stored.SocketPath) + pid, err := resolveLiveHypervisorPID(stored.HypervisorProcessIdentity, stored.SocketPath) if err != nil { return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index a6fe4e49a..1d0e4e3bf 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -483,10 +483,10 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. require.NoError(t, m.ensureDirectories("legacy-claimant")) pid := os.Getpid() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "legacy-claimant", - Name: "legacy-claimant", - GPUMdevUUID: "legacy-uuid", - HypervisorPID: &pid, + Id: "legacy-claimant", + Name: "legacy-claimant", + GPUMdevUUID: "legacy-uuid", + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, }})) claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/mdev/devices/legacy-uuid") @@ -538,11 +538,11 @@ func TestVGPUAssignmentClaimedByLiveInstanceGracesRecentDeadPIDClaim(t *testing. require.False(t, ProcessExists(deadPID)) assignedAt := time.Now().UTC() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: claimantID, - HypervisorPID: &deadPID, - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - GPUAssignedAt: &assignedAt, + Id: claimantID, + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, }})) // Same bounded grace as startup reconcile: a recent claim whose PID is @@ -569,10 +569,10 @@ func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) { require.NoError(t, m.ensureDirectories("dead-claimant")) deadPID := 1 << 30 require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "dead-claimant", - Name: "dead-claimant", - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - HypervisorPID: &deadPID, + Id: "dead-claimant", + Name: "dead-claimant", + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, }})) claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") From eb52f8c2d48af8afb0fb8304b8f2632d5917e8e8 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 17 Aug 2026 21:10:56 +0000 Subject: [PATCH 065/107] Deep-copy the phase tracker into the start rollback snapshot --- lib/instances/start.go | 1 + lib/instances/vgpu.go | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/instances/start.go b/lib/instances/start.go index 8914f2ae1..44dc50216 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -75,6 +75,7 @@ func (m *manager) startInstance( stored.HypervisorStartTime = 0 stored.HypervisorBootID = "" rollbackMeta := *meta + rollbackMeta.Phases = meta.Phases.Clone() // 2a. Clear stale exit info from previous run and apply command overrides stored.ExitCode = nil diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 54a0bf9c6..d0cd9b226 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -92,9 +92,7 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { // cleanup stack is LIFO, so cleanups registered after this one run before it // and this restore would clobber anything they persisted; it is safe only // while no such cleanup writes metadata and the instance lock serializes -// start. The snapshot is also a shallow copy (Phases shares its map), so it -// must be persisted before any Phases.Record on the live struct. Violating -// either invariant requires switching to targeted field restores. +// start. Violating that requires switching to targeted field restores. // // It reports whether the assignment was retained after a failed destroy and // whether that retention record was persisted, so start can surface the From 3f4c157949b0eaac44e53585e82416e446cbe851 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 18 Aug 2026 18:59:15 +0000 Subject: [PATCH 066/107] Retry orphaned vGPU releases in the background after delete MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A vGPU release during delete routinely fails when a GPU-busy VMM's kernel-side VFIO teardown outlives the force-kill wait. Delete's log-and-continue contract then deleted the metadata, stranding the VF until the next server restart and silently shrinking host GPU capacity. Hand the failed assignment to a bounded background retry (30s interval, 20 attempts) that re-runs the full release path — claim scan and destroy guards included — off the request path. The in-memory queue dedupes by device path; a restart abandons it and startup reconciliation sweeps the VF as before. --- lib/devices/GPU.md | 2 +- lib/instances/delete.go | 9 +- lib/instances/lifecycle_noop_test.go | 4 +- lib/instances/manager.go | 8 ++ lib/instances/vgpu_orphan.go | 78 ++++++++++++++ lib/instances/vgpu_orphan_test.go | 146 +++++++++++++++++++++++++++ 6 files changed, 241 insertions(+), 6 deletions(-) create mode 100644 lib/instances/vgpu_orphan.go create mode 100644 lib/instances/vgpu_orphan_test.go diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 3a1504428..ea5764ee0 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -97,7 +97,7 @@ Instance Create → Assign profile to VF → Attach VF to VM → Instance Runnin Instance Stop/Delete → Release profile → VF available again ``` -Hypeman reconciles orphaned assignments on server restart while preserving devices held open by a running VMM. +Hypeman reconciles orphaned assignments on server restart while preserving devices held open by a running VMM. A release that fails during delete (typically because a GPU-busy VMM's kernel-side VFIO teardown outlives the force-kill wait) is retried in the background for up to ten minutes, so a completed delete does not strand the VF until the next restart. ### Hypervisor Support diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 08781b977..f7646d64d 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -144,15 +144,18 @@ func (m *manager) deleteInstanceWithOptions( // or volume teardown. Release failure is logged and the delete continues, // matching the pre-refactor contract: the VMM is already confirmed dead, // the guards inside the release never destroy a device they cannot prove - // is unowned, and a skipped release is recovered by startup - // reconciliation. + // is unowned, and a skipped release is recovered by the background retry + // below or, after a restart, by startup reconciliation. hadVGPUAssignment := storedVGPUDevicePath(stored) != "" if hadVGPUAssignment { log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID) } if err := m.releaseStoredVGPU(ctx, stored); err != nil { - // Log error but continue with cleanup. + // Log error but continue with cleanup. The metadata is about to be + // deleted, so hand the assignment to the background retry — otherwise + // the VF stays allocated until the next startup reconciliation. log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) + m.scheduleOrphanedVGPURelease(ctx, *stored) } else if hadVGPUAssignment { if err := m.saveMetadata(meta); err != nil { log.WarnContext(ctx, "failed to save metadata after vGPU release", "instance_id", id, "error", err) diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index f5f495988..7868534a3 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -160,8 +160,8 @@ func TestDeleteContinuesWhenVGPUReleaseFails(t *testing.T) { require.NoError(t, m.saveMetadata(meta)) // A failed release is logged and the delete continues, matching the - // pre-refactor contract; the leaked assignment is recovered by startup - // reconciliation. + // pre-refactor contract; the leaked assignment is recovered by the + // background retry or startup reconciliation. require.NoError(t, m.DeleteInstance(context.Background(), id)) _, err = m.loadMetadata(id) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 1bb1e53ec..985eb29d7 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -213,6 +213,14 @@ type manager struct { // Periodic TAP garbage collection reconciler. tapGCOnce sync.Once + // vGPU assignments that survived a completed delete, keyed by device + // path, each with a background release retry in flight. + // orphanedVGPURetryDelay overrides the retry delay in tests; zero means + // the default. + orphanedVGPUMu sync.Mutex + orphanedVGPUs map[string]struct{} + orphanedVGPURetryDelay time.Duration + // Hypervisor support vmStarters map[hypervisor.Type]hypervisor.VMStarter defaultHypervisor hypervisor.Type // Default hypervisor type when not specified in request diff --git a/lib/instances/vgpu_orphan.go b/lib/instances/vgpu_orphan.go new file mode 100644 index 000000000..69a3545c5 --- /dev/null +++ b/lib/instances/vgpu_orphan.go @@ -0,0 +1,78 @@ +package instances + +import ( + "context" + "time" + + "github.com/kernel/hypeman/lib/logger" +) + +const ( + // orphanedVGPUReleaseMaxAttempts bounds the retry loop so a genuinely + // wedged VF degrades to one operator-actionable error instead of + // indefinite log churn. At the default interval this covers ten minutes, + // far beyond the seconds a dying VMM normally needs to finish kernel-side + // VFIO teardown. + orphanedVGPUReleaseMaxAttempts = 20 + defaultOrphanedVGPUReleaseRetryDelay = 30 * time.Second +) + +// scheduleOrphanedVGPURelease retries a vGPU release that failed during a +// completed delete, off the request path. Delete's log-and-continue contract +// is untouched — the caller already has its success — but without a retry the +// VF would stay allocated until the next startup reconciliation, silently +// shrinking host GPU capacity. A GPU-busy VMM routinely outlives delete's +// force-kill wait while the kernel finishes VFIO teardown, so this is the +// common case under load, not a tail case. +// +// Each attempt re-runs releaseStoredVGPU on a copy of the deleted instance's +// stored metadata, so the vendor VFIO claim scan and the destroy-side owner +// and open-handle guards apply on every retry exactly as they did on the +// original release. The queue is in-memory only: a restart abandons it and +// startup reconciliation sweeps the VF instead. +func (m *manager) scheduleOrphanedVGPURelease(ctx context.Context, stored StoredMetadata) { + path := storedVGPUDevicePath(&stored) + if path == "" { + return + } + m.orphanedVGPUMu.Lock() + if m.orphanedVGPUs == nil { + m.orphanedVGPUs = make(map[string]struct{}) + } + if _, pending := m.orphanedVGPUs[path]; pending { + m.orphanedVGPUMu.Unlock() + return + } + m.orphanedVGPUs[path] = struct{}{} + m.orphanedVGPUMu.Unlock() + + delay := m.orphanedVGPURetryDelay + if delay <= 0 { + delay = defaultOrphanedVGPUReleaseRetryDelay + } + // The request context ends with the delete; keep its values for logging + // but detach from its cancellation. + go m.retryOrphanedVGPURelease(context.WithoutCancel(ctx), stored, path, delay) +} + +func (m *manager) retryOrphanedVGPURelease(ctx context.Context, stored StoredMetadata, path string, delay time.Duration) { + log := logger.FromContext(ctx) + defer func() { + m.orphanedVGPUMu.Lock() + delete(m.orphanedVGPUs, path) + m.orphanedVGPUMu.Unlock() + }() + for attempt := 1; attempt <= orphanedVGPUReleaseMaxAttempts; attempt++ { + time.Sleep(delay) + if err := m.releaseStoredVGPU(ctx, &stored); err != nil { + log.WarnContext(ctx, "orphaned vGPU release retry failed", + "instance_id", stored.Id, "device_path", path, "attempt", attempt, "error", err) + continue + } + log.InfoContext(ctx, "released orphaned vGPU after delete", + "instance_id", stored.Id, "device_path", path, "attempt", attempt) + return + } + log.ErrorContext(ctx, "giving up on orphaned vGPU release; VF stays allocated until startup reconciliation or manual remediation", + "instance_id", stored.Id, "device_path", path, "attempts", orphanedVGPUReleaseMaxAttempts) +} diff --git a/lib/instances/vgpu_orphan_test.go b/lib/instances/vgpu_orphan_test.go new file mode 100644 index 000000000..25a350064 --- /dev/null +++ b/lib/instances/vgpu_orphan_test.go @@ -0,0 +1,146 @@ +package instances + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func waitForOrphanQueueEmpty(t *testing.T, m *manager) { + t.Helper() + require.Eventually(t, func() bool { + m.orphanedVGPUMu.Lock() + defer m.orphanedVGPUMu.Unlock() + return len(m.orphanedVGPUs) == 0 + }, 5*time.Second, 5*time.Millisecond, "orphan retry should finish and clear its queue entry") +} + +func TestScheduleOrphanedVGPUReleaseRetriesUntilSuccess(t *testing.T) { + t.Parallel() + + var attempts atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + orphanedVGPURetryDelay: time.Millisecond, + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + if attempts.Add(1) < 3 { + return errors.New("operation not permitted") + } + return nil + }, + } + m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{ + Id: "deleted-instance", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }) + + waitForOrphanQueueEmpty(t, m) + assert.Equal(t, int32(3), attempts.Load(), "release should succeed on the third attempt and stop retrying") +} + +func TestScheduleOrphanedVGPUReleaseGivesUpAfterMaxAttempts(t *testing.T) { + t.Parallel() + + var attempts atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + orphanedVGPURetryDelay: time.Millisecond, + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + attempts.Add(1) + return errors.New("vGPU destroy failed: 0xffffffff") + }, + } + m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{ + Id: "deleted-instance", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }) + + waitForOrphanQueueEmpty(t, m) + assert.Equal(t, int32(orphanedVGPUReleaseMaxAttempts), attempts.Load(), + "a wedged VF should get exactly the bounded number of attempts") +} + +func TestScheduleOrphanedVGPUReleaseDeduplicatesByDevicePath(t *testing.T) { + t.Parallel() + + var attempts atomic.Int32 + release := make(chan struct{}) + m := &manager{ + paths: paths.New(t.TempDir()), + orphanedVGPURetryDelay: 20 * time.Millisecond, + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + attempts.Add(1) + <-release + return nil + }, + } + stored := StoredMetadata{ + Id: "deleted-instance", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + } + m.scheduleOrphanedVGPURelease(context.Background(), stored) + m.scheduleOrphanedVGPURelease(context.Background(), stored) + + require.Eventually(t, func() bool { return attempts.Load() == 1 }, 5*time.Second, 5*time.Millisecond) + close(release) + waitForOrphanQueueEmpty(t, m) + assert.Equal(t, int32(1), attempts.Load(), "the second schedule for the same path must be dropped") +} + +func TestScheduleOrphanedVGPUReleaseIgnoresEmptyAssignment(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{Id: "no-gpu"}) + + m.orphanedVGPUMu.Lock() + defer m.orphanedVGPUMu.Unlock() + assert.Empty(t, m.orphanedVGPUs) +} + +// TestOrphanedVGPUReleaseReappliesClaimScan pins that the background retry +// goes through releaseStoredVGPU, not a raw destroy: a live claimant found by +// the vendor VFIO claim scan must keep blocking the release on every retry. +func TestOrphanedVGPUReleaseReappliesClaimScan(t *testing.T) { + t.Parallel() + + var destroys atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + orphanedVGPURetryDelay: time.Millisecond, + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + destroys.Add(1) + return nil + }, + } + // A claimant with a recent assignment and no persisted PID makes the scan + // fail closed, exactly like the synchronous release path. + require.NoError(t, m.ensureDirectories("mid-boot-claimant")) + assignedAt := time.Now() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "mid-boot-claimant", + Name: "mid-boot-claimant", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + }})) + + m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{ + Id: "deleted-instance", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }) + + waitForOrphanQueueEmpty(t, m) + assert.Zero(t, destroys.Load(), "no destroy may fire while the claim scan cannot clear the path") +} From 942a33d810ed556d5f2b6f89e29ff2cd990e19fd Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:40:11 +0000 Subject: [PATCH 067/107] Bound orphan retries in delete tests and tighten comments The two delete-continues tests triggered the new orphan retry with the default 30s delay, leaving a goroutine running ~10 minutes past the test. Use a millisecond delay and drain the queue before returning. --- lib/instances/delete.go | 5 ++--- lib/instances/lifecycle_noop_test.go | 4 ++++ lib/instances/vgpu_orphan.go | 18 ++++++------------ 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/lib/instances/delete.go b/lib/instances/delete.go index f7646d64d..49eead69c 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -151,9 +151,8 @@ func (m *manager) deleteInstanceWithOptions( log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID) } if err := m.releaseStoredVGPU(ctx, stored); err != nil { - // Log error but continue with cleanup. The metadata is about to be - // deleted, so hand the assignment to the background retry — otherwise - // the VF stays allocated until the next startup reconciliation. + // Log error but continue with cleanup; the background retry releases + // the VF once the metadata is gone. log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) m.scheduleOrphanedVGPURelease(ctx, *stored) } else if hadVGPUAssignment { diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 7868534a3..ebb8a8da1 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -152,6 +152,7 @@ func TestLifecycleNoopStandbyWithOptionsStillRejectsStandbyInstance(t *testing.T func TestDeleteContinuesWhenVGPUReleaseFails(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) + m.orphanedVGPURetryDelay = time.Millisecond meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" @@ -163,6 +164,7 @@ func TestDeleteContinuesWhenVGPUReleaseFails(t *testing.T) { // pre-refactor contract; the leaked assignment is recovered by the // background retry or startup reconciliation. require.NoError(t, m.DeleteInstance(context.Background(), id)) + waitForOrphanQueueEmpty(t, m) _, err = m.loadMetadata(id) require.Error(t, err, "instance data must be deleted despite the failed release") @@ -282,6 +284,7 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { func TestDeleteContinuesTeardownAfterFailedVGPURelease(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) + m.orphanedVGPURetryDelay = time.Millisecond deviceManager := &recordingDeviceManager{} m.deviceManager = deviceManager meta, err := m.loadMetadata(id) @@ -295,6 +298,7 @@ func TestDeleteContinuesTeardownAfterFailedVGPURelease(t *testing.T) { // The failed release must not block the rest of the teardown: devices // are detached and the instance is fully deleted. require.NoError(t, m.DeleteInstance(context.Background(), id)) + waitForOrphanQueueEmpty(t, m) assert.Equal(t, []string{"dev-1"}, deviceManager.detached) _, err = m.loadMetadata(id) diff --git a/lib/instances/vgpu_orphan.go b/lib/instances/vgpu_orphan.go index 69a3545c5..5f3fc2b38 100644 --- a/lib/instances/vgpu_orphan.go +++ b/lib/instances/vgpu_orphan.go @@ -18,18 +18,12 @@ const ( ) // scheduleOrphanedVGPURelease retries a vGPU release that failed during a -// completed delete, off the request path. Delete's log-and-continue contract -// is untouched — the caller already has its success — but without a retry the -// VF would stay allocated until the next startup reconciliation, silently -// shrinking host GPU capacity. A GPU-busy VMM routinely outlives delete's -// force-kill wait while the kernel finishes VFIO teardown, so this is the -// common case under load, not a tail case. -// -// Each attempt re-runs releaseStoredVGPU on a copy of the deleted instance's -// stored metadata, so the vendor VFIO claim scan and the destroy-side owner -// and open-handle guards apply on every retry exactly as they did on the -// original release. The queue is in-memory only: a restart abandons it and -// startup reconciliation sweeps the VF instead. +// completed delete, off the request path. A GPU-busy VMM routinely outlives +// delete's force-kill wait while the kernel finishes VFIO teardown, and once +// the metadata is deleted nothing else releases the VF until the next +// startup reconciliation. Each attempt re-runs releaseStoredVGPU, so the +// claim scan and destroy guards apply on every retry. The queue is in-memory +// only: a restart abandons it and startup reconciliation sweeps the VF. func (m *manager) scheduleOrphanedVGPURelease(ctx context.Context, stored StoredMetadata) { path := storedVGPUDevicePath(&stored) if path == "" { From 70de8d44c392644b5670b78ad86cdda2ede58956 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 19 Aug 2026 03:54:42 +0000 Subject: [PATCH 068/107] SIGTERM vGPU QEMU before SIGKILL during driver init A SIGKILL delivered to QEMU while the NVIDIA vGPU plugin is still initializing the VF wedges it near-deterministically: the guest driver loops on RmInitAdapter timeouts with no host-side signal, and only an SR-IOV cycle of the parent GPU recovers it. Voluntary QEMU exits run their VFIO teardown and are safe, as are hard kills after init. Start-failure cleanup and the force-kill fallback for initializing vGPU instances now send SIGTERM and wait a bounded grace before SIGKILL, and a hard kill inside the init window logs the affected device path. Clean creates, graceful stops, and running-instance deletes are unchanged. --- lib/devices/GPU.md | 14 +++- lib/hypervisor/qemu/process.go | 55 ++++++++++++- lib/hypervisor/qemu/process_test.go | 44 ++++++++++ lib/instances/delete.go | 11 +++ lib/instances/manager.go | 4 + lib/instances/process_identity.go | 14 ++++ lib/instances/process_identity_linux_test.go | 85 ++++++++++++++++++++ 7 files changed, 219 insertions(+), 8 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index ea5764ee0..2841d51c6 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -288,10 +288,16 @@ every request, so one wedged VF presents as all vGPU instances failing while `/resources` reports full capacity. The wedge itself leaves no host-side log: no kernel error, no XID, no plugin -crash. In the observed case it followed a period of heavy attach/teardown -churn on the VF, including QEMU processes that exited within seconds of -opening the VFIO device (failed start attempts that were then retried), so -suspect any workload that repeatedly kills the VMM mid-device-init. +crash. The trigger is a SIGKILL delivered to QEMU while the vGPU plugin is +still initializing the VF (roughly the first seconds after process start): +a single hard kill in that window wedges the VF near-deterministically, +while QEMU processes that exit voluntarily — error exits, QMP quit, SIGTERM — +run their VFIO teardown and never wedge, and hard kills of fully-initialized +vGPU VMs are also safe. Hypeman therefore SIGTERMs a vGPU QEMU first and only +escalates to SIGKILL after a grace period, both in start-failure cleanup and +when force-killing an instance that is still initializing; a hard kill in the +init window logs `VF may wedge` with the device path. External SIGKILLs +(OOM killer, manual `kill -9`) can still trigger it. Confirm by assigning the same profile on a different VF: if that guest initializes, the VF is wedged, not the driver stack. Remediate by cycling diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index c02c5beef..e7f8c84cc 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -40,6 +40,11 @@ const ( // socketDialTimeout is timeout for individual socket connection attempts socketDialTimeout = 100 * time.Millisecond + // vfioTermGrace is how long start-failure cleanup waits for a + // VFIO-attached QEMU to exit on SIGTERM before SIGKILL. Only failed + // starts pay it. + vfioTermGrace = 10 * time.Second + // clientCreateTimeout is how long to retry QMP client creation after the // socket appears. Under high parallel load the socket can accept connections // slightly later than file creation/availability. @@ -225,8 +230,14 @@ func buildQMPArgs(socketPath string) []string { } type startedProcess struct { - pid int - socketPath string + pid int + socketPath string + // termGrace, when non-zero, makes cleanup send SIGTERM and wait this long + // before SIGKILL. Set for VFIO-attached processes: hard-killing QEMU while + // the NVIDIA vGPU plugin is initializing can silently wedge the VF until + // its parent GPU's SR-IOV is cycled, while a terminating QEMU runs its + // device teardown and leaves the VF reusable. + termGrace time.Duration waitDone chan error waitConsumed bool waitErr error @@ -277,14 +288,47 @@ func (p *startedProcess) wait() error { return err } +// waitFor waits up to d for the process to exit, returning whether it did. +func (p *startedProcess) waitFor(d time.Duration) bool { + if _, exited := p.checkExited(); exited { + return true + } + select { + case err := <-p.waitDone: + p.waitConsumed = true + p.waitErr = err + return true + case <-time.After(d): + return false + } +} + func (p *startedProcess) cleanup() { if _, exited := p.checkExited(); !exited { - _ = syscall.Kill(p.pid, syscall.SIGKILL) - _ = p.wait() + terminated := false + if p.termGrace > 0 { + if syscall.Kill(p.pid, syscall.SIGTERM) == nil { + terminated = p.waitFor(p.termGrace) + } + } + if !terminated { + _ = syscall.Kill(p.pid, syscall.SIGKILL) + _ = p.wait() + } } _ = os.Remove(p.socketPath) } +// hasVFIODevice reports whether the QEMU command line attaches a VFIO device. +func hasVFIODevice(args []string) bool { + for _, arg := range args { + if strings.Contains(arg, "vfio-pci") { + return true + } + } + return false +} + // startQEMUProcess handles the common QEMU process startup logic. // Returns the PID, hypervisor client, and a cleanup function. // The cleanup function must be called on error; call cleanup.Release() on success. @@ -358,6 +402,9 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version } pid := proc.pid + if hasVFIODevice(args) { + proc.termGrace = vfioTermGrace + } log.DebugContext(processCtx, "QEMU process started", "pid", pid, "duration_ms", time.Since(processStartTime).Milliseconds()) // Setup cleanup to kill, reap, and remove the socket if subsequent steps fail. diff --git a/lib/hypervisor/qemu/process_test.go b/lib/hypervisor/qemu/process_test.go index e8be0dda2..b334887e3 100644 --- a/lib/hypervisor/qemu/process_test.go +++ b/lib/hypervisor/qemu/process_test.go @@ -1,6 +1,7 @@ package qemu import ( + "bufio" "context" "errors" "os" @@ -409,3 +410,46 @@ func TestWaitForSocketOrExitReturnsEarlyWhenProcessDies(t *testing.T) { require.NotNil(t, cmd.ProcessState) assert.True(t, cmd.ProcessState.Exited()) } + +func TestCleanupSIGTERMsProcessWithTermGrace(t *testing.T) { + socketPath := filepath.Join(t.TempDir(), "qemu.sock") + markerPath := filepath.Join(t.TempDir(), "terminated") + + cmd := exec.Command("sh", "-c", "trap 'touch "+markerPath+"; exit 0' TERM; echo ready; sleep 30 & wait") + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + proc, err := startManagedProcess(cmd, socketPath) + require.NoError(t, err) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + proc.termGrace = 5 * time.Second + + proc.cleanup() + + assert.FileExists(t, markerPath, "process must get SIGTERM, not SIGKILL, when termGrace is set") + require.NoFileExists(t, socketPath) + require.NotNil(t, cmd.ProcessState) +} + +func TestCleanupEscalatesToSIGKILLAfterTermGrace(t *testing.T) { + socketPath := filepath.Join(t.TempDir(), "qemu.sock") + + cmd := exec.Command("sh", "-c", "trap '' TERM; echo ready; sleep 30 & wait") + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + proc, err := startManagedProcess(cmd, socketPath) + require.NoError(t, err) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + proc.termGrace = 50 * time.Millisecond + + proc.cleanup() + + assert.ErrorIs(t, syscall.Kill(proc.pid, 0), syscall.ESRCH, "SIGTERM-ignoring process must still be hard-killed") + require.NoFileExists(t, socketPath) +} + +func TestHasVFIODevice(t *testing.T) { + assert.True(t, hasVFIODevice([]string{"-device", "vfio-pci,sysfsdev=/sys/bus/pci/devices/0000:82:00.4"})) + assert.False(t, hasVFIODevice([]string{"-device", "virtio-balloon-pci,id=balloon0"})) +} diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 49eead69c..c72729f21 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -241,6 +241,17 @@ func (m *manager) killHypervisor(ctx context.Context, inst *Instance) error { log.WarnContext(ctx, "stored hypervisor PID does not own the instance socket, killing the socket owner", "instance_id", inst.Id, "stored_pid", *inst.HypervisorPID, "owner_pid", pid) } + if inst.GPUProfile != "" && inst.State == StateInitializing { + // SIGKILL during vGPU driver init can silently wedge the VF until + // its parent GPU's SR-IOV is cycled, so ask the VMM to exit and + // run its VFIO teardown first. + if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { + os.Remove(inst.SocketPath) + return nil + } + log.WarnContext(ctx, "vGPU hypervisor did not exit on SIGTERM during driver init; hard-killing, VF may wedge", + "instance_id", inst.Id, "device_path", inst.GPUDevicePath) + } log.DebugContext(ctx, "killing hypervisor process", "instance_id", inst.Id, "pid", pid) if err := killProcessAndWait(pid); err != nil { return err diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 985eb29d7..c3709f53f 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -221,6 +221,10 @@ type manager struct { orphanedVGPUs map[string]struct{} orphanedVGPURetryDelay time.Duration + // vgpuInitTermGrace overrides killHypervisor's SIGTERM wait for vGPU + // instances still initializing; zero means the default. + vgpuInitTermGrace time.Duration + // Hypervisor support vmStarters map[hypervisor.Type]hypervisor.VMStarter defaultHypervisor hypervisor.Type // Default hypervisor type when not specified in request diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index be645e111..cca38e916 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -25,6 +25,20 @@ const linuxBootIDPath = "/proc/sys/kernel/random/boot_id" // does not unstick it, so the wait is short to keep stop and delete fast. const hypervisorSIGKILLWaitTimeout = 2 * time.Second +// defaultVGPUInitTermGrace is how long killHypervisor waits for a vGPU +// hypervisor still in driver init to exit on SIGTERM before SIGKILL. Only +// the force-kill fallback pays it, and only for initializing vGPU instances. +const defaultVGPUInitTermGrace = 10 * time.Second + +// vgpuTermGrace returns the SIGTERM wait used before hard-killing an +// initializing vGPU hypervisor. +func (m *manager) vgpuTermGrace() time.Duration { + if m.vgpuInitTermGrace > 0 { + return m.vgpuInitTermGrace + } + return defaultVGPUInitTermGrace +} + // killProcessAndWait SIGKILLs pid and waits for it to exit. A process that // survives the first wait gets its process group killed too (the hypervisor // may have spawned children in its own group) and a short grace period. An diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index 7ad259c41..2121d151b 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -691,3 +691,88 @@ func TestResolveRuntimeHypervisorPIDMintsIdentityOnlyWhenConfirmed(t *testing.T) assert.Empty(t, stored.HypervisorBootID, "a dead fallback must not mint the identity token") }) } + +// startTrapProcess starts a shell with the given TERM trap action (empty +// ignores the signal) and blocks until the trap is installed. It returns the +// PID and its boot-scoped identity. +func startTrapProcess(t *testing.T, trapAction string) (int, HypervisorProcessIdentity) { + t.Helper() + script := fmt.Sprintf("trap '%s' TERM; echo ready; sleep 30 & wait", trapAction) + process := exec.Command("sh", "-c", script) + stdout, err := process.StdoutPipe() + require.NoError(t, err) + require.NoError(t, process.Start()) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + waitDone := make(chan error, 1) + go func() { waitDone <- process.Wait() }() + t.Cleanup(func() { + _ = process.Process.Kill() + <-waitDone + }) + + pid := process.Process.Pid + startTime := processStartTime(pid) + require.NotZero(t, startTime) + return pid, HypervisorProcessIdentity{HypervisorPID: &pid, HypervisorStartTime: startTime, HypervisorBootID: hostBootID()} +} + +func TestKillHypervisorSIGTERMsInitializingVGPUHypervisor(t *testing.T) { + markerPath := filepath.Join(t.TempDir(), "terminated") + pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") + socketPath := filepath.Join(t.TempDir(), "missing.sock") + + m := &manager{} + require.NoError(t, m.killHypervisor(context.Background(), &Instance{ + State: StateInitializing, + StoredMetadata: StoredMetadata{ + Id: "kill-test", + GPUProfile: "NVIDIA L40S-1Q", + HypervisorProcessIdentity: identity, + SocketPath: socketPath, + }, + })) + + require.Eventually(t, func() bool { + return syscall.Kill(pid, 0) == syscall.ESRCH + }, 5*time.Second, 10*time.Millisecond) + assert.FileExists(t, markerPath, "hypervisor must be given SIGTERM, not SIGKILL, during vGPU driver init") +} + +func TestKillHypervisorEscalatesToSIGKILLWhenSIGTERMIgnored(t *testing.T) { + pid, identity := startTrapProcess(t, "") + socketPath := filepath.Join(t.TempDir(), "missing.sock") + + m := &manager{vgpuInitTermGrace: 50 * time.Millisecond} + require.NoError(t, m.killHypervisor(context.Background(), &Instance{ + State: StateInitializing, + StoredMetadata: StoredMetadata{ + Id: "kill-test", + GPUProfile: "NVIDIA L40S-1Q", + HypervisorProcessIdentity: identity, + SocketPath: socketPath, + }, + })) + + assert.ErrorIs(t, syscall.Kill(pid, 0), syscall.ESRCH, "SIGTERM-ignoring hypervisor must still be hard-killed") +} + +func TestKillHypervisorHardKillsVGPUHypervisorPastInit(t *testing.T) { + markerPath := filepath.Join(t.TempDir(), "terminated") + pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") + socketPath := filepath.Join(t.TempDir(), "missing.sock") + + m := &manager{} + require.NoError(t, m.killHypervisor(context.Background(), &Instance{ + State: StateRunning, + StoredMetadata: StoredMetadata{ + Id: "kill-test", + GPUProfile: "NVIDIA L40S-1Q", + HypervisorProcessIdentity: identity, + SocketPath: socketPath, + }, + })) + + assert.ErrorIs(t, syscall.Kill(pid, 0), syscall.ESRCH) + assert.NoFileExists(t, markerPath, "running vGPU hypervisors keep the direct SIGKILL path") +} From c64689dcba94232b7e39d465ce0b44844be0245a Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:07:44 +0000 Subject: [PATCH 069/107] Apply vGPU SIGTERM grace on stop's direct kill paths shutdownHypervisor (stop) could still SIGKILL an initializing vGPU QEMU directly: on QMP connect failure, on graceful-quit timeout, and when the hypervisor lacks graceful shutdown, bypassing the grace killHypervisor applies. Extract the SIGTERM-then-SIGKILL escalation into terminateThenKill and use it at all four force-kill sites. --- lib/hypervisor/qemu/process_test.go | 5 ----- lib/instances/delete.go | 13 +------------ lib/instances/manager.go | 2 +- lib/instances/process_identity.go | 21 +++++++++++++++++++-- lib/instances/standby.go | 6 +++--- 5 files changed, 24 insertions(+), 23 deletions(-) diff --git a/lib/hypervisor/qemu/process_test.go b/lib/hypervisor/qemu/process_test.go index b334887e3..2bdf27d4f 100644 --- a/lib/hypervisor/qemu/process_test.go +++ b/lib/hypervisor/qemu/process_test.go @@ -448,8 +448,3 @@ func TestCleanupEscalatesToSIGKILLAfterTermGrace(t *testing.T) { assert.ErrorIs(t, syscall.Kill(proc.pid, 0), syscall.ESRCH, "SIGTERM-ignoring process must still be hard-killed") require.NoFileExists(t, socketPath) } - -func TestHasVFIODevice(t *testing.T) { - assert.True(t, hasVFIODevice([]string{"-device", "vfio-pci,sysfsdev=/sys/bus/pci/devices/0000:82:00.4"})) - assert.False(t, hasVFIODevice([]string{"-device", "virtio-balloon-pci,id=balloon0"})) -} diff --git a/lib/instances/delete.go b/lib/instances/delete.go index c72729f21..2f90a8552 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -241,19 +241,8 @@ func (m *manager) killHypervisor(ctx context.Context, inst *Instance) error { log.WarnContext(ctx, "stored hypervisor PID does not own the instance socket, killing the socket owner", "instance_id", inst.Id, "stored_pid", *inst.HypervisorPID, "owner_pid", pid) } - if inst.GPUProfile != "" && inst.State == StateInitializing { - // SIGKILL during vGPU driver init can silently wedge the VF until - // its parent GPU's SR-IOV is cycled, so ask the VMM to exit and - // run its VFIO teardown first. - if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { - os.Remove(inst.SocketPath) - return nil - } - log.WarnContext(ctx, "vGPU hypervisor did not exit on SIGTERM during driver init; hard-killing, VF may wedge", - "instance_id", inst.Id, "device_path", inst.GPUDevicePath) - } log.DebugContext(ctx, "killing hypervisor process", "instance_id", inst.Id, "pid", pid) - if err := killProcessAndWait(pid); err != nil { + if err := m.terminateThenKill(ctx, inst, pid); err != nil { return err } } diff --git a/lib/instances/manager.go b/lib/instances/manager.go index c3709f53f..846b73ce5 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -221,7 +221,7 @@ type manager struct { orphanedVGPUs map[string]struct{} orphanedVGPURetryDelay time.Duration - // vgpuInitTermGrace overrides killHypervisor's SIGTERM wait for vGPU + // vgpuInitTermGrace overrides terminateThenKill's SIGTERM wait for vGPU // instances still initializing; zero means the default. vgpuInitTermGrace time.Duration diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index cca38e916..9bfe2cdaa 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -1,6 +1,7 @@ package instances import ( + "context" "errors" "fmt" "os" @@ -13,6 +14,7 @@ import ( "time" "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/logger" ) // linuxBootIDPath is the kernel-provided boot ID used to scope process @@ -25,9 +27,9 @@ const linuxBootIDPath = "/proc/sys/kernel/random/boot_id" // does not unstick it, so the wait is short to keep stop and delete fast. const hypervisorSIGKILLWaitTimeout = 2 * time.Second -// defaultVGPUInitTermGrace is how long killHypervisor waits for a vGPU +// defaultVGPUInitTermGrace is how long terminateThenKill waits for a vGPU // hypervisor still in driver init to exit on SIGTERM before SIGKILL. Only -// the force-kill fallback pays it, and only for initializing vGPU instances. +// force-kill paths pay it, and only for initializing vGPU instances. const defaultVGPUInitTermGrace = 10 * time.Second // vgpuTermGrace returns the SIGTERM wait used before hard-killing an @@ -39,6 +41,21 @@ func (m *manager) vgpuTermGrace() time.Duration { return defaultVGPUInitTermGrace } +// terminateThenKill hard-kills the hypervisor process, first giving a vGPU +// instance still in driver init a SIGTERM grace: SIGKILL in that window can +// silently wedge the VF until its parent GPU's SR-IOV is cycled (see +// lib/devices/GPU.md), while a terminating QEMU runs its VFIO teardown. +func (m *manager) terminateThenKill(ctx context.Context, inst *Instance, pid int) error { + if inst.GPUProfile != "" && inst.State == StateInitializing { + if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { + return nil + } + logger.FromContext(ctx).WarnContext(ctx, "vGPU hypervisor did not exit on SIGTERM during driver init; hard-killing, VF may wedge", + "instance_id", inst.Id, "device_path", inst.GPUDevicePath) + } + return killProcessAndWait(pid) +} + // killProcessAndWait SIGKILLs pid and waits for it to exit. A process that // survives the first wait gets its process group killed too (the hypervisor // may have spawned children in its own group) and a short grace period. An diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 4c6fd7d33..5c6e0173e 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -376,7 +376,7 @@ func (m *manager) shutdownHypervisor(ctx context.Context, inst *Instance) error // alive; teardown is committed, so kill it rather than report a // completed shutdown for a VMM that is still running. log.WarnContext(ctx, "could not connect to hypervisor, force killing resolved owner", "instance_id", inst.Id, "pid", pid, "error", err) - if err := killProcessAndWait(pid); err != nil { + if err := m.terminateThenKill(ctx, inst, pid); err != nil { return err } } @@ -405,13 +405,13 @@ func (m *manager) shutdownHypervisor(ctx context.Context, inst *Instance) error log.DebugContext(ctx, "hypervisor shutdown gracefully", "instance_id", inst.Id, "pid", pid) } else { log.WarnContext(ctx, "hypervisor did not exit gracefully in time, force killing process", "instance_id", inst.Id, "pid", pid) - if err := killProcessAndWait(pid); err != nil { + if err := m.terminateThenKill(ctx, inst, pid); err != nil { return err } } } else { log.DebugContext(ctx, "skipping graceful exit wait; force killing hypervisor process", "instance_id", inst.Id, "pid", pid) - if err := killProcessAndWait(pid); err != nil { + if err := m.terminateThenKill(ctx, inst, pid); err != nil { return err } } From 53ff4252b5d503825a5527623e221495689bba56 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:38:06 +0000 Subject: [PATCH 070/107] Lower vGPU SIGTERM grace to 5s Observed mid-init VFIO teardown completes in 1-2s, so 5s keeps 2-3x margin while halving the worst-case delay for a SIGTERM-ignoring process. --- lib/hypervisor/qemu/process.go | 5 +++-- lib/instances/process_identity.go | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index e7f8c84cc..cddd51824 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -42,8 +42,9 @@ const ( // vfioTermGrace is how long start-failure cleanup waits for a // VFIO-attached QEMU to exit on SIGTERM before SIGKILL. Only failed - // starts pay it. - vfioTermGrace = 10 * time.Second + // starts pay it, and only when the process ignores SIGTERM; observed + // mid-init VFIO teardown takes 1-2s. + vfioTermGrace = 5 * time.Second // clientCreateTimeout is how long to retry QMP client creation after the // socket appears. Under high parallel load the socket can accept connections diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 9bfe2cdaa..2f68b6c92 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -29,8 +29,10 @@ const hypervisorSIGKILLWaitTimeout = 2 * time.Second // defaultVGPUInitTermGrace is how long terminateThenKill waits for a vGPU // hypervisor still in driver init to exit on SIGTERM before SIGKILL. Only -// force-kill paths pay it, and only for initializing vGPU instances. -const defaultVGPUInitTermGrace = 10 * time.Second +// force-kill paths pay it, only for initializing vGPU instances, and only +// when the process ignores SIGTERM; observed mid-init VFIO teardown takes +// 1-2s. +const defaultVGPUInitTermGrace = 5 * time.Second // vgpuTermGrace returns the SIGTERM wait used before hard-killing an // initializing vGPU hypervisor. From c9e7ece535dcec3b79ba691d6af2324d8b515a59 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 20 Aug 2026 18:52:27 +0000 Subject: [PATCH 071/107] Apply vGPU SIGTERM grace in every instance state Running reports true ~4s before the guest driver finishes initializing and no host-side signal observes that boundary, so gating the SIGTERM grace on StateInitializing left a window where a failed stop or delete could SIGKILL QEMU mid-driver-init and wedge the VF. Post-init the SIGTERM is proven harmless and costs the grace only when the process ignores it. --- lib/devices/GPU.md | 7 ++--- lib/instances/process_identity.go | 27 +++++++++++--------- lib/instances/process_identity_linux_test.go | 25 ++++++++++++++++-- 3 files changed, 42 insertions(+), 17 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 2841d51c6..ff11c5cde 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -295,9 +295,10 @@ while QEMU processes that exit voluntarily — error exits, QMP quit, SIGTERM run their VFIO teardown and never wedge, and hard kills of fully-initialized vGPU VMs are also safe. Hypeman therefore SIGTERMs a vGPU QEMU first and only escalates to SIGKILL after a grace period, both in start-failure cleanup and -when force-killing an instance that is still initializing; a hard kill in the -init window logs `VF may wedge` with the device path. External SIGKILLs -(OOM killer, manual `kill -9`) can still trigger it. +when force-killing any vGPU instance (the instance reports Running seconds +before driver init completes, so no state reliably marks the window); a hard +kill after an ignored SIGTERM logs `VF may wedge` with the device path. +External SIGKILLs (OOM killer, manual `kill -9`) can still trigger it. Confirm by assigning the same profile on a different VF: if that guest initializes, the VF is wedged, not the driver stack. Remediate by cycling diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 2f68b6c92..0801aca6c 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -28,14 +28,13 @@ const linuxBootIDPath = "/proc/sys/kernel/random/boot_id" const hypervisorSIGKILLWaitTimeout = 2 * time.Second // defaultVGPUInitTermGrace is how long terminateThenKill waits for a vGPU -// hypervisor still in driver init to exit on SIGTERM before SIGKILL. Only -// force-kill paths pay it, only for initializing vGPU instances, and only -// when the process ignores SIGTERM; observed mid-init VFIO teardown takes -// 1-2s. +// hypervisor to exit on SIGTERM before SIGKILL. Only force-kill paths pay it, +// only for vGPU instances, and only when the process ignores SIGTERM; +// observed mid-init VFIO teardown takes 1-2s. const defaultVGPUInitTermGrace = 5 * time.Second -// vgpuTermGrace returns the SIGTERM wait used before hard-killing an -// initializing vGPU hypervisor. +// vgpuTermGrace returns the SIGTERM wait used before hard-killing a vGPU +// hypervisor. func (m *manager) vgpuTermGrace() time.Duration { if m.vgpuInitTermGrace > 0 { return m.vgpuInitTermGrace @@ -43,16 +42,20 @@ func (m *manager) vgpuTermGrace() time.Duration { return defaultVGPUInitTermGrace } -// terminateThenKill hard-kills the hypervisor process, first giving a vGPU -// instance still in driver init a SIGTERM grace: SIGKILL in that window can -// silently wedge the VF until its parent GPU's SR-IOV is cycled (see -// lib/devices/GPU.md), while a terminating QEMU runs its VFIO teardown. +// terminateThenKill hard-kills the hypervisor process, first giving any vGPU +// instance a SIGTERM grace: SIGKILL during guest driver init can silently +// wedge the VF until its parent GPU's SR-IOV is cycled (see +// lib/devices/GPU.md), while a terminating QEMU runs its VFIO teardown. The +// grace applies in every state, not just Initializing, because the instance +// reports Running seconds before the guest driver finishes initializing and +// nothing host-side observes that boundary; post-init the SIGTERM is proven +// harmless and costs the grace only when the process ignores it. func (m *manager) terminateThenKill(ctx context.Context, inst *Instance, pid int) error { - if inst.GPUProfile != "" && inst.State == StateInitializing { + if inst.GPUProfile != "" { if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { return nil } - logger.FromContext(ctx).WarnContext(ctx, "vGPU hypervisor did not exit on SIGTERM during driver init; hard-killing, VF may wedge", + logger.FromContext(ctx).WarnContext(ctx, "vGPU hypervisor did not exit on SIGTERM; hard-killing, VF may wedge if the guest driver was initializing", "instance_id", inst.Id, "device_path", inst.GPUDevicePath) } return killProcessAndWait(pid) diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index 2121d151b..e090b9a98 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -757,7 +757,7 @@ func TestKillHypervisorEscalatesToSIGKILLWhenSIGTERMIgnored(t *testing.T) { assert.ErrorIs(t, syscall.Kill(pid, 0), syscall.ESRCH, "SIGTERM-ignoring hypervisor must still be hard-killed") } -func TestKillHypervisorHardKillsVGPUHypervisorPastInit(t *testing.T) { +func TestKillHypervisorSIGTERMsRunningVGPUHypervisor(t *testing.T) { markerPath := filepath.Join(t.TempDir(), "terminated") pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") socketPath := filepath.Join(t.TempDir(), "missing.sock") @@ -773,6 +773,27 @@ func TestKillHypervisorHardKillsVGPUHypervisorPastInit(t *testing.T) { }, })) + require.Eventually(t, func() bool { + return syscall.Kill(pid, 0) == syscall.ESRCH + }, 5*time.Second, 10*time.Millisecond) + assert.FileExists(t, markerPath, "Running reports true before guest driver init completes, so vGPU hypervisors get SIGTERM in every state") +} + +func TestKillHypervisorHardKillsNonVGPUHypervisor(t *testing.T) { + markerPath := filepath.Join(t.TempDir(), "terminated") + pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") + socketPath := filepath.Join(t.TempDir(), "missing.sock") + + m := &manager{} + require.NoError(t, m.killHypervisor(context.Background(), &Instance{ + State: StateRunning, + StoredMetadata: StoredMetadata{ + Id: "kill-test", + HypervisorProcessIdentity: identity, + SocketPath: socketPath, + }, + })) + assert.ErrorIs(t, syscall.Kill(pid, 0), syscall.ESRCH) - assert.NoFileExists(t, markerPath, "running vGPU hypervisors keep the direct SIGKILL path") + assert.NoFileExists(t, markerPath, "non-vGPU hypervisors keep the direct SIGKILL path") } From 7948ce9b05528b93dc21843f0667a666a7888efe Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 20 Aug 2026 20:12:33 +0000 Subject: [PATCH 072/107] Harden the vGPU claim scan and retention-stub stop path A concurrent instance deletion between the claim scan's metadata listing and load turned ErrNotFound into a host-wide fail-closed release error, even though a vanished record cannot be a live claimant; skip it. Stop on a delete-only retention stub released its VF while leaving GPURetainedForCleanup set, so the stub's start/fork/snapshot errors kept claiming an assignment that no longer existed. Retention stubs now release only through delete, as documented. --- lib/instances/lifecycle_noop_test.go | 22 ++++++++++++++++++++++ lib/instances/vgpu.go | 11 +++++++++++ 2 files changed, 33 insertions(+) diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index ebb8a8da1..46f891379 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -367,6 +367,28 @@ func TestStopStoppedInstanceReleasesRetainedVGPU(t *testing.T) { assert.Empty(t, stored.GPUDevicePath) } +func TestStopStoppedInstanceLeavesRetentionStubForDelete(t *testing.T) { + m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFrameworkNone + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, m.saveMetadata(meta)) + + inst, err := m.StopInstance(context.Background(), id) + require.NoError(t, err) + require.NotNil(t, inst) + + // Retention stubs are delete-only: releasing on stop would leave a stub + // whose start/fork/snapshot errors still claim a retained assignment. + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) + assert.True(t, stored.GPURetainedForCleanup) +} + func TestStopStoppedInstanceVGPUReleaseFailureRemainsNoop(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) meta, err := m.loadMetadata(id) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index d0cd9b226..6b707e94c 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -187,6 +187,11 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu } meta, err := m.loadMetadata(id) if err != nil { + if errors.Is(err, ErrNotFound) { + // Deleted between listing and load; a vanished record cannot + // be a live claimant. + continue + } return false, fmt.Errorf("load metadata for vGPU release check: instance %s: %w", id, err) } stored := &meta.StoredMetadata @@ -228,6 +233,12 @@ func (m *manager) releaseRetainedVGPULocked(ctx context.Context, id string) { return } stored := &meta.StoredMetadata + if stored.GPURetainedForCleanup { + // Delete-only retention stubs release through delete. Releasing here + // would leave a stub whose start/fork/snapshot errors still claim a + // retained assignment that no longer exists. + return + } if storedVGPUDevicePath(stored) == "" { return } From bb8614ab866d2dbc40a9e14de11c3ae0a16899eb Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:10:59 +0000 Subject: [PATCH 073/107] Share one vGPU retention stub in create The two rollback paths carried near-identical retention-stub literals that would drift as fields are added. Build both from one helper, use nowUTC like the rest of the file, and drop the starter guard that is dead since create fails on a nil starter long before the vGPU block. --- lib/instances/create.go | 45 +++++++++++++++++------------------------ 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index e7ab3b6ad..eefd0a7ce 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -309,24 +309,27 @@ func (m *manager) createInstance( // Handle vGPU profile request if req.GPU != nil && req.GPU.Profile != "" { - log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) - gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) - if err != nil { - stub := StoredMetadata{ + // Identity fields a retention record keeps when rollback cannot + // release the assignment, so it lists as a recognizable, deletable + // instance. Create has already failed on a nil starter by this point. + retentionStub := func() StoredMetadata { + return StoredMetadata{ Id: id, Name: req.Name, Image: req.Image, ResolvedImage: resolvedImageRef, Platform: imageInfo.Platform, - CreatedAt: time.Now(), + CreatedAt: m.nowUTC(), HypervisorType: hvType, HypervisorVersion: hvVersion, + SocketPath: m.paths.InstanceSocket(id, starter.SocketName()), DataDir: m.paths.InstanceDir(id), } - if starterErr == nil { - stub.SocketPath = m.paths.InstanceSocket(id, starter.SocketName()) - } - retainedVGPU = retainedVGPUFromCreateError(stub, m.nowUTC(), err) + } + log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) + gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) + if err != nil { + retainedVGPU = retainedVGPUFromCreateError(retentionStub(), m.nowUTC(), err) log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) return nil, wrapCreateVGPUErr(req.GPU.Profile, err) } @@ -351,23 +354,13 @@ func (m *manager) createInstance( log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) retainedVGPU = stored if retainedVGPU == nil { - retainedVGPU = &StoredMetadata{ - Id: id, - Name: req.Name, - Image: req.Image, - ResolvedImage: resolvedImageRef, - Platform: imageInfo.Platform, - CreatedAt: time.Now(), - HypervisorType: hvType, - HypervisorVersion: hvVersion, - SocketPath: m.paths.InstanceSocket(id, starter.SocketName()), - DataDir: m.paths.InstanceDir(id), - GPUProfile: gpuDevice.ProfileName, - GPUFramework: gpuDevice.Framework, - GPUDevicePath: gpuDevice.SysfsPath, - GPUMdevUUID: gpuDevice.MdevUUID, - GPUAssignedAt: gpuAssignedAt, - } + stub := retentionStub() + stub.GPUProfile = gpuDevice.ProfileName + stub.GPUFramework = gpuDevice.Framework + stub.GPUDevicePath = gpuDevice.SysfsPath + stub.GPUMdevUUID = gpuDevice.MdevUUID + stub.GPUAssignedAt = gpuAssignedAt + retainedVGPU = &stub } } }) From 54c37b31aa3230f7a95140c69484f758a9aeca33 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:31:49 +0000 Subject: [PATCH 074/107] Name the strict metadata listing variant Replace the listMetadataFilesWithStatErrors(bool) mode flag with listMetadataFiles / listMetadataFilesStrict so call sites say which failure semantics they rely on. --- lib/instances/storage.go | 15 ++++++++++++--- lib/instances/vgpu.go | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/instances/storage.go b/lib/instances/storage.go index dd932d41a..6a2354624 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -187,12 +187,21 @@ func removeAllWithRetry(path string, removeAll func(string) error, sleep func(ti } } -// listMetadataFiles returns paths to all instance metadata files. +// listMetadataFiles returns paths to all instance metadata files, skipping +// entries whose metadata cannot be statted. func (m *manager) listMetadataFiles() ([]string, error) { - return m.listMetadataFilesWithStatErrors(false) + return m.walkMetadataFiles(false) } -func (m *manager) listMetadataFilesWithStatErrors(failOnStatError bool) ([]string, error) { +// listMetadataFilesStrict returns paths to all instance metadata files, +// failing on any stat error other than absence. Fail-closed callers (the +// vGPU release claim scan and startup reconcile protection) use it so an +// unreadable instance is an error instead of silently missing. +func (m *manager) listMetadataFilesStrict() ([]string, error) { + return m.walkMetadataFiles(true) +} + +func (m *manager) walkMetadataFiles(failOnStatError bool) ([]string, error) { guestsDir := m.paths.GuestsDir() // Ensure guests directory exists diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 6b707e94c..b38fb5225 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -176,7 +176,7 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) // assignment without a PID, or unverifiable process ownership returns an error // so the requester retains its assignment for a later retry. func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { - files, err := m.listMetadataFilesWithStatErrors(true) + files, err := m.listMetadataFilesStrict() if err != nil { return false, fmt.Errorf("list instances for vGPU release check: %w", err) } From b86d1da2e85bf95167505bb826b8c735a64d53c1 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:31:49 +0000 Subject: [PATCH 075/107] Skip instances deleted mid-listing in reconcile protection ListInstancesForReconcile failed hard when an instance was deleted between the metadata listing and its load. The startup call runs before the API serves, but the grace-period retry fires while deletes are in flight; one racing delete errored the whole list, which zeroed the retry and left vendor VFIO reconciliation disabled until the next restart. Skip ErrNotFound like the release claim scan does: a vanished record cannot claim a VF. --- lib/instances/manager.go | 10 ++++++++- lib/instances/query_test.go | 41 +++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 846b73ce5..54e9b00d6 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -2,6 +2,7 @@ package instances import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -755,7 +756,7 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { // needs raw metadata fields, and hydration would query the hypervisor of // every instance on the host before the API serves. func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, error) { - files, err := m.listMetadataFilesWithStatErrors(true) + files, err := m.listMetadataFilesStrict() if err != nil { return nil, err } @@ -764,6 +765,13 @@ func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, er id := filepath.Base(filepath.Dir(file)) meta, err := m.loadMetadata(id) if err != nil { + if errors.Is(err, ErrNotFound) { + // Deleted between listing and load; a vanished record cannot + // claim a VF. Failing here instead would zero the grace-period + // retry and disable the vendor VFIO sweep whenever it races a + // concurrent delete. + continue + } return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) } result = append(result, Instance{StoredMetadata: meta.StoredMetadata}) diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index ab3db29c3..b3dbfba41 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -42,6 +42,47 @@ func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { assert.Equal(t, "valid", listed[0].Id) } +// A concurrent delete can remove an instance between the reconcile listing +// and its metadata load. A vanished record cannot claim a VF, so it must be +// skipped like the release claim scan does — failing instead would zero the +// grace-period retry and silently disable the vendor VFIO sweep whenever it +// races a delete. +func TestListInstancesForReconcileSkipsInstanceDeletedDuringListing(t *testing.T) { + m := &manager{paths: paths.New(t.TempDir())} + + for _, id := range []string{"aaa-ghost", "zzz-live"} { + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + CreatedAt: time.Now(), + DataDir: m.paths.InstanceDir(id), + }})) + } + + // loadMetadata takes the snapshot-alias read lock, so holding the + // mutation lock parks the reconcile between listing and loading — the + // window a concurrent delete lands in. + unlock := hypervisor.LockSnapshotSourceAliasMutation() + type result struct { + listed []Instance + err error + } + done := make(chan result, 1) + go func() { + listed, err := m.ListInstancesForReconcile(context.Background()) + done <- result{listed, err} + }() + time.Sleep(100 * time.Millisecond) + require.NoError(t, os.Remove(m.paths.InstanceMetadata("aaa-ghost"))) + unlock() + + res := <-done + require.NoError(t, res.err) + require.Len(t, res.listed, 1) + assert.Equal(t, "zzz-live", res.listed[0].Id) +} + func TestParseExitSentinelLine(t *testing.T) { t.Parallel() tests := []struct { From f255bb2b78a27d5e11647f9739ae0db631b273e2 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:31:49 +0000 Subject: [PATCH 076/107] Count abandoned orphaned vGPU releases Giving up on an orphaned release leaves the VF allocated while /resources still advertises it, until startup reconciliation or manual remediation. That was visible only as a log line; count it so capacity leaks can alert. --- lib/instances/metrics.go | 21 +++++++++++++++++++++ lib/instances/vgpu_orphan.go | 1 + 2 files changed, 22 insertions(+) diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index 1ada5ac1e..bdffbdbce 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -94,6 +94,7 @@ type Metrics struct { lifecycleEventsDroppedTotal metric.Int64Counter forkMemFileShareFallbacksTotal metric.Int64Counter ttlReaperDeletionsTotal metric.Int64Counter + vgpuOrphanReleasesAbandonedTotal metric.Int64Counter tracer trace.Tracer } @@ -270,6 +271,14 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + vgpuOrphanReleasesAbandonedTotal, err := meter.Int64Counter( + "hypeman_instances_vgpu_orphan_releases_abandoned_total", + metric.WithDescription("Total orphaned vGPU release retries that gave up, leaving the VF allocated until startup reconciliation or manual remediation"), + ) + if err != nil { + return nil, err + } + // Register observable gauge for instance counts by state instancesTotal, err := meter.Int64ObservableGauge( "hypeman_instances_total", @@ -464,6 +473,7 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M lifecycleEventsDroppedTotal: lifecycleEventsDroppedTotal, forkMemFileShareFallbacksTotal: forkMemFileShareFallbacksTotal, ttlReaperDeletionsTotal: ttlReaperDeletionsTotal, + vgpuOrphanReleasesAbandonedTotal: vgpuOrphanReleasesAbandonedTotal, tracer: tracer, }, nil } @@ -563,6 +573,17 @@ func (m *manager) recordTimeToRunning(ctx context.Context, stored *StoredMetadat m.metrics.timeToRunning.Record(ctx, duration, metric.WithAttributes(attrs...)) } +// recordVGPUOrphanReleaseAbandoned records an orphaned vGPU release retry +// loop giving up: the VF stays allocated (capacity silently reduced) until +// startup reconciliation or manual remediation, so it must be visible beyond +// a log line. +func (m *manager) recordVGPUOrphanReleaseAbandoned(ctx context.Context) { + if m.metrics == nil { + return + } + m.metrics.vgpuOrphanReleasesAbandonedTotal.Add(ctx, 1) +} + // recordStateTransition records a state transition with hypervisor label. func (m *manager) recordStateTransition(ctx context.Context, fromState, toState string, hvType hypervisor.Type) { if m.metrics == nil { diff --git a/lib/instances/vgpu_orphan.go b/lib/instances/vgpu_orphan.go index 5f3fc2b38..b3ff2b33b 100644 --- a/lib/instances/vgpu_orphan.go +++ b/lib/instances/vgpu_orphan.go @@ -67,6 +67,7 @@ func (m *manager) retryOrphanedVGPURelease(ctx context.Context, stored StoredMet "instance_id", stored.Id, "device_path", path, "attempt", attempt) return } + m.recordVGPUOrphanReleaseAbandoned(ctx) log.ErrorContext(ctx, "giving up on orphaned vGPU release; VF stays allocated until startup reconciliation or manual remediation", "instance_id", stored.Id, "device_path", path, "attempts", orphanedVGPUReleaseMaxAttempts) } From 3644fbe1d799b453ad3dc08ddf3a9ca347edfd62 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:52:25 +0000 Subject: [PATCH 077/107] Define the vGPU retention-stub rejection once Start, fork, and snapshot each carried a verbatim copy of the rejection error and its rationale; the stub fill in create's cleanup closure duplicated retainedVGPUFromCreateError field-for-field. One error value and one device-to-stub helper replace the copies. --- lib/instances/create.go | 8 +------- lib/instances/fork.go | 5 +---- lib/instances/snapshot.go | 6 +----- lib/instances/start.go | 5 +---- lib/instances/vgpu.go | 16 ++++++++++++---- 5 files changed, 16 insertions(+), 24 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index eefd0a7ce..66c988b4f 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -354,13 +354,7 @@ func (m *manager) createInstance( log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) retainedVGPU = stored if retainedVGPU == nil { - stub := retentionStub() - stub.GPUProfile = gpuDevice.ProfileName - stub.GPUFramework = gpuDevice.Framework - stub.GPUDevicePath = gpuDevice.SysfsPath - stub.GPUMdevUUID = gpuDevice.MdevUUID - stub.GPUAssignedAt = gpuAssignedAt - retainedVGPU = &stub + retainedVGPU = retainedVGPUFromDevice(retentionStub(), gpuDevice, *gpuAssignedAt) } } }) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 266299e96..08ce4014a 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -220,10 +220,7 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin return nil, false, fmt.Errorf("%w: cannot fork from state %s (must be Stopped or Standby)", ErrInvalidState, source.State) } if stored.GPURetainedForCleanup { - // A delete-only retention stub from a failed create has no boot - // configuration, so a fork of it could never boot. Delete the stub to - // release its retained vGPU assignment. - return nil, false, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + return nil, false, errVGPURetentionStub } if !supportValidated { diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 297c7ef03..e30192ee9 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -67,11 +67,7 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps stored := &meta.StoredMetadata if stored.GPURetainedForCleanup { - // A delete-only retention stub from a failed create has no boot - // configuration, so a snapshot of it could never be restored or - // forked into a bootable instance. Delete the stub to release its - // retained vGPU assignment. - return nil, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + return nil, errVGPURetentionStub } if err := validateForkVolumeSafety(stored.Volumes); err != nil { return nil, fmt.Errorf("%w: snapshot requires readonly volume attachments: %v", ErrNotSupported, err) diff --git a/lib/instances/start.go b/lib/instances/start.go index 44dc50216..c7eb85493 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -48,11 +48,8 @@ func (m *manager) startInstance( return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } if stored.GPURetainedForCleanup { - // A delete-only retention stub from a failed create: it carries no - // boot configuration, so starting it would release the retained VF - // and then boot an incomplete record. Delete retries the release. log.ErrorContext(ctx, "refusing to start vGPU retention record", "instance_id", id) - return nil, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + return nil, errVGPURetentionStub } // Release any assignment retained by an earlier failed release and diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index b38fb5225..dcb6cff4d 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -33,6 +33,11 @@ func (e *VGPUCleanupPendingError) Error() string { func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } +// errVGPURetentionStub rejects every lifecycle verb except delete on a +// retention stub from a failed create: the record has no boot configuration, +// and only delete retries the release of its retained assignment. +var errVGPURetentionStub = fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + func (m *manager) createVGPUDevice(ctx context.Context, profileName, instanceID string) (*devices.VGPUDevice, error) { create := m.createVGPU if create == nil { @@ -58,11 +63,14 @@ func retainedVGPUFromCreateError(stub StoredMetadata, assignedAt time.Time, err if !ok { return nil } + return retainedVGPUFromDevice(stub, device, assignedAt) +} + +// retainedVGPUFromDevice fills stub with device's assignment fields so a +// failed rollback release retains a recognizable, deletable record. +func retainedVGPUFromDevice(stub StoredMetadata, device *devices.VGPUDevice, assignedAt time.Time) *StoredMetadata { stub.GPUProfile = device.ProfileName - stub.GPUFramework = device.Framework - stub.GPUDevicePath = device.SysfsPath - stub.GPUMdevUUID = device.MdevUUID - stub.GPUAssignedAt = &assignedAt + setStoredVGPUDevice(&stub, device, assignedAt) return &stub } From b4446633e46f2ef5bd1947c186bb8bbfd68340b9 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:52:25 +0000 Subject: [PATCH 078/107] Render pending vGPU cleanup responses through one helper The create and start handlers carried near-identical 20-line blocks deriving the vgpu_cleanup_pending message and inner error detail, differing only in the verb and release guidance. --- cmd/api/api/instances.go | 48 ++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index a277b5476..1c502c928 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -368,19 +368,11 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst // errors.Is case would match the cause and hide the pending vGPU cleanup. case errors.As(err, &vgpuPending): log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) - message := fmt.Sprintf("failed to create instance: %v; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.Err, vgpuPending.InstanceID) - innerCode := "vgpu_retained_instance" - if !vgpuPending.Retained { - message = fmt.Sprintf("failed to create instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.Err, vgpuPending.InstanceID) - innerCode = "vgpu_unretained_instance" - } + message, inner := vgpuCleanupPendingDetail(vgpuPending, "create", "delete it to retry") return oapi.CreateInstance500JSONResponse{ - Code: "vgpu_cleanup_pending", - Message: message, - InnerError: &oapi.ErrorDetail{ - Code: lo.ToPtr(innerCode), - Message: lo.ToPtr(vgpuPending.InstanceID), - }, + Code: "vgpu_cleanup_pending", + Message: message, + InnerError: inner, }, nil case errors.Is(err, instances.ErrImageNotReady): return oapi.CreateInstance400JSONResponse{ @@ -443,6 +435,22 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst return oapi.CreateInstance201JSONResponse(instanceToOAPI(*inst)), nil } +// vgpuCleanupPendingDetail renders a pending vGPU cleanup into the message +// and inner error detail shared by the create and start handlers. The +// retained guidance names the verb-specific way to release the assignment. +func vgpuCleanupPendingDetail(pending *instances.VGPUCleanupPendingError, action, retainedGuidance string) (string, *oapi.ErrorDetail) { + message := fmt.Sprintf("failed to %s instance: %v; vGPU release failed during rollback and instance %s retains the assignment, %s", action, pending.Err, pending.InstanceID, retainedGuidance) + innerCode := "vgpu_retained_instance" + if !pending.Retained { + message = fmt.Sprintf("failed to %s instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", action, pending.Err, pending.InstanceID) + innerCode = "vgpu_unretained_instance" + } + return message, &oapi.ErrorDetail{ + Code: lo.ToPtr(innerCode), + Message: lo.ToPtr(pending.InstanceID), + } +} + // GetInstance gets instance details // The id parameter can be an instance ID, name, or ID prefix // Note: Resolution is handled by ResolveResource middleware @@ -859,19 +867,11 @@ func (s *ApiService) StartInstance(ctx context.Context, request oapi.StartInstan // errors.Is case would match the cause and hide the pending vGPU cleanup. case errors.As(err, &vgpuPending): log.ErrorContext(ctx, "failed to start instance", "error", err) - message := fmt.Sprintf("failed to start instance: %v; vGPU release failed during rollback and instance %s retains the assignment, delete it or retry start to release it", vgpuPending.Err, vgpuPending.InstanceID) - innerCode := "vgpu_retained_instance" - if !vgpuPending.Retained { - message = fmt.Sprintf("failed to start instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.Err, vgpuPending.InstanceID) - innerCode = "vgpu_unretained_instance" - } + message, inner := vgpuCleanupPendingDetail(vgpuPending, "start", "delete it or retry start to release it") return oapi.StartInstance500JSONResponse{ - Code: "vgpu_cleanup_pending", - Message: message, - InnerError: &oapi.ErrorDetail{ - Code: lo.ToPtr(innerCode), - Message: lo.ToPtr(vgpuPending.InstanceID), - }, + Code: "vgpu_cleanup_pending", + Message: message, + InnerError: inner, }, nil case errors.Is(err, instances.ErrInvalidState): return oapi.StartInstance409JSONResponse{ From 4d21b0ba5c38b7e44996efc909d023afcf26b65e Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 01:27:54 +0000 Subject: [PATCH 079/107] Narrow the vGPU reconcile interface --- cmd/api/main.go | 10 +++++++++- cmd/api/main_test.go | 5 +++++ lib/instances/manager.go | 1 - 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 71e147ecf..a4db0f3e9 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -185,8 +185,16 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta }, logger), nil } +type vgpuReconcileInstanceLister interface { + ListInstancesForReconcile(context.Context) ([]instances.Instance, error) +} + func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, time.Duration, error) { - allInstances, err := instanceManager.ListInstancesForReconcile(ctx) + lister, ok := instanceManager.(vgpuReconcileInstanceLister) + if !ok { + return nil, 0, errors.New("instance manager does not support vGPU reconcile inventory") + } + allInstances, err := lister.ListInstancesForReconcile(ctx) if err != nil { return nil, 0, err } diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 85c132814..0718b4cf3 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -351,6 +351,11 @@ func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([] return s.list, nil } +func TestLiveInstanceVGPUDevicePathsRequiresReconcileInventory(t *testing.T) { + _, _, err := liveInstanceVGPUDevicePaths(context.Background(), struct{ instances.Manager }{}) + require.ErrorContains(t, err, "does not support vGPU reconcile inventory") +} + func TestLiveInstanceVGPUDevicePathsBoundsStartupProtection(t *testing.T) { dead := exec.Command("true") require.NoError(t, dead.Run()) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 54e9b00d6..f3da80470 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -29,7 +29,6 @@ import ( type Manager interface { ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) - ListInstancesForReconcile(ctx context.Context) ([]Instance, error) ListSnapshots(ctx context.Context, filter *ListSnapshotsFilter) ([]Snapshot, error) GetSnapshot(ctx context.Context, snapshotID string) (*Snapshot, error) CreateInstance(ctx context.Context, req CreateInstanceRequest) (*Instance, error) From 34deb09f5fe4957a5e8e6ce24f23f21e7e42304f Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:54:32 +0000 Subject: [PATCH 080/107] Encapsulate vGPU rollback retention --- lib/instances/create.go | 73 ++----------------- lib/instances/lifecycle_noop_test.go | 2 +- lib/instances/start.go | 17 ++--- lib/instances/vgpu.go | 95 +++++++++++++----------- lib/instances/vgpu_retention.go | 104 +++++++++++++++++++++++++++ lib/instances/vgpu_test.go | 46 ++++++++++-- 6 files changed, 212 insertions(+), 125 deletions(-) create mode 100644 lib/instances/vgpu_retention.go diff --git a/lib/instances/create.go b/lib/instances/create.go index 66c988b4f..f4ece0100 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -279,21 +279,13 @@ func (m *manager) createInstance( var gpuMdevUUID string var gpuAssignedAt *time.Time var stored *StoredMetadata - var retainedVGPU *StoredMetadata + retention := vgpuRetention{instanceID: id} // Setup cleanup stack early so device attachment errors trigger cleanup. - // When rollback cannot release a vGPU assignment, report whether its - // retention record was persisted. The wrapping defer is registered first - // so it runs after cu.Clean has attempted to retain the metadata. - vgpuPersisted := false - defer func() { - if retErr != nil && retainedVGPU != nil { - retErr = &VGPUCleanupPendingError{InstanceID: id, Retained: vgpuPersisted, Err: retErr} - } - }() + defer retention.deferWrapPending(&retErr) cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) - vgpuPersisted = m.cleanupFailedCreate(ctx, id, retainedVGPU) + m.persistVGPURetention(ctx, &retention) }) defer cu.Clean() @@ -329,7 +321,7 @@ func (m *manager) createInstance( log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) if err != nil { - retainedVGPU = retainedVGPUFromCreateError(retentionStub(), m.nowUTC(), err) + retention.retainFromCreateError(retentionStub(), m.nowUTC(), err) log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) return nil, wrapCreateVGPUErr(req.GPU.Profile, err) } @@ -352,9 +344,9 @@ func (m *manager) createInstance( } if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) - retainedVGPU = stored - if retainedVGPU == nil { - retainedVGPU = retainedVGPUFromDevice(retentionStub(), gpuDevice, *gpuAssignedAt) + retention.retain(stored) + if stored == nil { + retention.retainFromDevice(retentionStub(), gpuDevice, *gpuAssignedAt) } } }) @@ -647,57 +639,6 @@ func resolveCreateExpiration(req CreateInstanceRequest, now time.Time) (*time.Ti return &expiresAt, nil } -// cleanupFailedCreate reports whether the retention record for a vGPU -// assignment whose release failed during rollback was persisted. -func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) bool { - if retainedVGPU == nil { - m.deleteInstanceData(id) - return false - } - - log := logger.FromContext(ctx) - retentionSurvives := func() bool { - meta, err := m.loadMetadata(id) - if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { - return true - } - if err := m.deleteInstanceData(id); err != nil { - log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) - } - return false - } - if err := m.ensureDirectories(id); err != nil { - log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) - return retentionSurvives() - } - // Retain identity fields so the instance lists as a recognizable, - // deletable record rather than a nameless phantom, but drop resource - // claims (network, volumes, devices) that rollback already released. - retained := StoredMetadata{ - Id: id, - Name: retainedVGPU.Name, - Image: retainedVGPU.Image, - ResolvedImage: retainedVGPU.ResolvedImage, - Platform: retainedVGPU.Platform, - CreatedAt: retainedVGPU.CreatedAt, - HypervisorType: retainedVGPU.HypervisorType, - HypervisorVersion: retainedVGPU.HypervisorVersion, - SocketPath: retainedVGPU.SocketPath, - DataDir: retainedVGPU.DataDir, - GPUProfile: retainedVGPU.GPUProfile, - GPUFramework: retainedVGPU.GPUFramework, - GPUDevicePath: retainedVGPU.GPUDevicePath, - GPUMdevUUID: retainedVGPU.GPUMdevUUID, - GPUAssignedAt: retainedVGPU.GPUAssignedAt, - GPURetainedForCleanup: true, - } - if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { - log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) - return retentionSurvives() - } - return true -} - // validateCreateRequest validates the create instance request. // The request is mutated in-place to persist normalized egress/credential policy fields. func validateCreateRequest(req *CreateInstanceRequest) error { diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 46f891379..a43efabb0 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -201,7 +201,7 @@ func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) { // A failed create whose vGPU release also failed retains a minimal // GPU-fields-only stub, and the API tells the caller to delete it to retry // the release. Exercise that recovery path against the exact stub shape -// cleanupFailedCreate writes. +// persistVGPURetention writes. func TestDeleteReleasesRetainedCreateStub(t *testing.T) { p := paths.New(t.TempDir()) var destroyed []devices.VGPUAssignment diff --git a/lib/instances/start.go b/lib/instances/start.go index c7eb85493..a79663953 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -121,16 +121,8 @@ func (m *manager) startInstance( } // Setup cleanup stack for automatic rollback on errors - // Registered before cu.Clean so it runs after cleanup and can report a - // vGPU assignment that rollback failed to destroy, matching create's - // vgpu_cleanup_pending contract. - vgpuRetained := false - vgpuRetentionPersisted := false - defer func() { - if retErr != nil && vgpuRetained { - retErr = &VGPUCleanupPendingError{InstanceID: id, Retained: vgpuRetentionPersisted, Err: retErr} - } - }() + retention := vgpuRetention{instanceID: id} + defer retention.deferWrapPending(&retErr) cu := cleanup.Make(func() {}) defer cu.Clean() @@ -204,7 +196,10 @@ func (m *manager) startInstance( log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", stored.GPUProfile, "uuid", device.MdevUUID) // Add vGPU cleanup to stack cu.Add(func() { - vgpuRetained, vgpuRetentionPersisted = m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) + retained, persisted := m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) + if retained { + retention.markRetained(persisted) + } }) if err := m.saveMetadata(meta); err != nil { log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index dcb6cff4d..3561cbc5a 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -54,24 +54,18 @@ func vgpuDevicePendingCleanup(err error) (*devices.VGPUDevice, bool) { return &pending.Device, true } -// retainedVGPUFromCreateError fills stub with the pending device's assignment -// fields when err carries a failed device-layer cleanup. The caller provides -// identity fields on stub so the retained record lists as a recognizable, -// deletable instance. -func retainedVGPUFromCreateError(stub StoredMetadata, assignedAt time.Time, err error) *StoredMetadata { - device, ok := vgpuDevicePendingCleanup(err) - if !ok { - return nil +func vgpuAssignmentLiveness(stored *StoredMetadata, now time.Time, livePID bool) (live bool, graceRemaining time.Duration) { + if stored.HypervisorPID != nil && livePID { + return true, 0 } - return retainedVGPUFromDevice(stub, device, assignedAt) -} - -// retainedVGPUFromDevice fills stub with device's assignment fields so a -// failed rollback release retains a recognizable, deletable record. -func retainedVGPUFromDevice(stub StoredMetadata, device *devices.VGPUDevice, assignedAt time.Time) *StoredMetadata { - stub.GPUProfile = device.ProfileName - setStoredVGPUDevice(&stub, device, assignedAt) - return &stub + if stored.GPUAssignedAt == nil { + return false, 0 + } + remaining := VGPUAssignmentStartupGracePeriod - now.Sub(*stored.GPUAssignedAt) + if remaining <= 0 { + return false, 0 + } + return true, remaining } func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { @@ -96,15 +90,9 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUAssignedAt = nil } -// cleanupStartVGPU wholesale-restores the pre-start metadata snapshot. The -// cleanup stack is LIFO, so cleanups registered after this one run before it -// and this restore would clobber anything they persisted; it is safe only -// while no such cleanup writes metadata and the instance lock serializes -// start. Violating that requires switching to targeted field restores. -// -// It reports whether the assignment was retained after a failed destroy and -// whether that retention record was persisted, so start can surface the -// pending cleanup as a typed error like create does. +// cleanupStartVGPU reports whether the assignment was retained after a failed +// destroy and whether that retention record was persisted, so start can surface +// the pending cleanup as a typed error like create does. func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) (retained, persisted bool) { logger.FromContext(ctx).DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID) assignment := devices.VGPUAssignment{ @@ -113,14 +101,20 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic MdevUUID: device.MdevUUID, InstanceID: instanceID, } - cleanupMeta := rollbackMeta + cleanupMeta, err := m.loadMetadata(instanceID) + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to load current metadata for vGPU cleanup; restoring rollback snapshot", "instance_id", instanceID, "error", err) + cleanupMeta = &rollbackMeta + } else { + restoreStartMutatedFields(&cleanupMeta.StoredMetadata, &rollbackMeta.StoredMetadata) + } releaseErr := m.destroyVGPUAssignment(ctx, assignment) if releaseErr != nil { logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID, "error", releaseErr) setStoredVGPUDevice(&cleanupMeta.StoredMetadata, device, assignedAt) retained = true } - if err := m.saveMetadata(&cleanupMeta); err != nil { + if err := m.saveMetadata(cleanupMeta); err != nil { message := "failed to save metadata after vGPU cleanup" if releaseErr != nil { message = "failed to retain vGPU assignment metadata after cleanup failure" @@ -141,6 +135,27 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic return retained, retained } +// restoreStartMutatedFields must cover every field start mutates before the +// vGPU cleanup runs. +func restoreStartMutatedFields(dst, src *StoredMetadata) { + dst.HypervisorPID = src.HypervisorPID + dst.HypervisorStartTime = src.HypervisorStartTime + dst.HypervisorBootID = src.HypervisorBootID + dst.ExitCode = src.ExitCode + dst.ExitMessage = src.ExitMessage + dst.ProgramStartedAt = src.ProgramStartedAt + dst.GuestAgentReadyAt = src.GuestAgentReadyAt + dst.Entrypoint = src.Entrypoint + dst.Cmd = src.Cmd + dst.IP = src.IP + dst.MAC = src.MAC + dst.GPUFramework = src.GPUFramework + dst.GPUDevicePath = src.GPUDevicePath + dst.GPUMdevUUID = src.GPUMdevUUID + dst.GPUAssignedAt = src.GPUAssignedAt + dst.StartedAt = src.StartedAt +} + func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { path := storedVGPUDevicePath(stored) if path != "" { @@ -206,23 +221,21 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu if storedVGPUDevicePath(stored) != devicePath { continue } - if stored.HypervisorPID == nil { - if stored.GPUAssignedAt == nil || time.Since(*stored.GPUAssignedAt) >= VGPUAssignmentStartupGracePeriod { - continue + pid := 0 + if stored.HypervisorPID != nil { + pid, err = resolveLiveHypervisorPID(stored.HypervisorProcessIdentity, stored.SocketPath) + if err != nil { + return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) } - return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) } - pid, err := resolveLiveHypervisorPID(stored.HypervisorProcessIdentity, stored.SocketPath) - if err != nil { - return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) - } - if pid > 0 { + live, remaining := vgpuAssignmentLiveness(stored, time.Now(), pid > 0) + if pid > 0 && live { return true, nil } - // A dead PID with a recent assignment gets the same bounded grace as - // startup reconcile protection, so the two guards agree in the - // fail-closed direction while a mid-boot claimant hydrates. - if stored.GPUAssignedAt != nil && time.Since(*stored.GPUAssignedAt) < VGPUAssignmentStartupGracePeriod { + if remaining > 0 { + if stored.HypervisorPID == nil { + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) + } return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: recorded hypervisor is not running", id, devicePath) } } diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go new file mode 100644 index 000000000..a1a1aae0d --- /dev/null +++ b/lib/instances/vgpu_retention.go @@ -0,0 +1,104 @@ +package instances + +import ( + "context" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/logger" +) + +type vgpuRetention struct { + instanceID string + stub *StoredMetadata + retained bool + persisted bool +} + +func (r *vgpuRetention) retainFromCreateError(stub StoredMetadata, assignedAt time.Time, err error) { + device, ok := vgpuDevicePendingCleanup(err) + if !ok { + return + } + r.retainFromDevice(stub, device, assignedAt) +} + +func (r *vgpuRetention) retainFromDevice(stub StoredMetadata, device *devices.VGPUDevice, assignedAt time.Time) { + stub.GPUProfile = device.ProfileName + setStoredVGPUDevice(&stub, device, assignedAt) + r.stub = &stub + r.retained = true +} + +func (r *vgpuRetention) retain(stub *StoredMetadata) { + r.stub = stub + r.retained = stub != nil +} + +func (r *vgpuRetention) markRetained(persisted bool) { + r.retained = true + r.persisted = persisted +} + +func (r *vgpuRetention) wrapPending(err error) error { + if err == nil || !r.retained { + return err + } + return &VGPUCleanupPendingError{InstanceID: r.instanceID, Retained: r.persisted, Err: err} +} + +// deferWrapPending must be deferred before cleanup so it observes retention +// state recorded by rollback. +func (r *vgpuRetention) deferWrapPending(retErr *error) { + *retErr = r.wrapPending(*retErr) +} + +func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuRetention) { + if retention.stub == nil { + m.deleteInstanceData(retention.instanceID) + return + } + + id := retention.instanceID + retainedVGPU := retention.stub + log := logger.FromContext(ctx) + retentionSurvives := func() bool { + meta, err := m.loadMetadata(id) + if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { + return true + } + if err := m.deleteInstanceData(id); err != nil { + log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) + } + return false + } + if err := m.ensureDirectories(id); err != nil { + log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) + retention.persisted = retentionSurvives() + return + } + retained := StoredMetadata{ + Id: id, + Name: retainedVGPU.Name, + Image: retainedVGPU.Image, + ResolvedImage: retainedVGPU.ResolvedImage, + Platform: retainedVGPU.Platform, + CreatedAt: retainedVGPU.CreatedAt, + HypervisorType: retainedVGPU.HypervisorType, + HypervisorVersion: retainedVGPU.HypervisorVersion, + SocketPath: retainedVGPU.SocketPath, + DataDir: retainedVGPU.DataDir, + GPUProfile: retainedVGPU.GPUProfile, + GPUFramework: retainedVGPU.GPUFramework, + GPUDevicePath: retainedVGPU.GPUDevicePath, + GPUMdevUUID: retainedVGPU.GPUMdevUUID, + GPUAssignedAt: retainedVGPU.GPUAssignedAt, + GPURetainedForCleanup: true, + } + if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { + log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) + retention.persisted = retentionSurvives() + return + } + retention.persisted = true +} diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 1d0e4e3bf..c5ef4911b 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -18,6 +18,19 @@ import ( "github.com/stretchr/testify/require" ) +func persistTestVGPURetention(m *manager, ctx context.Context, id string, stub *StoredMetadata) bool { + retention := vgpuRetention{instanceID: id} + retention.retain(stub) + m.persistVGPURetention(ctx, &retention) + return retention.persisted +} + +func retainedVGPUFromCreateErrorForTest(stub StoredMetadata, assignedAt time.Time, err error) *StoredMetadata { + retention := vgpuRetention{} + retention.retainFromCreateError(stub, assignedAt, err) + return retention.stub +} + func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { t.Parallel() @@ -38,7 +51,7 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { DataDir: m.paths.InstanceDir("failed-create"), } - assert.True(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored)) + assert.True(t, persistTestVGPURetention(m, context.Background(), stored.Id, stored)) retained, err := m.loadMetadata(stored.Id) require.NoError(t, err) @@ -67,7 +80,7 @@ func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { m := &manager{paths: paths.New(t.TempDir())} require.NoError(t, m.ensureDirectories("failed-create")) - assert.False(t, m.cleanupFailedCreate(context.Background(), "failed-create", nil)) + assert.False(t, persistTestVGPURetention(m, context.Background(), "failed-create", nil)) _, err := m.loadMetadata("failed-create") require.Error(t, err) } @@ -85,7 +98,7 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", } - assert.False(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored)) + assert.False(t, persistTestVGPURetention(m, context.Background(), stored.Id, stored)) _, err := m.loadMetadata(id) require.Error(t, err) } @@ -109,13 +122,31 @@ func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T require.NoError(t, os.Chmod(instanceDir, 0o555)) t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) - assert.True(t, m.cleanupFailedCreate(context.Background(), id, stored)) + assert.True(t, persistTestVGPURetention(m, context.Background(), id, stored)) retained, err := m.loadMetadata(id) require.NoError(t, err) assert.Equal(t, stored.GPUFramework, retained.GPUFramework) assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) } +func TestVGPURetentionWrapPending(t *testing.T) { + cause := errors.New("boot failed") + + retention := vgpuRetention{instanceID: "inst-1"} + assert.Same(t, cause, retention.wrapPending(cause)) + + retention.retained = true + pending := retention.wrapPending(cause) + var cleanupPending *VGPUCleanupPendingError + require.ErrorAs(t, pending, &cleanupPending) + assert.False(t, cleanupPending.Retained) + + retention.persisted = true + pending = retention.wrapPending(cause) + require.ErrorAs(t, pending, &cleanupPending) + assert.True(t, cleanupPending.Retained) +} + func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { t.Parallel() @@ -145,7 +176,7 @@ func TestVGPUDevicePendingCleanup(t *testing.T) { assert.Equal(t, device, *actual) assignedAt := time.Now().UTC() - retained := retainedVGPUFromCreateError(StoredMetadata{Id: "inst-1", Name: "named", Image: "img"}, assignedAt, wrapped) + retained := retainedVGPUFromCreateErrorForTest(StoredMetadata{Id: "inst-1", Name: "named", Image: "img"}, assignedAt, wrapped) require.NotNil(t, retained) assert.Equal(t, "inst-1", retained.Id) assert.Equal(t, "named", retained.Name, "identity fields must survive into the retention stub") @@ -157,7 +188,7 @@ func TestVGPUDevicePendingCleanup(t *testing.T) { actual, ok = vgpuDevicePendingCleanup(cause) assert.False(t, ok) assert.Nil(t, actual) - assert.Nil(t, retainedVGPUFromCreateError(StoredMetadata{Id: "inst-1"}, assignedAt, cause)) + assert.Nil(t, retainedVGPUFromCreateErrorForTest(StoredMetadata{Id: "inst-1"}, assignedAt, cause)) } type startRetentionNetworkManager struct { @@ -427,6 +458,7 @@ func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { exitCode := 1 rollbackMeta := metadata{StoredMetadata: StoredMetadata{ Id: id, + Name: "original name", GPUProfile: "NVIDIA L40S-2Q", Entrypoint: []string{"old-entrypoint"}, Cmd: []string{"old-command"}, @@ -437,6 +469,7 @@ func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { }} partial := rollbackMeta + partial.Name = "concurrent update" partial.Entrypoint = []string{"new-entrypoint"} partial.Cmd = []string{"new-command"} partial.StartedAt = ptr(time.Now().UTC()) @@ -455,6 +488,7 @@ func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { stored, err := m.loadMetadata(id) require.NoError(t, err) + assert.Equal(t, "concurrent update", stored.Name) assert.Equal(t, rollbackMeta.Entrypoint, stored.Entrypoint) assert.Equal(t, rollbackMeta.Cmd, stored.Cmd) assert.Equal(t, rollbackMeta.StartedAt, stored.StartedAt) From 61c563e6d3d8f106ec7144302420716f0b242dea Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:54:32 +0000 Subject: [PATCH 081/107] Move vGPU reconciliation into instance manager --- cmd/api/main.go | 67 +------------------------- cmd/api/main_test.go | 41 ---------------- lib/builds/manager_test.go | 4 +- lib/devices/mdev_darwin.go | 2 +- lib/devices/vgpu_linux.go | 26 ++++++++-- lib/devices/vgpu_linux_test.go | 16 +++++++ lib/instances/manager.go | 1 + lib/instances/vgpu_reconcile.go | 61 ++++++++++++++++++++++++ lib/instances/vgpu_reconcile_test.go | 71 ++++++++++++++++++++++++++++ lib/instances/wait_test.go | 4 +- 10 files changed, 175 insertions(+), 118 deletions(-) create mode 100644 lib/instances/vgpu_reconcile.go create mode 100644 lib/instances/vgpu_reconcile_test.go diff --git a/cmd/api/main.go b/cmd/api/main.go index a4db0f3e9..da2748103 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -185,71 +185,6 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta }, logger), nil } -type vgpuReconcileInstanceLister interface { - ListInstancesForReconcile(context.Context) ([]instances.Instance, error) -} - -func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, time.Duration, error) { - lister, ok := instanceManager.(vgpuReconcileInstanceLister) - if !ok { - return nil, 0, errors.New("instance manager does not support vGPU reconcile inventory") - } - allInstances, err := lister.ListInstancesForReconcile(ctx) - if err != nil { - return nil, 0, err - } - protected := make(map[string]struct{}) - var retryAfter time.Duration - for _, inst := range allInstances { - if inst.GPUDevicePath == "" { - continue - } - if inst.HypervisorPID != nil && instances.HypervisorMayBeAlive(inst.HypervisorProcessIdentity, inst.SocketPath) { - protected[inst.GPUDevicePath] = struct{}{} - continue - } - if inst.GPUAssignedAt == nil { - continue - } - remaining := instances.VGPUAssignmentStartupGracePeriod - time.Since(*inst.GPUAssignedAt) - if remaining <= 0 { - continue - } - protected[inst.GPUDevicePath] = struct{}{} - if retryAfter == 0 || remaining < retryAfter { - retryAfter = remaining - } - } - return protected, retryAfter, nil -} - -func reconcileVGPUs(ctx context.Context, instanceManager instances.Manager, logger *slog.Logger) { - protected, retryAfter, err := liveInstanceVGPUDevicePaths(ctx, instanceManager) - if err != nil { - // Operator-actionable: vendor VFIO reconciliation stays disabled - // host-wide (and releases fail closed on the same inventory) until - // the unreadable instance metadata is repaired. - logger.Error("failed to list instances for vGPU reconcile protection; reconciling mdev only", "error", err) - protected = nil - retryAfter = 0 - } - if err := devices.ReconcileVGPUs(ctx, protected); err != nil { - logger.Warn("failed to reconcile vGPU devices", "error", err) - } - if retryAfter <= 0 { - return - } - go func() { - timer := time.NewTimer(retryAfter) - defer timer.Stop() - select { - case <-ctx.Done(): - case <-timer.C: - reconcileVGPUs(ctx, instanceManager, logger) - } - }() -} - func run() error { startupStarted := time.Now() slog.Info("starting hypeman initialization") @@ -451,7 +386,7 @@ func run() error { // Reconcile vGPU devices (clears orphaned vGPUs from previous runs) logger.Info("Reconciling vGPU devices...") - reconcileVGPUs(ctx, app.InstanceManager, logger) + app.InstanceManager.ReconcileVGPUs(ctx) // Wire up resource validator for aggregate limit checking // This enables the instance manager to validate CPU, memory, network, and GPU diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 0718b4cf3..b771e27fc 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -6,7 +6,6 @@ import ( "net/http" "net/http/httptest" "net/url" - "os/exec" "testing" "time" @@ -341,43 +340,3 @@ func TestImageNameWithSlashes_URLEncoding(t *testing.T) { }) } } - -type vgpuReconcileManagerStub struct { - instances.Manager - list []instances.Instance -} - -func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([]instances.Instance, error) { - return s.list, nil -} - -func TestLiveInstanceVGPUDevicePathsRequiresReconcileInventory(t *testing.T) { - _, _, err := liveInstanceVGPUDevicePaths(context.Background(), struct{ instances.Manager }{}) - require.ErrorContains(t, err, "does not support vGPU reconcile inventory") -} - -func TestLiveInstanceVGPUDevicePathsBoundsStartupProtection(t *testing.T) { - dead := exec.Command("true") - require.NoError(t, dead.Run()) - deadPID := dead.Process.Pid - recent := time.Now().Add(-time.Minute) - stale := time.Now().Add(-instances.VGPUAssignmentStartupGracePeriod - time.Minute) - - manager := vgpuReconcileManagerStub{list: []instances.Instance{ - {StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}}, - {StoredMetadata: instances.StoredMetadata{Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}}, - {StoredMetadata: instances.StoredMetadata{Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}}, - {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorProcessIdentity: instances.HypervisorProcessIdentity{HypervisorPID: &deadPID}}}, - {StoredMetadata: instances.StoredMetadata{Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorProcessIdentity: instances.HypervisorProcessIdentity{HypervisorPID: &deadPID}, GPUAssignedAt: &recent}}, - }} - - protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager) - require.NoError(t, err) - require.Positive(t, retryAfter) - require.LessOrEqual(t, retryAfter, instances.VGPUAssignmentStartupGracePeriod) - assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") - assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") - assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") - assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") - assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") -} diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index 44596bf68..ab390c72f 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -51,9 +51,7 @@ func (m *mockInstanceManager) ListInstances(ctx context.Context, filter *instanc return result, nil } -func (m *mockInstanceManager) ListInstancesForReconcile(ctx context.Context) ([]instances.Instance, error) { - return m.ListInstances(ctx, nil) -} +func (m *mockInstanceManager) ReconcileVGPUs(context.Context) {} func (m *mockInstanceManager) ListSnapshots(ctx context.Context, filter *instances.ListSnapshotsFilter) ([]instances.Snapshot, error) { return nil, nil diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index 4274063ed..4b726bb08 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -58,7 +58,7 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { return nil } -func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}) error { +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepVendorVFIO bool) error { return nil } diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index be92e8ee6..72827f7b2 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -105,20 +105,38 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { } // ReconcileVGPUs releases orphaned vGPU assignments. -func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}) error { +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepVendorVFIO bool) error { framework, _, err := DiscoverVGPU() if err != nil { return err } + return reconcileDiscoveredVGPUs( + ctx, + framework, + protectedDevicePaths, + sweepVendorVFIO, + func(ctx context.Context) error { return ReconcileMdevs(ctx, nil) }, + hostVendorVFIO.reconcile, + ) +} + +func reconcileDiscoveredVGPUs( + ctx context.Context, + framework VGPUFramework, + protectedDevicePaths map[string]struct{}, + sweepVendorVFIO bool, + reconcileMdev func(context.Context) error, + reconcileVendorVFIO func(context.Context, map[string]struct{}) error, +) error { switch framework { case VGPUFrameworkMdev: - return ReconcileMdevs(ctx, nil) + return reconcileMdev(ctx) case VGPUFrameworkVendorVFIO: - if protectedDevicePaths == nil { + if !sweepVendorVFIO { return nil } - return hostVendorVFIO.reconcile(ctx, protectedDevicePaths) + return reconcileVendorVFIO(ctx, protectedDevicePaths) default: return nil } diff --git a/lib/devices/vgpu_linux_test.go b/lib/devices/vgpu_linux_test.go index 7b03d9c26..32f9a30ea 100644 --- a/lib/devices/vgpu_linux_test.go +++ b/lib/devices/vgpu_linux_test.go @@ -3,6 +3,7 @@ package devices import ( + "context" "errors" "os" "path/filepath" @@ -12,6 +13,21 @@ import ( "github.com/stretchr/testify/require" ) +func TestReconcileDiscoveredVGPUsControlsVendorSweep(t *testing.T) { + protected := make(map[string]struct{}) + vendorCalls := 0 + reconcileVendor := func(context.Context, map[string]struct{}) error { + vendorCalls++ + return nil + } + + require.NoError(t, reconcileDiscoveredVGPUs(context.Background(), VGPUFrameworkVendorVFIO, protected, false, nil, reconcileVendor)) + assert.Zero(t, vendorCalls) + + require.NoError(t, reconcileDiscoveredVGPUs(context.Background(), VGPUFrameworkVendorVFIO, protected, true, nil, reconcileVendor)) + assert.Equal(t, 1, vendorCalls) +} + func TestDiscoverVGPUWithPropagatesMdevError(t *testing.T) { t.Parallel() diff --git a/lib/instances/manager.go b/lib/instances/manager.go index f3da80470..63e772c3e 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -29,6 +29,7 @@ import ( type Manager interface { ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) + ReconcileVGPUs(ctx context.Context) ListSnapshots(ctx context.Context, filter *ListSnapshotsFilter) ([]Snapshot, error) GetSnapshot(ctx context.Context, snapshotID string) (*Snapshot, error) CreateInstance(ctx context.Context, req CreateInstanceRequest) (*Instance, error) diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go new file mode 100644 index 000000000..85d3a6e0d --- /dev/null +++ b/lib/instances/vgpu_reconcile.go @@ -0,0 +1,61 @@ +package instances + +import ( + "context" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/logger" +) + +func (m *manager) liveVGPUReconcileProtection(ctx context.Context) (map[string]struct{}, time.Duration, error) { + allInstances, err := m.ListInstancesForReconcile(ctx) + if err != nil { + return nil, 0, err + } + protected := make(map[string]struct{}) + var retryAfter time.Duration + for i := range allInstances { + stored := &allInstances[i].StoredMetadata + if stored.GPUDevicePath == "" { + continue + } + livePID := stored.HypervisorPID != nil && HypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + live, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID) + if !live { + continue + } + protected[stored.GPUDevicePath] = struct{}{} + if remaining > 0 && (retryAfter == 0 || remaining < retryAfter) { + retryAfter = remaining + } + } + return protected, retryAfter, nil +} + +// ReconcileVGPUs releases orphaned vGPU assignments. +func (m *manager) ReconcileVGPUs(ctx context.Context) { + log := logger.FromContext(ctx) + protected, retryAfter, err := m.liveVGPUReconcileProtection(ctx) + sweepVendorVFIO := err == nil + if err != nil { + log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconcile, mdev reconcile still runs", "error", err) + protected = make(map[string]struct{}) + retryAfter = 0 + } + if err := devices.ReconcileVGPUs(ctx, protected, sweepVendorVFIO); err != nil { + log.WarnContext(ctx, "failed to reconcile vGPU devices", "error", err) + } + if retryAfter <= 0 { + return + } + go func() { + timer := time.NewTimer(retryAfter) + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + m.ReconcileVGPUs(ctx) + } + }() +} diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go new file mode 100644 index 000000000..1fbd90384 --- /dev/null +++ b/lib/instances/vgpu_reconcile_test.go @@ -0,0 +1,71 @@ +package instances + +import ( + "os/exec" + "testing" + "time" + + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLiveVGPUReconcileProtectionBoundsStartupProtection(t *testing.T) { + dead := exec.Command("true") + require.NoError(t, dead.Run()) + deadPID := dead.Process.Pid + now := time.Now().UTC() + recent := now.Add(-time.Minute) + stale := now.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + + m := &manager{paths: paths.New(t.TempDir()), now: func() time.Time { return now }} + instances := []StoredMetadata{ + {Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}, + {Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}, + {Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}, + {Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}}, + {Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, GPUAssignedAt: &recent}, + } + for i := range instances { + require.NoError(t, m.ensureDirectories(instances[i].Id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: instances[i]})) + } + + protected, retryAfter, err := m.liveVGPUReconcileProtection(t.Context()) + require.NoError(t, err) + assert.Equal(t, VGPUAssignmentStartupGracePeriod-time.Minute, retryAfter) + assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") + assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") +} + +func TestVGPUAssignmentLiveness(t *testing.T) { + now := time.Now().UTC() + recent := now.Add(-time.Minute) + stale := now.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + pid := 123 + + tests := []struct { + name string + stored StoredMetadata + livePID bool + live bool + remaining time.Duration + }{ + {name: "live PID", stored: StoredMetadata{HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}}, livePID: true, live: true}, + {name: "dead PID recent assignment", stored: StoredMetadata{HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, GPUAssignedAt: &recent}, live: true, remaining: VGPUAssignmentStartupGracePeriod - time.Minute}, + {name: "dead PID stale assignment", stored: StoredMetadata{HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, GPUAssignedAt: &stale}}, + {name: "no PID recent assignment", stored: StoredMetadata{GPUAssignedAt: &recent}, live: true, remaining: VGPUAssignmentStartupGracePeriod - time.Minute}, + {name: "no PID stale assignment", stored: StoredMetadata{GPUAssignedAt: &stale}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + live, remaining := vgpuAssignmentLiveness(&tt.stored, now, tt.livePID) + assert.Equal(t, tt.live, live) + assert.Equal(t, tt.remaining, remaining) + }) + } +} diff --git a/lib/instances/wait_test.go b/lib/instances/wait_test.go index 415003594..3ab02e402 100644 --- a/lib/instances/wait_test.go +++ b/lib/instances/wait_test.go @@ -32,9 +32,7 @@ func (s *stubManager) GetInstance(ctx context.Context, id string) (*Instance, er func (s *stubManager) ListInstances(context.Context, *ListInstancesFilter) ([]Instance, error) { return nil, nil } -func (s *stubManager) ListInstancesForReconcile(context.Context) ([]Instance, error) { - return nil, nil -} +func (s *stubManager) ReconcileVGPUs(context.Context) {} func (s *stubManager) ListSnapshots(context.Context, *ListSnapshotsFilter) ([]Snapshot, error) { return nil, nil } From e417c85792748e28b38243009a17a4a0b7c86452 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:54:32 +0000 Subject: [PATCH 082/107] Derive QEMU VFIO grace from VM config --- cmd/api/main_test.go | 2 -- lib/hypervisor/qemu/process.go | 28 +++++++++------------------- lib/hypervisor/qemu/process_test.go | 17 +++++++++++++++++ lib/hypervisor/vfio.go | 7 +++++++ lib/instances/process_identity.go | 8 +------- 5 files changed, 34 insertions(+), 28 deletions(-) create mode 100644 lib/hypervisor/vfio.go diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index b771e27fc..34dbba428 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -2,7 +2,6 @@ package main import ( "bytes" - "context" "net/http" "net/http/httptest" "net/url" @@ -12,7 +11,6 @@ import ( "github.com/getkin/kin-openapi/openapi3filter" "github.com/go-chi/chi/v5" "github.com/golang-jwt/jwt/v5" - "github.com/kernel/hypeman/lib/instances" mw "github.com/kernel/hypeman/lib/middleware" "github.com/kernel/hypeman/lib/oapi" nethttpmiddleware "github.com/oapi-codegen/nethttp-middleware" diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index cddd51824..367bf14e3 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -40,12 +40,6 @@ const ( // socketDialTimeout is timeout for individual socket connection attempts socketDialTimeout = 100 * time.Millisecond - // vfioTermGrace is how long start-failure cleanup waits for a - // VFIO-attached QEMU to exit on SIGTERM before SIGKILL. Only failed - // starts pay it, and only when the process ignores SIGTERM; observed - // mid-init VFIO teardown takes 1-2s. - vfioTermGrace = 5 * time.Second - // clientCreateTimeout is how long to retry QMP client creation after the // socket appears. Under high parallel load the socket can accept connections // slightly later than file creation/availability. @@ -320,20 +314,17 @@ func (p *startedProcess) cleanup() { _ = os.Remove(p.socketPath) } -// hasVFIODevice reports whether the QEMU command line attaches a VFIO device. -func hasVFIODevice(args []string) bool { - for _, arg := range args { - if strings.Contains(arg, "vfio-pci") { - return true - } +func vfioTermGraceFor(cfg hypervisor.VMConfig) time.Duration { + if cfg.VGPUDevicePath != "" || len(cfg.PCIDevices) > 0 { + return hypervisor.VFIOTermGrace } - return false + return 0 } // startQEMUProcess handles the common QEMU process startup logic. // Returns the PID, hypervisor client, and a cleanup function. // The cleanup function must be called on error; call cleanup.Release() on success. -func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version string, socketPath string, args []string) (int, *QEMU, *cleanup.Cleanup, error) { +func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version string, socketPath string, args []string, termGrace time.Duration) (int, *QEMU, *cleanup.Cleanup, error) { log := logger.FromContext(ctx) processAttrs := hypervisor.TraceAttributesFromContext(ctx) processAttrs = append(processAttrs, @@ -403,9 +394,8 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version } pid := proc.pid - if hasVFIODevice(args) { - proc.termGrace = vfioTermGrace - } + // Only failed starts pay the VFIO termination grace. + proc.termGrace = termGrace log.DebugContext(processCtx, "QEMU process started", "pid", pid, "duration_ms", time.Since(processStartTime).Milliseconds()) // Setup cleanup to kill, reap, and remove the socket if subsequent steps fail. @@ -522,7 +512,7 @@ func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, s // Build command arguments: QMP socket + VM configuration args := buildQMPArgs(socketPath) args = append(args, buildArgs(attempt, machineType)...) - pid, hv, cu, err = s.startQEMUProcess(ctx, p, version, socketPath, args) + pid, hv, cu, err = s.startQEMUProcess(ctx, p, version, socketPath, args, vfioTermGraceFor(attempt)) if err == nil { booted = attempt started = true @@ -657,7 +647,7 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, incomingURI := "exec:cat < " + memoryFile args = append(args, "-incoming", incomingURI) - pid, hv, cu, err := s.startQEMUProcess(ctx, p, version, socketPath, args) + pid, hv, cu, err := s.startQEMUProcess(ctx, p, version, socketPath, args, vfioTermGraceFor(config)) if err != nil { return 0, nil, err } diff --git a/lib/hypervisor/qemu/process_test.go b/lib/hypervisor/qemu/process_test.go index 2bdf27d4f..ad08aea24 100644 --- a/lib/hypervisor/qemu/process_test.go +++ b/lib/hypervisor/qemu/process_test.go @@ -411,6 +411,23 @@ func TestWaitForSocketOrExitReturnsEarlyWhenProcessDies(t *testing.T) { assert.True(t, cmd.ProcessState.Exited()) } +func TestVFIOTermGraceFor(t *testing.T) { + tests := []struct { + name string + cfg hypervisor.VMConfig + want time.Duration + }{ + {name: "vGPU", cfg: hypervisor.VMConfig{VGPUDevicePath: "/sys/bus/mdev/devices/test"}, want: hypervisor.VFIOTermGrace}, + {name: "PCI device", cfg: hypervisor.VMConfig{PCIDevices: []string{"0000:01:00.0"}}, want: hypervisor.VFIOTermGrace}, + {name: "no VFIO device", cfg: hypervisor.VMConfig{}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, vfioTermGraceFor(tt.cfg)) + }) + } +} + func TestCleanupSIGTERMsProcessWithTermGrace(t *testing.T) { socketPath := filepath.Join(t.TempDir(), "qemu.sock") markerPath := filepath.Join(t.TempDir(), "terminated") diff --git a/lib/hypervisor/vfio.go b/lib/hypervisor/vfio.go new file mode 100644 index 000000000..1744cfc84 --- /dev/null +++ b/lib/hypervisor/vfio.go @@ -0,0 +1,7 @@ +package hypervisor + +import "time" + +// VFIOTermGrace allows VFIO teardown to finish after SIGTERM before SIGKILL; +// SIGKILL during initialization can wedge the VF, while teardown takes 1-2s. +const VFIOTermGrace = 5 * time.Second diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 0801aca6c..83aa682bc 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -27,19 +27,13 @@ const linuxBootIDPath = "/proc/sys/kernel/random/boot_id" // does not unstick it, so the wait is short to keep stop and delete fast. const hypervisorSIGKILLWaitTimeout = 2 * time.Second -// defaultVGPUInitTermGrace is how long terminateThenKill waits for a vGPU -// hypervisor to exit on SIGTERM before SIGKILL. Only force-kill paths pay it, -// only for vGPU instances, and only when the process ignores SIGTERM; -// observed mid-init VFIO teardown takes 1-2s. -const defaultVGPUInitTermGrace = 5 * time.Second - // vgpuTermGrace returns the SIGTERM wait used before hard-killing a vGPU // hypervisor. func (m *manager) vgpuTermGrace() time.Duration { if m.vgpuInitTermGrace > 0 { return m.vgpuInitTermGrace } - return defaultVGPUInitTermGrace + return hypervisor.VFIOTermGrace } // terminateThenKill hard-kills the hypervisor process, first giving any vGPU From 648373ddaab7a266f824d4dd6fbe305ba0d961bd Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:25:09 +0000 Subject: [PATCH 083/107] Tighten vGPU lifecycle comments --- lib/hypervisor/qemu/process.go | 6 ++---- lib/instances/manager.go | 11 ++++------- lib/instances/metrics.go | 5 ++--- lib/instances/process_identity.go | 18 +++++++----------- lib/instances/storage.go | 5 ++--- lib/instances/vgpu.go | 24 +++++++++--------------- lib/instances/vgpu_orphan.go | 20 +++++++++----------- 7 files changed, 35 insertions(+), 54 deletions(-) diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index 367bf14e3..de560c10d 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -228,10 +228,8 @@ type startedProcess struct { pid int socketPath string // termGrace, when non-zero, makes cleanup send SIGTERM and wait this long - // before SIGKILL. Set for VFIO-attached processes: hard-killing QEMU while - // the NVIDIA vGPU plugin is initializing can silently wedge the VF until - // its parent GPU's SR-IOV is cycled, while a terminating QEMU runs its - // device teardown and leaves the VF reusable. + // before SIGKILL. Set for VFIO-attached processes: SIGKILL during vGPU + // plugin init can wedge the VF until its parent GPU is SR-IOV cycled. termGrace time.Duration waitDone chan error waitConsumed bool diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 63e772c3e..3954af19f 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -752,9 +752,8 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { } // ListInstancesForReconcile returns every instance's stored metadata or an -// invalid metadata error. It does not derive state: reconcile protection only -// needs raw metadata fields, and hydration would query the hypervisor of -// every instance on the host before the API serves. +// invalid metadata error. It does not derive state: hydration would query +// every hypervisor on the host before the API serves. func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, error) { files, err := m.listMetadataFilesStrict() if err != nil { @@ -766,10 +765,8 @@ func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, er meta, err := m.loadMetadata(id) if err != nil { if errors.Is(err, ErrNotFound) { - // Deleted between listing and load; a vanished record cannot - // claim a VF. Failing here instead would zero the grace-period - // retry and disable the vendor VFIO sweep whenever it races a - // concurrent delete. + // Deleted between listing and load; failing instead would + // disable the vendor VFIO sweep whenever it races a delete. continue } return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index bdffbdbce..085216fdf 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -574,9 +574,8 @@ func (m *manager) recordTimeToRunning(ctx context.Context, stored *StoredMetadat } // recordVGPUOrphanReleaseAbandoned records an orphaned vGPU release retry -// loop giving up: the VF stays allocated (capacity silently reduced) until -// startup reconciliation or manual remediation, so it must be visible beyond -// a log line. +// loop giving up: the VF stays allocated until startup reconciliation or +// manual remediation, so it must be visible beyond a log line. func (m *manager) recordVGPUOrphanReleaseAbandoned(ctx context.Context) { if m.metrics == nil { return diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 83aa682bc..dd546c420 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -37,13 +37,10 @@ func (m *manager) vgpuTermGrace() time.Duration { } // terminateThenKill hard-kills the hypervisor process, first giving any vGPU -// instance a SIGTERM grace: SIGKILL during guest driver init can silently -// wedge the VF until its parent GPU's SR-IOV is cycled (see -// lib/devices/GPU.md), while a terminating QEMU runs its VFIO teardown. The -// grace applies in every state, not just Initializing, because the instance -// reports Running seconds before the guest driver finishes initializing and -// nothing host-side observes that boundary; post-init the SIGTERM is proven -// harmless and costs the grace only when the process ignores it. +// instance a SIGTERM grace: SIGKILL during guest driver init can wedge the VF +// until its parent GPU is SR-IOV cycled (see lib/devices/GPU.md). The grace +// applies in every state because the instance reports Running seconds before +// driver init finishes and nothing host-side observes that boundary. func (m *manager) terminateThenKill(ctx context.Context, inst *Instance, pid int) error { if inst.GPUProfile != "" { if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { @@ -211,10 +208,9 @@ func classifyResolvedHypervisorOwner(socketPath string, stored, resolved int, er } // HypervisorMayBeAlive reports whether the recorded hypervisor process may -// still be running. It fails open: when ownership cannot be resolved it -// returns true, which is the safe direction for its callers (reconcile -// protection and claim checks, where true means "protect"). Do not use it to -// authorize teardown. +// still be running. It fails open (unresolvable ownership returns true, the +// safe direction for reconcile protection and claim checks); do not use it +// to authorize teardown. func HypervisorMayBeAlive(id HypervisorProcessIdentity, socketPath string) bool { pid, err := resolveLiveHypervisorPID(id, socketPath) return err != nil || pid > 0 diff --git a/lib/instances/storage.go b/lib/instances/storage.go index 6a2354624..40bbba684 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -194,9 +194,8 @@ func (m *manager) listMetadataFiles() ([]string, error) { } // listMetadataFilesStrict returns paths to all instance metadata files, -// failing on any stat error other than absence. Fail-closed callers (the -// vGPU release claim scan and startup reconcile protection) use it so an -// unreadable instance is an error instead of silently missing. +// failing on any stat error other than absence, so fail-closed callers see +// an unreadable instance as an error instead of silently missing. func (m *manager) listMetadataFilesStrict() ([]string, error) { return m.walkMetadataFiles(true) } diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 3561cbc5a..8ffe64dd8 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -123,10 +123,8 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic if !retained { return false, false } - // The mid-start save may already have persisted this assignment, in - // which case the on-disk record still points at the device and - // delete or a retried start can release it (matching create's - // retention-survives check). + // The mid-start save may already have persisted this assignment, so + // delete or a retried start can still release it. if meta, loadErr := m.loadMetadata(instanceID); loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath { return true, true } @@ -159,11 +157,10 @@ func restoreStartMutatedFields(dst, src *StoredMetadata) { func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { path := storedVGPUDevicePath(stored) if path != "" { - // Vendor VFIO VFs are reused across instances, so stale metadata can - // point at a path claimed by a live instance and the release must fail - // closed on an incomplete inventory. mdev UUIDs are unique and never - // reused, so skip the scan there — it would let one unreadable - // metadata file block every mdev release on the host. + // Vendor VFIO VFs are reused across instances, so the release must + // fail closed on an incomplete inventory. mdev UUIDs are never reused; + // scanning there would let one unreadable metadata file block every + // mdev release on the host. claimed := false if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { var err error @@ -192,12 +189,9 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) } // vgpuAssignmentClaimedByLiveInstance reports whether another live instance's -// stored metadata claims devicePath. It reads raw metadata instead of -// hydrating full instances: the scan runs on every vendor VFIO release, and -// deriving state would query the hypervisor of every instance on the host. -// A confirmed live claimant returns true. Unreadable metadata, a recent -// assignment without a PID, or unverifiable process ownership returns an error -// so the requester retains its assignment for a later retry. +// stored metadata claims devicePath. Unreadable metadata, a recent assignment +// without a PID, or unverifiable process ownership returns an error so the +// requester retains its assignment for a later retry. func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { files, err := m.listMetadataFilesStrict() if err != nil { diff --git a/lib/instances/vgpu_orphan.go b/lib/instances/vgpu_orphan.go index b3ff2b33b..d85a68bf3 100644 --- a/lib/instances/vgpu_orphan.go +++ b/lib/instances/vgpu_orphan.go @@ -8,22 +8,20 @@ import ( ) const ( - // orphanedVGPUReleaseMaxAttempts bounds the retry loop so a genuinely - // wedged VF degrades to one operator-actionable error instead of - // indefinite log churn. At the default interval this covers ten minutes, - // far beyond the seconds a dying VMM normally needs to finish kernel-side - // VFIO teardown. + // Bounds the retry loop (~10 minutes at the default interval, far beyond + // normal VFIO teardown) so a wedged VF degrades to one operator-actionable + // error instead of indefinite log churn. orphanedVGPUReleaseMaxAttempts = 20 defaultOrphanedVGPUReleaseRetryDelay = 30 * time.Second ) // scheduleOrphanedVGPURelease retries a vGPU release that failed during a -// completed delete, off the request path. A GPU-busy VMM routinely outlives -// delete's force-kill wait while the kernel finishes VFIO teardown, and once -// the metadata is deleted nothing else releases the VF until the next -// startup reconciliation. Each attempt re-runs releaseStoredVGPU, so the -// claim scan and destroy guards apply on every retry. The queue is in-memory -// only: a restart abandons it and startup reconciliation sweeps the VF. +// completed delete, off the request path: a GPU-busy VMM routinely outlives +// delete's force-kill wait, and once metadata is deleted nothing else +// releases the VF until startup reconciliation. Each attempt re-runs +// releaseStoredVGPU, so the claim scan and destroy guards apply on every +// retry. The queue is in-memory only; a restart abandons it and startup +// reconciliation sweeps the VF. func (m *manager) scheduleOrphanedVGPURelease(ctx context.Context, stored StoredMetadata) { path := storedVGPUDevicePath(&stored) if path == "" { From 928d61cec021cb9d09ec4f877dd4d7d9b067f609 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:25:12 +0000 Subject: [PATCH 084/107] Retry vGPU recovery paths that previously waited for restart - Retry the vendor VFIO reconcile sweep with a bounded delay when the startup instance listing fails, instead of disabling orphan recovery until the next process restart. One pending retry at a time. - Schedule the in-process orphaned-release retry when a rollback's retention record cannot be saved (create and start), instead of leaking the VF until restart. The retry scans claims without a self-exclusion because a restarted instance may hold the same VF. - Reject snapshot restore into a vGPU retention stub, matching start, fork, and snapshot. - Give passthrough PCI instances the same SIGTERM grace as vGPU instances on stop/delete, matching the QEMU-side vfioTermGraceFor. - Render the vgpu_cleanup_pending API detail from the error itself instead of duplicating its prose; use the manager clock in the claim scan; collapse the create-rollback retention branch. - Move ReconcileVGPUs off the Manager interface to a startup type assertion and unexport listInstancesForReconcile and hypervisorMayBeAlive. --- cmd/api/api/instances.go | 10 ++++---- cmd/api/main.go | 9 +++++-- lib/builds/manager_test.go | 2 -- lib/instances/create.go | 8 ++---- lib/instances/manager.go | 12 ++++++--- lib/instances/process_identity.go | 20 ++++++++------- lib/instances/query_test.go | 6 ++--- lib/instances/snapshot.go | 5 ++++ lib/instances/snapshot_test.go | 37 ++++++++++++++++++++++++++++ lib/instances/start.go | 4 +++ lib/instances/vgpu.go | 21 +++++++++++++--- lib/instances/vgpu_orphan.go | 17 +++++++------ lib/instances/vgpu_reconcile.go | 21 +++++++++++++--- lib/instances/vgpu_reconcile_test.go | 29 ++++++++++++++++++++++ lib/instances/vgpu_retention.go | 8 +++--- lib/instances/vgpu_test.go | 15 ++++++++--- lib/instances/wait_test.go | 1 - 17 files changed, 172 insertions(+), 53 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 1c502c928..46be5cec4 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -439,11 +439,11 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst // and inner error detail shared by the create and start handlers. The // retained guidance names the verb-specific way to release the assignment. func vgpuCleanupPendingDetail(pending *instances.VGPUCleanupPendingError, action, retainedGuidance string) (string, *oapi.ErrorDetail) { - message := fmt.Sprintf("failed to %s instance: %v; vGPU release failed during rollback and instance %s retains the assignment, %s", action, pending.Err, pending.InstanceID, retainedGuidance) - innerCode := "vgpu_retained_instance" - if !pending.Retained { - message = fmt.Sprintf("failed to %s instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", action, pending.Err, pending.InstanceID) - innerCode = "vgpu_unretained_instance" + message := fmt.Sprintf("failed to %s instance: %v", action, pending) + innerCode := "vgpu_unretained_instance" + if pending.Retained { + message += "; " + retainedGuidance + innerCode = "vgpu_retained_instance" } return message, &oapi.ErrorDetail{ Code: lo.ToPtr(innerCode), diff --git a/cmd/api/main.go b/cmd/api/main.go index da2748103..20ac8dc60 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -384,9 +384,14 @@ func run() error { return fmt.Errorf("reconcile device state: %w", err) } - // Reconcile vGPU devices (clears orphaned vGPUs from previous runs) + // Reconcile vGPU devices (clears orphaned vGPUs from previous runs). + // Type-asserted rather than added to instances.Manager so alternate + // Manager implementations compiled against the public module keep + // building without this startup-only method. logger.Info("Reconciling vGPU devices...") - app.InstanceManager.ReconcileVGPUs(ctx) + if r, ok := app.InstanceManager.(interface{ ReconcileVGPUs(context.Context) }); ok { + r.ReconcileVGPUs(ctx) + } // Wire up resource validator for aggregate limit checking // This enables the instance manager to validate CPU, memory, network, and GPU diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index ab390c72f..a137edc66 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -51,8 +51,6 @@ func (m *mockInstanceManager) ListInstances(ctx context.Context, filter *instanc return result, nil } -func (m *mockInstanceManager) ReconcileVGPUs(context.Context) {} - func (m *mockInstanceManager) ListSnapshots(ctx context.Context, filter *instances.ListSnapshotsFilter) ([]instances.Snapshot, error) { return nil, nil } diff --git a/lib/instances/create.go b/lib/instances/create.go index f4ece0100..de301a95d 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -278,7 +278,6 @@ func (m *manager) createInstance( var gpuDevicePath string var gpuMdevUUID string var gpuAssignedAt *time.Time - var stored *StoredMetadata retention := vgpuRetention{instanceID: id} // Setup cleanup stack early so device attachment errors trigger cleanup. @@ -344,10 +343,7 @@ func (m *manager) createInstance( } if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) - retention.retain(stored) - if stored == nil { - retention.retainFromDevice(retentionStub(), gpuDevice, *gpuAssignedAt) - } + retention.retainFromDevice(retentionStub(), gpuDevice, *gpuAssignedAt) } }) } @@ -388,7 +384,7 @@ func (m *manager) createInstance( if err != nil { return nil, err } - stored = &StoredMetadata{ + stored := &StoredMetadata{ Id: id, Name: req.Name, Image: req.Image, diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 3954af19f..da1fe59d3 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "time" "github.com/kernel/hypeman/lib/devices" @@ -29,7 +30,6 @@ import ( type Manager interface { ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) - ReconcileVGPUs(ctx context.Context) ListSnapshots(ctx context.Context, filter *ListSnapshotsFilter) ([]Snapshot, error) GetSnapshot(ctx context.Context, snapshotID string) (*Snapshot, error) CreateInstance(ctx context.Context, req CreateInstanceRequest) (*Instance, error) @@ -222,6 +222,12 @@ type manager struct { orphanedVGPUs map[string]struct{} orphanedVGPURetryDelay time.Duration + // One pending vGPU reconcile retry at a time, for both the startup-grace + // and listing-failure paths. vgpuReconcileRetryDelay overrides the + // listing-failure delay in tests; zero means the default. + vgpuReconcileRetryPending atomic.Bool + vgpuReconcileRetryDelay time.Duration + // vgpuInitTermGrace overrides terminateThenKill's SIGTERM wait for vGPU // instances still initializing; zero means the default. vgpuInitTermGrace time.Duration @@ -751,10 +757,10 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { return m.defaultHypervisor } -// ListInstancesForReconcile returns every instance's stored metadata or an +// listInstancesForReconcile returns every instance's stored metadata or an // invalid metadata error. It does not derive state: hydration would query // every hypervisor on the host before the API serves. -func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, error) { +func (m *manager) listInstancesForReconcile(ctx context.Context) ([]Instance, error) { files, err := m.listMetadataFilesStrict() if err != nil { return nil, err diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index dd546c420..cd8ff2fe1 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -36,17 +36,19 @@ func (m *manager) vgpuTermGrace() time.Duration { return hypervisor.VFIOTermGrace } -// terminateThenKill hard-kills the hypervisor process, first giving any vGPU -// instance a SIGTERM grace: SIGKILL during guest driver init can wedge the VF -// until its parent GPU is SR-IOV cycled (see lib/devices/GPU.md). The grace -// applies in every state because the instance reports Running seconds before -// driver init finishes and nothing host-side observes that boundary. +// terminateThenKill hard-kills the hypervisor process, first giving any +// instance with VFIO devices (a vGPU VF or passthrough PCI devices, matching +// the QEMU-side vfioTermGraceFor) a SIGTERM grace: SIGKILL during guest +// driver init can wedge the device until its parent GPU is SR-IOV cycled +// (see lib/devices/GPU.md). The grace applies in every state because the +// instance reports Running seconds before driver init finishes and nothing +// host-side observes that boundary. func (m *manager) terminateThenKill(ctx context.Context, inst *Instance, pid int) error { - if inst.GPUProfile != "" { + if inst.GPUProfile != "" || len(inst.Devices) > 0 { if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { return nil } - logger.FromContext(ctx).WarnContext(ctx, "vGPU hypervisor did not exit on SIGTERM; hard-killing, VF may wedge if the guest driver was initializing", + logger.FromContext(ctx).WarnContext(ctx, "hypervisor with VFIO devices did not exit on SIGTERM; hard-killing, device may wedge if the guest driver was initializing", "instance_id", inst.Id, "device_path", inst.GPUDevicePath) } return killProcessAndWait(pid) @@ -207,11 +209,11 @@ func classifyResolvedHypervisorOwner(socketPath string, stored, resolved int, er return 0, fmt.Errorf("cannot confirm ownership of socket %s: %w", socketPath, err) } -// HypervisorMayBeAlive reports whether the recorded hypervisor process may +// hypervisorMayBeAlive reports whether the recorded hypervisor process may // still be running. It fails open (unresolvable ownership returns true, the // safe direction for reconcile protection and claim checks); do not use it // to authorize teardown. -func HypervisorMayBeAlive(id HypervisorProcessIdentity, socketPath string) bool { +func hypervisorMayBeAlive(id HypervisorProcessIdentity, socketPath string) bool { pid, err := resolveLiveHypervisorPID(id, socketPath) return err != nil || pid > 0 } diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index b3dbfba41..3d12c739d 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -31,12 +31,12 @@ func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { require.NoError(t, err) require.Len(t, listed, 1) - _, err = m.ListInstancesForReconcile(context.Background()) + _, err = m.listInstancesForReconcile(context.Background()) require.Error(t, err) assert.ErrorContains(t, err, "load metadata for instance invalid") require.NoError(t, os.Remove(m.paths.InstanceMetadata("invalid"))) - listed, err = m.ListInstancesForReconcile(context.Background()) + listed, err = m.listInstancesForReconcile(context.Background()) require.NoError(t, err) require.Len(t, listed, 1) assert.Equal(t, "valid", listed[0].Id) @@ -70,7 +70,7 @@ func TestListInstancesForReconcileSkipsInstanceDeletedDuringListing(t *testing.T } done := make(chan result, 1) go func() { - listed, err := m.ListInstancesForReconcile(context.Background()) + listed, err := m.listInstancesForReconcile(context.Background()) done <- result{listed, err} }() time.Sleep(100 * time.Millisecond) diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index e30192ee9..1d8456ae5 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -266,6 +266,11 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str if sourceInst.State == StateRunning { return nil, fmt.Errorf("%w: cannot restore snapshot while source is %s", ErrInvalidState, sourceInst.State) } + if sourceMeta.GPURetainedForCleanup { + // Restoring would rebuild boot config from the snapshot record, whose + // retention flag is false, silently clearing the delete-only marker. + return nil, errVGPURetentionStub + } targetState, err := resolveSnapshotTargetState(rec.Snapshot.Kind, req.TargetState) if err != nil { diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index f8c022fe9..e4fcc048f 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -77,6 +77,43 @@ func TestCreateSnapshotRejectsVGPURetentionRecord(t *testing.T) { require.ErrorContains(t, err, "delete it to release the assignment") } +func TestRestoreSnapshotRejectsVGPURetentionRecord(t *testing.T) { + mgr, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "snapshot-vgpu-restore-retention" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, sourceID, mgr.defaultHypervisor) + + snapshot, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Name: "snapshot-vgpu-restore-retention", + }) + require.NoError(t, err) + + meta, err := mgr.loadMetadata(sourceID) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, mgr.saveMetadata(meta)) + + // Restoring into the delete-only stub would rebuild boot config from the + // snapshot record, whose retention flag is false, clearing the marker. + _, err = mgr.RestoreSnapshot(ctx, sourceID, snapshot.Id, RestoreSnapshotRequest{ + TargetState: StateStopped, + TargetHypervisor: mgr.defaultHypervisor, + }) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") + + // The retained assignment must survive the rejected restore for delete. + stored, err := mgr.loadMetadata(sourceID) + require.NoError(t, err) + assert.True(t, stored.GPURetainedForCleanup) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) +} + func TestRestoreSnapshotDoesNotResurrectStaleVGPUAssignment(t *testing.T) { mgr, _ := setupTestManager(t) ctx := context.Background() diff --git a/lib/instances/start.go b/lib/instances/start.go index a79663953..1089a35a3 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -185,6 +185,10 @@ func (m *manager) startInstance( wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) + // No on-disk record points at the device; retry the release + // in the background instead of waiting for the next startup + // reconcile. + m.scheduleOrphanedVGPURelease(ctx, retentionMeta.StoredMetadata) return nil, &VGPUCleanupPendingError{InstanceID: id, Err: fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr)} } return nil, &VGPUCleanupPendingError{InstanceID: id, Retained: true, Err: wrapped} diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 8ffe64dd8..b82a07d7d 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -17,7 +17,8 @@ const VGPUAssignmentStartupGracePeriod = 5 * time.Minute // VGPUCleanupPendingError reports a failed create whose vGPU release also // failed during rollback. When Retained is true, deleting the retained instance -// retries the release; otherwise startup reconciliation recovers the assignment. +// retries the release; otherwise a background retry and startup reconciliation +// recover the assignment. type VGPUCleanupPendingError struct { InstanceID string Retained bool @@ -28,7 +29,7 @@ func (e *VGPUCleanupPendingError) Error() string { if e.Retained { return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID) } - return fmt.Sprintf("%v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", e.Err, e.InstanceID) + return fmt.Sprintf("%v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the release is retried in the background and by the next startup reconcile", e.Err, e.InstanceID) } func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } @@ -128,6 +129,9 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic if meta, loadErr := m.loadMetadata(instanceID); loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath { return true, true } + // No on-disk record points at the device; retry the release in the + // background instead of waiting for the next startup reconcile. + m.scheduleOrphanedVGPURelease(ctx, cleanupMeta.StoredMetadata) return true, false } return retained, retained @@ -155,6 +159,15 @@ func restoreStartMutatedFields(dst, src *StoredMetadata) { } func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { + return m.releaseStoredVGPUExcluding(ctx, stored, stored.Id) +} + +// releaseStoredVGPUExcluding releases stored's assignment while treating +// excludeID's metadata as not a claimant. Callers releasing an instance's own +// persisted assignment exclude that instance; the orphan retry passes no +// exclusion because its instance may have been restarted onto the same VF, +// and that live claim must block the release. +func (m *manager) releaseStoredVGPUExcluding(ctx context.Context, stored *StoredMetadata, excludeID string) error { path := storedVGPUDevicePath(stored) if path != "" { // Vendor VFIO VFs are reused across instances, so the release must @@ -164,7 +177,7 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) claimed := false if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { var err error - claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path) + claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, excludeID, path) if err != nil { return err } @@ -222,7 +235,7 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) } } - live, remaining := vgpuAssignmentLiveness(stored, time.Now(), pid > 0) + live, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), pid > 0) if pid > 0 && live { return true, nil } diff --git a/lib/instances/vgpu_orphan.go b/lib/instances/vgpu_orphan.go index d85a68bf3..513cf9bc9 100644 --- a/lib/instances/vgpu_orphan.go +++ b/lib/instances/vgpu_orphan.go @@ -15,12 +15,13 @@ const ( defaultOrphanedVGPUReleaseRetryDelay = 30 * time.Second ) -// scheduleOrphanedVGPURelease retries a vGPU release that failed during a -// completed delete, off the request path: a GPU-busy VMM routinely outlives -// delete's force-kill wait, and once metadata is deleted nothing else -// releases the VF until startup reconciliation. Each attempt re-runs -// releaseStoredVGPU, so the claim scan and destroy guards apply on every -// retry. The queue is in-memory only; a restart abandons it and startup +// scheduleOrphanedVGPURelease retries a vGPU release for an assignment no +// on-disk metadata points at anymore: a release that failed during a +// completed delete (a GPU-busy VMM routinely outlives delete's force-kill +// wait), or a rollback whose retention record could not be saved. Without a +// record, nothing else releases the VF until startup reconciliation. Each +// attempt re-runs the release, so the claim scan and destroy guards apply on +// every retry. The queue is in-memory only; a restart abandons it and startup // reconciliation sweeps the VF. func (m *manager) scheduleOrphanedVGPURelease(ctx context.Context, stored StoredMetadata) { path := storedVGPUDevicePath(&stored) @@ -56,7 +57,9 @@ func (m *manager) retryOrphanedVGPURelease(ctx context.Context, stored StoredMet }() for attempt := 1; attempt <= orphanedVGPUReleaseMaxAttempts; attempt++ { time.Sleep(delay) - if err := m.releaseStoredVGPU(ctx, &stored); err != nil { + // No claim-scan exclusion: unlike delete, a failed start keeps its + // instance record, and a restarted instance may hold this same VF. + if err := m.releaseStoredVGPUExcluding(ctx, &stored, ""); err != nil { log.WarnContext(ctx, "orphaned vGPU release retry failed", "instance_id", stored.Id, "device_path", path, "attempt", attempt, "error", err) continue diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 85d3a6e0d..4890b216a 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -8,8 +8,13 @@ import ( "github.com/kernel/hypeman/lib/logger" ) +// vgpuReconcileListRetryDelay spaces retries of the vendor VFIO sweep when +// the instance listing fails: without a retry, one transient stat error at +// startup would disable orphan recovery until the next restart. +const vgpuReconcileListRetryDelay = time.Minute + func (m *manager) liveVGPUReconcileProtection(ctx context.Context) (map[string]struct{}, time.Duration, error) { - allInstances, err := m.ListInstancesForReconcile(ctx) + allInstances, err := m.listInstancesForReconcile(ctx) if err != nil { return nil, 0, err } @@ -20,7 +25,7 @@ func (m *manager) liveVGPUReconcileProtection(ctx context.Context) (map[string]s if stored.GPUDevicePath == "" { continue } - livePID := stored.HypervisorPID != nil && HypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + livePID := stored.HypervisorPID != nil && hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) live, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID) if !live { continue @@ -41,7 +46,10 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { if err != nil { log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconcile, mdev reconcile still runs", "error", err) protected = make(map[string]struct{}) - retryAfter = 0 + retryAfter = vgpuReconcileListRetryDelay + if m.vgpuReconcileRetryDelay > 0 { + retryAfter = m.vgpuReconcileRetryDelay + } } if err := devices.ReconcileVGPUs(ctx, protected, sweepVendorVFIO); err != nil { log.WarnContext(ctx, "failed to reconcile vGPU devices", "error", err) @@ -49,12 +57,19 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { if retryAfter <= 0 { return } + // One pending retry at a time: overlapping calls would fork parallel + // retry chains. + if !m.vgpuReconcileRetryPending.CompareAndSwap(false, true) { + return + } go func() { timer := time.NewTimer(retryAfter) defer timer.Stop() select { case <-ctx.Done(): + m.vgpuReconcileRetryPending.Store(false) case <-timer.C: + m.vgpuReconcileRetryPending.Store(false) m.ReconcileVGPUs(ctx) } }() diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 1fbd90384..6fd299af7 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -1,7 +1,10 @@ package instances import ( + "context" + "os" "os/exec" + "path/filepath" "testing" "time" @@ -41,6 +44,32 @@ func TestLiveVGPUReconcileProtectionBoundsStartupProtection(t *testing.T) { assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") } +func TestReconcileVGPUsRetriesAfterListingFailure(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m := &manager{paths: paths.New(t.TempDir()), vgpuReconcileRetryDelay: 250 * time.Millisecond} + const id = "unreadable" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{Id: id}})) + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o000)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + m.ReconcileVGPUs(ctx) + require.True(t, m.vgpuReconcileRetryPending.Load(), + "a listing failure must schedule a retry instead of disabling the vendor sweep until restart") + + // Once the listing recovers, the retry runs the sweep and stops rearming. + require.NoError(t, os.Chmod(instanceDir, 0o755)) + require.Eventually(t, func() bool { + return !m.vgpuReconcileRetryPending.Load() + }, 5*time.Second, 10*time.Millisecond) +} + func TestVGPUAssignmentLiveness(t *testing.T) { now := time.Now().UTC() recent := now.Add(-time.Minute) diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index a1a1aae0d..f6cd4b02b 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -30,11 +30,6 @@ func (r *vgpuRetention) retainFromDevice(stub StoredMetadata, device *devices.VG r.retained = true } -func (r *vgpuRetention) retain(stub *StoredMetadata) { - r.stub = stub - r.retained = stub != nil -} - func (r *vgpuRetention) markRetained(persisted bool) { r.retained = true r.persisted = persisted @@ -70,6 +65,9 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten if err := m.deleteInstanceData(id); err != nil { log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) } + // No on-disk record points at the device; retry the release in the + // background instead of waiting for the next startup reconcile. + m.scheduleOrphanedVGPURelease(ctx, *retainedVGPU) return false } if err := m.ensureDirectories(id); err != nil { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index c5ef4911b..ea3b18429 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -19,8 +19,7 @@ import ( ) func persistTestVGPURetention(m *manager, ctx context.Context, id string, stub *StoredMetadata) bool { - retention := vgpuRetention{instanceID: id} - retention.retain(stub) + retention := vgpuRetention{instanceID: id, stub: stub, retained: stub != nil} m.persistVGPURetention(ctx, &retention) return retention.persisted } @@ -101,6 +100,11 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { assert.False(t, persistTestVGPURetention(m, context.Background(), stored.Id, stored)) _, err := m.loadMetadata(id) require.Error(t, err) + + m.orphanedVGPUMu.Lock() + _, queued := m.orphanedVGPUs[stored.GPUDevicePath] + m.orphanedVGPUMu.Unlock() + assert.True(t, queued, "unpersisted retention must queue a background release") } func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T) { @@ -157,7 +161,7 @@ func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { unpersisted := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause} assert.ErrorIs(t, unpersisted, cause) - assert.Equal(t, "boot failed; vGPU release failed during rollback and the retention record for instance inst-1 could not be saved; the assignment is recovered on the next startup reconcile", unpersisted.Error()) + assert.Equal(t, "boot failed; vGPU release failed during rollback and the retention record for instance inst-1 could not be saved; the release is retried in the background and by the next startup reconcile", unpersisted.Error()) } func TestVGPUDevicePendingCleanup(t *testing.T) { @@ -334,6 +338,11 @@ func TestStartReportsUnretainedVGPUWhenRetentionSaveFails(t *testing.T) { stored, err := m.loadMetadata(id) require.NoError(t, err) assert.Empty(t, stored.GPUDevicePath, "retention save failed, so no assignment should be recorded") + + m.orphanedVGPUMu.Lock() + _, queued := m.orphanedVGPUs[device.SysfsPath] + m.orphanedVGPUMu.Unlock() + assert.True(t, queued, "unpersisted retention must queue a background release") } func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { diff --git a/lib/instances/wait_test.go b/lib/instances/wait_test.go index 3ab02e402..dbb630185 100644 --- a/lib/instances/wait_test.go +++ b/lib/instances/wait_test.go @@ -32,7 +32,6 @@ func (s *stubManager) GetInstance(ctx context.Context, id string) (*Instance, er func (s *stubManager) ListInstances(context.Context, *ListInstancesFilter) ([]Instance, error) { return nil, nil } -func (s *stubManager) ReconcileVGPUs(context.Context) {} func (s *stubManager) ListSnapshots(context.Context, *ListSnapshotsFilter) ([]Snapshot, error) { return nil, nil } From e361540757632342d27f238135fd93c93d9c9d98 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 20:08:05 +0000 Subject: [PATCH 085/107] Validate live vGPU claim test identity --- lib/instances/vgpu_test.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index ea3b18429..7ae6f3cdf 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -526,10 +526,14 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. require.NoError(t, m.ensureDirectories("legacy-claimant")) pid := os.Getpid() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "legacy-claimant", - Name: "legacy-claimant", - GPUMdevUUID: "legacy-uuid", - HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, + Id: "legacy-claimant", + Name: "legacy-claimant", + GPUMdevUUID: "legacy-uuid", + HypervisorProcessIdentity: HypervisorProcessIdentity{ + HypervisorPID: &pid, + HypervisorStartTime: processStartTime(pid), + HypervisorBootID: hostBootID(), + }, }})) claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/mdev/devices/legacy-uuid") From a9387fc7d2257b28dd2cf89f610501a4c3255117 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 21:00:23 +0000 Subject: [PATCH 086/107] Clean guest data before retaining vGPU assignment --- lib/instances/vgpu_retention.go | 5 +++++ lib/instances/vgpu_test.go | 17 ++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index f6cd4b02b..eaca96289 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -70,6 +70,11 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten m.scheduleOrphanedVGPURelease(ctx, *retainedVGPU) return false } + if err := m.deleteInstanceData(id); err != nil { + log.ErrorContext(ctx, "failed to clean instance data before retaining vGPU assignment", "instance_id", id, "error", err) + retention.persisted = retentionSurvives() + return + } if err := m.ensureDirectories(id); err != nil { log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) retention.persisted = retentionSurvives() diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 7ae6f3cdf..cb54c509b 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -49,8 +49,16 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { HypervisorType: "qemu", DataDir: m.paths.InstanceDir("failed-create"), } + require.NoError(t, m.ensureDirectories(stored.Id)) + require.NoError(t, os.WriteFile(m.paths.InstanceOverlay(stored.Id), []byte("overlay"), 0o644)) + require.NoError(t, os.WriteFile(m.paths.InstanceConfigDisk(stored.Id), []byte("config"), 0o644)) + require.NoError(t, os.MkdirAll(m.paths.InstanceVolumeOverlaysDir(stored.Id), 0o755)) + require.NoError(t, os.WriteFile(m.paths.InstanceVolumeOverlay(stored.Id, "volume"), []byte("volume overlay"), 0o644)) assert.True(t, persistTestVGPURetention(m, context.Background(), stored.Id, stored)) + assert.NoFileExists(t, m.paths.InstanceOverlay(stored.Id)) + assert.NoFileExists(t, m.paths.InstanceConfigDisk(stored.Id)) + assert.NoDirExists(t, m.paths.InstanceVolumeOverlaysDir(stored.Id)) retained, err := m.loadMetadata(stored.Id) require.NoError(t, err) @@ -85,12 +93,15 @@ func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { } func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { - t.Parallel() + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } m := &manager{paths: paths.New(t.TempDir())} const id = "failed-create" - require.NoError(t, m.ensureDirectories(id)) - require.NoError(t, os.Mkdir(m.paths.InstanceMetadata(id), 0o755)) + require.NoError(t, os.MkdirAll(m.paths.GuestsDir(), 0o755)) + require.NoError(t, os.Chmod(m.paths.GuestsDir(), 0o555)) + t.Cleanup(func() { _ = os.Chmod(m.paths.GuestsDir(), 0o755) }) stored := &StoredMetadata{ Id: id, From 9b3ece3d35a1cea4cfb0006f6ca67241ae2b1fe7 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:21:58 +0000 Subject: [PATCH 087/107] Retry failed vGPU reconciliation --- lib/instances/manager.go | 7 ++++--- lib/instances/vgpu_reconcile.go | 22 ++++++++++++++++------ lib/instances/vgpu_reconcile_test.go | 24 ++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index da1fe59d3..91b25aebc 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -186,6 +186,7 @@ type manager struct { deleteInstanceFn func(context.Context, string) error createVGPU func(context.Context, string, string) (*devices.VGPUDevice, error) destroyVGPU func(context.Context, devices.VGPUAssignment) error + reconcileVGPUDevices func(context.Context, map[string]struct{}, bool) error deleteSnapshotFn func(context.Context, string) error ttlReaperDeleteTimeout time.Duration egressProxy *egressproxy.Service @@ -222,9 +223,9 @@ type manager struct { orphanedVGPUs map[string]struct{} orphanedVGPURetryDelay time.Duration - // One pending vGPU reconcile retry at a time, for both the startup-grace - // and listing-failure paths. vgpuReconcileRetryDelay overrides the - // listing-failure delay in tests; zero means the default. + // One pending vGPU reconcile retry at a time, for startup grace and + // reconciliation failures. vgpuReconcileRetryDelay overrides the retry + // delay in tests; zero means the default. vgpuReconcileRetryPending atomic.Bool vgpuReconcileRetryDelay time.Duration diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 4890b216a..24be04fe3 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -8,10 +8,9 @@ import ( "github.com/kernel/hypeman/lib/logger" ) -// vgpuReconcileListRetryDelay spaces retries of the vendor VFIO sweep when -// the instance listing fails: without a retry, one transient stat error at -// startup would disable orphan recovery until the next restart. -const vgpuReconcileListRetryDelay = time.Minute +// vgpuReconcileFailureRetryDelay spaces retries after a transient metadata or +// device error would otherwise disable orphan recovery until the next restart. +const vgpuReconcileFailureRetryDelay = time.Minute func (m *manager) liveVGPUReconcileProtection(ctx context.Context) (map[string]struct{}, time.Duration, error) { allInstances, err := m.listInstancesForReconcile(ctx) @@ -46,13 +45,24 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { if err != nil { log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconcile, mdev reconcile still runs", "error", err) protected = make(map[string]struct{}) - retryAfter = vgpuReconcileListRetryDelay + retryAfter = vgpuReconcileFailureRetryDelay if m.vgpuReconcileRetryDelay > 0 { retryAfter = m.vgpuReconcileRetryDelay } } - if err := devices.ReconcileVGPUs(ctx, protected, sweepVendorVFIO); err != nil { + reconcileDevices := m.reconcileVGPUDevices + if reconcileDevices == nil { + reconcileDevices = devices.ReconcileVGPUs + } + if err := reconcileDevices(ctx, protected, sweepVendorVFIO); err != nil { log.WarnContext(ctx, "failed to reconcile vGPU devices", "error", err) + deviceRetryAfter := vgpuReconcileFailureRetryDelay + if m.vgpuReconcileRetryDelay > 0 { + deviceRetryAfter = m.vgpuReconcileRetryDelay + } + if retryAfter <= 0 || deviceRetryAfter < retryAfter { + retryAfter = deviceRetryAfter + } } if retryAfter <= 0 { return diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 6fd299af7..815415d09 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -2,9 +2,11 @@ package instances import ( "context" + "errors" "os" "os/exec" "path/filepath" + "sync/atomic" "testing" "time" @@ -70,6 +72,28 @@ func TestReconcileVGPUsRetriesAfterListingFailure(t *testing.T) { }, 5*time.Second, 10*time.Millisecond) } +func TestReconcileVGPUsRetriesAfterDeviceFailure(t *testing.T) { + var calls atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + vgpuReconcileRetryDelay: 10 * time.Millisecond, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { + if calls.Add(1) == 1 { + return errors.New("transient device error") + } + return nil + }, + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + m.ReconcileVGPUs(ctx) + require.True(t, m.vgpuReconcileRetryPending.Load()) + require.Eventually(t, func() bool { + return calls.Load() >= 2 && !m.vgpuReconcileRetryPending.Load() + }, 5*time.Second, 10*time.Millisecond) +} + func TestVGPUAssignmentLiveness(t *testing.T) { now := time.Now().UTC() recent := now.Add(-time.Minute) From c2737529ec245aa6e995f98e27e3e8ddfceb2505 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 15:19:57 +0000 Subject: [PATCH 088/107] Trim vGPU lifecycle comments and tests --- cmd/api/api/instances.go | 7 ---- cmd/api/api/instances_test.go | 5 --- cmd/api/main.go | 4 --- lib/hypervisor/qemu/process.go | 9 ++--- lib/instances/create.go | 4 --- lib/instances/delete.go | 9 +---- lib/instances/fork_test.go | 2 -- lib/instances/lifecycle_noop_test.go | 7 ---- lib/instances/manager.go | 14 -------- lib/instances/metrics.go | 3 -- lib/instances/process_identity.go | 15 ++------ lib/instances/process_identity_linux_test.go | 27 +-------------- lib/instances/query_test.go | 8 ----- lib/instances/snapshot.go | 2 -- lib/instances/snapshot_test.go | 5 --- lib/instances/start.go | 4 --- lib/instances/storage.go | 5 --- lib/instances/types.go | 15 ++++---- lib/instances/vgpu.go | 36 ++------------------ lib/instances/vgpu_orphan.go | 15 -------- lib/instances/vgpu_orphan_test.go | 16 --------- lib/instances/vgpu_reconcile.go | 4 --- lib/instances/vgpu_reconcile_test.go | 30 ---------------- lib/instances/vgpu_retention.go | 2 -- lib/instances/vgpu_test.go | 23 ------------- 25 files changed, 14 insertions(+), 257 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 46be5cec4..e424da8e4 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -364,8 +364,6 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst if err != nil { var vgpuPending *instances.VGPUCleanupPendingError switch { - // Checked first: it wraps the original create error, so a later - // errors.Is case would match the cause and hide the pending vGPU cleanup. case errors.As(err, &vgpuPending): log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) message, inner := vgpuCleanupPendingDetail(vgpuPending, "create", "delete it to retry") @@ -435,9 +433,6 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst return oapi.CreateInstance201JSONResponse(instanceToOAPI(*inst)), nil } -// vgpuCleanupPendingDetail renders a pending vGPU cleanup into the message -// and inner error detail shared by the create and start handlers. The -// retained guidance names the verb-specific way to release the assignment. func vgpuCleanupPendingDetail(pending *instances.VGPUCleanupPendingError, action, retainedGuidance string) (string, *oapi.ErrorDetail) { message := fmt.Sprintf("failed to %s instance: %v", action, pending) innerCode := "vgpu_unretained_instance" @@ -863,8 +858,6 @@ func (s *ApiService) StartInstance(ctx context.Context, request oapi.StartInstan if err != nil { var vgpuPending *instances.VGPUCleanupPendingError switch { - // Checked first: it wraps the original start error, so a later - // errors.Is case would match the cause and hide the pending vGPU cleanup. case errors.As(err, &vgpuPending): log.ErrorContext(ctx, "failed to start instance", "error", err) message, inner := vgpuCleanupPendingDetail(vgpuPending, "start", "delete it or retry start to release it") diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 738cf547c..fd506023d 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -57,8 +57,6 @@ func (m createErrorInstanceManager) CreateInstance(context.Context, instances.Cr return nil, m.err } -// A retained-assignment error must win over the mapping of the create error -// it wraps, or the response omits the instance the caller has to delete. func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { t.Parallel() svc := newTestService(t) @@ -1082,9 +1080,6 @@ func TestRestoreInstance_ErrorMapping(t *testing.T) { } } -// A retained-assignment error must win over the mapping of the start error -// it wraps, or the response omits the pending vGPU cleanup the caller has to -// resolve. func TestStartInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { t.Parallel() diff --git a/cmd/api/main.go b/cmd/api/main.go index 20ac8dc60..3bd0fff39 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -384,10 +384,6 @@ func run() error { return fmt.Errorf("reconcile device state: %w", err) } - // Reconcile vGPU devices (clears orphaned vGPUs from previous runs). - // Type-asserted rather than added to instances.Manager so alternate - // Manager implementations compiled against the public module keep - // building without this startup-only method. logger.Info("Reconciling vGPU devices...") if r, ok := app.InstanceManager.(interface{ ReconcileVGPUs(context.Context) }); ok { r.ReconcileVGPUs(ctx) diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index de560c10d..f16d606eb 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -225,11 +225,8 @@ func buildQMPArgs(socketPath string) []string { } type startedProcess struct { - pid int - socketPath string - // termGrace, when non-zero, makes cleanup send SIGTERM and wait this long - // before SIGKILL. Set for VFIO-attached processes: SIGKILL during vGPU - // plugin init can wedge the VF until its parent GPU is SR-IOV cycled. + pid int + socketPath string termGrace time.Duration waitDone chan error waitConsumed bool @@ -281,7 +278,6 @@ func (p *startedProcess) wait() error { return err } -// waitFor waits up to d for the process to exit, returning whether it did. func (p *startedProcess) waitFor(d time.Duration) bool { if _, exited := p.checkExited(); exited { return true @@ -392,7 +388,6 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version } pid := proc.pid - // Only failed starts pay the VFIO termination grace. proc.termGrace = termGrace log.DebugContext(processCtx, "QEMU process started", "pid", pid, "duration_ms", time.Since(processStartTime).Milliseconds()) diff --git a/lib/instances/create.go b/lib/instances/create.go index de301a95d..525f2c262 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -280,7 +280,6 @@ func (m *manager) createInstance( var gpuAssignedAt *time.Time retention := vgpuRetention{instanceID: id} - // Setup cleanup stack early so device attachment errors trigger cleanup. defer retention.deferWrapPending(&retErr) cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) @@ -300,9 +299,6 @@ func (m *manager) createInstance( // Handle vGPU profile request if req.GPU != nil && req.GPU.Profile != "" { - // Identity fields a retention record keeps when rollback cannot - // release the assignment, so it lists as a recognizable, deletable - // instance. Create has already failed on a nil starter by this point. retentionStub := func() StoredMetadata { return StoredMetadata{ Id: id, diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 2f90a8552..160ae807d 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -140,19 +140,12 @@ func (m *manager) deleteInstanceWithOptions( } m.closeFirecrackerUFFDSession(ctx, stored) - // 5b. Release the vGPU assignment if present, before any network, device, - // or volume teardown. Release failure is logged and the delete continues, - // matching the pre-refactor contract: the VMM is already confirmed dead, - // the guards inside the release never destroy a device they cannot prove - // is unowned, and a skipped release is recovered by the background retry - // below or, after a restart, by startup reconciliation. + // Release before deleting metadata so a failed release can be retried safely. hadVGPUAssignment := storedVGPUDevicePath(stored) != "" if hadVGPUAssignment { log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID) } if err := m.releaseStoredVGPU(ctx, stored); err != nil { - // Log error but continue with cleanup; the background retry releases - // the VF once the metadata is gone. log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) m.scheduleOrphanedVGPURelease(ctx, *stored) } else if hadVGPUAssignment { diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index ef802d1ad..c0bed670e 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -81,8 +81,6 @@ func TestForkInstanceRejectsVGPURetentionRecord(t *testing.T) { meta.GPURetainedForCleanup = true require.NoError(t, manager.saveMetadata(meta)) - // The delete-only retention stub has no boot configuration, so a fork of - // it could never boot; only delete may act on it. _, err = manager.ForkInstance(ctx, sourceID, ForkInstanceRequest{Name: "fork-vgpu-retention-copy"}) require.ErrorIs(t, err, ErrInvalidState) require.ErrorContains(t, err, "delete it to release the assignment") diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index a43efabb0..90f2e7aa1 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -198,10 +198,6 @@ func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) { assert.Equal(t, restartpolicy.BlockedReasonManualStop, persisted.RestartStatus.BlockedReason) } -// A failed create whose vGPU release also failed retains a minimal -// GPU-fields-only stub, and the API tells the caller to delete it to retry -// the release. Exercise that recovery path against the exact stub shape -// persistVGPURetention writes. func TestDeleteReleasesRetainedCreateStub(t *testing.T) { p := paths.New(t.TempDir()) var destroyed []devices.VGPUAssignment @@ -342,7 +338,6 @@ func TestStartRejectsVGPURetentionRecord(t *testing.T) { require.ErrorIs(t, err, ErrInvalidState) require.ErrorContains(t, err, "delete it to release the assignment") - // The retained assignment must survive the rejected start for delete. stored, err := m.loadMetadata(id) require.NoError(t, err) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) @@ -381,8 +376,6 @@ func TestStopStoppedInstanceLeavesRetentionStubForDelete(t *testing.T) { require.NoError(t, err) require.NotNil(t, inst) - // Retention stubs are delete-only: releasing on stop would leave a stub - // whose start/fork/snapshot errors still claim a retained assignment. stored, err := m.loadMetadata(id) require.NoError(t, err) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 91b25aebc..7cd94262a 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -215,22 +215,13 @@ type manager struct { // Periodic TAP garbage collection reconciler. tapGCOnce sync.Once - // vGPU assignments that survived a completed delete, keyed by device - // path, each with a background release retry in flight. - // orphanedVGPURetryDelay overrides the retry delay in tests; zero means - // the default. orphanedVGPUMu sync.Mutex orphanedVGPUs map[string]struct{} orphanedVGPURetryDelay time.Duration - // One pending vGPU reconcile retry at a time, for startup grace and - // reconciliation failures. vgpuReconcileRetryDelay overrides the retry - // delay in tests; zero means the default. vgpuReconcileRetryPending atomic.Bool vgpuReconcileRetryDelay time.Duration - // vgpuInitTermGrace overrides terminateThenKill's SIGTERM wait for vGPU - // instances still initializing; zero means the default. vgpuInitTermGrace time.Duration // Hypervisor support @@ -758,9 +749,6 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { return m.defaultHypervisor } -// listInstancesForReconcile returns every instance's stored metadata or an -// invalid metadata error. It does not derive state: hydration would query -// every hypervisor on the host before the API serves. func (m *manager) listInstancesForReconcile(ctx context.Context) ([]Instance, error) { files, err := m.listMetadataFilesStrict() if err != nil { @@ -772,8 +760,6 @@ func (m *manager) listInstancesForReconcile(ctx context.Context) ([]Instance, er meta, err := m.loadMetadata(id) if err != nil { if errors.Is(err, ErrNotFound) { - // Deleted between listing and load; failing instead would - // disable the vendor VFIO sweep whenever it races a delete. continue } return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index 085216fdf..de71c2686 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -573,9 +573,6 @@ func (m *manager) recordTimeToRunning(ctx context.Context, stored *StoredMetadat m.metrics.timeToRunning.Record(ctx, duration, metric.WithAttributes(attrs...)) } -// recordVGPUOrphanReleaseAbandoned records an orphaned vGPU release retry -// loop giving up: the VF stays allocated until startup reconciliation or -// manual remediation, so it must be visible beyond a log line. func (m *manager) recordVGPUOrphanReleaseAbandoned(ctx context.Context) { if m.metrics == nil { return diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index cd8ff2fe1..02f43fcda 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -27,8 +27,6 @@ const linuxBootIDPath = "/proc/sys/kernel/random/boot_id" // does not unstick it, so the wait is short to keep stop and delete fast. const hypervisorSIGKILLWaitTimeout = 2 * time.Second -// vgpuTermGrace returns the SIGTERM wait used before hard-killing a vGPU -// hypervisor. func (m *manager) vgpuTermGrace() time.Duration { if m.vgpuInitTermGrace > 0 { return m.vgpuInitTermGrace @@ -36,13 +34,7 @@ func (m *manager) vgpuTermGrace() time.Duration { return hypervisor.VFIOTermGrace } -// terminateThenKill hard-kills the hypervisor process, first giving any -// instance with VFIO devices (a vGPU VF or passthrough PCI devices, matching -// the QEMU-side vfioTermGraceFor) a SIGTERM grace: SIGKILL during guest -// driver init can wedge the device until its parent GPU is SR-IOV cycled -// (see lib/devices/GPU.md). The grace applies in every state because the -// instance reports Running seconds before driver init finishes and nothing -// host-side observes that boundary. +// SIGKILL during guest driver init can wedge a VF until the parent GPU is reset. func (m *manager) terminateThenKill(ctx context.Context, inst *Instance, pid int) error { if inst.GPUProfile != "" || len(inst.Devices) > 0 { if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { @@ -209,10 +201,7 @@ func classifyResolvedHypervisorOwner(socketPath string, stored, resolved int, er return 0, fmt.Errorf("cannot confirm ownership of socket %s: %w", socketPath, err) } -// hypervisorMayBeAlive reports whether the recorded hypervisor process may -// still be running. It fails open (unresolvable ownership returns true, the -// safe direction for reconcile protection and claim checks); do not use it -// to authorize teardown. +// Ambiguous ownership is treated as live; this must not authorize teardown. func hypervisorMayBeAlive(id HypervisorProcessIdentity, socketPath string) bool { pid, err := resolveLiveHypervisorPID(id, socketPath) return err != nil || pid > 0 diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index e090b9a98..b208a6ef3 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -692,9 +692,6 @@ func TestResolveRuntimeHypervisorPIDMintsIdentityOnlyWhenConfirmed(t *testing.T) }) } -// startTrapProcess starts a shell with the given TERM trap action (empty -// ignores the signal) and blocks until the trap is installed. It returns the -// PID and its boot-scoped identity. func startTrapProcess(t *testing.T, trapAction string) (int, HypervisorProcessIdentity) { t.Helper() script := fmt.Sprintf("trap '%s' TERM; echo ready; sleep 30 & wait", trapAction) @@ -717,7 +714,7 @@ func startTrapProcess(t *testing.T, trapAction string) (int, HypervisorProcessId return pid, HypervisorProcessIdentity{HypervisorPID: &pid, HypervisorStartTime: startTime, HypervisorBootID: hostBootID()} } -func TestKillHypervisorSIGTERMsInitializingVGPUHypervisor(t *testing.T) { +func TestKillHypervisorSIGTERMsVGPUHypervisor(t *testing.T) { markerPath := filepath.Join(t.TempDir(), "terminated") pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") socketPath := filepath.Join(t.TempDir(), "missing.sock") @@ -757,28 +754,6 @@ func TestKillHypervisorEscalatesToSIGKILLWhenSIGTERMIgnored(t *testing.T) { assert.ErrorIs(t, syscall.Kill(pid, 0), syscall.ESRCH, "SIGTERM-ignoring hypervisor must still be hard-killed") } -func TestKillHypervisorSIGTERMsRunningVGPUHypervisor(t *testing.T) { - markerPath := filepath.Join(t.TempDir(), "terminated") - pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") - socketPath := filepath.Join(t.TempDir(), "missing.sock") - - m := &manager{} - require.NoError(t, m.killHypervisor(context.Background(), &Instance{ - State: StateRunning, - StoredMetadata: StoredMetadata{ - Id: "kill-test", - GPUProfile: "NVIDIA L40S-1Q", - HypervisorProcessIdentity: identity, - SocketPath: socketPath, - }, - })) - - require.Eventually(t, func() bool { - return syscall.Kill(pid, 0) == syscall.ESRCH - }, 5*time.Second, 10*time.Millisecond) - assert.FileExists(t, markerPath, "Running reports true before guest driver init completes, so vGPU hypervisors get SIGTERM in every state") -} - func TestKillHypervisorHardKillsNonVGPUHypervisor(t *testing.T) { markerPath := filepath.Join(t.TempDir(), "terminated") pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index 3d12c739d..795b00baf 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -42,11 +42,6 @@ func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { assert.Equal(t, "valid", listed[0].Id) } -// A concurrent delete can remove an instance between the reconcile listing -// and its metadata load. A vanished record cannot claim a VF, so it must be -// skipped like the release claim scan does — failing instead would zero the -// grace-period retry and silently disable the vendor VFIO sweep whenever it -// races a delete. func TestListInstancesForReconcileSkipsInstanceDeletedDuringListing(t *testing.T) { m := &manager{paths: paths.New(t.TempDir())} @@ -60,9 +55,6 @@ func TestListInstancesForReconcileSkipsInstanceDeletedDuringListing(t *testing.T }})) } - // loadMetadata takes the snapshot-alias read lock, so holding the - // mutation lock parks the reconcile between listing and loading — the - // window a concurrent delete lands in. unlock := hypervisor.LockSnapshotSourceAliasMutation() type result struct { listed []Instance diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 1d8456ae5..a87acb614 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -267,8 +267,6 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str return nil, fmt.Errorf("%w: cannot restore snapshot while source is %s", ErrInvalidState, sourceInst.State) } if sourceMeta.GPURetainedForCleanup { - // Restoring would rebuild boot config from the snapshot record, whose - // retention flag is false, silently clearing the delete-only marker. return nil, errVGPURetentionStub } diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index e4fcc048f..31f0a4343 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -67,8 +67,6 @@ func TestCreateSnapshotRejectsVGPURetentionRecord(t *testing.T) { meta.GPURetainedForCleanup = true require.NoError(t, mgr.saveMetadata(meta)) - // The delete-only retention stub has no boot configuration, so a snapshot - // of it could never be restored or forked into a bootable instance. _, err = mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ Kind: SnapshotKindStopped, Name: "snapshot-vgpu-retention", @@ -98,8 +96,6 @@ func TestRestoreSnapshotRejectsVGPURetentionRecord(t *testing.T) { meta.GPURetainedForCleanup = true require.NoError(t, mgr.saveMetadata(meta)) - // Restoring into the delete-only stub would rebuild boot config from the - // snapshot record, whose retention flag is false, clearing the marker. _, err = mgr.RestoreSnapshot(ctx, sourceID, snapshot.Id, RestoreSnapshotRequest{ TargetState: StateStopped, TargetHypervisor: mgr.defaultHypervisor, @@ -107,7 +103,6 @@ func TestRestoreSnapshotRejectsVGPURetentionRecord(t *testing.T) { require.ErrorIs(t, err, ErrInvalidState) require.ErrorContains(t, err, "delete it to release the assignment") - // The retained assignment must survive the rejected restore for delete. stored, err := mgr.loadMetadata(sourceID) require.NoError(t, err) assert.True(t, stored.GPURetainedForCleanup) diff --git a/lib/instances/start.go b/lib/instances/start.go index 1089a35a3..8986d7fb3 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -67,7 +67,6 @@ func (m *manager) startInstance( } } - // Do not persist the previous VMM's identity with a new vGPU assignment. stored.HypervisorPID = nil stored.HypervisorStartTime = 0 stored.HypervisorBootID = "" @@ -185,9 +184,6 @@ func (m *manager) startInstance( wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) - // No on-disk record points at the device; retry the release - // in the background instead of waiting for the next startup - // reconcile. m.scheduleOrphanedVGPURelease(ctx, retentionMeta.StoredMetadata) return nil, &VGPUCleanupPendingError{InstanceID: id, Err: fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr)} } diff --git a/lib/instances/storage.go b/lib/instances/storage.go index 40bbba684..1a4d325b0 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -187,15 +187,10 @@ func removeAllWithRetry(path string, removeAll func(string) error, sleep func(ti } } -// listMetadataFiles returns paths to all instance metadata files, skipping -// entries whose metadata cannot be statted. func (m *manager) listMetadataFiles() ([]string, error) { return m.walkMetadataFiles(false) } -// listMetadataFilesStrict returns paths to all instance metadata files, -// failing on any stat error other than absence, so fail-closed callers see -// an unreadable instance as an error instead of silently missing. func (m *manager) listMetadataFilesStrict() ([]string, error) { return m.walkMetadataFiles(true) } diff --git a/lib/instances/types.go b/lib/instances/types.go index ab41178cd..ed32f5b5d 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -151,15 +151,12 @@ type StoredMetadata struct { Devices []string // Device IDs attached to this instance // GPU configuration (vGPU mode) - GPUProfile string // vGPU profile name (e.g., "L40S-1Q") - GPUFramework devices.VGPUFramework - GPUDevicePath string - GPUMdevUUID string // populated for mdev-backed vGPUs - GPUAssignedAt *time.Time // set before hypervisor startup to bound crash recovery protection - // GPURetainedForCleanup marks a delete-only retention stub written when a - // failed create could not release its vGPU: the record has no boot - // configuration, so only delete (which retries the release) may act on it. - GPURetainedForCleanup bool + GPUProfile string // vGPU profile name (e.g., "L40S-1Q") + GPUFramework devices.VGPUFramework + GPUDevicePath string + GPUMdevUUID string // populated for mdev-backed vGPUs + GPUAssignedAt *time.Time // set before hypervisor startup to bound crash recovery protection + GPURetainedForCleanup bool // delete-only stub holding a vGPU assignment // Command overrides (like docker run ) Entrypoint []string // Override image entrypoint (nil = use image default) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index b82a07d7d..0e09f2f34 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -15,10 +15,7 @@ import ( // persisted hypervisor PID is treated as potentially live. const VGPUAssignmentStartupGracePeriod = 5 * time.Minute -// VGPUCleanupPendingError reports a failed create whose vGPU release also -// failed during rollback. When Retained is true, deleting the retained instance -// retries the release; otherwise a background retry and startup reconciliation -// recover the assignment. +// VGPUCleanupPendingError reports a failed rollback that left a vGPU assigned. type VGPUCleanupPendingError struct { InstanceID string Retained bool @@ -34,9 +31,6 @@ func (e *VGPUCleanupPendingError) Error() string { func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } -// errVGPURetentionStub rejects every lifecycle verb except delete on a -// retention stub from a failed create: the record has no boot configuration, -// and only delete retries the release of its retained assignment. var errVGPURetentionStub = fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) func (m *manager) createVGPUDevice(ctx context.Context, profileName, instanceID string) (*devices.VGPUDevice, error) { @@ -91,9 +85,6 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUAssignedAt = nil } -// cleanupStartVGPU reports whether the assignment was retained after a failed -// destroy and whether that retention record was persisted, so start can surface -// the pending cleanup as a typed error like create does. func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) (retained, persisted bool) { logger.FromContext(ctx).DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID) assignment := devices.VGPUAssignment{ @@ -124,21 +115,15 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic if !retained { return false, false } - // The mid-start save may already have persisted this assignment, so - // delete or a retried start can still release it. if meta, loadErr := m.loadMetadata(instanceID); loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath { return true, true } - // No on-disk record points at the device; retry the release in the - // background instead of waiting for the next startup reconcile. m.scheduleOrphanedVGPURelease(ctx, cleanupMeta.StoredMetadata) return true, false } return retained, retained } -// restoreStartMutatedFields must cover every field start mutates before the -// vGPU cleanup runs. func restoreStartMutatedFields(dst, src *StoredMetadata) { dst.HypervisorPID = src.HypervisorPID dst.HypervisorStartTime = src.HypervisorStartTime @@ -162,18 +147,10 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) return m.releaseStoredVGPUExcluding(ctx, stored, stored.Id) } -// releaseStoredVGPUExcluding releases stored's assignment while treating -// excludeID's metadata as not a claimant. Callers releasing an instance's own -// persisted assignment exclude that instance; the orphan retry passes no -// exclusion because its instance may have been restarted onto the same VF, -// and that live claim must block the release. func (m *manager) releaseStoredVGPUExcluding(ctx context.Context, stored *StoredMetadata, excludeID string) error { path := storedVGPUDevicePath(stored) if path != "" { - // Vendor VFIO VFs are reused across instances, so the release must - // fail closed on an incomplete inventory. mdev UUIDs are never reused; - // scanning there would let one unreadable metadata file block every - // mdev release on the host. + // Vendor VFIO VFs are reusable, so release fails closed on an incomplete inventory. claimed := false if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { var err error @@ -201,10 +178,6 @@ func (m *manager) releaseStoredVGPUExcluding(ctx context.Context, stored *Stored return nil } -// vgpuAssignmentClaimedByLiveInstance reports whether another live instance's -// stored metadata claims devicePath. Unreadable metadata, a recent assignment -// without a PID, or unverifiable process ownership returns an error so the -// requester retains its assignment for a later retry. func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { files, err := m.listMetadataFilesStrict() if err != nil { @@ -218,8 +191,6 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu meta, err := m.loadMetadata(id) if err != nil { if errors.Is(err, ErrNotFound) { - // Deleted between listing and load; a vanished record cannot - // be a live claimant. continue } return false, fmt.Errorf("load metadata for vGPU release check: instance %s: %w", id, err) @@ -262,9 +233,6 @@ func (m *manager) releaseRetainedVGPULocked(ctx context.Context, id string) { } stored := &meta.StoredMetadata if stored.GPURetainedForCleanup { - // Delete-only retention stubs release through delete. Releasing here - // would leave a stub whose start/fork/snapshot errors still claim a - // retained assignment that no longer exists. return } if storedVGPUDevicePath(stored) == "" { diff --git a/lib/instances/vgpu_orphan.go b/lib/instances/vgpu_orphan.go index 513cf9bc9..041402642 100644 --- a/lib/instances/vgpu_orphan.go +++ b/lib/instances/vgpu_orphan.go @@ -8,21 +8,10 @@ import ( ) const ( - // Bounds the retry loop (~10 minutes at the default interval, far beyond - // normal VFIO teardown) so a wedged VF degrades to one operator-actionable - // error instead of indefinite log churn. orphanedVGPUReleaseMaxAttempts = 20 defaultOrphanedVGPUReleaseRetryDelay = 30 * time.Second ) -// scheduleOrphanedVGPURelease retries a vGPU release for an assignment no -// on-disk metadata points at anymore: a release that failed during a -// completed delete (a GPU-busy VMM routinely outlives delete's force-kill -// wait), or a rollback whose retention record could not be saved. Without a -// record, nothing else releases the VF until startup reconciliation. Each -// attempt re-runs the release, so the claim scan and destroy guards apply on -// every retry. The queue is in-memory only; a restart abandons it and startup -// reconciliation sweeps the VF. func (m *manager) scheduleOrphanedVGPURelease(ctx context.Context, stored StoredMetadata) { path := storedVGPUDevicePath(&stored) if path == "" { @@ -43,8 +32,6 @@ func (m *manager) scheduleOrphanedVGPURelease(ctx context.Context, stored Stored if delay <= 0 { delay = defaultOrphanedVGPUReleaseRetryDelay } - // The request context ends with the delete; keep its values for logging - // but detach from its cancellation. go m.retryOrphanedVGPURelease(context.WithoutCancel(ctx), stored, path, delay) } @@ -57,8 +44,6 @@ func (m *manager) retryOrphanedVGPURelease(ctx context.Context, stored StoredMet }() for attempt := 1; attempt <= orphanedVGPUReleaseMaxAttempts; attempt++ { time.Sleep(delay) - // No claim-scan exclusion: unlike delete, a failed start keeps its - // instance record, and a restarted instance may hold this same VF. if err := m.releaseStoredVGPUExcluding(ctx, &stored, ""); err != nil { log.WarnContext(ctx, "orphaned vGPU release retry failed", "instance_id", stored.Id, "device_path", path, "attempt", attempt, "error", err) diff --git a/lib/instances/vgpu_orphan_test.go b/lib/instances/vgpu_orphan_test.go index 25a350064..ee8576a3b 100644 --- a/lib/instances/vgpu_orphan_test.go +++ b/lib/instances/vgpu_orphan_test.go @@ -97,20 +97,6 @@ func TestScheduleOrphanedVGPUReleaseDeduplicatesByDevicePath(t *testing.T) { assert.Equal(t, int32(1), attempts.Load(), "the second schedule for the same path must be dropped") } -func TestScheduleOrphanedVGPUReleaseIgnoresEmptyAssignment(t *testing.T) { - t.Parallel() - - m := &manager{paths: paths.New(t.TempDir())} - m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{Id: "no-gpu"}) - - m.orphanedVGPUMu.Lock() - defer m.orphanedVGPUMu.Unlock() - assert.Empty(t, m.orphanedVGPUs) -} - -// TestOrphanedVGPUReleaseReappliesClaimScan pins that the background retry -// goes through releaseStoredVGPU, not a raw destroy: a live claimant found by -// the vendor VFIO claim scan must keep blocking the release on every retry. func TestOrphanedVGPUReleaseReappliesClaimScan(t *testing.T) { t.Parallel() @@ -123,8 +109,6 @@ func TestOrphanedVGPUReleaseReappliesClaimScan(t *testing.T) { return nil }, } - // A claimant with a recent assignment and no persisted PID makes the scan - // fail closed, exactly like the synchronous release path. require.NoError(t, m.ensureDirectories("mid-boot-claimant")) assignedAt := time.Now() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 24be04fe3..aab2da2a1 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -8,8 +8,6 @@ import ( "github.com/kernel/hypeman/lib/logger" ) -// vgpuReconcileFailureRetryDelay spaces retries after a transient metadata or -// device error would otherwise disable orphan recovery until the next restart. const vgpuReconcileFailureRetryDelay = time.Minute func (m *manager) liveVGPUReconcileProtection(ctx context.Context) (map[string]struct{}, time.Duration, error) { @@ -67,8 +65,6 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { if retryAfter <= 0 { return } - // One pending retry at a time: overlapping calls would fork parallel - // retry chains. if !m.vgpuReconcileRetryPending.CompareAndSwap(false, true) { return } diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 815415d09..03851f40d 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -65,7 +65,6 @@ func TestReconcileVGPUsRetriesAfterListingFailure(t *testing.T) { require.True(t, m.vgpuReconcileRetryPending.Load(), "a listing failure must schedule a retry instead of disabling the vendor sweep until restart") - // Once the listing recovers, the retry runs the sweep and stops rearming. require.NoError(t, os.Chmod(instanceDir, 0o755)) require.Eventually(t, func() bool { return !m.vgpuReconcileRetryPending.Load() @@ -93,32 +92,3 @@ func TestReconcileVGPUsRetriesAfterDeviceFailure(t *testing.T) { return calls.Load() >= 2 && !m.vgpuReconcileRetryPending.Load() }, 5*time.Second, 10*time.Millisecond) } - -func TestVGPUAssignmentLiveness(t *testing.T) { - now := time.Now().UTC() - recent := now.Add(-time.Minute) - stale := now.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) - pid := 123 - - tests := []struct { - name string - stored StoredMetadata - livePID bool - live bool - remaining time.Duration - }{ - {name: "live PID", stored: StoredMetadata{HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}}, livePID: true, live: true}, - {name: "dead PID recent assignment", stored: StoredMetadata{HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, GPUAssignedAt: &recent}, live: true, remaining: VGPUAssignmentStartupGracePeriod - time.Minute}, - {name: "dead PID stale assignment", stored: StoredMetadata{HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, GPUAssignedAt: &stale}}, - {name: "no PID recent assignment", stored: StoredMetadata{GPUAssignedAt: &recent}, live: true, remaining: VGPUAssignmentStartupGracePeriod - time.Minute}, - {name: "no PID stale assignment", stored: StoredMetadata{GPUAssignedAt: &stale}}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - live, remaining := vgpuAssignmentLiveness(&tt.stored, now, tt.livePID) - assert.Equal(t, tt.live, live) - assert.Equal(t, tt.remaining, remaining) - }) - } -} diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index eaca96289..feac72ed2 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -65,8 +65,6 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten if err := m.deleteInstanceData(id); err != nil { log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) } - // No on-disk record points at the device; retry the release in the - // background instead of waiting for the next startup reconcile. m.scheduleOrphanedVGPURelease(ctx, *retainedVGPU) return false } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index cb54c509b..4b31bf7c7 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -67,17 +67,13 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) assert.Equal(t, stored.GPUMdevUUID, retained.GPUMdevUUID) assert.Equal(t, stored.GPUAssignedAt, retained.GPUAssignedAt) - // Identity fields survive so the retained record lists as a - // recognizable, deletable instance instead of a nameless phantom. assert.Equal(t, stored.Name, retained.Name) assert.Equal(t, stored.GPUProfile, retained.GPUProfile) assert.Equal(t, stored.HypervisorType, retained.HypervisorType) assert.Equal(t, stored.DataDir, retained.DataDir) - // Resource claims released by rollback stay dropped. assert.False(t, retained.NetworkEnabled) assert.Empty(t, retained.IP) assert.Empty(t, retained.Volumes) - // The stub has no boot configuration, so it is marked delete-only. assert.True(t, retained.GPURetainedForCleanup) } @@ -162,19 +158,6 @@ func TestVGPURetentionWrapPending(t *testing.T) { assert.True(t, cleanupPending.Retained) } -func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { - t.Parallel() - - cause := errors.New("boot failed") - retained := &VGPUCleanupPendingError{InstanceID: "inst-1", Retained: true, Err: cause} - assert.ErrorIs(t, retained, cause) - assert.Equal(t, "boot failed; vGPU release failed during rollback, instance inst-1 retains the assignment", retained.Error()) - - unpersisted := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause} - assert.ErrorIs(t, unpersisted, cause) - assert.Equal(t, "boot failed; vGPU release failed during rollback and the retention record for instance inst-1 could not be saved; the release is retried in the background and by the next startup reconcile", unpersisted.Error()) -} - func TestVGPUDevicePendingCleanup(t *testing.T) { t.Parallel() @@ -445,15 +428,12 @@ func TestCleanupStartVGPUReportsRetainedWhenMidStartSaveSurvives(t *testing.T) { } assignedAt := time.Now().UTC() - // The mid-start save already persisted the assignment. meta, err := m.loadMetadata(id) require.NoError(t, err) rollbackMeta := *meta setStoredVGPUDevice(&meta.StoredMetadata, &device, assignedAt) require.NoError(t, m.saveMetadata(meta)) - // The cleanup save fails, but the surviving on-disk record still points - // at the device, so retention must be reported as persisted. instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) require.NoError(t, os.Chmod(instanceDir, 0o555)) t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) @@ -603,12 +583,9 @@ func TestVGPUAssignmentClaimedByLiveInstanceGracesRecentDeadPIDClaim(t *testing. GPUAssignedAt: &assignedAt, }})) - // Same bounded grace as startup reconcile: a recent claim whose PID is - // dead fails closed instead of being treated as unclaimed. _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", "/sys/bus/pci/devices/0000:82:00.4") require.Error(t, err) - // Past the grace period the dead claim no longer blocks the release. stale := assignedAt.Add(-2 * VGPUAssignmentStartupGracePeriod) meta, err := m.loadMetadata(claimantID) require.NoError(t, err) From 78d8fa39a7547032bda748f86be7baa464a42b27 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 20:32:09 +0000 Subject: [PATCH 089/107] Track vendor VFIO assignment age and owner for periodic sweeps Record the owning instance and assignment time for each vendor VFIO VF so a reconcile sweep can run while instances are being created: recently assigned VFs get a grace period before they are eligible (mirroring orphanedMdevGracePeriod), and owned VFs are destroyed with their recorded owner ID instead of failing the ownership check. --- lib/devices/vendor_vfio_linux.go | 34 +++++++++++++++++++------ lib/devices/vendor_vfio_linux_test.go | 36 ++++++++++++++++++++++++--- 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 2c576b4d2..74eae4948 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "log/slog" + "maps" "os" "path/filepath" "sort" @@ -14,6 +15,7 @@ import ( "strings" "sync" "syscall" + "time" "github.com/kernel/hypeman/lib/logger" ) @@ -21,13 +23,23 @@ import ( const ( pciDevicesPath = "/sys/bus/pci/devices" vfioDevicesPath = "/dev/vfio/devices" + + // vendorVFIOAssignmentGracePeriod protects assignments created by this + // process from the periodic sweep until their owning instance has had time + // to persist metadata and boot, mirroring orphanedMdevGracePeriod. + vendorVFIOAssignmentGracePeriod = 5 * time.Minute ) +type vendorVFIOOwner struct { + instanceID string + assignedAt time.Time +} + type vendorVFIOSysfs struct { pciDevicesPath string procPath string vfioDevicesPath string - owners map[string]string + owners map[string]vendorVFIOOwner framebufferByType map[string]int openVFIOPathsFunc func() (map[string]struct{}, error) } @@ -37,7 +49,7 @@ var ( pciDevicesPath: pciDevicesPath, procPath: procPath, vfioDevicesPath: vfioDevicesPath, - owners: make(map[string]string), + owners: make(map[string]vendorVFIOOwner), framebufferByType: make(map[string]int), } vendorVFIOMu sync.Mutex @@ -196,7 +208,7 @@ func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID str verifyErr := fmt.Errorf("verify vGPU on VF %s: type is %s, want %s", targetVF, currentType, requested.TypeName) return nil, s.rollbackCreate(currentTypePath, targetVF, instanceID, device, verifyErr) } - s.owners[targetVF] = instanceID + s.owners[targetVF] = vendorVFIOOwner{instanceID: instanceID, assignedAt: time.Now()} logger.FromContext(ctx).InfoContext(ctx, "created vendor VFIO vGPU", "profile", profileName, @@ -229,10 +241,10 @@ func (s vendorVFIOSysfs) destroy(ctx context.Context, vfAddress, instanceID stri if instanceID == "" { return fmt.Errorf("cannot release vendor VFIO vGPU on VF %s without instance ID", vfAddress) } - if owner != instanceID { + if owner.instanceID != instanceID { log.WarnContext(ctx, "skipping vendor VFIO vGPU release owned by another instance", "vf", vfAddress, - "owner_instance_id", owner, + "owner_instance_id", owner.instanceID, "requesting_instance_id", instanceID, ) return nil @@ -264,6 +276,9 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map if err != nil { return err } + vendorVFIOMu.Lock() + owners := maps.Clone(s.owners) + vendorVFIOMu.Unlock() log := logger.FromContext(ctx) protectedVFs := make(map[string]struct{}, len(protectedDevicePaths)) for path := range protectedDevicePaths { @@ -278,6 +293,11 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map log.DebugContext(ctx, "skipping vendor VFIO vGPU held by a live instance", "vf", vf.PCIAddress) continue } + owner := owners[vf.PCIAddress] + if !owner.assignedAt.IsZero() && time.Since(owner.assignedAt) < vendorVFIOAssignmentGracePeriod { + log.DebugContext(ctx, "skipping recently assigned vendor VFIO vGPU during grace period", "vf", vf.PCIAddress) + continue + } if openPaths == nil { if openPaths, err = s.openVFIOPaths(); err != nil { return fmt.Errorf("scan open VFIO handles: %w", err) @@ -292,7 +312,7 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map log.WarnContext(ctx, "preserving vendor VFIO vGPU held open without a live instance claim", "vf", vf.PCIAddress) continue } - if err := s.destroy(ctx, vf.PCIAddress, ""); err != nil { + if err := s.destroy(ctx, vf.PCIAddress, owner.instanceID); err != nil { log.WarnContext(ctx, "failed to destroy orphaned vendor VFIO vGPU", "vf", vf.PCIAddress, "error", err) } } @@ -514,7 +534,7 @@ func framebufferFromProfileName(name string) int { func (s vendorVFIOSysfs) rollbackCreate(currentTypePath, vfAddress, instanceID string, device VGPUDevice, verifyErr error) error { if err := os.WriteFile(currentTypePath, []byte("0"), 0200); err != nil { - s.owners[vfAddress] = instanceID + s.owners[vfAddress] = vendorVFIOOwner{instanceID: instanceID, assignedAt: time.Now()} return &VGPUCreateCleanupPendingError{ Device: device, Err: errors.Join(verifyErr, fmt.Errorf("roll back vGPU on VF %s: %w", vfAddress, err)), diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index bd7293922..f7b0d6c3f 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -372,6 +373,32 @@ func TestVendorVFIOReconcile(t *testing.T) { assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:e3:00.4", "nvidia", "current_vgpu_type"), "1148") } +func TestVendorVFIOReconcileSkipsRecentlyAssignedVF(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.owners["0000:82:00.4"] = vendorVFIOOwner{instanceID: "mid-create", assignedAt: time.Now()} + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcileDestroysOwnedVFPastGracePeriod(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.owners["0000:82:00.4"] = vendorVFIOOwner{ + instanceID: "deleted-instance", + assignedAt: time.Now().Add(-vendorVFIOAssignmentGracePeriod - time.Minute), + } + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "0") + assert.Empty(t, sysfs.owners) +} + func TestVendorVFIOReconcileRechecksOpenHandlesBeforeDestroy(t *testing.T) { t.Parallel() @@ -518,7 +545,7 @@ func TestRollbackVendorVFIOCreate(t *testing.T) { t.Run("preserves verification error", func(t *testing.T) { currentTypePath := filepath.Join(t.TempDir(), "current_vgpu_type") require.NoError(t, os.WriteFile(currentTypePath, []byte("1148"), 0644)) - sysfs := vendorVFIOSysfs{owners: make(map[string]string)} + sysfs := vendorVFIOSysfs{owners: make(map[string]vendorVFIOOwner)} err := sysfs.rollbackCreate(currentTypePath, device.VFAddress, "instance-1", device, verifyErr) require.ErrorIs(t, err, verifyErr) @@ -528,7 +555,7 @@ func TestRollbackVendorVFIOCreate(t *testing.T) { t.Run("retains assignment when rollback fails", func(t *testing.T) { currentTypePath := filepath.Join(t.TempDir(), "missing", "current_vgpu_type") - sysfs := vendorVFIOSysfs{owners: make(map[string]string)} + sysfs := vendorVFIOSysfs{owners: make(map[string]vendorVFIOOwner)} err := sysfs.rollbackCreate(currentTypePath, device.VFAddress, "instance-1", device, verifyErr) require.ErrorIs(t, err, verifyErr) @@ -536,7 +563,8 @@ func TestRollbackVendorVFIOCreate(t *testing.T) { var pending *VGPUCreateCleanupPendingError require.ErrorAs(t, err, &pending) assert.Equal(t, device, pending.Device) - assert.Equal(t, "instance-1", sysfs.owners[device.VFAddress]) + assert.Equal(t, "instance-1", sysfs.owners[device.VFAddress].instanceID) + assert.False(t, sysfs.owners[device.VFAddress].assignedAt.IsZero()) }) } @@ -566,7 +594,7 @@ func newTestVendorVFIOSysfs(t *testing.T) testVendorVFIOSysfs { pciDevicesPath: pci, procPath: proc, vfioDevicesPath: vfio, - owners: make(map[string]string), + owners: make(map[string]vendorVFIOOwner), framebufferByType: make(map[string]int), }} } From 937caf1f66ec10ad9d35535326ac9a40e33c2545 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 20:32:16 +0000 Subject: [PATCH 090/107] Replace vGPU release retry machinery with a periodic reconciler Run the fail-closed vGPU reconcile once at startup and every minute after, skipping hosts without a vGPU framework. Each pass retries releases for assignments whose owner is no longer live (re-verified under the instance lock) and then sweeps device-level leftovers with no live metadata claim. This deletes the per-path orphan retry goroutines - whose path-keyed dedup could drop cleanup for a newer assignment reusing the same VF - the CAS/timer retry in ReconcileVGPUs, the stopped-instance release special case in StopInstance, the retention fallbacks that scheduled background retries, and the orphan-abandoned metric. --- cmd/api/main.go | 4 +- lib/devices/GPU.md | 2 +- lib/instances/delete.go | 3 +- lib/instances/lifecycle_noop_test.go | 44 ++---- lib/instances/manager.go | 17 +- lib/instances/metrics.go | 17 -- lib/instances/start.go | 3 +- lib/instances/vgpu.go | 30 +--- lib/instances/vgpu_orphan.go | 59 ------- lib/instances/vgpu_orphan_test.go | 130 --------------- lib/instances/vgpu_reconcile.go | 142 +++++++++++------ lib/instances/vgpu_reconcile_test.go | 226 +++++++++++++++++++++++---- lib/instances/vgpu_retention.go | 3 +- lib/instances/vgpu_test.go | 12 +- 14 files changed, 321 insertions(+), 371 deletions(-) delete mode 100644 lib/instances/vgpu_orphan.go delete mode 100644 lib/instances/vgpu_orphan_test.go diff --git a/cmd/api/main.go b/cmd/api/main.go index 3bd0fff39..faa9ac15a 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -385,8 +385,8 @@ func run() error { } logger.Info("Reconciling vGPU devices...") - if r, ok := app.InstanceManager.(interface{ ReconcileVGPUs(context.Context) }); ok { - r.ReconcileVGPUs(ctx) + if r, ok := app.InstanceManager.(interface{ StartVGPUReconciler(context.Context) }); ok { + r.StartVGPUReconciler(ctx) } // Wire up resource validator for aggregate limit checking diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index ff11c5cde..d04dcf599 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -97,7 +97,7 @@ Instance Create → Assign profile to VF → Attach VF to VM → Instance Runnin Instance Stop/Delete → Release profile → VF available again ``` -Hypeman reconciles orphaned assignments on server restart while preserving devices held open by a running VMM. A release that fails during delete (typically because a GPU-busy VMM's kernel-side VFIO teardown outlives the force-kill wait) is retried in the background for up to ten minutes, so a completed delete does not strand the VF until the next restart. +Hypeman reconciles orphaned assignments with a periodic fail-closed pass: once at startup and every minute afterward (skipped entirely on hosts without GPUs). Each pass releases assignments whose owning instance is no longer live and clears their metadata, then sweeps device-level leftovers with no live metadata claim. Devices held open by a running VMM and assignments younger than five minutes are preserved, so a release that fails during stop or delete (typically because a GPU-busy VMM's kernel-side VFIO teardown outlives the force-kill wait) is simply retried on later passes until the device is free. ### Hypervisor Support diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 160ae807d..d2f0e63d9 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -146,8 +146,7 @@ func (m *manager) deleteInstanceWithOptions( log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID) } if err := m.releaseStoredVGPU(ctx, stored); err != nil { - log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) - m.scheduleOrphanedVGPURelease(ctx, *stored) + log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup; the periodic vGPU reconcile releases it once free", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) } else if hadVGPUAssignment { if err := m.saveMetadata(meta); err != nil { log.WarnContext(ctx, "failed to save metadata after vGPU release", "instance_id", id, "error", err) diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 90f2e7aa1..1843485e3 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -152,7 +152,6 @@ func TestLifecycleNoopStandbyWithOptionsStillRejectsStandbyInstance(t *testing.T func TestDeleteContinuesWhenVGPUReleaseFails(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) - m.orphanedVGPURetryDelay = time.Millisecond meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" @@ -162,9 +161,8 @@ func TestDeleteContinuesWhenVGPUReleaseFails(t *testing.T) { // A failed release is logged and the delete continues, matching the // pre-refactor contract; the leaked assignment is recovered by the - // background retry or startup reconciliation. + // periodic vGPU reconcile. require.NoError(t, m.DeleteInstance(context.Background(), id)) - waitForOrphanQueueEmpty(t, m) _, err = m.loadMetadata(id) require.Error(t, err, "instance data must be deleted despite the failed release") @@ -280,7 +278,6 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { func TestDeleteContinuesTeardownAfterFailedVGPURelease(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) - m.orphanedVGPURetryDelay = time.Millisecond deviceManager := &recordingDeviceManager{} m.deviceManager = deviceManager meta, err := m.loadMetadata(id) @@ -294,7 +291,6 @@ func TestDeleteContinuesTeardownAfterFailedVGPURelease(t *testing.T) { // The failed release must not block the rest of the teardown: devices // are detached and the instance is fully deleted. require.NoError(t, m.DeleteInstance(context.Background(), id)) - waitForOrphanQueueEmpty(t, m) assert.Equal(t, []string{"dev-1"}, deviceManager.detached) _, err = m.loadMetadata(id) @@ -343,8 +339,9 @@ func TestStartRejectsVGPURetentionRecord(t *testing.T) { assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) } -func TestStopStoppedInstanceReleasesRetainedVGPU(t *testing.T) { +func TestReconcileReleasesRetainedVGPUOnStoppedInstance(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) + m.reconcileVGPUDevices = func(context.Context, map[string]struct{}, bool) error { return nil } meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" @@ -352,38 +349,26 @@ func TestStopStoppedInstanceReleasesRetainedVGPU(t *testing.T) { meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" require.NoError(t, m.saveMetadata(meta)) + // Stop on an already-stopped instance is a no-op for the assignment; the + // periodic reconcile retries the release. inst, err := m.StopInstance(context.Background(), id) require.NoError(t, err) require.NotNil(t, inst) assert.Equal(t, StateStopped, inst.State) - stored, err := m.loadMetadata(id) require.NoError(t, err) - assert.Empty(t, stored.GPUDevicePath) -} - -func TestStopStoppedInstanceLeavesRetentionStubForDelete(t *testing.T) { - m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) - meta, err := m.loadMetadata(id) - require.NoError(t, err) - meta.GPUProfile = "NVIDIA L40S-2Q" - meta.GPUFramework = devices.VGPUFrameworkNone - meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" - meta.GPURetainedForCleanup = true - require.NoError(t, m.saveMetadata(meta)) - - inst, err := m.StopInstance(context.Background(), id) - require.NoError(t, err) - require.NotNil(t, inst) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) - stored, err := m.loadMetadata(id) + m.ReconcileVGPUs(context.Background()) + stored, err = m.loadMetadata(id) require.NoError(t, err) - assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) - assert.True(t, stored.GPURetainedForCleanup) + assert.Empty(t, stored.GPUDevicePath) + assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile, "profile is kept for the next start") } -func TestStopStoppedInstanceVGPUReleaseFailureRemainsNoop(t *testing.T) { +func TestReconcileVGPUReleaseFailureKeepsStoppedInstanceUsable(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) + m.reconcileVGPUDevices = func(context.Context, map[string]struct{}, bool) error { return nil } meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" @@ -391,10 +376,7 @@ func TestStopStoppedInstanceVGPUReleaseFailureRemainsNoop(t *testing.T) { meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" require.NoError(t, m.saveMetadata(meta)) - inst, err := m.StopInstance(context.Background(), id) - require.NoError(t, err) - require.NotNil(t, inst) - assert.Equal(t, StateStopped, inst.State) + m.ReconcileVGPUs(context.Background()) stored, err := m.loadMetadata(id) require.NoError(t, err) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 7cd94262a..8c298ed24 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -8,7 +8,6 @@ import ( "path/filepath" "strings" "sync" - "sync/atomic" "time" "github.com/kernel/hypeman/lib/devices" @@ -215,12 +214,10 @@ type manager struct { // Periodic TAP garbage collection reconciler. tapGCOnce sync.Once - orphanedVGPUMu sync.Mutex - orphanedVGPUs map[string]struct{} - orphanedVGPURetryDelay time.Duration - - vgpuReconcileRetryPending atomic.Bool - vgpuReconcileRetryDelay time.Duration + // Periodic vGPU reconciler. + vgpuReconcileOnce sync.Once + vgpuReconcileInterval time.Duration + discoverVGPU func() (devices.VGPUFramework, []devices.VirtualFunction, error) vgpuInitTermGrace time.Duration @@ -665,12 +662,6 @@ func (m *manager) StopInstance(ctx context.Context, id string) (*Instance, error if err := m.markRestartManualStopLocked(ctx, id); err != nil { return nil, err } - // A stopped instance can retain a vGPU assignment when the release - // failed during the original stop. Retry it here so the vGPU slot is - // not held until the next start, delete, or hypeman restart. A failed - // retry only logs, keeping stop's no-op contract for already-stopped - // instances. - m.releaseRetainedVGPULocked(ctx, id) updated, err := m.currentInstanceWithoutHydration(ctx, id) if err != nil { return nil, err diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index de71c2686..1ada5ac1e 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -94,7 +94,6 @@ type Metrics struct { lifecycleEventsDroppedTotal metric.Int64Counter forkMemFileShareFallbacksTotal metric.Int64Counter ttlReaperDeletionsTotal metric.Int64Counter - vgpuOrphanReleasesAbandonedTotal metric.Int64Counter tracer trace.Tracer } @@ -271,14 +270,6 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } - vgpuOrphanReleasesAbandonedTotal, err := meter.Int64Counter( - "hypeman_instances_vgpu_orphan_releases_abandoned_total", - metric.WithDescription("Total orphaned vGPU release retries that gave up, leaving the VF allocated until startup reconciliation or manual remediation"), - ) - if err != nil { - return nil, err - } - // Register observable gauge for instance counts by state instancesTotal, err := meter.Int64ObservableGauge( "hypeman_instances_total", @@ -473,7 +464,6 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M lifecycleEventsDroppedTotal: lifecycleEventsDroppedTotal, forkMemFileShareFallbacksTotal: forkMemFileShareFallbacksTotal, ttlReaperDeletionsTotal: ttlReaperDeletionsTotal, - vgpuOrphanReleasesAbandonedTotal: vgpuOrphanReleasesAbandonedTotal, tracer: tracer, }, nil } @@ -573,13 +563,6 @@ func (m *manager) recordTimeToRunning(ctx context.Context, stored *StoredMetadat m.metrics.timeToRunning.Record(ctx, duration, metric.WithAttributes(attrs...)) } -func (m *manager) recordVGPUOrphanReleaseAbandoned(ctx context.Context) { - if m.metrics == nil { - return - } - m.metrics.vgpuOrphanReleasesAbandonedTotal.Add(ctx, 1) -} - // recordStateTransition records a state transition with hypervisor label. func (m *manager) recordStateTransition(ctx context.Context, fromState, toState string, hvType hypervisor.Type) { if m.metrics == nil { diff --git a/lib/instances/start.go b/lib/instances/start.go index 8986d7fb3..19bb64697 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -55,7 +55,7 @@ func (m *manager) startInstance( // Release any assignment retained by an earlier failed release and // persist the cleared fields immediately, so a failure later in start // cannot leave on-disk metadata pointing at a device that is already - // gone (matching releaseRetainedVGPULocked). + // gone. if storedVGPUDevicePath(stored) != "" { if err := m.releaseStoredVGPU(ctx, stored); err != nil { log.ErrorContext(ctx, "failed to release stale vGPU before start", "instance_id", id, "error", err) @@ -184,7 +184,6 @@ func (m *manager) startInstance( wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) - m.scheduleOrphanedVGPURelease(ctx, retentionMeta.StoredMetadata) return nil, &VGPUCleanupPendingError{InstanceID: id, Err: fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr)} } return nil, &VGPUCleanupPendingError{InstanceID: id, Retained: true, Err: wrapped} diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 0e09f2f34..3fb26778a 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -26,7 +26,7 @@ func (e *VGPUCleanupPendingError) Error() string { if e.Retained { return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID) } - return fmt.Sprintf("%v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the release is retried in the background and by the next startup reconcile", e.Err, e.InstanceID) + return fmt.Sprintf("%v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the periodic vGPU reconcile retries the release", e.Err, e.InstanceID) } func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } @@ -118,7 +118,6 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic if meta, loadErr := m.loadMetadata(instanceID); loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath { return true, true } - m.scheduleOrphanedVGPURelease(ctx, cleanupMeta.StoredMetadata) return true, false } return retained, retained @@ -220,33 +219,6 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu return false, nil } -// releaseRetainedVGPULocked releases a vGPU assignment retained on a stopped -// instance after a failed release during the original stop. It is a no-op -// when no assignment is retained, and a failed retry only logs so the -// metadata stays for the next retry. The caller must hold the instance lock. -func (m *manager) releaseRetainedVGPULocked(ctx context.Context, id string) { - log := logger.FromContext(ctx) - meta, err := m.loadMetadata(id) - if err != nil { - log.WarnContext(ctx, "failed to load metadata for retained vGPU release", "instance_id", id, "error", err) - return - } - stored := &meta.StoredMetadata - if stored.GPURetainedForCleanup { - return - } - if storedVGPUDevicePath(stored) == "" { - return - } - if err := m.releaseStoredVGPU(ctx, stored); err != nil { - log.WarnContext(ctx, "failed to destroy retained vGPU; retaining assignment metadata", "instance_id", id, "error", err) - return - } - if err := m.saveMetadata(meta); err != nil { - log.WarnContext(ctx, "failed to save metadata after retained vGPU release", "instance_id", id, "error", err) - } -} - func storedVGPUDevicePath(stored *StoredMetadata) string { if stored.GPUDevicePath != "" { return stored.GPUDevicePath diff --git a/lib/instances/vgpu_orphan.go b/lib/instances/vgpu_orphan.go deleted file mode 100644 index 041402642..000000000 --- a/lib/instances/vgpu_orphan.go +++ /dev/null @@ -1,59 +0,0 @@ -package instances - -import ( - "context" - "time" - - "github.com/kernel/hypeman/lib/logger" -) - -const ( - orphanedVGPUReleaseMaxAttempts = 20 - defaultOrphanedVGPUReleaseRetryDelay = 30 * time.Second -) - -func (m *manager) scheduleOrphanedVGPURelease(ctx context.Context, stored StoredMetadata) { - path := storedVGPUDevicePath(&stored) - if path == "" { - return - } - m.orphanedVGPUMu.Lock() - if m.orphanedVGPUs == nil { - m.orphanedVGPUs = make(map[string]struct{}) - } - if _, pending := m.orphanedVGPUs[path]; pending { - m.orphanedVGPUMu.Unlock() - return - } - m.orphanedVGPUs[path] = struct{}{} - m.orphanedVGPUMu.Unlock() - - delay := m.orphanedVGPURetryDelay - if delay <= 0 { - delay = defaultOrphanedVGPUReleaseRetryDelay - } - go m.retryOrphanedVGPURelease(context.WithoutCancel(ctx), stored, path, delay) -} - -func (m *manager) retryOrphanedVGPURelease(ctx context.Context, stored StoredMetadata, path string, delay time.Duration) { - log := logger.FromContext(ctx) - defer func() { - m.orphanedVGPUMu.Lock() - delete(m.orphanedVGPUs, path) - m.orphanedVGPUMu.Unlock() - }() - for attempt := 1; attempt <= orphanedVGPUReleaseMaxAttempts; attempt++ { - time.Sleep(delay) - if err := m.releaseStoredVGPUExcluding(ctx, &stored, ""); err != nil { - log.WarnContext(ctx, "orphaned vGPU release retry failed", - "instance_id", stored.Id, "device_path", path, "attempt", attempt, "error", err) - continue - } - log.InfoContext(ctx, "released orphaned vGPU after delete", - "instance_id", stored.Id, "device_path", path, "attempt", attempt) - return - } - m.recordVGPUOrphanReleaseAbandoned(ctx) - log.ErrorContext(ctx, "giving up on orphaned vGPU release; VF stays allocated until startup reconciliation or manual remediation", - "instance_id", stored.Id, "device_path", path, "attempts", orphanedVGPUReleaseMaxAttempts) -} diff --git a/lib/instances/vgpu_orphan_test.go b/lib/instances/vgpu_orphan_test.go deleted file mode 100644 index ee8576a3b..000000000 --- a/lib/instances/vgpu_orphan_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package instances - -import ( - "context" - "errors" - "sync/atomic" - "testing" - "time" - - "github.com/kernel/hypeman/lib/devices" - "github.com/kernel/hypeman/lib/paths" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func waitForOrphanQueueEmpty(t *testing.T, m *manager) { - t.Helper() - require.Eventually(t, func() bool { - m.orphanedVGPUMu.Lock() - defer m.orphanedVGPUMu.Unlock() - return len(m.orphanedVGPUs) == 0 - }, 5*time.Second, 5*time.Millisecond, "orphan retry should finish and clear its queue entry") -} - -func TestScheduleOrphanedVGPUReleaseRetriesUntilSuccess(t *testing.T) { - t.Parallel() - - var attempts atomic.Int32 - m := &manager{ - paths: paths.New(t.TempDir()), - orphanedVGPURetryDelay: time.Millisecond, - destroyVGPU: func(context.Context, devices.VGPUAssignment) error { - if attempts.Add(1) < 3 { - return errors.New("operation not permitted") - } - return nil - }, - } - m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{ - Id: "deleted-instance", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - }) - - waitForOrphanQueueEmpty(t, m) - assert.Equal(t, int32(3), attempts.Load(), "release should succeed on the third attempt and stop retrying") -} - -func TestScheduleOrphanedVGPUReleaseGivesUpAfterMaxAttempts(t *testing.T) { - t.Parallel() - - var attempts atomic.Int32 - m := &manager{ - paths: paths.New(t.TempDir()), - orphanedVGPURetryDelay: time.Millisecond, - destroyVGPU: func(context.Context, devices.VGPUAssignment) error { - attempts.Add(1) - return errors.New("vGPU destroy failed: 0xffffffff") - }, - } - m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{ - Id: "deleted-instance", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - }) - - waitForOrphanQueueEmpty(t, m) - assert.Equal(t, int32(orphanedVGPUReleaseMaxAttempts), attempts.Load(), - "a wedged VF should get exactly the bounded number of attempts") -} - -func TestScheduleOrphanedVGPUReleaseDeduplicatesByDevicePath(t *testing.T) { - t.Parallel() - - var attempts atomic.Int32 - release := make(chan struct{}) - m := &manager{ - paths: paths.New(t.TempDir()), - orphanedVGPURetryDelay: 20 * time.Millisecond, - destroyVGPU: func(context.Context, devices.VGPUAssignment) error { - attempts.Add(1) - <-release - return nil - }, - } - stored := StoredMetadata{ - Id: "deleted-instance", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - } - m.scheduleOrphanedVGPURelease(context.Background(), stored) - m.scheduleOrphanedVGPURelease(context.Background(), stored) - - require.Eventually(t, func() bool { return attempts.Load() == 1 }, 5*time.Second, 5*time.Millisecond) - close(release) - waitForOrphanQueueEmpty(t, m) - assert.Equal(t, int32(1), attempts.Load(), "the second schedule for the same path must be dropped") -} - -func TestOrphanedVGPUReleaseReappliesClaimScan(t *testing.T) { - t.Parallel() - - var destroys atomic.Int32 - m := &manager{ - paths: paths.New(t.TempDir()), - orphanedVGPURetryDelay: time.Millisecond, - destroyVGPU: func(context.Context, devices.VGPUAssignment) error { - destroys.Add(1) - return nil - }, - } - require.NoError(t, m.ensureDirectories("mid-boot-claimant")) - assignedAt := time.Now() - require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "mid-boot-claimant", - Name: "mid-boot-claimant", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - GPUAssignedAt: &assignedAt, - }})) - - m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{ - Id: "deleted-instance", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - }) - - waitForOrphanQueueEmpty(t, m) - assert.Zero(t, destroys.Load(), "no destroy may fire while the claim scan cannot clear the path") -} diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index aab2da2a1..966bf1118 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -2,51 +2,62 @@ package instances import ( "context" + "errors" "time" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/logger" ) -const vgpuReconcileFailureRetryDelay = time.Minute +const defaultVGPUReconcileInterval = time.Minute -func (m *manager) liveVGPUReconcileProtection(ctx context.Context) (map[string]struct{}, time.Duration, error) { - allInstances, err := m.listInstancesForReconcile(ctx) +// StartVGPUReconciler runs one reconcile pass and then keeps reconciling +// periodically until ctx is cancelled. Hosts without a vGPU framework skip +// reconciliation entirely. A discovery failure starts the reconciler anyway: +// a transient sysfs error must not disable cleanup on a GPU host. +func (m *manager) StartVGPUReconciler(ctx context.Context) { + discover := m.discoverVGPU + if discover == nil { + discover = devices.DiscoverVGPU + } + framework, _, err := discover() + if err == nil && framework == devices.VGPUFrameworkNone { + return + } if err != nil { - return nil, 0, err + logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU framework; starting vGPU reconciler anyway", "error", err) } - protected := make(map[string]struct{}) - var retryAfter time.Duration - for i := range allInstances { - stored := &allInstances[i].StoredMetadata - if stored.GPUDevicePath == "" { - continue + m.ReconcileVGPUs(ctx) + m.vgpuReconcileOnce.Do(func() { + interval := m.vgpuReconcileInterval + if interval <= 0 { + interval = defaultVGPUReconcileInterval } - livePID := stored.HypervisorPID != nil && hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) - live, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID) - if !live { - continue - } - protected[stored.GPUDevicePath] = struct{}{} - if remaining > 0 && (retryAfter == 0 || remaining < retryAfter) { - retryAfter = remaining - } - } - return protected, retryAfter, nil + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + m.ReconcileVGPUs(ctx) + } + } + }() + }) } -// ReconcileVGPUs releases orphaned vGPU assignments. +// ReconcileVGPUs runs one fail-closed reconcile pass: stale instance-held +// assignments are released, then device-level leftovers not claimed by a live +// instance are swept. Failures only log; the next periodic pass retries. func (m *manager) ReconcileVGPUs(ctx context.Context) { log := logger.FromContext(ctx) - protected, retryAfter, err := m.liveVGPUReconcileProtection(ctx) + protected, err := m.reconcileVGPUAssignments(ctx) sweepVendorVFIO := err == nil if err != nil { - log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconcile, mdev reconcile still runs", "error", err) + log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping vendor VFIO sweep until the next pass, mdev reconcile still runs", "error", err) protected = make(map[string]struct{}) - retryAfter = vgpuReconcileFailureRetryDelay - if m.vgpuReconcileRetryDelay > 0 { - retryAfter = m.vgpuReconcileRetryDelay - } } reconcileDevices := m.reconcileVGPUDevices if reconcileDevices == nil { @@ -54,29 +65,68 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { } if err := reconcileDevices(ctx, protected, sweepVendorVFIO); err != nil { log.WarnContext(ctx, "failed to reconcile vGPU devices", "error", err) - deviceRetryAfter := vgpuReconcileFailureRetryDelay - if m.vgpuReconcileRetryDelay > 0 { - deviceRetryAfter = m.vgpuReconcileRetryDelay + } +} + +// reconcileVGPUAssignments retries releases for assignments whose owner is no +// longer live and returns the device paths still protected by live instances. +// Listing fails closed: any unreadable metadata aborts the pass so the vendor +// VFIO sweep cannot clear a VF whose claim it could not read. +func (m *manager) reconcileVGPUAssignments(ctx context.Context) (map[string]struct{}, error) { + allInstances, err := m.listInstancesForReconcile(ctx) + if err != nil { + return nil, err + } + protected := make(map[string]struct{}) + for i := range allInstances { + stored := &allInstances[i].StoredMetadata + if storedVGPUDevicePath(stored) == "" { + continue } - if retryAfter <= 0 || deviceRetryAfter < retryAfter { - retryAfter = deviceRetryAfter + livePID := stored.HypervisorPID != nil && hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + if live, _ := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID); live { + if stored.GPUDevicePath != "" { + protected[stored.GPUDevicePath] = struct{}{} + } + continue } + m.releaseStaleVGPUAssignment(ctx, stored.Id) } - if retryAfter <= 0 { + return protected, nil +} + +// releaseStaleVGPUAssignment retries a release that previously failed, under +// the instance lock. Liveness is re-verified after locking so a concurrent +// start or restore keeps its assignment. A failed release only logs and keeps +// the metadata for the next pass. +func (m *manager) releaseStaleVGPUAssignment(ctx context.Context, id string) { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + log := logger.FromContext(ctx) + meta, err := m.loadMetadata(id) + if err != nil { + if !errors.Is(err, ErrNotFound) { + log.WarnContext(ctx, "failed to load metadata for stale vGPU release", "instance_id", id, "error", err) + } return } - if !m.vgpuReconcileRetryPending.CompareAndSwap(false, true) { + stored := &meta.StoredMetadata + path := storedVGPUDevicePath(stored) + if path == "" { return } - go func() { - timer := time.NewTimer(retryAfter) - defer timer.Stop() - select { - case <-ctx.Done(): - m.vgpuReconcileRetryPending.Store(false) - case <-timer.C: - m.vgpuReconcileRetryPending.Store(false) - m.ReconcileVGPUs(ctx) - } - }() + livePID := stored.HypervisorPID != nil && hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + if live, _ := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID); live { + return + } + if err := m.releaseStoredVGPU(ctx, stored); err != nil { + log.WarnContext(ctx, "failed to release stale vGPU assignment; retrying on the next reconcile pass", "instance_id", id, "device_path", path, "error", err) + return + } + if err := m.saveMetadata(meta); err != nil { + log.WarnContext(ctx, "failed to save metadata after stale vGPU release", "instance_id", id, "error", err) + return + } + log.InfoContext(ctx, "released stale vGPU assignment", "instance_id", id, "device_path", path) } diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 03851f40d..468591dee 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -10,12 +10,13 @@ import ( "testing" "time" + "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestLiveVGPUReconcileProtectionBoundsStartupProtection(t *testing.T) { +func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { dead := exec.Command("true") require.NoError(t, dead.Run()) deadPID := dead.Process.Pid @@ -23,7 +24,17 @@ func TestLiveVGPUReconcileProtectionBoundsStartupProtection(t *testing.T) { recent := now.Add(-time.Minute) stale := now.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) - m := &manager{paths: paths.New(t.TempDir()), now: func() time.Time { return now }} + var protected map[string]struct{} + m := &manager{ + paths: paths.New(t.TempDir()), + now: func() time.Time { return now }, + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, + reconcileVGPUDevices: func(_ context.Context, p map[string]struct{}, sweepVendorVFIO bool) error { + protected = p + assert.True(t, sweepVendorVFIO) + return nil + }, + } instances := []StoredMetadata{ {Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}, {Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}, @@ -36,22 +47,38 @@ func TestLiveVGPUReconcileProtectionBoundsStartupProtection(t *testing.T) { require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: instances[i]})) } - protected, retryAfter, err := m.liveVGPUReconcileProtection(t.Context()) - require.NoError(t, err) - assert.Equal(t, VGPUAssignmentStartupGracePeriod-time.Minute, retryAfter) + m.ReconcileVGPUs(t.Context()) assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") + + for _, id := range []string{"orphaned", "legacy", "dead"} { + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "stale assignment on %s must be released", id) + } + for _, id := range []string{"booting", "stale-pid-booting"} { + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.NotEmpty(t, stored.GPUDevicePath, "live assignment on %s must be kept", id) + } } -func TestReconcileVGPUsRetriesAfterListingFailure(t *testing.T) { +func TestReconcileVGPUsSkipsVendorSweepWhenListingFails(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root bypasses directory permissions") } - m := &manager{paths: paths.New(t.TempDir()), vgpuReconcileRetryDelay: 250 * time.Millisecond} + var sweeps []bool + m := &manager{ + paths: paths.New(t.TempDir()), + reconcileVGPUDevices: func(_ context.Context, _ map[string]struct{}, sweepVendorVFIO bool) error { + sweeps = append(sweeps, sweepVendorVFIO) + return nil + }, + } const id = "unreadable" require.NoError(t, m.ensureDirectories(id)) require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{Id: id}})) @@ -59,36 +86,181 @@ func TestReconcileVGPUsRetriesAfterListingFailure(t *testing.T) { require.NoError(t, os.Chmod(instanceDir, 0o000)) t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - m.ReconcileVGPUs(ctx) - require.True(t, m.vgpuReconcileRetryPending.Load(), - "a listing failure must schedule a retry instead of disabling the vendor sweep until restart") + m.ReconcileVGPUs(t.Context()) + require.Equal(t, []bool{false}, sweeps, + "a listing failure must skip the vendor sweep, not run it with an empty protection set") require.NoError(t, os.Chmod(instanceDir, 0o755)) - require.Eventually(t, func() bool { - return !m.vgpuReconcileRetryPending.Load() - }, 5*time.Second, 10*time.Millisecond) + m.ReconcileVGPUs(t.Context()) + assert.Equal(t, []bool{false, true}, sweeps, "the next pass retries the vendor sweep") +} + +func TestReconcileVGPUsReleasesStaleAssignment(t *testing.T) { + t.Parallel() + + var destroyed []devices.VGPUAssignment + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, + } + const id = "stopped-retained" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUProfile: "NVIDIA L40S-2Q", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }})) + + m.ReconcileVGPUs(t.Context()) + + require.Len(t, destroyed, 1) + assert.Equal(t, devices.VGPUAssignment{ + Framework: devices.VGPUFrameworkVendorVFIO, + DevicePath: "/sys/bus/pci/devices/0000:82:00.4", + InstanceID: id, + }, destroyed[0]) + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath) + assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile, "profile is kept for the next start") +} + +func TestReconcileVGPUsKeepsAssignmentWhenReleaseFails(t *testing.T) { + t.Parallel() + + var attempts atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + if attempts.Add(1) == 1 { + return errors.New("vGPU destroy failed: 0xffffffff") + } + return nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, + } + const id = "wedged" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }})) + + m.ReconcileVGPUs(t.Context()) + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath, + "a failed release must keep the assignment metadata for the next pass") + + m.ReconcileVGPUs(t.Context()) + stored, err = m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "the next pass retries the release") + assert.Equal(t, int32(2), attempts.Load()) +} + +func TestReconcileVGPUsDefersToUnconfirmedClaimant(t *testing.T) { + t.Parallel() + + var destroys atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + destroys.Add(1) + return nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, + } + const path = "/sys/bus/pci/devices/0000:82:00.4" + assignedAt := time.Now() + for _, stored := range []StoredMetadata{ + {Id: "stale", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: path}, + {Id: "mid-boot-claimant", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: path, GPUAssignedAt: &assignedAt}, + } { + require.NoError(t, m.ensureDirectories(stored.Id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: stored})) + } + + m.ReconcileVGPUs(t.Context()) + + assert.Zero(t, destroys.Load(), "no destroy may fire while the claim scan cannot clear the path") + stale, err := m.loadMetadata("stale") + require.NoError(t, err) + assert.Equal(t, path, stale.GPUDevicePath, "the stale release retries once the claimant's liveness is decidable") +} + +func TestReconcileVGPUsReleasesRetentionStubAssignment(t *testing.T) { + t.Parallel() + + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, + } + const id = "retention-stub" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + Name: "failed-create", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPURetainedForCleanup: true, + }})) + + m.ReconcileVGPUs(t.Context()) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "the stub's wedged assignment is released once free") + assert.True(t, stored.GPURetainedForCleanup, "the stub stays a delete-only record of the failed create") } -func TestReconcileVGPUsRetriesAfterDeviceFailure(t *testing.T) { - var calls atomic.Int32 +func TestStartVGPUReconcilerSkipsHostsWithoutGPUs(t *testing.T) { + t.Parallel() + + var passes atomic.Int32 m := &manager{ - paths: paths.New(t.TempDir()), - vgpuReconcileRetryDelay: 10 * time.Millisecond, + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkNone, nil, nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { + passes.Add(1) + return nil + }, + vgpuReconcileInterval: time.Millisecond, + } + + m.StartVGPUReconciler(t.Context()) + time.Sleep(20 * time.Millisecond) + assert.Zero(t, passes.Load(), "a host without GPUs must not reconcile at all") +} + +func TestStartVGPUReconcilerRunsPeriodically(t *testing.T) { + t.Parallel() + + var passes atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkVendorVFIO, nil, nil + }, reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { - if calls.Add(1) == 1 { + if passes.Add(1) == 1 { return errors.New("transient device error") } return nil }, + vgpuReconcileInterval: time.Millisecond, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - m.ReconcileVGPUs(ctx) - require.True(t, m.vgpuReconcileRetryPending.Load()) - require.Eventually(t, func() bool { - return calls.Load() >= 2 && !m.vgpuReconcileRetryPending.Load() - }, 5*time.Second, 10*time.Millisecond) + m.StartVGPUReconciler(t.Context()) + require.Eventually(t, func() bool { return passes.Load() >= 3 }, 5*time.Second, time.Millisecond, + "periodic passes must keep running after a failed pass") } diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index feac72ed2..121c13cb0 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -57,6 +57,8 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten id := retention.instanceID retainedVGPU := retention.stub log := logger.FromContext(ctx) + // When the retention record is lost, the assignment has no metadata claim + // left; the periodic vGPU reconcile sweeps the device once it is free. retentionSurvives := func() bool { meta, err := m.loadMetadata(id) if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { @@ -65,7 +67,6 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten if err := m.deleteInstanceData(id); err != nil { log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) } - m.scheduleOrphanedVGPURelease(ctx, *retainedVGPU) return false } if err := m.deleteInstanceData(id); err != nil { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 4b31bf7c7..e5ed8d13f 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -106,12 +106,7 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { } assert.False(t, persistTestVGPURetention(m, context.Background(), stored.Id, stored)) _, err := m.loadMetadata(id) - require.Error(t, err) - - m.orphanedVGPUMu.Lock() - _, queued := m.orphanedVGPUs[stored.GPUDevicePath] - m.orphanedVGPUMu.Unlock() - assert.True(t, queued, "unpersisted retention must queue a background release") + require.Error(t, err, "the lost retention leaves no metadata claim, so the periodic sweep releases the VF") } func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T) { @@ -332,11 +327,6 @@ func TestStartReportsUnretainedVGPUWhenRetentionSaveFails(t *testing.T) { stored, err := m.loadMetadata(id) require.NoError(t, err) assert.Empty(t, stored.GPUDevicePath, "retention save failed, so no assignment should be recorded") - - m.orphanedVGPUMu.Lock() - _, queued := m.orphanedVGPUs[device.SysfsPath] - m.orphanedVGPUMu.Unlock() - assert.True(t, queued, "unpersisted retention must queue a background release") } func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { From d1ac2f2742e964232184221cd672194dd0ab3d9d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:21:01 +0000 Subject: [PATCH 091/107] Update vgpu_cleanup_pending assertions for the periodic reconcile message --- cmd/api/api/instances_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index fd506023d..5f826dc9c 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -104,7 +104,7 @@ func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance( assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") assert.Contains(t, pending.Message, network.ErrNameExists.Error(), "the underlying create failure must survive the cleanup guidance") - assert.Contains(t, pending.Message, "startup reconcile") + assert.Contains(t, pending.Message, "periodic vGPU reconcile") assert.NotContains(t, pending.Message, "delete") require.NotNil(t, pending.InnerError) require.NotNil(t, pending.InnerError.Code) @@ -1129,7 +1129,7 @@ func TestStartInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") - assert.Contains(t, pending.Message, "startup reconcile") + assert.Contains(t, pending.Message, "periodic vGPU reconcile") assert.NotContains(t, pending.Message, "delete") require.NotNil(t, pending.InnerError) require.NotNil(t, pending.InnerError.Code) From 93cc7550fcb763c14734d9d578ccdf4c77248b44 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:21:01 +0000 Subject: [PATCH 092/107] Check socket ownership for vGPU liveness without a persisted PID An assignment could lose its persisted hypervisor PID while its VM stays alive - a post-boot metadata save failure or a hypeman crash before the save. The liveness checks gated the socket-ownership scan on a non-nil PID, so once the startup grace expired the reconciler considered such an assignment stale and could remove an mdev out from under the live VM (DestroyMdev has no in-use guard). Run the socket-ownership scan unconditionally: a live VMM always holds its control-socket listener, and a missing listener still resolves to not-alive, so genuinely stopped instances are released as before. The claim scan in releaseStoredVGPUExcluding gets the same treatment. --- lib/instances/vgpu.go | 15 +++--- lib/instances/vgpu_reconcile.go | 8 ++- lib/instances/vgpu_reconcile_linux_test.go | 62 ++++++++++++++++++++++ 3 files changed, 75 insertions(+), 10 deletions(-) create mode 100644 lib/instances/vgpu_reconcile_linux_test.go diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 3fb26778a..bb230f3d1 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -13,7 +13,7 @@ import ( // VGPUAssignmentStartupGracePeriod bounds how long an assignment without a // persisted hypervisor PID is treated as potentially live. -const VGPUAssignmentStartupGracePeriod = 5 * time.Minute +const VGPUAssignmentStartupGracePeriod = devices.VGPUAssignmentGracePeriod // VGPUCleanupPendingError reports a failed rollback that left a vGPU assigned. type VGPUCleanupPendingError struct { @@ -50,7 +50,7 @@ func vgpuDevicePendingCleanup(err error) (*devices.VGPUDevice, bool) { } func vgpuAssignmentLiveness(stored *StoredMetadata, now time.Time, livePID bool) (live bool, graceRemaining time.Duration) { - if stored.HypervisorPID != nil && livePID { + if livePID { return true, 0 } if stored.GPUAssignedAt == nil { @@ -198,12 +198,11 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu if storedVGPUDevicePath(stored) != devicePath { continue } - pid := 0 - if stored.HypervisorPID != nil { - pid, err = resolveLiveHypervisorPID(stored.HypervisorProcessIdentity, stored.SocketPath) - if err != nil { - return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) - } + // Resolve even without a persisted PID: the socket-ownership scan can + // still prove a claimant whose post-boot metadata save failed is live. + pid, err := resolveLiveHypervisorPID(stored.HypervisorProcessIdentity, stored.SocketPath) + if err != nil { + return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) } live, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), pid > 0) if pid > 0 && live { diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 966bf1118..5cfb43d4c 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -83,7 +83,11 @@ func (m *manager) reconcileVGPUAssignments(ctx context.Context) (map[string]stru if storedVGPUDevicePath(stored) == "" { continue } - livePID := stored.HypervisorPID != nil && hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + // The socket-ownership check runs even without a persisted PID: a VMM + // whose post-boot metadata save failed still holds its control-socket + // listener, and releasing its device would tear the vGPU out from + // under a live VM. + livePID := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) if live, _ := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID); live { if stored.GPUDevicePath != "" { protected[stored.GPUDevicePath] = struct{}{} @@ -116,7 +120,7 @@ func (m *manager) releaseStaleVGPUAssignment(ctx context.Context, id string) { if path == "" { return } - livePID := stored.HypervisorPID != nil && hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + livePID := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) if live, _ := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID); live { return } diff --git a/lib/instances/vgpu_reconcile_linux_test.go b/lib/instances/vgpu_reconcile_linux_test.go new file mode 100644 index 000000000..8708a1dd7 --- /dev/null +++ b/lib/instances/vgpu_reconcile_linux_test.go @@ -0,0 +1,62 @@ +//go:build linux + +package instances + +import ( + "context" + "net" + "path/filepath" + "testing" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A VMM whose post-boot metadata save failed has no persisted PID, but it +// still holds its control-socket listener. The reconciler must protect its +// assignment past the startup grace period instead of releasing the device +// out from under the live VM. +func TestReconcileVGPUsProtectsSocketOwnerWithoutPersistedPID(t *testing.T) { + t.Parallel() + + socketPath := filepath.Join(t.TempDir(), "test.sock") + listener, err := net.Listen("unix", socketPath) + require.NoError(t, err) + defer listener.Close() + + var destroyed []devices.VGPUAssignment + var protected map[string]struct{} + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }, + reconcileVGPUDevices: func(_ context.Context, p map[string]struct{}, _ bool) error { + protected = p + return nil + }, + } + const id = "pid-save-failed" + stale := time.Now().UTC().Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + SocketPath: socketPath, + GPUFramework: devices.VGPUFrameworkMdev, + GPUDevicePath: "/sys/bus/mdev/devices/test-mdev", + GPUMdevUUID: "test-mdev", + GPUAssignedAt: &stale, + }})) + + m.ReconcileVGPUs(t.Context()) + + assert.Empty(t, destroyed, "a live socket owner must block the release even with a nil persisted PID") + assert.Contains(t, protected, "/sys/bus/mdev/devices/test-mdev") + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/mdev/devices/test-mdev", stored.GPUDevicePath) +} From a7886fe6c2ac1d026c8a76f8f4d3b60680e1f5a0 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:21:01 +0000 Subject: [PATCH 093/107] Share one vGPU assignment grace period constant The five-minute fresh-assignment protection existed three times: the instances startup grace, the mdev orphan grace, and the vendor VFIO sweep grace. Define it once in lib/devices and alias the instances constant to it. --- lib/devices/mdev_linux.go | 13 ++++++------- lib/devices/types.go | 5 +++++ lib/devices/vendor_vfio_linux.go | 7 +------ lib/devices/vendor_vfio_linux_test.go | 2 +- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/lib/devices/mdev_linux.go b/lib/devices/mdev_linux.go index 6ef9f6c60..0908e8765 100644 --- a/lib/devices/mdev_linux.go +++ b/lib/devices/mdev_linux.go @@ -23,10 +23,9 @@ import ( ) const ( - mdevBusPath = "/sys/class/mdev_bus" - mdevDevices = "/sys/bus/mdev/devices" - orphanedMdevGracePeriod = 5 * time.Minute - procPath = "/proc" + mdevBusPath = "/sys/class/mdev_bus" + mdevDevices = "/sys/bus/mdev/devices" + procPath = "/proc" ) // mdevMu protects mdev creation/destruction to prevent race conditions @@ -747,7 +746,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro log.InfoContext(ctx, "reconciling mdev devices", "total_mdevs", len(mdevs), "managed_vfs", len(managedVFs), - "grace_period", orphanedMdevGracePeriod.String(), + "grace_period", VGPUAssignmentGracePeriod.String(), ) groupInUseCache := make(map[int]bool) @@ -782,7 +781,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro continue } - pastGracePeriod, age, err := mdevPastGracePeriod(mdev.UUID, orphanedMdevGracePeriod) + pastGracePeriod, age, err := mdevPastGracePeriod(mdev.UUID, VGPUAssignmentGracePeriod) if err != nil { log.WarnContext(ctx, "failed to determine mdev age, skipping cleanup", "uuid", mdev.UUID, "error", err) skippedProbeError++ @@ -792,7 +791,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro log.DebugContext(ctx, "skipping recently created mdev during grace period", "uuid", mdev.UUID, "age", age.String(), - "grace_period", orphanedMdevGracePeriod.String(), + "grace_period", VGPUAssignmentGracePeriod.String(), ) skippedGrace++ continue diff --git a/lib/devices/types.go b/lib/devices/types.go index 31ebd90ef..369bb3268 100644 --- a/lib/devices/types.go +++ b/lib/devices/types.go @@ -60,6 +60,11 @@ func ValidateDeviceName(name string) bool { // GPUMode represents the host's GPU configuration mode type GPUMode string +// VGPUAssignmentGracePeriod protects a fresh vGPU assignment from cleanup +// until its VM has had time to boot and become identifiable — by a persisted +// hypervisor PID, a control-socket owner, or an open VFIO handle. +const VGPUAssignmentGracePeriod = 5 * time.Minute + type VGPUFramework string const ( diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 74eae4948..a1378a3c7 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -23,11 +23,6 @@ import ( const ( pciDevicesPath = "/sys/bus/pci/devices" vfioDevicesPath = "/dev/vfio/devices" - - // vendorVFIOAssignmentGracePeriod protects assignments created by this - // process from the periodic sweep until their owning instance has had time - // to persist metadata and boot, mirroring orphanedMdevGracePeriod. - vendorVFIOAssignmentGracePeriod = 5 * time.Minute ) type vendorVFIOOwner struct { @@ -294,7 +289,7 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map continue } owner := owners[vf.PCIAddress] - if !owner.assignedAt.IsZero() && time.Since(owner.assignedAt) < vendorVFIOAssignmentGracePeriod { + if !owner.assignedAt.IsZero() && time.Since(owner.assignedAt) < VGPUAssignmentGracePeriod { log.DebugContext(ctx, "skipping recently assigned vendor VFIO vGPU during grace period", "vf", vf.PCIAddress) continue } diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index f7b0d6c3f..098d016e3 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -391,7 +391,7 @@ func TestVendorVFIOReconcileDestroysOwnedVFPastGracePeriod(t *testing.T) { sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") sysfs.owners["0000:82:00.4"] = vendorVFIOOwner{ instanceID: "deleted-instance", - assignedAt: time.Now().Add(-vendorVFIOAssignmentGracePeriod - time.Minute), + assignedAt: time.Now().Add(-VGPUAssignmentGracePeriod - time.Minute), } require.NoError(t, sysfs.reconcile(context.Background(), nil)) From 34f606a17c6a8b1ce226e15309f29ac57cfa0924 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:09:21 +0000 Subject: [PATCH 094/107] Preserve delete-only vGPU retention records --- lib/instances/vgpu_retention.go | 81 +++++++++++++++++++++++++-------- lib/instances/vgpu_test.go | 6 ++- 2 files changed, 66 insertions(+), 21 deletions(-) diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index 121c13cb0..e630484ae 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -2,9 +2,13 @@ package instances import ( "context" + "encoding/json" + "fmt" + "os" "time" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) @@ -62,7 +66,16 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten retentionSurvives := func() bool { meta, err := m.loadMetadata(id) if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { - return true + saveErr := m.saveVGPURetentionStub(retainedVGPU) + if saveErr == nil { + return true + } + log.ErrorContext(ctx, "failed to replace surviving instance metadata with vGPU retention stub", "instance_id", id, "error", saveErr) + overwriteErr := m.overwriteVGPURetentionStub(retainedVGPU) + if overwriteErr == nil { + return true + } + log.ErrorContext(ctx, "failed to overwrite surviving instance metadata with vGPU retention stub", "instance_id", id, "error", overwriteErr) } if err := m.deleteInstanceData(id); err != nil { log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) @@ -79,28 +92,56 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten retention.persisted = retentionSurvives() return } - retained := StoredMetadata{ - Id: id, - Name: retainedVGPU.Name, - Image: retainedVGPU.Image, - ResolvedImage: retainedVGPU.ResolvedImage, - Platform: retainedVGPU.Platform, - CreatedAt: retainedVGPU.CreatedAt, - HypervisorType: retainedVGPU.HypervisorType, - HypervisorVersion: retainedVGPU.HypervisorVersion, - SocketPath: retainedVGPU.SocketPath, - DataDir: retainedVGPU.DataDir, - GPUProfile: retainedVGPU.GPUProfile, - GPUFramework: retainedVGPU.GPUFramework, - GPUDevicePath: retainedVGPU.GPUDevicePath, - GPUMdevUUID: retainedVGPU.GPUMdevUUID, - GPUAssignedAt: retainedVGPU.GPUAssignedAt, - GPURetainedForCleanup: true, - } - if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { + if err := m.saveVGPURetentionStub(retainedVGPU); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) retention.persisted = retentionSurvives() return } retention.persisted = true } + +func vgpuRetentionMetadata(source *StoredMetadata) *metadata { + return &metadata{StoredMetadata: StoredMetadata{ + Id: source.Id, + Name: source.Name, + Image: source.Image, + ResolvedImage: source.ResolvedImage, + Platform: source.Platform, + CreatedAt: source.CreatedAt, + HypervisorType: source.HypervisorType, + HypervisorVersion: source.HypervisorVersion, + SocketPath: source.SocketPath, + DataDir: source.DataDir, + GPUProfile: source.GPUProfile, + GPUFramework: source.GPUFramework, + GPUDevicePath: source.GPUDevicePath, + GPUMdevUUID: source.GPUMdevUUID, + GPUAssignedAt: source.GPUAssignedAt, + GPURetainedForCleanup: true, + }} +} + +func (m *manager) saveVGPURetentionStub(source *StoredMetadata) error { + return m.saveMetadata(vgpuRetentionMetadata(source)) +} + +// overwriteVGPURetentionStub handles a surviving metadata file when its +// directory cannot create the temporary file used by saveMetadata. +func (m *manager) overwriteVGPURetentionStub(source *StoredMetadata) error { + retained := vgpuRetentionMetadata(source) + data, err := json.MarshalIndent(retained, "", " ") + if err != nil { + return fmt.Errorf("marshal metadata: %w", err) + } + unlockAliasReaders := hypervisor.LockSnapshotSourceAliasReaders() + defer unlockAliasReaders() + writeFile := m.writeFile + if writeFile == nil { + writeFile = os.WriteFile + } + if err := writeFile(m.paths.InstanceMetadata(source.Id), data, 0644); err != nil { + return fmt.Errorf("write metadata: %w", err) + } + m.syncAdmissionAllocation(retained) + return nil +} diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index e5ed8d13f..b9f7124d2 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -109,7 +109,7 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { require.Error(t, err, "the lost retention leaves no metadata claim, so the periodic sweep releases the VF") } -func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T) { +func TestCleanupFailedCreateRewritesFullMetadataWhenDirectoryIsReadOnly(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root bypasses directory permissions") } @@ -133,6 +133,10 @@ func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T require.NoError(t, err) assert.Equal(t, stored.GPUFramework, retained.GPUFramework) assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) + assert.True(t, retained.GPURetainedForCleanup) + + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{}) + assert.ErrorIs(t, err, errVGPURetentionStub) } func TestVGPURetentionWrapPending(t *testing.T) { From 742281ec1ff2b208541450050bd0d53f8a6db31a Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:44:10 +0000 Subject: [PATCH 095/107] Trim redundant vGPU coverage --- cmd/api/api/instances_test.go | 164 +++++++++------------------ lib/instances/lifecycle_noop_test.go | 18 +-- lib/instances/vgpu.go | 10 +- lib/instances/vgpu_reconcile_test.go | 55 +++------ lib/instances/vgpu_test.go | 117 +++++++------------ 5 files changed, 116 insertions(+), 248 deletions(-) diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 5f826dc9c..ccaeb3a86 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -48,69 +48,37 @@ func TestGetInstance_NotFound(t *testing.T) { require.Error(t, err) } -type createErrorInstanceManager struct { - instances.Manager - err error -} - -func (m createErrorInstanceManager) CreateInstance(context.Context, instances.CreateInstanceRequest) (*instances.Instance, error) { - return nil, m.err -} - -func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { +func TestVGPUCleanupPendingDetail(t *testing.T) { t.Parallel() - svc := newTestService(t) - svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{ - InstanceID: "inst-1", - Retained: true, - Err: network.ErrNameExists, - }} - - resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ - Body: &oapi.CreateInstanceRequest{Image: "test-image"}, - }) - require.NoError(t, err) - - pending, ok := resp.(oapi.CreateInstance500JSONResponse) - require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) - assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) - assert.Contains(t, pending.Message, "inst-1") - assert.Contains(t, pending.Message, network.ErrNameExists.Error(), - "the underlying create failure must survive the cleanup guidance") - assert.Contains(t, pending.Message, "delete it to retry") - require.NotNil(t, pending.InnerError) - require.NotNil(t, pending.InnerError.Code) - assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code) - require.NotNil(t, pending.InnerError.Message) - assert.Equal(t, "inst-1", *pending.InnerError.Message) -} - -func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance(t *testing.T) { - t.Parallel() - svc := newTestService(t) - svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{ - InstanceID: "inst-1", - Err: network.ErrNameExists, - }} - - resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ - Body: &oapi.CreateInstanceRequest{Image: "test-image"}, - }) - require.NoError(t, err) - - pending, ok := resp.(oapi.CreateInstance500JSONResponse) - require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) - assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) - assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") - assert.Contains(t, pending.Message, network.ErrNameExists.Error(), - "the underlying create failure must survive the cleanup guidance") - assert.Contains(t, pending.Message, "periodic vGPU reconcile") - assert.NotContains(t, pending.Message, "delete") - require.NotNil(t, pending.InnerError) - require.NotNil(t, pending.InnerError.Code) - assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code) - require.NotNil(t, pending.InnerError.Message) - assert.Equal(t, "inst-1", *pending.InnerError.Message) + for _, tt := range []struct { + name string + retained bool + code string + guidance string + exclude string + }{ + {name: "retained", retained: true, code: "vgpu_retained_instance", guidance: "delete it to retry"}, + {name: "unretained", code: "vgpu_unretained_instance", guidance: "periodic vGPU reconcile", exclude: "delete"}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + message, inner := vgpuCleanupPendingDetail(&instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Retained: tt.retained, + Err: network.ErrNameExists, + }, "create", "delete it to retry") + assert.Contains(t, message, "inst-1") + assert.Contains(t, message, network.ErrNameExists.Error()) + assert.Contains(t, message, tt.guidance) + if tt.exclude != "" { + assert.NotContains(t, message, tt.exclude) + } + require.NotNil(t, inner.Code) + assert.Equal(t, tt.code, *inner.Code) + require.NotNil(t, inner.Message) + assert.Equal(t, "inst-1", *inner.Message) + }) + } } func TestCreateInstance_AutoPullImage(t *testing.T) { @@ -956,6 +924,16 @@ func TestCreateInstance_ErrorStatusMapping(t *testing.T) { wantCode string wantMessage string }{ + { + name: "vGPU cleanup pending beats wrapped name conflict -> 500", + err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Retained: true, + Err: network.ErrNameExists, + }, + wantType: oapi.CreateInstance500JSONResponse{}, + wantCode: "vgpu_cleanup_pending", + }, { name: "platform not available -> 404", err: fmt.Errorf("resolve image: %w", images.ErrPlatformNotAvailable), @@ -1082,61 +1060,23 @@ func TestRestoreInstance_ErrorMapping(t *testing.T) { func TestStartInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { t.Parallel() - + svc := newTestService(t) resolved := &instances.Instance{ StoredMetadata: instances.StoredMetadata{Id: "inst-1", Name: "inst-1"}, State: instances.StateStopped, } + svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{ + InstanceID: resolved.Id, + Retained: true, + Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources), + }} - t.Run("retained", func(t *testing.T) { - t.Parallel() - svc := newTestService(t) - svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{ - InstanceID: "inst-1", - Retained: true, - Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources), - }} - - resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id}) - require.NoError(t, rerr) - - pending, ok := resp.(oapi.StartInstance500JSONResponse) - require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) - assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) - assert.Contains(t, pending.Message, "inst-1") - assert.Contains(t, pending.Message, instances.ErrInsufficientResources.Error(), - "the underlying start failure must survive the cleanup guidance") - assert.Contains(t, pending.Message, "delete it or retry start") - require.NotNil(t, pending.InnerError) - require.NotNil(t, pending.InnerError.Code) - assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code) - require.NotNil(t, pending.InnerError.Message) - assert.Equal(t, "inst-1", *pending.InnerError.Message) - }) - - t.Run("unretained", func(t *testing.T) { - t.Parallel() - svc := newTestService(t) - svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{ - InstanceID: "inst-1", - Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources), - }} - - resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id}) - require.NoError(t, rerr) - - pending, ok := resp.(oapi.StartInstance500JSONResponse) - require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) - assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) - assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") - assert.Contains(t, pending.Message, "periodic vGPU reconcile") - assert.NotContains(t, pending.Message, "delete") - require.NotNil(t, pending.InnerError) - require.NotNil(t, pending.InnerError.Code) - assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code) - require.NotNil(t, pending.InnerError.Message) - assert.Equal(t, "inst-1", *pending.InnerError.Message) - }) + resp, err := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id}) + require.NoError(t, err) + pending, ok := resp.(oapi.StartInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "delete it or retry start") } func TestInstanceActions_ImageNotFoundMapsTo404(t *testing.T) { diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 1843485e3..85083766b 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -214,10 +214,11 @@ func TestDeleteReleasesRetainedCreateStub(t *testing.T) { require.NoError(t, m.ensureDirectories(id)) assignedAt := time.Now().UTC() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: id, - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - GPUAssignedAt: &assignedAt, + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + GPURetainedForCleanup: true, }})) require.NoError(t, m.DeleteInstance(context.Background(), id)) @@ -339,9 +340,8 @@ func TestStartRejectsVGPURetentionRecord(t *testing.T) { assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) } -func TestReconcileReleasesRetainedVGPUOnStoppedInstance(t *testing.T) { +func TestStopStoppedInstanceLeavesVGPUForReconcile(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) - m.reconcileVGPUDevices = func(context.Context, map[string]struct{}, bool) error { return nil } meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" @@ -358,12 +358,6 @@ func TestReconcileReleasesRetainedVGPUOnStoppedInstance(t *testing.T) { stored, err := m.loadMetadata(id) require.NoError(t, err) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) - - m.ReconcileVGPUs(context.Background()) - stored, err = m.loadMetadata(id) - require.NoError(t, err) - assert.Empty(t, stored.GPUDevicePath) - assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile, "profile is kept for the next start") } func TestReconcileVGPUReleaseFailureKeepsStoppedInstanceUsable(t *testing.T) { diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index bb230f3d1..37127e921 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -143,17 +143,13 @@ func restoreStartMutatedFields(dst, src *StoredMetadata) { } func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { - return m.releaseStoredVGPUExcluding(ctx, stored, stored.Id) -} - -func (m *manager) releaseStoredVGPUExcluding(ctx context.Context, stored *StoredMetadata, excludeID string) error { path := storedVGPUDevicePath(stored) if path != "" { // Vendor VFIO VFs are reusable, so release fails closed on an incomplete inventory. claimed := false if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { var err error - claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, excludeID, path) + claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path) if err != nil { return err } @@ -204,8 +200,8 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu if err != nil { return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) } - live, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), pid > 0) - if pid > 0 && live { + _, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), pid > 0) + if pid > 0 { return true, nil } if remaining > 0 { diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 468591dee..395010532 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -25,10 +25,14 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { stale := now.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) var protected map[string]struct{} + var destroyed []devices.VGPUAssignment m := &manager{ - paths: paths.New(t.TempDir()), - now: func() time.Time { return now }, - destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, + paths: paths.New(t.TempDir()), + now: func() time.Time { return now }, + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }, reconcileVGPUDevices: func(_ context.Context, p map[string]struct{}, sweepVendorVFIO bool) error { protected = p assert.True(t, sweepVendorVFIO) @@ -37,7 +41,7 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { } instances := []StoredMetadata{ {Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}, - {Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}, + {Id: "orphaned", GPUProfile: "NVIDIA L40S-2Q", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}, {Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}, {Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}}, {Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, GPUAssignedAt: &recent}, @@ -59,6 +63,14 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { require.NoError(t, err) assert.Empty(t, stored.GPUDevicePath, "stale assignment on %s must be released", id) } + assert.Contains(t, destroyed, devices.VGPUAssignment{ + Framework: devices.VGPUFrameworkVendorVFIO, + DevicePath: "/sys/bus/pci/devices/0000:82:00.5", + InstanceID: "orphaned", + }) + orphaned, err := m.loadMetadata("orphaned") + require.NoError(t, err) + assert.Equal(t, "NVIDIA L40S-2Q", orphaned.GPUProfile) for _, id := range []string{"booting", "stale-pid-booting"} { stored, err := m.loadMetadata(id) require.NoError(t, err) @@ -95,41 +107,6 @@ func TestReconcileVGPUsSkipsVendorSweepWhenListingFails(t *testing.T) { assert.Equal(t, []bool{false, true}, sweeps, "the next pass retries the vendor sweep") } -func TestReconcileVGPUsReleasesStaleAssignment(t *testing.T) { - t.Parallel() - - var destroyed []devices.VGPUAssignment - m := &manager{ - paths: paths.New(t.TempDir()), - destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { - destroyed = append(destroyed, assignment) - return nil - }, - reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, - } - const id = "stopped-retained" - require.NoError(t, m.ensureDirectories(id)) - require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: id, - GPUProfile: "NVIDIA L40S-2Q", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - }})) - - m.ReconcileVGPUs(t.Context()) - - require.Len(t, destroyed, 1) - assert.Equal(t, devices.VGPUAssignment{ - Framework: devices.VGPUFrameworkVendorVFIO, - DevicePath: "/sys/bus/pci/devices/0000:82:00.4", - InstanceID: id, - }, destroyed[0]) - stored, err := m.loadMetadata(id) - require.NoError(t, err) - assert.Empty(t, stored.GPUDevicePath) - assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile, "profile is kept for the next start") -} - func TestReconcileVGPUsKeepsAssignmentWhenReleaseFails(t *testing.T) { t.Parallel() diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index b9f7124d2..55f88f4d9 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -526,87 +526,48 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. assert.True(t, claimed) } -func TestVGPUAssignmentClaimedByLiveInstanceErrorsOnRecentNilPIDClaim(t *testing.T) { +func TestVGPUAssignmentClaimedByLiveInstanceLiveness(t *testing.T) { t.Parallel() - m := &manager{paths: paths.New(t.TempDir())} - require.NoError(t, m.ensureDirectories("booting-claimant")) - assignedAt := time.Now().UTC() - require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "booting-claimant", - Name: "booting-claimant", - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - GPUAssignedAt: &assignedAt, - }})) - - _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") - require.Error(t, err) - assert.Contains(t, err.Error(), "booting-claimant") -} - -func TestVGPUAssignmentClaimedByLiveInstanceIgnoresStaleNilPIDClaim(t *testing.T) { - t.Parallel() - - m := &manager{paths: paths.New(t.TempDir())} - require.NoError(t, m.ensureDirectories("stale-claimant")) - assignedAt := time.Now().Add(-VGPUAssignmentStartupGracePeriod - time.Minute) - require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "stale-claimant", - Name: "stale-claimant", - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - GPUAssignedAt: &assignedAt, - }})) - - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") - require.NoError(t, err) - assert.False(t, claimed) -} - -func TestVGPUAssignmentClaimedByLiveInstanceGracesRecentDeadPIDClaim(t *testing.T) { - m := &manager{paths: paths.New(t.TempDir())} - claimantID := "claimant-dead-pid" - require.NoError(t, m.ensureDirectories(claimantID)) - deadPID := 1<<22 - 1 - require.False(t, ProcessExists(deadPID)) - assignedAt := time.Now().UTC() - require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: claimantID, - HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - GPUAssignedAt: &assignedAt, - }})) - - _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", "/sys/bus/pci/devices/0000:82:00.4") - require.Error(t, err) - - stale := assignedAt.Add(-2 * VGPUAssignmentStartupGracePeriod) - meta, err := m.loadMetadata(claimantID) - require.NoError(t, err) - meta.GPUAssignedAt = &stale - require.NoError(t, m.saveMetadata(meta)) - - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", "/sys/bus/pci/devices/0000:82:00.4") - require.NoError(t, err) - assert.False(t, claimed) -} - -func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) { - t.Parallel() - - m := &manager{paths: paths.New(t.TempDir())} - require.NoError(t, m.ensureDirectories("dead-claimant")) + const devicePath = "/sys/bus/pci/devices/0000:82:00.4" deadPID := 1 << 30 - require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "dead-claimant", - Name: "dead-claimant", - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, - }})) - - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") - require.NoError(t, err) - assert.False(t, claimed, "a claim whose hypervisor is gone must not block the release") + require.False(t, ProcessExists(deadPID)) + recent := time.Now().UTC() + stale := recent.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + tests := []struct { + name string + assignedAt *time.Time + pid *int + wantErr string + }{ + {name: "recent without PID", assignedAt: &recent, wantErr: "no persisted hypervisor PID"}, + {name: "stale without PID", assignedAt: &stale}, + {name: "recent dead PID", assignedAt: &recent, pid: &deadPID, wantErr: "recorded hypervisor is not running"}, + {name: "legacy dead PID", pid: &deadPID}, + } + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + m := &manager{paths: paths.New(t.TempDir())} + id := fmt.Sprintf("claimant-%d", i) + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + GPUAssignedAt: tt.assignedAt, + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: tt.pid}, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", devicePath) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.False(t, claimed) + }) + } } func TestReleaseStoredVGPUSkipsClaimScanForMdev(t *testing.T) { From 33f58e10ee89ee69c67ff9b699b34efc2e2127bb Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:40:31 +0000 Subject: [PATCH 096/107] Add counters for vGPU cleanup failure paths A wedged or leaked VF presents as reduced GPU capacity while /resources still reports full capacity, and reconcile, stale-release, and retention failures were visible only as log lines. Count failed reconcile stages, failed stale releases, and retained assignments (by operation and whether the retention record persisted) so sustained failure can alert. --- lib/instances/metrics.go | 81 +++++++++++++++++++++++++++++++++ lib/instances/metrics_test.go | 45 ++++++++++++++++++ lib/instances/start.go | 3 ++ lib/instances/vgpu_reconcile.go | 3 ++ lib/instances/vgpu_retention.go | 3 ++ 5 files changed, 135 insertions(+) diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index 1ada5ac1e..a380ff09f 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -73,6 +73,20 @@ type lifecycleEventDropReason string const lifecycleEventDropReasonBufferFull lifecycleEventDropReason = "buffer_full" +type vgpuReconcileStage string + +const ( + vgpuReconcileStageListInstances vgpuReconcileStage = "list_instances" + vgpuReconcileStageReconcileDevices vgpuReconcileStage = "reconcile_devices" +) + +type vgpuRetentionOperation string + +const ( + vgpuRetentionOperationCreate vgpuRetentionOperation = "create" + vgpuRetentionOperationStart vgpuRetentionOperation = "start" +) + // Metrics holds the metrics instruments for instance operations. type Metrics struct { createDuration metric.Float64Histogram @@ -94,6 +108,9 @@ type Metrics struct { lifecycleEventsDroppedTotal metric.Int64Counter forkMemFileShareFallbacksTotal metric.Int64Counter ttlReaperDeletionsTotal metric.Int64Counter + vgpuReconcileFailuresTotal metric.Int64Counter + vgpuStaleReleaseFailuresTotal metric.Int64Counter + vgpuAssignmentsRetainedTotal metric.Int64Counter tracer trace.Tracer } @@ -270,6 +287,30 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + vgpuReconcileFailuresTotal, err := meter.Int64Counter( + "hypeman_instances_vgpu_reconcile_failures_total", + metric.WithDescription("Total number of vGPU reconcile pass stages that failed, leaving stale assignments or device leftovers allocated while /resources still advertises the capacity"), + ) + if err != nil { + return nil, err + } + + vgpuStaleReleaseFailuresTotal, err := meter.Int64Counter( + "hypeman_instances_vgpu_stale_release_failures_total", + metric.WithDescription("Total number of stale vGPU assignment releases that failed, keeping the VF allocated until a later reconcile pass succeeds"), + ) + if err != nil { + return nil, err + } + + vgpuAssignmentsRetainedTotal, err := meter.Int64Counter( + "hypeman_instances_vgpu_assignments_retained_total", + metric.WithDescription("Total number of failed rollbacks that left a vGPU assignment behind, by whether the retention record the periodic reconcile needs was persisted"), + ) + if err != nil { + return nil, err + } + // Register observable gauge for instance counts by state instancesTotal, err := meter.Int64ObservableGauge( "hypeman_instances_total", @@ -464,6 +505,9 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M lifecycleEventsDroppedTotal: lifecycleEventsDroppedTotal, forkMemFileShareFallbacksTotal: forkMemFileShareFallbacksTotal, ttlReaperDeletionsTotal: ttlReaperDeletionsTotal, + vgpuReconcileFailuresTotal: vgpuReconcileFailuresTotal, + vgpuStaleReleaseFailuresTotal: vgpuStaleReleaseFailuresTotal, + vgpuAssignmentsRetainedTotal: vgpuAssignmentsRetainedTotal, tracer: tracer, }, nil } @@ -563,6 +607,43 @@ func (m *manager) recordTimeToRunning(ctx context.Context, stored *StoredMetadat m.metrics.timeToRunning.Record(ctx, duration, metric.WithAttributes(attrs...)) } +// recordVGPUReconcileFailure records a vGPU reconcile stage failing: stale +// assignments or device leftovers stay allocated (capacity silently reduced +// while /resources reports full) until a later pass succeeds, so sustained +// failure must be alertable beyond a log line. +func (m *manager) recordVGPUReconcileFailure(ctx context.Context, stage vgpuReconcileStage) { + if m.metrics == nil { + return + } + m.metrics.vgpuReconcileFailuresTotal.Add(ctx, 1, metric.WithAttributes( + attribute.String("stage", string(stage)), + )) +} + +// recordVGPUStaleReleaseFailure records a stale vGPU release failing: the VF +// stays allocated while /resources still advertises it, so a wedged release +// that fails every pass must be alertable beyond a log line. +func (m *manager) recordVGPUStaleReleaseFailure(ctx context.Context) { + if m.metrics == nil { + return + } + m.metrics.vgpuStaleReleaseFailuresTotal.Add(ctx, 1) +} + +// recordVGPURetainedAssignment records a failed rollback leaving a vGPU +// assignment behind. An unpersisted retention record has no metadata claim, +// so the device is only recovered by the periodic reconcile sweep; either way +// the VF is unavailable while /resources still advertises it. +func (m *manager) recordVGPURetainedAssignment(ctx context.Context, operation vgpuRetentionOperation, persisted bool) { + if m.metrics == nil { + return + } + m.metrics.vgpuAssignmentsRetainedTotal.Add(ctx, 1, metric.WithAttributes( + attribute.String("operation", string(operation)), + attribute.String("persisted", strconv.FormatBool(persisted)), + )) +} + // recordStateTransition records a state transition with hypervisor label. func (m *manager) recordStateTransition(ctx context.Context, fromState, toState string, hvType hypervisor.Type) { if m.metrics == nil { diff --git a/lib/instances/metrics_test.go b/lib/instances/metrics_test.go index 811571e36..525002cbb 100644 --- a/lib/instances/metrics_test.go +++ b/lib/instances/metrics_test.go @@ -2,6 +2,7 @@ package instances import ( "context" + "errors" "os" "path/filepath" "testing" @@ -556,6 +557,50 @@ func TestEnsureSnapshotMemoryReadySkipsPendingCompressionWithoutPreemptionMetric assert.False(t, metricExists(rm, "hypeman_snapshot_compression_preemptions_total"), "pending-delay cancellation should not record a preemption") } +func TestVGPUReconcileFailureMetric_RecordStages(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + reader := otelmetric.NewManualReader() + provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) + + m := &manager{ + paths: paths.New(t.TempDir()), + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { + return errors.New("sweep failed") + }, + } + metrics, err := newInstanceMetrics(provider.Meter("test"), nil, m) + require.NoError(t, err) + m.metrics = metrics + + const id = "unreadable" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{Id: id}})) + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o000)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + m.ReconcileVGPUs(t.Context()) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(t.Context(), &rm)) + + failuresMetric := findMetric(t, rm, "hypeman_instances_vgpu_reconcile_failures_total") + failures, ok := failuresMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, failures.DataPoints, 2) + for _, point := range failures.DataPoints { + switch metricLabel(t, point.Attributes, "stage") { + case "list_instances", "reconcile_devices": + assert.Equal(t, int64(1), point.Value) + default: + t.Fatalf("unexpected reconcile failure stage datapoint: %s", metricLabel(t, point.Attributes, "stage")) + } + } +} + func assertMetricNames(t *testing.T, rm metricdata.ResourceMetrics, expected []string) { t.Helper() diff --git a/lib/instances/start.go b/lib/instances/start.go index 19bb64697..ea8c2a6f4 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -183,9 +183,11 @@ func (m *manager) startInstance( setStoredVGPUDevice(&retentionMeta.StoredMetadata, pendingDevice, assignedAt) wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { + m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, false) log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) return nil, &VGPUCleanupPendingError{InstanceID: id, Err: fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr)} } + m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, true) return nil, &VGPUCleanupPendingError{InstanceID: id, Retained: true, Err: wrapped} } return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) @@ -198,6 +200,7 @@ func (m *manager) startInstance( retained, persisted := m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) if retained { retention.markRetained(persisted) + m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, persisted) } }) if err := m.saveMetadata(meta); err != nil { diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 5cfb43d4c..9ca8cef72 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -56,6 +56,7 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { protected, err := m.reconcileVGPUAssignments(ctx) sweepVendorVFIO := err == nil if err != nil { + m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageListInstances) log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping vendor VFIO sweep until the next pass, mdev reconcile still runs", "error", err) protected = make(map[string]struct{}) } @@ -64,6 +65,7 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { reconcileDevices = devices.ReconcileVGPUs } if err := reconcileDevices(ctx, protected, sweepVendorVFIO); err != nil { + m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageReconcileDevices) log.WarnContext(ctx, "failed to reconcile vGPU devices", "error", err) } } @@ -125,6 +127,7 @@ func (m *manager) releaseStaleVGPUAssignment(ctx context.Context, id string) { return } if err := m.releaseStoredVGPU(ctx, stored); err != nil { + m.recordVGPUStaleReleaseFailure(ctx) log.WarnContext(ctx, "failed to release stale vGPU assignment; retrying on the next reconcile pass", "instance_id", id, "device_path", path, "error", err) return } diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index e630484ae..691ae5eaa 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -61,6 +61,9 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten id := retention.instanceID retainedVGPU := retention.stub log := logger.FromContext(ctx) + defer func() { + m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationCreate, retention.persisted) + }() // When the retention record is lost, the assignment has no metadata claim // left; the periodic vGPU reconcile sweeps the device once it is free. retentionSurvives := func() bool { From ea55f132937e78518c2085a75c848f70dbc80c65 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 26 Aug 2026 18:28:01 +0000 Subject: [PATCH 097/107] Simplify vGPU lifecycle cleanup --- lib/devices/vgpu_linux.go | 28 +-- lib/devices/vgpu_linux_test.go | 16 -- lib/instances/lifecycle_noop_test.go | 36 --- lib/instances/manager.go | 10 +- lib/instances/metrics.go | 11 - lib/instances/process_identity.go | 8 +- lib/instances/process_identity_linux_test.go | 4 +- lib/instances/query_test.go | 24 +- lib/instances/vgpu.go | 96 ++------ lib/instances/vgpu_reconcile.go | 18 +- lib/instances/vgpu_reconcile_linux_test.go | 6 +- lib/instances/vgpu_reconcile_test.go | 2 +- lib/instances/vgpu_retention.go | 42 +--- lib/instances/vgpu_test.go | 233 +++++-------------- 14 files changed, 113 insertions(+), 421 deletions(-) diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index 72827f7b2..36bf86c9e 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -111,33 +111,13 @@ func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{ return err } - return reconcileDiscoveredVGPUs( - ctx, - framework, - protectedDevicePaths, - sweepVendorVFIO, - func(ctx context.Context) error { return ReconcileMdevs(ctx, nil) }, - hostVendorVFIO.reconcile, - ) -} - -func reconcileDiscoveredVGPUs( - ctx context.Context, - framework VGPUFramework, - protectedDevicePaths map[string]struct{}, - sweepVendorVFIO bool, - reconcileMdev func(context.Context) error, - reconcileVendorVFIO func(context.Context, map[string]struct{}) error, -) error { switch framework { case VGPUFrameworkMdev: - return reconcileMdev(ctx) + return ReconcileMdevs(ctx, nil) case VGPUFrameworkVendorVFIO: - if !sweepVendorVFIO { - return nil + if sweepVendorVFIO { + return hostVendorVFIO.reconcile(ctx, protectedDevicePaths) } - return reconcileVendorVFIO(ctx, protectedDevicePaths) - default: - return nil } + return nil } diff --git a/lib/devices/vgpu_linux_test.go b/lib/devices/vgpu_linux_test.go index 32f9a30ea..7b03d9c26 100644 --- a/lib/devices/vgpu_linux_test.go +++ b/lib/devices/vgpu_linux_test.go @@ -3,7 +3,6 @@ package devices import ( - "context" "errors" "os" "path/filepath" @@ -13,21 +12,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestReconcileDiscoveredVGPUsControlsVendorSweep(t *testing.T) { - protected := make(map[string]struct{}) - vendorCalls := 0 - reconcileVendor := func(context.Context, map[string]struct{}) error { - vendorCalls++ - return nil - } - - require.NoError(t, reconcileDiscoveredVGPUs(context.Background(), VGPUFrameworkVendorVFIO, protected, false, nil, reconcileVendor)) - assert.Zero(t, vendorCalls) - - require.NoError(t, reconcileDiscoveredVGPUs(context.Background(), VGPUFrameworkVendorVFIO, protected, true, nil, reconcileVendor)) - assert.Equal(t, 1, vendorCalls) -} - func TestDiscoverVGPUWithPropagatesMdevError(t *testing.T) { t.Parallel() diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 85083766b..77d15d733 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -150,24 +150,6 @@ func TestLifecycleNoopStandbyWithOptionsStillRejectsStandbyInstance(t *testing.T assertNoLifecycleEvent(t, events) } -func TestDeleteContinuesWhenVGPUReleaseFails(t *testing.T) { - m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) - meta, err := m.loadMetadata(id) - require.NoError(t, err) - meta.GPUProfile = "NVIDIA L40S-2Q" - meta.GPUFramework = devices.VGPUFramework("future-framework") - meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" - require.NoError(t, m.saveMetadata(meta)) - - // A failed release is logged and the delete continues, matching the - // pre-refactor contract; the leaked assignment is recovered by the - // periodic vGPU reconcile. - require.NoError(t, m.DeleteInstance(context.Background(), id)) - - _, err = m.loadMetadata(id) - require.Error(t, err, "instance data must be deleted despite the failed release") -} - func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) var persisted *metadata @@ -360,24 +342,6 @@ func TestStopStoppedInstanceLeavesVGPUForReconcile(t *testing.T) { assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) } -func TestReconcileVGPUReleaseFailureKeepsStoppedInstanceUsable(t *testing.T) { - m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) - m.reconcileVGPUDevices = func(context.Context, map[string]struct{}, bool) error { return nil } - meta, err := m.loadMetadata(id) - require.NoError(t, err) - meta.GPUProfile = "NVIDIA L40S-2Q" - meta.GPUFramework = devices.VGPUFramework("future-framework") - meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" - require.NoError(t, m.saveMetadata(meta)) - - m.ReconcileVGPUs(context.Background()) - - stored, err := m.loadMetadata(id) - require.NoError(t, err) - assert.Equal(t, devices.VGPUFramework("future-framework"), stored.GPUFramework) - assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) -} - // recordingDeviceManager is a devices.Manager stub that records passthrough // teardown calls. Only the methods delete exercises are implemented. type recordingDeviceManager struct { diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 8c298ed24..d44ef3014 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -219,7 +219,7 @@ type manager struct { vgpuReconcileInterval time.Duration discoverVGPU func() (devices.VGPUFramework, []devices.VirtualFunction, error) - vgpuInitTermGrace time.Duration + vfioTermGrace time.Duration // Hypervisor support vmStarters map[hypervisor.Type]hypervisor.VMStarter @@ -293,8 +293,6 @@ func NewManagerWithConfigE(p *paths.Paths, imageManager images.Manager, systemMa defaultHypervisor: defaultHypervisor, now: time.Now, writeFile: os.WriteFile, - createVGPU: devices.CreateVGPU, - destroyVGPU: devices.DestroyVGPU, meter: meter, tracer: tracer, guestMemoryPolicy: policy, @@ -740,12 +738,12 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { return m.defaultHypervisor } -func (m *manager) listInstancesForReconcile(ctx context.Context) ([]Instance, error) { +func (m *manager) listMetadataForReconcile() ([]StoredMetadata, error) { files, err := m.listMetadataFilesStrict() if err != nil { return nil, err } - result := make([]Instance, 0, len(files)) + result := make([]StoredMetadata, 0, len(files)) for _, file := range files { id := filepath.Base(filepath.Dir(file)) meta, err := m.loadMetadata(id) @@ -755,7 +753,7 @@ func (m *manager) listInstancesForReconcile(ctx context.Context) ([]Instance, er } return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) } - result = append(result, Instance{StoredMetadata: meta.StoredMetadata}) + result = append(result, meta.StoredMetadata) } return result, nil } diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index a380ff09f..e14deb49b 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -607,10 +607,6 @@ func (m *manager) recordTimeToRunning(ctx context.Context, stored *StoredMetadat m.metrics.timeToRunning.Record(ctx, duration, metric.WithAttributes(attrs...)) } -// recordVGPUReconcileFailure records a vGPU reconcile stage failing: stale -// assignments or device leftovers stay allocated (capacity silently reduced -// while /resources reports full) until a later pass succeeds, so sustained -// failure must be alertable beyond a log line. func (m *manager) recordVGPUReconcileFailure(ctx context.Context, stage vgpuReconcileStage) { if m.metrics == nil { return @@ -620,9 +616,6 @@ func (m *manager) recordVGPUReconcileFailure(ctx context.Context, stage vgpuReco )) } -// recordVGPUStaleReleaseFailure records a stale vGPU release failing: the VF -// stays allocated while /resources still advertises it, so a wedged release -// that fails every pass must be alertable beyond a log line. func (m *manager) recordVGPUStaleReleaseFailure(ctx context.Context) { if m.metrics == nil { return @@ -630,10 +623,6 @@ func (m *manager) recordVGPUStaleReleaseFailure(ctx context.Context) { m.metrics.vgpuStaleReleaseFailuresTotal.Add(ctx, 1) } -// recordVGPURetainedAssignment records a failed rollback leaving a vGPU -// assignment behind. An unpersisted retention record has no metadata claim, -// so the device is only recovered by the periodic reconcile sweep; either way -// the VF is unavailable while /resources still advertises it. func (m *manager) recordVGPURetainedAssignment(ctx context.Context, operation vgpuRetentionOperation, persisted bool) { if m.metrics == nil { return diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 02f43fcda..7e8500402 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -27,9 +27,9 @@ const linuxBootIDPath = "/proc/sys/kernel/random/boot_id" // does not unstick it, so the wait is short to keep stop and delete fast. const hypervisorSIGKILLWaitTimeout = 2 * time.Second -func (m *manager) vgpuTermGrace() time.Duration { - if m.vgpuInitTermGrace > 0 { - return m.vgpuInitTermGrace +func (m *manager) vfioTerminationGrace() time.Duration { + if m.vfioTermGrace > 0 { + return m.vfioTermGrace } return hypervisor.VFIOTermGrace } @@ -37,7 +37,7 @@ func (m *manager) vgpuTermGrace() time.Duration { // SIGKILL during guest driver init can wedge a VF until the parent GPU is reset. func (m *manager) terminateThenKill(ctx context.Context, inst *Instance, pid int) error { if inst.GPUProfile != "" || len(inst.Devices) > 0 { - if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { + if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vfioTerminationGrace()) { return nil } logger.FromContext(ctx).WarnContext(ctx, "hypervisor with VFIO devices did not exit on SIGTERM; hard-killing, device may wedge if the guest driver was initializing", diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index b208a6ef3..2ec201450 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -546,7 +546,7 @@ func TestVGPUAssignmentClaimedByLiveInstanceProtectsReusedPIDClaim(t *testing.T) SocketPath: socketPath, }})) - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", devicePath) + claimed, err := m.vgpuAssignmentClaimedByLiveInstance("other-instance", devicePath) require.NoError(t, err) assert.True(t, claimed) } @@ -740,7 +740,7 @@ func TestKillHypervisorEscalatesToSIGKILLWhenSIGTERMIgnored(t *testing.T) { pid, identity := startTrapProcess(t, "") socketPath := filepath.Join(t.TempDir(), "missing.sock") - m := &manager{vgpuInitTermGrace: 50 * time.Millisecond} + m := &manager{vfioTermGrace: 50 * time.Millisecond} require.NoError(t, m.killHypervisor(context.Background(), &Instance{ State: StateInitializing, StoredMetadata: StoredMetadata{ diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index 795b00baf..222e85a30 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -14,7 +14,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { +func TestListMetadataForReconcileFailsOnInvalidMetadata(t *testing.T) { m := &manager{paths: paths.New(t.TempDir())} require.NoError(t, m.ensureDirectories("valid")) @@ -31,18 +31,18 @@ func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { require.NoError(t, err) require.Len(t, listed, 1) - _, err = m.listInstancesForReconcile(context.Background()) + _, err = m.listMetadataForReconcile() require.Error(t, err) assert.ErrorContains(t, err, "load metadata for instance invalid") require.NoError(t, os.Remove(m.paths.InstanceMetadata("invalid"))) - listed, err = m.listInstancesForReconcile(context.Background()) + metadata, err := m.listMetadataForReconcile() require.NoError(t, err) - require.Len(t, listed, 1) - assert.Equal(t, "valid", listed[0].Id) + require.Len(t, metadata, 1) + assert.Equal(t, "valid", metadata[0].Id) } -func TestListInstancesForReconcileSkipsInstanceDeletedDuringListing(t *testing.T) { +func TestListMetadataForReconcileSkipsInstanceDeletedDuringListing(t *testing.T) { m := &manager{paths: paths.New(t.TempDir())} for _, id := range []string{"aaa-ghost", "zzz-live"} { @@ -57,13 +57,13 @@ func TestListInstancesForReconcileSkipsInstanceDeletedDuringListing(t *testing.T unlock := hypervisor.LockSnapshotSourceAliasMutation() type result struct { - listed []Instance - err error + metadata []StoredMetadata + err error } done := make(chan result, 1) go func() { - listed, err := m.listInstancesForReconcile(context.Background()) - done <- result{listed, err} + metadata, err := m.listMetadataForReconcile() + done <- result{metadata, err} }() time.Sleep(100 * time.Millisecond) require.NoError(t, os.Remove(m.paths.InstanceMetadata("aaa-ghost"))) @@ -71,8 +71,8 @@ func TestListInstancesForReconcileSkipsInstanceDeletedDuringListing(t *testing.T res := <-done require.NoError(t, res.err) - require.Len(t, res.listed, 1) - assert.Equal(t, "zzz-live", res.listed[0].Id) + require.Len(t, res.metadata, 1) + assert.Equal(t, "zzz-live", res.metadata[0].Id) } func TestParseExitSentinelLine(t *testing.T) { diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 37127e921..7b70e807c 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -11,10 +11,6 @@ import ( "github.com/kernel/hypeman/lib/logger" ) -// VGPUAssignmentStartupGracePeriod bounds how long an assignment without a -// persisted hypervisor PID is treated as potentially live. -const VGPUAssignmentStartupGracePeriod = devices.VGPUAssignmentGracePeriod - // VGPUCleanupPendingError reports a failed rollback that left a vGPU assigned. type VGPUCleanupPendingError struct { InstanceID string @@ -49,18 +45,9 @@ func vgpuDevicePendingCleanup(err error) (*devices.VGPUDevice, bool) { return &pending.Device, true } -func vgpuAssignmentLiveness(stored *StoredMetadata, now time.Time, livePID bool) (live bool, graceRemaining time.Duration) { - if livePID { - return true, 0 - } - if stored.GPUAssignedAt == nil { - return false, 0 - } - remaining := VGPUAssignmentStartupGracePeriod - now.Sub(*stored.GPUAssignedAt) - if remaining <= 0 { - return false, 0 - } - return true, remaining +func vgpuAssignmentMayBeLive(stored *StoredMetadata, now time.Time, hypervisorLive bool) bool { + return hypervisorLive || + stored.GPUAssignedAt != nil && now.Sub(*stored.GPUAssignedAt) < devices.VGPUAssignmentGracePeriod } func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { @@ -87,61 +74,32 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) (retained, persisted bool) { logger.FromContext(ctx).DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID) - assignment := devices.VGPUAssignment{ + releaseErr := m.destroyVGPUAssignment(ctx, devices.VGPUAssignment{ Framework: device.Framework, DevicePath: device.SysfsPath, MdevUUID: device.MdevUUID, InstanceID: instanceID, - } - cleanupMeta, err := m.loadMetadata(instanceID) - if err != nil { - logger.FromContext(ctx).WarnContext(ctx, "failed to load current metadata for vGPU cleanup; restoring rollback snapshot", "instance_id", instanceID, "error", err) - cleanupMeta = &rollbackMeta - } else { - restoreStartMutatedFields(&cleanupMeta.StoredMetadata, &rollbackMeta.StoredMetadata) - } - releaseErr := m.destroyVGPUAssignment(ctx, assignment) + }) if releaseErr != nil { logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID, "error", releaseErr) - setStoredVGPUDevice(&cleanupMeta.StoredMetadata, device, assignedAt) + setStoredVGPUDevice(&rollbackMeta.StoredMetadata, device, assignedAt) retained = true } - if err := m.saveMetadata(cleanupMeta); err != nil { + if err := m.saveMetadata(&rollbackMeta); err != nil { message := "failed to save metadata after vGPU cleanup" - if releaseErr != nil { + if retained { message = "failed to retain vGPU assignment metadata after cleanup failure" } logger.FromContext(ctx).ErrorContext(ctx, message, "instance_id", instanceID, "error", err) if !retained { return false, false } - if meta, loadErr := m.loadMetadata(instanceID); loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath { - return true, true - } - return true, false + meta, loadErr := m.loadMetadata(instanceID) + return true, loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath } return retained, retained } -func restoreStartMutatedFields(dst, src *StoredMetadata) { - dst.HypervisorPID = src.HypervisorPID - dst.HypervisorStartTime = src.HypervisorStartTime - dst.HypervisorBootID = src.HypervisorBootID - dst.ExitCode = src.ExitCode - dst.ExitMessage = src.ExitMessage - dst.ProgramStartedAt = src.ProgramStartedAt - dst.GuestAgentReadyAt = src.GuestAgentReadyAt - dst.Entrypoint = src.Entrypoint - dst.Cmd = src.Cmd - dst.IP = src.IP - dst.MAC = src.MAC - dst.GPUFramework = src.GPUFramework - dst.GPUDevicePath = src.GPUDevicePath - dst.GPUMdevUUID = src.GPUMdevUUID - dst.GPUAssignedAt = src.GPUAssignedAt - dst.StartedAt = src.StartedAt -} - func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { path := storedVGPUDevicePath(stored) if path != "" { @@ -149,7 +107,7 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) claimed := false if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { var err error - claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path) + claimed, err = m.vgpuAssignmentClaimedByLiveInstance(stored.Id, path) if err != nil { return err } @@ -173,42 +131,28 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) return nil } -func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { - files, err := m.listMetadataFilesStrict() +func (m *manager) vgpuAssignmentClaimedByLiveInstance(excludeID, devicePath string) (bool, error) { + allMetadata, err := m.listMetadataForReconcile() if err != nil { return false, fmt.Errorf("list instances for vGPU release check: %w", err) } - for _, file := range files { - id := filepath.Base(filepath.Dir(file)) - if id == excludeID { - continue - } - meta, err := m.loadMetadata(id) - if err != nil { - if errors.Is(err, ErrNotFound) { - continue - } - return false, fmt.Errorf("load metadata for vGPU release check: instance %s: %w", id, err) - } - stored := &meta.StoredMetadata - if storedVGPUDevicePath(stored) != devicePath { + for i := range allMetadata { + stored := &allMetadata[i] + if stored.Id == excludeID || storedVGPUDevicePath(stored) != devicePath { continue } - // Resolve even without a persisted PID: the socket-ownership scan can - // still prove a claimant whose post-boot metadata save failed is live. pid, err := resolveLiveHypervisorPID(stored.HypervisorProcessIdentity, stored.SocketPath) if err != nil { - return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) + return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", stored.Id, devicePath, err) } - _, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), pid > 0) if pid > 0 { return true, nil } - if remaining > 0 { + if vgpuAssignmentMayBeLive(stored, m.nowUTC(), false) { if stored.HypervisorPID == nil { - return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", stored.Id, devicePath) } - return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: recorded hypervisor is not running", id, devicePath) + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: recorded hypervisor is not running", stored.Id, devicePath) } } return false, nil diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 9ca8cef72..066c9bdee 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -75,22 +75,18 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { // Listing fails closed: any unreadable metadata aborts the pass so the vendor // VFIO sweep cannot clear a VF whose claim it could not read. func (m *manager) reconcileVGPUAssignments(ctx context.Context) (map[string]struct{}, error) { - allInstances, err := m.listInstancesForReconcile(ctx) + allMetadata, err := m.listMetadataForReconcile() if err != nil { return nil, err } protected := make(map[string]struct{}) - for i := range allInstances { - stored := &allInstances[i].StoredMetadata + for i := range allMetadata { + stored := &allMetadata[i] if storedVGPUDevicePath(stored) == "" { continue } - // The socket-ownership check runs even without a persisted PID: a VMM - // whose post-boot metadata save failed still holds its control-socket - // listener, and releasing its device would tear the vGPU out from - // under a live VM. - livePID := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) - if live, _ := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID); live { + hypervisorLive := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + if vgpuAssignmentMayBeLive(stored, m.nowUTC(), hypervisorLive) { if stored.GPUDevicePath != "" { protected[stored.GPUDevicePath] = struct{}{} } @@ -122,8 +118,8 @@ func (m *manager) releaseStaleVGPUAssignment(ctx context.Context, id string) { if path == "" { return } - livePID := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) - if live, _ := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID); live { + hypervisorLive := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + if vgpuAssignmentMayBeLive(stored, m.nowUTC(), hypervisorLive) { return } if err := m.releaseStoredVGPU(ctx, stored); err != nil { diff --git a/lib/instances/vgpu_reconcile_linux_test.go b/lib/instances/vgpu_reconcile_linux_test.go index 8708a1dd7..553c64c15 100644 --- a/lib/instances/vgpu_reconcile_linux_test.go +++ b/lib/instances/vgpu_reconcile_linux_test.go @@ -15,10 +15,6 @@ import ( "github.com/stretchr/testify/require" ) -// A VMM whose post-boot metadata save failed has no persisted PID, but it -// still holds its control-socket listener. The reconciler must protect its -// assignment past the startup grace period instead of releasing the device -// out from under the live VM. func TestReconcileVGPUsProtectsSocketOwnerWithoutPersistedPID(t *testing.T) { t.Parallel() @@ -41,7 +37,7 @@ func TestReconcileVGPUsProtectsSocketOwnerWithoutPersistedPID(t *testing.T) { }, } const id = "pid-save-failed" - stale := time.Now().UTC().Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + stale := time.Now().UTC().Add(-devices.VGPUAssignmentGracePeriod - time.Minute) require.NoError(t, m.ensureDirectories(id)) require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ Id: id, diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 395010532..7a9b66731 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -22,7 +22,7 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { deadPID := dead.Process.Pid now := time.Now().UTC() recent := now.Add(-time.Minute) - stale := now.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + stale := now.Add(-devices.VGPUAssignmentGracePeriod - time.Minute) var protected map[string]struct{} var destroyed []devices.VGPUAssignment diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index 691ae5eaa..2804e8abf 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -2,13 +2,9 @@ package instances import ( "context" - "encoding/json" - "fmt" - "os" "time" "github.com/kernel/hypeman/lib/devices" - "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) @@ -46,8 +42,7 @@ func (r *vgpuRetention) wrapPending(err error) error { return &VGPUCleanupPendingError{InstanceID: r.instanceID, Retained: r.persisted, Err: err} } -// deferWrapPending must be deferred before cleanup so it observes retention -// state recorded by rollback. +// Defer before cleanup so rollback records retention before this wraps the error. func (r *vgpuRetention) deferWrapPending(retErr *error) { *retErr = r.wrapPending(*retErr) } @@ -64,21 +59,13 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten defer func() { m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationCreate, retention.persisted) }() - // When the retention record is lost, the assignment has no metadata claim - // left; the periodic vGPU reconcile sweeps the device once it is free. retentionSurvives := func() bool { meta, err := m.loadMetadata(id) if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { - saveErr := m.saveVGPURetentionStub(retainedVGPU) - if saveErr == nil { - return true + if err := m.saveVGPURetentionStub(retainedVGPU); err != nil { + log.ErrorContext(ctx, "failed to replace surviving instance metadata with vGPU retention stub; preserving existing assignment claim", "instance_id", id, "error", err) } - log.ErrorContext(ctx, "failed to replace surviving instance metadata with vGPU retention stub", "instance_id", id, "error", saveErr) - overwriteErr := m.overwriteVGPURetentionStub(retainedVGPU) - if overwriteErr == nil { - return true - } - log.ErrorContext(ctx, "failed to overwrite surviving instance metadata with vGPU retention stub", "instance_id", id, "error", overwriteErr) + return true } if err := m.deleteInstanceData(id); err != nil { log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) @@ -127,24 +114,3 @@ func vgpuRetentionMetadata(source *StoredMetadata) *metadata { func (m *manager) saveVGPURetentionStub(source *StoredMetadata) error { return m.saveMetadata(vgpuRetentionMetadata(source)) } - -// overwriteVGPURetentionStub handles a surviving metadata file when its -// directory cannot create the temporary file used by saveMetadata. -func (m *manager) overwriteVGPURetentionStub(source *StoredMetadata) error { - retained := vgpuRetentionMetadata(source) - data, err := json.MarshalIndent(retained, "", " ") - if err != nil { - return fmt.Errorf("marshal metadata: %w", err) - } - unlockAliasReaders := hypervisor.LockSnapshotSourceAliasReaders() - defer unlockAliasReaders() - writeFile := m.writeFile - if writeFile == nil { - writeFile = os.WriteFile - } - if err := writeFile(m.paths.InstanceMetadata(source.Id), data, 0644); err != nil { - return fmt.Errorf("write metadata: %w", err) - } - m.syncAdmissionAllocation(retained) - return nil -} diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 55f88f4d9..b575c7546 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -24,53 +24,51 @@ func persistTestVGPURetention(m *manager, ctx context.Context, id string, stub * return retention.persisted } -func retainedVGPUFromCreateErrorForTest(stub StoredMetadata, assignedAt time.Time, err error) *StoredMetadata { - retention := vgpuRetention{} - retention.retainFromCreateError(stub, assignedAt, err) - return retention.stub -} - func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { t.Parallel() m := &manager{paths: paths.New(t.TempDir())} - assignedAt := time.Now().UTC() - stored := &StoredMetadata{ - Id: "failed-create", - Name: "failed-create", - GPUProfile: "NVIDIA L40S-2Q", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - GPUMdevUUID: "mdev-uuid", - GPUAssignedAt: &assignedAt, + const id = "failed-create" + stub := StoredMetadata{ + Id: id, + Name: id, NetworkEnabled: true, IP: "192.0.2.1", Volumes: []VolumeAttachment{{VolumeID: "volume"}}, HypervisorType: "qemu", - DataDir: m.paths.InstanceDir("failed-create"), + DataDir: m.paths.InstanceDir(id), } - require.NoError(t, m.ensureDirectories(stored.Id)) - require.NoError(t, os.WriteFile(m.paths.InstanceOverlay(stored.Id), []byte("overlay"), 0o644)) - require.NoError(t, os.WriteFile(m.paths.InstanceConfigDisk(stored.Id), []byte("config"), 0o644)) - require.NoError(t, os.MkdirAll(m.paths.InstanceVolumeOverlaysDir(stored.Id), 0o755)) - require.NoError(t, os.WriteFile(m.paths.InstanceVolumeOverlay(stored.Id, "volume"), []byte("volume overlay"), 0o644)) - - assert.True(t, persistTestVGPURetention(m, context.Background(), stored.Id, stored)) - assert.NoFileExists(t, m.paths.InstanceOverlay(stored.Id)) - assert.NoFileExists(t, m.paths.InstanceConfigDisk(stored.Id)) - assert.NoDirExists(t, m.paths.InstanceVolumeOverlaysDir(stored.Id)) - - retained, err := m.loadMetadata(stored.Id) + device := devices.VGPUDevice{ + ProfileName: "NVIDIA L40S-2Q", + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + assignedAt := time.Now().UTC() + retention := vgpuRetention{instanceID: id} + retention.retainFromCreateError(stub, assignedAt, &devices.VGPUCreateCleanupPendingError{Device: device, Err: errors.New("rollback failed")}) + + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, os.WriteFile(m.paths.InstanceOverlay(id), []byte("overlay"), 0o644)) + require.NoError(t, os.WriteFile(m.paths.InstanceConfigDisk(id), []byte("config"), 0o644)) + require.NoError(t, os.MkdirAll(m.paths.InstanceVolumeOverlaysDir(id), 0o755)) + require.NoError(t, os.WriteFile(m.paths.InstanceVolumeOverlay(id, "volume"), []byte("volume overlay"), 0o644)) + + m.persistVGPURetention(context.Background(), &retention) + assert.True(t, retention.persisted) + assert.NoFileExists(t, m.paths.InstanceOverlay(id)) + assert.NoFileExists(t, m.paths.InstanceConfigDisk(id)) + assert.NoDirExists(t, m.paths.InstanceVolumeOverlaysDir(id)) + + retained, err := m.loadMetadata(id) require.NoError(t, err) - assert.Equal(t, stored.Id, retained.Id) - assert.Equal(t, stored.GPUFramework, retained.GPUFramework) - assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) - assert.Equal(t, stored.GPUMdevUUID, retained.GPUMdevUUID) - assert.Equal(t, stored.GPUAssignedAt, retained.GPUAssignedAt) - assert.Equal(t, stored.Name, retained.Name) - assert.Equal(t, stored.GPUProfile, retained.GPUProfile) - assert.Equal(t, stored.HypervisorType, retained.HypervisorType) - assert.Equal(t, stored.DataDir, retained.DataDir) + assert.Equal(t, id, retained.Id) + assert.Equal(t, device.Framework, retained.GPUFramework) + assert.Equal(t, device.SysfsPath, retained.GPUDevicePath) + assert.Equal(t, assignedAt, *retained.GPUAssignedAt) + assert.Equal(t, stub.Name, retained.Name) + assert.Equal(t, device.ProfileName, retained.GPUProfile) + assert.Equal(t, stub.HypervisorType, retained.HypervisorType) + assert.Equal(t, stub.DataDir, retained.DataDir) assert.False(t, retained.NetworkEnabled) assert.Empty(t, retained.IP) assert.Empty(t, retained.Volumes) @@ -109,7 +107,7 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { require.Error(t, err, "the lost retention leaves no metadata claim, so the periodic sweep releases the VF") } -func TestCleanupFailedCreateRewritesFullMetadataWhenDirectoryIsReadOnly(t *testing.T) { +func TestCleanupFailedCreatePreservesClaimWhenStubSaveFails(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root bypasses directory permissions") } @@ -119,6 +117,7 @@ func TestCleanupFailedCreateRewritesFullMetadataWhenDirectoryIsReadOnly(t *testi require.NoError(t, m.ensureDirectories(id)) stored := &StoredMetadata{ Id: id, + Name: "surviving metadata", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", } @@ -131,61 +130,10 @@ func TestCleanupFailedCreateRewritesFullMetadataWhenDirectoryIsReadOnly(t *testi assert.True(t, persistTestVGPURetention(m, context.Background(), id, stored)) retained, err := m.loadMetadata(id) require.NoError(t, err) + assert.Equal(t, stored.Name, retained.Name) assert.Equal(t, stored.GPUFramework, retained.GPUFramework) assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) - assert.True(t, retained.GPURetainedForCleanup) - - _, err = m.startInstance(context.Background(), id, StartInstanceRequest{}) - assert.ErrorIs(t, err, errVGPURetentionStub) -} - -func TestVGPURetentionWrapPending(t *testing.T) { - cause := errors.New("boot failed") - - retention := vgpuRetention{instanceID: "inst-1"} - assert.Same(t, cause, retention.wrapPending(cause)) - - retention.retained = true - pending := retention.wrapPending(cause) - var cleanupPending *VGPUCleanupPendingError - require.ErrorAs(t, pending, &cleanupPending) - assert.False(t, cleanupPending.Retained) - - retention.persisted = true - pending = retention.wrapPending(cause) - require.ErrorAs(t, pending, &cleanupPending) - assert.True(t, cleanupPending.Retained) -} - -func TestVGPUDevicePendingCleanup(t *testing.T) { - t.Parallel() - - device := devices.VGPUDevice{ - Framework: devices.VGPUFrameworkVendorVFIO, - SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", - } - cause := errors.New("rollback failed") - pending := &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} - - wrapped := fmt.Errorf("create failed: %w", pending) - actual, ok := vgpuDevicePendingCleanup(wrapped) - require.True(t, ok) - assert.Equal(t, device, *actual) - - assignedAt := time.Now().UTC() - retained := retainedVGPUFromCreateErrorForTest(StoredMetadata{Id: "inst-1", Name: "named", Image: "img"}, assignedAt, wrapped) - require.NotNil(t, retained) - assert.Equal(t, "inst-1", retained.Id) - assert.Equal(t, "named", retained.Name, "identity fields must survive into the retention stub") - assert.Equal(t, "img", retained.Image) - assert.Equal(t, device.Framework, retained.GPUFramework) - assert.Equal(t, device.SysfsPath, retained.GPUDevicePath) - assert.Equal(t, assignedAt, *retained.GPUAssignedAt) - - actual, ok = vgpuDevicePendingCleanup(cause) - assert.False(t, ok) - assert.Nil(t, actual) - assert.Nil(t, retainedVGPUFromCreateErrorForTest(StoredMetadata{Id: "inst-1"}, assignedAt, cause)) + assert.False(t, retained.GPURetainedForCleanup) } type startRetentionNetworkManager struct { @@ -333,24 +281,6 @@ func TestStartReportsUnretainedVGPUWhenRetentionSaveFails(t *testing.T) { assert.Empty(t, stored.GPUDevicePath, "retention save failed, so no assignment should be recorded") } -func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { - m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { - return nil - }) - meta, err := m.loadMetadata(id) - require.NoError(t, err) - meta.HypervisorType = hypervisor.TypeCloudHypervisor - require.NoError(t, m.saveMetadata(meta)) - - cause := errors.New("create failed") - m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { - return nil, cause - } - - _, err = m.startInstance(context.Background(), id, StartInstanceRequest{}) - assert.ErrorIs(t, err, cause) -} - func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { var destroyed []devices.VGPUAssignment m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { @@ -439,37 +369,24 @@ func TestCleanupStartVGPUReportsRetainedWhenMidStartSaveSurvives(t *testing.T) { func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { m := &manager{ - paths: paths.New(t.TempDir()), - destroyVGPU: func(context.Context, devices.VGPUAssignment) error { - return nil - }, + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, } const id = "failed-start" require.NoError(t, m.ensureDirectories(id)) previousStart := time.Now().Add(-time.Hour).UTC() - previousProgramStart := previousStart.Add(time.Second) exitCode := 1 rollbackMeta := metadata{StoredMetadata: StoredMetadata{ - Id: id, - Name: "original name", - GPUProfile: "NVIDIA L40S-2Q", - Entrypoint: []string{"old-entrypoint"}, - Cmd: []string{"old-command"}, - StartedAt: &previousStart, - ProgramStartedAt: &previousProgramStart, - ExitCode: &exitCode, - ExitMessage: "previous exit", + Id: id, + Name: "original name", + GPUProfile: "NVIDIA L40S-2Q", + Entrypoint: []string{"old-entrypoint"}, + StartedAt: &previousStart, + ExitCode: &exitCode, + ExitMessage: "previous exit", }} - - partial := rollbackMeta - partial.Name = "concurrent update" - partial.Entrypoint = []string{"new-entrypoint"} - partial.Cmd = []string{"new-command"} - partial.StartedAt = ptr(time.Now().UTC()) - partial.ProgramStartedAt = nil - partial.ExitCode = nil - partial.ExitMessage = "" + partial := metadata{StoredMetadata: StoredMetadata{Id: id, Name: "partial start"}} assignedAt := time.Now().UTC() device := &devices.VGPUDevice{ Framework: devices.VGPUFrameworkVendorVFIO, @@ -482,15 +399,7 @@ func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { stored, err := m.loadMetadata(id) require.NoError(t, err) - assert.Equal(t, "concurrent update", stored.Name) - assert.Equal(t, rollbackMeta.Entrypoint, stored.Entrypoint) - assert.Equal(t, rollbackMeta.Cmd, stored.Cmd) - assert.Equal(t, rollbackMeta.StartedAt, stored.StartedAt) - assert.Equal(t, rollbackMeta.ProgramStartedAt, stored.ProgramStartedAt) - assert.Equal(t, rollbackMeta.ExitCode, stored.ExitCode) - assert.Equal(t, rollbackMeta.ExitMessage, stored.ExitMessage) - assert.Empty(t, stored.GPUDevicePath) - assert.Nil(t, stored.GPUAssignedAt) + assert.Equal(t, rollbackMeta.StoredMetadata, stored.StoredMetadata) } func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { @@ -500,7 +409,7 @@ func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) require.NoError(t, m.ensureDirectories("invalid-instance")) require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid-instance"), []byte("{"), 0o644)) - _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + _, err := m.vgpuAssignmentClaimedByLiveInstance("other-instance", "/sys/bus/pci/devices/0000:82:00.4") require.Error(t, err) } @@ -521,7 +430,7 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. }, }})) - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/mdev/devices/legacy-uuid") + claimed, err := m.vgpuAssignmentClaimedByLiveInstance("other-instance", "/sys/bus/mdev/devices/legacy-uuid") require.NoError(t, err) assert.True(t, claimed) } @@ -533,7 +442,7 @@ func TestVGPUAssignmentClaimedByLiveInstanceLiveness(t *testing.T) { deadPID := 1 << 30 require.False(t, ProcessExists(deadPID)) recent := time.Now().UTC() - stale := recent.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + stale := recent.Add(-devices.VGPUAssignmentGracePeriod - time.Minute) tests := []struct { name string assignedAt *time.Time @@ -559,7 +468,7 @@ func TestVGPUAssignmentClaimedByLiveInstanceLiveness(t *testing.T) { HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: tt.pid}, }})) - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", devicePath) + claimed, err := m.vgpuAssignmentClaimedByLiveInstance("requester", devicePath) if tt.wantErr != "" { require.ErrorContains(t, err, tt.wantErr) return @@ -635,37 +544,3 @@ func TestStoredVGPUDevicePath(t *testing.T) { })) assert.Empty(t, storedVGPUDevicePath(&StoredMetadata{})) } - -func TestReleaseStoredVGPURetainsMetadataOnFailure(t *testing.T) { - t.Parallel() - - m := &manager{paths: paths.New(t.TempDir())} - stored := &StoredMetadata{ - GPUFramework: devices.VGPUFramework("future-framework"), - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - } - err := m.releaseStoredVGPU(context.Background(), stored) - assert.Error(t, err) - assert.Equal(t, devices.VGPUFramework("future-framework"), stored.GPUFramework) - assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) -} - -func TestSetAndClearStoredVGPUDevice(t *testing.T) { - t.Parallel() - - assignedAt := time.Now().UTC() - stored := &StoredMetadata{} - setStoredVGPUDevice(stored, &devices.VGPUDevice{ - Framework: devices.VGPUFrameworkVendorVFIO, - SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", - }, assignedAt) - assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) - assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) - assert.Equal(t, assignedAt, *stored.GPUAssignedAt) - - clearStoredVGPUDevice(stored) - assert.Empty(t, stored.GPUFramework) - assert.Empty(t, stored.GPUDevicePath) - assert.Empty(t, stored.GPUMdevUUID) - assert.Nil(t, stored.GPUAssignedAt) -} From be458d17f21e06fd66e62c3338ab51dfea05df30 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:11:57 +0000 Subject: [PATCH 098/107] Simplify vGPU retention persistence to a single attempt --- lib/instances/vgpu.go | 5 +++-- lib/instances/vgpu_retention.go | 19 +++---------------- lib/instances/vgpu_test.go | 33 ++------------------------------- 3 files changed, 8 insertions(+), 49 deletions(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 7b70e807c..6d45d82f6 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -94,8 +94,7 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic if !retained { return false, false } - meta, loadErr := m.loadMetadata(instanceID) - return true, loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath + return true, false } return retained, retained } @@ -132,6 +131,8 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) } func (m *manager) vgpuAssignmentClaimedByLiveInstance(excludeID, devicePath string) (bool, error) { + // Each vendor VFIO release lists and loads every instance's metadata. This is + // acceptable at GPU-host scale (tens of VFs and instances); revisit at hundreds. allMetadata, err := m.listMetadataForReconcile() if err != nil { return false, fmt.Errorf("list instances for vGPU release check: %w", err) diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index 2804e8abf..a0e048bdc 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -59,32 +59,19 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten defer func() { m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationCreate, retention.persisted) }() - retentionSurvives := func() bool { - meta, err := m.loadMetadata(id) - if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { - if err := m.saveVGPURetentionStub(retainedVGPU); err != nil { - log.ErrorContext(ctx, "failed to replace surviving instance metadata with vGPU retention stub; preserving existing assignment claim", "instance_id", id, "error", err) - } - return true - } - if err := m.deleteInstanceData(id); err != nil { - log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) - } - return false - } + + // An unpersisted retention leaves no metadata claim. The periodic reconciler releases + // the VF after its grace period once no open VFIO handles remain. if err := m.deleteInstanceData(id); err != nil { log.ErrorContext(ctx, "failed to clean instance data before retaining vGPU assignment", "instance_id", id, "error", err) - retention.persisted = retentionSurvives() return } if err := m.ensureDirectories(id); err != nil { log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) - retention.persisted = retentionSurvives() return } if err := m.saveVGPURetentionStub(retainedVGPU); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) - retention.persisted = retentionSurvives() return } retention.persisted = true diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index b575c7546..d8814e17e 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -107,35 +107,6 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { require.Error(t, err, "the lost retention leaves no metadata claim, so the periodic sweep releases the VF") } -func TestCleanupFailedCreatePreservesClaimWhenStubSaveFails(t *testing.T) { - if os.Geteuid() == 0 { - t.Skip("root bypasses directory permissions") - } - - m := &manager{paths: paths.New(t.TempDir())} - const id = "failed-create" - require.NoError(t, m.ensureDirectories(id)) - stored := &StoredMetadata{ - Id: id, - Name: "surviving metadata", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - } - require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: *stored})) - - instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) - require.NoError(t, os.Chmod(instanceDir, 0o555)) - t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) - - assert.True(t, persistTestVGPURetention(m, context.Background(), id, stored)) - retained, err := m.loadMetadata(id) - require.NoError(t, err) - assert.Equal(t, stored.Name, retained.Name) - assert.Equal(t, stored.GPUFramework, retained.GPUFramework) - assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) - assert.False(t, retained.GPURetainedForCleanup) -} - type startRetentionNetworkManager struct { network.Manager config network.NetworkConfig @@ -338,7 +309,7 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { assert.Empty(t, stored.Entrypoint) } -func TestCleanupStartVGPUReportsRetainedWhenMidStartSaveSurvives(t *testing.T) { +func TestCleanupStartVGPUReportsUnpersistedRetentionWhenRollbackSaveFails(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root bypasses directory permissions") } @@ -364,7 +335,7 @@ func TestCleanupStartVGPUReportsRetainedWhenMidStartSaveSurvives(t *testing.T) { retained, persisted := m.cleanupStartVGPU(context.Background(), id, &device, assignedAt, rollbackMeta) assert.True(t, retained) - assert.True(t, persisted, "a surviving mid-start save keeps the assignment recoverable via delete") + assert.False(t, persisted) } func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { From a43dee1d1b61d35d54fe3882b883639d9d980566 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:52:33 +0000 Subject: [PATCH 099/107] Report vGPU retention as persisted when the assignment claim survives When rollback release fails and the subsequent retention save also fails, the assignment save from earlier in start may still hold the claim on disk. Check for that surviving claim before reporting the retention as unpersisted, so the API does not emit vgpu_unretained_instance for an assignment that is still durably attributed. --- lib/instances/vgpu.go | 7 +++++ lib/instances/vgpu_test.go | 52 ++++++++++++++++++++++++-------------- 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 6d45d82f6..00e32ba01 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -94,6 +94,13 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic if !retained { return false, false } + // The pre-cleanup assignment save may already hold this claim on disk, in + // which case the retention is durable despite the failed rollback save. + if onDisk, loadErr := m.loadMetadata(instanceID); loadErr == nil && + onDisk.GPUDevicePath == device.SysfsPath && + onDisk.GPUAssignedAt != nil && onDisk.GPUAssignedAt.Equal(assignedAt) { + return true, true + } return true, false } return retained, retained diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index d8814e17e..e82773115 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -309,33 +309,47 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { assert.Empty(t, stored.Entrypoint) } -func TestCleanupStartVGPUReportsUnpersistedRetentionWhenRollbackSaveFails(t *testing.T) { +func TestCleanupStartVGPUReportsRetentionWhenRollbackSaveFails(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root bypasses directory permissions") } - m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { - return errors.New("destroy failed") - }) - device := devices.VGPUDevice{ - Framework: devices.VGPUFrameworkVendorVFIO, - SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + tests := []struct { + name string + assignmentSaved bool + wantPersisted bool + }{ + {name: "assignment save survived", assignmentSaved: true, wantPersisted: true}, + {name: "assignment never saved", assignmentSaved: false, wantPersisted: false}, } - assignedAt := time.Now().UTC() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return errors.New("destroy failed") + }) + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + assignedAt := time.Now().UTC() - meta, err := m.loadMetadata(id) - require.NoError(t, err) - rollbackMeta := *meta - setStoredVGPUDevice(&meta.StoredMetadata, &device, assignedAt) - require.NoError(t, m.saveMetadata(meta)) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + rollbackMeta := *meta + if tt.assignmentSaved { + setStoredVGPUDevice(&meta.StoredMetadata, &device, assignedAt) + require.NoError(t, m.saveMetadata(meta)) + } - instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) - require.NoError(t, os.Chmod(instanceDir, 0o555)) - t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) - retained, persisted := m.cleanupStartVGPU(context.Background(), id, &device, assignedAt, rollbackMeta) - assert.True(t, retained) - assert.False(t, persisted) + retained, persisted := m.cleanupStartVGPU(context.Background(), id, &device, assignedAt, rollbackMeta) + assert.True(t, retained) + assert.Equal(t, tt.wantPersisted, persisted) + }) + } } func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { From 07da77245f8a66efbd2d0fa88c4e800885c11fc8 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:49:27 +0000 Subject: [PATCH 100/107] Protect claimed mdevs during reconciliation --- lib/devices/mdev_darwin.go | 2 +- lib/devices/mdev_linux.go | 22 +++++++++++++++++++--- lib/devices/vgpu_linux.go | 22 +++++++++++++++++----- lib/devices/vgpu_linux_test.go | 24 ++++++++++++++++++++++++ lib/instances/vgpu_reconcile.go | 13 ++++++------- lib/instances/vgpu_reconcile_test.go | 20 +++++++++++--------- 6 files changed, 78 insertions(+), 25 deletions(-) diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index 4b726bb08..93d3adbf7 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -58,7 +58,7 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { return nil } -func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepVendorVFIO bool) error { +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepDevices bool) error { return nil } diff --git a/lib/devices/mdev_linux.go b/lib/devices/mdev_linux.go index 0908e8765..423d8473d 100644 --- a/lib/devices/mdev_linux.go +++ b/lib/devices/mdev_linux.go @@ -707,17 +707,27 @@ func mdevPastGracePeriod(mdevUUID string, gracePeriod time.Duration) (bool, time return age >= gracePeriod, age, nil } +func protectedMdevUUIDs(instanceInfos []MdevReconcileInfo) map[string]struct{} { + protected := make(map[string]struct{}, len(instanceInfos)) + for _, info := range instanceInfos { + if info.MdevUUID != "" && info.IsRunning { + protected[info.MdevUUID] = struct{}{} + } + } + return protected +} + // ReconcileMdevs destroys orphaned mdevs on managed VFs. -// This is called on server startup to clean up stale mdevs from previous runs. // // Policy: // - Consider only mdevs whose parent VF is currently managed by hypeman (discoverable via /sys/class/mdev_bus) +// - Keep mdevs claimed by live instance metadata // - Keep mdevs whose VFIO group has an open file handle (/dev/vfio/) // - Keep mdevs younger than a short grace period to avoid racing very recent state transitions // - Delete all remaining mdevs func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) error { log := logger.FromContext(ctx) - _ = instanceInfos + protectedMdevs := protectedMdevUUIDs(instanceInfos) vfs, err := discoverMdevVFs() if err != nil { @@ -750,13 +760,18 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro ) groupInUseCache := make(map[int]bool) - var destroyed, failedDestroy, skippedUnmanagedVF, skippedInUse, skippedGrace, skippedProbeError int + var destroyed, failedDestroy, skippedUnmanagedVF, skippedClaimed, skippedInUse, skippedGrace, skippedProbeError int for _, mdev := range mdevs { if _, ok := managedVFs[mdev.VFAddress]; !ok { log.DebugContext(ctx, "skipping mdev on unmanaged VF", "uuid", mdev.UUID, "vf", mdev.VFAddress) skippedUnmanagedVF++ continue } + if _, ok := protectedMdevs[mdev.UUID]; ok { + log.DebugContext(ctx, "skipping mdev claimed by live instance", "uuid", mdev.UUID) + skippedClaimed++ + continue + } group, err := mdevIOMMUGroup(mdev.UUID) if err != nil { @@ -817,6 +832,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro "destroyed", destroyed, "failed_destroy", failedDestroy, "skipped_unmanaged_vf", skippedUnmanagedVF, + "skipped_claimed", skippedClaimed, "skipped_in_use", skippedInUse, "skipped_grace", skippedGrace, "skipped_probe_error", skippedProbeError, diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index 36bf86c9e..354217fd5 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -104,20 +104,32 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { } } +func mdevReconcileInfos(protectedDevicePaths map[string]struct{}) []MdevReconcileInfo { + instanceInfos := make([]MdevReconcileInfo, 0, len(protectedDevicePaths)) + for devicePath := range protectedDevicePaths { + instanceInfos = append(instanceInfos, MdevReconcileInfo{ + MdevUUID: filepath.Base(devicePath), + IsRunning: true, + }) + } + return instanceInfos +} + // ReconcileVGPUs releases orphaned vGPU assignments. -func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepVendorVFIO bool) error { +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepDevices bool) error { framework, _, err := DiscoverVGPU() if err != nil { return err } + if !sweepDevices { + return nil + } switch framework { case VGPUFrameworkMdev: - return ReconcileMdevs(ctx, nil) + return ReconcileMdevs(ctx, mdevReconcileInfos(protectedDevicePaths)) case VGPUFrameworkVendorVFIO: - if sweepVendorVFIO { - return hostVendorVFIO.reconcile(ctx, protectedDevicePaths) - } + return hostVendorVFIO.reconcile(ctx, protectedDevicePaths) } return nil } diff --git a/lib/devices/vgpu_linux_test.go b/lib/devices/vgpu_linux_test.go index 7b03d9c26..5bb52da1d 100644 --- a/lib/devices/vgpu_linux_test.go +++ b/lib/devices/vgpu_linux_test.go @@ -12,6 +12,30 @@ import ( "github.com/stretchr/testify/require" ) +func TestMdevReconcileInfosProtectClaimedDevicePath(t *testing.T) { + t.Parallel() + + infos := mdevReconcileInfos(map[string]struct{}{ + "/sys/bus/mdev/devices/claimed": {}, + }) + + assert.Equal(t, []MdevReconcileInfo{{MdevUUID: "claimed", IsRunning: true}}, infos) +} + +func TestProtectedMdevUUIDs(t *testing.T) { + t.Parallel() + + protected := protectedMdevUUIDs([]MdevReconcileInfo{ + {MdevUUID: "claimed", IsRunning: true}, + {MdevUUID: "stale"}, + {IsRunning: true}, + }) + + assert.Contains(t, protected, "claimed") + assert.NotContains(t, protected, "stale") + assert.Len(t, protected, 1) +} + func TestDiscoverVGPUWithPropagatesMdevError(t *testing.T) { t.Parallel() diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 066c9bdee..27ecb9e01 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -54,17 +54,17 @@ func (m *manager) StartVGPUReconciler(ctx context.Context) { func (m *manager) ReconcileVGPUs(ctx context.Context) { log := logger.FromContext(ctx) protected, err := m.reconcileVGPUAssignments(ctx) - sweepVendorVFIO := err == nil + sweepDevices := err == nil if err != nil { m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageListInstances) - log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping vendor VFIO sweep until the next pass, mdev reconcile still runs", "error", err) + log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping device sweep until the next pass", "error", err) protected = make(map[string]struct{}) } reconcileDevices := m.reconcileVGPUDevices if reconcileDevices == nil { reconcileDevices = devices.ReconcileVGPUs } - if err := reconcileDevices(ctx, protected, sweepVendorVFIO); err != nil { + if err := reconcileDevices(ctx, protected, sweepDevices); err != nil { m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageReconcileDevices) log.WarnContext(ctx, "failed to reconcile vGPU devices", "error", err) } @@ -82,14 +82,13 @@ func (m *manager) reconcileVGPUAssignments(ctx context.Context) (map[string]stru protected := make(map[string]struct{}) for i := range allMetadata { stored := &allMetadata[i] - if storedVGPUDevicePath(stored) == "" { + devicePath := storedVGPUDevicePath(stored) + if devicePath == "" { continue } hypervisorLive := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) if vgpuAssignmentMayBeLive(stored, m.nowUTC(), hypervisorLive) { - if stored.GPUDevicePath != "" { - protected[stored.GPUDevicePath] = struct{}{} - } + protected[devicePath] = struct{}{} continue } m.releaseStaleVGPUAssignment(ctx, stored.Id) diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 7a9b66731..8b8d71b66 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -33,9 +33,9 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { destroyed = append(destroyed, assignment) return nil }, - reconcileVGPUDevices: func(_ context.Context, p map[string]struct{}, sweepVendorVFIO bool) error { + reconcileVGPUDevices: func(_ context.Context, p map[string]struct{}, sweepDevices bool) error { protected = p - assert.True(t, sweepVendorVFIO) + assert.True(t, sweepDevices) return nil }, } @@ -45,6 +45,7 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { {Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}, {Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}}, {Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, GPUAssignedAt: &recent}, + {Id: "legacy-mdev-booting", GPUMdevUUID: "test-mdev", GPUAssignedAt: &recent}, } for i := range instances { require.NoError(t, m.ensureDirectories(instances[i].Id)) @@ -57,6 +58,7 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") + assert.Contains(t, protected, "/sys/bus/mdev/devices/test-mdev") for _, id := range []string{"orphaned", "legacy", "dead"} { stored, err := m.loadMetadata(id) @@ -71,14 +73,14 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { orphaned, err := m.loadMetadata("orphaned") require.NoError(t, err) assert.Equal(t, "NVIDIA L40S-2Q", orphaned.GPUProfile) - for _, id := range []string{"booting", "stale-pid-booting"} { + for _, id := range []string{"booting", "stale-pid-booting", "legacy-mdev-booting"} { stored, err := m.loadMetadata(id) require.NoError(t, err) - assert.NotEmpty(t, stored.GPUDevicePath, "live assignment on %s must be kept", id) + assert.NotEmpty(t, storedVGPUDevicePath(&stored.StoredMetadata), "live assignment on %s must be kept", id) } } -func TestReconcileVGPUsSkipsVendorSweepWhenListingFails(t *testing.T) { +func TestReconcileVGPUsSkipsDeviceSweepWhenListingFails(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root bypasses directory permissions") } @@ -86,8 +88,8 @@ func TestReconcileVGPUsSkipsVendorSweepWhenListingFails(t *testing.T) { var sweeps []bool m := &manager{ paths: paths.New(t.TempDir()), - reconcileVGPUDevices: func(_ context.Context, _ map[string]struct{}, sweepVendorVFIO bool) error { - sweeps = append(sweeps, sweepVendorVFIO) + reconcileVGPUDevices: func(_ context.Context, _ map[string]struct{}, sweepDevices bool) error { + sweeps = append(sweeps, sweepDevices) return nil }, } @@ -100,11 +102,11 @@ func TestReconcileVGPUsSkipsVendorSweepWhenListingFails(t *testing.T) { m.ReconcileVGPUs(t.Context()) require.Equal(t, []bool{false}, sweeps, - "a listing failure must skip the vendor sweep, not run it with an empty protection set") + "a listing failure must skip the device sweep, not run it with an empty protection set") require.NoError(t, os.Chmod(instanceDir, 0o755)) m.ReconcileVGPUs(t.Context()) - assert.Equal(t, []bool{false, true}, sweeps, "the next pass retries the vendor sweep") + assert.Equal(t, []bool{false, true}, sweeps, "the next pass retries the device sweep") } func TestReconcileVGPUsKeepsAssignmentWhenReleaseFails(t *testing.T) { From 4f13a3c174a3a2089212fbf94e7c054cd99784ab Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:56:07 +0000 Subject: [PATCH 101/107] Unify vGPU retention wrapping and dedupe test fixtures --- lib/instances/create.go | 3 ++- lib/instances/start.go | 19 +++++++++--------- lib/instances/vgpu_retention.go | 5 ----- lib/instances/vgpu_test.go | 35 +++++++++++++-------------------- 4 files changed, 26 insertions(+), 36 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index 525f2c262..cd50deb5c 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -280,7 +280,8 @@ func (m *manager) createInstance( var gpuAssignedAt *time.Time retention := vgpuRetention{instanceID: id} - defer retention.deferWrapPending(&retErr) + // Deferred before cu.Clean so rollback records retention before this wraps the error. + defer func() { retErr = retention.wrapPending(retErr) }() cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) m.persistVGPURetention(ctx, &retention) diff --git a/lib/instances/start.go b/lib/instances/start.go index ea8c2a6f4..37b836427 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -121,7 +121,8 @@ func (m *manager) startInstance( // Setup cleanup stack for automatic rollback on errors retention := vgpuRetention{instanceID: id} - defer retention.deferWrapPending(&retErr) + // Deferred before cu.Clean so rollback records retention before this wraps the error. + defer func() { retErr = retention.wrapPending(retErr) }() cu := cleanup.Make(func() {}) defer cu.Clean() @@ -177,20 +178,20 @@ func (m *manager) startInstance( device, err := m.createVGPUDevice(ctx, stored.GPUProfile, id) if err != nil { log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) + wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) if pendingDevice, ok := vgpuDevicePendingCleanup(err); ok { - assignedAt := m.nowUTC() retentionMeta := rollbackMeta - setStoredVGPUDevice(&retentionMeta.StoredMetadata, pendingDevice, assignedAt) - wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) + setStoredVGPUDevice(&retentionMeta.StoredMetadata, pendingDevice, m.nowUTC()) + persisted := true if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { - m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, false) log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) - return nil, &VGPUCleanupPendingError{InstanceID: id, Err: fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr)} + wrapped = fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr) + persisted = false } - m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, true) - return nil, &VGPUCleanupPendingError{InstanceID: id, Retained: true, Err: wrapped} + retention.markRetained(persisted) + m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, persisted) } - return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) + return nil, wrapped } assignedAt := m.nowUTC() setStoredVGPUDevice(stored, device, assignedAt) diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index a0e048bdc..1bcba2c42 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -42,11 +42,6 @@ func (r *vgpuRetention) wrapPending(err error) error { return &VGPUCleanupPendingError{InstanceID: r.instanceID, Retained: r.persisted, Err: err} } -// Defer before cleanup so rollback records retention before this wraps the error. -func (r *vgpuRetention) deferWrapPending(retErr *error) { - *retErr = r.wrapPending(*retErr) -} - func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuRetention) { if retention.stub == nil { m.deleteInstanceData(retention.instanceID) diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index e82773115..07cdcb8e5 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -18,6 +18,16 @@ import ( "github.com/stretchr/testify/require" ) +func testVendorVFIODevice(profileName string) devices.VGPUDevice { + return devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + ProfileType: "1148", + ProfileName: profileName, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } +} + func persistTestVGPURetention(m *manager, ctx context.Context, id string, stub *StoredMetadata) bool { retention := vgpuRetention{instanceID: id, stub: stub, retained: stub != nil} m.persistVGPURetention(ctx, &retention) @@ -131,13 +141,8 @@ func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, dev instanceLocks: sync.Map{}, bootMarkerScans: sync.Map{}, createVGPU: func(_ context.Context, profileName, _ string) (*devices.VGPUDevice, error) { - return &devices.VGPUDevice{ - Framework: devices.VGPUFrameworkVendorVFIO, - VFAddress: "0000:82:00.4", - ProfileType: "1148", - ProfileName: profileName, - SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", - }, nil + device := testVendorVFIODevice(profileName) + return &device, nil }, destroyVGPU: destroy, } @@ -180,13 +185,7 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { meta.ExitMessage = "previous exit" require.NoError(t, m.saveMetadata(meta)) - device := devices.VGPUDevice{ - Framework: devices.VGPUFrameworkVendorVFIO, - VFAddress: "0000:82:00.4", - ProfileType: "1148", - ProfileName: "NVIDIA L40S-2Q", - SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", - } + device := testVendorVFIODevice("NVIDIA L40S-2Q") cause := errors.New("create verification and rollback failed") m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} @@ -225,13 +224,7 @@ func TestStartReportsUnretainedVGPUWhenRetentionSaveFails(t *testing.T) { m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { return nil }) - device := devices.VGPUDevice{ - Framework: devices.VGPUFrameworkVendorVFIO, - VFAddress: "0000:82:00.4", - ProfileType: "1148", - ProfileName: "NVIDIA L40S-2Q", - SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", - } + device := testVendorVFIODevice("NVIDIA L40S-2Q") cause := errors.New("create verification and rollback failed") m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) From f0b0e11d0199158215c5b90d4fe690295c8cc973 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:23:35 +0000 Subject: [PATCH 102/107] Quarantine unhealthy vGPU VFs via a persisted health store Add a VF health store persisted at /gpu/vf-health.json: init failures reported against a VF are tallied per instance assignment, and once failures accumulate from gpu.vf_quarantine_threshold distinct assignments (default 2) the VF is quarantined. Quarantined VFs are excluded from placement and advertised profile availability, cards with quarantined VFs are deprioritized, and selection among equivalent free VFs is randomized. An exact-assignment success report clears the match and older tallies and rescinds that assignment's quarantine. An unreadable or invalid state file fails closed: mutations are refused, placement and advertised availability are disabled, and loads are retried after repair. Writes fsync before and after the rename. GET /resources reports allocatable_slots and quarantined_slots, and GPU admission gates on the allocatable count. GPU.md documents the store semantics, draining the parent GPU, the SR-IOV recovery cycle, and clearing quarantine state. --- cmd/api/api/resources.go | 8 +- cmd/api/config/config.go | 9 +- cmd/api/config/config_test.go | 12 + cmd/api/main.go | 1 + config.example.yaml | 6 + lib/devices/GPU.md | 65 +++- lib/devices/manager.go | 4 + lib/devices/vendor_vfio_linux.go | 49 ++- lib/devices/vendor_vfio_linux_test.go | 66 ++++ lib/devices/vf_health.go | 416 +++++++++++++++++++++ lib/devices/vf_health_test.go | 408 +++++++++++++++++++++ lib/oapi/oapi.go | 498 +++++++++++++------------- lib/paths/paths.go | 5 + lib/resources/gpu.go | 53 +-- lib/resources/gpu_test.go | 88 +++++ lib/resources/monitoring_test.go | 4 +- lib/resources/resource.go | 17 +- openapi.yaml | 12 +- 18 files changed, 1417 insertions(+), 304 deletions(-) create mode 100644 lib/devices/vf_health.go create mode 100644 lib/devices/vf_health_test.go create mode 100644 lib/resources/gpu_test.go diff --git a/cmd/api/api/resources.go b/cmd/api/api/resources.go index ebb6ad951..dec9f35eb 100644 --- a/cmd/api/api/resources.go +++ b/cmd/api/api/resources.go @@ -87,9 +87,11 @@ func convertResourceStatus(rs resources.ResourceStatus) oapi.ResourceStatus { func convertGPUResourceStatus(gs *resources.GPUResourceStatus) oapi.GPUResourceStatus { result := oapi.GPUResourceStatus{ - Mode: oapi.GPUResourceStatusMode(gs.Mode), - TotalSlots: gs.TotalSlots, - UsedSlots: gs.UsedSlots, + Mode: oapi.GPUResourceStatusMode(gs.Mode), + TotalSlots: gs.TotalSlots, + UsedSlots: gs.UsedSlots, + AllocatableSlots: gs.AllocatableSlots, + QuarantinedSlots: gs.QuarantinedSlots, } // Convert profiles (vGPU mode) diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index b46f4f2e5..c250a3621 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -269,7 +269,8 @@ type SnapshotConfig struct { // GPUConfig holds GPU-related settings. type GPUConfig struct { - ProfileCacheTTL string `koanf:"profile_cache_ttl"` + ProfileCacheTTL string `koanf:"profile_cache_ttl"` + VFQuarantineThreshold int `koanf:"vf_quarantine_threshold"` } // Config is the top-level Hypeman server configuration. @@ -494,7 +495,8 @@ func defaultConfig() *Config { }, GPU: GPUConfig{ - ProfileCacheTTL: "30m", + ProfileCacheTTL: "30m", + VFQuarantineThreshold: 2, }, } } @@ -647,6 +649,9 @@ func (c *Config) Validate() error { if c.Build.MaxConcurrentSourceBuilds <= 0 { return fmt.Errorf("build.max_concurrent_source_builds must be positive, got %d", c.Build.MaxConcurrentSourceBuilds) } + if c.GPU.VFQuarantineThreshold < 1 { + return fmt.Errorf("gpu.vf_quarantine_threshold must be >= 1, got %d", c.GPU.VFQuarantineThreshold) + } if c.Limits.MaxConcurrentPushes <= 0 { return fmt.Errorf("limits.max_concurrent_pushes must be positive, got %d", c.Limits.MaxConcurrentPushes) } diff --git a/cmd/api/config/config_test.go b/cmd/api/config/config_test.go index 5660d878e..efd52f208 100644 --- a/cmd/api/config/config_test.go +++ b/cmd/api/config/config_test.go @@ -250,6 +250,18 @@ func TestValidateRejectsInvalidMetricsPort(t *testing.T) { } } +func TestValidateRejectsInvalidVFQuarantineThreshold(t *testing.T) { + for _, threshold := range []int{0, -1} { + cfg := defaultConfig() + cfg.GPU.VFQuarantineThreshold = threshold + + err := cfg.Validate() + if err == nil { + t.Fatalf("expected validation error for vf_quarantine_threshold %d", threshold) + } + } +} + func TestValidateRejectsInvalidMetricExportInterval(t *testing.T) { cfg := defaultConfig() cfg.Otel.MetricExportInterval = "not-a-duration" diff --git a/cmd/api/main.go b/cmd/api/main.go index faa9ac15a..d6fba1b43 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -204,6 +204,7 @@ func run() error { // Configure GPU profile cache TTL devices.SetGPUProfileCacheTTL(cfg.GPU.ProfileCacheTTL) + devices.SetVFQuarantineThreshold(cfg.GPU.VFQuarantineThreshold) // Initialize OpenTelemetry (before wire initialization) otelCfg := otel.Config{ diff --git a/config.example.yaml b/config.example.yaml index ebef41257..70d55fa68 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -170,6 +170,12 @@ data_dir: /var/lib/hypeman # idle_ttl: "" # delete builders idle this long (e.g. "24h"); # # destructive, empty = disabled +# gpu: +# profile_cache_ttl: 30m # vGPU profile metadata cache TTL +# vf_quarantine_threshold: 2 # distinct instance assignments that must report +# # a guest driver init failure before the VF is +# # quarantined (must be >= 1) + # ============================================================================= # Resource Limits # ============================================================================= diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index d04dcf599..17c387723 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -49,8 +49,10 @@ curl -s http://localhost:4973/resources | jq .gpu "mode": "vgpu", "total_slots": 64, "used_slots": 5, + "allocatable_slots": 57, + "quarantined_slots": 2, "profiles": [ - {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 59}, + {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 57}, {"name": "L40S-2Q", "framebuffer_mb": 2048, "available": 30}, {"name": "L40S-4Q", "framebuffer_mb": 4096, "available": 16} ] @@ -121,6 +123,8 @@ curl -s http://localhost:4973/resources | jq .gpu "mode": "passthrough", "total_slots": 4, "used_slots": 2, + "allocatable_slots": 2, + "quarantined_slots": 0, "devices": [ {"name": "NVIDIA L40S", "available": true}, {"name": "NVIDIA L40S", "available": false} @@ -185,8 +189,10 @@ Returns GPU status along with other resources: "mode": "vgpu", "total_slots": 64, "used_slots": 5, + "allocatable_slots": 57, + "quarantined_slots": 2, "profiles": [ - {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 59} + {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 57} ] } } @@ -282,10 +288,25 @@ NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884) ``` (0x65 = timeout; the guest's init requests are never answered, and -`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle). Because -placement is deterministic least-loaded, an idle host re-picks the same VF for -every request, so one wedged VF presents as all vGPU instances failing while -`/resources` reports full capacity. +`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle). + +Hypeman tracks these failures in `/gpu/vf-health.json` (it survives +restarts): each reported init failure is tallied per instance assignment, and +once failures accumulate from `gpu.vf_quarantine_threshold` distinct +assignments (default 2), the VF is quarantined: excluded from placement and +from advertised profile availability, and its parent GPU becomes +overflow-only — deprioritized for new placements. Selection among a card's +equivalent free VFs is randomized so a wedged VF cannot capture every +placement. A reported init success clears failures only when that exact +assignment has a recorded failure, removing the match and older tallies; if +that assignment crossed the threshold, its later success also rescinds the +quarantine. If the state file exists but cannot be loaded, placement and +advertised availability fail closed until it is repaired or removed. + +`used_slots` includes quarantined VFs still held by running instances, so it +can overlap `quarantined_slots`; use `allocatable_slots` for admission. + +Quarantine only removes capacity — it never touches a running instance. The wedge itself leaves no host-side log: no kernel error, no XID, no plugin crash. The trigger is a SIGKILL delivered to QEMU while the vGPU plugin is @@ -303,18 +324,44 @@ External SIGKILLs (OOM killer, manual `kill -9`) can still trigger it. Confirm by assigning the same profile on a different VF: if that guest initializes, the VF is wedged, not the driver stack. Remediate by cycling SR-IOV on the parent GPU (this destroys and recreates all of its VFs, so it -requires no vGPU assignments on that GPU): +requires no vGPU assignments on that GPU). The DCGM quiesce is not optional: +with `nv-hostengine`/`dcgm-exporter` holding the GPUs open, `sriov-manage -d` +fails with `Cannot obtain unbindLock` on first contact. + +Any manual edit to `vf-health.json` needs an immediate hypeman restart: the +store loads only at startup, and a failure report landing first re-persists +the in-memory set over your edit. The restart does not disturb running VMs — +startup reconciliation protects live VFs. + +**Draining the parent GPU.** Overflow-only is a preference, not a cordon: +under capacity pressure new placements still land on the card's healthy VFs +and refill it. To drain the card, quarantine all of its VFs by hand — add +records to the versioned `vf-health.json` (`{"version": 1, "records": +[{"vf_address": "...", "quarantined_at": "..."}]}`) and restart. Running +instances are untouched and +drain through their normal lifecycle: standby is blocked for vGPU instances, +so only a running VM pins a VF, and each stop or delete frees one for good. +Monitor by listing instances whose `gpu.device_path` sits under the parent +GPU; once none remain, run the cycle below. ```bash +# 1. Quiesce the services holding the GPU (required for the unbind lock). +systemctl stop nvidia-dcgm-exporter nvidia-dcgm + +# 2. Cycle SR-IOV on the parent GPU. /usr/lib/nvidia/sriov-manage -d /usr/lib/nvidia/sriov-manage -e + +# 3. Restart the quiesced services. +systemctl start nvidia-dcgm nvidia-dcgm-exporter ``` +After the cycle, remove the card's entries from `vf-health.json`, restart, +and boot a GPU instance to verify recovery. + Do not unbind/rebind the VF from the nvidia driver — it breaks the nvidia-vgpu-vfio core-device registration (`vfio_pci_core_device not found`) and the VF stops accepting assignments entirely until the SR-IOV cycle. -Services holding the GPU (DCGM, persistenced) must be stopped for the cycle -to obtain the unbind lock. ### vGPU assignment fails diff --git a/lib/devices/manager.go b/lib/devices/manager.go index 30763c04d..6b9f6340a 100644 --- a/lib/devices/manager.go +++ b/lib/devices/manager.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "log/slog" "os" "runtime" "strings" @@ -85,6 +86,9 @@ type manager struct { // NewManager creates a new device manager. // Use SetLivenessChecker after construction to enable accurate orphan detection. func NewManager(p *paths.Paths) Manager { + if err := initVFHealth(p.VFHealthState()); err != nil { + slog.Default().Error("failed to load VF health state; vGPU placement is disabled until the state file is repaired or removed", "error", err) + } return &manager{ paths: p, vfioBinder: NewVFIOBinder(), diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index a1378a3c7..b923439a3 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -8,12 +8,12 @@ import ( "fmt" "log/slog" "maps" + "math/rand/v2" "os" "path/filepath" "sort" "strconv" "strings" - "sync" "syscall" "time" @@ -37,18 +37,16 @@ type vendorVFIOSysfs struct { owners map[string]vendorVFIOOwner framebufferByType map[string]int openVFIOPathsFunc func() (map[string]struct{}, error) + pickVFIndex func(n int) int } -var ( - hostVendorVFIO = vendorVFIOSysfs{ - pciDevicesPath: pciDevicesPath, - procPath: procPath, - vfioDevicesPath: vfioDevicesPath, - owners: make(map[string]vendorVFIOOwner), - framebufferByType: make(map[string]int), - } - vendorVFIOMu sync.Mutex -) +var hostVendorVFIO = vendorVFIOSysfs{ + pciDevicesPath: pciDevicesPath, + procPath: procPath, + vfioDevicesPath: vfioDevicesPath, + owners: make(map[string]vendorVFIOOwner), + framebufferByType: make(map[string]int), +} func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { entries, err := os.ReadDir(s.pciDevicesPath) @@ -110,6 +108,10 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { // available_instances. This is a best-effort snapshot because creating on one // VF may revoke the type from siblings that share its GPU framebuffer. func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { + quarantined, err := vfHealth.checkedAddresses() + if err != nil { + return nil, err + } profilesByType := make(map[string]profileMetadata) creatableVFs := make(map[string]int) for _, vf := range vfs { @@ -121,9 +123,10 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) continue } + _, bad := quarantined[vf.PCIAddress] for _, profile := range creatable { profilesByType[profile.TypeName] = profile - if !vf.Allocated { + if !vf.Allocated && !bad { creatableVFs[profile.TypeName]++ } } @@ -315,10 +318,22 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map } func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType string) (string, error) { + quarantined, err := vfHealth.checkedAddresses() + if err != nil { + return "", err + } usageByGPU := make(map[string]int) unknownUsageByGPU := make(map[string]bool) + quarantinedByGPU := make(map[string]int) freeByGPU := make(map[string][]VirtualFunction) for _, vf := range vfs { + _, bad := quarantined[vf.PCIAddress] + if bad { + quarantinedByGPU[vf.ParentGPU]++ + if !vf.Allocated { + continue + } + } if vf.Allocated { // framebufferByType only covers currently creatable profiles, so // after a restart an allocated type can be missing when its @@ -352,6 +367,9 @@ func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType gpus = append(gpus, gpu) } sort.Slice(gpus, func(i, j int) bool { + if quarantinedByGPU[gpus[i]] != quarantinedByGPU[gpus[j]] { + return quarantinedByGPU[gpus[i]] < quarantinedByGPU[gpus[j]] + } if unknownUsageByGPU[gpus[i]] != unknownUsageByGPU[gpus[j]] { return !unknownUsageByGPU[gpus[i]] } @@ -363,7 +381,12 @@ func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType if len(gpus) == 0 { return "", nil } - return freeByGPU[gpus[0]][0].PCIAddress, nil + candidates := freeByGPU[gpus[0]] + pick := s.pickVFIndex + if pick == nil { + pick = rand.IntN + } + return candidates[pick(len(candidates))].PCIAddress, nil } func (s vendorVFIOSysfs) profileMetadata(vfs []VirtualFunction) ([]profileMetadata, error) { diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 098d016e3..a65fd0a88 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -621,3 +621,69 @@ func assertFileValue(t *testing.T, path, expected string) { require.NoError(t, err) assert.Equal(t, expected, string(value)) } + +func TestVendorVFIOSkipsQuarantinedVF(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.pickVFIndex = func(int) int { return 0 } + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", device.VFAddress) +} + +func TestVendorVFIONoVFWhenAllQuarantined(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + + _, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.ErrorContains(t, err, "no available VF") +} + +func TestVendorVFIOCardBiasAvoidsGPUWithQuarantinedVF(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.pickVFIndex = func(int) int { return 0 } + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress) +} + +func TestVendorVFIOSelectUsesTiebreakAmongFreeVFs(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.pickVFIndex = func(n int) int { return n - 1 } + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", device.VFAddress) +} + +func TestVendorVFIOListProfilesExcludesQuarantinedFromAvailability(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-1Q")) +} diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go new file mode 100644 index 000000000..b47ff11a5 --- /dev/null +++ b/lib/devices/vf_health.go @@ -0,0 +1,416 @@ +package devices + +import ( + "encoding/json" + "fmt" + "log/slog" + "os" + "path/filepath" + "regexp" + "sort" + "sync" + "time" +) + +const ( + vfHealthFileVersion = 1 + defaultVFQuarantineThreshold = 2 +) + +type vfInitFailure struct { + InstanceID string `json:"instance_id,omitempty"` + AssignedAt string `json:"assigned_at,omitempty"` + ReportedAt time.Time `json:"reported_at"` +} + +type vfHealthRecord struct { + VFAddress string `json:"vf_address"` + Failures []vfInitFailure `json:"failures,omitempty"` + QuarantinedAt *time.Time `json:"quarantined_at,omitempty"` +} + +type vfHealthFile struct { + Version int `json:"version"` + Records []vfHealthRecord `json:"records"` +} + +// VFInitFailureReport describes one guest-reported driver init failure. +type VFInitFailureReport struct { + VFAddress string + InstanceID string + AssignedAt string +} + +// VFInitSuccessReport identifies the assignment that successfully initialized. +type VFInitSuccessReport struct { + VFAddress string + InstanceID string + AssignedAt string +} + +// VFReportOutcome describes how a failure report changed a VF's health state. +type VFReportOutcome int + +const ( + // VFReportUnchanged means the VF was already quarantined or this + // assignment was already recorded. + VFReportUnchanged VFReportOutcome = iota + // VFReportRecorded means the failure was tallied below the quarantine threshold. + VFReportRecorded + // VFReportQuarantined means this report crossed the threshold and quarantined the VF. + VFReportQuarantined +) + +// VFReportResult is the outcome of recording a driver init failure. +type VFReportResult struct { + Outcome VFReportOutcome + Failures int + Threshold int +} + +// VFSuccessResult describes how a successful init changed a VF's health state. +type VFSuccessResult struct { + Cleared int + Rescinded bool +} + +type vfHealthStore struct { + mu sync.Mutex + path string + records map[string]vfHealthRecord + threshold int + loadErr error +} + +var vfHealthAddressPattern = regexp.MustCompile(`^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$`) + +var ( + vfHealth = &vfHealthStore{records: make(map[string]vfHealthRecord), threshold: defaultVFQuarantineThreshold} + vendorVFIOMu sync.Mutex +) + +func initVFHealth(path string) error { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.path = path + return vfHealth.loadLocked() +} + +// SetVFQuarantineThreshold configures the number of failed assignments +// required to quarantine a VF. +func SetVFQuarantineThreshold(n int) { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.threshold = n +} + +func (s *vfHealthStore) loadLocked() error { + s.records = make(map[string]vfHealthRecord) + s.loadErr = nil + + data, err := os.ReadFile(s.path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + s.loadErr = fmt.Errorf("read VF health state: %w", err) + return s.loadErr + } + var state vfHealthFile + if err := json.Unmarshal(data, &state); err != nil { + s.loadErr = fmt.Errorf("unmarshal VF health state: %w", err) + return s.loadErr + } + if state.Version != vfHealthFileVersion { + s.loadErr = fmt.Errorf("validate VF health state: unsupported version %d", state.Version) + return s.loadErr + } + if state.Records == nil { + s.loadErr = fmt.Errorf("validate VF health state: expected a records array") + return s.loadErr + } + loaded := make(map[string]vfHealthRecord, len(state.Records)) + for i, record := range state.Records { + if !vfHealthAddressPattern.MatchString(record.VFAddress) { + s.loadErr = fmt.Errorf("validate VF health state record %d: invalid VF address %q", i, record.VFAddress) + return s.loadErr + } + if record.QuarantinedAt != nil && record.QuarantinedAt.IsZero() { + s.loadErr = fmt.Errorf("validate VF health state record %d: missing quarantine timestamp", i) + return s.loadErr + } + if record.QuarantinedAt == nil && len(record.Failures) == 0 { + s.loadErr = fmt.Errorf("validate VF health state record %d: neither quarantined nor any recorded failures", i) + return s.loadErr + } + assignments := make(map[string]struct{}, len(record.Failures)) + for j, failure := range record.Failures { + if failure.ReportedAt.IsZero() { + s.loadErr = fmt.Errorf("validate VF health state record %d failure %d: missing report timestamp", i, j) + return s.loadErr + } + key := failure.InstanceID + "\x00" + failure.AssignedAt + if _, exists := assignments[key]; exists { + s.loadErr = fmt.Errorf("validate VF health state record %d: duplicate failure for assignment %q", i, failure.InstanceID) + return s.loadErr + } + assignments[key] = struct{}{} + } + if _, exists := loaded[record.VFAddress]; exists { + s.loadErr = fmt.Errorf("validate VF health state record %d: duplicate VF address %q", i, record.VFAddress) + return s.loadErr + } + loaded[record.VFAddress] = record + } + s.records = loaded + return nil +} + +func (s *vfHealthStore) ensureLoadedLocked() error { + if s.loadErr == nil { + return nil + } + return s.loadLocked() +} + +func (s *vfHealthStore) checkedAddresses() (map[string]struct{}, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.ensureLoadedLocked(); err != nil { + return nil, fmt.Errorf("VF health state unavailable: %w", err) + } + addresses := make(map[string]struct{}, len(s.records)) + for address, record := range s.records { + if record.QuarantinedAt != nil { + addresses[address] = struct{}{} + } + } + return addresses, nil +} + +// VGPUAvailability returns free allocatable and quarantined VF counts. +func VGPUAvailability(framework VGPUFramework, vfs []VirtualFunction) (allocatable, quarantined int, err error) { + if framework != VGPUFrameworkVendorVFIO { + return countFreeVFs(vfs, nil), 0, nil + } + addresses, err := vfHealth.checkedAddresses() + if err != nil { + return 0, 0, err + } + for _, vf := range vfs { + if _, ok := addresses[vf.PCIAddress]; ok { + quarantined++ + } + } + return countFreeVFs(vfs, addresses), quarantined, nil +} + +func countFreeVFs(vfs []VirtualFunction, quarantined map[string]struct{}) int { + available := 0 + for _, vf := range vfs { + if vf.Allocated { + continue + } + if _, ok := quarantined[vf.PCIAddress]; !ok { + available++ + } + } + return available +} + +// ReportVFInitFailure records a guest-reported driver init failure and +// quarantines the VF once failures from enough distinct assignments accumulate. +func ReportVFInitFailure(report VFInitFailureReport) (VFReportResult, error) { + // Lock order is vendorVFIOMu before vfHealth.mu. This serializes quarantine + // mutations with vendor-VFIO create, destroy, and reconciliation so placement + // cannot select a VF while it is being quarantined. + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + return vfHealth.reportFailure(report) +} + +// ReportVFInitSuccess clears failures through an exactly matched successful +// assignment. A quarantine is rescinded only when that assignment triggered it. +func ReportVFInitSuccess(report VFInitSuccessReport) (VFSuccessResult, error) { + // Lock order is vendorVFIOMu before vfHealth.mu. This serializes quarantine + // mutations with vendor-VFIO create, destroy, and reconciliation so placement + // cannot select a VF while it is being quarantined. + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + return vfHealth.reportSuccess(report) +} + +// VFHealthStoreUnavailable reports whether persisted state failed to load. +func VFHealthStoreUnavailable() bool { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + return vfHealth.loadErr != nil +} + +// TotalQuarantinedVFs returns the number of quarantined VFs in persisted state. +func TotalQuarantinedVFs() int { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + count := 0 + for _, record := range vfHealth.records { + if record.QuarantinedAt != nil { + count++ + } + } + return count +} + +func (s *vfHealthStore) sortedRecordsLocked() []vfHealthRecord { + records := make([]vfHealthRecord, 0, len(s.records)) + for _, record := range s.records { + records = append(records, record) + } + sort.Slice(records, func(i, j int) bool { return records[i].VFAddress < records[j].VFAddress }) + return records +} + +func (s *vfHealthStore) reportFailure(report VFInitFailureReport) (VFReportResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.ensureLoadedLocked(); err != nil { + return VFReportResult{}, err + } + if !vfHealthAddressPattern.MatchString(report.VFAddress) { + return VFReportResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress) + } + + previous, existed := s.records[report.VFAddress] + result := VFReportResult{Failures: len(previous.Failures), Threshold: s.threshold} + if previous.QuarantinedAt != nil { + return result, nil + } + for _, failure := range previous.Failures { + if sameVFAssignment(failure, report.InstanceID, report.AssignedAt) { + return result, nil + } + } + + record := vfHealthRecord{ + VFAddress: report.VFAddress, + Failures: append(append([]vfInitFailure(nil), previous.Failures...), vfInitFailure{ + InstanceID: report.InstanceID, + AssignedAt: report.AssignedAt, + ReportedAt: time.Now().UTC(), + }), + } + result.Failures = len(record.Failures) + result.Outcome = VFReportRecorded + if result.Failures >= s.threshold { + now := time.Now().UTC() + record.QuarantinedAt = &now + result.Outcome = VFReportQuarantined + } + s.records[report.VFAddress] = record + if err := s.persistLocked(); err != nil { + if existed { + s.records[report.VFAddress] = previous + } else { + delete(s.records, report.VFAddress) + } + return VFReportResult{}, err + } + return result, nil +} + +func sameVFAssignment(failure vfInitFailure, instanceID, assignedAt string) bool { + return failure.InstanceID == instanceID && failure.AssignedAt == assignedAt +} + +func (s *vfHealthStore) reportSuccess(report VFInitSuccessReport) (VFSuccessResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.ensureLoadedLocked(); err != nil { + return VFSuccessResult{}, err + } + if !vfHealthAddressPattern.MatchString(report.VFAddress) { + return VFSuccessResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress) + } + previous, ok := s.records[report.VFAddress] + if !ok || len(previous.Failures) == 0 { + return VFSuccessResult{}, nil + } + + match := -1 + for i, failure := range previous.Failures { + if sameVFAssignment(failure, report.InstanceID, report.AssignedAt) { + match = i + break + } + } + if match < 0 || (previous.QuarantinedAt != nil && match != len(previous.Failures)-1) { + return VFSuccessResult{}, nil + } + + remaining := append([]vfInitFailure(nil), previous.Failures[match+1:]...) + result := VFSuccessResult{ + Cleared: len(previous.Failures) - len(remaining), + Rescinded: previous.QuarantinedAt != nil, + } + if len(remaining) == 0 { + delete(s.records, report.VFAddress) + } else { + record := previous + record.Failures = remaining + s.records[report.VFAddress] = record + } + if err := s.persistLocked(); err != nil { + s.records[report.VFAddress] = previous + return VFSuccessResult{}, err + } + return result, nil +} + +func (s *vfHealthStore) persistLocked() error { + if s.path == "" { + return nil + } + data, err := json.MarshalIndent(vfHealthFile{ + Version: vfHealthFileVersion, + Records: s.sortedRecordsLocked(), + }, "", " ") + if err != nil { + return fmt.Errorf("marshal VF health state: %w", err) + } + if err := os.MkdirAll(filepath.Dir(s.path), 0755); err != nil { + return fmt.Errorf("create VF health state dir: %w", err) + } + tmp := s.path + ".tmp" + f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return fmt.Errorf("create VF health state: %w", err) + } + if _, err := f.Write(data); err != nil { + f.Close() + os.Remove(tmp) + return fmt.Errorf("write VF health state: %w", err) + } + if err := f.Sync(); err != nil { + f.Close() + os.Remove(tmp) + return fmt.Errorf("sync VF health state: %w", err) + } + if err := f.Close(); err != nil { + os.Remove(tmp) + return fmt.Errorf("close VF health state: %w", err) + } + if err := os.Rename(tmp, s.path); err != nil { + os.Remove(tmp) + return fmt.Errorf("rename VF health state: %w", err) + } + dirPath := filepath.Dir(s.path) + dir, err := os.Open(dirPath) + if err != nil { + slog.Default().Warn("failed to open VF health state directory for sync", "path", dirPath, "error", err) + return nil + } + if err := dir.Sync(); err != nil { + slog.Default().Warn("failed to sync VF health state directory", "path", dirPath, "error", err) + } + _ = dir.Close() + return nil +} diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go new file mode 100644 index 000000000..dd286c296 --- /dev/null +++ b/lib/devices/vf_health_test.go @@ -0,0 +1,408 @@ +package devices + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func resetVFHealthStore(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "vf-health.json") + require.NoError(t, initVFHealth(path)) + t.Cleanup(func() { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.path = "" + vfHealth.records = make(map[string]vfHealthRecord) + vfHealth.threshold = defaultVFQuarantineThreshold + vfHealth.loadErr = nil + }) + return path +} + +func quarantinedVFs() []vfHealthRecord { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + records := vfHealth.sortedRecordsLocked() + result := records[:0] + for _, record := range records { + if record.QuarantinedAt != nil { + result = append(result, record) + } + } + return result +} + +func quarantineVF(t *testing.T, address string) { + t.Helper() + SetVFQuarantineThreshold(1) + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: address, InstanceID: "quarantine-helper"}) + require.NoError(t, err) + require.Equal(t, VFReportQuarantined, result.Outcome) + SetVFQuarantineThreshold(defaultVFQuarantineThreshold) +} + +func TestVGPUAvailability(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + vfs := []VirtualFunction{ + {PCIAddress: "0000:82:00.4"}, + {PCIAddress: "0000:82:00.5", Allocated: true}, + {PCIAddress: "0000:82:00.6"}, + } + + available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, vfs) + require.NoError(t, err) + assert.Equal(t, 1, available) + assert.Equal(t, 1, quarantined) + + available, quarantined, err = VGPUAvailability(VGPUFrameworkMdev, vfs) + require.NoError(t, err) + assert.Equal(t, 2, available) + assert.Zero(t, quarantined) +} + +func TestVGPUAvailabilityExcludesOnlyQuarantinedVFs(t *testing.T) { + resetVFHealthStore(t) + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:82:00.4", InstanceID: "instance-1"}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) + + available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.NoError(t, err) + assert.Equal(t, 1, available, "a below-threshold failure tally must not remove the VF from placement") + assert.Zero(t, quarantined) +} + +func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { + path := resetVFHealthStore(t) + require.NoError(t, os.WriteFile(path, []byte("not json"), 0o644)) + require.Error(t, initVFHealth(path)) + + _, _, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.ErrorContains(t, err, "VF health state unavailable") + + available, quarantined, err := VGPUAvailability(VGPUFrameworkMdev, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.NoError(t, err) + assert.Equal(t, 1, available) + assert.Zero(t, quarantined) +} + +func TestReportVFInitFailureQuarantinesAtThreshold(t *testing.T) { + path := resetVFHealthStore(t) + + result, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + }) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, 1, result.Failures) + assert.Equal(t, defaultVFQuarantineThreshold, result.Threshold) + assert.Empty(t, quarantinedVFs(), "one failure must not quarantine at the default threshold") + + result, err = ReportVFInitFailure(VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-2", + AssignedAt: "2026-08-20T16:00:00Z", + }) + require.NoError(t, err) + assert.Equal(t, VFReportQuarantined, result.Outcome) + assert.Equal(t, 2, result.Failures) + + records := quarantinedVFs() + require.Len(t, records, 1) + assert.Equal(t, 1, TotalQuarantinedVFs()) + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) + require.NotNil(t, records[0].QuarantinedAt) + require.Len(t, records[0].Failures, 2) + assert.Equal(t, "instance-1", records[0].Failures[0].InstanceID) + + result, err = ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportUnchanged, result.Outcome) + + require.NoError(t, initVFHealth(path)) + reloaded := quarantinedVFs() + require.Len(t, reloaded, 1) + assert.Equal(t, "0000:e3:00.4", reloaded[0].VFAddress) + require.Len(t, reloaded[0].Failures, 2) +} + +func TestReportVFInitFailureDeduplicatesAssignments(t *testing.T) { + resetVFHealthStore(t) + + report := VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + } + result, err := ReportVFInitFailure(report) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + + result, err = ReportVFInitFailure(report) + require.NoError(t, err) + assert.Equal(t, VFReportUnchanged, result.Outcome) + assert.Equal(t, 1, result.Failures) + assert.Empty(t, quarantinedVFs(), "a rescanned assignment must not count toward the threshold twice") +} + +func TestReportVFInitFailureRespectsConfiguredThreshold(t *testing.T) { + resetVFHealthStore(t) + SetVFQuarantineThreshold(3) + + for i, instance := range []string{"instance-1", "instance-2"} { + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, i+1, result.Failures) + assert.Equal(t, 3, result.Threshold) + } + + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportQuarantined, result.Outcome) +} + +func TestReportVFInitSuccessClearsFailureTally(t *testing.T) { + path := resetVFHealthStore(t) + report := VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + } + + _, err := ReportVFInitFailure(report) + require.NoError(t, err) + + success := VFInitSuccessReport{ + VFAddress: report.VFAddress, + InstanceID: report.InstanceID, + AssignedAt: report.AssignedAt, + } + successResult, err := ReportVFInitSuccess(success) + require.NoError(t, err) + assert.Equal(t, 1, successResult.Cleared) + assert.False(t, successResult.Rescinded) + + successResult, err = ReportVFInitSuccess(success) + require.NoError(t, err) + assert.Zero(t, successResult.Cleared) + + require.NoError(t, initVFHealth(path)) + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: report.VFAddress, InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, 1, result.Failures) +} + +func TestReportVFInitSuccessRescindsQuarantineTriggeredByAssignment(t *testing.T) { + resetVFHealthStore(t) + vf := "0000:e3:00.4" + _, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: vf, + InstanceID: "instance-1", + AssignedAt: "2026-08-20T14:00:00Z", + }) + require.NoError(t, err) + trigger := VFInitFailureReport{ + VFAddress: vf, + InstanceID: "instance-2", + AssignedAt: "2026-08-20T15:00:00Z", + } + result, err := ReportVFInitFailure(trigger) + require.NoError(t, err) + require.Equal(t, VFReportQuarantined, result.Outcome) + + success, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: trigger.VFAddress, + InstanceID: trigger.InstanceID, + AssignedAt: trigger.AssignedAt, + }) + require.NoError(t, err) + assert.Equal(t, 2, success.Cleared) + assert.True(t, success.Rescinded) + assert.Empty(t, quarantinedVFs()) +} + +func TestReportVFInitSuccessWithoutMatchingFailureClearsNothing(t *testing.T) { + resetVFHealthStore(t) + _, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + }) + require.NoError(t, err) + + result, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-2", + AssignedAt: "2026-08-20T16:00:00Z", + }) + require.NoError(t, err) + assert.Zero(t, result.Cleared) + assert.False(t, result.Rescinded) +} + +func TestReportVFInitSuccessNeverClearsAnotherAssignmentsQuarantine(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:e3:00.4") + + result, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "another-instance", + }) + require.NoError(t, err) + assert.Zero(t, result.Cleared) + assert.False(t, result.Rescinded) + require.Len(t, quarantinedVFs(), 1) +} + +func TestReportVFInitFailureRejectsInvalidAddress(t *testing.T) { + resetVFHealthStore(t) + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "not-a-pci-address"}) + require.ErrorContains(t, err, "invalid VF address") + _, err = ReportVFInitSuccess(VFInitSuccessReport{VFAddress: "not-a-pci-address"}) + require.ErrorContains(t, err, "invalid VF address") + assert.Empty(t, quarantinedVFs()) +} + +func TestReportVFInitFailureRollsBackOnPersistFailure(t *testing.T) { + resetVFHealthStore(t) + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0644)) + vfHealth.path = filepath.Join(blocker, "vf-health.json") + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.Error(t, err) + + vfHealth.mu.Lock() + _, exists := vfHealth.records["0000:e3:00.4"] + vfHealth.mu.Unlock() + assert.False(t, exists, "a failure whose persist failed must be retried by the next report") +} + +func TestReportVFInitSuccessRollsBackOnPersistFailure(t *testing.T) { + resetVFHealthStore(t) + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.NoError(t, err) + + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0644)) + goodPath := vfHealth.path + vfHealth.path = filepath.Join(blocker, "vf-health.json") + + _, err = ReportVFInitSuccess(VFInitSuccessReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.Error(t, err) + vfHealth.path = goodPath + + vfHealth.mu.Lock() + record, exists := vfHealth.records["0000:e3:00.4"] + vfHealth.mu.Unlock() + require.True(t, exists, "a clear whose persist failed must be restored in memory") + assert.Len(t, record.Failures, 1) +} + +func TestCheckedAddressesFailsClosedOnUnloadedState(t *testing.T) { + path := resetVFHealthStore(t) + quarantineVF(t, "0000:e3:00.4") + + require.NoError(t, os.WriteFile(path, []byte("not json"), 0644)) + require.Error(t, initVFHealth(path)) + + _, err := vfHealth.checkedAddresses() + require.Error(t, err) + + restored := `{"version":1,"records":[{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-20T00:00:00Z"}]}` + require.NoError(t, os.WriteFile(path, []byte(restored), 0644)) + addresses, err := vfHealth.checkedAddresses() + require.NoError(t, err) + assert.Contains(t, addresses, "0000:e3:00.4") +} + +func TestCheckedAddressesFailsClosedOnInvalidRecord(t *testing.T) { + tests := []struct { + name string + state string + wantErr string + }{ + { + name: "unsupported version", + state: `{"version":2,"records":[]}`, + wantErr: "unsupported version 2", + }, + { + name: "missing records", + state: `{"version":1}`, + wantErr: "expected a records array", + }, + { + name: "invalid address", + state: `{"version":1,"records":[{"vf_address":"not-a-pci-address","quarantined_at":"2026-08-20T00:00:00Z"}]}`, + wantErr: "invalid VF address", + }, + { + name: "neither quarantined nor failed", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4"}]}`, + wantErr: "neither quarantined nor any recorded failures", + }, + { + name: "failure missing report timestamp", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","failures":[{"instance_id":"instance-1"}]}]}`, + wantErr: "missing report timestamp", + }, + { + name: "duplicate assignment", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","failures":[{"instance_id":"instance-1","assigned_at":"a","reported_at":"2026-08-20T00:00:00Z"},{"instance_id":"instance-1","assigned_at":"a","reported_at":"2026-08-21T00:00:00Z"}]}]}`, + wantErr: "duplicate failure for assignment", + }, + { + name: "duplicate address", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-20T00:00:00Z"},{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-21T00:00:00Z"}]}`, + wantErr: "duplicate VF address", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path := resetVFHealthStore(t) + require.NoError(t, os.WriteFile(path, []byte(tt.state), 0644)) + require.ErrorContains(t, initVFHealth(path), tt.wantErr) + assert.True(t, VFHealthStoreUnavailable()) + assert.Empty(t, quarantinedVFs()) + + _, err := vfHealth.checkedAddresses() + require.Error(t, err) + }) + } +} + +func TestReportVFInitFailureRefusesToClobberUnloadedState(t *testing.T) { + path := resetVFHealthStore(t) + quarantineVF(t, "0000:e3:00.4") + + require.NoError(t, os.WriteFile(path, []byte("not json"), 0644)) + require.Error(t, initVFHealth(path)) + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.5"}) + require.Error(t, err) + _, err = ReportVFInitSuccess(VFInitSuccessReport{VFAddress: "0000:e3:00.5"}) + require.Error(t, err) + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, "not json", string(data), "a failed load must not be overwritten by later reports") + + restored := `{"version":1,"records":[{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-20T00:00:00Z"}]}` + require.NoError(t, os.WriteFile(path, []byte(restored), 0644)) + quarantineVF(t, "0000:e3:00.5") + records := quarantinedVFs() + require.Len(t, records, 2, "reload must recover the previously persisted quarantine") + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) +} diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 27d459268..0a12662cf 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1018,6 +1018,9 @@ type GPUProfile struct { // GPUResourceStatus GPU resource status. Null if no GPUs available. type GPUResourceStatus struct { + // AllocatableSlots Free slots eligible for placement, matching admission control (excludes quarantined VFs; 0 while VF health state is unavailable) + AllocatableSlots int `json:"allocatable_slots"` + // Devices Physical GPUs (only in passthrough mode) Devices *[]PassthroughDevice `json:"devices,omitempty"` @@ -1027,10 +1030,13 @@ type GPUResourceStatus struct { // Profiles Available vGPU profiles (only in vGPU mode) Profiles *[]GPUProfile `json:"profiles,omitempty"` + // QuarantinedSlots VFs quarantined after guest driver init failures (vGPU mode only). May overlap used_slots until the affected instance releases its VF. + QuarantinedSlots int `json:"quarantined_slots"` + // TotalSlots Total slots (VFs for vGPU, physical GPUs for passthrough) TotalSlots int `json:"total_slots"` - // UsedSlots Slots currently in use + // UsedSlots Slots currently in use. Includes quarantined VFs that are still assigned, so this can overlap quarantined_slots. UsedSlots int `json:"used_slots"` } @@ -19032,250 +19038,252 @@ var swaggerSpec = []string{ "b/uvJmpekx9IbJSZ6KGUx7UaT/b9dhfRy7N3Z01jKqouIX90S3NaAe+zHO0yyZmp++TZVLCtx2gR1wwe", "j+msj34kUvXIZMKFOrCBLOCQ8mq+4gUSJM41PfjQZJwhSccJnNGiVy3w61/0hAAIZJxPJkRUUQeeh2Rp", "7+1RGjDu/aSfI/NCAXtw+mNVntZye1ut/axCDaC2T3BE2XSz9XYHDIK1aazDFXx59u6NLRvUhLisl7Io", - "LWTAlvvo16Kwll5qWaIq9QOWwjpyX0Mi2dlsIWmEE9OiqYpBmW/gg9PQWkI/Kz+0ptCAnB6ux+ZOHtqY", - "T7Mczv35m97J6/dbaUzm3cqYIJBvxhOix73psae5QzMos94qXGneZGkxhCHbnlhvrQqW0XqRPAYRWB3F", - "FU5GMuGhmKG3+iGCh2jj/U8mHVmPoIuyylbq332oIZ++nwRPDCDYN3R7Dh3WTbaVAx7UXesQaJ3q9Cqd", - "ho6KSaValrGWiyHyy+pG88v1BfhMI839Hrlsr5pR3QqyyCSFGbxGFwaBzKfGam5NM5JkWGBFkkUtHrMK", - "qE6WPdrkmkQ3SDd7oV//ZIpo5IKM1EwQOeNJNQ5it7tciFVCzPGc2NpTZk6e4V9xlGJxCTexE+RRzswK", - "VEPWd9fhy8yUym4wqZ/fvj0z2r0iYo6TetKDXPLwH5MEL9CYqCtCmJsKlgj7Ma/1pFHZUP9HqFFGBOXV", - "NezsBvo9N3HQaCpwRJD5ypVxtlsiIb6o7VLaXgKIhVFEpGzY3+1V+2s/neRJuz0ODWt7bdXz6CYb/Pbo", - "zNWxKeoEu2XeWV7lMyJ65si5gsGrt3ZHrq6o5LpiBoh2SWAYEyixZDNN/ERQF98EJaT055XkS485SD9d", - "0PYDp8AsVdec8w+tpMv6cQ858lPM4lC9ZZNgYFLzp4DXBVHOIgfRj8Ym+MmgBRg9wM+TEATHlBEpa3nN", - "US6STrfTm9hZHWxtJTzCCUAW7u1uP9taHUa6Mn7YhkuNYrpKv3RBVSbsxmXRGrw3mHSVJLZwlrWwwJl1", - "XHM/AHtajleEGsv6bvMkPBfAPhgspWhf40i5qnBgkqu4XLF/bAFYuDIfaDA1hV60qP3cP5+DoA88w2pW", - "Jf+tJdqHuBiIadQ0YswdtXU0RP4xKFFxESr9xYWyGX9j4oIei/vQhRQ6CN2Kc27wzJ/lk/393f11fAiY", - "Te2Y23MXmKp5u5rTTLzltufXNgDhWVWZwx3p1dVL9bqsoSnNEZdIavWC8oywG63n/t7uzs3Ws+1ETlxY", - "WI0vhSBhjk6PjUwUcaYwZUSglCgcY4WrTAZsWZrLQG0ZTFJIC5p8v5q1NMRP+Bgvty2M9aW87w018t44", - "SOgUMzrRDNm+6fcsZ3hn/8mBqeQZk8ne/pN+v39T5IsXJdRFq63YMkF6HghGX84+bx/uAOCizVz+6Jwd", - "vv1ZM7JcCnNpbckxZQfev4t/lg/gD/PPMWVhYIw2xV/pZKnoazUeLbfIwyQ+QGV9byf3tIkPajBGQ3Qy", - "oPEEYeYqUZp3hydX0Ditlg24QW7wilxZLa68ZsmicW1uXce1rHCuvPqtfrZZi1qu9ONq/7ozd8E7tk8D", - "QF2UuV32rN+qULFcWctxqdRXRlhRvTFJzF8RZwDsGyrlWLki3bMWlcDgGrElv4ou/R+L3r0fj/yBeL+7", - "SmLeT7am44cbhsSsFEj/tiyHrudCThxdc5jDtsfiVmhbP9fi0AVjwR/4LrxN2Fi199fT//r9/8izp3/f", - "/v3V+/f/PX/5X8e/0v9+n5y9/iyMk9UQhA+KI/jFoANNTXgfP7AtKZ1iFQVsdFr9a1hh+8RYHFQ0g4qf", - "aEwOhqyHXlFFhKkfV0t+HHbQBgFNCb7S4i4UxzF5Z5v64zPj0dQf/+HE4E/1NmKblC7shhRYIzIfxzzF", - "lG0O2ZDZtpCbiAS9QP8VowhnpggcZUjrvws0FlCxz7qYys676A+cZZ82h8xW6Tdo2hmGomeTIuuLOQex", - "HZUJg7WvkwJ2wmQkDllxWxcYfMbP2C8B+ylJ6jlDDYuyWn+zmtOzQQitEPJZ9EZCURpQQQrK1mRUJNqg", - "Z4PNZX1ujY5R0NAK8rOOeJPweJiHzMVNSZLHJKYR8BWXJzizmaRFiqahNGvEywS/XsDevDHJazHCuZpp", - "XhTZxPqI80tKurClXXCHQeQHfGn8+TOe9caL3oxnBcgCFibaBRuPeFXJ/j89O9HeeyLoxPYUzJXXJBIQ", - "OuHI2JmZlMPCurA0sbemngfTos+c2NdNYRhpaoGYUG6VC+bKVRAoWgloHgX1kZBM/j2KEgpWJznjeRKj", - "GQDuKd1MCDOvMyhSZvA4ismk/u9qSMPO/hPQYN2/d3daZ6yapVtFZXkS0GlTx/pacGzDJmEARjwYOUP4", - "miAkfQNaPy7YKRSH/54j11B54gpGYrxTJodO2uoHifSy5zaDaU72GFjIkBG2p6nNfbR0CitZUy1aMNEJ", - "8FnSAqzkhcnWfPvqHCkiUpc/vxHp3YFTYpApelTK3BbiOjw6fbHZ7wSBliquKtiqlVlV1UEHsBZstEJT", - "EEZpo8Ep6aKTY8iWtddKqYtBesNPXKDE3IrlZXQAYB1Vcw82JfxOjq0AmizKkAcjtgw7m67FrH69HaA3", - "hQqIi6EUeY8lbbkmy8sEmrUBcCb3Yqn1Wpos+Mes+mfvY8i0gPqGhhcDWGTj/dXe5uigp/RFVbOQ3fhC", - "8qNQGu1f3t5/aVjlLy+j795MRrc+4FE2wzJE3TPfqwkvLe2770ausnvRHDNU6XckafBs/c1VefGuIUX0", - "PVf5PIQjut/b3n67vXdz891NEXGrUFgeTF4BitsezfYuUGEDGK9UjRqDy5F+bEPJnV3k/SmaYcm+U/Cw", - "Zh3Z3n3axigBvbYNy/YDsvnEDKngUg5XqwgnNghjlzRJjAAj6ZThBD1HG+cnL385efVqE/XQ69en9a1Y", - "9UVwf24Bjgu3AKyjyTALQCtVIAFQkTv49u0rOFwJgfQLI4df3h4yd61psQWErhvcy7N34PjHcuQCN5tz", - "FXGZ70uuqVRyGVWtVfzz50D2mk/bFfl3kzRtlLX+V+P+/lwBpg3C5G3eAWCvC15fWs4HwLJ9yCTBrw9H", - "dyXy7efC11o7wx2h1zZeaSHk1xoiwn7T7XZ7HNo7GU4FUCbEtnwJx2Vw3xr4tduhgezVQ6kvHhKjk7Oy", - "0lPpjHDN1+b0fKe//eQZFCvdHrRh7CmOVvR9enjUvvPBjrllDvD4IIoPQGG/rc/KErZRQXByhRdQ7M8s", - "7bBjLkxPu/WOrVUkW8XXLOPr3g5Oty7GNQDmgjjrApfkKF1Za6RFemIdNC3NLSRkSpOEShJxFsuqjDzD", - "EsnMIKGamhuFBD9kMMAuKkofg5SCcBSJvDQ9Wunayvt5Zuke6n5mnGkdAID/fyELiVIKTtCiewh9lKjI", - "gomHbEO4jKkiNQpKfsb6B8g/6NrI9lgPjSqoLaI/GDI5y5VmYpt9dMSZzFMirFUWjSl4jDaRzI1KC+OF", - "1VhohilpTMSQ6dcCWKt/FOrJwZPBYDDodgpNblf/exCipjt1fvYtlrDJ+QW0P2ZRhQFGUOQM5Swmoqj/", - "TQw51EPkbug4/UwQYfd5O/HKfl7KVeGDuQ5zuB2Y8OciuMJQG/RziA69hXK+f3sRvVXOkZNfbbaR/Wp0", - "kwgGgiKeJ7HW+Mb6tjMGORJbM6QkynBn8y6V6J2pvVmdug09Vhz9nhOxQO9PTythD4JMNA9oN3HgEg37", - "wLMbbcPOGhvJ2tHcxL3s4d3eB8ZtXVLxJMQvjmjrexhdErSh0Iphq6I4r7Kvaa0ymEVCmdknTTQrJhhK", - "kRmVgZF+q3IhJxbf0YrS2MlDBm+6gH8onfpyIbfGudzKIrpl82+2AJvjGWBz7AWTp2MyH+V5SDXSjxwY", - "y7t3J8doA34BbFlIoawSMMZPtp8Nnj3vPRtvP+ntxYPtHt7efdLb2ceDyW70dHd7Z3dFIkyLbLrbJ8gF", - "NeZAHHMRtT5y0fOhoOam3IWabGLjsa8oi/lV5foLBsj6vdvg23XdL4fWtx5CMCEnwVIZ80UDJzuFS55E", - "um0TkG4zNotSS2FD55O3g+3Ptf7A4BruiLciZ8atajAGChdC6g3Y36zqOG/H8mFALvFl3Wr5nbdftMHB", - "/vOD/c9dNJe8sW6MdXK6x81tighzGMy17BCXoejZkZyBsmNlImPVt8kknW6nyHeBv0EYqMVSF49bJXE1", - "HdhumI2sulYakqdPKvoKRKoYDL74QEsqTh+Big5Fir4WgY4SnsfIs8UZSDLww514uotuBtxi1kRnIEZN", - "MobWcQDTGipHUKYZMfgfdSM20/oAvYR34RFOjVpnB2Hql/iuNxwvTLyMPl+ua6NkrR7yudWv4ButbCH9", - "L5i2XgZrsl3dhJHODtCvHL4ptD3G67Zf8zqoWcuv1+3EGxau2yFnQGdW1DxAPxXiZSGgWoF0QxL758gy", - "rBKwZrMCG2B3vKOppdw5LwW+2zEr2ul23EJBqvxy0vy7kuqXzp9PiqFAMoITOMtljnCuaGJhumEmVCoa", - "SZs8oje3SeyxpZVIPDLKU1NMqkk8tQpW8ZGTqt6fog1AYvwLsoZt/a/NIn61ctftPN97/uTpzvMnrfCW", - "ygGuF42PIC16eXBr5eQoy0fWNtI09aOzd8b2ERmrQhH78v7Uh7fIBNesR8/cNeh3/rz/3IeZink+TjzH", - "osWkM6i2sGFBJLWCFzXEQf5OkzmdTNjvH6PLnb8Lmm5fP5E74+0G+FzTUdjsduIHFyzZqMm4Z8okhZGA", - "gKCEbATLekMkzACdE4WAfnoIR6DeFNnMluQcpJZd8SBh7e3u7j57ur/Tiq7s6LyDMwIjXOBStiPwjhi8", - "iTbenJ+jLY/gTJsOUwIQzplVfcPnDNkax4OqQNrfHuyGqKTh4i6pxrY9TxuX/L1VH+2k7KJDUnahWi6d", - "8uBq7+4Onu7tP9tvd4yteXgkrldzGJeyZJbHAvH7O78B0uTbwzMECcETHFVtOy5C7EajUjcaFRSRMODv", - "NxjYs6dP9vd2d7bbob6Fgk4snmHlwFZ5V+DQBYgisBuBpVhmvd2m2yIkThkCe0OiBNP0MHIpFrXbx4C8", - "j4R5rdyENheD1cCXLq4W37YybhUmK5OgY0QDLlDOitIi/fUu2S/iWW3m2uZ6WM/VQ2k5TK+ehScyJdRu", - "sZSZIHPKc/kFGuLK5MxOEs7Fjb5tUljeEJknythsqETvT78DnqJpDUlFsqoOZalxBYjTLSd3o/NcIZEw", - "kTctVqvdaLP1qybcbTi13VWAGhVu0AidFmvOlbP1wZ9HOIlyKKaDi/3UswIMMIAEyLJkYWL7k4RzhqIZ", - "ZuAkER7iEZrxJO4HI2H1k9EkGFXBr1DCDejzJSGZrTNjBqE/0yIMnRO04VdYM6RUq3u6nxomYyuJVKlx", - "Pw0XcMQylKxWpMLr9cSKe3jE5pOKJTThUwlKoYKshX4dBj/DwiQjYGbqJs1To0sGAq4DQ6wx89CNam5S", - "PrEKrhU5INHcrCSOBJcSkYROoUbP+9Na/vKKnLcii3l9QGd1sC1I1zg0A1eZQcNqXV4tdD8G8nk+54YE", - "GoacwRWhks44mWKWQ+UZj5CtIb7fOhxyxqUaFbhUNxysVCMoJ5ELUqLlFVn3hT3IvRO8Fx1ru81y2bjj", - "W329RFXhppoG2MxTgysaXq1uQYMhMl5G5loJBlaii9WhpG4CVlfWH6ASWqUebBnagJwXjy15EHSbbYJk", - "wiqr7mdJW7XVQV/tDc7bwrqtRnE7w2p2wiY8gP1xA8+ps0TbaNWMiJRCQRUUE0ZJ7HTJwoVqTV2QMJ5I", - "guKc2JUz8qnAdsGxOd7gs2LORkbZtMbr6x22MQ+bMayuNgH92hfbhDvJcELtW5HDWpl4RYlwmVrbKgiU", - "ylHYnbXcsCDTPMECWUDGNkOWizSh7LJN63KRjnlCI6Q/qPvFJzxJ+NVIP5I/wFw2W81OfzBqKk10bgZn", - "8wLNhtT6Lafwg57lZi0rGSwxW+b7LXCMtokeC0aK/0QTYtH93jF67RF6FY59b2fQlC3f0GglT34ZGfKm", - "nNuSbPDE5zKQW7hSynFFlUhsMfKN2JPl0tR3aXErORBW5wK8nUenmjjyedAkR4Zf14BJ0JhA3o+b2jLX", - "aMEW20wlWFoilzP0dz6uGkTbhv0GCpZtsBIiQ5BJML4fdnSlQdq8sbQm3u7eBIMC2KqeKHx0Q2iHdaXd", - "yviqJn7yZqnK2YzYJaNujqbiWYsKHi7+o4AvsL22xzGo16MLxCsDSo1UC6jsCuV0Fl6RRYnGXAhAoNYS", - "DmduNgC7omUevdYO9wq9nZEFEiTFlA0ZZYWRFMDUCGJkToSXJcuFVrKmJO6jv3kqHmB2p5laWDB4MJ5/", - "JxG/YsUYh8wfpG48l7qdQ2YsiyLPVKVcpG4WtD5NKJC1DE4wJaB+IlUzNBFEzvy5h2pmahnviou4sRjR", - "ArlXoMYN+FiR4peE+aysaCaoGpqGRuar5Sg+U/AWnlr9E1Vq0KJ6jdnV/eWSiLCQWEypeKVV6Ip3VDzl", - "xIDAACIK1Ba0fxkWX6CgtMA8KZv/q2uy/OmsaLz6W+01D9fEwQwfGrNt0AQbmTSeWrBP1ZO2NlQF0uBW", - "odks+xLQhguhdgVaqpKAVyil1T3ZLhOvnizgRrMlSVTtfe/Z/tMnLSvVfJazzqB3fWnX3Dxd4ZJr2KnT", - "Nn6fZ/vPnj/f3dt/vnMjD4vLK2nYn6bcEn9/0Aa5VvqwJv/6xz/fn9a8PvsQgz240aBMZkl4SA3ZJdUB", - "vT/91z/+6UZ16wGFGM0yQniD374xSifxd9IFClRdeO2cZCv0+8OKkQAXbAZtkMmEgBl0ZNatVw6mBgPS", - "TgrGGY6oWgQYOb4y0e7FKzWk6zbuoOpgQyKvaduiomrOJfNxmXS64TpH/2l8wzVaeNa64JXMx01+6Nf1", - "Xo0XuvRa+DEOLUIMZFFrfdnAXcznCstKQLf+O4K8C5dhtpxtY95YjbpbT4WAKBZb180LBQyhtdfkSfuR", - "v/217fT8lhWzTn3FP6w4h81H8EZW38CNHDD6RutTa2v8wV6At/tqNPZL0a2s9VepW1feujfvt0X28HKd", - "hOIGu3l/XsLkTT6sYwIDPdox2CUv2+5WSKKBmrxcmIABjSekVwTq2UQZJHPjEdRn3sLMBzI4o0s+mVSx", - "bvebsdEB9geSvVwvWCmtmXQRuXY2izqwtsH4GXb25bCjVYBhZzsddmpuq2D6ZIqvR7aDKrbLYBVYeZn+", - "XhukdDMYJzy6NFXWoHh3Hw1QSjCTKGdw+Gtete3Bau9Qt5N5e1NAgxMT4rTEtmBMYzLDcwoVKaxPZVoJ", - "xCTXVEkIGIV2DlDMDdpTpcSsnaF+zSQ3HpSThksHs4VtWDeo3+PMRbSW74KBbwKFbdlHInjXghVojv36", - "9WnXBDBA6KEZWCW+0U3UjEAzyKKLWnmF8vdw/PA4ISMYdx2uP11eRz8nHTyrgkiipMXvLsmhRgQo4jlT", - "dRz/tJ0iV00rW76ScgbBfjb8A3DZbO+GQFBMIjiRcvksVgn9FsRdyxuwKx1KHNgNkTAcCvAlhX3Fb6xD", - "uD4AY2zwqkObdvy4buMlHEnFbTmx4lSPyHVESFwH/Ay/0jZW3n4ZjJV/hS1GUFG42b4N8c7Ls+vfXYIX", - "jLVptf2YfsZZD9BJ3JZaJBEDDWixaqqEVoEe9yAtRiF41dALbTKuyfXqtf6VXCvAR4/zxIDehUnXsip7", - "Ga1b8VtnNjYdaC7I2vJ8d1C2zsSb36pwnQ1Vf4jadfatO6lXt7Q750S5d88tGTXuULXQS8Wl5QL+3SvV", - "GBtDSl1kL3i0nW7WSHBvFraKWFDeljmaDKdklAkyodcriMe8YBTjKqxJeZCKDAaDL7qR4mu09xRFMyxk", - "beyMTmcqWVQDcPYCWEqfVdRREEWYMxS22flyN92Hy9Fudjv91kPC8bkHDbRU0sSKpKNVuNlHpbfNWucz", - "vAArTqOT8Onu3mCwuzO4FXC2G9YNluuo/MSWQKy205RS531nHf2VKFW/hSLJermu7pWgkKtdLJNUguD0", - "ABJvMhwRlJAJgOQVCa3rPYv1rlcP3gpUNou2oH+3UXbfnA++WjKn6MpijrtpdJxzsYpB5D9f4xBtYDPR", - "EqReIOdutzd48nZ792D/ycH29l2AXReL1JTt8fTj9tXTZAdP9pJni6e/b8+eTnfS3aAedklNZaA2tPqL", - "frcxyqa8JKtYRhWWhjbsHDIi6gWT64XGJUkoIz1ZZEitT1NcwQuM/33t+b+Znd/MYKXscF6dpC9CYFUu", - "ToWyHgZ/y05mpe+iPpuT49WzuFUGUn0gYXqrDwXIq91goELFduczkRly1vIaeue92PoiWpkVt+4qCnnY", - "4aQHd7lhxUPkXQNm8Ga96gJfvuQCttMpF1TN0tW3RfFaASMOcdMfpYqreE99dDJlUC3d/7kIk/OVKP1x", - "p9tJPu5Vz4z9vT3yl0UgLgjQbrUvFbQII4Ni/KtXAV4pFQ9hItm1rq7H/MN2b/s5xCEkH/d+GPSeVyMO", - "uma1/OXbdm9Xfh20WUO/BKArHbX9/EYR1249V1HQLzRUwK68ly02saXxsja1uzpcwm1lg8vHS3tcQ/Jp", - "FEA/V9Kzl9vIF5pikuBFCJveM9TKmvboExkakyllso3ddndQGG7302Gnjw4tQDjosooX/fjNQw16j05o", - "mpKYahnTqP7NGQw7LW1xdV3iZrVJ3FcBaa0fFteer4dIWJdwte6a7H9GPu5nab/tNN5V6B1gV3MqKmCI", - "wYtdRCcIs1qBUsrmOKGxTaSHxEiIVztwQG0lyVoeIEs50NlJumjKFSpT6Fva23LWbBcsxk+uwd66AjPD", - "EMTOFwFEKQDE6Cr2dXKMMsHjPCrzRxMYdIn4IfIaRNsKIX99SO5d2jcgMXvCBVpv32gyaLSzTzbtd802", - "qQm2eau3B+u3+k6MIt1OnsXreZh5qR0HuxFy+5oUxICJprrsNUnQm8yHFhz9jb+CyzqvsSVHWiTKM+dg", - "0TS1TEkBdwu4GEJxvcckIfqaWm4E8SQusySoLLnoepa6/eTZrMnFCR6p5YH8QkimdRXAP4L+UswWwYG5", - "sqPFXbIxcGjf0ji8eqZckV2t6uCerpXEGrfKN+E2lVAwXL5m8zZ4KZee+bvA+PZFs2UEFMfwK0Lam+YS", - "APZLF/bWaD++C7PcQwppr63roQbb6kCFC3R0138ZC6zFuirx7oXc8yGyeGs14yYo2noWqG91Puz9j7Ey", - "o1H/YOuHv/zfvQ//GbQ21/RmSUQvJhMINLoki54pPqR19H4ViBUqH2hhempJheAUbEgAcm4Poz/e/UHB", - "NBa/4nRpChCh5VUO2l47ob/8R3N8k7eM74BPriXZzy4MchcFVBV319FGSsTUxZK7RLLN/pBBbtolWUjk", - "1SOzIo0j1O9k8YkXgY4ujBjYJ2x+gcYUCjzKIdNaLY4ikmltwpa4oaZKOQfuIwhO/HZsXTSX+G0dkiae", - "gKD3p0sovq/fvf3x9btfj0evz178engy+uXFf0OIx1XP9BD3NO3t7T+xtcn9ldwO1se4eZmHPjq1YfrW", - "1T/JQaEFnC6J0lzlEBRCrqMkl3TuHIQquX1Bh+Vk3dsXSPhMBGClklBUgkWqTuiEgF8frhMbVEOlI0Yq", - "oai7NW5QhpZvbEM4ww5wUq8mf6icht6K8GqXG1td9CezdizUYKMGDjtkvEL1/YD2QiXgVbjYD+9ltAGZ", - "I67yrEuc3bwZVuth0WAw8vALFxgaPP8SRUDfraz6OedJT6s3DZUSgtZksxbByHloymQkdJqcDtNxQIa3", - "pt0pneKAnyHkT/gixTrdgNZmTC3tf2PVsnAew3G9jIQ5lmapamUPakYCqXrNaQ6plmobgHcBWdjkrlIv", - "tq6aqJoytWWL6obwMmIOYOarspXLU+bQEXvw0fok3JV6lTczbyTNe3Pq1IeagrNigc700lzNiCDeRsAH", - "JTz/DZfM5uW0QGExRQkzIsqYVZfUo6VScDdLtFFYftwSFNnGy+bw1eUXTvF10QO4UrBc8j/CPMryT9sv", - "fwSo/Deu5CWduCZgGDXlLgwMX6WiVWviqGp5M3yqWp63eT948CyvWsH9ms5WjTjLPiqkGaLHv2GqfuIC", - "1MFmzJM7x5eHyz8mAjDg6ujxraDXaUriEc/V6vNvK+rbK78oi1qW1XWqLwYijirpvE28wKFylGNYXmm9", - "HCTKBVWLc71eNpgb0iBdLVtYSOgIfi47hvqhnz6B0XgSSBh5SRgRNILqrPo8ppiBxoTen3pF+ky9xiW8", - "VhCBXh+dWHODg/wF9ZEqID0Xd3l4dtLpduZEGJW7M+jv9gdwmDPCcEY7B53d/nZ/0AGtagZT3IKK+jZ/", - "2uYbF4rrSWwloR/dS/pLgVOi4IvfAkgAEHdoXwcVBE89JTLDVFgtMksAocAQDNVfQ7kBd6EemFu5a5a9", - "tc0U0owh+4Vkr+3mfgBBGc4OTHNnMLDA5spev5C7YxIGtv5uo0fLfltJdXaJAuj7S2qeky2Lpf/U7ewN", - "tm80plVDgbMb6vgdwzaJl4B2vn/DhbhVpyfMpOXZJGsbDuWfOCAk/6z99kHvmczTFIuFWzB/tTIumwRj", - "IhF27xo9TkkUaVYBNYL66DUj5jnCCmETuSxyBqWV3YeaQqunwLTtNrkAKfqRx4svtoSVPpyN4lOVnenj", - "8mmJnr8c7RRkvLyR9pFD2DZUew8E9CMu6oI/2EnZGzy/+06POJskNFKoVxCwjUemEkJ+EsALd9hDXKDf", - "c64wKsL5H9GRtjLruCC3bnkVbf1B40/meCckZAY/IyLFzCRHmHfWHPql42xcEuVxXnmrOcI/Oe7Ym8qB", - "8JiLCgS56hH1r626MLh8He0FEBhsn2Z68QMS/t49nHA72aI07EMeOSjIiXJJHtNxsi62cSmEBGW5l0R9", - "LTQ/uM8ryxYR+BOeosdCwC9JIeGVu7V0KWxlImdGAQ5KgG/KhEX73XdV4e9t+cSLkgG/hm4aylko41fF", - "8aKP3JoapV8tAGJJEJhnvHytnOnhfS0nbOc+ThjMuPAUfbumvl1Tq065oRY3BTiY3ilvYYO4kQXiz2d/", - "uLH14Zvtob3toZXlgZEra134Ox/3kY1IjXhMkJzxPInRmCCDd+RiTxQW/elHhEU0o3MCoHZQpC1PFM2w", - "gMiSFMVYYeNDbzRMrDRLFM1t6eZ6Lg6xXOA6joUkI8DhGzXhT5YRiJQxEiP9iYXuK+EEl8qJm7MfNLAX", - "DZZXI7qacUkKPD+mvNsc0pul0Y6h2f6QvbVAr3oBIZja8RpJEoCrXWH/4QzhIbMffO9YiAsEkzgtORcW", - "gBlIDTKl2Zbl1DY90pGMeAhr5y1hmKmezEhEJzSy07okCxvPGWywVd0lPWA3zvenRcIG2tkM47UBPGMY", - "nPe4eIYsJVX9NwyCoKMkj0snl4MQwmKMkyRYmGOa8DFORmZ9LknAJ/gS3rCLUjpcSm8S4zExJeSzhZpx", - "Zv7OxzlTufl7LPiVJGLY2ewPGSRi2LUmcbcUENEVFHJLM67PmeCp6XPLDHHrj0uy+NQfssM4pcxRBHyC", - "E8kRuYbvoL4VYGYY7tVAD+Y0hf3gR7lUPPWRTx3dmWHyXGW5shklkqhuCPVzyBRHfzhsx09bf5Q9fgJn", - "McGxphPvFTMlkK2bRi1HWM9+BK8G3O0EFmDY0RepCfOYCsyUge0swCnR1N/SjaI6AlRMra9whBnKeGYq", - "SwBRzbAmuUobgNWAkwQpOEruWy24w042zMdC76XjRtw9A5RWO0aUodMfvcM02HsWPk+SRIKEIkr+6/z1", - "rwhuZb0H5rUyXMukdDAtMKA4B9ep42kvcDRDxlEFxQSHHRoPO4U7N96EsebShsv0euBT/EEP7QfTTZfG", - "P/T7uinjrjxAv/1hWjnQZylLDQ7osPOpi7wHU6pm+bh49iG8oE3wZecVRoA2zDW3CZwEU0Ca8W58c0Vi", - "FiNub4FkgTAqOZAfuDKmDIvFqkTCwNLbFeQTE8noLcYfQ4hcHHYOhi52cdjpDjuEzeE3G+A47HwKr4D1", - "WjZXroP7rHBuFkT0ZDDYXI+Ebdc34LNs4Rj4wjpgo1ZUlN3UO2hhWP9c/oF/a/2zcP1gpjsvoYmM4u+M", - "74/QAeFJ7L4mGnBB1MRuzCKSOLF7vaHn/p0HerMikiT3TaAPRZ6Fe6xA6n9U5AibVR6jleb7B6a4wX1d", - "KhWz/cPQ76Oznwes59Z2TuYu1DlcpwQwaKwqjczLCEt0DmPqnWvl+wX82rf/dbofYCpeJHx6cWBUd5Tw", - "KUoos/kAXqCyFg/sWsJHBoam+M6i0rgicRtGkvjXP/4Jg6Js+q9//NNiu//rH/+E475l4NWgxvTFjGCh", - "xgSriwP0CyFZDyd0TtxkoAosmROxQLsDa/OHR8grdW+lNDlkQ/aGqFwwL2/C1GuTtkHrKtDzoSwn0sL4", - "6BfpxBaTMbGNAbuNO8tmKe/1RHcDcIgwA28C+lZ0NABYctQU2raaaCdsMjVzrhhN62GaS8F66/mLItfK", - "UG/PDPCGDAaWOHTu4IGdNNo4P3+x2UegbRmqgIJBoDuUzVg1ov+NJ63nSYajVBkKrLLhTRHO8Jgm1Jkc", - "G6qdmCOY4mhGGSnjiwuscdfEgRup5jGHZyfIBkJ24dUhe32+BSZWRSKVC9K1nEBYhNGyHBq3eS7QA/Av", - "qiA6rGffHbIJwZAndHJsmIAHwl3kAxYNMwDygBhXqiqV17pDZpBkLXKxPngpj0kCH0H/U6zIFV50UVHr", - "1lVHSbDSCrHs6peHzGC92jXoAVQJ8obZB35mhtRzkbw2Z0uQSaJVY4jAN2W/oe+NCRfIRjh7Vf5ddybJ", - "0gxLL1qKo9fnen5T0AS5sQdCS6/P3W5sdpHkKEooUEOE2ZBNIRDIgfdyVtnVIqFshkXci7i+BHwwp0vG", - "rxIST5t47JFPZHcoyVT6CRynn+vk+tiEi9nyBPQhNgB1qz13x/addq472+KfyXdnC0HewHlnLLjE8Buz", - "ut8ceS0ceeF1c069kGft2CEw3l3Er+nigQJ+He0tr7l54i3ZQ1j00IaDtgGvCBfo7OgE4TgWRMrNf297", - "n56podJS/tP3o2bFDxF6YsfChQX9s/aWKoE8Fnbwxo4aYTeven1d/37bqhTfabzpijo85ZV397dHrdOb", - "XCOl0FvS2rebZG2wLZURhzKDJbX0QDRKSCG+FOfUp6J1VmUTxltcOSvFJcueT47dgbw/+7LtOmf1u+Ee", - "mOJxjSE+ICOsplr7VbMfEzW/K3bRoU2vMD9/XaQ5uD8p6L5N0SEyf0zqYlxbNs0FDdBJ4wX6kigDb3KX", - "errtITDxcyLcqTYDXZhZF9MynyKD0wITAkvMat33xLzSTvU17f2ZNF9YnptILHbJv4koLZTdcq1WKbgn", - "tgT03em30MON1NsvF7ZiCSywyGBFHTu3E1hWN7BcsGjzW+TKF6doE9dYKrHCzZvEhSXboCkVetZ9yXWH", - "zK83rmU6q9dShiYJnc6sEyCmE4jVU379bhjlzj2MsqiTLbAiNkTxMeb9nulFtl7gOREKvT46MevvX6lb", - "f0DQ6npVyTGvlbfruzeveoRFPC6cJ80yqX3yhRUmQ/+VXN77P3WPMJ+VOvGgSWD8jP03weTIxL/3Kf9f", - "Oz8ldCywWPyvnZ9wklFG/tfuYYIVkWrzzohlcF833X0rMI+Y+LT+QquLBqyJTQEydo3AX7zVUuZ37/+p", - "xH4z6RsJ/sW6fpP928j+/nKtFP/tVtypAmD6eCAPV0FsodWGR98gbe7BaGop0oO0qXiRSlCbGZcKHj2+", - "/GYbVE4LivOvjZbW//JArrw+HOmeHHdhIaGiNFS0sOmD9+QLcOO4d+HW9nv/joDDdEynOc+ln5mYYhXN", - "iLRZuwmpMuDHJnaX13Oj4P0VU+ngPq+Oe5erv9H9HUn89Q01zNs49NbJ/O6ttjK/fV/L/AbR1GY227Ib", - "XVeSabMh0NphmrYl4wr063IAeGhcIV0EvdOKSqkuINAgDobsf2v94zdFcPrhB5dCmQ8GO0/gd8LmH35w", - "WZTs1JEKYUpQW0Hv8Ndj8KJOIVAWiuyVCdv1cZia3UB6rqzAv52CVDqS22tIjgq/aUitNCRvuVZrSHYv", - "7lZFqpYmuXcdydFbaMEtpvifU0v6k7tHKhqczCcTGlHCoMALJKbLpXhAo8l984zcMiGZWX+kF0xUkURa", - "q5EF11ojoZc1pb9ktE63EeedI6wUSTOFpgJHZJInpjICkrNcxfyKOdh3mKCrIETL+YSud9fUyDUSTkIL", - "V/9tq+kWFb/uW9V1tbYfZxYYz2zxWqtclqJNs3b5sMR7tzpli6v2/rXKx0xiRn1bXrpMawiBMkamgFWa", - "m5S54ssSAa2P3r595dLjtHoiXFEsxV0lLFckdMj8Slh99KIsMWZecC1o9YHENp0WkgZtbamY4DihjEA8", - "MZGhTLZq/boHPRZfXgIOF+drJQHf87G05VYfTgJ+MFZwL7LmSaWKNS8NEn7dvuK0OHkTTs2j4leWAQUY", - "T0jW28K54j2bcLs14waFLQxEeZbgCHAo9WsGIs1iHBhMRL8pAC4QPEmIMNB3Wa6cuDVkxeAo8wrSW8ns", - "Qjc/ypmiyUXXhPMAfolEmC0s/tOQVTqzMh/kIUOOPYxQkMyMuFapUg+a8lzCW5Ay7HeJcHKFF3LIbOay", - "+Ryq+goSGZTIJOmjnzmARiA8xZR5jNeUS/xODtkFjRMyspgPF4hKJGdcKMJIjFI+J7LaL8EioUTAJI6w", - "XjmJUrwA8DWDQ2nWh2fEAJxVkCW4/jdmMYXCe7rnYsoHQ4bRzmCAUoKZtHniEk/gwrFtIBhEZUDfI4z2", - "Bs/tV7V9A4Bgt/wb+jQJQeY8wuNkgYimYkCqUJuwgakthGkKCuvtm1AhzX4V9k1b4ayysVS6uo5xF+Ws", - "zIQHW3/OisR1vV0qFwzmab2AhIriGrTgH2MSYb2ejFf7AdhFHkW5CF2Qequ9iqz/joKjN71zWKpwnnkC", - "JoOIxLDnjKsZnGkOR2nz+waqKonqz3HRBA8JFwgjj65LiwaJcmCNGwBTeFGWF2SuXPDF5vfu7OjjaxmB", - "O/4GKPCx3E9ARHwyqRzA9VeTOcCr8juWSfjPek6PXF1Zn8XFFE8Zl4pGjhnWy9B/UwhbK4SrVzZIzRMu", - "Ln3Zqkq/P3Fx2VYDs+Cn9HEpYv4Mv0JHhB4eAE0/vD8CrOFGWdFEc+9KWp2+ilMKQhdV0gU6c5RwNtWn", - "qLTK37vbwNfqNgxonL5MhXF2FxA/WgkZ2R9NaVo9GVv4E1wMkW31oXmR7v0enFG/coVomiUkJVC6tmeI", - "TW92CQcFZf6p9ECRbsYr9anyc5eNLihN/EHXiUNAV27DNkB6X96uIFNN+HQ96GDRuUPYC6AODtk7aeDA", - "L4zr6QIVPFgLtAbiH13NaDQDBELQW3X7BqAQZ9lFAb68eYBewkH2Maih8w0D7K9pTfKEGGDBeZpeHCwX", - "Z31/egofGfBBU4b14gC5gqzF/SH1Wz6ioJ5FgqVCv1qcxI1CGYcdvVBY65vF/DYt1mAJjj1kIdxBRq5s", - "g3SCLjwIwosGfCzHb1/xqfxqXEVlSQMzF8WRVR2BNgmLO01BHjQJO362B4MQ0nZLJEQzjDsGQlwazCs+", - "LcopVEgZZ1lb8rXDBCqep+kKGkYbHqyaVDHP1V+kiokQ8LGl7ibiRhs4sqW08KUmVAui5w72JpBfMJTJ", - "4JsHl0oz1U63Q1iedg5+s/+ap2mn27Hj8XDRbyDcr0GUrDe4HHKjd8aDjfwmlt8EELLK7D1EyNrNYdXp", - "Zon8jXnhT+8tdDa7ByRDkA9qRtyvSQT1xls1+DBeIFvCyJ7fx8gA/hJFCZek4uB5POBZ1tBVkxmbDUVu", - "jXt6eHHuqg21iWA5t5+euy+/At17XayIGzNy0733oJHlETzmRGC5NJsJF3XEpXXRJF89IX25LVmaahsK", - "+UabN7cytiJMrScsswj7QWyqz+Fc8RQrGkHlo2jGufTIvoBHNjXKrPG4oEwwrRgt12YQXGhSvbBm6Aur", - "RhxYkxnC/iPbRx8+t3kH4S/co/KLnzyrQMHxu070h+oAUJpdUDJBGc4l0VJdnhIULSLNFU2pK4KjGYpw", - "pnJBoIofQSllNM1TH/da79gcA0bHxXZ60UXjXKEEiyloZeahC7aJeJoSFhOwzw3ZjOA51SqlQAlWhEWL", - "niRQ/XdO0BUXlwnHMZgYshiDpweqBwqiKRBAxFOicIwVBkHnQp/4kUliuigKAhu1npHrkhriIRM5+95U", - "NNDNXriBXiACkN1UzorCkRGOCYuCUNbnXzcb+/K26HOi6hN9oMigW/HShwwV8m2ubjhfRxTRI4vF5sJu", - "Yxs2v0Lolc0qbDX7w5HRv+eRNnN1c3wgB1OxxKtO8dfhWSqI7qvxLj28+4gLFOemO+9UApn/WX1CBUPx", - "g60gs9Rs420dQ0WFvGKZb8Tztv5wf57cwpb3lXDCbqNi31SLqZz018By7areiuc+kBHT2pJ8m9zDsWAX", - "0fVg4hMXHpd7LMZWy7DN0Sz4ts+dlMCgfXH2jW3X2bYNeLgt23a22SWXvsfIKetBjGiYg1szbiOrtqaD", - "f9NslNrsPJb54Cyy9FzcG1s8KRihYY0ZXiQcx3+GIOEV/qOIC2HgLwBQ4zHBr3pWQz89AGxzZZG3rsvW", - "fH96utnEJYRaySOEesQcwkvJ0Z+l8bIB9/WcCEFji1KKjk6PbbgulUjkrI9ep1QhxdElIVmZ0QJZhX09", - "PwcEslxQvoL40e0QpsQi45SptaMoX72bwXy6VRn6e+aTFs/7mzu8tTscLPuPj50Bl4GcDTOB1Zqpwmpt", - "nVHKJlykRi7DY57r1jUP0suk99MgFUxoQuRCKpKaqMRJnsBxg9oQtv6v/c7schdicvXJMelyGREplZJy", - "JofM5opkROi+9ee6fS/AKugQULjgr2eGSX4dwXt6MCZeDaumVQPIJqgr2jnobOEs24qxwg0BYnZ4nzGk", - "nyAaD8lFOuYJjVBC2aVEGwm9NOoJmkuU6D82V4bzjeC7L13d+PYnS6/0CZvwYO04Q7MFMf+psrosW3OO", - "yUfH1l4S/7A4/gMbHWZr6+snC4KTHtQjdsA9KFc0oR8Nq9ONUKloZFKOcLF2708LptofslOihH4HQ2pb", - "khhEA9AutzLBo61hPhjsRhkF9LddAoMDhtf8OIUej87emTRUknKx6A6Z/gc0/PbwzHh3J9haE7yB2sLJ", - "6GTr9ZoA53NYpn/jCEEzwZXoBcEN/+YSvDnGSOMZkg1HlGerVCWe/elDWK0E982u8DjtCgDyVMxmowD2", - "cmhcYRvCnCd5qv9h/jhZh2umcDR7D69+NdKuGc7abtwEH8WhtHOKialt+SBOD7NgjzVmVS+cmwIIMZVo", - "wOAtcKj+jNT95c33/jp+he5Ou6KubuxXc7bu++azY3AIG/56PJZjbijNzUTx1danK0ybrU8/Jjy6lBaK", - "xTcbar0N8NX1jyUetnURgpgAmaHIQhgZoCwiu0NWM0AaxB+JMFJEpJThZAvmbBoBZG9nxcJzTiFBO4I8", - "lZ6kMWAmJQDfDfB3ejZgqHINeB5daStr+e/4zkjF0ZhEPCUO7XwzpLr9DVP1ExdV6PKvhS++9dYfIAEx", - "BXv7GrT25h4/C739FF9DqHScW4eyG9HGS17+aExBXQR7M+zsDuSw00XDzk467OgdOMJgQsUK7aOUslwR", - "2UfHxr4FKbhPBkiSiLNYOtB1Z8HbHcimhFxDlg3ZnU/gu/sUeyxVwVK+sZ2E2IN+D+nvIWkHbfgHzp7J", - "uAuHLkY8V8bcb8+VfSsmCswjm/fuq/XOyDfdvg0n/5s9vhUeBbus2aW39YazZ7mckWaT2ytTyChXYwDz", - "dsVF5Qz9nY9lFzFyZazhQqr+Et/TX5+ZDu6j0IDu6iZFBuzcv1UYaFFhoFyrMFijCbDUV7KjDoPYSK4z", - "LhSgONpce0NDoEkAcgSPcIJeH50MWaRZkYEWFCTlwJ0sHrq5hQ//do5eHL3pomModIl+zsebffSaJQtX", - "btz4aIbMSGKGeUWYobGhWhKHrmczdqCeuwwW1x08UOVoczICnhW3Vy5IvNuZERyDRPJH5xU3nQVQh9+8", - "0gcIgH/Nl8W2d1YKH503RIlF73CiiFhu9tTmSbECM8Ne0g6CzgpuBvhSdygd8lrZp5ENDDTG7k4ngJTx", - "6VvRh7svkHo/XjITJ2LK7Y1zQBplkGSA48XjimWSM1QwxxAL9K/romxCU5aw5WUrFQzosiny+ysyua/k", - "XRVs+X/X0wUzfbSOpqyyT5qIi3Iraz29Ljl4ZuCQraMqwhmOqFp0EU4Se0fZm6CISOkV4u9YEHwZ8yvW", - "H7I3RaEXm9CLjs7edZ2jFsVUXpoWrC+2j17PiZD5uBgcgoNmvMaw5iQeMsVRhJMoT7S4QSYTEkEuLtRv", - "kQ2+3GIonTs8O2UnwWIzXlR7/uhq3IVpAnavJIs6xW2Zrd4SJEowTZvBx62gBgGHEGow1o1yhiibJDak", - "KhJcSmSb6pGETuk4sQFCso/ezgiSOCVDliWYMSJQLk1UvB56LxNEytwkeOsGAKTXUFQXlcCCmeDKhiYk", - "nAtpogk0hb8/RVKRbAWZvTEtn8Kc70i2NY3bnh7ISF0bQ7MpxL6C9IYYSjELrukoT1wA472GopsBPbSU", - "+FgO/ltBp1Mi9KnAhsmacDxzrN1ymkNfyVhurHd5XrzVrt5l0aqXlehl7K0EhhuVWNtx52ZRf4HOL2kj", - "dqB9dLMs4l/0Ry37rmarhgdhH33mLEOlO/8dq2See0mCbQ1YJYU/NnOSN/LKUa0k2q6H1WqdWXuXma6t", - "8bMeDDbrMaNl4Ur6bJPC+/URwuB+UR7uu8ja46atCtpVRTdtSPlfj6b/VVDg3cDoPzDKyS1g9L+qvHvA", - "OX84/JPgQX2oPPqK79kV2/3TI+HfVfq8gcMHOLam9HnD9Wzw6kpF6b19p52aZFv8M0nwNt7xBvK7W/Zv", - "Wn8LlcFbrHUuaE3wJM3UwgW0WV9lGXQm6UfSb3AEF3Grd+cKvkVI55cjD0enjQGdf87a+A8SM2pLB1KJ", - "To4DRecfGcagf+YqF8uWvnV6WEQzOifNRvfqCbZLlAnSy3gGzpXYLJhdD3eXKSz604/INm8xV+2/oPYk", - "QPWTGMVUkEglC1MHVHME08d3EgmuNQF4zsWiOUrEHJGfBE8P7WzW3If2TFljWBlnmC56MVa4N3fcZoUJ", - "7TOiO108pWZ4iDL08ke0Qa6VMBUu0ERrPohOiiUl1xEhsQSa3PQHvD1osGzSj2Q0HbcZ5YpaJa9tLRgU", - "5VLx1O39yTHagNpnU8L0XmhRfwKSbCb4nMYkroyxM+eJWdXthgW9qd1VCxVF4TqnXJjBPYgM0+ZCmn6k", - "WZUtFCExY8owDG5tVZDqmTJJ/Lo/TJkLwLF75Ebx7Qqzmt+GU3Y0JUIdTruIinMD8bz57Zp7zNecnwzl", - "7rTKbefCc1Ybr9vlR7VMW7qLwg9F7tz9mq3ffz0pPVQ+ymweazqfFwppk9n86yLBwf3dD/dtLn//iFNA", - "XxKnfHumcmhAtxgimFcQ0x2TOUl4lkI9dHi30+3kIukcdGZKZQdbWxD7PeNSHew9f7rb+fTh0/8fAAD/", - "//uqpNOh8AEA", + "LWTAlvvo16Kwll5qWaIq9QOWwho9mQBq/cJIJjwUjfKTIATBs7Kgpz574ME2GL1F1ACOAaOJM1dWEG1A", + "lFdMJPo9xwIzRRmJ0fuf5PdoYF3m739CJovGMiAq/cDGqqb4NLSljdlwZ7OFpBFOzLKY0h6U+VZKONKt", + "1Yyz8kNrzw0oG+Gico59oI35NMthAc/f9E5ev99KYzLvVsYE0YgznhA97k2Px84dJEOZuldhrfMmc5Gh", + "btmW7XhrVfC91ovkcbnA6nhU0ERw73+qEou5eEyqqjXXUkYVQG8B3vJGMUiDEN1Hp3hhtfoMOLjpykNA", + "xZOJAaAsOJsgCcESqlFL9P6n/towQcUVTprm8FY/tKdmQ09I76keZhdlFaKEk+QhP/ndPgkysHI+AacH", + "dFi3oPfRCQsfQhOLAthtBoZJSjplWtOQNhwlwqxYyaW9q3LyoJGijnXXqS5cZTrdADsKUUyId5rcumWh", + "e7k6Jr+sHhp+ub4io2mkud8jl/5X87JYzcbxN8gSdHExyHxq3CjWVidJhgVWJFnUAnSrCPtkOcSBXJPo", + "BvmHL/Trn0xVlVyQkZoJImc8qQbG7HaXK/NKCEKfE1uMzMzJ8wQpjlIsLuGUOc0O5cysQDWHYXcd4NBM", + "qewGk/r57dszY+5RRMxxUs+CkUshH8ckwQs0JuqKEOamgiXCfhB0PYtYNhSEEmqUEUF5dQ07u4F+z01g", + "PJoKHBFkvnJ1vQu2po9e26W0vQQgLKOISNmwv9ur9td+OsmTdnscGtb22jL40U02+O3RmStsVBSOdsu8", + "s7zKZ0T0zJFzFaRXb+2OXF1iy3XFDDLxkgQ5JlBzy6Ye+ZnBLuANaorpzyvZuB5zkH7+qO0HToFZqq45", + "5x9aqRv14x6K7Egxi0MFuE3GicFqmAKAG4S9ixx0ARqbaDhzJ3v3s02cEQTHlBEpa4nuUS6STrfTm9hZ", + "HWxtaX6fAIbl3u72s63VccUrA8pt/NwopqsMDi7KzsRhubRqAwAIk66SxBbOshYmWbOOa+4HYE/LAaxQ", + "dFvfbZ7I7zIaBoOlnP1rHClXJhBstBUfPPaPLSBNVwUZ3WBqKv9o3eu5fz4HwaCIDKtZlfy3lmgfAqUg", + "yFXTiLF/1dbREPnHoHTKRagWHBfKpoCOiYuCLe5DF2PqMJUr3trBM3+WT/b3d/fX8SFgNrVjbs9dYKrm", + "7WqSO/GW255f2wDE61VlDnekV5ez1euyhqY0R1wiqdULyjPCbrSe+3u7Ozdbz7YTOXFxgjW+FMIIOjo9", + "NjKR1iwxZUSglCgcY4WrTAaMm5rLQLEhTFLIE5t8v5q1NATU+KA/t62U9qXCMRqKJr5xGOEpZnQCSpJ5", + "0+9ZzvDO/pMDU9o1JpO9/Sf9fv+mUCgvSuyTVluxZaI2PVSUvpx93j7cAeJJm7n80Tk7fPuzZmS5FObS", + "2pJjyg68fxf/LB/AH+afY8rCSCltqgHTyVIV4GqAYm6hqEl8gMqC707uaRMw1uCdgHB1gGcK4g5Wwnbv", + "DmCwoHFarSNxg2TxFcnTWlx5zZJF49rcurBvWfJeeQV9fbtCi+K+9OPqgAtn/4R3bJ/G0lHUPV4OtbhV", + "5Wq5srjnUu23jLCinGeSmL8izgDpOVTbs3JFumctSsPBNWJrwBVd+j8WvXs/HvkD8X53peW8n2yRzw83", + "jJFaKZD+bVkOXc+FnDi65jCHjdHFrdC2oLIFJgwmBzzwXXibOMJq76+n//X7/5FnT/++/fur9+//e/7y", + "v45/pf/9Pjl7/VmgN6sxKR8UWPKLYUlC8FwFULItKZ1iFQVsdFr9a1hh+8RYHFQ0gxKwaEwOhqyHXlFF", + "hCkoWMuGHXbQBgFNCb7S4i5USzKJiJv64zPj4tYf/+HE4E/1NmKLUiDshhTgMzIfxzzFlG0O2ZDZtpCb", + "iAS9QP8VowhnpiogZUjrvws0FlDC0focy8676A+cZZ82hwyssuTawKtnGKrgTYo0QOYiBuyoTFy0fZ0U", + "OCQmRXXIitu6AGU0jud+WcGBkqSeRNawKKv1N6s5PRuE4CshwUlvJFQpAhWkoGxNRkXmFXo22FzW59bo", + "GAUNrSA/G5lhMmAP85C5uClr9pjENAK+4hJHZza1uMjZNZRmjXiZ4NcL2Js3JpsxRjhXM82LIou0EHF+", + "SUkXtrQL/lEIBYIvTYDHjGe98aI341mBuoGFCX/CJkSiqmT/n56daO89EXRiewqCJ2gSCQidcGTszEwO", + "amFdWJrYW1PghWnRZ07s66ZSkDTFYUxsv8oFc/VLCFQxBXiXgvpISCb/HkUJBauTnPE8idEMEBiVbiYE", + "otgZFDlUeBzFZFL/dzXGZWf/CWiw7t+7O61TmM3SraKyPAnotKljfS04tmGTMAAjHoycIXxNVJq+Aa1j", + "H+wUisN/z5FrqDxxBSMxnj6TVCltOYxEeumUm8G8N3sMLIbMCNvT1OY+WjqFlTS6Fi2YcBX4LGmBXvPC", + "pO++fXWOFBGpA1TYiPTuwCkxUCU9KmVuK7MdHp2+2Ox3gshbFZcWbNXKNLvqoAPgGzZ8pSkqp7TR4JR0", + "0ckxpE/ba6XUxSDf5ScuUGJuxfIyOgD0lqq5B5uajifHVgBNFmUMjBFbhp1N12JWv94O0JtCBcTFUIpE", + "2JK2XJPlZQLN2ohIk4yz1Hotbxr8Y1b9s/cxpN5AwUvDiwE9tPH+am9zdFhk+qKqWchufCH5YUmN9i9v", + "7780zvaXl9F3byajWy/0KJthGaLume/VhJeW9t13ZFfZvWgOIqv0O5I0eLb+5sr+eNeQIvqeq3weApbd", + "721vv93eu7n57qYQyVVsNA83sUBJbg9vfBcwwQHQX6pGjdkGSD+2uQXOLvL+FM2wZN8peFizjmzvPm1j", + "lIBe28bp+xH6fGKGVHApB7RWxJcbyLlLmiRGgJF0ynCCnqON85OXv5y8erWJeuj169P6Vqz6Irg/t0BL", + "hlsA1tGkHAawtioYEahIJn379hUcroRAPo6Rwy9vj6G81rTYAlPZDe7l2Ttw/GM5cpG8zcmruEwAJ9dU", + "KrkMs9cqIP5zMJzNp6VhrM0kTRs2xG8tEPTPFaTiIG7i5h0gOLtshqXlfABw44fMGv36gJVXQiF/Lp6x", + "tTPcEZxx45UWggKuQWTsN91utwcmvpPhVBCGQmzLl3BcSv+tkYC7HRpIZz60cXzo5Kws/VU6I1zztTk9", + "3+lvP3kG1Wu3B20Ye4qjFX2fHh6173ywY26ZAzw+iOIDUNhv67OyhG1UEJxc4QVUfzRLO+yYC9PTbr1j", + "axXJVvE1y4DLt8NXrotxDQjKIM66wCU5SlcWn2mRr1pH0UtzixGa0iShkkScxbIqI8+wRDIz0LimCEsh", + "wQ8ZDLCLilrYIKUgHEUiL02PVrq28n6eWbqHQrAZZ1oHgEoQv5CFRCkFJ2jRPYQ+SlSkRcVDtiFcCl2R", + "Kwc1YGP9AySkdG2qQ9yFqGEoNqM/GDI5y5VmYpt9dMSZzFMirFUWjSl4jDaRzI1KC+OF1VhohilpTMSQ", + "6dcC4Lt/FOrJwZPBYDDodgpNblf/exCipjt1fvYtuLRJAgf4R2ZhpgFXUuQM5SwmoigITww51EPkbug4", + "/UxUafd5O/HKfl7KVeGDuQ6Euh269OdC+sJQG/RziA69hXK+f3sRvVUSmpNfbfqZ/Wp0kwgGgiKeJ7HW", + "+Mb6tjMGORJbM6QkynDnIhPknSnGWp26DT1WHP2eE7FA709PK2EPgkw0D2g3ceASDfvAsxttw84aG8na", + "0dzEvewBIN8H6HFdUvEkxC8Ocex7GF1WvKHQimGrojivsq9prTKYkUOZ2SdNNCsmWKvHazBEysBIv1W5", + "kBML+GlFaZdTYQHICzyQ0qkvF3JrnMutLKJbNpdpC8BangFYy14wmz4m81Geh1Qj/cih87x7d3KMNuAX", + "ABs2CTKV7jF+sv1s8Ox579l4+0lvLx5s9/D27pPezj4eTHajp7vbO7srkopapFfePmMyqDEH4piLqPWR", + "i54PBTU35S7UZBMbj31FWcyvKtdfMEDW790G367rfjm0vvUQgilBCZbKmC8aONkpXPIk0m2bgHSbwlvU", + "3gobOp+8HWx/rvUHBtdwR7wVOTNuVQM6UbgQUm/A/mZVx3k7lg8Dcokv61bL77z9og0O9p8f7H/uornk", + "jXVjrJPTPW5uU0SYA+WuZYe4lFXPjuQMlB0rExmrvk0m6XQ7Rb4L/A3CQC2WunjcKomr6cB2w2xk1bXS", + "kE1/UtFXIFLFgDLGB1pScfoIlPgoMBu0CHSU8DxGni3OYNSBH+7E0110M+AWsyY6k0ZrkjEgM5JKm89H", + "mWbE4H/UjdjU+wP0Et6FRzg1ap0dhClo47vecLww8TL6fLmujZK1esjnVr+Cb7SyhfS/YNp6GazJdnUT", + "Rjo7QL9y+KbQ9hiv237N66BmLb9etxNvWPx2B6UCnVlR8wD9VIiXhYBqBdINSeyfI8uwSgSjzQqOhN3x", + "jqaWcuc8TIRux6xop9txCwXYCcsoCu9Kql86fz4phgLJCE7gLJdJ47miicVth5lQqWgkbfKI3twmscfm", + "ZpJ4ZJSnpphUk/pqFaziIydVvT9FGwDN+RdkDdv6X5tF/Grlrtt5vvf8ydOd509aAXCVA1wvGh9Bnvzy", + "4NbKyVGWj6xtpGnqR2fvjO0jMlaFIvbl/amPd5IJrlmPnrlr0O/8ef+5jzsW83yceI5FC1JoYI5hw4LQ", + "egUvaoiD/J0mczqZsN8/Rpc7fxc03b5+InfG2w14yqajsNntxA8uWLJRk3HP1M0KQ0MBQQnZiJ72hkiY", + "ATonCgH99BCOQL0p8qktyTmMNbviQcLa293dffZ0f6cVXdnReQdnBEa4wKVsR+AdMXgTbbw5P0dbHsGZ", + "Nh3ICEDeM6v6hs8ZskWvB1WBtL892A1RScPFXVKNbXueNi75e6s+2knZRYfk7UK1XDrlwdXe3R083dt/", + "tt/uGFvz8Ehcr+YwLmXJLI+tzODv/AZIk28PzxAkBE9wVLXtuAixG41K3WhUUFXEVAO4wcCePX2yv7e7", + "s90OBjAUdGIBLisHtsq7AocuQBSB3QgsxTLr7TbdFiFxyhDYGxIlmKaHkUuxqN0+BvV/JMxr5Sa0uRis", + "Br50cbX4tpVxqzBZmQQdIxpwgXJW1Jrpr3fJfhHPajPXNtfDeq4eSsthevUsXpWpqXeLpcwEmVOeyy/Q", + "EFcmZ3aScC5u9G2TwvKGyDxRxmZDJXp/+h3wFE1rSCqSVXUoS40rUL1uObkbnecKiYSJvGmxWu1Gm61f", + "NeFuw6ntrgLUqHCDRiy9WHOunK0P/jzCSZRDdSVc7KeeFYDCASRAliULE9ufJJwzFM0wAyeJ8CCw0Iwn", + "cT8YCaufjCbBqAp+hRJuUMAvCcls4SEzCP2ZFmHonKANv+SeIaVaIdz91DAZW1qmSo37abiiJ5ahZLUi", + "FV6vJ1bcA6g2n1QsoQmfSlAKFWQt9Ot1ETIsTDICZqaQ1jw1umQg4DowxBozD92o5iblE6vgWpEDEs3N", + "SuJIcOmBU70/reUvr8h5K7KY1wd0VgfbgnSNQzNwlRl4tNb19kL3YyCf53NuSKBhyBlcESrpjJMpZjmU", + "IvII2Rri+63DIWdcqlEB8nXDwUo1gvoiuSAlfGKRdV/Yg9w7wXvRsbbbLJeNO77V10tUFW6qaYDNPDW4", + "ouHV6hY0GCLjZZSzlcBqJdxcHczqJuiFZUEKKqFV6uHYoQ3IefHYkodJuNkmSCassup+lrRVWy721d7g", + "vC3O32pYvzOsZidswgPYHzfwnDpLtI1WzYhwUHsxYZTETpcsXKjW1AUJ44kkKM6JXTkjnwpsFxyb4w0+", + "K+ZsZJRNa7y+3mEb87AZw+ryI9CvfbFNuJMMJ9S+FTmslYlXlAiXqbWtgkCpHIXdWcsNCzLNEyyQRehs", + "M2S5SBPKLtu0LhfpmCc0QvqDul98wpOEX430I/kDzGWz1ez0B6OmWlXnZnA2L9BsSK3fcgo/6Flu1rKS", + "wRKzZb7fAsdom+ixYKT4TzQhFinxHaPXHqFX8fn3dgZN2fINjVby5JehQm/KuS3JBk98LgO5hSulHFdl", + "i8S2aIIRe7JcmoI/LW4lh8rrXIC38+hUE0c+D5rkyPDrGjAJGhPI+3FTW+YaLdhim6kEa43kcob+zsdV", + "g2jbsN9ABbsNVkJkCDIJxvfDjq40SJs3ltbE292bYFAAW9UThY9uCO2wrtZfGV/VxE/eLJW9mxG7ZNTN", + "0ZTAa1HSxcV/FPAFttf2OAb1AoWBeGVAqZFqAaV+ob7Swqu6KdGYCwGQ5FrC4czNBmBXtMyj19rhXqG3", + "M7JAgqSYsiGjrDCSApgaQYzMifCyZLnQStaUxH30N0/FAxD3NFMLWx0AjOffScSvWDHGIfMHqRvPpW7n", + "kBnLosgzVakfqpsFrU8TCmQtgxNMCSioSdUMTQSRM3/uoSKqWsa74iJurE61QO4VKHoEPlak+CVhPisr", + "mgmqhqahkflqOYrPVECGp1b/RJWixKhedHh1f7kkIiwkFlMqXmkVuuIdFU85MSAwgIgCxSbtX4bFFygo", + "LTBPyub/6posfzorGq/+VnvNwzVxuNOHxmwbNMFGJo2nFuxT9aStDVWBNLhVaDbLvgS04UKoXcWeqiTg", + "Vc5pdU+2y8SrJwu40WxJElV733u2//RJy9JFn+WsM+hdX9o1N09XuOQaduq0jd/n2f6z58939/af79zI", + "w+LyShr2pym3xN8ftEGulT6syb/+8c/3pzWvzz7EYA9uNCiTWRIeUkN2SXVA70//9Y9/ulHdekAhRrMM", + "Gd/gt2+M0kn8nXSBAlUXXjsn2Qr9/rBiJMAFm0EbBKC46ZyMzLr1ysHUYEDaScE4wxFViwAjx1cm2r14", + "pYa13cYdVB1sSOQ1bVtUVM25ZD4uk043XOfoP41vuEYLz1pXQJP5uMkP/breq/FCl14LP8ahRYiBLIrv", + "Lxu4i/lcYVkJ6NZ/R5B34TLMlrNtzBurUXfrqRAQxWIL/XmhgCHk+5o8aT/yt7+2nZ7fsmLWqa/4hxXn", + "sPkI3sjqG7iRA0bfaH1qbY0/2Avwdl+Nxn5twpXFHyuFDMtb9+b9tsgeXi6cUdxgN+/PS5i8yYd1TGCg", + "RzsGu+Rl290KSTRQk5cLEzCg8YT0vOIFpvCTzI1HUJ95CzMfyOCMLvlkUsW63W/GRgfYH0j2cr1gpbRm", + "0kXk2tks6sDaBuNn2NmXw45WAYad7XTYqbmtgumTKb4e2Q6q2C6DVWDlZfp7bZDSzWCc8OjSlN2Dau59", + "NEApwUyinMHhr3nVtgervUPdTubtTQENTkyI0xLbgjGNyQzPKVT3sD6VaSUQk1xTJSFgFNo5QDE3aE+V", + "msN2hvo1k9x4UE4aLh3MFrZh3aB+jzMX0Vq+Cwa+CVQ6Zh+J4F0LVqA59uvXp10TwAChh2ZglfhGN1Ez", + "As0giy5q5RXK38Pxw+OEjGDcdbj+dHkd/Zx08KwKIomSFr+7JIcaEaCI50zVcfzTdopcNa1s+UrKGQT7", + "2fAPwGWzvRsCQTGJ4ETK5bNYJfRbEHctb8CudChxYDdEwnAowJcU9hW/sQ7h+gCMscErF27a8eO6jZdw", + "JBW39eWKUz0i1xEhcR3wM/xK21h5+2UwVv4VthhBRSVv+zbEOy/Prn93CV4w1qbV9mP6GWc9QCdxW2qR", + "RAw0oMWqqRJaBXrcg7QYheBVQy+0ybgm16vX+ldyrQAfPc4TA3oXJl3LquxltG7Fb53Z2HSguSBr6zXe", + "QR1DE29+q0qGNlT9IYoZ2rfupIDh0u6cE+XePbdk1LhD1UIvFZeWC/h3r1RjbAwpdZG94NF2ulkjwb1Z", + "2CpiQXlb5mgynJJRJsiEXq8gHvOCUYyrsCblQSoyGAy+6EaKr9HeUxTNsJC1sTM6nalkUQ3A2QtgKX1W", + "lU9BFGHOUNhm58vddB8uR7vZ7fRbDwnH5x400FJJEyuSjlbhZh+V3jZrnc/wAqw4jU7Cp7t7g8HuzuBW", + "wNluWDdYrqPyE1sTs9pOU0qd95119FeiVP0WiiTr5ULLV4JCrnaxTFIJgtMDSLzJcERQQiYAklcktK73", + "LNa7Xj14K1DZLNqC/t1G2X1zPvhqyZyiK4s57qbRcc7FKgaR/3yNQ7SBzURLkHqBnLvd3uDJ2+3dg/0n", + "B9vbdwF2XSxSU7bH04/bV0+THTzZS54tnv6+PXs63Ul3g3rYJTWVgdrQ6i/63cYom/KSrGIZVVga2rBz", + "yIioV9CuV56XJKGM9GSRIbU+TXEFLzD+97Xn/2Z2fjODlbLDeXWSvgiBVbk4Fcp6GPwtO5mVvov6bE6O", + "V8/iVhlI9YGE6a0+FCCvdoOBChXbnc9EZshZy2vonfdi64toZVbcuqso5GGHkx7c5YYVD5F3DZjBm/Wq", + "C3z5kgvYTqdcUDVLV98WxWsFjDjETX+UKq7iPfXRyZRB+Xz/5yJMzlei9Medbif5uFc9M/b39shfFoG4", + "IEC71b5U0CKMLCFzkqxeBXilVDyEiWTXuroe8w/bve3nEIeQfNz7YdB7Xo046JrV8pdv271d+XXQZg39", + "EoCudNT28xtFXLv1XEVBv9BQAbvyXrbYxJbGy2Ll7upwCbeVDS4fL+1xDcmnUQD9XEnPXm4jX2iKSYIX", + "IWx6z1Ara9qjT2RoTKaUyTZ2291BYbjdT4edPjq0AOGgyype9OM3r2nFpxOapiSmWsY0qn9zBsNOS1tc", + "XZe4WW0S91VAWuuHxbXn6yES1iVcrbsm+5+Rj/tZ2m87jXcVegfY1ZyKChhi8GIX0QnCrFaglLI5Tmhs", + "E+khMRLi1Q4cUFtJspYHyFIOdHaSLppyhcoU+pb2tpw12wWL8ZNrsLeuwMwwBLHzRQBRCgAxuop9nRyj", + "TPA4j8r80QQGXSJ+iLwG0bZCyF8fknuX9g1IzJ5wgdbbN5oMGu3sk037XbNNaoJt3urtwfqtvhOjSLeT", + "Z/F6HmZeasfBboTcviYFMWCiqS57TRL0JvOhBUd/46/gss5rbMmRFonyzDlYNE0tU1LA3QIuhlBc7zFJ", + "iL6mlhtBPInLLAkqSy66nqVuP3k2a3JxgkdqeSC/EJJpXQXwj6C/FLNFcGCu7Ghxl2wMHNq3NA6vnilX", + "ZFerOrinayWxxq3yTbhNJRQMl6/ZvA1eyqVn/i4wvn3RbBkBxTH8ipD2prkEgP3Shb012o/vwiz3kELa", + "a+t6qMG2OlDhAh3d9V/GAmuxrkq8eyH3fIgs3lrNuAmKtp4F6ludD3v/Y6zMaNQ/2PrhL/9378N/Bq3N", + "Nb1ZEtGLyQQCjS7JomeKD2kdvV8FYoXKB1qYnlpSITgFGxKAnNvD6I93f1AwjcWvOF2aAkRoeZWDttdO", + "6C//0Rzf5C3jO+CTa0n2swuD3EUBVcXddbSREjF1seQukWyzP2SQm3ZJFhJ59cisSOMI9TtZfOJFoKML", + "Iwb2CZtfoDGFAo9yyLRWi6OIZFqbsCVuqKlSzoH7CIITvx1bF80lfluHpIknIOj96RKK7+t3b398/e7X", + "49Hrsxe/Hp6Mfnnx3xDicdUzPcQ9TXt7+09sbXJ/JbeD9TFuXuahj05tmL519U9yUGgBp0uiNFc5BIWQ", + "6yjJJZ07B6FKbl/QYTlZ9/YFEj4TAVipJBSVYJGqEzoh4NeH68QG1VDpiJFKKOpujRuUoeUb2xDOsAOc", + "1KvJHyqnobcivNrlxlYX/cmsHQs12KiBww4Zr1B9P6C9UAl4FS72w3sZbUDmiKs86xJnN2+G1XpYNBiM", + "PPzCBYYGz79EEdB3K6t+znnS0+pNQ6WEoDXZrEUwch6aMhkJnSanw3QckOGtaXdKpzjgZwj5E75IsU43", + "oLUZU0v731i1LJzHcFwvI2GOpVmqWtmDmpFAql5zmkOqpdoG4F1AFja5q9SLrasmqqZMbdmiuiG8jJgD", + "mPmqbOXylDl0xB58tD4Jd6Ve5c3MG0nz3pw69aGm4KxYoDO9NFczIoi3EfBBCc9/wyWzeTktUFhMUcKM", + "iDJm1SX1aKkU3M0SbRSWH7cERbbxsjl8dfmFU3xd9ACuFCyX/I8wj7L80/bLHwEq/40reUknrgkYRk25", + "CwPDV6lo1Zo4qlreDJ+qludt3g8ePMurVnC/prNVI86yjwpphujxb5iqn7gAdbAZ8+TO8eXh8o+JAAy4", + "Onp8K+h1mpJ4xHO1+vzbivr2yi/KopZldZ3qi4GIo0o6bxMvcKgc5RiWV1ovB4lyQdXiXK+XDeaGNEhX", + "yxYWEjqCn8uOoX7op09gNJ4EEkZeEkYEjaA6qz6PKWagMaH3p16RPlOvcQmvFUSg10cn1tzgIH9BfaQK", + "SM/FXR6enXS6nTkRRuXuDPq7/QEc5owwnNHOQWe3v90fdECrmsEUt6Civs2ftvnGheJ6EltJ6Ef3kv5S", + "4JQo+OK3ABIAxB3a10EFwVNPicwwFVaLzBJAKDAEQ/XXUG7AXagH5lbummVvbTOFNGPIfiHZa7u5H0BQ", + "hrMD09wZDCywubLXL+TumISBrb/b6NGy31ZSnV2iAPr+kprnZMti6T91O3uD7RuNadVQ4OyGOn7HsE3i", + "JaCd799wIW7V6QkzaXk2ydqGQ/knDgjJP2u/fdB7JvM0xWLhFsxfrYzLJsGYSITdu0aPUxJFmlVAjaA+", + "es2IeY6wQthELoucQWll96Gm0OopMG27TS5Ain7k8eKLLWGlD2ej+FRlZ/q4fFqi5y9HOwUZL2+kfeQQ", + "tg3V3gMB/YiLuuAPdlL2Bs/vvtMjziYJjRTqFQRs45GphJCfBPDCHfYQF+j3nCuMinD+R3Skrcw6Lsit", + "W15FW3/Q+JM53gkJmcHPiEgxM8kR5p01h37pOBuXRHmcV95qjvBPjjv2pnIgPOaiAkGuekT9a6suDC5f", + "R3sBBAbbp5le/ICEv3cPJ9xOtigN+5BHDgpyolySx3ScrIttXAohQVnuJVFfC80P7vPKskUE/oSn6LEQ", + "8EtSSHjlbi1dCluZyJlRgIMS4JsyYdF+911V+HtbPvGiZMCvoZuGchbK+FVxvOgjt6ZG6VcLgFgSBOYZ", + "L18rZ3p4X8sJ27mPEwYzLjxF366pb9fUqlNuqMVNAQ6md8pb2CBuZIH489kfbmx9+GZ7aG97aGV5YOTK", + "Whf+zsd9ZCNSIx4TJGc8T2I0JsjgHbnYE4VFf/oRYRHN6JwAqB0UacsTRTMsILIkRTFW2PjQGw0TK80S", + "RXNburmei0MsF7iOYyHJCHD4Rk34k2UEImWMxEh/YqH7SjjBpXLi5uwHDexFg+XViK5mXJICz48p7zaH", + "9GZptGNotj9kby3Qq15ACKZ2vEaSBOBqV9h/OEN4yOwH3zsW4gLBJE5LzoUFYAZSg0xptmU5tU2PdCQj", + "HsLaeUsYZqonMxLRCY3stC7JwsZzBhtsVXdJD9iN8/1pkbCBdjbDeG0AzxgG5z0uniFLSVX/DYMg6CjJ", + "49LJ5SCEsBjjJAkW5pgmfIyTkVmfSxLwCb6EN+yilA6X0pvEeExMCflsoWacmb/zcc5Ubv4eC34liRh2", + "NvtDBokYdq1J3C0FRHQFhdzSjOtzJnhq+twyQ9z645IsPvWH7DBOKXMUAZ/gRHJEruE7qG8FmBmGezXQ", + "gzlNYT/4US4VT33kU0d3Zpg8V1mubEaJJKobQv0cMsXRHw7b8dPWH2WPn8BZTHCs6cR7xUwJZOumUcsR", + "1rMfwasBdzuBBRh29EVqwjymAjNlYDsLcEo09bd0o6iOABVT6yscYYYynpnKEkBUM6xJrtIGYDXgJEEK", + "jpL7VgvusJMN87HQe+m4EXfPAKXVjhFl6PRH7zAN9p6Fz5MkkSChiJL/On/9K4JbWe+Bea0M1zIpHUwL", + "DCjOwXXqeNoLHM2QcVRBMcFhh8bDTuHOjTdhrLm04TK9HvgUf9BD+8F006XxD/2+bsq4Kw/Qb3+YVg70", + "WcpSgwM67HzqIu/BlKpZPi6efQgvaBN82XmFEaANc81tAifBFJBmvBvfXJGYxYjbWyBZIIxKDuQHrowp", + "w2KxKpEwsPR2BfnERDJ6i/HHECIXh52DoYtdHHa6ww5hc/jNBjgOO5/CK2C9ls2V6+A+K5ybBRE9GQw2", + "1yNh2/UN+CxbOAa+sA7YqBUVZTf1DloY1j+Xf+DfWv8sXD+Y6c5LaCKj+Dvj+yN0QHgSu6+JBlwQNbEb", + "s4gkTuxeb+i5f+eB3qyIJMl9E+hDkWfhHiuQ+h8VOcJmlcdopfn+gSlucF+XSsVs/zD0++js5wHrubWd", + "k7kLdQ7XKQEMGqtKI/MywhKdw5h651r5fgG/9u1/ne4HmIoXCZ9eHBjVHSV8ihLKbD6AF6isxQO7lvCR", + "gaEpvrOoNK5I3IaRJP71j3/CoCib/usf/7TY7v/6xz/huG8ZeDWoMX0xI1ioMcHq4gD9QkjWwwmdEzcZ", + "qAJL5kQs0O7A2vzhEfJK3VspTQ7ZkL0hKhfMy5sw9dqkbdC6CvR8KMuJtDA++kU6scVkTGxjwG7jzrJZ", + "yns90d0AHCLMwJuAvhUdDQCWHDWFtq0m2gmbTM2cK0bTepjmUrDeev6iyLUy1NszA7whg4ElDp07eGAn", + "jTbOz19s9hFoW4YqoGAQ6A5lM1aN6H/jSet5kuEoVYYCq2x4U4QzPKYJdSbHhmon5gimOJpRRsr44gJr", + "3DVx4Eaqeczh2QmygZBdeHXIXp9vgYlVkUjlgnQtJxAWYbQsh8Ztngv0APyLKogO69l3h2xCMOQJnRwb", + "JuCBcBf5gEXDDIA8IMaVqkrlte6QGSRZi1ysD17KY5LAR9D/FCtyhRddVNS6ddVREqy0Qiy7+uUhM1iv", + "dg16AFWCvGH2gZ+ZIfVcJK/N2RJkkmjVGCLwTdlv6HtjwgWyEc5elX/XnUmyNMPSi5bi6PW5nt8UNEFu", + "7IHQ0utztxubXSQ5ihIK1BBhNmRTCARy4L2cVXa1SCibYRH3Iq4vAR/M6ZLxq4TE0yYee+QT2R1KMpV+", + "Asfp5zq5PjbhYrY8AX2IDUDdas/dsX2nnevOtvhn8t3ZQpA3cN4ZCy4x/Mas7jdHXgtHXnjdnFMv5Fk7", + "dgiMdxfxa7p4oIBfR3vLa26eeEv2EBY9tOGgbcArwgU6OzpBOI4FkXLz39vep2dqqLSU//T9qFnxQ4Se", + "2LFwYUH/rL2lSiCPhR28saNG2M2rXl/Xv9+2KsV3Gm+6og5PeeXd/e1R6/Qm10gp9Ja09u0mWRtsS2XE", + "ocxgSS09EI0SUogvxTn1qWidVdmE8RZXzkpxybLnk2N3IO/Pvmy7zln9brgHpnhcY4gPyAirqdZ+1ezH", + "RM3vil10aNMrzM9fF2kO7k8Kum9TdIjMH5O6GNeWTXNBA3TSeIG+JMrAm9ylnm57CEz8nAh3qs1AF2bW", + "xbTMp8jgtMCEwBKzWvc9Ma+0U31Ne38mzReW5yYSi13ybyJKC2W3XKtVCu6JLQF9d/ot9HAj9fbLha1Y", + "AgssMlhRx87tBJbVDSwXLNr8FrnyxSnaxDWWSqxw8yZxYck2aEqFnnVfct0h8+uNa5nO6rWUoUlCpzPr", + "BIjpBGL1lF+/G0a5cw+jLOpkC6yIDVF8jHm/Z3qRrRd4ToRCr49OzPr7V+rWHxC0ul5Vcsxr5e367s2r", + "HmERjwvnSbNMap98YYXJ0H8ll/f+T90jzGelTjxoEhg/Y/9NMDky8e99yv/Xzk8JHQssFv9r5yecZJSR", + "/7V7mGBFpNq8M2IZ3NdNd98KzCMmPq2/0OqiAWtiU4CMXSPwF2+1lPnd+38qsd9M+kaCf7Gu32T/NrK/", + "v1wrxX+7FXeqAJg+HsjDVRBbaLXh0TdIm3swmlqK9CBtKl6kEtRmxqWCR48vv9kGldOC4vxro6X1vzyQ", + "K68PR7onx11YSKgoDRUtbPrgPfkC3DjuXbi1/d6/I+AwHdNpznPpZyamWEUzIm3WbkKqDPixid3l9dwo", + "eH/FVDq4z6vj3uXqb3R/RxJ/fUMN8zYOvXUyv3urrcxv39cyv0E0tZnNtuxG15Vk2mwItHaYpm3JuAL9", + "uhwAHhpXSBdB77SiUqoLCDSIgyH731r/+E0RnH74waVQ5oPBzhP4nbD5hx9cFiU7daRCmBLUVtA7/PUY", + "vKhTCJSFIntlwnZ9HKZmN5CeKyvwb6cglY7k9hqSo8JvGlIrDclbrtUakt2Lu1WRqqVJ7l1HcvQWWnCL", + "Kf7n1JL+5O6RigYn88mERpQwKPACielyKR7QaHLfPCO3TEhm1h/pBRNVJJHWamTBtdZI6GVN6S8ZrdNt", + "xHnnCCtF0kyhqcARmeSJqYyA5CxXMb9iDvYdJugqCNFyPqHr3TU1co2Ek9DC1X/barpFxa/7VnVdre3H", + "mQXGM1u81iqXpWjTrF0+LPHerU7Z4qq9f63yMZOYUd+Wly7TGkKgjJEpYJXmJmWu+LJEQOujt29fufQ4", + "rZ4IVxRLcVcJyxUJHTK/ElYfvShLjJkXXAtafSCxTaeFpEFbWyomOE4oIxBPTGQok61av+5Bj8WXl4DD", + "xflaScD3fCxtudWHk4AfjBXci6x5UqlizUuDhF+3rzgtTt6EU/Oo+JVlQAHGE5L1tnCueM8m3G7NuEFh", + "CwNRniU4AhxK/ZqBSLMYBwYT0W8KgAsETxIiDPRdlisnbg1ZMTjKvIL0VjK70M2PcqZoctE14TyAXyIR", + "ZguL/zRklc6szAd5yJBjDyMUJDMjrlWq1IOmPJfwFqQM+10inFzhhRwym7lsPoeqvoJEBiUySfroZw6g", + "EQhPMWUe4zXlEr+TQ3ZB44SMLObDBaISyRkXijASo5TPiaz2S7BIKBEwiSOsV06iFC8AfM3gUJr14Rkx", + "AGcVZAmu/41ZTKHwnu65mPLBkGG0MxiglGAmbZ64xBO4cGwbCAZRGdD3CKO9wXP7VW3fACDYLf+GPk1C", + "kDmP8DhZIKKpGJAq1CZsYGoLYZqCwnr7JlRIs1+FfdNWOKtsLJWurmPcRTkrM+HB1p+zInFdb5fKBYN5", + "Wi8goaK4Bi34x5hEWK8n49V+AHaRR1EuQhek3mqvIuu/o+DoTe8cliqcZ56AySAiMew542oGZ5rDUdr8", + "voGqSqL6c1w0wUPCBcLIo+vSokGiHFjjBsAUXpTlBZkrF3yx+b07O/r4Wkbgjr8BCnws9xMQEZ9MKgdw", + "/dVkDvCq/I5lEv6zntMjV1fWZ3ExxVPGpaKRY4b1MvTfFMLWCuHqlQ1S84SLS1+2qtLvT1xcttXALPgp", + "fVyKmD/Dr9ARoYcHQNMP748Aa7hRVjTR3LuSVqev4pSC0EWVdIHOHCWcTfUpKq3y9+428LW6DQMapy9T", + "YZzdBcSPVkJG9kdTmlZPxhb+BBdDZFt9aF6ke78HZ9SvXCGaZglJCZSu7Rli05tdwkFBmX8qPVCkm/FK", + "far83GWjC0oTf9B14hDQlduwDZDel7cryFQTPl0POlh07hD2AqiDQ/ZOGjjwC+N6ukAFD9YCrYH4R1cz", + "Gs0AgRD0Vt2+ASjEWXZRgC9vHqCXcJB9DGrofMMA+2takzwhBlhwnqYXB8vFWd+fnsJHBnzQlGG9OECu", + "IGtxf0j9lo8oqGeRYKnQrxYncaNQxmFHLxTW+mYxv02LNViCYw9ZCHeQkSvbIJ2gCw+C8KIBH8vx21d8", + "Kr8aV1FZ0sDMRXFkVUegTcLiTlOQB03Cjp/twSCEtN0SCdEM446BEJcG84pPi3IKFVLGWdaWfO0wgYrn", + "abqChtGGB6smVcxz9RepYiIEfGypu4m40QaObCktfKkJ1YLouYO9CeQXDGUy+ObBpdJMtdPtEJannYPf", + "7L/madrpdux4PFz0Gwj3axAl6w0uh9zonfFgI7+J5TcBhKwyew8RsnZzWHW6WSJ/Y17403sLnc3uAckQ", + "5IOaEfdrEkG98VYNPowXyJYwsuf3MTKAv0RRwiWpOHgeD3iWNXTVZMZmQ5Fb454eXpy7akNtIljO7afn", + "7suvQPdeFyvixozcdO89aGR5BI85EVguzWbCRR1xaV00yVdPSF9uS5am2oZCvtHmza2MrQhT6wnLLMJ+", + "EJvqczhXPMWKRlD5KJpxLj2yL+CRTY0yazwuKBNMK0bLtRkEF5pUL6wZ+sKqEQfWZIaw/8j20YfPbd5B", + "+Av3qPziJ88qUHD8rhP9oToAlGYXlExQhnNJtFSXpwRFi0hzRVPqiuBohiKcqVwQqOJHUEoZTfPUx73W", + "OzbHgNFxsZ1edNE4VyjBYgpamXnogm0inqaExQTsc0M2I3hOtUopUIIVYdGiJwlU/50TdMXFZcJxDCaG", + "LMbg6YHqgYJoCgQQ8ZQoHGOFQdC50Cd+ZJKYLoqCwEatZ+S6pIZ4yETOvjcVDXSzF26gF4gAZDeVs6Jw", + "ZIRjwqIglPX5183Gvrwt+pyo+kQfKDLoVrz0IUOFfJurG87XEUX0yGKxubDb2IbNrxB6ZbMKW83+cGT0", + "73mkzVzdHB/IwVQs8apT/HV4lgqi+2q8Sw/vPuICxbnpzjuVQOZ/Vp9QwVD8YCvILDXbeFvHUFEhr1jm", + "G/G8rT/cnye3sOV9JZyw26jYN9ViKif9NbBcu6q34rkPZMS0tiTfJvdwLNhFdD2Y+MSFx+Uei7HVMmxz", + "NAu+7XMnJTBoX5x9Y9t1tm0DHm7Ltp1tdsml7zFyynoQIxrm4NaM28iqreng3zQbpTY7j2U+OIssPRf3", + "xhZPCkZoWGOGFwnH8Z8hSHiF/yjiQhj4CwDUeEzwq57V0E8PANtcWeSt67I135+ebjZxCaFW8gihHjGH", + "8FJy9GdpvGzAfT0nQtDYopSio9NjG65LJRI566PXKVVIcXRJSFZmtEBWYV/PzwGBLBeUryB+dDuEKbHI", + "OGVq7SjKV+9mMJ9uVYb+nvmkxfP+5g5v7Q4Hy/7jY2fAZSBnw0xgtWaqsFpbZ5SyCRepkcvwmOe6dc2D", + "9DLp/TRIBROaELmQiqQmKnGSJ3DcoDaErf9rvzO73IWYXH1yTLpcRkRKpaScySGzuSIZEbpv/blu3wuw", + "CjoEFC7465lhkl9H8J4ejIlXw6pp1QCyCeqKdg46WzjLtmKscEOAmB3eZwzpJ4jGQ3KRjnlCI5RQdinR", + "RkIvjXqC5hIl+o/NleF8I/juS1c3vv3J0it9wiY8WDvO0GxBzH+qrC7L1pxj8tGxtZfEPyyO/8BGh9na", + "+vrJguCkB/WIHXAPyhVN6EfD6nQjVCoamZQjXKzd+9OCqfaH7JQood/BkNqWJAbRALTLrUzwaGuYDwa7", + "UUYB/W2XwOCA4TU/TqHHo7N3Jg2VpFwsukOm/wENvz08M97dCbbWBG+gtnAyOtl6vSbA+RyW6d84QtBM", + "cCV6QXDDv7kEb44x0niGZMMR5dkqVYlnf/oQVivBfbMrPE67AoA8FbPZKIC9HBpX2IYw50me6n+YP07W", + "4ZopHM3ew6tfjbRrhrO2GzfBR3Eo7ZxiYmpbPojTwyzYY41Z1QvnpgBCTCUaMHgLHKo/I3V/efO9v45f", + "obvTrqirG/vVnK37vvnsGBzChr8ej+WYG0pzM1F8tfXpCtNm69OPCY8upYVi8c2GWm8DfHX9Y4mHbV2E", + "ICZAZiiyEEYGKIvI7pDVDJAG8UcijBQRKWU42YI5m0YA2dtZsfCcU0jQjiBPpSdpDJhJCcB3A/ydng0Y", + "qlwDnkdX2spa/ju+M1JxNCYRT4lDO98MqW5/w1T9xEUVuvxr4YtvvfUHSEBMwd6+Bq29ucfPQm8/xdcQ", + "Kh3n1qHsRrTxkpc/GlNQF8HeDDu7AznsdNGws5MOO3oHjjCYULFC+yilLFdE9tGxsW9BCu6TAZIk4iyW", + "DnTdWfB2B7IpIdeQZUN25xP47j7FHktVsJRvbCch9qDfQ/p7SNpBG/6Bs2cy7sKhixHPlTH323Nl34qJ", + "AvPI5r37ar0z8k23b8PJ/2aPb4VHwS5rdultveHsWS5npNnk9soUMsrVGMC8XXFROUN/52PZRYxcGWu4", + "kKq/xPf012emg/soNKC7ukmRATv3bxUGWlQYKNcqDNZoAiz1leyowyA2kuuMCwUojjbX3tAQaBKAHMEj", + "nKDXRydDFmlWZKAFBUk5cCeLh25u4cO/naMXR2+66BgKXaKf8/FmH71mycKVGzc+miEzkphhXhFmaGyo", + "lsSh69mMHajnLoPFdQcPVDnanIyAZ8XtlQsS73ZmBMcgkfzRecVNZwHU4Tev9AEC4F/zZbHtnZXCR+cN", + "UWLRO5woIpabPbV5UqzAzLCXtIOgs4KbAb7UHUqHvFb2aWQDA42xu9MJIGV8+lb04e4LpN6Pl8zEiZhy", + "e+MckEYZJBngePG4YpnkDBXMMcQC/eu6KJvQlCVsedlKBQO6bIr8/opM7it5VwVb/t/1dMFMH62jKavs", + "kybiotzKWk+vSw6eGThk66iKcIYjqhZdhJPE3lH2JigiUnqF+DsWBF/G/Ir1h+xNUejFJvSio7N3Xeeo", + "RTGVl6YF64vto9dzImQ+LgaH4KAZrzGsOYmHTHEU4STKEy1ukMmERJCLC/VbZIMvtxhK5w7PTtlJsNiM", + "F9WeP7oad2GagN0ryaJOcVtmq7cEiRJM02bwcSuoQcAhhBqMdaOcIcomiQ2pigSXEtmmeiShUzpObICQ", + "7KO3M4IkTsmQZQlmjAiUSxMVr4feywSRMjcJ3roBAOk1FNVFJbBgJriyoQkJ50KaaAJN4e9PkVQkW0Fm", + "b0zLpzDnO5JtTeO2pwcyUtfG0GwKsa8gvSGGUsyCazrKExfAeK+h6GZADy0lPpaD/1bQ6ZQIfSqwYbIm", + "HM8ca7ec5tBXMpYb612eF2+1q3dZtOplJXoZeyuB4UYl1nbcuVnUX6DzS9qIHWgf3SyL+Bf9Ucu+q9mq", + "4UHYR585y1Dpzn/HKpnnXpJgWwNWSeGPzZzkjbxyVCuJtuthtVpn1t5lpmtr/KwHg816zGhZuJI+26Tw", + "fn2EMLhflIf7LrL2uGmrgnZV0U0bUv7Xo+l/FRR4NzD6D4xycgsY/a8q7x5wzh8O/yR4UB8qj77ie3bF", + "dv/0SPh3lT5v4PABjq0pfd5wPRu8ulJRem/faacm2Rb/TBK8jXe8gfzulv2b1t9CZfAWa50LWhM8STO1", + "cAFt1ldZBp1J+pH0GxzBRdzq3bmCbxHS+eXIw9FpY0Dnn7M2/oPEjNrSgVSik+NA0flHhjHon7nKxbKl", + "b50eFtGMzkmz0b16gu0SZYL0Mp6BcyU2C2bXw91lCov+9COyzVvMVfsvqD0JUP0kRjEVJFLJwtQB1RzB", + "9PGdRIJrTQCec7FojhIxR+QnwdNDO5s196E9U9YYVsYZpotejBXuzR23WWFC+4zoThdPqRkeogy9/BFt", + "kGslTIULNNGaD6KTYknJdURILIEmN/0Bbw8aLJv0IxlNx21GuaJWyWtbCwZFuVQ8dXt/cow2oPbZlDC9", + "F1rUn4Akmwk+pzGJK2PszHliVnW7YUFvanfVQkVRuM4pF2ZwDyLDtLmQph9pVmULRUjMmDIMg1tbFaR6", + "pkwSv+4PU+YCcOweuVF8u8Ks5rfhlB1NiVCH0y6i4txAPG9+u+Ye8zXnJ0O5O61y27nwnNXG63b5US3T", + "lu6i8EORO3e/Zuv3X09KD5WPMpvHms7nhULaZDb/ukhwcH/3w32by98/4hTQl8Qp356pHBrQLYYI5hXE", + "dMdkThKepVAPHd7tdDu5SDoHnZlS2cHWFsR+z7hUB3vPn+52Pn349P8HAAD//19ApuOy8gEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/paths/paths.go b/lib/paths/paths.go index 814dc1432..833a7a911 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -343,6 +343,11 @@ func (p *Paths) DeviceMetadata(id string) string { return filepath.Join(p.DeviceDir(id), "metadata.json") } +// VFHealthState returns the path to the persisted vGPU VF health file. +func (p *Paths) VFHealthState() string { + return filepath.Join(p.dataDir, "gpu", "vf-health.json") +} + // Volume path methods // VolumesDir returns the root volumes directory. diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 054e3744e..2c7843532 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -10,16 +10,19 @@ import ( // GPUResourceStatus represents the GPU resource status for the API response. // Returns nil if no GPU is available on the host. type GPUResourceStatus struct { - Mode string `json:"mode"` // "vgpu" or "passthrough" - TotalSlots int `json:"total_slots"` // VFs for vGPU, physical GPUs for passthrough - UsedSlots int `json:"used_slots"` // Slots currently in use - Profiles []devices.GPUProfile `json:"profiles,omitempty"` // vGPU mode only - Devices []devices.PassthroughDevice `json:"devices,omitempty"` // passthrough mode only + Mode string `json:"mode"` // "vgpu" or "passthrough" + TotalSlots int `json:"total_slots"` // VFs for vGPU, physical GPUs for passthrough + UsedSlots int `json:"used_slots"` // Slots currently in use, including assigned quarantined VFs + AllocatableSlots int `json:"allocatable_slots"` // Healthy free slots used by admission control + QuarantinedSlots int `json:"quarantined_slots"` // Quarantined VFs; may overlap UsedSlots + Profiles []devices.GPUProfile `json:"profiles,omitempty"` // vGPU mode only + Devices []devices.PassthroughDevice `json:"devices,omitempty"` // passthrough mode only } -// GetGPUStatus returns the current GPU resource status. -// Returns nil if no GPU is available or the mode is "none". -func GetGPUStatus(ctx context.Context) *GPUResourceStatus { +// GetGPUStatus returns the current GPU resource status and any error that +// prevents determining allocatable vGPU capacity. It returns nil if no GPU is +// available or the mode is "none". +func GetGPUStatus(ctx context.Context) (*GPUResourceStatus, error) { framework, vfs, err := devices.DiscoverVGPU() if err != nil { // Only report passthrough once vGPU discovery confirms no vGPU @@ -27,16 +30,16 @@ func GetGPUStatus(ctx context.Context) *GPUResourceStatus { // expose the PFs/VFs as available passthrough slots while active vGPU // assignments exist. logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU state", "error", err) - return nil + return nil, nil } if framework != devices.VGPUFrameworkNone { return getVGPUStatus(ctx, framework, vfs) } - return getPassthroughStatus() + return getPassthroughStatus(), nil } // getVGPUStatus returns GPU status for vGPU mode (SR-IOV). -func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []devices.VirtualFunction) *GPUResourceStatus { +func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []devices.VirtualFunction) (*GPUResourceStatus, error) { usedSlots := 0 // Count used VFs (those with a vGPU assigned) for _, vf := range vfs { @@ -51,13 +54,20 @@ func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []d logger.FromContext(ctx).WarnContext(ctx, "failed to list vGPU profiles; reporting none", "framework", framework, "error", err) profiles = nil } - - return &GPUResourceStatus{ - Mode: string(devices.GPUModeVGPU), - TotalSlots: len(vfs), - UsedSlots: usedSlots, - Profiles: profiles, + allocatableSlots, quarantinedSlots, err := devices.VGPUAvailability(framework, vfs) + status := &GPUResourceStatus{ + Mode: string(devices.GPUModeVGPU), + TotalSlots: len(vfs), + UsedSlots: usedSlots, + AllocatableSlots: allocatableSlots, + QuarantinedSlots: quarantinedSlots, + Profiles: profiles, + } + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to count allocatable vGPU slots; reporting none", "framework", framework, "error", err) + status.AllocatableSlots = 0 } + return status, err } // getPassthroughStatus returns GPU status for whole-GPU passthrough mode. @@ -92,9 +102,10 @@ func getPassthroughStatus() *GPUResourceStatus { } return &GPUResourceStatus{ - Mode: string(devices.GPUModePassthrough), - TotalSlots: len(passthroughDevices), - UsedSlots: usedSlots, - Devices: passthroughDevices, + Mode: string(devices.GPUModePassthrough), + TotalSlots: len(passthroughDevices), + UsedSlots: usedSlots, + AllocatableSlots: len(passthroughDevices) - usedSlots, + Devices: passthroughDevices, } } diff --git a/lib/resources/gpu_test.go b/lib/resources/gpu_test.go new file mode 100644 index 000000000..825a20260 --- /dev/null +++ b/lib/resources/gpu_test.go @@ -0,0 +1,88 @@ +package resources + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func initVFHealthForTest(t *testing.T, state []byte) { + t.Helper() + dataDir := t.TempDir() + if state != nil { + path := paths.New(dataDir).VFHealthState() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, state, 0o644)) + } + devices.NewManager(paths.New(dataDir)) + resetDir := t.TempDir() + t.Cleanup(func() { devices.NewManager(paths.New(resetDir)) }) +} + +func TestGetVGPUStatusFailsClosedWhenVFHealthIsUnavailable(t *testing.T) { + initVFHealthForTest(t, []byte("not json")) + + status, err := getVGPUStatus(context.Background(), devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + assert.Zero(t, status.AllocatableSlots) + assert.Zero(t, status.QuarantinedSlots) + require.ErrorContains(t, err, "VF health state unavailable") +} + +func TestGetVGPUStatusReportsQuarantinedSlots(t *testing.T) { + initVFHealthForTest(t, nil) + for _, instance := range []string{"instance-1", "instance-2"} { + _, err := devices.ReportVFInitFailure(devices.VFInitFailureReport{VFAddress: "0000:82:00.4", InstanceID: instance}) + require.NoError(t, err) + } + + status, err := getVGPUStatus(context.Background(), devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4"}, + {PCIAddress: "0000:82:00.5", Allocated: true}, + {PCIAddress: "0000:82:00.6"}, + }) + require.NoError(t, err) + assert.Equal(t, 3, status.TotalSlots) + assert.Equal(t, 1, status.UsedSlots) + assert.Equal(t, 1, status.AllocatableSlots) + assert.Equal(t, 1, status.QuarantinedSlots) +} + +func TestReserveAllocationUsesAllocatableGPUSlots(t *testing.T) { + status := &GPUResourceStatus{ + Mode: string(devices.GPUModeVGPU), + TotalSlots: 4, + UsedSlots: 1, + AllocatableSlots: 0, + } + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return status, nil }) + t.Cleanup(func() { setGPUStatusProvider(nil) }) + + mgr := NewManager(&config.Config{}, paths.New(t.TempDir())) + ctx := context.Background() + + err := mgr.ValidateAllocation(ctx, 0, 0, 0, 0, 0, 0, true) + require.ErrorContains(t, err, "no allocatable vgpu slots") + + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { + return status, errors.New("VF health state unavailable: read failed") + }) + err = mgr.ValidateAllocation(ctx, 0, 0, 0, 0, 0, 0, true) + require.ErrorContains(t, err, "vGPU placement is disabled: VF health state unavailable") + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return status, nil }) + + status.AllocatableSlots = 1 + require.NoError(t, mgr.ReserveAllocation(ctx, "pending-a", 0, 0, 0, 0, 0, 0, true)) + err = mgr.ReserveAllocation(ctx, "pending-b", 0, 0, 0, 0, 0, 0, true) + require.ErrorContains(t, err, "no allocatable vgpu slots") + + mgr.FinishAllocation("pending-a") + require.NoError(t, mgr.ReserveAllocation(ctx, "pending-b", 0, 0, 0, 0, 0, 0, true)) +} diff --git a/lib/resources/monitoring_test.go b/lib/resources/monitoring_test.go index bef0740dc..39166856e 100644 --- a/lib/resources/monitoring_test.go +++ b/lib/resources/monitoring_test.go @@ -198,7 +198,7 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { mgr, _, _ := monitoringTestManager(t) originalProvider := currentGPUStatusProvider() - setGPUStatusProvider(func(context.Context) *GPUResourceStatus { + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return &GPUResourceStatus{ Mode: "vgpu", TotalSlots: 8, @@ -207,7 +207,7 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { {Name: "L40S-1Q", Available: 5}, {Name: "L40S-2Q", Available: 2}, }, - } + }, nil }) defer func() { setGPUStatusProvider(originalProvider) diff --git a/lib/resources/resource.go b/lib/resources/resource.go index 86f644bda..9c28fda0f 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -37,13 +37,13 @@ var ( gpuStatusProvider = GetGPUStatus ) -func currentGPUStatusProvider() func(context.Context) *GPUResourceStatus { +func currentGPUStatusProvider() func(context.Context) (*GPUResourceStatus, error) { gpuStatusProviderMu.RLock() defer gpuStatusProviderMu.RUnlock() return gpuStatusProvider } -func setGPUStatusProvider(fn func(context.Context) *GPUResourceStatus) { +func setGPUStatusProvider(fn func(context.Context) (*GPUResourceStatus, error)) { if fn == nil { fn = GetGPUStatus } @@ -427,7 +427,7 @@ func (m *Manager) GetFullStatus(ctx context.Context) (*FullResourceStatus, error } // Get GPU status - gpuStatus := currentGPUStatusProvider()(ctx) + gpuStatus, _ := currentGPUStatusProvider()(ctx) return &FullResourceStatus{ CPU: *cpuStatus, @@ -691,15 +691,18 @@ func (m *Manager) validateAllocationLocked(ctx context.Context, excludeID string // Check GPU if needed if req.GPUSlots > 0 { - gpuStatus := currentGPUStatusProvider()(ctx) + gpuStatus, gpuStatusErr := currentGPUStatusProvider()(ctx) if gpuStatus == nil { return fmt.Errorf("insufficient GPU: no GPU available on this host") } - availableSlots := gpuStatus.TotalSlots - gpuStatus.UsedSlots - pending.GPUSlots + availableSlots := gpuStatus.AllocatableSlots - pending.GPUSlots if availableSlots < req.GPUSlots { + if gpuStatusErr != nil { + return fmt.Errorf("insufficient GPU: vGPU placement is disabled: %w", gpuStatusErr) + } if availableSlots <= 0 { - return fmt.Errorf("insufficient GPU: all %d %s slots are in use", - gpuStatus.TotalSlots, gpuStatus.Mode) + return fmt.Errorf("insufficient GPU: no allocatable %s slots available (%d total, %d in use)", + gpuStatus.Mode, gpuStatus.TotalSlots, gpuStatus.UsedSlots) } return fmt.Errorf("insufficient GPU: requested %d %s slot(s), but only %d available", req.GPUSlots, gpuStatus.Mode, availableSlots) diff --git a/openapi.yaml b/openapi.yaml index c33c796d8..cea2abe3f 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1800,7 +1800,7 @@ components: type: object description: GPU resource status. Null if no GPUs available. nullable: true - required: [mode, total_slots, used_slots] + required: [mode, total_slots, used_slots, allocatable_slots, quarantined_slots] properties: mode: type: string @@ -1813,8 +1813,16 @@ components: example: 64 used_slots: type: integer - description: Slots currently in use + description: Slots currently in use. Includes quarantined VFs that are still assigned, so this can overlap quarantined_slots. example: 5 + allocatable_slots: + type: integer + description: Free slots eligible for placement, matching admission control (excludes quarantined VFs; 0 while VF health state is unavailable) + example: 57 + quarantined_slots: + type: integer + description: VFs quarantined after guest driver init failures (vGPU mode only). May overlap used_slots until the affected instance releases its VF. + example: 2 profiles: type: array description: Available vGPU profiles (only in vGPU mode) From def61def247337d0340e4340ea0a4fe69a5538ce Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:56:12 +0000 Subject: [PATCH 103/107] Fail vGPU placement closed on VF health persist failures A failed state write previously rolled memory back and left the store reporting healthy, so a VF whose threshold-crossing failure could not be persisted stayed allocatable. Latch write failures and refuse placement until a later write succeeds; re-reported markers retry the write. Also make acknowledged reports crash-durable (fsync the parent when the state dir is first created, treat directory sync failures as persist failures instead of logging success), and re-evaluate persisted tallies against the configured threshold at load and on threshold changes so a lowered gpu.vf_quarantine_threshold applies to existing failures. --- lib/devices/GPU.md | 7 ++- lib/devices/vf_health.go | 85 +++++++++++++++++++++++++++-------- lib/devices/vf_health_test.go | 73 ++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 21 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 17c387723..678871562 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -300,8 +300,11 @@ equivalent free VFs is randomized so a wedged VF cannot capture every placement. A reported init success clears failures only when that exact assignment has a recorded failure, removing the match and older tallies; if that assignment crossed the threshold, its later success also rescinds the -quarantine. If the state file exists but cannot be loaded, placement and -advertised availability fail closed until it is repaired or removed. +quarantine. If the state file exists but cannot be loaded, or the last write +to it failed, placement and advertised availability fail closed until a load +or write succeeds. Recorded tallies are re-evaluated against the configured +threshold at load, so lowering `gpu.vf_quarantine_threshold` quarantines VFs +whose persisted failures already meet the new value. `used_slots` includes quarantined VFs still held by running instances, so it can overlap `quarantined_slots`; use `allocatable_slots` for admission. diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go index b47ff11a5..b0870e212 100644 --- a/lib/devices/vf_health.go +++ b/lib/devices/vf_health.go @@ -75,11 +75,12 @@ type VFSuccessResult struct { } type vfHealthStore struct { - mu sync.Mutex - path string - records map[string]vfHealthRecord - threshold int - loadErr error + mu sync.Mutex + path string + records map[string]vfHealthRecord + threshold int + loadErr error + persistErr error } var vfHealthAddressPattern = regexp.MustCompile(`^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$`) @@ -97,16 +98,39 @@ func initVFHealth(path string) error { } // SetVFQuarantineThreshold configures the number of failed assignments -// required to quarantine a VF. +// required to quarantine a VF. Already-recorded tallies are re-evaluated so a +// lowered threshold applies to failures persisted before the change. func SetVFQuarantineThreshold(n int) { vfHealth.mu.Lock() defer vfHealth.mu.Unlock() vfHealth.threshold = n + vfHealth.requarantineLocked() +} + +// requarantineLocked quarantines records whose failure tallies meet the +// current threshold, so threshold changes and loaded state agree. +func (s *vfHealthStore) requarantineLocked() { + changed := false + for address, record := range s.records { + if record.QuarantinedAt != nil || len(record.Failures) < s.threshold { + continue + } + now := time.Now().UTC() + record.QuarantinedAt = &now + s.records[address] = record + changed = true + } + if changed { + if err := s.persistLocked(); err != nil { + slog.Default().Error("failed to persist re-evaluated VF quarantines; vGPU placement is disabled until a write succeeds", "error", err) + } + } } func (s *vfHealthStore) loadLocked() error { s.records = make(map[string]vfHealthRecord) s.loadErr = nil + s.persistErr = nil data, err := os.ReadFile(s.path) if err != nil { @@ -163,6 +187,7 @@ func (s *vfHealthStore) loadLocked() error { loaded[record.VFAddress] = record } s.records = loaded + s.requarantineLocked() return nil } @@ -179,6 +204,9 @@ func (s *vfHealthStore) checkedAddresses() (map[string]struct{}, error) { if err := s.ensureLoadedLocked(); err != nil { return nil, fmt.Errorf("VF health state unavailable: %w", err) } + if s.persistErr != nil { + return nil, fmt.Errorf("VF health state unavailable: last write failed: %w", s.persistErr) + } addresses := make(map[string]struct{}, len(s.records)) for address, record := range s.records { if record.QuarantinedAt != nil { @@ -240,11 +268,12 @@ func ReportVFInitSuccess(report VFInitSuccessReport) (VFSuccessResult, error) { return vfHealth.reportSuccess(report) } -// VFHealthStoreUnavailable reports whether persisted state failed to load. +// VFHealthStoreUnavailable reports whether persisted state failed to load or +// the last write failed. func VFHealthStoreUnavailable() bool { vfHealth.mu.Lock() defer vfHealth.mu.Unlock() - return vfHealth.loadErr != nil + return vfHealth.loadErr != nil || vfHealth.persistErr != nil } // TotalQuarantinedVFs returns the number of quarantined VFs in persisted state. @@ -365,10 +394,19 @@ func (s *vfHealthStore) reportSuccess(report VFInitSuccessReport) (VFSuccessResu return result, nil } +// persistLocked writes the current records to disk. A failure is latched and +// fails placement closed until a later write succeeds, because in-memory +// rollback alone would leave a reported-unhealthy VF allocatable. func (s *vfHealthStore) persistLocked() error { if s.path == "" { return nil } + err := s.writeStateLocked() + s.persistErr = err + return err +} + +func (s *vfHealthStore) writeStateLocked() error { data, err := json.MarshalIndent(vfHealthFile{ Version: vfHealthFileVersion, Records: s.sortedRecordsLocked(), @@ -376,8 +414,15 @@ func (s *vfHealthStore) persistLocked() error { if err != nil { return fmt.Errorf("marshal VF health state: %w", err) } - if err := os.MkdirAll(filepath.Dir(s.path), 0755); err != nil { - return fmt.Errorf("create VF health state dir: %w", err) + dirPath := filepath.Dir(s.path) + if _, err := os.Stat(dirPath); os.IsNotExist(err) { + if err := os.MkdirAll(dirPath, 0755); err != nil { + return fmt.Errorf("create VF health state dir: %w", err) + } + // Make the new directory entry itself durable. + if err := syncDir(filepath.Dir(dirPath)); err != nil { + return fmt.Errorf("sync VF health state parent dir: %w", err) + } } tmp := s.path + ".tmp" f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) @@ -402,15 +447,17 @@ func (s *vfHealthStore) persistLocked() error { os.Remove(tmp) return fmt.Errorf("rename VF health state: %w", err) } - dirPath := filepath.Dir(s.path) - dir, err := os.Open(dirPath) - if err != nil { - slog.Default().Warn("failed to open VF health state directory for sync", "path", dirPath, "error", err) - return nil - } - if err := dir.Sync(); err != nil { - slog.Default().Warn("failed to sync VF health state directory", "path", dirPath, "error", err) + if err := syncDir(dirPath); err != nil { + return fmt.Errorf("sync VF health state dir: %w", err) } - _ = dir.Close() return nil } + +func syncDir(path string) error { + dir, err := os.Open(path) + if err != nil { + return err + } + defer dir.Close() + return dir.Sync() +} diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go index dd286c296..75d8fbabf 100644 --- a/lib/devices/vf_health_test.go +++ b/lib/devices/vf_health_test.go @@ -1,6 +1,7 @@ package devices import ( + "encoding/json" "os" "path/filepath" "testing" @@ -20,6 +21,7 @@ func resetVFHealthStore(t *testing.T) string { vfHealth.records = make(map[string]vfHealthRecord) vfHealth.threshold = defaultVFQuarantineThreshold vfHealth.loadErr = nil + vfHealth.persistErr = nil }) return path } @@ -92,6 +94,77 @@ func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { assert.Zero(t, quarantined) } +func TestVGPUAvailabilityFailsClosedAfterPersistFailure(t *testing.T) { + resetVFHealthStore(t) + SetVFQuarantineThreshold(1) + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0o644)) + goodPath := vfHealth.path + vfHealth.path = filepath.Join(blocker, "vf-health.json") + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.Error(t, err) + assert.True(t, VFHealthStoreUnavailable()) + _, _, err = VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) + require.ErrorContains(t, err, "last write failed") + + vfHealth.path = goodPath + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.NoError(t, err) + assert.Equal(t, VFReportQuarantined, result.Outcome) + assert.False(t, VFHealthStoreUnavailable()) + + available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) + require.NoError(t, err) + assert.Zero(t, available) + assert.Equal(t, 1, quarantined) +} + +func TestSetVFQuarantineThresholdReevaluatesRecordedFailures(t *testing.T) { + path := resetVFHealthStore(t) + SetVFQuarantineThreshold(3) + for _, instance := range []string{"instance-1", "instance-2"} { + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) + } + + SetVFQuarantineThreshold(2) + + records := quarantinedVFs() + require.Len(t, records, 1) + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) + + data, err := os.ReadFile(path) + require.NoError(t, err) + var state vfHealthFile + require.NoError(t, json.Unmarshal(data, &state)) + require.Len(t, state.Records, 1) + assert.NotNil(t, state.Records[0].QuarantinedAt, "the re-evaluated quarantine must be persisted") +} + +func TestLoadReevaluatesTalliesAgainstConfiguredThreshold(t *testing.T) { + path := resetVFHealthStore(t) + SetVFQuarantineThreshold(3) + for _, instance := range []string{"instance-1", "instance-2"} { + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) + } + + // Simulate a restart where the threshold is configured lower before the + // persisted tallies are loaded. + vfHealth.mu.Lock() + vfHealth.records = make(map[string]vfHealthRecord) + vfHealth.threshold = 2 + vfHealth.mu.Unlock() + require.NoError(t, initVFHealth(path)) + + records := quarantinedVFs() + require.Len(t, records, 1) + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) +} + func TestReportVFInitFailureQuarantinesAtThreshold(t *testing.T) { path := resetVFHealthStore(t) From 4527e647c5284d711467503f06ad7b94d3c0334d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:51:36 +0000 Subject: [PATCH 104/107] Preserve quarantines across sync failures --- lib/devices/vendor_vfio_linux.go | 119 ++++++++++++++++++------------- lib/devices/vf_health.go | 93 +++++++++++++++--------- lib/devices/vf_health_test.go | 75 +++++++++++++++++++ 3 files changed, 203 insertions(+), 84 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index b923439a3..0d8a27383 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -30,6 +30,26 @@ type vendorVFIOOwner struct { assignedAt time.Time } +type vendorVFIOGPUPlacement struct { + usage int + unknownUsage bool + quarantined int + freeVFs []VirtualFunction +} + +func (p *vendorVFIOGPUPlacement) preferredTo(other *vendorVFIOGPUPlacement, gpu, otherGPU string) bool { + if p.quarantined != other.quarantined { + return p.quarantined < other.quarantined + } + if p.unknownUsage != other.unknownUsage { + return !p.unknownUsage + } + if p.usage != other.usage { + return p.usage < other.usage + } + return gpu < otherGPU +} + type vendorVFIOSysfs struct { pciDevicesPath string procPath string @@ -317,71 +337,70 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map return nil } +func (s vendorVFIOSysfs) addVFToPlacement(placement *vendorVFIOGPUPlacement, vf VirtualFunction, profileType string, quarantined bool) { + if quarantined { + placement.quarantined++ + if !vf.Allocated { + return + } + } + if vf.Allocated { + // framebufferByType only covers currently creatable profiles, so + // after a restart an allocated type can be missing when its + // capacity is exhausted. Prefer GPUs whose load is fully known + // instead of rejecting placement outright; the kernel driver + // still enforces real capacity through creatable_vgpu_types. + framebuffer, ok := s.framebufferByType[vf.ProfileType] + if !ok { + placement.unknownUsage = true + return + } + placement.usage += framebuffer + return + } + profiles, err := s.readCreatableProfiles(vf.PCIAddress) + if err != nil { + // An unreadable free VF is just not a placement candidate. + slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) + return + } + for _, profile := range profiles { + if profile.TypeName == profileType { + placement.freeVFs = append(placement.freeVFs, vf) + return + } + } +} + func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType string) (string, error) { quarantined, err := vfHealth.checkedAddresses() if err != nil { return "", err } - usageByGPU := make(map[string]int) - unknownUsageByGPU := make(map[string]bool) - quarantinedByGPU := make(map[string]int) - freeByGPU := make(map[string][]VirtualFunction) + placementByGPU := make(map[string]*vendorVFIOGPUPlacement) for _, vf := range vfs { - _, bad := quarantined[vf.PCIAddress] - if bad { - quarantinedByGPU[vf.ParentGPU]++ - if !vf.Allocated { - continue - } - } - if vf.Allocated { - // framebufferByType only covers currently creatable profiles, so - // after a restart an allocated type can be missing when its - // capacity is exhausted. Prefer GPUs whose load is fully known - // instead of rejecting placement outright; the kernel driver - // still enforces real capacity through creatable_vgpu_types. - framebuffer, ok := s.framebufferByType[vf.ProfileType] - if !ok { - unknownUsageByGPU[vf.ParentGPU] = true - continue - } - usageByGPU[vf.ParentGPU] += framebuffer - continue - } - profiles, err := s.readCreatableProfiles(vf.PCIAddress) - if err != nil { - // An unreadable free VF is just not a placement candidate. - slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) - continue - } - for _, profile := range profiles { - if profile.TypeName == profileType { - freeByGPU[vf.ParentGPU] = append(freeByGPU[vf.ParentGPU], vf) - break - } + placement := placementByGPU[vf.ParentGPU] + if placement == nil { + placement = &vendorVFIOGPUPlacement{} + placementByGPU[vf.ParentGPU] = placement } + _, bad := quarantined[vf.PCIAddress] + s.addVFToPlacement(placement, vf, profileType, bad) } - gpus := make([]string, 0, len(freeByGPU)) - for gpu := range freeByGPU { - gpus = append(gpus, gpu) + gpus := make([]string, 0, len(placementByGPU)) + for gpu, placement := range placementByGPU { + if len(placement.freeVFs) > 0 { + gpus = append(gpus, gpu) + } } sort.Slice(gpus, func(i, j int) bool { - if quarantinedByGPU[gpus[i]] != quarantinedByGPU[gpus[j]] { - return quarantinedByGPU[gpus[i]] < quarantinedByGPU[gpus[j]] - } - if unknownUsageByGPU[gpus[i]] != unknownUsageByGPU[gpus[j]] { - return !unknownUsageByGPU[gpus[i]] - } - if usageByGPU[gpus[i]] == usageByGPU[gpus[j]] { - return gpus[i] < gpus[j] - } - return usageByGPU[gpus[i]] < usageByGPU[gpus[j]] + return placementByGPU[gpus[i]].preferredTo(placementByGPU[gpus[j]], gpus[i], gpus[j]) }) if len(gpus) == 0 { return "", nil } - candidates := freeByGPU[gpus[0]] + candidates := placementByGPU[gpus[0]].freeVFs pick := s.pickVFIndex if pick == nil { pick = rand.IntN diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go index b0870e212..479696661 100644 --- a/lib/devices/vf_health.go +++ b/lib/devices/vf_health.go @@ -75,18 +75,23 @@ type VFSuccessResult struct { } type vfHealthStore struct { - mu sync.Mutex - path string - records map[string]vfHealthRecord - threshold int - loadErr error - persistErr error + mu sync.Mutex + path string + records map[string]vfHealthRecord + threshold int + loadErr error + persistErr error + syncDirFunc func(string) error } var vfHealthAddressPattern = regexp.MustCompile(`^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$`) var ( - vfHealth = &vfHealthStore{records: make(map[string]vfHealthRecord), threshold: defaultVFQuarantineThreshold} + vfHealth = &vfHealthStore{ + records: make(map[string]vfHealthRecord), + threshold: defaultVFQuarantineThreshold, + syncDirFunc: syncDir, + } vendorVFIOMu sync.Mutex ) @@ -121,7 +126,7 @@ func (s *vfHealthStore) requarantineLocked() { changed = true } if changed { - if err := s.persistLocked(); err != nil { + if _, err := s.persistLocked(); err != nil { slog.Default().Error("failed to persist re-evaluated VF quarantines; vGPU placement is disabled until a write succeeds", "error", err) } } @@ -307,6 +312,9 @@ func (s *vfHealthStore) reportFailure(report VFInitFailureReport) (VFReportResul if !vfHealthAddressPattern.MatchString(report.VFAddress) { return VFReportResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress) } + if err := s.retryPersistLocked(); err != nil { + return VFReportResult{}, err + } previous, existed := s.records[report.VFAddress] result := VFReportResult{Failures: len(previous.Failures), Threshold: s.threshold} @@ -335,11 +343,14 @@ func (s *vfHealthStore) reportFailure(report VFInitFailureReport) (VFReportResul result.Outcome = VFReportQuarantined } s.records[report.VFAddress] = record - if err := s.persistLocked(); err != nil { - if existed { - s.records[report.VFAddress] = previous - } else { - delete(s.records, report.VFAddress) + renamed, err := s.persistLocked() + if err != nil { + if !renamed { + if existed { + s.records[report.VFAddress] = previous + } else { + delete(s.records, report.VFAddress) + } } return VFReportResult{}, err } @@ -359,6 +370,9 @@ func (s *vfHealthStore) reportSuccess(report VFInitSuccessReport) (VFSuccessResu if !vfHealthAddressPattern.MatchString(report.VFAddress) { return VFSuccessResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress) } + if err := s.retryPersistLocked(); err != nil { + return VFSuccessResult{}, err + } previous, ok := s.records[report.VFAddress] if !ok || len(previous.Failures) == 0 { return VFSuccessResult{}, nil @@ -387,70 +401,81 @@ func (s *vfHealthStore) reportSuccess(report VFInitSuccessReport) (VFSuccessResu record.Failures = remaining s.records[report.VFAddress] = record } - if err := s.persistLocked(); err != nil { - s.records[report.VFAddress] = previous + renamed, err := s.persistLocked() + if err != nil { + if !renamed { + s.records[report.VFAddress] = previous + } return VFSuccessResult{}, err } return result, nil } +func (s *vfHealthStore) retryPersistLocked() error { + if s.persistErr == nil { + return nil + } + _, err := s.persistLocked() + return err +} + // persistLocked writes the current records to disk. A failure is latched and -// fails placement closed until a later write succeeds, because in-memory -// rollback alone would leave a reported-unhealthy VF allocatable. -func (s *vfHealthStore) persistLocked() error { +// fails placement closed until a later write succeeds. The returned boolean +// reports whether the rename made the new state visible. +func (s *vfHealthStore) persistLocked() (bool, error) { if s.path == "" { - return nil + return false, nil } - err := s.writeStateLocked() + renamed, err := s.writeStateLocked() s.persistErr = err - return err + return renamed, err } -func (s *vfHealthStore) writeStateLocked() error { +func (s *vfHealthStore) writeStateLocked() (bool, error) { data, err := json.MarshalIndent(vfHealthFile{ Version: vfHealthFileVersion, Records: s.sortedRecordsLocked(), }, "", " ") if err != nil { - return fmt.Errorf("marshal VF health state: %w", err) + return false, fmt.Errorf("marshal VF health state: %w", err) } dirPath := filepath.Dir(s.path) if _, err := os.Stat(dirPath); os.IsNotExist(err) { if err := os.MkdirAll(dirPath, 0755); err != nil { - return fmt.Errorf("create VF health state dir: %w", err) + return false, fmt.Errorf("create VF health state dir: %w", err) } // Make the new directory entry itself durable. - if err := syncDir(filepath.Dir(dirPath)); err != nil { - return fmt.Errorf("sync VF health state parent dir: %w", err) + if err := s.syncDirFunc(filepath.Dir(dirPath)); err != nil { + return false, fmt.Errorf("sync VF health state parent dir: %w", err) } } tmp := s.path + ".tmp" f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { - return fmt.Errorf("create VF health state: %w", err) + return false, fmt.Errorf("create VF health state: %w", err) } if _, err := f.Write(data); err != nil { f.Close() os.Remove(tmp) - return fmt.Errorf("write VF health state: %w", err) + return false, fmt.Errorf("write VF health state: %w", err) } if err := f.Sync(); err != nil { f.Close() os.Remove(tmp) - return fmt.Errorf("sync VF health state: %w", err) + return false, fmt.Errorf("sync VF health state: %w", err) } if err := f.Close(); err != nil { os.Remove(tmp) - return fmt.Errorf("close VF health state: %w", err) + return false, fmt.Errorf("close VF health state: %w", err) } if err := os.Rename(tmp, s.path); err != nil { os.Remove(tmp) - return fmt.Errorf("rename VF health state: %w", err) + return false, fmt.Errorf("rename VF health state: %w", err) } - if err := syncDir(dirPath); err != nil { - return fmt.Errorf("sync VF health state dir: %w", err) + if err := s.syncDirFunc(dirPath); err != nil { + return true, fmt.Errorf("sync VF health state dir: %w", err) } - return nil + return true, nil } func syncDir(path string) error { diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go index 75d8fbabf..5166a76d2 100644 --- a/lib/devices/vf_health_test.go +++ b/lib/devices/vf_health_test.go @@ -2,6 +2,7 @@ package devices import ( "encoding/json" + "errors" "os" "path/filepath" "testing" @@ -22,6 +23,7 @@ func resetVFHealthStore(t *testing.T) string { vfHealth.threshold = defaultVFQuarantineThreshold vfHealth.loadErr = nil vfHealth.persistErr = nil + vfHealth.syncDirFunc = syncDir }) return path } @@ -362,6 +364,79 @@ func TestReportVFInitFailureRollsBackOnPersistFailure(t *testing.T) { assert.False(t, exists, "a failure whose persist failed must be retried by the next report") } +func TestReportVFInitFailureRetainsRenamedStateAfterSyncFailure(t *testing.T) { + path := resetVFHealthStore(t) + vf := "0000:e3:00.4" + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-1"}) + require.NoError(t, err) + + vfHealth.syncDirFunc = func(string) error { return errors.New("injected sync failure") } + _, err = ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-2"}) + require.ErrorContains(t, err, "sync VF health state dir") + + data, err := os.ReadFile(path) + require.NoError(t, err) + var state vfHealthFile + require.NoError(t, json.Unmarshal(data, &state)) + require.Len(t, state.Records, 1) + assert.NotNil(t, state.Records[0].QuarantinedAt) + require.Len(t, quarantinedVFs(), 1, "memory must retain state already renamed into place") + assert.True(t, VFHealthStoreUnavailable()) + + vfHealth.syncDirFunc = syncDir + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.5", InstanceID: "other-instance"}) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.False(t, VFHealthStoreUnavailable()) + + data, err = os.ReadFile(path) + require.NoError(t, err) + state = vfHealthFile{} + require.NoError(t, json.Unmarshal(data, &state)) + found := false + for _, record := range state.Records { + if record.VFAddress == vf { + found = true + assert.NotNil(t, record.QuarantinedAt, "a later write must not erase the renamed quarantine") + } + } + require.True(t, found) + + available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: vf}}) + require.NoError(t, err) + assert.Zero(t, available) + assert.Equal(t, 1, quarantined) +} + +func TestReportRetriesFailedThresholdPersistence(t *testing.T) { + path := resetVFHealthStore(t) + vf := "0000:e3:00.4" + SetVFQuarantineThreshold(3) + for _, instance := range []string{"instance-1", "instance-2"} { + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: instance}) + require.NoError(t, err) + } + + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0644)) + vfHealth.path = filepath.Join(blocker, "vf-health.json") + SetVFQuarantineThreshold(2) + assert.True(t, VFHealthStoreUnavailable()) + + vfHealth.path = path + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportUnchanged, result.Outcome) + assert.False(t, VFHealthStoreUnavailable()) + + data, err := os.ReadFile(path) + require.NoError(t, err) + var state vfHealthFile + require.NoError(t, json.Unmarshal(data, &state)) + require.Len(t, state.Records, 1) + assert.NotNil(t, state.Records[0].QuarantinedAt) +} + func TestReportVFInitSuccessRollsBackOnPersistFailure(t *testing.T) { resetVFHealthStore(t) _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) From 54f03c429cf938c0a1a3418b888f1e576dc81c3a Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:41:07 +0000 Subject: [PATCH 105/107] Retry VF health parent directory sync --- lib/devices/vf_health.go | 13 +++++------ lib/devices/vf_health_test.go | 41 ++++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go index 479696661..843060ca9 100644 --- a/lib/devices/vf_health.go +++ b/lib/devices/vf_health.go @@ -440,14 +440,11 @@ func (s *vfHealthStore) writeStateLocked() (bool, error) { return false, fmt.Errorf("marshal VF health state: %w", err) } dirPath := filepath.Dir(s.path) - if _, err := os.Stat(dirPath); os.IsNotExist(err) { - if err := os.MkdirAll(dirPath, 0755); err != nil { - return false, fmt.Errorf("create VF health state dir: %w", err) - } - // Make the new directory entry itself durable. - if err := s.syncDirFunc(filepath.Dir(dirPath)); err != nil { - return false, fmt.Errorf("sync VF health state parent dir: %w", err) - } + if err := os.MkdirAll(dirPath, 0755); err != nil { + return false, fmt.Errorf("create VF health state dir: %w", err) + } + if err := s.syncDirFunc(filepath.Dir(dirPath)); err != nil { + return false, fmt.Errorf("sync VF health state parent dir: %w", err) } tmp := s.path + ".tmp" f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go index 5166a76d2..260c5cc83 100644 --- a/lib/devices/vf_health_test.go +++ b/lib/devices/vf_health_test.go @@ -364,13 +364,52 @@ func TestReportVFInitFailureRollsBackOnPersistFailure(t *testing.T) { assert.False(t, exists, "a failure whose persist failed must be retried by the next report") } +func TestReportVFInitFailureRetriesParentSyncAfterFailure(t *testing.T) { + resetVFHealthStore(t) + parentDir := t.TempDir() + vfHealth.path = filepath.Join(parentDir, "gpu", "vf-health.json") + + parentSyncs := 0 + retrySawPersistErr := false + vfHealth.syncDirFunc = func(path string) error { + if path != parentDir { + return syncDir(path) + } + parentSyncs++ + if parentSyncs == 1 { + return errors.New("injected parent sync failure") + } + if parentSyncs == 2 { + retrySawPersistErr = vfHealth.persistErr != nil + } + return syncDir(path) + } + + report := VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"} + _, err := ReportVFInitFailure(report) + require.ErrorContains(t, err, "sync VF health state parent dir") + assert.True(t, VFHealthStoreUnavailable()) + + result, err := ReportVFInitFailure(report) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, 3, parentSyncs) + assert.True(t, retrySawPersistErr, "retry must sync the parent before clearing the write failure") + assert.False(t, VFHealthStoreUnavailable()) +} + func TestReportVFInitFailureRetainsRenamedStateAfterSyncFailure(t *testing.T) { path := resetVFHealthStore(t) vf := "0000:e3:00.4" _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-1"}) require.NoError(t, err) - vfHealth.syncDirFunc = func(string) error { return errors.New("injected sync failure") } + vfHealth.syncDirFunc = func(path string) error { + if path == filepath.Dir(vfHealth.path) { + return errors.New("injected sync failure") + } + return syncDir(path) + } _, err = ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-2"}) require.ErrorContains(t, err, "sync VF health state dir") From 294405abddcf728fb49970a2be15b21d477916a7 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:44:17 +0000 Subject: [PATCH 106/107] Deduplicate VF health lock-order comment --- lib/devices/vf_health.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go index 843060ca9..3ae8d02c0 100644 --- a/lib/devices/vf_health.go +++ b/lib/devices/vf_health.go @@ -92,6 +92,9 @@ var ( threshold: defaultVFQuarantineThreshold, syncDirFunc: syncDir, } + // vendorVFIOMu is acquired before vfHealth.mu. It serializes quarantine + // mutations with vendor-VFIO create, destroy, and reconciliation so + // placement cannot select a VF while it is being quarantined. vendorVFIOMu sync.Mutex ) @@ -254,9 +257,6 @@ func countFreeVFs(vfs []VirtualFunction, quarantined map[string]struct{}) int { // ReportVFInitFailure records a guest-reported driver init failure and // quarantines the VF once failures from enough distinct assignments accumulate. func ReportVFInitFailure(report VFInitFailureReport) (VFReportResult, error) { - // Lock order is vendorVFIOMu before vfHealth.mu. This serializes quarantine - // mutations with vendor-VFIO create, destroy, and reconciliation so placement - // cannot select a VF while it is being quarantined. vendorVFIOMu.Lock() defer vendorVFIOMu.Unlock() return vfHealth.reportFailure(report) @@ -265,9 +265,6 @@ func ReportVFInitFailure(report VFInitFailureReport) (VFReportResult, error) { // ReportVFInitSuccess clears failures through an exactly matched successful // assignment. A quarantine is rescinded only when that assignment triggered it. func ReportVFInitSuccess(report VFInitSuccessReport) (VFSuccessResult, error) { - // Lock order is vendorVFIOMu before vfHealth.mu. This serializes quarantine - // mutations with vendor-VFIO create, destroy, and reconciliation so placement - // cannot select a VF while it is being quarantined. vendorVFIOMu.Lock() defer vendorVFIOMu.Unlock() return vfHealth.reportSuccess(report) From bf0fd622f06d393cd51db2db5b3d19c7387052ed Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:44:17 +0000 Subject: [PATCH 107/107] Collapse redundant VF health test cases Fold the below-threshold placement assertion into TestVGPUAvailability and the repaired-state recovery assertion into TestVGPUAvailabilityFailsWhenStoreUnavailable, exercising both through the public API. Drop TestReportVFInitFailureRespectsConfiguredThreshold and TestCheckedAddressesFailsClosedOnUnloadedState, whose remaining coverage is subsumed by the threshold re-evaluation and invalid-record tests. --- lib/devices/vf_health_test.go | 58 +++++++---------------------------- 1 file changed, 11 insertions(+), 47 deletions(-) diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go index 260c5cc83..00f14dbf7 100644 --- a/lib/devices/vf_health_test.go +++ b/lib/devices/vf_health_test.go @@ -53,6 +53,9 @@ func quarantineVF(t *testing.T, address string) { func TestVGPUAvailability(t *testing.T) { resetVFHealthStore(t) quarantineVF(t, "0000:82:00.4") + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:82:00.6", InstanceID: "instance-1"}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) vfs := []VirtualFunction{ {PCIAddress: "0000:82:00.4"}, {PCIAddress: "0000:82:00.5", Allocated: true}, @@ -61,7 +64,7 @@ func TestVGPUAvailability(t *testing.T) { available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, vfs) require.NoError(t, err) - assert.Equal(t, 1, available) + assert.Equal(t, 1, available, "a below-threshold failure tally must not remove the VF from placement") assert.Equal(t, 1, quarantined) available, quarantined, err = VGPUAvailability(VGPUFrameworkMdev, vfs) @@ -70,18 +73,6 @@ func TestVGPUAvailability(t *testing.T) { assert.Zero(t, quarantined) } -func TestVGPUAvailabilityExcludesOnlyQuarantinedVFs(t *testing.T) { - resetVFHealthStore(t) - result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:82:00.4", InstanceID: "instance-1"}) - require.NoError(t, err) - require.Equal(t, VFReportRecorded, result.Outcome) - - available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) - require.NoError(t, err) - assert.Equal(t, 1, available, "a below-threshold failure tally must not remove the VF from placement") - assert.Zero(t, quarantined) -} - func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { path := resetVFHealthStore(t) require.NoError(t, os.WriteFile(path, []byte("not json"), 0o644)) @@ -94,6 +85,13 @@ func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { require.NoError(t, err) assert.Equal(t, 1, available) assert.Zero(t, quarantined) + + restored := `{"version":1,"records":[{"vf_address":"0000:82:00.4","quarantined_at":"2026-08-20T00:00:00Z"}]}` + require.NoError(t, os.WriteFile(path, []byte(restored), 0o644)) + available, quarantined, err = VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.NoError(t, err, "a repaired state file must re-enable placement without a new report") + assert.Zero(t, available) + assert.Equal(t, 1, quarantined) } func TestVGPUAvailabilityFailsClosedAfterPersistFailure(t *testing.T) { @@ -228,23 +226,6 @@ func TestReportVFInitFailureDeduplicatesAssignments(t *testing.T) { assert.Empty(t, quarantinedVFs(), "a rescanned assignment must not count toward the threshold twice") } -func TestReportVFInitFailureRespectsConfiguredThreshold(t *testing.T) { - resetVFHealthStore(t) - SetVFQuarantineThreshold(3) - - for i, instance := range []string{"instance-1", "instance-2"} { - result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) - require.NoError(t, err) - assert.Equal(t, VFReportRecorded, result.Outcome) - assert.Equal(t, i+1, result.Failures) - assert.Equal(t, 3, result.Threshold) - } - - result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-3"}) - require.NoError(t, err) - assert.Equal(t, VFReportQuarantined, result.Outcome) -} - func TestReportVFInitSuccessClearsFailureTally(t *testing.T) { path := resetVFHealthStore(t) report := VFInitFailureReport{ @@ -497,23 +478,6 @@ func TestReportVFInitSuccessRollsBackOnPersistFailure(t *testing.T) { assert.Len(t, record.Failures, 1) } -func TestCheckedAddressesFailsClosedOnUnloadedState(t *testing.T) { - path := resetVFHealthStore(t) - quarantineVF(t, "0000:e3:00.4") - - require.NoError(t, os.WriteFile(path, []byte("not json"), 0644)) - require.Error(t, initVFHealth(path)) - - _, err := vfHealth.checkedAddresses() - require.Error(t, err) - - restored := `{"version":1,"records":[{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-20T00:00:00Z"}]}` - require.NoError(t, os.WriteFile(path, []byte(restored), 0644)) - addresses, err := vfHealth.checkedAddresses() - require.NoError(t, err) - assert.Contains(t, addresses, "0000:e3:00.4") -} - func TestCheckedAddressesFailsClosedOnInvalidRecord(t *testing.T) { tests := []struct { name string