diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 669d38633..e424da8e4 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -362,7 +362,16 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst inst, err := s.InstanceManager.CreateInstance(ctx, domainReq) if err != nil { + var vgpuPending *instances.VGPUCleanupPendingError switch { + case errors.As(err, &vgpuPending): + log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) + message, inner := vgpuCleanupPendingDetail(vgpuPending, "create", "delete it to retry") + return oapi.CreateInstance500JSONResponse{ + Code: "vgpu_cleanup_pending", + Message: message, + InnerError: inner, + }, nil case errors.Is(err, instances.ErrImageNotReady): return oapi.CreateInstance400JSONResponse{ Code: "image_not_ready", @@ -424,6 +433,19 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst return oapi.CreateInstance201JSONResponse(instanceToOAPI(*inst)), nil } +func vgpuCleanupPendingDetail(pending *instances.VGPUCleanupPendingError, action, retainedGuidance string) (string, *oapi.ErrorDetail) { + message := fmt.Sprintf("failed to %s instance: %v", action, pending) + innerCode := "vgpu_unretained_instance" + if pending.Retained { + message += "; " + retainedGuidance + innerCode = "vgpu_retained_instance" + } + return message, &oapi.ErrorDetail{ + Code: lo.ToPtr(innerCode), + Message: lo.ToPtr(pending.InstanceID), + } +} + // GetInstance gets instance details // The id parameter can be an instance ID, name, or ID prefix // Note: Resolution is handled by ResolveResource middleware @@ -834,7 +856,16 @@ func (s *ApiService) StartInstance(ctx context.Context, request oapi.StartInstan result, err := s.InstanceManager.StartInstance(ctx, inst.Id, startReq) if err != nil { + var vgpuPending *instances.VGPUCleanupPendingError switch { + case errors.As(err, &vgpuPending): + log.ErrorContext(ctx, "failed to start instance", "error", err) + message, inner := vgpuCleanupPendingDetail(vgpuPending, "start", "delete it or retry start to release it") + return oapi.StartInstance500JSONResponse{ + Code: "vgpu_cleanup_pending", + Message: message, + InnerError: inner, + }, nil case errors.Is(err, instances.ErrInvalidState): return oapi.StartInstance409JSONResponse{ Code: "invalid_state", @@ -1269,6 +1300,9 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { if inst.GPUMdevUUID != "" { gpu.MdevUuid = lo.ToPtr(inst.GPUMdevUUID) } + if inst.GPUDevicePath != "" { + gpu.DevicePath = lo.ToPtr(inst.GPUDevicePath) + } oapiInst.Gpu = gpu } diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index cc94c572d..ccaeb3a86 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -17,6 +17,7 @@ import ( "github.com/kernel/hypeman/lib/instances" "github.com/kernel/hypeman/lib/instances/phasetracking" mw "github.com/kernel/hypeman/lib/middleware" + "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" "github.com/kernel/hypeman/lib/paths" restartpolicy "github.com/kernel/hypeman/lib/restart-policy" @@ -47,6 +48,39 @@ func TestGetInstance_NotFound(t *testing.T) { require.Error(t, err) } +func TestVGPUCleanupPendingDetail(t *testing.T) { + t.Parallel() + for _, tt := range []struct { + name string + retained bool + code string + guidance string + exclude string + }{ + {name: "retained", retained: true, code: "vgpu_retained_instance", guidance: "delete it to retry"}, + {name: "unretained", code: "vgpu_unretained_instance", guidance: "periodic vGPU reconcile", exclude: "delete"}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + message, inner := vgpuCleanupPendingDetail(&instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Retained: tt.retained, + Err: network.ErrNameExists, + }, "create", "delete it to retry") + assert.Contains(t, message, "inst-1") + assert.Contains(t, message, network.ErrNameExists.Error()) + assert.Contains(t, message, tt.guidance) + if tt.exclude != "" { + assert.NotContains(t, message, tt.exclude) + } + require.NotNil(t, inner.Code) + assert.Equal(t, tt.code, *inner.Code) + require.NotNil(t, inner.Message) + assert.Equal(t, "inst-1", *inner.Message) + }) + } +} + func TestCreateInstance_AutoPullImage(t *testing.T) { t.Parallel() if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { @@ -890,6 +924,16 @@ func TestCreateInstance_ErrorStatusMapping(t *testing.T) { wantCode string wantMessage string }{ + { + name: "vGPU cleanup pending beats wrapped name conflict -> 500", + err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Retained: true, + Err: network.ErrNameExists, + }, + wantType: oapi.CreateInstance500JSONResponse{}, + wantCode: "vgpu_cleanup_pending", + }, { name: "platform not available -> 404", err: fmt.Errorf("resolve image: %w", images.ErrPlatformNotAvailable), @@ -1014,6 +1058,27 @@ func TestRestoreInstance_ErrorMapping(t *testing.T) { } } +func TestStartInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { + t.Parallel() + svc := newTestService(t) + resolved := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{Id: "inst-1", Name: "inst-1"}, + State: instances.StateStopped, + } + svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{ + InstanceID: resolved.Id, + Retained: true, + Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources), + }} + + resp, err := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id}) + require.NoError(t, err) + pending, ok := resp.(oapi.StartInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "delete it or retry start") +} + func TestInstanceActions_ImageNotFoundMapsTo404(t *testing.T) { t.Parallel() diff --git a/cmd/api/api/resources.go b/cmd/api/api/resources.go index ebb6ad951..dec9f35eb 100644 --- a/cmd/api/api/resources.go +++ b/cmd/api/api/resources.go @@ -87,9 +87,11 @@ func convertResourceStatus(rs resources.ResourceStatus) oapi.ResourceStatus { func convertGPUResourceStatus(gs *resources.GPUResourceStatus) oapi.GPUResourceStatus { result := oapi.GPUResourceStatus{ - Mode: oapi.GPUResourceStatusMode(gs.Mode), - TotalSlots: gs.TotalSlots, - UsedSlots: gs.UsedSlots, + Mode: oapi.GPUResourceStatusMode(gs.Mode), + TotalSlots: gs.TotalSlots, + UsedSlots: gs.UsedSlots, + AllocatableSlots: gs.AllocatableSlots, + QuarantinedSlots: gs.QuarantinedSlots, } // Convert profiles (vGPU mode) diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index b46f4f2e5..c250a3621 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -269,7 +269,8 @@ type SnapshotConfig struct { // GPUConfig holds GPU-related settings. type GPUConfig struct { - ProfileCacheTTL string `koanf:"profile_cache_ttl"` + ProfileCacheTTL string `koanf:"profile_cache_ttl"` + VFQuarantineThreshold int `koanf:"vf_quarantine_threshold"` } // Config is the top-level Hypeman server configuration. @@ -494,7 +495,8 @@ func defaultConfig() *Config { }, GPU: GPUConfig{ - ProfileCacheTTL: "30m", + ProfileCacheTTL: "30m", + VFQuarantineThreshold: 2, }, } } @@ -647,6 +649,9 @@ func (c *Config) Validate() error { if c.Build.MaxConcurrentSourceBuilds <= 0 { return fmt.Errorf("build.max_concurrent_source_builds must be positive, got %d", c.Build.MaxConcurrentSourceBuilds) } + if c.GPU.VFQuarantineThreshold < 1 { + return fmt.Errorf("gpu.vf_quarantine_threshold must be >= 1, got %d", c.GPU.VFQuarantineThreshold) + } if c.Limits.MaxConcurrentPushes <= 0 { return fmt.Errorf("limits.max_concurrent_pushes must be positive, got %d", c.Limits.MaxConcurrentPushes) } diff --git a/cmd/api/config/config_test.go b/cmd/api/config/config_test.go index 5660d878e..efd52f208 100644 --- a/cmd/api/config/config_test.go +++ b/cmd/api/config/config_test.go @@ -250,6 +250,18 @@ func TestValidateRejectsInvalidMetricsPort(t *testing.T) { } } +func TestValidateRejectsInvalidVFQuarantineThreshold(t *testing.T) { + for _, threshold := range []int{0, -1} { + cfg := defaultConfig() + cfg.GPU.VFQuarantineThreshold = threshold + + err := cfg.Validate() + if err == nil { + t.Fatalf("expected validation error for vf_quarantine_threshold %d", threshold) + } + } +} + func TestValidateRejectsInvalidMetricExportInterval(t *testing.T) { cfg := defaultConfig() cfg.Otel.MetricExportInterval = "not-a-duration" diff --git a/cmd/api/main.go b/cmd/api/main.go index f1bbfcf3d..d6fba1b43 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -204,6 +204,7 @@ func run() error { // Configure GPU profile cache TTL devices.SetGPUProfileCacheTTL(cfg.GPU.ProfileCacheTTL) + devices.SetVFQuarantineThreshold(cfg.GPU.VFQuarantineThreshold) // Initialize OpenTelemetry (before wire initialization) otelCfg := otel.Config{ @@ -384,11 +385,9 @@ func run() error { return fmt.Errorf("reconcile device state: %w", err) } - // Reconcile mdev devices (clears orphaned vGPUs from previous runs) - logger.Info("Reconciling mdev devices...") - if err := devices.ReconcileMdevs(app.Ctx, nil); err != nil { - // Log but don't fail - mdev cleanup is best-effort - logger.Warn("failed to reconcile mdev devices", "error", err) + logger.Info("Reconciling vGPU devices...") + if r, ok := app.InstanceManager.(interface{ StartVGPUReconciler(context.Context) }); ok { + r.StartVGPUReconciler(ctx) } // Wire up resource validator for aggregate limit checking diff --git a/config.example.yaml b/config.example.yaml index ebef41257..70d55fa68 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -170,6 +170,12 @@ data_dir: /var/lib/hypeman # idle_ttl: "" # delete builders idle this long (e.g. "24h"); # # destructive, empty = disabled +# gpu: +# profile_cache_ttl: 30m # vGPU profile metadata cache TTL +# vf_quarantine_threshold: 2 # distinct instance assignments that must report +# # a guest driver init failure before the VF is +# # quarantined (must be >= 1) + # ============================================================================= # Resource Limits # ============================================================================= diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 12861a6a7..1f1a82776 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -4,6 +4,8 @@ import ( "bytes" "context" "os" + "path/filepath" + "strings" "testing" "time" @@ -21,21 +23,23 @@ import ( "github.com/stretchr/testify/require" ) -// TestVGPU is an integration test that verifies vGPU (SR-IOV mdev) support works. +// TestVGPU is an integration test that verifies vGPU (SR-IOV) support works +// on the host's framework: mdev or NVIDIA's vendor-specific VFIO. // // This test automatically detects vGPU availability and skips if: -// - No SR-IOV VFs are found in /sys/class/mdev_bus/ +// - No vGPU framework (mdev or vendor VFIO) is discovered // - No vGPU profiles are available -// - Not running as root (required for mdev creation) +// - Not running as root (required for sysfs vGPU assignment) // - KVM is not available // // To run manually: // // sudo go test -v -run TestVGPU -timeout 5m ./integration/... // -// Note: This test verifies mdev creation and PCI device visibility inside the VM. -// It does NOT test nvidia-smi or CUDA functionality since that requires NVIDIA -// guest drivers pre-installed in the image. +// Note: This test verifies vGPU assignment, release on stop, reacquisition on +// start, and PCI device visibility inside the VM. It does NOT test nvidia-smi +// or CUDA functionality since that requires NVIDIA guest drivers pre-installed +// in the image. func TestVGPU(t *testing.T) { t.Parallel() if testing.Short() { @@ -84,9 +88,15 @@ func TestVGPU(t *testing.T) { // Cleanup any orphaned instances and mdevs t.Cleanup(func() { - if instanceID != "" { - t.Log("Cleanup: Deleting instance...") - instanceManager.DeleteInstance(ctx, instanceID) + if instanceID == "" { + return + } + if _, err := instanceManager.StopInstance(ctx, instanceID); err != nil { + t.Logf("Cleanup: stop instance: %v", err) + } + t.Log("Cleanup: Deleting instance...") + if err := instanceManager.DeleteInstance(ctx, instanceID); err != nil { + t.Errorf("cleanup: delete instance: %v", err) } }) @@ -159,9 +169,18 @@ func TestVGPU(t *testing.T) { instanceID = inst.Id t.Logf("Instance created: %s", inst.Id) - // Verify mdev UUID was assigned - require.NotEmpty(t, inst.GPUMdevUUID, "Instance should have mdev UUID assigned") - t.Logf("mdev UUID: %s", inst.GPUMdevUUID) + // Verify the assignment matches the host's framework + require.NotEmpty(t, inst.GPUDevicePath, "Instance should have a vGPU device path assigned") + switch inst.GPUFramework { + case devices.VGPUFrameworkMdev: + require.NotEmpty(t, inst.GPUMdevUUID, "mdev instance should have a UUID assigned") + t.Logf("mdev UUID: %s", inst.GPUMdevUUID) + case devices.VGPUFrameworkVendorVFIO: + require.Empty(t, inst.GPUMdevUUID, "vendor VFIO instance should not have an mdev UUID") + t.Logf("vendor VFIO VF: %s", inst.GPUDevicePath) + default: + t.Fatalf("unexpected vGPU framework %q", inst.GPUFramework) + } // Step 5: Check GPU resources AFTER creating instance t.Run("ResourcesDecrementedAfterCreation", func(t *testing.T) { @@ -180,12 +199,9 @@ func TestVGPU(t *testing.T) { assert.Less(t, availableAfter, availableBefore, "available instances should decrease after creating VM") }) - // Step 6: Verify mdev was created in sysfs - t.Run("MdevCreated", func(t *testing.T) { - mdevPath := "/sys/bus/mdev/devices/" + inst.GPUMdevUUID - _, err := os.Stat(mdevPath) - assert.NoError(t, err, "mdev device should exist at %s", mdevPath) - t.Logf("mdev exists at: %s", mdevPath) + // Step 6: Verify the assignment exists in sysfs + t.Run("VGPUAssignedInSysfs", func(t *testing.T) { + assertVGPUAssigned(t, inst.GPUFramework, inst.GPUDevicePath) }) // Step 7: Wait for guest agent to be ready @@ -225,13 +241,68 @@ func TestVGPU(t *testing.T) { require.NoError(t, err) assert.Equal(t, profile, actualInst.GPUProfile, "GPU profile should match") - assert.NotEmpty(t, actualInst.GPUMdevUUID, "mdev UUID should be set") - t.Logf("Instance GPU: profile=%s, mdev=%s", actualInst.GPUProfile, actualInst.GPUMdevUUID) + assert.Equal(t, inst.GPUFramework, actualInst.GPUFramework, "framework should match") + assert.NotEmpty(t, actualInst.GPUDevicePath, "device path should be set") + if inst.GPUFramework == devices.VGPUFrameworkMdev { + assert.NotEmpty(t, actualInst.GPUMdevUUID, "mdev UUID should be set") + } + t.Logf("Instance GPU: profile=%s, framework=%s, device=%s", actualInst.GPUProfile, actualInst.GPUFramework, actualInst.GPUDevicePath) + }) + + t.Log("Step 10: Stopping instance...") + _, err = instanceManager.StopInstance(ctx, inst.Id) + require.NoError(t, err, "stop should succeed") + + t.Run("VGPUReleasedOnStop", func(t *testing.T) { + stopped, err := instanceManager.GetInstance(ctx, inst.Id) + require.NoError(t, err) + assert.Empty(t, stopped.GPUDevicePath, "assignment metadata should be cleared on stop") + assertVGPUReleased(t, inst.GPUFramework, inst.GPUDevicePath) + }) + + t.Log("Step 11: Starting instance to reacquire a vGPU...") + started, err := instanceManager.StartInstance(ctx, inst.Id, instances.StartInstanceRequest{}) + require.NoError(t, err, "start should succeed") + + t.Run("VGPUReacquiredOnStart", func(t *testing.T) { + require.NotEmpty(t, started.GPUDevicePath, "start should assign a vGPU") + assert.Equal(t, inst.GPUFramework, started.GPUFramework, "framework should match") + assertVGPUAssigned(t, started.GPUFramework, started.GPUDevicePath) }) t.Log("✅ vGPU test PASSED!") } +func assertVGPUAssigned(t *testing.T, framework devices.VGPUFramework, devicePath string) { + t.Helper() + switch framework { + case devices.VGPUFrameworkMdev: + _, err := os.Stat(devicePath) + assert.NoError(t, err, "mdev device should exist at %s", devicePath) + case devices.VGPUFrameworkVendorVFIO: + data, err := os.ReadFile(filepath.Join(devicePath, "nvidia", "current_vgpu_type")) + require.NoError(t, err, "VF should expose current_vgpu_type") + assert.NotEqual(t, "0", strings.TrimSpace(string(data)), "VF should have a vGPU type assigned") + default: + t.Fatalf("unexpected vGPU framework %q", framework) + } +} + +func assertVGPUReleased(t *testing.T, framework devices.VGPUFramework, devicePath string) { + t.Helper() + switch framework { + case devices.VGPUFrameworkMdev: + _, err := os.Stat(devicePath) + assert.True(t, os.IsNotExist(err), "mdev device should be gone from %s", devicePath) + case devices.VGPUFrameworkVendorVFIO: + data, err := os.ReadFile(filepath.Join(devicePath, "nvidia", "current_vgpu_type")) + require.NoError(t, err, "VF should expose current_vgpu_type") + assert.Equal(t, "0", strings.TrimSpace(string(data)), "VF assignment should be released") + default: + t.Fatalf("unexpected vGPU framework %q", framework) + } +} + // checkVGPUTestPrerequisites checks if vGPU test can run. // Returns (skipReason, profileName) - skipReason is empty if all prerequisites are met. func checkVGPUTestPrerequisites() (string, string) { @@ -245,10 +316,13 @@ func checkVGPUTestPrerequisites() (string, string) { return "vGPU test requires root (sudo) for mdev creation", "" } - // Check for vGPU mode (SR-IOV VFs present) - mode := devices.DetectHostGPUMode() - if mode != devices.GPUModeVGPU { - return "vGPU test requires SR-IOV VFs in /sys/class/mdev_bus/", "" + // Check for a vGPU framework (mdev or vendor VFIO) + framework, _, err := devices.DiscoverVGPU() + if err != nil { + return "vGPU test failed to discover vGPU framework: " + err.Error(), "" + } + if framework == devices.VGPUFrameworkNone { + return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", "" } // Check for available profiles diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 54a19c472..678871562 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -8,16 +8,17 @@ hypeman supports two GPU modes, automatically detected based on host configurati | Mode | Description | Use Case | |------|-------------|----------| -| **vGPU (SR-IOV)** | Virtual GPUs via mdev on SR-IOV VFs | Multi-tenant, shared GPU resources | +| **vGPU (SR-IOV)** | Virtual GPUs on SR-IOV VFs via mdev or vendor VFIO | Multi-tenant, shared GPU resources | | **Passthrough** | Whole GPU VFIO passthrough | Dedicated GPU per instance | The host's GPU mode is determined by the host driver configuration: -- If `/sys/class/mdev_bus/` contains VFs → vGPU mode -- If NVIDIA GPUs are available for VFIO → passthrough mode +- If `/sys/class/mdev_bus/` contains VFs → mdev vGPU mode +- If VFs expose `/sys/bus/pci/devices//nvidia/current_vgpu_type` → vendor VFIO vGPU mode +- If NVIDIA GPUs are available for whole-device VFIO → passthrough mode ## vGPU Mode (Recommended) -vGPU mode uses NVIDIA's SR-IOV technology to create Virtual Functions (VFs), each capable of hosting an mdev (mediated device) representing a vGPU. +vGPU mode uses NVIDIA's SR-IOV technology to create Virtual Functions (VFs). Hosts on older kernels represent each vGPU as an mdev. Hosts using NVIDIA's vendor VFIO framework assign the profile directly to the VF through `current_vgpu_type`. ### How It Works @@ -48,8 +49,10 @@ curl -s http://localhost:4973/resources | jq .gpu "mode": "vgpu", "total_slots": 64, "used_slots": 5, + "allocatable_slots": 57, + "quarantined_slots": 2, "profiles": [ - {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 59}, + {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 57}, {"name": "L40S-2Q", "framebuffer_mb": 2048, "available": 30}, {"name": "L40S-4Q", "framebuffer_mb": 4096, "available": 16} ] @@ -74,7 +77,7 @@ curl -X POST http://localhost:4973/instances \ }' ``` -The response includes the assigned mdev UUID: +On an mdev host, the response also includes the assigned mdev UUID: ```json { @@ -87,19 +90,23 @@ The response includes the assigned mdev UUID: } ``` -### Ephemeral mdev Lifecycle +### Ephemeral vGPU Lifecycle -mdev devices are **ephemeral**: created on instance start, destroyed on instance delete. +vGPU assignments are created on instance start and released on stop or delete. Hypeman creates/removes an mdev on mdev hosts and writes the profile ID/`0` to `current_vgpu_type` on vendor VFIO hosts. ``` -Instance Create → Create mdev → Attach to VM → Instance Running -Instance Delete → Stop VM → Destroy mdev → VF available again +Instance Create → Assign profile to VF → Attach VF to VM → Instance Running +Instance Stop/Delete → Release profile → VF available again ``` -This ensures: -- **Security**: No VRAM data leakage between instances -- **Clean state**: Fresh vGPU for each instance -- **Automatic cleanup**: Orphaned mdevs cleaned up on server restart +Hypeman reconciles orphaned assignments with a periodic fail-closed pass: once at startup and every minute afterward (skipped entirely on hosts without GPUs). Each pass releases assignments whose owning instance is no longer live and clears their metadata, then sweeps device-level leftovers with no live metadata claim. Devices held open by a running VMM and assignments younger than five minutes are preserved, so a release that fails during stop or delete (typically because a GPU-busy VMM's kernel-side VFIO teardown outlives the force-kill wait) is simply retried on later passes until the device is free. + +### Hypervisor Support + +Hypervisor selection for vGPU instances is caller policy; hypeman does not enforce it. In practice **QEMU is the only hypervisor with working vGPU support**: + +- **QEMU**: fully supported and validated on both mdev and vendor VFIO hosts. +- **Cloud Hypervisor**: vendor VFIO vGPUs are known broken upstream ([cloud-hypervisor#7572](https://github.com/cloud-hypervisor/cloud-hypervisor/issues/7572)) — the VM boots and the VF attaches, but VFIO region reads fail, the guest driver cannot initialize, and the vGPU is non-functional. Do not place vGPU instances on Cloud Hypervisor. ## Passthrough Mode @@ -116,6 +123,8 @@ curl -s http://localhost:4973/resources | jq .gpu "mode": "passthrough", "total_slots": 4, "used_slots": 2, + "allocatable_slots": 2, + "quarantined_slots": 0, "devices": [ {"name": "NVIDIA L40S", "available": true}, {"name": "NVIDIA L40S", "available": false} @@ -180,8 +189,10 @@ Returns GPU status along with other resources: "mode": "vgpu", "total_slots": 64, "used_slots": 5, + "allocatable_slots": 57, + "quarantined_slots": 2, "profiles": [ - {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 59} + {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 57} ] } } @@ -241,7 +252,8 @@ To upgrade the NVIDIA driver version: 1. Check host GPU mode detection: ```bash - ls /sys/class/mdev_bus/ # Should show VFs for vGPU mode + ls /sys/class/mdev_bus/ + find /sys/bus/pci/devices -path '*/nvidia/current_vgpu_type' ``` 2. Verify NVIDIA drivers are loaded on host: @@ -265,17 +277,107 @@ curl -s http://localhost:4973/resources | jq '.gpu.profiles' curl http://localhost:4973/instances//logs?source=app ``` -### mdev creation fails +### Guest driver init times out on one VF (vendor VFIO) -1. Check if VFs are available: - ```bash - ls /sys/class/mdev_bus/ - ``` +A VF can be wedged inside the NVIDIA stack while its sysfs interface stays +healthy: assignment succeeds, the host plugin logs `display_init inst: 0 +successful`, but the guest driver loops on -2. Verify mdev types: - ```bash - cat /sys/class/mdev_bus/*/mdev_supported_types/*/available_instances - ``` +``` +NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884) +``` + +(0x65 = timeout; the guest's init requests are never answered, and +`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle). + +Hypeman tracks these failures in `/gpu/vf-health.json` (it survives +restarts): each reported init failure is tallied per instance assignment, and +once failures accumulate from `gpu.vf_quarantine_threshold` distinct +assignments (default 2), the VF is quarantined: excluded from placement and +from advertised profile availability, and its parent GPU becomes +overflow-only — deprioritized for new placements. Selection among a card's +equivalent free VFs is randomized so a wedged VF cannot capture every +placement. A reported init success clears failures only when that exact +assignment has a recorded failure, removing the match and older tallies; if +that assignment crossed the threshold, its later success also rescinds the +quarantine. If the state file exists but cannot be loaded, or the last write +to it failed, placement and advertised availability fail closed until a load +or write succeeds. Recorded tallies are re-evaluated against the configured +threshold at load, so lowering `gpu.vf_quarantine_threshold` quarantines VFs +whose persisted failures already meet the new value. + +`used_slots` includes quarantined VFs still held by running instances, so it +can overlap `quarantined_slots`; use `allocatable_slots` for admission. + +Quarantine only removes capacity — it never touches a running instance. + +The wedge itself leaves no host-side log: no kernel error, no XID, no plugin +crash. The trigger is a SIGKILL delivered to QEMU while the vGPU plugin is +still initializing the VF (roughly the first seconds after process start): +a single hard kill in that window wedges the VF near-deterministically, +while QEMU processes that exit voluntarily — error exits, QMP quit, SIGTERM — +run their VFIO teardown and never wedge, and hard kills of fully-initialized +vGPU VMs are also safe. Hypeman therefore SIGTERMs a vGPU QEMU first and only +escalates to SIGKILL after a grace period, both in start-failure cleanup and +when force-killing any vGPU instance (the instance reports Running seconds +before driver init completes, so no state reliably marks the window); a hard +kill after an ignored SIGTERM logs `VF may wedge` with the device path. +External SIGKILLs (OOM killer, manual `kill -9`) can still trigger it. + +Confirm by assigning the same profile on a different VF: if that guest +initializes, the VF is wedged, not the driver stack. Remediate by cycling +SR-IOV on the parent GPU (this destroys and recreates all of its VFs, so it +requires no vGPU assignments on that GPU). The DCGM quiesce is not optional: +with `nv-hostengine`/`dcgm-exporter` holding the GPUs open, `sriov-manage -d` +fails with `Cannot obtain unbindLock` on first contact. + +Any manual edit to `vf-health.json` needs an immediate hypeman restart: the +store loads only at startup, and a failure report landing first re-persists +the in-memory set over your edit. The restart does not disturb running VMs — +startup reconciliation protects live VFs. + +**Draining the parent GPU.** Overflow-only is a preference, not a cordon: +under capacity pressure new placements still land on the card's healthy VFs +and refill it. To drain the card, quarantine all of its VFs by hand — add +records to the versioned `vf-health.json` (`{"version": 1, "records": +[{"vf_address": "...", "quarantined_at": "..."}]}`) and restart. Running +instances are untouched and +drain through their normal lifecycle: standby is blocked for vGPU instances, +so only a running VM pins a VF, and each stop or delete frees one for good. +Monitor by listing instances whose `gpu.device_path` sits under the parent +GPU; once none remain, run the cycle below. + +```bash +# 1. Quiesce the services holding the GPU (required for the unbind lock). +systemctl stop nvidia-dcgm-exporter nvidia-dcgm + +# 2. Cycle SR-IOV on the parent GPU. +/usr/lib/nvidia/sriov-manage -d +/usr/lib/nvidia/sriov-manage -e + +# 3. Restart the quiesced services. +systemctl start nvidia-dcgm nvidia-dcgm-exporter +``` + +After the cycle, remove the card's entries from `vf-health.json`, restart, +and boot a GPU instance to verify recovery. + +Do not unbind/rebind the VF from the nvidia driver — it breaks the +nvidia-vgpu-vfio core-device registration (`vfio_pci_core_device not found`) +and the VF stops accepting assignments entirely until the SR-IOV cycle. + +### vGPU assignment fails + +Check the files for the framework detected on the host: + +```bash +# mdev +cat /sys/class/mdev_bus/*/mdev_supported_types/*/available_instances + +# vendor VFIO +cat /sys/bus/pci/devices/*/nvidia/creatable_vgpu_types +cat /sys/bus/pci/devices/*/nvidia/current_vgpu_type +``` ## Performance Tuning diff --git a/lib/devices/gpu_mode.go b/lib/devices/gpu_mode.go deleted file mode 100644 index 40b3b2ba0..000000000 --- a/lib/devices/gpu_mode.go +++ /dev/null @@ -1,30 +0,0 @@ -package devices - -import ( - "os" -) - -// DetectHostGPUMode determines the host's GPU configuration mode. -// -// Returns: -// - GPUModeVGPU if /sys/class/mdev_bus has entries (SR-IOV VFs present) -// - GPUModePassthrough if NVIDIA GPUs are available for VFIO passthrough -// - GPUModeNone if no GPUs are available -// -// Note: A host is configured for either vGPU or passthrough, not both, -// because the host driver determines which mode is available. -func DetectHostGPUMode() GPUMode { - // Check for vGPU mode first (SR-IOV VFs present) - entries, err := os.ReadDir("/sys/class/mdev_bus") - if err == nil && len(entries) > 0 { - return GPUModeVGPU - } - - // Check for passthrough mode (physical GPUs available) - gpus, err := DiscoverAvailableDevices() - if err == nil && len(gpus) > 0 { - return GPUModePassthrough - } - - return GPUModeNone -} diff --git a/lib/devices/manager.go b/lib/devices/manager.go index 30763c04d..6b9f6340a 100644 --- a/lib/devices/manager.go +++ b/lib/devices/manager.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "log/slog" "os" "runtime" "strings" @@ -85,6 +86,9 @@ type manager struct { // NewManager creates a new device manager. // Use SetLivenessChecker after construction to enable accurate orphan detection. func NewManager(p *paths.Paths) Manager { + if err := initVFHealth(p.VFHealthState()); err != nil { + slog.Default().Error("failed to load VF health state; vGPU placement is disabled until the state file is repaired or removed", "error", err) + } return &manager{ paths: p, vfioBinder: NewVFIOBinder(), diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index 1427a5095..93d3adbf7 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -12,10 +12,9 @@ func SetGPUProfileCacheTTL(ttl string) { // No-op on macOS } -// DiscoverVFs returns an empty list on macOS. -// SR-IOV Virtual Functions are not available on macOS. -func DiscoverVFs() ([]VirtualFunction, error) { - return []VirtualFunction{}, nil +// DiscoverVGPU reports no vGPU framework on macOS. +func DiscoverVGPU() (VGPUFramework, []VirtualFunction, error) { + return VGPUFrameworkNone, nil, nil } // ListGPUProfiles returns an empty list on macOS. @@ -24,7 +23,7 @@ func ListGPUProfiles() ([]GPUProfile, error) { } // ListGPUProfilesWithVFs returns an empty list on macOS. -func ListGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { +func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction) ([]GPUProfile, error) { return []GPUProfile{}, nil } @@ -59,6 +58,10 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { return nil } +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepDevices bool) error { + return nil +} + // ReconcileMdevs is a no-op on macOS. func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) error { return nil diff --git a/lib/devices/mdev_linux.go b/lib/devices/mdev_linux.go index 1a398a418..423d8473d 100644 --- a/lib/devices/mdev_linux.go +++ b/lib/devices/mdev_linux.go @@ -5,7 +5,9 @@ package devices import ( "bufio" "context" + "errors" "fmt" + "log/slog" "os" "os/exec" "path/filepath" @@ -21,10 +23,9 @@ import ( ) const ( - mdevBusPath = "/sys/class/mdev_bus" - mdevDevices = "/sys/bus/mdev/devices" - orphanedMdevGracePeriod = 5 * time.Minute - procPath = "/proc" + mdevBusPath = "/sys/class/mdev_bus" + mdevDevices = "/sys/bus/mdev/devices" + procPath = "/proc" ) // mdevMu protects mdev creation/destruction to prevent race conditions @@ -89,32 +90,54 @@ func getCachedProfiles(firstVF string) []profileMetadata { return cachedProfiles } -// DiscoverVFs returns all SR-IOV Virtual Functions available for vGPU. -// These are discovered by scanning /sys/class/mdev_bus/ which contains -// VFs that can host mdev devices. -func DiscoverVFs() ([]VirtualFunction, error) { - entries, err := os.ReadDir(mdevBusPath) +// discoverMdevVFs returns all SR-IOV Virtual Functions available for vGPU, +// discovered by scanning /sys/class/mdev_bus/. +func discoverMdevVFs() ([]VirtualFunction, error) { + return discoverMdevVFsWith(mdevBusPath, pciDevicesPath, ListMdevDevices) +} + +func discoverMdevVFsWith(busPath, pciPath string, listMdevs func() ([]MdevDevice, error)) ([]VirtualFunction, error) { + entries, err := os.ReadDir(busPath) if err != nil { if os.IsNotExist(err) { - return nil, nil // No mdev_bus means no vGPU support + return nil, nil // No mdev_bus means no mdev vGPU support } return nil, fmt.Errorf("read mdev_bus: %w", err) } // List mdevs once and build a lookup map to avoid O(n*m) performance - mdevs, _ := ListMdevDevices() + mdevs, _ := listMdevs() mdevByVF := make(map[string]bool, len(mdevs)) for _, mdev := range mdevs { mdevByVF[mdev.VFAddress] = true } var vfs []VirtualFunction + var vfErrs []error for _, entry := range entries { vfAddr := entry.Name() + types, err := os.ReadDir(filepath.Join(busPath, vfAddr, "mdev_supported_types")) + if err != nil { + if os.IsNotExist(err) { + continue + } + vfErrs = append(vfErrs, fmt.Errorf("read mdev supported types for VF %s: %w", vfAddr, err)) + continue + } + usable := false + for _, typ := range types { + if typ.IsDir() { + usable = true + break + } + } + if !usable { + continue + } // Find parent GPU by checking physfn symlink // VFs have a physfn symlink pointing to their parent Physical Function - physfnPath := filepath.Join("/sys/bus/pci/devices", vfAddr, "physfn") + physfnPath := filepath.Join(pciPath, vfAddr, "physfn") parentGPU := "" if target, err := os.Readlink(physfnPath); err == nil { parentGPU = filepath.Base(target) @@ -129,24 +152,23 @@ func DiscoverVFs() ([]VirtualFunction, error) { Allocated: hasMdev, }) } + if len(vfErrs) > 0 { + // Mirror vendor VFIO discovery: one unreadable VF must not blank out + // the host's GPU capacity. Only when no VF is readable does discovery + // fail, so a wholesale sysfs outage cannot demote an mdev host to + // vendor VFIO or passthrough while assignments exist. + if len(vfs) == 0 { + return nil, errors.Join(vfErrs...) + } + slog.Default().Warn("skipping unreadable mdev VFs", "error", errors.Join(vfErrs...)) + } return vfs, nil } -// ListGPUProfiles returns available vGPU profiles with availability counts. -// Profiles are discovered from the first VF's mdev_supported_types directory. -func ListGPUProfiles() ([]GPUProfile, error) { - vfs, err := DiscoverVFs() - if err != nil { - return nil, err - } - return ListGPUProfilesWithVFs(vfs) -} - -// ListGPUProfilesWithVFs returns available vGPU profiles using pre-discovered VFs. -// This avoids redundant VF discovery when the caller already has the list. -// Uses parallel sysfs reads for fast availability counting. -func ListGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { +// listMdevGPUProfilesWithVFs returns available vGPU profiles using +// pre-discovered VFs. Uses parallel sysfs reads for fast availability counting. +func listMdevGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { if len(vfs) == 0 { return nil, nil } @@ -305,7 +327,7 @@ func countAvailableForSingleProfile(freeVFsByParent map[string][]VirtualFunction // findProfileType finds the internal type name (e.g., "nvidia-556") for a profile name (e.g., "L40S-1Q") func findProfileType(profileName string) (string, error) { - vfs, err := DiscoverVFs() + vfs, err := discoverMdevVFs() if err != nil || len(vfs) == 0 { return "", fmt.Errorf("no VFs available") } @@ -531,7 +553,7 @@ func CreateMdev(ctx context.Context, profileName, instanceID string) (*MdevDevic } // Discover all VFs - vfs, err := DiscoverVFs() + vfs, err := discoverMdevVFs() if err != nil { return nil, fmt.Errorf("discover VFs: %w", err) } @@ -685,19 +707,29 @@ func mdevPastGracePeriod(mdevUUID string, gracePeriod time.Duration) (bool, time return age >= gracePeriod, age, nil } +func protectedMdevUUIDs(instanceInfos []MdevReconcileInfo) map[string]struct{} { + protected := make(map[string]struct{}, len(instanceInfos)) + for _, info := range instanceInfos { + if info.MdevUUID != "" && info.IsRunning { + protected[info.MdevUUID] = struct{}{} + } + } + return protected +} + // ReconcileMdevs destroys orphaned mdevs on managed VFs. -// This is called on server startup to clean up stale mdevs from previous runs. // // Policy: // - Consider only mdevs whose parent VF is currently managed by hypeman (discoverable via /sys/class/mdev_bus) +// - Keep mdevs claimed by live instance metadata // - Keep mdevs whose VFIO group has an open file handle (/dev/vfio/) // - Keep mdevs younger than a short grace period to avoid racing very recent state transitions // - Delete all remaining mdevs func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) error { log := logger.FromContext(ctx) - _ = instanceInfos + protectedMdevs := protectedMdevUUIDs(instanceInfos) - vfs, err := DiscoverVFs() + vfs, err := discoverMdevVFs() if err != nil { return fmt.Errorf("discover managed VFs: %w", err) } @@ -724,17 +756,22 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro log.InfoContext(ctx, "reconciling mdev devices", "total_mdevs", len(mdevs), "managed_vfs", len(managedVFs), - "grace_period", orphanedMdevGracePeriod.String(), + "grace_period", VGPUAssignmentGracePeriod.String(), ) groupInUseCache := make(map[int]bool) - var destroyed, failedDestroy, skippedUnmanagedVF, skippedInUse, skippedGrace, skippedProbeError int + var destroyed, failedDestroy, skippedUnmanagedVF, skippedClaimed, skippedInUse, skippedGrace, skippedProbeError int for _, mdev := range mdevs { if _, ok := managedVFs[mdev.VFAddress]; !ok { log.DebugContext(ctx, "skipping mdev on unmanaged VF", "uuid", mdev.UUID, "vf", mdev.VFAddress) skippedUnmanagedVF++ continue } + if _, ok := protectedMdevs[mdev.UUID]; ok { + log.DebugContext(ctx, "skipping mdev claimed by live instance", "uuid", mdev.UUID) + skippedClaimed++ + continue + } group, err := mdevIOMMUGroup(mdev.UUID) if err != nil { @@ -759,7 +796,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro continue } - pastGracePeriod, age, err := mdevPastGracePeriod(mdev.UUID, orphanedMdevGracePeriod) + pastGracePeriod, age, err := mdevPastGracePeriod(mdev.UUID, VGPUAssignmentGracePeriod) if err != nil { log.WarnContext(ctx, "failed to determine mdev age, skipping cleanup", "uuid", mdev.UUID, "error", err) skippedProbeError++ @@ -769,7 +806,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro log.DebugContext(ctx, "skipping recently created mdev during grace period", "uuid", mdev.UUID, "age", age.String(), - "grace_period", orphanedMdevGracePeriod.String(), + "grace_period", VGPUAssignmentGracePeriod.String(), ) skippedGrace++ continue @@ -795,6 +832,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro "destroyed", destroyed, "failed_destroy", failedDestroy, "skipped_unmanaged_vf", skippedUnmanagedVF, + "skipped_claimed", skippedClaimed, "skipped_in_use", skippedInUse, "skipped_grace", skippedGrace, "skipped_probe_error", skippedProbeError, diff --git a/lib/devices/types.go b/lib/devices/types.go index 809d669fe..369bb3268 100644 --- a/lib/devices/types.go +++ b/lib/devices/types.go @@ -60,15 +60,21 @@ func ValidateDeviceName(name string) bool { // GPUMode represents the host's GPU configuration mode type GPUMode string +// VGPUAssignmentGracePeriod protects a fresh vGPU assignment from cleanup +// until its VM has had time to boot and become identifiable — by a persisted +// hypervisor PID, a control-socket owner, or an open VFIO handle. +const VGPUAssignmentGracePeriod = 5 * time.Minute + type VGPUFramework string const ( - VGPUFrameworkNone VGPUFramework = "" - VGPUFrameworkMdev VGPUFramework = "mdev" + VGPUFrameworkNone VGPUFramework = "" + VGPUFrameworkMdev VGPUFramework = "mdev" + VGPUFrameworkVendorVFIO VGPUFramework = "vendor-vfio" // GPUModePassthrough indicates whole GPU VFIO passthrough GPUModePassthrough GPUMode = "passthrough" - // GPUModeVGPU indicates SR-IOV + mdev based vGPU + // GPUModeVGPU indicates vGPU mode GPUModeVGPU GPUMode = "vgpu" // GPUModeNone indicates no GPU available GPUModeNone GPUMode = "none" @@ -76,9 +82,10 @@ const ( // VirtualFunction represents an SR-IOV Virtual Function for vGPU type VirtualFunction struct { - PCIAddress string `json:"pci_address"` // e.g., "0000:82:00.4" - ParentGPU string `json:"parent_gpu"` // e.g., "0000:82:00.0" - Allocated bool `json:"allocated"` // true if a vGPU is assigned to this VF + PCIAddress string `json:"pci_address"` // e.g., "0000:82:00.4" + ParentGPU string `json:"parent_gpu"` // e.g., "0000:82:00.0" + Allocated bool `json:"allocated"` // true if a vGPU is assigned to this VF + ProfileType string `json:"profile_type,omitempty"` } // VGPUAssignment identifies an existing vGPU assignment to release. @@ -86,6 +93,7 @@ type VGPUAssignment struct { Framework VGPUFramework DevicePath string MdevUUID string + InstanceID string } type VGPUDevice struct { @@ -97,6 +105,16 @@ type VGPUDevice struct { MdevUUID string } +// VGPUCreateCleanupPendingError reports a failed create whose assignment could +// not be rolled back. Device identifies the assignment that still needs release. +type VGPUCreateCleanupPendingError struct { + Device VGPUDevice + Err error +} + +func (e *VGPUCreateCleanupPendingError) Error() string { return e.Err.Error() } +func (e *VGPUCreateCleanupPendingError) Unwrap() error { return e.Err } + // MdevDevice represents an active mediated device (vGPU instance) type MdevDevice struct { UUID string `json:"uuid"` // e.g., "aa618089-8b16-4d01-a136-25a0f3c73123" diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go new file mode 100644 index 000000000..0d8a27383 --- /dev/null +++ b/lib/devices/vendor_vfio_linux.go @@ -0,0 +1,593 @@ +//go:build linux + +package devices + +import ( + "context" + "errors" + "fmt" + "log/slog" + "maps" + "math/rand/v2" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "syscall" + "time" + + "github.com/kernel/hypeman/lib/logger" +) + +const ( + pciDevicesPath = "/sys/bus/pci/devices" + vfioDevicesPath = "/dev/vfio/devices" +) + +type vendorVFIOOwner struct { + instanceID string + assignedAt time.Time +} + +type vendorVFIOGPUPlacement struct { + usage int + unknownUsage bool + quarantined int + freeVFs []VirtualFunction +} + +func (p *vendorVFIOGPUPlacement) preferredTo(other *vendorVFIOGPUPlacement, gpu, otherGPU string) bool { + if p.quarantined != other.quarantined { + return p.quarantined < other.quarantined + } + if p.unknownUsage != other.unknownUsage { + return !p.unknownUsage + } + if p.usage != other.usage { + return p.usage < other.usage + } + return gpu < otherGPU +} + +type vendorVFIOSysfs struct { + pciDevicesPath string + procPath string + vfioDevicesPath string + owners map[string]vendorVFIOOwner + framebufferByType map[string]int + openVFIOPathsFunc func() (map[string]struct{}, error) + pickVFIndex func(n int) int +} + +var hostVendorVFIO = vendorVFIOSysfs{ + pciDevicesPath: pciDevicesPath, + procPath: procPath, + vfioDevicesPath: vfioDevicesPath, + owners: make(map[string]vendorVFIOOwner), + framebufferByType: make(map[string]int), +} + +func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { + entries, err := os.ReadDir(s.pciDevicesPath) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read PCI devices: %w", err) + } + + vfs := make([]VirtualFunction, 0) + var vfErrs []error + for _, entry := range entries { + vfPath := filepath.Join(s.pciDevicesPath, entry.Name()) + nvidiaPath := filepath.Join(vfPath, "nvidia") + if _, err := os.Stat(filepath.Join(nvidiaPath, "creatable_vgpu_types")); err != nil { + if os.IsNotExist(err) { + continue + } + vfErrs = append(vfErrs, fmt.Errorf("stat creatable vGPU types for VF %s: %w", entry.Name(), err)) + continue + } + + currentType, err := readCurrentVGPUType(filepath.Join(nvidiaPath, "current_vgpu_type")) + if err != nil { + vfErrs = append(vfErrs, fmt.Errorf("read current vGPU type for VF %s: %w", entry.Name(), err)) + continue + } + + parentGPU := "" + if target, err := os.Readlink(filepath.Join(vfPath, "physfn")); err == nil { + parentGPU = filepath.Base(target) + } + vfs = append(vfs, VirtualFunction{ + PCIAddress: entry.Name(), + ParentGPU: parentGPU, + Allocated: currentType != "0", + ProfileType: currentType, + }) + } + if len(vfErrs) > 0 { + // One unreadable VF must not blank out the host's GPU capacity: skip + // it and keep the readable inventory. A skipped VF is never selected + // for placement and never reconciled, both safe directions. Only when + // no VF is readable does discovery fail, so a wholesale sysfs outage + // cannot demote a vGPU host to passthrough while assignments exist. + if len(vfs) == 0 { + return nil, errors.Join(vfErrs...) + } + slog.Default().Warn("skipping unreadable vendor VFIO VFs", "error", errors.Join(vfErrs...)) + } + + sort.Slice(vfs, func(i, j int) bool { return vfs[i].PCIAddress < vfs[j].PCIAddress }) + return vfs, nil +} + +// listProfiles counts each free VF advertising a type as one creatable +// instance, matching the driver-reported units that mdev sums through +// available_instances. This is a best-effort snapshot because creating on one +// VF may revoke the type from siblings that share its GPU framebuffer. +func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { + quarantined, err := vfHealth.checkedAddresses() + if err != nil { + return nil, err + } + profilesByType := make(map[string]profileMetadata) + creatableVFs := make(map[string]int) + for _, vf := range vfs { + creatable, err := s.readCreatableProfiles(vf.PCIAddress) + if err != nil { + // Mirror discoverVFs: one unreadable VF must not blank the host's + // advertised capacity. Skipping only underreports availability, + // the safe direction for status and admission. + slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) + continue + } + _, bad := quarantined[vf.PCIAddress] + for _, profile := range creatable { + profilesByType[profile.TypeName] = profile + if !vf.Allocated && !bad { + creatableVFs[profile.TypeName]++ + } + } + } + + metadata := make([]profileMetadata, 0, len(profilesByType)) + for _, profile := range profilesByType { + metadata = append(metadata, profile) + } + sort.Slice(metadata, func(i, j int) bool { return metadata[i].Name < metadata[j].Name }) + + profiles := make([]GPUProfile, 0, len(metadata)) + for _, profile := range metadata { + profiles = append(profiles, GPUProfile{ + Name: profile.Name, + FramebufferMB: profile.FramebufferMB, + Available: creatableVFs[profile.TypeName], + }) + } + return profiles, nil +} + +func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) { + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + + vfs, err := s.discoverVFs() + if err != nil { + return nil, err + } + metadata, err := s.profileMetadata(vfs) + if err != nil { + return nil, err + } + + var requested profileMetadata + found := false + for _, profile := range metadata { + if profile.Name == profileName { + requested = profile + found = true + break + } + } + if !found { + if len(metadata) == 0 && len(vfs) > 0 { + return nil, fmt.Errorf("no creatable vGPU profiles on any VF, GPUs may be at capacity: profile %q", profileName) + } + return nil, fmt.Errorf("profile %q is not creatable on any VF (unknown profile or insufficient capacity)", profileName) + } + + targetVF, err := s.selectLeastLoadedVF(vfs, requested.TypeName) + if err != nil { + return nil, err + } + if targetVF == "" { + return nil, fmt.Errorf("no available VF for profile %q", profileName) + } + + currentTypePath := filepath.Join(s.pciDevicesPath, targetVF, "nvidia", "current_vgpu_type") + if err := os.WriteFile(currentTypePath, []byte(requested.TypeName), 0200); err != nil { + return nil, fmt.Errorf("create vGPU on VF %s: %w", targetVF, err) + } + device := VGPUDevice{ + Framework: VGPUFrameworkVendorVFIO, + VFAddress: targetVF, + ProfileType: requested.TypeName, + ProfileName: profileName, + SysfsPath: filepath.Join(s.pciDevicesPath, targetVF), + } + currentType, err := readCurrentVGPUType(currentTypePath) + if err != nil { + verifyErr := fmt.Errorf("verify vGPU on VF %s: %w", targetVF, err) + return nil, s.rollbackCreate(currentTypePath, targetVF, instanceID, device, verifyErr) + } + if currentType != requested.TypeName { + verifyErr := fmt.Errorf("verify vGPU on VF %s: type is %s, want %s", targetVF, currentType, requested.TypeName) + return nil, s.rollbackCreate(currentTypePath, targetVF, instanceID, device, verifyErr) + } + s.owners[targetVF] = vendorVFIOOwner{instanceID: instanceID, assignedAt: time.Now()} + + logger.FromContext(ctx).InfoContext(ctx, "created vendor VFIO vGPU", + "profile", profileName, + "vf", targetVF, + "instance_id", instanceID, + ) + return &device, nil +} + +func (s vendorVFIOSysfs) destroy(ctx context.Context, vfAddress, instanceID string) error { + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + + log := logger.FromContext(ctx) + currentTypePath := filepath.Join(s.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type") + currentType, err := readCurrentVGPUType(currentTypePath) + if err != nil { + if os.IsNotExist(err) { + delete(s.owners, vfAddress) + return nil + } + return fmt.Errorf("read current vGPU type for VF %s: %w", vfAddress, err) + } + if currentType == "0" { + delete(s.owners, vfAddress) + return nil + } + + if owner, ok := s.owners[vfAddress]; ok { + if instanceID == "" { + return fmt.Errorf("cannot release vendor VFIO vGPU on VF %s without instance ID", vfAddress) + } + if owner.instanceID != instanceID { + log.WarnContext(ctx, "skipping vendor VFIO vGPU release owned by another instance", + "vf", vfAddress, + "owner_instance_id", owner.instanceID, + "requesting_instance_id", instanceID, + ) + return nil + } + } + + openPaths, err := s.openVFIOPaths() + if err != nil { + return fmt.Errorf("scan open VFIO handles: %w", err) + } + inUse, err := s.vfioDeviceInUse(vfAddress, openPaths) + if err != nil { + return fmt.Errorf("check vendor VFIO vGPU usage for VF %s: %w", vfAddress, err) + } + if inUse { + return fmt.Errorf("vendor VFIO vGPU on VF %s is still in use", vfAddress) + } + + if err := os.WriteFile(currentTypePath, []byte("0"), 0200); err != nil { + return fmt.Errorf("destroy vGPU on VF %s: %w", vfAddress, err) + } + delete(s.owners, vfAddress) + log.InfoContext(ctx, "destroyed vendor VFIO vGPU", "vf", vfAddress) + return nil +} + +func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map[string]struct{}) error { + vfs, err := s.discoverVFs() + if err != nil { + return err + } + vendorVFIOMu.Lock() + owners := maps.Clone(s.owners) + vendorVFIOMu.Unlock() + log := logger.FromContext(ctx) + protectedVFs := make(map[string]struct{}, len(protectedDevicePaths)) + for path := range protectedDevicePaths { + protectedVFs[filepath.Base(path)] = struct{}{} + } + var openPaths map[string]struct{} + for _, vf := range vfs { + if !vf.Allocated { + continue + } + if _, ok := protectedVFs[vf.PCIAddress]; ok { + log.DebugContext(ctx, "skipping vendor VFIO vGPU held by a live instance", "vf", vf.PCIAddress) + continue + } + owner := owners[vf.PCIAddress] + if !owner.assignedAt.IsZero() && time.Since(owner.assignedAt) < VGPUAssignmentGracePeriod { + log.DebugContext(ctx, "skipping recently assigned vendor VFIO vGPU during grace period", "vf", vf.PCIAddress) + continue + } + if openPaths == nil { + if openPaths, err = s.openVFIOPaths(); err != nil { + return fmt.Errorf("scan open VFIO handles: %w", err) + } + } + inUse, err := s.vfioDeviceInUse(vf.PCIAddress, openPaths) + if err != nil { + log.WarnContext(ctx, "failed to check vendor VFIO vGPU usage", "vf", vf.PCIAddress, "error", err) + continue + } + if inUse { + log.WarnContext(ctx, "preserving vendor VFIO vGPU held open without a live instance claim", "vf", vf.PCIAddress) + continue + } + if err := s.destroy(ctx, vf.PCIAddress, owner.instanceID); err != nil { + log.WarnContext(ctx, "failed to destroy orphaned vendor VFIO vGPU", "vf", vf.PCIAddress, "error", err) + } + } + return nil +} + +func (s vendorVFIOSysfs) addVFToPlacement(placement *vendorVFIOGPUPlacement, vf VirtualFunction, profileType string, quarantined bool) { + if quarantined { + placement.quarantined++ + if !vf.Allocated { + return + } + } + if vf.Allocated { + // framebufferByType only covers currently creatable profiles, so + // after a restart an allocated type can be missing when its + // capacity is exhausted. Prefer GPUs whose load is fully known + // instead of rejecting placement outright; the kernel driver + // still enforces real capacity through creatable_vgpu_types. + framebuffer, ok := s.framebufferByType[vf.ProfileType] + if !ok { + placement.unknownUsage = true + return + } + placement.usage += framebuffer + return + } + profiles, err := s.readCreatableProfiles(vf.PCIAddress) + if err != nil { + // An unreadable free VF is just not a placement candidate. + slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) + return + } + for _, profile := range profiles { + if profile.TypeName == profileType { + placement.freeVFs = append(placement.freeVFs, vf) + return + } + } +} + +func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType string) (string, error) { + quarantined, err := vfHealth.checkedAddresses() + if err != nil { + return "", err + } + placementByGPU := make(map[string]*vendorVFIOGPUPlacement) + for _, vf := range vfs { + placement := placementByGPU[vf.ParentGPU] + if placement == nil { + placement = &vendorVFIOGPUPlacement{} + placementByGPU[vf.ParentGPU] = placement + } + _, bad := quarantined[vf.PCIAddress] + s.addVFToPlacement(placement, vf, profileType, bad) + } + + gpus := make([]string, 0, len(placementByGPU)) + for gpu, placement := range placementByGPU { + if len(placement.freeVFs) > 0 { + gpus = append(gpus, gpu) + } + } + sort.Slice(gpus, func(i, j int) bool { + return placementByGPU[gpus[i]].preferredTo(placementByGPU[gpus[j]], gpus[i], gpus[j]) + }) + if len(gpus) == 0 { + return "", nil + } + candidates := placementByGPU[gpus[0]].freeVFs + pick := s.pickVFIndex + if pick == nil { + pick = rand.IntN + } + return candidates[pick(len(candidates))].PCIAddress, nil +} + +func (s vendorVFIOSysfs) profileMetadata(vfs []VirtualFunction) ([]profileMetadata, error) { + profilesByType := make(map[string]profileMetadata) + for _, vf := range vfs { + profiles, err := s.readCreatableProfiles(vf.PCIAddress) + if err != nil { + // Mirror listProfiles: an unreadable VF must not fail placement + // while /resources still advertises the remaining capacity. + slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) + continue + } + for _, profile := range profiles { + profilesByType[profile.TypeName] = profile + s.framebufferByType[profile.TypeName] = profile.FramebufferMB + } + } + profiles := make([]profileMetadata, 0, len(profilesByType)) + for _, profile := range profilesByType { + profiles = append(profiles, profile) + } + sort.Slice(profiles, func(i, j int) bool { return profiles[i].Name < profiles[j].Name }) + return profiles, nil +} + +func (s vendorVFIOSysfs) readCreatableProfiles(vfAddress string) ([]profileMetadata, error) { + path := filepath.Join(s.pciDevicesPath, vfAddress, "nvidia", "creatable_vgpu_types") + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read creatable vGPU types for VF %s: %w", vfAddress, err) + } + return parseCreatableVGPUTypes(string(data)) +} + +func (s vendorVFIOSysfs) vfioDeviceInUse(vfAddress string, openPaths map[string]struct{}) (bool, error) { + devicePaths := make([]string, 0, 2) + probeErrs := make([]error, 0, 2) + + vfioDevices, err := os.ReadDir(filepath.Join(s.pciDevicesPath, vfAddress, "vfio-dev")) + if err != nil { + if !os.IsNotExist(err) { + probeErrs = append(probeErrs, fmt.Errorf("read VFIO devices for VF %s: %w", vfAddress, err)) + } + } else { + for _, device := range vfioDevices { + devicePaths = append(devicePaths, filepath.Join(s.vfioDevicesPath, device.Name())) + } + } + + target, err := os.Readlink(filepath.Join(s.pciDevicesPath, vfAddress, "iommu_group")) + if err != nil { + if !os.IsNotExist(err) { + probeErrs = append(probeErrs, fmt.Errorf("read IOMMU group for VF %s: %w", vfAddress, err)) + } + } else { + devicePaths = append(devicePaths, filepath.Join(filepath.Dir(s.vfioDevicesPath), filepath.Base(target))) + } + + for _, path := range devicePaths { + if _, ok := openPaths[path]; ok { + return true, nil + } + } + if len(probeErrs) > 0 { + return false, errors.Join(probeErrs...) + } + return false, nil +} + +// openVFIOPaths hard-fails on any unreadable /proc entry, unlike mdev's +// isVFIOGroupInUse which skips them. That is deliberate: this scan authorizes +// clearing current_vgpu_type on a VF path that is reused across assignments, +// so an incomplete scan must fail the release (which retains metadata for a +// later retry) rather than risk a false "not in use" answer. The exception +// is a process that exits mid-scan (ENOENT/ESRCH): a dead process holds +// nothing open, so skipping it cannot produce that false answer. +func (s vendorVFIOSysfs) openVFIOPaths() (map[string]struct{}, error) { + if s.openVFIOPathsFunc != nil { + return s.openVFIOPathsFunc() + } + processes, err := os.ReadDir(s.procPath) + if err != nil { + return nil, err + } + prefix := filepath.Dir(s.vfioDevicesPath) + string(filepath.Separator) + open := make(map[string]struct{}) + for _, process := range processes { + if _, err := strconv.Atoi(process.Name()); err != nil { + continue + } + fdPath := filepath.Join(s.procPath, process.Name(), "fd") + fds, err := os.ReadDir(fdPath) + if err != nil { + if os.IsNotExist(err) || errors.Is(err, syscall.ESRCH) { + continue + } + return nil, fmt.Errorf("read process %s file descriptors: %w", process.Name(), err) + } + for _, fd := range fds { + target, err := os.Readlink(filepath.Join(fdPath, fd.Name())) + if err != nil { + if os.IsNotExist(err) || errors.Is(err, syscall.ESRCH) { + continue + } + return nil, fmt.Errorf("read process %s file descriptor %s: %w", process.Name(), fd.Name(), err) + } + if strings.HasPrefix(target, prefix) { + open[target] = struct{}{} + } + } + } + return open, nil +} + +func parseCreatableVGPUTypes(value string) ([]profileMetadata, error) { + profiles := make([]profileMetadata, 0) + for lineNumber, line := range strings.Split(value, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + typeID, name, found := strings.Cut(line, ":") + typeID = strings.TrimSpace(typeID) + name = strings.TrimSpace(name) + if typeID == "ID" { + continue + } + if !found || name == "" { + return nil, fmt.Errorf("parse creatable vGPU types line %d: %q", lineNumber+1, line) + } + if _, err := strconv.Atoi(typeID); err != nil { + return nil, fmt.Errorf("parse vGPU type ID %q: %w", typeID, err) + } + profiles = append(profiles, profileMetadata{ + TypeName: typeID, + Name: name, + FramebufferMB: framebufferFromProfileName(name), + }) + } + return profiles, nil +} + +// framebufferFromProfileName parses the framebuffer size from names like +// "NVIDIA L40S-12Q". NVIDIA's 512 MB 0Q/0B profiles parse as 0 and would read +// as free VRAM in least-loaded placement; no supported GPU exposes them today, +// so placement only skews if such a profile ever appears. +func framebufferFromProfileName(name string) int { + series := strings.LastIndexAny(name, "ABCQ") + if series <= 0 { + return 0 + } + dash := strings.LastIndex(name[:series], "-") + if dash < 0 { + return 0 + } + gb, err := strconv.Atoi(name[dash+1 : series]) + if err != nil { + return 0 + } + return gb * 1024 +} + +func (s vendorVFIOSysfs) rollbackCreate(currentTypePath, vfAddress, instanceID string, device VGPUDevice, verifyErr error) error { + if err := os.WriteFile(currentTypePath, []byte("0"), 0200); err != nil { + s.owners[vfAddress] = vendorVFIOOwner{instanceID: instanceID, assignedAt: time.Now()} + return &VGPUCreateCleanupPendingError{ + Device: device, + Err: errors.Join(verifyErr, fmt.Errorf("roll back vGPU on VF %s: %w", vfAddress, err)), + } + } + return verifyErr +} + +func readCurrentVGPUType(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + value := strings.TrimSpace(string(data)) + if _, err := strconv.Atoi(value); err != nil { + return "", fmt.Errorf("invalid current vGPU type %q", value) + } + return value, nil +} diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go new file mode 100644 index 000000000..a65fd0a88 --- /dev/null +++ b/lib/devices/vendor_vfio_linux_test.go @@ -0,0 +1,689 @@ +//go:build linux + +package devices + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testCreatableTypes = `ID : vGPU Name +1147 : NVIDIA L40S-1Q +1148 : NVIDIA L40S-2Q +1159 : NVIDIA L40S-48Q +` + +func TestParseCreatableVGPUTypes(t *testing.T) { + t.Parallel() + + profiles, err := parseCreatableVGPUTypes(testCreatableTypes) + require.NoError(t, err) + require.Len(t, profiles, 3) + assert.Equal(t, profileMetadata{TypeName: "1147", Name: "NVIDIA L40S-1Q", FramebufferMB: 1024}, profiles[0]) + assert.Equal(t, profileMetadata{TypeName: "1159", Name: "NVIDIA L40S-48Q", FramebufferMB: 48 * 1024}, profiles[2]) +} + +func TestVendorVFIODiscoverSkipsUnreadableVF(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + // Replace one VF's current_vgpu_type with a directory so reading it fails. + badType := filepath.Join(sysfs.pciDevicesPath, "0000:82:00.5", "nvidia", "current_vgpu_type") + require.NoError(t, os.Remove(badType)) + require.NoError(t, os.Mkdir(badType, 0755)) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + require.Len(t, vfs, 1) + assert.Equal(t, "0000:82:00.4", vfs[0].PCIAddress) +} + +func TestVendorVFIODiscoverFailsWhenNoVFIsReadable(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + badType := filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type") + require.NoError(t, os.Remove(badType)) + require.NoError(t, os.Mkdir(badType, 0755)) + + // With every VF unreadable, discovery must fail rather than report an + // empty inventory that would demote the host to passthrough. + _, err := sysfs.discoverVFs() + require.Error(t, err) +} + +func TestVendorVFIOListProfilesSkipsUnreadableVF(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + badCreatable := filepath.Join(sysfs.pciDevicesPath, "0000:82:00.5", "nvidia", "creatable_vgpu_types") + require.NoError(t, os.Remove(badCreatable)) + require.NoError(t, os.Mkdir(badCreatable, 0755)) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + require.NotEmpty(t, profiles, "readable VFs must keep advertising capacity") + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-1Q"), + "only the readable VF counts toward availability") +} + +func TestVendorVFIOCreateSkipsUnreadableVF(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + badCreatable := filepath.Join(sysfs.pciDevicesPath, "0000:82:00.5", "nvidia", "creatable_vgpu_types") + require.NoError(t, os.Remove(badCreatable)) + require.NoError(t, os.Mkdir(badCreatable, 0755)) + + // Create places on the readable VF instead of failing the placement. + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "inst-1") + require.NoError(t, err) + assert.Equal(t, "0000:82:00.4", device.VFAddress) +} + +func TestVendorVFIOCreateAndDestroy(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + require.Len(t, vfs, 1) + assert.False(t, vfs[0].Allocated) + assert.Equal(t, "0000:82:00.0", vfs[0].ParentGPU) + + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-2Q")) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-2Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, VGPUFrameworkVendorVFIO, device.Framework) + assert.Equal(t, "0000:82:00.4", device.VFAddress) + assert.Equal(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4"), device.SysfsPath) + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "1148") + + require.NoError(t, sysfs.destroy(context.Background(), device.VFAddress, "instance-1")) + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "0") +} + +func TestVendorVFIODestroySkipsAssignmentOwnedByAnotherInstance(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-2Q", "instance-1") + require.NoError(t, err) + + require.NoError(t, sysfs.destroy(context.Background(), device.VFAddress, "stale-instance")) + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "1148") + + require.NoError(t, sysfs.destroy(context.Background(), device.VFAddress, "instance-1")) + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "0") +} + +func TestVendorVFIODestroyRejectsMissingInstanceIDForOwnedVF(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-2Q", "instance-1") + require.NoError(t, err) + + err = sysfs.destroy(context.Background(), device.VFAddress, "") + require.ErrorContains(t, err, "without instance ID") + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIODestroyRetainsAssignmentInUse(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + + activeDevice := filepath.Join(sysfs.vfioDevicesPath, "vfio42") + fdDir := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdDir, 0755)) + require.NoError(t, os.Symlink(activeDevice, filepath.Join(fdDir, "5"))) + + err := sysfs.destroy(context.Background(), vfAddress, "instance-1") + require.Error(t, err) + assert.ErrorContains(t, err, "still in use") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIODestroyReleasesUnboundVF(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + unbind func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) + }{ + { + name: "missing vfio device directory", + unbind: func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) { + require.NoError(t, os.RemoveAll(filepath.Join(sysfs.pciDevicesPath, vfAddress, "vfio-dev"))) + }, + }, + { + name: "missing iommu group symlink", + unbind: func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) { + require.NoError(t, os.Remove(filepath.Join(sysfs.pciDevicesPath, vfAddress, "iommu_group"))) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + tt.unbind(t, sysfs, vfAddress) + + require.NoError(t, sysfs.destroy(context.Background(), vfAddress, "instance-1")) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "0") + }) + } +} + +func TestVendorVFIODestroyRetainsAssignmentWhenOneVFIOPathIsMissing(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + remove func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) + open func(sysfs testVendorVFIOSysfs) string + }{ + { + name: "missing iommu group", + remove: func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) { + require.NoError(t, os.Remove(filepath.Join(sysfs.pciDevicesPath, vfAddress, "iommu_group"))) + }, + open: func(sysfs testVendorVFIOSysfs) string { + return filepath.Join(sysfs.vfioDevicesPath, "vfio42") + }, + }, + { + name: "missing vfio device directory", + remove: func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) { + require.NoError(t, os.RemoveAll(filepath.Join(sysfs.pciDevicesPath, vfAddress, "vfio-dev"))) + }, + open: func(sysfs testVendorVFIOSysfs) string { + return filepath.Join(filepath.Dir(sysfs.vfioDevicesPath), "42") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + tt.remove(t, sysfs, vfAddress) + + fdDir := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdDir, 0755)) + require.NoError(t, os.Symlink(tt.open(sysfs), filepath.Join(fdDir, "5"))) + + err := sysfs.destroy(context.Background(), vfAddress, "instance-1") + require.ErrorContains(t, err, "still in use") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") + }) + } +} + +func TestVendorVFIOListProfilesCountsFreeVFs(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-48Q"), + "each free VF advertising the type counts as one creatable instance") +} + +func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "ID : vGPU Name\n") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "1159", "") + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Empty(t, profiles) + + _, err = sysfs.create(context.Background(), "NVIDIA L40S-2Q", "instance-1") + require.Error(t, err) + assert.ErrorContains(t, err, "GPUs may be at capacity") +} + +func TestVendorVFIOCreateReportsAmbiguousMissingProfile(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "ID : vGPU Name\n1148 : NVIDIA L40S-2Q\n") + + _, err := sysfs.create(context.Background(), "NVIDIA L40S-48Q", "instance-1") + require.Error(t, err) + assert.ErrorContains(t, err, "not creatable on any VF") + assert.ErrorContains(t, err, "unknown profile or insufficient capacity") +} + +func TestVendorVFIOSelectsLeastLoadedGPU(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress) +} + +func TestVendorVFIOSelectsLeastLoadedGPUWithConsumedType(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + _, err = sysfs.profileMetadata(vfs) + require.NoError(t, err) + for _, vfAddress := range []string{"0000:82:00.5", "0000:e3:00.4"} { + creatableTypesPath := filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "creatable_vgpu_types") + require.NoError(t, os.Chmod(creatableTypesPath, 0644)) + require.NoError(t, os.WriteFile(creatableTypesPath, []byte("1147 : NVIDIA L40S-1Q\n"), 0444)) + } + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress) +} + +func TestVendorVFIOPlacementPrefersKnownLoadWhenAllocatedTypeIsUnknown(t *testing.T) { + t.Parallel() + + // Simulates a restart: type 1159 is allocated but no longer creatable + // anywhere, so its framebuffer size is unknown. + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "1147 : NVIDIA L40S-1Q\n") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", "1147 : NVIDIA L40S-1Q\n") + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress, "the GPU with unknown load should be picked last") +} + +func TestVendorVFIOPlacesOnGPUWithUnknownLoadWhenItHasTheOnlyCapacity(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "1147 : NVIDIA L40S-1Q\n") + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", device.VFAddress) +} + +func TestVendorVFIOReconcile(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "43", "1148", "") + + activeDevice := filepath.Join(sysfs.vfioDevicesPath, "vfio43") + fdDir := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdDir, 0755)) + require.NoError(t, os.Symlink(activeDevice, filepath.Join(fdDir, "5"))) + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "0") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:e3:00.4", "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcileSkipsRecentlyAssignedVF(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.owners["0000:82:00.4"] = vendorVFIOOwner{instanceID: "mid-create", assignedAt: time.Now()} + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcileDestroysOwnedVFPastGracePeriod(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.owners["0000:82:00.4"] = vendorVFIOOwner{ + instanceID: "deleted-instance", + assignedAt: time.Now().Add(-VGPUAssignmentGracePeriod - time.Minute), + } + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "0") + assert.Empty(t, sysfs.owners) +} + +func TestVendorVFIOReconcileRechecksOpenHandlesBeforeDestroy(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + + activeDevice := filepath.Join(sysfs.vfioDevicesPath, "vfio42") + scans := 0 + sysfs.openVFIOPathsFunc = func() (map[string]struct{}, error) { + scans++ + if scans == 1 { + return map[string]struct{}{}, nil + } + return map[string]struct{}{activeDevice: {}}, nil + } + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assert.Equal(t, 2, scans) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcileSkipsProtectedVF(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "43", "1148", "") + + protected := map[string]struct{}{ + filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4"): {}, + } + require.NoError(t, sysfs.reconcile(context.Background(), protected)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "1148") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:e3:00.4", "nvidia", "current_vgpu_type"), "0") +} + +func TestVendorVFIOReconcilePreservesLegacyGroupFD(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "43", "1148", "") + + legacyGroup := filepath.Join(filepath.Dir(sysfs.vfioDevicesPath), "43") + fdDir := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdDir, 0755)) + require.NoError(t, os.Symlink(legacyGroup, filepath.Join(fdDir, "5"))) + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "0") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:e3:00.4", "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcilePreservesVFWhenVFIODeviceProbeFails(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + vfioDevPath := filepath.Join(sysfs.pciDevicesPath, vfAddress, "vfio-dev") + require.NoError(t, os.RemoveAll(vfioDevPath)) + require.NoError(t, os.WriteFile(vfioDevPath, nil, 0644)) + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcilePreservesVFWhenIOMMUGroupProbeFails(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + iommuGroupPath := filepath.Join(sysfs.pciDevicesPath, vfAddress, "iommu_group") + require.NoError(t, os.Remove(iommuGroupPath)) + require.NoError(t, os.WriteFile(iommuGroupPath, nil, 0644)) + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcilePreservesVFWhenProcFDDirectoryScanFails(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + + processPath := filepath.Join(sysfs.procPath, "123") + require.NoError(t, os.MkdirAll(processPath, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(processPath, "fd"), nil, 0644)) + + err := sysfs.reconcile(context.Background(), nil) + require.Error(t, err) + assert.ErrorContains(t, err, "read process 123 file descriptors") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcilePreservesVFWhenProcFDLinkScanFails(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + + fdPath := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdPath, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(fdPath, "5"), nil, 0644)) + + err := sysfs.reconcile(context.Background(), nil) + require.Error(t, err) + assert.ErrorContains(t, err, "read process 123 file descriptor 5") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestParseCreatableVGPUTypesHeaderOnly(t *testing.T) { + t.Parallel() + + profiles, err := parseCreatableVGPUTypes("ID : vGPU Name\n") + require.NoError(t, err) + assert.Empty(t, profiles) +} + +func TestParseCreatableVGPUTypesRejectsMalformedLine(t *testing.T) { + t.Parallel() + + _, err := parseCreatableVGPUTypes("NVIDIA") + require.Error(t, err) + + _, err = parseCreatableVGPUTypes("not-an-id : NVIDIA L40S-1Q") + require.Error(t, err) +} + +func TestRollbackVendorVFIOCreate(t *testing.T) { + t.Parallel() + + verifyErr := errors.New("verification failed") + device := VGPUDevice{ + Framework: VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + t.Run("preserves verification error", func(t *testing.T) { + currentTypePath := filepath.Join(t.TempDir(), "current_vgpu_type") + require.NoError(t, os.WriteFile(currentTypePath, []byte("1148"), 0644)) + sysfs := vendorVFIOSysfs{owners: make(map[string]vendorVFIOOwner)} + + err := sysfs.rollbackCreate(currentTypePath, device.VFAddress, "instance-1", device, verifyErr) + require.ErrorIs(t, err, verifyErr) + assertFileValue(t, currentTypePath, "0") + assert.Empty(t, sysfs.owners) + }) + + t.Run("retains assignment when rollback fails", func(t *testing.T) { + currentTypePath := filepath.Join(t.TempDir(), "missing", "current_vgpu_type") + sysfs := vendorVFIOSysfs{owners: make(map[string]vendorVFIOOwner)} + + err := sysfs.rollbackCreate(currentTypePath, device.VFAddress, "instance-1", device, verifyErr) + require.ErrorIs(t, err, verifyErr) + assert.ErrorContains(t, err, "roll back vGPU on VF 0000:82:00.4") + var pending *VGPUCreateCleanupPendingError + require.ErrorAs(t, err, &pending) + assert.Equal(t, device, pending.Device) + assert.Equal(t, "instance-1", sysfs.owners[device.VFAddress].instanceID) + assert.False(t, sysfs.owners[device.VFAddress].assignedAt.IsZero()) + }) +} + +func profileAvailability(profiles []GPUProfile, name string) int { + for _, profile := range profiles { + if profile.Name == name { + return profile.Available + } + } + return -1 +} + +type testVendorVFIOSysfs struct { + vendorVFIOSysfs +} + +func newTestVendorVFIOSysfs(t *testing.T) testVendorVFIOSysfs { + t.Helper() + root := t.TempDir() + pci := filepath.Join(root, "sys", "bus", "pci", "devices") + proc := filepath.Join(root, "proc") + vfio := filepath.Join(root, "dev", "vfio", "devices") + require.NoError(t, os.MkdirAll(pci, 0755)) + require.NoError(t, os.MkdirAll(proc, 0755)) + require.NoError(t, os.MkdirAll(vfio, 0755)) + return testVendorVFIOSysfs{vendorVFIOSysfs{ + pciDevicesPath: pci, + procPath: proc, + vfioDevicesPath: vfio, + owners: make(map[string]vendorVFIOOwner), + framebufferByType: make(map[string]int), + }} +} + +func (s testVendorVFIOSysfs) addVF(t *testing.T, parent, address, vfioID, currentType, creatableTypes string) { + t.Helper() + parentPath := filepath.Join(s.pciDevicesPath, parent) + vfPath := filepath.Join(s.pciDevicesPath, address) + nvidiaPath := filepath.Join(vfPath, "nvidia") + require.NoError(t, os.MkdirAll(parentPath, 0755)) + require.NoError(t, os.MkdirAll(nvidiaPath, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "current_vgpu_type"), []byte(currentType), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "creatable_vgpu_types"), []byte(creatableTypes), 0444)) + require.NoError(t, os.Symlink(parentPath, filepath.Join(vfPath, "physfn"))) + vfioName := "vfio" + vfioID + require.NoError(t, os.MkdirAll(filepath.Join(vfPath, "vfio-dev", vfioName), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(s.vfioDevicesPath, vfioName), nil, 0600)) + require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "kernel", "iommu_groups", vfioID), filepath.Join(vfPath, "iommu_group"))) +} + +func assertFileValue(t *testing.T, path, expected string) { + t.Helper() + value, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, expected, string(value)) +} + +func TestVendorVFIOSkipsQuarantinedVF(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.pickVFIndex = func(int) int { return 0 } + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", device.VFAddress) +} + +func TestVendorVFIONoVFWhenAllQuarantined(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + + _, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.ErrorContains(t, err, "no available VF") +} + +func TestVendorVFIOCardBiasAvoidsGPUWithQuarantinedVF(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.pickVFIndex = func(int) int { return 0 } + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress) +} + +func TestVendorVFIOSelectUsesTiebreakAmongFreeVFs(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.pickVFIndex = func(n int) int { return n - 1 } + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", device.VFAddress) +} + +func TestVendorVFIOListProfilesExcludesQuarantinedFromAvailability(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-1Q")) +} diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go new file mode 100644 index 000000000..3ae8d02c0 --- /dev/null +++ b/lib/devices/vf_health.go @@ -0,0 +1,482 @@ +package devices + +import ( + "encoding/json" + "fmt" + "log/slog" + "os" + "path/filepath" + "regexp" + "sort" + "sync" + "time" +) + +const ( + vfHealthFileVersion = 1 + defaultVFQuarantineThreshold = 2 +) + +type vfInitFailure struct { + InstanceID string `json:"instance_id,omitempty"` + AssignedAt string `json:"assigned_at,omitempty"` + ReportedAt time.Time `json:"reported_at"` +} + +type vfHealthRecord struct { + VFAddress string `json:"vf_address"` + Failures []vfInitFailure `json:"failures,omitempty"` + QuarantinedAt *time.Time `json:"quarantined_at,omitempty"` +} + +type vfHealthFile struct { + Version int `json:"version"` + Records []vfHealthRecord `json:"records"` +} + +// VFInitFailureReport describes one guest-reported driver init failure. +type VFInitFailureReport struct { + VFAddress string + InstanceID string + AssignedAt string +} + +// VFInitSuccessReport identifies the assignment that successfully initialized. +type VFInitSuccessReport struct { + VFAddress string + InstanceID string + AssignedAt string +} + +// VFReportOutcome describes how a failure report changed a VF's health state. +type VFReportOutcome int + +const ( + // VFReportUnchanged means the VF was already quarantined or this + // assignment was already recorded. + VFReportUnchanged VFReportOutcome = iota + // VFReportRecorded means the failure was tallied below the quarantine threshold. + VFReportRecorded + // VFReportQuarantined means this report crossed the threshold and quarantined the VF. + VFReportQuarantined +) + +// VFReportResult is the outcome of recording a driver init failure. +type VFReportResult struct { + Outcome VFReportOutcome + Failures int + Threshold int +} + +// VFSuccessResult describes how a successful init changed a VF's health state. +type VFSuccessResult struct { + Cleared int + Rescinded bool +} + +type vfHealthStore struct { + mu sync.Mutex + path string + records map[string]vfHealthRecord + threshold int + loadErr error + persistErr error + syncDirFunc func(string) error +} + +var vfHealthAddressPattern = regexp.MustCompile(`^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$`) + +var ( + vfHealth = &vfHealthStore{ + records: make(map[string]vfHealthRecord), + threshold: defaultVFQuarantineThreshold, + syncDirFunc: syncDir, + } + // vendorVFIOMu is acquired before vfHealth.mu. It serializes quarantine + // mutations with vendor-VFIO create, destroy, and reconciliation so + // placement cannot select a VF while it is being quarantined. + vendorVFIOMu sync.Mutex +) + +func initVFHealth(path string) error { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.path = path + return vfHealth.loadLocked() +} + +// SetVFQuarantineThreshold configures the number of failed assignments +// required to quarantine a VF. Already-recorded tallies are re-evaluated so a +// lowered threshold applies to failures persisted before the change. +func SetVFQuarantineThreshold(n int) { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.threshold = n + vfHealth.requarantineLocked() +} + +// requarantineLocked quarantines records whose failure tallies meet the +// current threshold, so threshold changes and loaded state agree. +func (s *vfHealthStore) requarantineLocked() { + changed := false + for address, record := range s.records { + if record.QuarantinedAt != nil || len(record.Failures) < s.threshold { + continue + } + now := time.Now().UTC() + record.QuarantinedAt = &now + s.records[address] = record + changed = true + } + if changed { + if _, err := s.persistLocked(); err != nil { + slog.Default().Error("failed to persist re-evaluated VF quarantines; vGPU placement is disabled until a write succeeds", "error", err) + } + } +} + +func (s *vfHealthStore) loadLocked() error { + s.records = make(map[string]vfHealthRecord) + s.loadErr = nil + s.persistErr = nil + + data, err := os.ReadFile(s.path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + s.loadErr = fmt.Errorf("read VF health state: %w", err) + return s.loadErr + } + var state vfHealthFile + if err := json.Unmarshal(data, &state); err != nil { + s.loadErr = fmt.Errorf("unmarshal VF health state: %w", err) + return s.loadErr + } + if state.Version != vfHealthFileVersion { + s.loadErr = fmt.Errorf("validate VF health state: unsupported version %d", state.Version) + return s.loadErr + } + if state.Records == nil { + s.loadErr = fmt.Errorf("validate VF health state: expected a records array") + return s.loadErr + } + loaded := make(map[string]vfHealthRecord, len(state.Records)) + for i, record := range state.Records { + if !vfHealthAddressPattern.MatchString(record.VFAddress) { + s.loadErr = fmt.Errorf("validate VF health state record %d: invalid VF address %q", i, record.VFAddress) + return s.loadErr + } + if record.QuarantinedAt != nil && record.QuarantinedAt.IsZero() { + s.loadErr = fmt.Errorf("validate VF health state record %d: missing quarantine timestamp", i) + return s.loadErr + } + if record.QuarantinedAt == nil && len(record.Failures) == 0 { + s.loadErr = fmt.Errorf("validate VF health state record %d: neither quarantined nor any recorded failures", i) + return s.loadErr + } + assignments := make(map[string]struct{}, len(record.Failures)) + for j, failure := range record.Failures { + if failure.ReportedAt.IsZero() { + s.loadErr = fmt.Errorf("validate VF health state record %d failure %d: missing report timestamp", i, j) + return s.loadErr + } + key := failure.InstanceID + "\x00" + failure.AssignedAt + if _, exists := assignments[key]; exists { + s.loadErr = fmt.Errorf("validate VF health state record %d: duplicate failure for assignment %q", i, failure.InstanceID) + return s.loadErr + } + assignments[key] = struct{}{} + } + if _, exists := loaded[record.VFAddress]; exists { + s.loadErr = fmt.Errorf("validate VF health state record %d: duplicate VF address %q", i, record.VFAddress) + return s.loadErr + } + loaded[record.VFAddress] = record + } + s.records = loaded + s.requarantineLocked() + return nil +} + +func (s *vfHealthStore) ensureLoadedLocked() error { + if s.loadErr == nil { + return nil + } + return s.loadLocked() +} + +func (s *vfHealthStore) checkedAddresses() (map[string]struct{}, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.ensureLoadedLocked(); err != nil { + return nil, fmt.Errorf("VF health state unavailable: %w", err) + } + if s.persistErr != nil { + return nil, fmt.Errorf("VF health state unavailable: last write failed: %w", s.persistErr) + } + addresses := make(map[string]struct{}, len(s.records)) + for address, record := range s.records { + if record.QuarantinedAt != nil { + addresses[address] = struct{}{} + } + } + return addresses, nil +} + +// VGPUAvailability returns free allocatable and quarantined VF counts. +func VGPUAvailability(framework VGPUFramework, vfs []VirtualFunction) (allocatable, quarantined int, err error) { + if framework != VGPUFrameworkVendorVFIO { + return countFreeVFs(vfs, nil), 0, nil + } + addresses, err := vfHealth.checkedAddresses() + if err != nil { + return 0, 0, err + } + for _, vf := range vfs { + if _, ok := addresses[vf.PCIAddress]; ok { + quarantined++ + } + } + return countFreeVFs(vfs, addresses), quarantined, nil +} + +func countFreeVFs(vfs []VirtualFunction, quarantined map[string]struct{}) int { + available := 0 + for _, vf := range vfs { + if vf.Allocated { + continue + } + if _, ok := quarantined[vf.PCIAddress]; !ok { + available++ + } + } + return available +} + +// ReportVFInitFailure records a guest-reported driver init failure and +// quarantines the VF once failures from enough distinct assignments accumulate. +func ReportVFInitFailure(report VFInitFailureReport) (VFReportResult, error) { + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + return vfHealth.reportFailure(report) +} + +// ReportVFInitSuccess clears failures through an exactly matched successful +// assignment. A quarantine is rescinded only when that assignment triggered it. +func ReportVFInitSuccess(report VFInitSuccessReport) (VFSuccessResult, error) { + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + return vfHealth.reportSuccess(report) +} + +// VFHealthStoreUnavailable reports whether persisted state failed to load or +// the last write failed. +func VFHealthStoreUnavailable() bool { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + return vfHealth.loadErr != nil || vfHealth.persistErr != nil +} + +// TotalQuarantinedVFs returns the number of quarantined VFs in persisted state. +func TotalQuarantinedVFs() int { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + count := 0 + for _, record := range vfHealth.records { + if record.QuarantinedAt != nil { + count++ + } + } + return count +} + +func (s *vfHealthStore) sortedRecordsLocked() []vfHealthRecord { + records := make([]vfHealthRecord, 0, len(s.records)) + for _, record := range s.records { + records = append(records, record) + } + sort.Slice(records, func(i, j int) bool { return records[i].VFAddress < records[j].VFAddress }) + return records +} + +func (s *vfHealthStore) reportFailure(report VFInitFailureReport) (VFReportResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.ensureLoadedLocked(); err != nil { + return VFReportResult{}, err + } + if !vfHealthAddressPattern.MatchString(report.VFAddress) { + return VFReportResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress) + } + if err := s.retryPersistLocked(); err != nil { + return VFReportResult{}, err + } + + previous, existed := s.records[report.VFAddress] + result := VFReportResult{Failures: len(previous.Failures), Threshold: s.threshold} + if previous.QuarantinedAt != nil { + return result, nil + } + for _, failure := range previous.Failures { + if sameVFAssignment(failure, report.InstanceID, report.AssignedAt) { + return result, nil + } + } + + record := vfHealthRecord{ + VFAddress: report.VFAddress, + Failures: append(append([]vfInitFailure(nil), previous.Failures...), vfInitFailure{ + InstanceID: report.InstanceID, + AssignedAt: report.AssignedAt, + ReportedAt: time.Now().UTC(), + }), + } + result.Failures = len(record.Failures) + result.Outcome = VFReportRecorded + if result.Failures >= s.threshold { + now := time.Now().UTC() + record.QuarantinedAt = &now + result.Outcome = VFReportQuarantined + } + s.records[report.VFAddress] = record + renamed, err := s.persistLocked() + if err != nil { + if !renamed { + if existed { + s.records[report.VFAddress] = previous + } else { + delete(s.records, report.VFAddress) + } + } + return VFReportResult{}, err + } + return result, nil +} + +func sameVFAssignment(failure vfInitFailure, instanceID, assignedAt string) bool { + return failure.InstanceID == instanceID && failure.AssignedAt == assignedAt +} + +func (s *vfHealthStore) reportSuccess(report VFInitSuccessReport) (VFSuccessResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.ensureLoadedLocked(); err != nil { + return VFSuccessResult{}, err + } + if !vfHealthAddressPattern.MatchString(report.VFAddress) { + return VFSuccessResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress) + } + if err := s.retryPersistLocked(); err != nil { + return VFSuccessResult{}, err + } + previous, ok := s.records[report.VFAddress] + if !ok || len(previous.Failures) == 0 { + return VFSuccessResult{}, nil + } + + match := -1 + for i, failure := range previous.Failures { + if sameVFAssignment(failure, report.InstanceID, report.AssignedAt) { + match = i + break + } + } + if match < 0 || (previous.QuarantinedAt != nil && match != len(previous.Failures)-1) { + return VFSuccessResult{}, nil + } + + remaining := append([]vfInitFailure(nil), previous.Failures[match+1:]...) + result := VFSuccessResult{ + Cleared: len(previous.Failures) - len(remaining), + Rescinded: previous.QuarantinedAt != nil, + } + if len(remaining) == 0 { + delete(s.records, report.VFAddress) + } else { + record := previous + record.Failures = remaining + s.records[report.VFAddress] = record + } + renamed, err := s.persistLocked() + if err != nil { + if !renamed { + s.records[report.VFAddress] = previous + } + return VFSuccessResult{}, err + } + return result, nil +} + +func (s *vfHealthStore) retryPersistLocked() error { + if s.persistErr == nil { + return nil + } + _, err := s.persistLocked() + return err +} + +// persistLocked writes the current records to disk. A failure is latched and +// fails placement closed until a later write succeeds. The returned boolean +// reports whether the rename made the new state visible. +func (s *vfHealthStore) persistLocked() (bool, error) { + if s.path == "" { + return false, nil + } + renamed, err := s.writeStateLocked() + s.persistErr = err + return renamed, err +} + +func (s *vfHealthStore) writeStateLocked() (bool, error) { + data, err := json.MarshalIndent(vfHealthFile{ + Version: vfHealthFileVersion, + Records: s.sortedRecordsLocked(), + }, "", " ") + if err != nil { + return false, fmt.Errorf("marshal VF health state: %w", err) + } + dirPath := filepath.Dir(s.path) + if err := os.MkdirAll(dirPath, 0755); err != nil { + return false, fmt.Errorf("create VF health state dir: %w", err) + } + if err := s.syncDirFunc(filepath.Dir(dirPath)); err != nil { + return false, fmt.Errorf("sync VF health state parent dir: %w", err) + } + tmp := s.path + ".tmp" + f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return false, fmt.Errorf("create VF health state: %w", err) + } + if _, err := f.Write(data); err != nil { + f.Close() + os.Remove(tmp) + return false, fmt.Errorf("write VF health state: %w", err) + } + if err := f.Sync(); err != nil { + f.Close() + os.Remove(tmp) + return false, fmt.Errorf("sync VF health state: %w", err) + } + if err := f.Close(); err != nil { + os.Remove(tmp) + return false, fmt.Errorf("close VF health state: %w", err) + } + if err := os.Rename(tmp, s.path); err != nil { + os.Remove(tmp) + return false, fmt.Errorf("rename VF health state: %w", err) + } + if err := s.syncDirFunc(dirPath); err != nil { + return true, fmt.Errorf("sync VF health state dir: %w", err) + } + return true, nil +} + +func syncDir(path string) error { + dir, err := os.Open(path) + if err != nil { + return err + } + defer dir.Close() + return dir.Sync() +} diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go new file mode 100644 index 000000000..00f14dbf7 --- /dev/null +++ b/lib/devices/vf_health_test.go @@ -0,0 +1,559 @@ +package devices + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func resetVFHealthStore(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "vf-health.json") + require.NoError(t, initVFHealth(path)) + t.Cleanup(func() { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.path = "" + vfHealth.records = make(map[string]vfHealthRecord) + vfHealth.threshold = defaultVFQuarantineThreshold + vfHealth.loadErr = nil + vfHealth.persistErr = nil + vfHealth.syncDirFunc = syncDir + }) + return path +} + +func quarantinedVFs() []vfHealthRecord { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + records := vfHealth.sortedRecordsLocked() + result := records[:0] + for _, record := range records { + if record.QuarantinedAt != nil { + result = append(result, record) + } + } + return result +} + +func quarantineVF(t *testing.T, address string) { + t.Helper() + SetVFQuarantineThreshold(1) + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: address, InstanceID: "quarantine-helper"}) + require.NoError(t, err) + require.Equal(t, VFReportQuarantined, result.Outcome) + SetVFQuarantineThreshold(defaultVFQuarantineThreshold) +} + +func TestVGPUAvailability(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:82:00.6", InstanceID: "instance-1"}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) + vfs := []VirtualFunction{ + {PCIAddress: "0000:82:00.4"}, + {PCIAddress: "0000:82:00.5", Allocated: true}, + {PCIAddress: "0000:82:00.6"}, + } + + available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, vfs) + require.NoError(t, err) + assert.Equal(t, 1, available, "a below-threshold failure tally must not remove the VF from placement") + assert.Equal(t, 1, quarantined) + + available, quarantined, err = VGPUAvailability(VGPUFrameworkMdev, vfs) + require.NoError(t, err) + assert.Equal(t, 2, available) + assert.Zero(t, quarantined) +} + +func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { + path := resetVFHealthStore(t) + require.NoError(t, os.WriteFile(path, []byte("not json"), 0o644)) + require.Error(t, initVFHealth(path)) + + _, _, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.ErrorContains(t, err, "VF health state unavailable") + + available, quarantined, err := VGPUAvailability(VGPUFrameworkMdev, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.NoError(t, err) + assert.Equal(t, 1, available) + assert.Zero(t, quarantined) + + restored := `{"version":1,"records":[{"vf_address":"0000:82:00.4","quarantined_at":"2026-08-20T00:00:00Z"}]}` + require.NoError(t, os.WriteFile(path, []byte(restored), 0o644)) + available, quarantined, err = VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.NoError(t, err, "a repaired state file must re-enable placement without a new report") + assert.Zero(t, available) + assert.Equal(t, 1, quarantined) +} + +func TestVGPUAvailabilityFailsClosedAfterPersistFailure(t *testing.T) { + resetVFHealthStore(t) + SetVFQuarantineThreshold(1) + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0o644)) + goodPath := vfHealth.path + vfHealth.path = filepath.Join(blocker, "vf-health.json") + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.Error(t, err) + assert.True(t, VFHealthStoreUnavailable()) + _, _, err = VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) + require.ErrorContains(t, err, "last write failed") + + vfHealth.path = goodPath + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.NoError(t, err) + assert.Equal(t, VFReportQuarantined, result.Outcome) + assert.False(t, VFHealthStoreUnavailable()) + + available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) + require.NoError(t, err) + assert.Zero(t, available) + assert.Equal(t, 1, quarantined) +} + +func TestSetVFQuarantineThresholdReevaluatesRecordedFailures(t *testing.T) { + path := resetVFHealthStore(t) + SetVFQuarantineThreshold(3) + for _, instance := range []string{"instance-1", "instance-2"} { + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) + } + + SetVFQuarantineThreshold(2) + + records := quarantinedVFs() + require.Len(t, records, 1) + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) + + data, err := os.ReadFile(path) + require.NoError(t, err) + var state vfHealthFile + require.NoError(t, json.Unmarshal(data, &state)) + require.Len(t, state.Records, 1) + assert.NotNil(t, state.Records[0].QuarantinedAt, "the re-evaluated quarantine must be persisted") +} + +func TestLoadReevaluatesTalliesAgainstConfiguredThreshold(t *testing.T) { + path := resetVFHealthStore(t) + SetVFQuarantineThreshold(3) + for _, instance := range []string{"instance-1", "instance-2"} { + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) + } + + // Simulate a restart where the threshold is configured lower before the + // persisted tallies are loaded. + vfHealth.mu.Lock() + vfHealth.records = make(map[string]vfHealthRecord) + vfHealth.threshold = 2 + vfHealth.mu.Unlock() + require.NoError(t, initVFHealth(path)) + + records := quarantinedVFs() + require.Len(t, records, 1) + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) +} + +func TestReportVFInitFailureQuarantinesAtThreshold(t *testing.T) { + path := resetVFHealthStore(t) + + result, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + }) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, 1, result.Failures) + assert.Equal(t, defaultVFQuarantineThreshold, result.Threshold) + assert.Empty(t, quarantinedVFs(), "one failure must not quarantine at the default threshold") + + result, err = ReportVFInitFailure(VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-2", + AssignedAt: "2026-08-20T16:00:00Z", + }) + require.NoError(t, err) + assert.Equal(t, VFReportQuarantined, result.Outcome) + assert.Equal(t, 2, result.Failures) + + records := quarantinedVFs() + require.Len(t, records, 1) + assert.Equal(t, 1, TotalQuarantinedVFs()) + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) + require.NotNil(t, records[0].QuarantinedAt) + require.Len(t, records[0].Failures, 2) + assert.Equal(t, "instance-1", records[0].Failures[0].InstanceID) + + result, err = ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportUnchanged, result.Outcome) + + require.NoError(t, initVFHealth(path)) + reloaded := quarantinedVFs() + require.Len(t, reloaded, 1) + assert.Equal(t, "0000:e3:00.4", reloaded[0].VFAddress) + require.Len(t, reloaded[0].Failures, 2) +} + +func TestReportVFInitFailureDeduplicatesAssignments(t *testing.T) { + resetVFHealthStore(t) + + report := VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + } + result, err := ReportVFInitFailure(report) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + + result, err = ReportVFInitFailure(report) + require.NoError(t, err) + assert.Equal(t, VFReportUnchanged, result.Outcome) + assert.Equal(t, 1, result.Failures) + assert.Empty(t, quarantinedVFs(), "a rescanned assignment must not count toward the threshold twice") +} + +func TestReportVFInitSuccessClearsFailureTally(t *testing.T) { + path := resetVFHealthStore(t) + report := VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + } + + _, err := ReportVFInitFailure(report) + require.NoError(t, err) + + success := VFInitSuccessReport{ + VFAddress: report.VFAddress, + InstanceID: report.InstanceID, + AssignedAt: report.AssignedAt, + } + successResult, err := ReportVFInitSuccess(success) + require.NoError(t, err) + assert.Equal(t, 1, successResult.Cleared) + assert.False(t, successResult.Rescinded) + + successResult, err = ReportVFInitSuccess(success) + require.NoError(t, err) + assert.Zero(t, successResult.Cleared) + + require.NoError(t, initVFHealth(path)) + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: report.VFAddress, InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, 1, result.Failures) +} + +func TestReportVFInitSuccessRescindsQuarantineTriggeredByAssignment(t *testing.T) { + resetVFHealthStore(t) + vf := "0000:e3:00.4" + _, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: vf, + InstanceID: "instance-1", + AssignedAt: "2026-08-20T14:00:00Z", + }) + require.NoError(t, err) + trigger := VFInitFailureReport{ + VFAddress: vf, + InstanceID: "instance-2", + AssignedAt: "2026-08-20T15:00:00Z", + } + result, err := ReportVFInitFailure(trigger) + require.NoError(t, err) + require.Equal(t, VFReportQuarantined, result.Outcome) + + success, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: trigger.VFAddress, + InstanceID: trigger.InstanceID, + AssignedAt: trigger.AssignedAt, + }) + require.NoError(t, err) + assert.Equal(t, 2, success.Cleared) + assert.True(t, success.Rescinded) + assert.Empty(t, quarantinedVFs()) +} + +func TestReportVFInitSuccessWithoutMatchingFailureClearsNothing(t *testing.T) { + resetVFHealthStore(t) + _, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + }) + require.NoError(t, err) + + result, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-2", + AssignedAt: "2026-08-20T16:00:00Z", + }) + require.NoError(t, err) + assert.Zero(t, result.Cleared) + assert.False(t, result.Rescinded) +} + +func TestReportVFInitSuccessNeverClearsAnotherAssignmentsQuarantine(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:e3:00.4") + + result, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "another-instance", + }) + require.NoError(t, err) + assert.Zero(t, result.Cleared) + assert.False(t, result.Rescinded) + require.Len(t, quarantinedVFs(), 1) +} + +func TestReportVFInitFailureRejectsInvalidAddress(t *testing.T) { + resetVFHealthStore(t) + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "not-a-pci-address"}) + require.ErrorContains(t, err, "invalid VF address") + _, err = ReportVFInitSuccess(VFInitSuccessReport{VFAddress: "not-a-pci-address"}) + require.ErrorContains(t, err, "invalid VF address") + assert.Empty(t, quarantinedVFs()) +} + +func TestReportVFInitFailureRollsBackOnPersistFailure(t *testing.T) { + resetVFHealthStore(t) + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0644)) + vfHealth.path = filepath.Join(blocker, "vf-health.json") + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.Error(t, err) + + vfHealth.mu.Lock() + _, exists := vfHealth.records["0000:e3:00.4"] + vfHealth.mu.Unlock() + assert.False(t, exists, "a failure whose persist failed must be retried by the next report") +} + +func TestReportVFInitFailureRetriesParentSyncAfterFailure(t *testing.T) { + resetVFHealthStore(t) + parentDir := t.TempDir() + vfHealth.path = filepath.Join(parentDir, "gpu", "vf-health.json") + + parentSyncs := 0 + retrySawPersistErr := false + vfHealth.syncDirFunc = func(path string) error { + if path != parentDir { + return syncDir(path) + } + parentSyncs++ + if parentSyncs == 1 { + return errors.New("injected parent sync failure") + } + if parentSyncs == 2 { + retrySawPersistErr = vfHealth.persistErr != nil + } + return syncDir(path) + } + + report := VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"} + _, err := ReportVFInitFailure(report) + require.ErrorContains(t, err, "sync VF health state parent dir") + assert.True(t, VFHealthStoreUnavailable()) + + result, err := ReportVFInitFailure(report) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, 3, parentSyncs) + assert.True(t, retrySawPersistErr, "retry must sync the parent before clearing the write failure") + assert.False(t, VFHealthStoreUnavailable()) +} + +func TestReportVFInitFailureRetainsRenamedStateAfterSyncFailure(t *testing.T) { + path := resetVFHealthStore(t) + vf := "0000:e3:00.4" + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-1"}) + require.NoError(t, err) + + vfHealth.syncDirFunc = func(path string) error { + if path == filepath.Dir(vfHealth.path) { + return errors.New("injected sync failure") + } + return syncDir(path) + } + _, err = ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-2"}) + require.ErrorContains(t, err, "sync VF health state dir") + + data, err := os.ReadFile(path) + require.NoError(t, err) + var state vfHealthFile + require.NoError(t, json.Unmarshal(data, &state)) + require.Len(t, state.Records, 1) + assert.NotNil(t, state.Records[0].QuarantinedAt) + require.Len(t, quarantinedVFs(), 1, "memory must retain state already renamed into place") + assert.True(t, VFHealthStoreUnavailable()) + + vfHealth.syncDirFunc = syncDir + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.5", InstanceID: "other-instance"}) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.False(t, VFHealthStoreUnavailable()) + + data, err = os.ReadFile(path) + require.NoError(t, err) + state = vfHealthFile{} + require.NoError(t, json.Unmarshal(data, &state)) + found := false + for _, record := range state.Records { + if record.VFAddress == vf { + found = true + assert.NotNil(t, record.QuarantinedAt, "a later write must not erase the renamed quarantine") + } + } + require.True(t, found) + + available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: vf}}) + require.NoError(t, err) + assert.Zero(t, available) + assert.Equal(t, 1, quarantined) +} + +func TestReportRetriesFailedThresholdPersistence(t *testing.T) { + path := resetVFHealthStore(t) + vf := "0000:e3:00.4" + SetVFQuarantineThreshold(3) + for _, instance := range []string{"instance-1", "instance-2"} { + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: instance}) + require.NoError(t, err) + } + + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0644)) + vfHealth.path = filepath.Join(blocker, "vf-health.json") + SetVFQuarantineThreshold(2) + assert.True(t, VFHealthStoreUnavailable()) + + vfHealth.path = path + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportUnchanged, result.Outcome) + assert.False(t, VFHealthStoreUnavailable()) + + data, err := os.ReadFile(path) + require.NoError(t, err) + var state vfHealthFile + require.NoError(t, json.Unmarshal(data, &state)) + require.Len(t, state.Records, 1) + assert.NotNil(t, state.Records[0].QuarantinedAt) +} + +func TestReportVFInitSuccessRollsBackOnPersistFailure(t *testing.T) { + resetVFHealthStore(t) + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.NoError(t, err) + + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0644)) + goodPath := vfHealth.path + vfHealth.path = filepath.Join(blocker, "vf-health.json") + + _, err = ReportVFInitSuccess(VFInitSuccessReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.Error(t, err) + vfHealth.path = goodPath + + vfHealth.mu.Lock() + record, exists := vfHealth.records["0000:e3:00.4"] + vfHealth.mu.Unlock() + require.True(t, exists, "a clear whose persist failed must be restored in memory") + assert.Len(t, record.Failures, 1) +} + +func TestCheckedAddressesFailsClosedOnInvalidRecord(t *testing.T) { + tests := []struct { + name string + state string + wantErr string + }{ + { + name: "unsupported version", + state: `{"version":2,"records":[]}`, + wantErr: "unsupported version 2", + }, + { + name: "missing records", + state: `{"version":1}`, + wantErr: "expected a records array", + }, + { + name: "invalid address", + state: `{"version":1,"records":[{"vf_address":"not-a-pci-address","quarantined_at":"2026-08-20T00:00:00Z"}]}`, + wantErr: "invalid VF address", + }, + { + name: "neither quarantined nor failed", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4"}]}`, + wantErr: "neither quarantined nor any recorded failures", + }, + { + name: "failure missing report timestamp", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","failures":[{"instance_id":"instance-1"}]}]}`, + wantErr: "missing report timestamp", + }, + { + name: "duplicate assignment", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","failures":[{"instance_id":"instance-1","assigned_at":"a","reported_at":"2026-08-20T00:00:00Z"},{"instance_id":"instance-1","assigned_at":"a","reported_at":"2026-08-21T00:00:00Z"}]}]}`, + wantErr: "duplicate failure for assignment", + }, + { + name: "duplicate address", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-20T00:00:00Z"},{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-21T00:00:00Z"}]}`, + wantErr: "duplicate VF address", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path := resetVFHealthStore(t) + require.NoError(t, os.WriteFile(path, []byte(tt.state), 0644)) + require.ErrorContains(t, initVFHealth(path), tt.wantErr) + assert.True(t, VFHealthStoreUnavailable()) + assert.Empty(t, quarantinedVFs()) + + _, err := vfHealth.checkedAddresses() + require.Error(t, err) + }) + } +} + +func TestReportVFInitFailureRefusesToClobberUnloadedState(t *testing.T) { + path := resetVFHealthStore(t) + quarantineVF(t, "0000:e3:00.4") + + require.NoError(t, os.WriteFile(path, []byte("not json"), 0644)) + require.Error(t, initVFHealth(path)) + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.5"}) + require.Error(t, err) + _, err = ReportVFInitSuccess(VFInitSuccessReport{VFAddress: "0000:e3:00.5"}) + require.Error(t, err) + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, "not json", string(data), "a failed load must not be overwritten by later reports") + + restored := `{"version":1,"records":[{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-20T00:00:00Z"}]}` + require.NoError(t, os.WriteFile(path, []byte(restored), 0644)) + quarantineVF(t, "0000:e3:00.5") + records := quarantinedVFs() + require.Len(t, records, 2, "reload must recover the previously persisted quarantine") + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) +} diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index eaf210b42..354217fd5 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -8,31 +8,128 @@ import ( "path/filepath" ) +// DiscoverVGPU returns the host's active vGPU framework and virtual functions. +func DiscoverVGPU() (VGPUFramework, []VirtualFunction, error) { + return discoverVGPUWith(discoverMdevVFs, hostVendorVFIO.discoverVFs) +} + +func discoverVGPUWith(discoverMdev, discoverVendorVFIO func() ([]VirtualFunction, error)) (VGPUFramework, []VirtualFunction, error) { + vfs, err := discoverMdev() + if err != nil { + return VGPUFrameworkNone, nil, fmt.Errorf("discover mdev VFs: %w", err) + } + if len(vfs) > 0 { + return VGPUFrameworkMdev, vfs, nil + } + + vfs, err = discoverVendorVFIO() + if err != nil { + return VGPUFrameworkNone, nil, fmt.Errorf("discover vendor VFIO VFs: %w", err) + } + if len(vfs) == 0 { + return VGPUFrameworkNone, nil, nil + } + return VGPUFrameworkVendorVFIO, vfs, nil +} + +// ListGPUProfiles returns available vGPU profiles with availability counts. +func ListGPUProfiles() ([]GPUProfile, error) { + framework, vfs, err := DiscoverVGPU() + if err != nil { + return nil, err + } + return ListGPUProfilesWithVFs(framework, vfs) +} + +// ListGPUProfilesWithVFs returns available profiles for discovered VFs. +func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction) ([]GPUProfile, error) { + switch framework { + case VGPUFrameworkMdev: + return listMdevGPUProfilesWithVFs(vfs) + case VGPUFrameworkVendorVFIO: + return hostVendorVFIO.listProfiles(vfs) + default: + return nil, nil + } +} + func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) { - mdev, err := CreateMdev(ctx, profileName, instanceID) + framework, _, err := DiscoverVGPU() if err != nil { return nil, err } - return &VGPUDevice{ - Framework: VGPUFrameworkMdev, - VFAddress: mdev.VFAddress, - ProfileType: mdev.ProfileType, - ProfileName: mdev.ProfileName, - SysfsPath: mdev.SysfsPath, - MdevUUID: mdev.UUID, - }, nil + switch framework { + case VGPUFrameworkMdev: + mdev, err := CreateMdev(ctx, profileName, instanceID) + if err != nil { + return nil, err + } + return &VGPUDevice{ + Framework: VGPUFrameworkMdev, + VFAddress: mdev.VFAddress, + ProfileType: mdev.ProfileType, + ProfileName: mdev.ProfileName, + SysfsPath: mdev.SysfsPath, + MdevUUID: mdev.UUID, + }, nil + case VGPUFrameworkVendorVFIO: + return hostVendorVFIO.create(ctx, profileName, instanceID) + default: + return nil, fmt.Errorf("vGPU framework not available") + } } func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { - if assignment.Framework != VGPUFrameworkNone && assignment.Framework != VGPUFrameworkMdev { - return fmt.Errorf("unknown vGPU framework %q", assignment.Framework) + framework := assignment.Framework + if framework == VGPUFrameworkNone && assignment.MdevUUID != "" { + framework = VGPUFrameworkMdev } - mdevUUID := assignment.MdevUUID - if mdevUUID == "" { - if assignment.DevicePath == "" { - return nil + + switch framework { + case VGPUFrameworkMdev: + mdevUUID := assignment.MdevUUID + if mdevUUID == "" { + if assignment.DevicePath == "" { + return nil + } + mdevUUID = filepath.Base(assignment.DevicePath) } - mdevUUID = filepath.Base(assignment.DevicePath) + return DestroyMdev(ctx, mdevUUID) + case VGPUFrameworkVendorVFIO: + return hostVendorVFIO.destroy(ctx, filepath.Base(assignment.DevicePath), assignment.InstanceID) + case VGPUFrameworkNone: + return nil + default: + return fmt.Errorf("unknown vGPU framework %q", framework) + } +} + +func mdevReconcileInfos(protectedDevicePaths map[string]struct{}) []MdevReconcileInfo { + instanceInfos := make([]MdevReconcileInfo, 0, len(protectedDevicePaths)) + for devicePath := range protectedDevicePaths { + instanceInfos = append(instanceInfos, MdevReconcileInfo{ + MdevUUID: filepath.Base(devicePath), + IsRunning: true, + }) + } + return instanceInfos +} + +// ReconcileVGPUs releases orphaned vGPU assignments. +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepDevices bool) error { + framework, _, err := DiscoverVGPU() + if err != nil { + return err + } + if !sweepDevices { + return nil + } + + switch framework { + case VGPUFrameworkMdev: + return ReconcileMdevs(ctx, mdevReconcileInfos(protectedDevicePaths)) + case VGPUFrameworkVendorVFIO: + return hostVendorVFIO.reconcile(ctx, protectedDevicePaths) } - return DestroyMdev(ctx, mdevUUID) + return nil } diff --git a/lib/devices/vgpu_linux_test.go b/lib/devices/vgpu_linux_test.go new file mode 100644 index 000000000..5bb52da1d --- /dev/null +++ b/lib/devices/vgpu_linux_test.go @@ -0,0 +1,136 @@ +//go:build linux + +package devices + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMdevReconcileInfosProtectClaimedDevicePath(t *testing.T) { + t.Parallel() + + infos := mdevReconcileInfos(map[string]struct{}{ + "/sys/bus/mdev/devices/claimed": {}, + }) + + assert.Equal(t, []MdevReconcileInfo{{MdevUUID: "claimed", IsRunning: true}}, infos) +} + +func TestProtectedMdevUUIDs(t *testing.T) { + t.Parallel() + + protected := protectedMdevUUIDs([]MdevReconcileInfo{ + {MdevUUID: "claimed", IsRunning: true}, + {MdevUUID: "stale"}, + {IsRunning: true}, + }) + + assert.Contains(t, protected, "claimed") + assert.NotContains(t, protected, "stale") + assert.Len(t, protected, 1) +} + +func TestDiscoverVGPUWithPropagatesMdevError(t *testing.T) { + t.Parallel() + + discoveryErr := errors.New("mdev discovery failed") + vendorCalled := false + framework, vfs, err := discoverVGPUWith( + func() ([]VirtualFunction, error) { + return nil, discoveryErr + }, + func() ([]VirtualFunction, error) { + vendorCalled = true + return []VirtualFunction{{PCIAddress: "0000:82:00.4"}}, nil + }, + ) + + require.ErrorIs(t, err, discoveryErr) + assert.Equal(t, VGPUFrameworkNone, framework) + assert.Nil(t, vfs) + assert.False(t, vendorCalled) +} + +func TestDiscoverVGPUWithFallsBackFromTypelessMdevBus(t *testing.T) { + t.Parallel() + + root := t.TempDir() + busPath := filepath.Join(root, "sys", "class", "mdev_bus") + require.NoError(t, os.MkdirAll(filepath.Join(busPath, "0000:82:00.4", "mdev_supported_types"), 0755)) + + framework, vfs, err := discoverVGPUWith( + func() ([]VirtualFunction, error) { + return discoverMdevVFsWith(busPath, filepath.Join(root, "sys", "bus", "pci", "devices"), func() ([]MdevDevice, error) { + return nil, nil + }) + }, + func() ([]VirtualFunction, error) { + return []VirtualFunction{{PCIAddress: "0000:82:00.4"}}, nil + }, + ) + + require.NoError(t, err) + assert.Equal(t, VGPUFrameworkVendorVFIO, framework) + assert.Equal(t, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}, vfs) +} + +func TestDiscoverVGPUWithPropagatesVendorVFIOError(t *testing.T) { + t.Parallel() + + discoveryErr := errors.New("vendor VFIO discovery failed") + framework, vfs, err := discoverVGPUWith( + func() ([]VirtualFunction, error) { + return nil, nil + }, + func() ([]VirtualFunction, error) { + return nil, discoveryErr + }, + ) + + require.ErrorIs(t, err, discoveryErr) + assert.Equal(t, VGPUFrameworkNone, framework) + assert.Nil(t, vfs) +} + +func TestDiscoverMdevVFsSkipsUnreadableVF(t *testing.T) { + t.Parallel() + + root := t.TempDir() + busPath := filepath.Join(root, "sys", "class", "mdev_bus") + pciPath := filepath.Join(root, "sys", "bus", "pci", "devices") + require.NoError(t, os.MkdirAll(filepath.Join(busPath, "0000:82:00.4", "mdev_supported_types", "nvidia-556"), 0755)) + // A regular file makes the supported-types read fail without IsNotExist. + require.NoError(t, os.MkdirAll(filepath.Join(busPath, "0000:82:00.5"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(busPath, "0000:82:00.5", "mdev_supported_types"), nil, 0644)) + + vfs, err := discoverMdevVFsWith(busPath, pciPath, func() ([]MdevDevice, error) { + return nil, nil + }) + require.NoError(t, err) + require.Len(t, vfs, 1) + assert.Equal(t, "0000:82:00.4", vfs[0].PCIAddress) +} + +func TestDiscoverMdevVFsFailsWhenNoVFIsReadable(t *testing.T) { + t.Parallel() + + root := t.TempDir() + busPath := filepath.Join(root, "sys", "class", "mdev_bus") + pciPath := filepath.Join(root, "sys", "bus", "pci", "devices") + require.NoError(t, os.MkdirAll(filepath.Join(busPath, "0000:82:00.4"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(busPath, "0000:82:00.4", "mdev_supported_types"), nil, 0644)) + + // With every VF unreadable, discovery must fail rather than report an + // empty inventory that would demote the host to vendor VFIO or + // passthrough. + _, err := discoverMdevVFsWith(busPath, pciPath, func() ([]MdevDevice, error) { + return nil, nil + }) + require.Error(t, err) +} diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index c02c5beef..f16d606eb 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -227,6 +227,7 @@ func buildQMPArgs(socketPath string) []string { type startedProcess struct { pid int socketPath string + termGrace time.Duration waitDone chan error waitConsumed bool waitErr error @@ -277,18 +278,47 @@ func (p *startedProcess) wait() error { return err } +func (p *startedProcess) waitFor(d time.Duration) bool { + if _, exited := p.checkExited(); exited { + return true + } + select { + case err := <-p.waitDone: + p.waitConsumed = true + p.waitErr = err + return true + case <-time.After(d): + return false + } +} + func (p *startedProcess) cleanup() { if _, exited := p.checkExited(); !exited { - _ = syscall.Kill(p.pid, syscall.SIGKILL) - _ = p.wait() + terminated := false + if p.termGrace > 0 { + if syscall.Kill(p.pid, syscall.SIGTERM) == nil { + terminated = p.waitFor(p.termGrace) + } + } + if !terminated { + _ = syscall.Kill(p.pid, syscall.SIGKILL) + _ = p.wait() + } } _ = os.Remove(p.socketPath) } +func vfioTermGraceFor(cfg hypervisor.VMConfig) time.Duration { + if cfg.VGPUDevicePath != "" || len(cfg.PCIDevices) > 0 { + return hypervisor.VFIOTermGrace + } + return 0 +} + // startQEMUProcess handles the common QEMU process startup logic. // Returns the PID, hypervisor client, and a cleanup function. // The cleanup function must be called on error; call cleanup.Release() on success. -func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version string, socketPath string, args []string) (int, *QEMU, *cleanup.Cleanup, error) { +func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version string, socketPath string, args []string, termGrace time.Duration) (int, *QEMU, *cleanup.Cleanup, error) { log := logger.FromContext(ctx) processAttrs := hypervisor.TraceAttributesFromContext(ctx) processAttrs = append(processAttrs, @@ -358,6 +388,7 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version } pid := proc.pid + proc.termGrace = termGrace log.DebugContext(processCtx, "QEMU process started", "pid", pid, "duration_ms", time.Since(processStartTime).Milliseconds()) // Setup cleanup to kill, reap, and remove the socket if subsequent steps fail. @@ -474,7 +505,7 @@ func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, s // Build command arguments: QMP socket + VM configuration args := buildQMPArgs(socketPath) args = append(args, buildArgs(attempt, machineType)...) - pid, hv, cu, err = s.startQEMUProcess(ctx, p, version, socketPath, args) + pid, hv, cu, err = s.startQEMUProcess(ctx, p, version, socketPath, args, vfioTermGraceFor(attempt)) if err == nil { booted = attempt started = true @@ -609,7 +640,7 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, incomingURI := "exec:cat < " + memoryFile args = append(args, "-incoming", incomingURI) - pid, hv, cu, err := s.startQEMUProcess(ctx, p, version, socketPath, args) + pid, hv, cu, err := s.startQEMUProcess(ctx, p, version, socketPath, args, vfioTermGraceFor(config)) if err != nil { return 0, nil, err } diff --git a/lib/hypervisor/qemu/process_test.go b/lib/hypervisor/qemu/process_test.go index e8be0dda2..ad08aea24 100644 --- a/lib/hypervisor/qemu/process_test.go +++ b/lib/hypervisor/qemu/process_test.go @@ -1,6 +1,7 @@ package qemu import ( + "bufio" "context" "errors" "os" @@ -409,3 +410,58 @@ func TestWaitForSocketOrExitReturnsEarlyWhenProcessDies(t *testing.T) { require.NotNil(t, cmd.ProcessState) assert.True(t, cmd.ProcessState.Exited()) } + +func TestVFIOTermGraceFor(t *testing.T) { + tests := []struct { + name string + cfg hypervisor.VMConfig + want time.Duration + }{ + {name: "vGPU", cfg: hypervisor.VMConfig{VGPUDevicePath: "/sys/bus/mdev/devices/test"}, want: hypervisor.VFIOTermGrace}, + {name: "PCI device", cfg: hypervisor.VMConfig{PCIDevices: []string{"0000:01:00.0"}}, want: hypervisor.VFIOTermGrace}, + {name: "no VFIO device", cfg: hypervisor.VMConfig{}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, vfioTermGraceFor(tt.cfg)) + }) + } +} + +func TestCleanupSIGTERMsProcessWithTermGrace(t *testing.T) { + socketPath := filepath.Join(t.TempDir(), "qemu.sock") + markerPath := filepath.Join(t.TempDir(), "terminated") + + cmd := exec.Command("sh", "-c", "trap 'touch "+markerPath+"; exit 0' TERM; echo ready; sleep 30 & wait") + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + proc, err := startManagedProcess(cmd, socketPath) + require.NoError(t, err) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + proc.termGrace = 5 * time.Second + + proc.cleanup() + + assert.FileExists(t, markerPath, "process must get SIGTERM, not SIGKILL, when termGrace is set") + require.NoFileExists(t, socketPath) + require.NotNil(t, cmd.ProcessState) +} + +func TestCleanupEscalatesToSIGKILLAfterTermGrace(t *testing.T) { + socketPath := filepath.Join(t.TempDir(), "qemu.sock") + + cmd := exec.Command("sh", "-c", "trap '' TERM; echo ready; sleep 30 & wait") + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + proc, err := startManagedProcess(cmd, socketPath) + require.NoError(t, err) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + proc.termGrace = 50 * time.Millisecond + + proc.cleanup() + + assert.ErrorIs(t, syscall.Kill(proc.pid, 0), syscall.ESRCH, "SIGTERM-ignoring process must still be hard-killed") + require.NoFileExists(t, socketPath) +} diff --git a/lib/hypervisor/vfio.go b/lib/hypervisor/vfio.go new file mode 100644 index 000000000..1744cfc84 --- /dev/null +++ b/lib/hypervisor/vfio.go @@ -0,0 +1,7 @@ +package hypervisor + +import "time" + +// VFIOTermGrace allows VFIO teardown to finish after SIGTERM before SIGKILL; +// SIGKILL during initialization can wedge the VF, while teardown takes 1-2s. +const VFIOTermGrace = 5 * time.Second diff --git a/lib/instances/create.go b/lib/instances/create.go index add25e4e7..cd50deb5c 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -56,7 +56,7 @@ func wrapCreateVGPUErr(profile string, err error) error { if errors.Is(err, devices.ErrVGPUNotSupportedOnMacOS) { return fmt.Errorf("%w: %w", ErrInvalidRequest, err) } - return fmt.Errorf("create vGPU mdev for profile %s: %w", profile, err) + return fmt.Errorf("create vGPU for profile %s: %w", profile, err) } // generateVsockCID converts first 8 chars of instance ID to a unique CID @@ -277,11 +277,14 @@ func (m *manager) createInstance( var gpuFramework devices.VGPUFramework var gpuDevicePath string var gpuMdevUUID string + var gpuAssignedAt *time.Time + retention := vgpuRetention{instanceID: id} - // Setup cleanup stack early so device attachment errors trigger cleanup + // Deferred before cu.Clean so rollback records retention before this wraps the error. + defer func() { retErr = retention.wrapPending(retErr) }() cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) - m.deleteInstanceData(id) + m.persistVGPURetention(ctx, &retention) }) defer cu.Clean() @@ -297,9 +300,24 @@ func (m *manager) createInstance( // Handle vGPU profile request if req.GPU != nil && req.GPU.Profile != "" { + retentionStub := func() StoredMetadata { + return StoredMetadata{ + Id: id, + Name: req.Name, + Image: req.Image, + ResolvedImage: resolvedImageRef, + Platform: imageInfo.Platform, + CreatedAt: m.nowUTC(), + HypervisorType: hvType, + HypervisorVersion: hvVersion, + SocketPath: m.paths.InstanceSocket(id, starter.SocketName()), + DataDir: m.paths.InstanceDir(id), + } + } log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) - gpuDevice, err = devices.CreateVGPU(ctx, req.GPU.Profile, id) + gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) if err != nil { + retention.retainFromCreateError(retentionStub(), m.nowUTC(), err) log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) return nil, wrapCreateVGPUErr(req.GPU.Profile, err) } @@ -307,6 +325,8 @@ func (m *manager) createInstance( gpuFramework = gpuDevice.Framework gpuDevicePath = gpuDevice.SysfsPath gpuMdevUUID = gpuDevice.MdevUUID + assignedAt := m.nowUTC() + gpuAssignedAt = &assignedAt log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", gpuProfile, "uuid", gpuMdevUUID) // Add vGPU cleanup to stack @@ -316,9 +336,11 @@ func (m *manager) createInstance( Framework: gpuDevice.Framework, DevicePath: gpuDevice.SysfsPath, MdevUUID: gpuDevice.MdevUUID, + InstanceID: id, } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { + if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) + retention.retainFromDevice(retentionStub(), gpuDevice, *gpuAssignedAt) } }) } @@ -395,6 +417,7 @@ func (m *manager) createInstance( GPUFramework: gpuFramework, GPUDevicePath: gpuDevicePath, GPUMdevUUID: gpuMdevUUID, + GPUAssignedAt: gpuAssignedAt, Entrypoint: req.Entrypoint, Cmd: req.Cmd, SkipKernelHeaders: req.SkipKernelHeaders, diff --git a/lib/instances/create_mdev_test.go b/lib/instances/create_mdev_test.go index e6e4f55c5..05b3e9d8c 100644 --- a/lib/instances/create_mdev_test.go +++ b/lib/instances/create_mdev_test.go @@ -63,7 +63,7 @@ func TestWrapCreateVGPUErr(t *testing.T) { { name: "other vGPU error", err: errors.New("boom"), - wantMessage: "create vGPU mdev for profile profile: boom", + wantMessage: "create vGPU for profile profile: boom", }, } { t.Run(tc.name, func(t *testing.T) { diff --git a/lib/instances/delete.go b/lib/instances/delete.go index e897ff049..d2f0e63d9 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -140,19 +140,13 @@ func (m *manager) deleteInstanceWithOptions( } m.closeFirecrackerUFFDSession(ctx, stored) - // 5b. Release the vGPU assignment if present, before any network, device, - // or volume teardown. Release failure is logged and the delete continues, - // matching the pre-refactor contract: the VMM is already confirmed dead, - // the guards inside the release never destroy a device they cannot prove - // is unowned, and a skipped release is recovered by startup - // reconciliation. + // Release before deleting metadata so a failed release can be retried safely. hadVGPUAssignment := storedVGPUDevicePath(stored) != "" if hadVGPUAssignment { log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID) } - if err := releaseStoredVGPU(ctx, stored); err != nil { - // Log error but continue with cleanup. - log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) + if err := m.releaseStoredVGPU(ctx, stored); err != nil { + log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup; the periodic vGPU reconcile releases it once free", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) } else if hadVGPUAssignment { if err := m.saveMetadata(meta); err != nil { log.WarnContext(ctx, "failed to save metadata after vGPU release", "instance_id", id, "error", err) @@ -240,7 +234,7 @@ func (m *manager) killHypervisor(ctx context.Context, inst *Instance) error { "instance_id", inst.Id, "stored_pid", *inst.HypervisorPID, "owner_pid", pid) } log.DebugContext(ctx, "killing hypervisor process", "instance_id", inst.Id, "pid", pid) - if err := killProcessAndWait(pid); err != nil { + if err := m.terminateThenKill(ctx, inst, pid); err != nil { return err } } diff --git a/lib/instances/fork.go b/lib/instances/fork.go index e0c778860..08ce4014a 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -219,6 +219,9 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin default: return nil, false, fmt.Errorf("%w: cannot fork from state %s (must be Stopped or Standby)", ErrInvalidState, source.State) } + if stored.GPURetainedForCleanup { + return nil, false, errVGPURetentionStub + } if !supportValidated { if err := m.validateForkSupport(ctx, stored.HypervisorType); err != nil { diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index 26bc6cfcb..c0bed670e 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -63,6 +63,29 @@ func TestForkInstanceClearsVGPUAssignment(t *testing.T) { assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", source.GPUDevicePath) } +func TestForkInstanceRejectsVGPURetentionRecord(t *testing.T) { + manager, _ := setupTestManager(t) + ctx := context.Background() + hvType := hypervisor.Type("fork-vgpu-retention-test") + hypervisor.RegisterCapabilities(hvType, hypervisor.Capabilities{SupportsConcurrentForkPrepare: true}) + manager.vmStarters[hvType] = concurrentForkPrepareTestStarter{} + + sourceID := "fork-vgpu-retention-source" + createStoppedSnapshotSourceFixture(t, manager, sourceID, sourceID, hvType) + + meta, err := manager.loadMetadata(sourceID) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, manager.saveMetadata(meta)) + + _, err = manager.ForkInstance(ctx, sourceID, ForkInstanceRequest{Name: "fork-vgpu-retention-copy"}) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") +} + func TestForkInstance_VZStoppedSourceSupported(t *testing.T) { t.Parallel() manager, _ := setupTestManager(t) diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index a9b918dfb..77d15d733 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -3,6 +3,7 @@ package instances import ( "context" "errors" + "net" "os" "path/filepath" "sync" @@ -149,24 +150,6 @@ func TestLifecycleNoopStandbyWithOptionsStillRejectsStandbyInstance(t *testing.T assertNoLifecycleEvent(t, events) } -func TestDeleteContinuesWhenVGPUReleaseFails(t *testing.T) { - m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) - meta, err := m.loadMetadata(id) - require.NoError(t, err) - meta.GPUProfile = "NVIDIA L40S-2Q" - meta.GPUFramework = devices.VGPUFramework("future-framework") - meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" - require.NoError(t, m.saveMetadata(meta)) - - // A failed release is logged and the delete continues, matching the - // pre-refactor contract; the leaked assignment is recovered by startup - // reconciliation. - require.NoError(t, m.DeleteInstance(context.Background(), id)) - - _, err = m.loadMetadata(id) - require.Error(t, err, "instance data must be deleted despite the failed release") -} - func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) var persisted *metadata @@ -195,6 +178,87 @@ func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) { assert.Equal(t, restartpolicy.BlockedReasonManualStop, persisted.RestartStatus.BlockedReason) } +func TestDeleteReleasesRetainedCreateStub(t *testing.T) { + p := paths.New(t.TempDir()) + var destroyed []devices.VGPUAssignment + m := &manager{ + paths: p, + instanceLocks: sync.Map{}, + bootMarkerScans: sync.Map{}, + now: time.Now, + lifecycleEvents: newLifecycleSubscribers(), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }, + } + const id = "retained-stub" + require.NoError(t, m.ensureDirectories(id)) + assignedAt := time.Now().UTC() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + GPURetainedForCleanup: true, + }})) + + require.NoError(t, m.DeleteInstance(context.Background(), id)) + + require.Len(t, destroyed, 1) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", destroyed[0].DevicePath) + assert.Equal(t, id, destroyed[0].InstanceID) + _, err := m.loadMetadata(id) + require.Error(t, err, "retained stub must be fully deleted") +} + +func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { + now := time.Now().UTC() + m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, now) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFrameworkVendorVFIO + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + require.NoError(t, m.saveMetadata(meta)) + + claimantID := "inst-live-claimant" + require.NoError(t, m.ensureDirectories(claimantID)) + pid := os.Getpid() + // Bind under /tmp: a t.TempDir()-derived path exceeds the macOS AF_UNIX + // path limit. + socketDir, err := os.MkdirTemp("/tmp", "hypeman-claimant-socket-") + require.NoError(t, err) + t.Cleanup(func() { + _ = os.RemoveAll(socketDir) + }) + socketPath := filepath.Join(socketDir, "noop.sock") + listener, err := net.Listen("unix", socketPath) + require.NoError(t, err) + defer listener.Close() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: claimantID, + Name: claimantID, + Image: "test-image", + CreatedAt: now, + HypervisorType: lifecycleNoopHypervisorType, + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, + SocketPath: socketPath, + DataDir: m.paths.InstanceDir(claimantID), + GPUProfile: "NVIDIA L40S-2Q", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }})) + + require.NoError(t, m.DeleteInstance(context.Background(), id)) + + _, err = m.loadMetadata(id) + require.Error(t, err, "deleted instance metadata should be gone") + claimant, err := m.loadMetadata(claimantID) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", claimant.GPUDevicePath, "live claimant keeps its assignment") +} + func TestDeleteContinuesTeardownAfterFailedVGPURelease(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) deviceManager := &recordingDeviceManager{} @@ -239,42 +303,42 @@ func TestStartPersistsStaleVGPUReleaseImmediately(t *testing.T) { assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile, "profile is kept for the next start") } -func TestStopStoppedInstanceReleasesRetainedVGPU(t *testing.T) { +func TestStartRejectsVGPURetentionRecord(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" meta.GPUFramework = devices.VGPUFrameworkNone meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true require.NoError(t, m.saveMetadata(meta)) - inst, err := m.StopInstance(context.Background(), id) - require.NoError(t, err) - require.NotNil(t, inst) - assert.Equal(t, StateStopped, inst.State) + _, err = m.StartInstance(context.Background(), id, StartInstanceRequest{}) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") stored, err := m.loadMetadata(id) require.NoError(t, err) - assert.Empty(t, stored.GPUDevicePath) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) } -func TestStopStoppedInstanceVGPUReleaseFailureRemainsNoop(t *testing.T) { +func TestStopStoppedInstanceLeavesVGPUForReconcile(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" - meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUFramework = devices.VGPUFrameworkNone meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" require.NoError(t, m.saveMetadata(meta)) + // Stop on an already-stopped instance is a no-op for the assignment; the + // periodic reconcile retries the release. inst, err := m.StopInstance(context.Background(), id) require.NoError(t, err) require.NotNil(t, inst) assert.Equal(t, StateStopped, inst.State) - stored, err := m.loadMetadata(id) require.NoError(t, err) - assert.Equal(t, devices.VGPUFramework("future-framework"), stored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) } @@ -300,6 +364,20 @@ func (m *recordingDeviceManager) UnbindFromVFIO(ctx context.Context, id string) return nil } +func TestLifecycleNoopStandbyRejectsVendorVFIOVGPU(t *testing.T) { + m, id := newLifecycleNoopManagerWithInstance(t, StateRunning, time.Now().UTC()) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFrameworkVendorVFIO + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + require.NoError(t, m.saveMetadata(meta)) + + _, err = m.StandbyInstance(context.Background(), id, StandbyInstanceRequest{}) + require.ErrorIs(t, err, ErrInvalidState) + assert.ErrorContains(t, err, "standby is not supported for instances with vGPU attached") +} + func newLifecycleNoopManagerWithInstance(t *testing.T, state State, now time.Time) (*manager, string) { t.Helper() diff --git a/lib/instances/manager.go b/lib/instances/manager.go index bc23fdf74..d44ef3014 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -2,8 +2,10 @@ package instances import ( "context" + "errors" "fmt" "os" + "path/filepath" "strings" "sync" "time" @@ -181,6 +183,9 @@ type manager struct { now func() time.Time writeFile func(string, []byte, os.FileMode) error deleteInstanceFn func(context.Context, string) error + createVGPU func(context.Context, string, string) (*devices.VGPUDevice, error) + destroyVGPU func(context.Context, devices.VGPUAssignment) error + reconcileVGPUDevices func(context.Context, map[string]struct{}, bool) error deleteSnapshotFn func(context.Context, string) error ttlReaperDeleteTimeout time.Duration egressProxy *egressproxy.Service @@ -209,6 +214,13 @@ type manager struct { // Periodic TAP garbage collection reconciler. tapGCOnce sync.Once + // Periodic vGPU reconciler. + vgpuReconcileOnce sync.Once + vgpuReconcileInterval time.Duration + discoverVGPU func() (devices.VGPUFramework, []devices.VirtualFunction, error) + + vfioTermGrace time.Duration + // Hypervisor support vmStarters map[hypervisor.Type]hypervisor.VMStarter defaultHypervisor hypervisor.Type // Default hypervisor type when not specified in request @@ -648,12 +660,6 @@ func (m *manager) StopInstance(ctx context.Context, id string) (*Instance, error if err := m.markRestartManualStopLocked(ctx, id); err != nil { return nil, err } - // A stopped instance can retain a vGPU assignment when the release - // failed during the original stop. Retry it here so the vGPU slot is - // not held until the next start, delete, or hypeman restart. A failed - // retry only logs, keeping stop's no-op contract for already-stopped - // instances. - m.releaseRetainedVGPULocked(ctx, id) updated, err := m.currentInstanceWithoutHydration(ctx, id) if err != nil { return nil, err @@ -732,6 +738,26 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { return m.defaultHypervisor } +func (m *manager) listMetadataForReconcile() ([]StoredMetadata, error) { + files, err := m.listMetadataFilesStrict() + if err != nil { + return nil, err + } + result := make([]StoredMetadata, 0, len(files)) + for _, file := range files { + id := filepath.Base(filepath.Dir(file)) + meta, err := m.loadMetadata(id) + if err != nil { + if errors.Is(err, ErrNotFound) { + continue + } + return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) + } + result = append(result, meta.StoredMetadata) + } + return result, nil +} + // ListInstances returns instances, optionally filtered by the given criteria. // Pass nil to return all instances. func (m *manager) ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) { diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index 1ada5ac1e..e14deb49b 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -73,6 +73,20 @@ type lifecycleEventDropReason string const lifecycleEventDropReasonBufferFull lifecycleEventDropReason = "buffer_full" +type vgpuReconcileStage string + +const ( + vgpuReconcileStageListInstances vgpuReconcileStage = "list_instances" + vgpuReconcileStageReconcileDevices vgpuReconcileStage = "reconcile_devices" +) + +type vgpuRetentionOperation string + +const ( + vgpuRetentionOperationCreate vgpuRetentionOperation = "create" + vgpuRetentionOperationStart vgpuRetentionOperation = "start" +) + // Metrics holds the metrics instruments for instance operations. type Metrics struct { createDuration metric.Float64Histogram @@ -94,6 +108,9 @@ type Metrics struct { lifecycleEventsDroppedTotal metric.Int64Counter forkMemFileShareFallbacksTotal metric.Int64Counter ttlReaperDeletionsTotal metric.Int64Counter + vgpuReconcileFailuresTotal metric.Int64Counter + vgpuStaleReleaseFailuresTotal metric.Int64Counter + vgpuAssignmentsRetainedTotal metric.Int64Counter tracer trace.Tracer } @@ -270,6 +287,30 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + vgpuReconcileFailuresTotal, err := meter.Int64Counter( + "hypeman_instances_vgpu_reconcile_failures_total", + metric.WithDescription("Total number of vGPU reconcile pass stages that failed, leaving stale assignments or device leftovers allocated while /resources still advertises the capacity"), + ) + if err != nil { + return nil, err + } + + vgpuStaleReleaseFailuresTotal, err := meter.Int64Counter( + "hypeman_instances_vgpu_stale_release_failures_total", + metric.WithDescription("Total number of stale vGPU assignment releases that failed, keeping the VF allocated until a later reconcile pass succeeds"), + ) + if err != nil { + return nil, err + } + + vgpuAssignmentsRetainedTotal, err := meter.Int64Counter( + "hypeman_instances_vgpu_assignments_retained_total", + metric.WithDescription("Total number of failed rollbacks that left a vGPU assignment behind, by whether the retention record the periodic reconcile needs was persisted"), + ) + if err != nil { + return nil, err + } + // Register observable gauge for instance counts by state instancesTotal, err := meter.Int64ObservableGauge( "hypeman_instances_total", @@ -464,6 +505,9 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M lifecycleEventsDroppedTotal: lifecycleEventsDroppedTotal, forkMemFileShareFallbacksTotal: forkMemFileShareFallbacksTotal, ttlReaperDeletionsTotal: ttlReaperDeletionsTotal, + vgpuReconcileFailuresTotal: vgpuReconcileFailuresTotal, + vgpuStaleReleaseFailuresTotal: vgpuStaleReleaseFailuresTotal, + vgpuAssignmentsRetainedTotal: vgpuAssignmentsRetainedTotal, tracer: tracer, }, nil } @@ -563,6 +607,32 @@ func (m *manager) recordTimeToRunning(ctx context.Context, stored *StoredMetadat m.metrics.timeToRunning.Record(ctx, duration, metric.WithAttributes(attrs...)) } +func (m *manager) recordVGPUReconcileFailure(ctx context.Context, stage vgpuReconcileStage) { + if m.metrics == nil { + return + } + m.metrics.vgpuReconcileFailuresTotal.Add(ctx, 1, metric.WithAttributes( + attribute.String("stage", string(stage)), + )) +} + +func (m *manager) recordVGPUStaleReleaseFailure(ctx context.Context) { + if m.metrics == nil { + return + } + m.metrics.vgpuStaleReleaseFailuresTotal.Add(ctx, 1) +} + +func (m *manager) recordVGPURetainedAssignment(ctx context.Context, operation vgpuRetentionOperation, persisted bool) { + if m.metrics == nil { + return + } + m.metrics.vgpuAssignmentsRetainedTotal.Add(ctx, 1, metric.WithAttributes( + attribute.String("operation", string(operation)), + attribute.String("persisted", strconv.FormatBool(persisted)), + )) +} + // recordStateTransition records a state transition with hypervisor label. func (m *manager) recordStateTransition(ctx context.Context, fromState, toState string, hvType hypervisor.Type) { if m.metrics == nil { diff --git a/lib/instances/metrics_test.go b/lib/instances/metrics_test.go index 811571e36..525002cbb 100644 --- a/lib/instances/metrics_test.go +++ b/lib/instances/metrics_test.go @@ -2,6 +2,7 @@ package instances import ( "context" + "errors" "os" "path/filepath" "testing" @@ -556,6 +557,50 @@ func TestEnsureSnapshotMemoryReadySkipsPendingCompressionWithoutPreemptionMetric assert.False(t, metricExists(rm, "hypeman_snapshot_compression_preemptions_total"), "pending-delay cancellation should not record a preemption") } +func TestVGPUReconcileFailureMetric_RecordStages(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + reader := otelmetric.NewManualReader() + provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) + + m := &manager{ + paths: paths.New(t.TempDir()), + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { + return errors.New("sweep failed") + }, + } + metrics, err := newInstanceMetrics(provider.Meter("test"), nil, m) + require.NoError(t, err) + m.metrics = metrics + + const id = "unreadable" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{Id: id}})) + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o000)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + m.ReconcileVGPUs(t.Context()) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(t.Context(), &rm)) + + failuresMetric := findMetric(t, rm, "hypeman_instances_vgpu_reconcile_failures_total") + failures, ok := failuresMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, failures.DataPoints, 2) + for _, point := range failures.DataPoints { + switch metricLabel(t, point.Attributes, "stage") { + case "list_instances", "reconcile_devices": + assert.Equal(t, int64(1), point.Value) + default: + t.Fatalf("unexpected reconcile failure stage datapoint: %s", metricLabel(t, point.Attributes, "stage")) + } + } +} + func assertMetricNames(t *testing.T, rm metricdata.ResourceMetrics, expected []string) { t.Helper() diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 1d885d0b5..7e8500402 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -1,6 +1,7 @@ package instances import ( + "context" "errors" "fmt" "os" @@ -13,6 +14,7 @@ import ( "time" "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/logger" ) // linuxBootIDPath is the kernel-provided boot ID used to scope process @@ -25,6 +27,25 @@ const linuxBootIDPath = "/proc/sys/kernel/random/boot_id" // does not unstick it, so the wait is short to keep stop and delete fast. const hypervisorSIGKILLWaitTimeout = 2 * time.Second +func (m *manager) vfioTerminationGrace() time.Duration { + if m.vfioTermGrace > 0 { + return m.vfioTermGrace + } + return hypervisor.VFIOTermGrace +} + +// SIGKILL during guest driver init can wedge a VF until the parent GPU is reset. +func (m *manager) terminateThenKill(ctx context.Context, inst *Instance, pid int) error { + if inst.GPUProfile != "" || len(inst.Devices) > 0 { + if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vfioTerminationGrace()) { + return nil + } + logger.FromContext(ctx).WarnContext(ctx, "hypervisor with VFIO devices did not exit on SIGTERM; hard-killing, device may wedge if the guest driver was initializing", + "instance_id", inst.Id, "device_path", inst.GPUDevicePath) + } + return killProcessAndWait(pid) +} + // killProcessAndWait SIGKILLs pid and waits for it to exit. A process that // survives the first wait gets its process group killed too (the hypervisor // may have spawned children in its own group) and a short grace period. An @@ -180,6 +201,12 @@ func classifyResolvedHypervisorOwner(socketPath string, stored, resolved int, er return 0, fmt.Errorf("cannot confirm ownership of socket %s: %w", socketPath, err) } +// Ambiguous ownership is treated as live; this must not authorize teardown. +func hypervisorMayBeAlive(id HypervisorProcessIdentity, socketPath string) bool { + pid, err := resolveLiveHypervisorPID(id, socketPath) + return err != nil || pid > 0 +} + // ProcessExists reports whether pid belongs to a live, non-zombie process. func ProcessExists(pid int) bool { if pid <= 0 { diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index 7450ed112..2ec201450 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -17,7 +17,9 @@ import ( "testing" "time" + "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -508,6 +510,47 @@ func TestRefreshHypervisorPIDResolvesSocketOwnerWhenStoredPIDIsDead(t *testing.T assert.Equal(t, hostBootID(), stored.HypervisorBootID) } +func TestVGPUAssignmentClaimedByLiveInstanceProtectsReusedPIDClaim(t *testing.T) { + socketPath := filepath.Join(t.TempDir(), "test.sock") + owner := exec.Command(os.Args[0], "-test.run=^TestSocketListenerHelper$") + owner.Env = append(os.Environ(), "HYPERVISOR_SOCKET_HELPER=1", "HYPERVISOR_SOCKET_PATH="+socketPath) + stdin, err := owner.StdinPipe() + require.NoError(t, err) + stdout, err := owner.StdoutPipe() + require.NoError(t, err) + require.NoError(t, owner.Start()) + t.Cleanup(func() { + _ = stdin.Close() + _ = owner.Process.Kill() + _ = owner.Wait() + }) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + + stale := exec.Command("sleep", "30") + require.NoError(t, stale.Start()) + t.Cleanup(func() { + _ = stale.Process.Kill() + _ = stale.Wait() + }) + + m := &manager{paths: paths.New(t.TempDir())} + const devicePath = "/sys/bus/pci/devices/0000:82:00.4" + stalePID := stale.Process.Pid + require.NoError(t, m.ensureDirectories("live-claimant")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "live-claimant", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &stalePID}, + SocketPath: socketPath, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance("other-instance", devicePath) + require.NoError(t, err) + assert.True(t, claimed) +} + func TestKillHypervisorSurvivesConcurrentReaper(t *testing.T) { socketPath := filepath.Join(t.TempDir(), "test.sock") process := exec.Command(os.Args[0], "-test.run=^TestSocketListenerHelper$") @@ -648,3 +691,84 @@ func TestResolveRuntimeHypervisorPIDMintsIdentityOnlyWhenConfirmed(t *testing.T) assert.Empty(t, stored.HypervisorBootID, "a dead fallback must not mint the identity token") }) } + +func startTrapProcess(t *testing.T, trapAction string) (int, HypervisorProcessIdentity) { + t.Helper() + script := fmt.Sprintf("trap '%s' TERM; echo ready; sleep 30 & wait", trapAction) + process := exec.Command("sh", "-c", script) + stdout, err := process.StdoutPipe() + require.NoError(t, err) + require.NoError(t, process.Start()) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + waitDone := make(chan error, 1) + go func() { waitDone <- process.Wait() }() + t.Cleanup(func() { + _ = process.Process.Kill() + <-waitDone + }) + + pid := process.Process.Pid + startTime := processStartTime(pid) + require.NotZero(t, startTime) + return pid, HypervisorProcessIdentity{HypervisorPID: &pid, HypervisorStartTime: startTime, HypervisorBootID: hostBootID()} +} + +func TestKillHypervisorSIGTERMsVGPUHypervisor(t *testing.T) { + markerPath := filepath.Join(t.TempDir(), "terminated") + pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") + socketPath := filepath.Join(t.TempDir(), "missing.sock") + + m := &manager{} + require.NoError(t, m.killHypervisor(context.Background(), &Instance{ + State: StateInitializing, + StoredMetadata: StoredMetadata{ + Id: "kill-test", + GPUProfile: "NVIDIA L40S-1Q", + HypervisorProcessIdentity: identity, + SocketPath: socketPath, + }, + })) + + require.Eventually(t, func() bool { + return syscall.Kill(pid, 0) == syscall.ESRCH + }, 5*time.Second, 10*time.Millisecond) + assert.FileExists(t, markerPath, "hypervisor must be given SIGTERM, not SIGKILL, during vGPU driver init") +} + +func TestKillHypervisorEscalatesToSIGKILLWhenSIGTERMIgnored(t *testing.T) { + pid, identity := startTrapProcess(t, "") + socketPath := filepath.Join(t.TempDir(), "missing.sock") + + m := &manager{vfioTermGrace: 50 * time.Millisecond} + require.NoError(t, m.killHypervisor(context.Background(), &Instance{ + State: StateInitializing, + StoredMetadata: StoredMetadata{ + Id: "kill-test", + GPUProfile: "NVIDIA L40S-1Q", + HypervisorProcessIdentity: identity, + SocketPath: socketPath, + }, + })) + + assert.ErrorIs(t, syscall.Kill(pid, 0), syscall.ESRCH, "SIGTERM-ignoring hypervisor must still be hard-killed") +} + +func TestKillHypervisorHardKillsNonVGPUHypervisor(t *testing.T) { + markerPath := filepath.Join(t.TempDir(), "terminated") + pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") + socketPath := filepath.Join(t.TempDir(), "missing.sock") + + m := &manager{} + require.NoError(t, m.killHypervisor(context.Background(), &Instance{ + State: StateRunning, + StoredMetadata: StoredMetadata{ + Id: "kill-test", + HypervisorProcessIdentity: identity, + SocketPath: socketPath, + }, + })) + + assert.ErrorIs(t, syscall.Kill(pid, 0), syscall.ESRCH) + assert.NoFileExists(t, markerPath, "non-vGPU hypervisors keep the direct SIGKILL path") +} diff --git a/lib/instances/query.go b/lib/instances/query.go index 8e3ed61f1..0621460da 100644 --- a/lib/instances/query.go +++ b/lib/instances/query.go @@ -784,7 +784,7 @@ func parseSentinelTimestamp(line, sentinelPrefix string) (time.Time, bool) { return time.Time{}, false } -// listInstances returns all instances +// listInstances returns all instances, skipping metadata files that cannot be loaded. func (m *manager) listInstances(ctx context.Context) ([]Instance, error) { ctx, span := m.tracerOrDefault().Start(ctx, "instances.list_metadata") defer span.End() diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index 8bb0bb464..222e85a30 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -14,6 +14,67 @@ import ( "github.com/stretchr/testify/require" ) +func TestListMetadataForReconcileFailsOnInvalidMetadata(t *testing.T) { + m := &manager{paths: paths.New(t.TempDir())} + + require.NoError(t, m.ensureDirectories("valid")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "valid", + Name: "valid", + CreatedAt: time.Now(), + DataDir: m.paths.InstanceDir("valid"), + }})) + require.NoError(t, m.ensureDirectories("invalid")) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid"), []byte("{"), 0644)) + + listed, err := m.ListInstances(context.Background(), nil) + require.NoError(t, err) + require.Len(t, listed, 1) + + _, err = m.listMetadataForReconcile() + require.Error(t, err) + assert.ErrorContains(t, err, "load metadata for instance invalid") + + require.NoError(t, os.Remove(m.paths.InstanceMetadata("invalid"))) + metadata, err := m.listMetadataForReconcile() + require.NoError(t, err) + require.Len(t, metadata, 1) + assert.Equal(t, "valid", metadata[0].Id) +} + +func TestListMetadataForReconcileSkipsInstanceDeletedDuringListing(t *testing.T) { + m := &manager{paths: paths.New(t.TempDir())} + + for _, id := range []string{"aaa-ghost", "zzz-live"} { + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + CreatedAt: time.Now(), + DataDir: m.paths.InstanceDir(id), + }})) + } + + unlock := hypervisor.LockSnapshotSourceAliasMutation() + type result struct { + metadata []StoredMetadata + err error + } + done := make(chan result, 1) + go func() { + metadata, err := m.listMetadataForReconcile() + done <- result{metadata, err} + }() + time.Sleep(100 * time.Millisecond) + require.NoError(t, os.Remove(m.paths.InstanceMetadata("aaa-ghost"))) + unlock() + + res := <-done + require.NoError(t, res.err) + require.Len(t, res.metadata, 1) + assert.Equal(t, "zzz-live", res.metadata[0].Id) +} + func TestParseExitSentinelLine(t *testing.T) { t.Parallel() tests := []struct { diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 2d2676c72..a87acb614 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -66,6 +66,9 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata + if stored.GPURetainedForCleanup { + return nil, errVGPURetentionStub + } if err := validateForkVolumeSafety(stored.Volumes); err != nil { return nil, fmt.Errorf("%w: snapshot requires readonly volume attachments: %v", ErrNotSupported, err) } @@ -263,6 +266,9 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str if sourceInst.State == StateRunning { return nil, fmt.Errorf("%w: cannot restore snapshot while source is %s", ErrInvalidState, sourceInst.State) } + if sourceMeta.GPURetainedForCleanup { + return nil, errVGPURetentionStub + } targetState, err := resolveSnapshotTargetState(rec.Snapshot.Kind, req.TargetState) if err != nil { @@ -312,6 +318,7 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str restored.GPUFramework = sourceMeta.GPUFramework restored.GPUDevicePath = sourceMeta.GPUDevicePath restored.GPUMdevUUID = sourceMeta.GPUMdevUUID + restored.GPUAssignedAt = sourceMeta.GPUAssignedAt restored.HypervisorType = targetHypervisor restored.HypervisorVersion = targetHypervisorVersion restored.SocketPath = m.paths.InstanceSocket(id, starter.SocketName()) diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index b23e364ee..31f0a4343 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -52,6 +52,63 @@ func TestForkSnapshotClearsVGPUAssignment(t *testing.T) { assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", source.GPUDevicePath) } +func TestCreateSnapshotRejectsVGPURetentionRecord(t *testing.T) { + mgr, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "snapshot-vgpu-retention" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, sourceID, mgr.defaultHypervisor) + + meta, err := mgr.loadMetadata(sourceID) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, mgr.saveMetadata(meta)) + + _, err = mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Name: "snapshot-vgpu-retention", + }) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") +} + +func TestRestoreSnapshotRejectsVGPURetentionRecord(t *testing.T) { + mgr, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "snapshot-vgpu-restore-retention" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, sourceID, mgr.defaultHypervisor) + + snapshot, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Name: "snapshot-vgpu-restore-retention", + }) + require.NoError(t, err) + + meta, err := mgr.loadMetadata(sourceID) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, mgr.saveMetadata(meta)) + + _, err = mgr.RestoreSnapshot(ctx, sourceID, snapshot.Id, RestoreSnapshotRequest{ + TargetState: StateStopped, + TargetHypervisor: mgr.defaultHypervisor, + }) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") + + stored, err := mgr.loadMetadata(sourceID) + require.NoError(t, err) + assert.True(t, stored.GPURetainedForCleanup) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) +} + func TestRestoreSnapshotDoesNotResurrectStaleVGPUAssignment(t *testing.T) { mgr, _ := setupTestManager(t) ctx := context.Background() @@ -113,6 +170,8 @@ func TestRestoreSnapshotKeepsCurrentVGPUAssignment(t *testing.T) { meta.GPUFramework = devices.VGPUFramework("future-framework") meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" meta.GPUMdevUUID = "retained-uuid" + assignedAt := time.Now().UTC().Truncate(time.Second) + meta.GPUAssignedAt = &assignedAt require.NoError(t, mgr.saveMetadata(meta)) _, err = mgr.RestoreSnapshot(ctx, sourceID, snapshot.Id, RestoreSnapshotRequest{ @@ -126,6 +185,8 @@ func TestRestoreSnapshotKeepsCurrentVGPUAssignment(t *testing.T) { assert.Equal(t, devices.VGPUFramework("future-framework"), restored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", restored.GPUDevicePath) assert.Equal(t, "retained-uuid", restored.GPUMdevUUID) + require.NotNil(t, restored.GPUAssignedAt) + assert.True(t, assignedAt.Equal(*restored.GPUAssignedAt)) } func TestStoppedSnapshotLifecycleAndForkAfterSourceDeletion(t *testing.T) { diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 4c6fd7d33..5c6e0173e 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -376,7 +376,7 @@ func (m *manager) shutdownHypervisor(ctx context.Context, inst *Instance) error // alive; teardown is committed, so kill it rather than report a // completed shutdown for a VMM that is still running. log.WarnContext(ctx, "could not connect to hypervisor, force killing resolved owner", "instance_id", inst.Id, "pid", pid, "error", err) - if err := killProcessAndWait(pid); err != nil { + if err := m.terminateThenKill(ctx, inst, pid); err != nil { return err } } @@ -405,13 +405,13 @@ func (m *manager) shutdownHypervisor(ctx context.Context, inst *Instance) error log.DebugContext(ctx, "hypervisor shutdown gracefully", "instance_id", inst.Id, "pid", pid) } else { log.WarnContext(ctx, "hypervisor did not exit gracefully in time, force killing process", "instance_id", inst.Id, "pid", pid) - if err := killProcessAndWait(pid); err != nil { + if err := m.terminateThenKill(ctx, inst, pid); err != nil { return err } } } else { log.DebugContext(ctx, "skipping graceful exit wait; force killing hypervisor process", "instance_id", inst.Id, "pid", pid) - if err := killProcessAndWait(pid); err != nil { + if err := m.terminateThenKill(ctx, inst, pid); err != nil { return err } } diff --git a/lib/instances/start.go b/lib/instances/start.go index 7e7855eac..37b836427 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/egressproxy" "github.com/kernel/hypeman/lib/instances/phasetracking" "github.com/kernel/hypeman/lib/logger" @@ -48,13 +47,17 @@ func (m *manager) startInstance( log.ErrorContext(ctx, "invalid state for start", "instance_id", id, "state", inst.State) return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } + if stored.GPURetainedForCleanup { + log.ErrorContext(ctx, "refusing to start vGPU retention record", "instance_id", id) + return nil, errVGPURetentionStub + } // Release any assignment retained by an earlier failed release and // persist the cleared fields immediately, so a failure later in start // cannot leave on-disk metadata pointing at a device that is already - // gone (matching releaseRetainedVGPULocked). + // gone. if storedVGPUDevicePath(stored) != "" { - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { log.ErrorContext(ctx, "failed to release stale vGPU before start", "instance_id", id, "error", err) return nil, fmt.Errorf("release stale vGPU before start: %w", err) } @@ -64,6 +67,12 @@ func (m *manager) startInstance( } } + stored.HypervisorPID = nil + stored.HypervisorStartTime = 0 + stored.HypervisorBootID = "" + rollbackMeta := *meta + rollbackMeta.Phases = meta.Phases.Clone() + // 2a. Clear stale exit info from previous run and apply command overrides stored.ExitCode = nil stored.ExitMessage = "" @@ -111,6 +120,9 @@ func (m *manager) startInstance( } // Setup cleanup stack for automatic rollback on errors + retention := vgpuRetention{instanceID: id} + // Deferred before cu.Clean so rollback records retention before this wraps the error. + defer func() { retErr = retention.wrapPending(retErr) }() cu := cleanup.Make(func() {}) defer cu.Clean() @@ -162,26 +174,40 @@ func (m *manager) startInstance( // 4b. Recreate the vGPU if this instance had a GPU profile // Note: GPU availability was already validated in step 2b if stored.GPUProfile != "" { - log.InfoContext(ctx, "creating vGPU mdev for start", "instance_id", id, "profile", stored.GPUProfile) - device, err := devices.CreateVGPU(ctx, stored.GPUProfile, id) + log.InfoContext(ctx, "creating vGPU for start", "instance_id", id, "profile", stored.GPUProfile) + device, err := m.createVGPUDevice(ctx, stored.GPUProfile, id) if err != nil { log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) - return nil, fmt.Errorf("create vGPU mdev for profile %s: %w", stored.GPUProfile, err) + wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) + if pendingDevice, ok := vgpuDevicePendingCleanup(err); ok { + retentionMeta := rollbackMeta + setStoredVGPUDevice(&retentionMeta.StoredMetadata, pendingDevice, m.nowUTC()) + persisted := true + if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { + log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) + wrapped = fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr) + persisted = false + } + retention.markRetained(persisted) + m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, persisted) + } + return nil, wrapped } - setStoredVGPUDevice(stored, device) + assignedAt := m.nowUTC() + setStoredVGPUDevice(stored, device, assignedAt) log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", stored.GPUProfile, "uuid", device.MdevUUID) // Add vGPU cleanup to stack cu.Add(func() { - log.DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID) - assignment := devices.VGPUAssignment{ - Framework: device.Framework, - DevicePath: device.SysfsPath, - MdevUUID: device.MdevUUID, - } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { - log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID, "error", err) + retained, persisted := m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) + if retained { + retention.markRetained(persisted) + m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, persisted) } }) + if err := m.saveMetadata(meta); err != nil { + log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err) + return nil, fmt.Errorf("save metadata after vGPU creation: %w", err) + } } // 5. Regenerate config disk with new network configuration diff --git a/lib/instances/stop.go b/lib/instances/stop.go index 7eddeb034..ebcf58708 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -232,7 +232,7 @@ func (m *manager) stopInstance( // 7. Release the vGPU assignment if present (frees the vGPU slot for other VMs). if path := storedVGPUDevicePath(stored); path != "" { log.InfoContext(ctx, "destroying vGPU on stop", "instance_id", id, "device_path", path) - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on stop; retaining assignment metadata", "instance_id", id, "device_path", path, "error", err) } } diff --git a/lib/instances/storage.go b/lib/instances/storage.go index a293fc6e1..1a4d325b0 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -187,8 +187,15 @@ func removeAllWithRetry(path string, removeAll func(string) error, sleep func(ti } } -// listMetadataFiles returns paths to all instance metadata files func (m *manager) listMetadataFiles() ([]string, error) { + return m.walkMetadataFiles(false) +} + +func (m *manager) listMetadataFilesStrict() ([]string, error) { + return m.walkMetadataFiles(true) +} + +func (m *manager) walkMetadataFiles(failOnStatError bool) ([]string, error) { guestsDir := m.paths.GuestsDir() // Ensure guests directory exists @@ -210,6 +217,8 @@ func (m *manager) listMetadataFiles() ([]string, error) { metaPath := filepath.Join(guestsDir, entry.Name(), "metadata.json") if _, err := os.Stat(metaPath); err == nil { metaFiles = append(metaFiles, metaPath) + } else if failOnStatError && !os.IsNotExist(err) { + return nil, fmt.Errorf("stat metadata for instance %s: %w", entry.Name(), err) } } diff --git a/lib/instances/types.go b/lib/instances/types.go index 6aac15985..ed32f5b5d 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -151,10 +151,12 @@ type StoredMetadata struct { Devices []string // Device IDs attached to this instance // GPU configuration (vGPU mode) - GPUProfile string // vGPU profile name (e.g., "L40S-1Q") - GPUFramework devices.VGPUFramework - GPUDevicePath string - GPUMdevUUID string // populated for mdev-backed vGPUs + GPUProfile string // vGPU profile name (e.g., "L40S-1Q") + GPUFramework devices.VGPUFramework + GPUDevicePath string + GPUMdevUUID string // populated for mdev-backed vGPUs + GPUAssignedAt *time.Time // set before hypervisor startup to bound crash recovery protection + GPURetainedForCleanup bool // delete-only stub holding a vGPU assignment // Command overrides (like docker run ) Entrypoint []string // Override image entrypoint (nil = use image default) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index cffe2ac1d..00e32ba01 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -2,62 +2,168 @@ package instances import ( "context" + "errors" + "fmt" "path/filepath" + "time" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/logger" ) -func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice) { +// VGPUCleanupPendingError reports a failed rollback that left a vGPU assigned. +type VGPUCleanupPendingError struct { + InstanceID string + Retained bool + Err error +} + +func (e *VGPUCleanupPendingError) Error() string { + if e.Retained { + return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID) + } + return fmt.Sprintf("%v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the periodic vGPU reconcile retries the release", e.Err, e.InstanceID) +} + +func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } + +var errVGPURetentionStub = fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + +func (m *manager) createVGPUDevice(ctx context.Context, profileName, instanceID string) (*devices.VGPUDevice, error) { + create := m.createVGPU + if create == nil { + create = devices.CreateVGPU + } + return create(ctx, profileName, instanceID) +} + +func vgpuDevicePendingCleanup(err error) (*devices.VGPUDevice, bool) { + var pending *devices.VGPUCreateCleanupPendingError + if !errors.As(err, &pending) { + return nil, false + } + return &pending.Device, true +} + +func vgpuAssignmentMayBeLive(stored *StoredMetadata, now time.Time, hypervisorLive bool) bool { + return hypervisorLive || + stored.GPUAssignedAt != nil && now.Sub(*stored.GPUAssignedAt) < devices.VGPUAssignmentGracePeriod +} + +func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { + destroy := m.destroyVGPU + if destroy == nil { + destroy = devices.DestroyVGPU + } + return destroy(ctx, assignment) +} + +func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice, assignedAt time.Time) { stored.GPUFramework = device.Framework stored.GPUDevicePath = device.SysfsPath stored.GPUMdevUUID = device.MdevUUID + stored.GPUAssignedAt = &assignedAt } func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUFramework = devices.VGPUFrameworkNone stored.GPUDevicePath = "" stored.GPUMdevUUID = "" + stored.GPUAssignedAt = nil } -func releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { +func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) (retained, persisted bool) { + logger.FromContext(ctx).DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID) + releaseErr := m.destroyVGPUAssignment(ctx, devices.VGPUAssignment{ + Framework: device.Framework, + DevicePath: device.SysfsPath, + MdevUUID: device.MdevUUID, + InstanceID: instanceID, + }) + if releaseErr != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID, "error", releaseErr) + setStoredVGPUDevice(&rollbackMeta.StoredMetadata, device, assignedAt) + retained = true + } + if err := m.saveMetadata(&rollbackMeta); err != nil { + message := "failed to save metadata after vGPU cleanup" + if retained { + message = "failed to retain vGPU assignment metadata after cleanup failure" + } + logger.FromContext(ctx).ErrorContext(ctx, message, "instance_id", instanceID, "error", err) + if !retained { + return false, false + } + // The pre-cleanup assignment save may already hold this claim on disk, in + // which case the retention is durable despite the failed rollback save. + if onDisk, loadErr := m.loadMetadata(instanceID); loadErr == nil && + onDisk.GPUDevicePath == device.SysfsPath && + onDisk.GPUAssignedAt != nil && onDisk.GPUAssignedAt.Equal(assignedAt) { + return true, true + } + return true, false + } + return retained, retained +} + +func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { path := storedVGPUDevicePath(stored) if path != "" { - assignment := devices.VGPUAssignment{ - Framework: stored.GPUFramework, - DevicePath: path, - MdevUUID: stored.GPUMdevUUID, + // Vendor VFIO VFs are reusable, so release fails closed on an incomplete inventory. + claimed := false + if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { + var err error + claimed, err = m.vgpuAssignmentClaimedByLiveInstance(stored.Id, path) + if err != nil { + return err + } } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { - return err + if claimed { + logger.FromContext(ctx).WarnContext(ctx, "dropping stale vGPU assignment claimed by another live instance", + "instance_id", stored.Id, "device_path", path) + } else { + assignment := devices.VGPUAssignment{ + Framework: stored.GPUFramework, + DevicePath: path, + MdevUUID: stored.GPUMdevUUID, + InstanceID: stored.Id, + } + if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { + return err + } } } clearStoredVGPUDevice(stored) return nil } -// releaseRetainedVGPULocked releases a vGPU assignment retained on a stopped -// instance after a failed release during the original stop. It is a no-op -// when no assignment is retained, and a failed retry only logs so the -// metadata stays for the next retry. The caller must hold the instance lock. -func (m *manager) releaseRetainedVGPULocked(ctx context.Context, id string) { - log := logger.FromContext(ctx) - meta, err := m.loadMetadata(id) +func (m *manager) vgpuAssignmentClaimedByLiveInstance(excludeID, devicePath string) (bool, error) { + // Each vendor VFIO release lists and loads every instance's metadata. This is + // acceptable at GPU-host scale (tens of VFs and instances); revisit at hundreds. + allMetadata, err := m.listMetadataForReconcile() if err != nil { - log.WarnContext(ctx, "failed to load metadata for retained vGPU release", "instance_id", id, "error", err) - return - } - stored := &meta.StoredMetadata - if storedVGPUDevicePath(stored) == "" { - return + return false, fmt.Errorf("list instances for vGPU release check: %w", err) } - if err := releaseStoredVGPU(ctx, stored); err != nil { - log.WarnContext(ctx, "failed to destroy retained vGPU; retaining assignment metadata", "instance_id", id, "error", err) - return - } - if err := m.saveMetadata(meta); err != nil { - log.WarnContext(ctx, "failed to save metadata after retained vGPU release", "instance_id", id, "error", err) + for i := range allMetadata { + stored := &allMetadata[i] + if stored.Id == excludeID || storedVGPUDevicePath(stored) != devicePath { + continue + } + pid, err := resolveLiveHypervisorPID(stored.HypervisorProcessIdentity, stored.SocketPath) + if err != nil { + return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", stored.Id, devicePath, err) + } + if pid > 0 { + return true, nil + } + if vgpuAssignmentMayBeLive(stored, m.nowUTC(), false) { + if stored.HypervisorPID == nil { + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", stored.Id, devicePath) + } + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: recorded hypervisor is not running", stored.Id, devicePath) + } } + return false, nil } func storedVGPUDevicePath(stored *StoredMetadata) string { diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go new file mode 100644 index 000000000..27ecb9e01 --- /dev/null +++ b/lib/instances/vgpu_reconcile.go @@ -0,0 +1,134 @@ +package instances + +import ( + "context" + "errors" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/logger" +) + +const defaultVGPUReconcileInterval = time.Minute + +// StartVGPUReconciler runs one reconcile pass and then keeps reconciling +// periodically until ctx is cancelled. Hosts without a vGPU framework skip +// reconciliation entirely. A discovery failure starts the reconciler anyway: +// a transient sysfs error must not disable cleanup on a GPU host. +func (m *manager) StartVGPUReconciler(ctx context.Context) { + discover := m.discoverVGPU + if discover == nil { + discover = devices.DiscoverVGPU + } + framework, _, err := discover() + if err == nil && framework == devices.VGPUFrameworkNone { + return + } + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU framework; starting vGPU reconciler anyway", "error", err) + } + m.ReconcileVGPUs(ctx) + m.vgpuReconcileOnce.Do(func() { + interval := m.vgpuReconcileInterval + if interval <= 0 { + interval = defaultVGPUReconcileInterval + } + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + m.ReconcileVGPUs(ctx) + } + } + }() + }) +} + +// ReconcileVGPUs runs one fail-closed reconcile pass: stale instance-held +// assignments are released, then device-level leftovers not claimed by a live +// instance are swept. Failures only log; the next periodic pass retries. +func (m *manager) ReconcileVGPUs(ctx context.Context) { + log := logger.FromContext(ctx) + protected, err := m.reconcileVGPUAssignments(ctx) + sweepDevices := err == nil + if err != nil { + m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageListInstances) + log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping device sweep until the next pass", "error", err) + protected = make(map[string]struct{}) + } + reconcileDevices := m.reconcileVGPUDevices + if reconcileDevices == nil { + reconcileDevices = devices.ReconcileVGPUs + } + if err := reconcileDevices(ctx, protected, sweepDevices); err != nil { + m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageReconcileDevices) + log.WarnContext(ctx, "failed to reconcile vGPU devices", "error", err) + } +} + +// reconcileVGPUAssignments retries releases for assignments whose owner is no +// longer live and returns the device paths still protected by live instances. +// Listing fails closed: any unreadable metadata aborts the pass so the vendor +// VFIO sweep cannot clear a VF whose claim it could not read. +func (m *manager) reconcileVGPUAssignments(ctx context.Context) (map[string]struct{}, error) { + allMetadata, err := m.listMetadataForReconcile() + if err != nil { + return nil, err + } + protected := make(map[string]struct{}) + for i := range allMetadata { + stored := &allMetadata[i] + devicePath := storedVGPUDevicePath(stored) + if devicePath == "" { + continue + } + hypervisorLive := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + if vgpuAssignmentMayBeLive(stored, m.nowUTC(), hypervisorLive) { + protected[devicePath] = struct{}{} + continue + } + m.releaseStaleVGPUAssignment(ctx, stored.Id) + } + return protected, nil +} + +// releaseStaleVGPUAssignment retries a release that previously failed, under +// the instance lock. Liveness is re-verified after locking so a concurrent +// start or restore keeps its assignment. A failed release only logs and keeps +// the metadata for the next pass. +func (m *manager) releaseStaleVGPUAssignment(ctx context.Context, id string) { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + log := logger.FromContext(ctx) + meta, err := m.loadMetadata(id) + if err != nil { + if !errors.Is(err, ErrNotFound) { + log.WarnContext(ctx, "failed to load metadata for stale vGPU release", "instance_id", id, "error", err) + } + return + } + stored := &meta.StoredMetadata + path := storedVGPUDevicePath(stored) + if path == "" { + return + } + hypervisorLive := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + if vgpuAssignmentMayBeLive(stored, m.nowUTC(), hypervisorLive) { + return + } + if err := m.releaseStoredVGPU(ctx, stored); err != nil { + m.recordVGPUStaleReleaseFailure(ctx) + log.WarnContext(ctx, "failed to release stale vGPU assignment; retrying on the next reconcile pass", "instance_id", id, "device_path", path, "error", err) + return + } + if err := m.saveMetadata(meta); err != nil { + log.WarnContext(ctx, "failed to save metadata after stale vGPU release", "instance_id", id, "error", err) + return + } + log.InfoContext(ctx, "released stale vGPU assignment", "instance_id", id, "device_path", path) +} diff --git a/lib/instances/vgpu_reconcile_linux_test.go b/lib/instances/vgpu_reconcile_linux_test.go new file mode 100644 index 000000000..553c64c15 --- /dev/null +++ b/lib/instances/vgpu_reconcile_linux_test.go @@ -0,0 +1,58 @@ +//go:build linux + +package instances + +import ( + "context" + "net" + "path/filepath" + "testing" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReconcileVGPUsProtectsSocketOwnerWithoutPersistedPID(t *testing.T) { + t.Parallel() + + socketPath := filepath.Join(t.TempDir(), "test.sock") + listener, err := net.Listen("unix", socketPath) + require.NoError(t, err) + defer listener.Close() + + var destroyed []devices.VGPUAssignment + var protected map[string]struct{} + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }, + reconcileVGPUDevices: func(_ context.Context, p map[string]struct{}, _ bool) error { + protected = p + return nil + }, + } + const id = "pid-save-failed" + stale := time.Now().UTC().Add(-devices.VGPUAssignmentGracePeriod - time.Minute) + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + SocketPath: socketPath, + GPUFramework: devices.VGPUFrameworkMdev, + GPUDevicePath: "/sys/bus/mdev/devices/test-mdev", + GPUMdevUUID: "test-mdev", + GPUAssignedAt: &stale, + }})) + + m.ReconcileVGPUs(t.Context()) + + assert.Empty(t, destroyed, "a live socket owner must block the release even with a nil persisted PID") + assert.Contains(t, protected, "/sys/bus/mdev/devices/test-mdev") + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/mdev/devices/test-mdev", stored.GPUDevicePath) +} diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go new file mode 100644 index 000000000..8b8d71b66 --- /dev/null +++ b/lib/instances/vgpu_reconcile_test.go @@ -0,0 +1,245 @@ +package instances + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { + dead := exec.Command("true") + require.NoError(t, dead.Run()) + deadPID := dead.Process.Pid + now := time.Now().UTC() + recent := now.Add(-time.Minute) + stale := now.Add(-devices.VGPUAssignmentGracePeriod - time.Minute) + + var protected map[string]struct{} + var destroyed []devices.VGPUAssignment + m := &manager{ + paths: paths.New(t.TempDir()), + now: func() time.Time { return now }, + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }, + reconcileVGPUDevices: func(_ context.Context, p map[string]struct{}, sweepDevices bool) error { + protected = p + assert.True(t, sweepDevices) + return nil + }, + } + instances := []StoredMetadata{ + {Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}, + {Id: "orphaned", GPUProfile: "NVIDIA L40S-2Q", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}, + {Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}, + {Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}}, + {Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, GPUAssignedAt: &recent}, + {Id: "legacy-mdev-booting", GPUMdevUUID: "test-mdev", GPUAssignedAt: &recent}, + } + for i := range instances { + require.NoError(t, m.ensureDirectories(instances[i].Id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: instances[i]})) + } + + m.ReconcileVGPUs(t.Context()) + assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") + assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") + assert.Contains(t, protected, "/sys/bus/mdev/devices/test-mdev") + + for _, id := range []string{"orphaned", "legacy", "dead"} { + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "stale assignment on %s must be released", id) + } + assert.Contains(t, destroyed, devices.VGPUAssignment{ + Framework: devices.VGPUFrameworkVendorVFIO, + DevicePath: "/sys/bus/pci/devices/0000:82:00.5", + InstanceID: "orphaned", + }) + orphaned, err := m.loadMetadata("orphaned") + require.NoError(t, err) + assert.Equal(t, "NVIDIA L40S-2Q", orphaned.GPUProfile) + for _, id := range []string{"booting", "stale-pid-booting", "legacy-mdev-booting"} { + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.NotEmpty(t, storedVGPUDevicePath(&stored.StoredMetadata), "live assignment on %s must be kept", id) + } +} + +func TestReconcileVGPUsSkipsDeviceSweepWhenListingFails(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + var sweeps []bool + m := &manager{ + paths: paths.New(t.TempDir()), + reconcileVGPUDevices: func(_ context.Context, _ map[string]struct{}, sweepDevices bool) error { + sweeps = append(sweeps, sweepDevices) + return nil + }, + } + const id = "unreadable" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{Id: id}})) + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o000)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + m.ReconcileVGPUs(t.Context()) + require.Equal(t, []bool{false}, sweeps, + "a listing failure must skip the device sweep, not run it with an empty protection set") + + require.NoError(t, os.Chmod(instanceDir, 0o755)) + m.ReconcileVGPUs(t.Context()) + assert.Equal(t, []bool{false, true}, sweeps, "the next pass retries the device sweep") +} + +func TestReconcileVGPUsKeepsAssignmentWhenReleaseFails(t *testing.T) { + t.Parallel() + + var attempts atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + if attempts.Add(1) == 1 { + return errors.New("vGPU destroy failed: 0xffffffff") + } + return nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, + } + const id = "wedged" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }})) + + m.ReconcileVGPUs(t.Context()) + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath, + "a failed release must keep the assignment metadata for the next pass") + + m.ReconcileVGPUs(t.Context()) + stored, err = m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "the next pass retries the release") + assert.Equal(t, int32(2), attempts.Load()) +} + +func TestReconcileVGPUsDefersToUnconfirmedClaimant(t *testing.T) { + t.Parallel() + + var destroys atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + destroys.Add(1) + return nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, + } + const path = "/sys/bus/pci/devices/0000:82:00.4" + assignedAt := time.Now() + for _, stored := range []StoredMetadata{ + {Id: "stale", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: path}, + {Id: "mid-boot-claimant", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: path, GPUAssignedAt: &assignedAt}, + } { + require.NoError(t, m.ensureDirectories(stored.Id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: stored})) + } + + m.ReconcileVGPUs(t.Context()) + + assert.Zero(t, destroys.Load(), "no destroy may fire while the claim scan cannot clear the path") + stale, err := m.loadMetadata("stale") + require.NoError(t, err) + assert.Equal(t, path, stale.GPUDevicePath, "the stale release retries once the claimant's liveness is decidable") +} + +func TestReconcileVGPUsReleasesRetentionStubAssignment(t *testing.T) { + t.Parallel() + + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, + } + const id = "retention-stub" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + Name: "failed-create", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPURetainedForCleanup: true, + }})) + + m.ReconcileVGPUs(t.Context()) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "the stub's wedged assignment is released once free") + assert.True(t, stored.GPURetainedForCleanup, "the stub stays a delete-only record of the failed create") +} + +func TestStartVGPUReconcilerSkipsHostsWithoutGPUs(t *testing.T) { + t.Parallel() + + var passes atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkNone, nil, nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { + passes.Add(1) + return nil + }, + vgpuReconcileInterval: time.Millisecond, + } + + m.StartVGPUReconciler(t.Context()) + time.Sleep(20 * time.Millisecond) + assert.Zero(t, passes.Load(), "a host without GPUs must not reconcile at all") +} + +func TestStartVGPUReconcilerRunsPeriodically(t *testing.T) { + t.Parallel() + + var passes atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkVendorVFIO, nil, nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { + if passes.Add(1) == 1 { + return errors.New("transient device error") + } + return nil + }, + vgpuReconcileInterval: time.Millisecond, + } + + m.StartVGPUReconciler(t.Context()) + require.Eventually(t, func() bool { return passes.Load() >= 3 }, 5*time.Second, time.Millisecond, + "periodic passes must keep running after a failed pass") +} diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go new file mode 100644 index 000000000..1bcba2c42 --- /dev/null +++ b/lib/instances/vgpu_retention.go @@ -0,0 +1,98 @@ +package instances + +import ( + "context" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/logger" +) + +type vgpuRetention struct { + instanceID string + stub *StoredMetadata + retained bool + persisted bool +} + +func (r *vgpuRetention) retainFromCreateError(stub StoredMetadata, assignedAt time.Time, err error) { + device, ok := vgpuDevicePendingCleanup(err) + if !ok { + return + } + r.retainFromDevice(stub, device, assignedAt) +} + +func (r *vgpuRetention) retainFromDevice(stub StoredMetadata, device *devices.VGPUDevice, assignedAt time.Time) { + stub.GPUProfile = device.ProfileName + setStoredVGPUDevice(&stub, device, assignedAt) + r.stub = &stub + r.retained = true +} + +func (r *vgpuRetention) markRetained(persisted bool) { + r.retained = true + r.persisted = persisted +} + +func (r *vgpuRetention) wrapPending(err error) error { + if err == nil || !r.retained { + return err + } + return &VGPUCleanupPendingError{InstanceID: r.instanceID, Retained: r.persisted, Err: err} +} + +func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuRetention) { + if retention.stub == nil { + m.deleteInstanceData(retention.instanceID) + return + } + + id := retention.instanceID + retainedVGPU := retention.stub + log := logger.FromContext(ctx) + defer func() { + m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationCreate, retention.persisted) + }() + + // An unpersisted retention leaves no metadata claim. The periodic reconciler releases + // the VF after its grace period once no open VFIO handles remain. + if err := m.deleteInstanceData(id); err != nil { + log.ErrorContext(ctx, "failed to clean instance data before retaining vGPU assignment", "instance_id", id, "error", err) + return + } + if err := m.ensureDirectories(id); err != nil { + log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) + return + } + if err := m.saveVGPURetentionStub(retainedVGPU); err != nil { + log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) + return + } + retention.persisted = true +} + +func vgpuRetentionMetadata(source *StoredMetadata) *metadata { + return &metadata{StoredMetadata: StoredMetadata{ + Id: source.Id, + Name: source.Name, + Image: source.Image, + ResolvedImage: source.ResolvedImage, + Platform: source.Platform, + CreatedAt: source.CreatedAt, + HypervisorType: source.HypervisorType, + HypervisorVersion: source.HypervisorVersion, + SocketPath: source.SocketPath, + DataDir: source.DataDir, + GPUProfile: source.GPUProfile, + GPUFramework: source.GPUFramework, + GPUDevicePath: source.GPUDevicePath, + GPUMdevUUID: source.GPUMdevUUID, + GPUAssignedAt: source.GPUAssignedAt, + GPURetainedForCleanup: true, + }} +} + +func (m *manager) saveVGPURetentionStub(source *StoredMetadata) error { + return m.saveMetadata(vgpuRetentionMetadata(source)) +} diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 6f2c46819..07cdcb8e5 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -2,53 +2,523 @@ package instances import ( "context" + "errors" + "fmt" + "os" + "path/filepath" + "sync" "testing" + "time" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/network" + "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestStoredVGPUDevicePath(t *testing.T) { +func testVendorVFIODevice(profileName string) devices.VGPUDevice { + return devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + ProfileType: "1148", + ProfileName: profileName, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } +} + +func persistTestVGPURetention(m *manager, ctx context.Context, id string, stub *StoredMetadata) bool { + retention := vgpuRetention{instanceID: id, stub: stub, retained: stub != nil} + m.persistVGPURetention(ctx, &retention) + return retention.persisted +} + +func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { t.Parallel() - assert.Equal(t, "/sys/bus/mdev/devices/new-uuid", storedVGPUDevicePath(&StoredMetadata{ - GPUDevicePath: "/sys/bus/mdev/devices/new-uuid", - GPUMdevUUID: "legacy-uuid", - })) - assert.Equal(t, "/sys/bus/mdev/devices/legacy-uuid", storedVGPUDevicePath(&StoredMetadata{ - GPUMdevUUID: "legacy-uuid", - })) - assert.Empty(t, storedVGPUDevicePath(&StoredMetadata{})) + m := &manager{paths: paths.New(t.TempDir())} + const id = "failed-create" + stub := StoredMetadata{ + Id: id, + Name: id, + NetworkEnabled: true, + IP: "192.0.2.1", + Volumes: []VolumeAttachment{{VolumeID: "volume"}}, + HypervisorType: "qemu", + DataDir: m.paths.InstanceDir(id), + } + device := devices.VGPUDevice{ + ProfileName: "NVIDIA L40S-2Q", + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + assignedAt := time.Now().UTC() + retention := vgpuRetention{instanceID: id} + retention.retainFromCreateError(stub, assignedAt, &devices.VGPUCreateCleanupPendingError{Device: device, Err: errors.New("rollback failed")}) + + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, os.WriteFile(m.paths.InstanceOverlay(id), []byte("overlay"), 0o644)) + require.NoError(t, os.WriteFile(m.paths.InstanceConfigDisk(id), []byte("config"), 0o644)) + require.NoError(t, os.MkdirAll(m.paths.InstanceVolumeOverlaysDir(id), 0o755)) + require.NoError(t, os.WriteFile(m.paths.InstanceVolumeOverlay(id, "volume"), []byte("volume overlay"), 0o644)) + + m.persistVGPURetention(context.Background(), &retention) + assert.True(t, retention.persisted) + assert.NoFileExists(t, m.paths.InstanceOverlay(id)) + assert.NoFileExists(t, m.paths.InstanceConfigDisk(id)) + assert.NoDirExists(t, m.paths.InstanceVolumeOverlaysDir(id)) + + retained, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, id, retained.Id) + assert.Equal(t, device.Framework, retained.GPUFramework) + assert.Equal(t, device.SysfsPath, retained.GPUDevicePath) + assert.Equal(t, assignedAt, *retained.GPUAssignedAt) + assert.Equal(t, stub.Name, retained.Name) + assert.Equal(t, device.ProfileName, retained.GPUProfile) + assert.Equal(t, stub.HypervisorType, retained.HypervisorType) + assert.Equal(t, stub.DataDir, retained.DataDir) + assert.False(t, retained.NetworkEnabled) + assert.Empty(t, retained.IP) + assert.Empty(t, retained.Volumes) + assert.True(t, retained.GPURetainedForCleanup) } -func TestReleaseStoredVGPURetainsMetadataOnFailure(t *testing.T) { +func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { t.Parallel() + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("failed-create")) + + assert.False(t, persistTestVGPURetention(m, context.Background(), "failed-create", nil)) + _, err := m.loadMetadata("failed-create") + require.Error(t, err) +} + +func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m := &manager{paths: paths.New(t.TempDir())} + const id = "failed-create" + require.NoError(t, os.MkdirAll(m.paths.GuestsDir(), 0o755)) + require.NoError(t, os.Chmod(m.paths.GuestsDir(), 0o555)) + t.Cleanup(func() { _ = os.Chmod(m.paths.GuestsDir(), 0o755) }) + stored := &StoredMetadata{ - GPUFramework: devices.VGPUFramework("future-framework"), + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", } - err := releaseStoredVGPU(context.Background(), stored) - assert.Error(t, err) - assert.Equal(t, devices.VGPUFramework("future-framework"), stored.GPUFramework) - assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) + assert.False(t, persistTestVGPURetention(m, context.Background(), stored.Id, stored)) + _, err := m.loadMetadata(id) + require.Error(t, err, "the lost retention leaves no metadata claim, so the periodic sweep releases the VF") } -func TestSetAndClearStoredVGPUDevice(t *testing.T) { - t.Parallel() +type startRetentionNetworkManager struct { + network.Manager + config network.NetworkConfig + releaseCalls int +} + +func (m *startRetentionNetworkManager) CreateAllocation(context.Context, network.AllocateRequest) (*network.NetworkConfig, error) { + config := m.config + return &config, nil +} - stored := &StoredMetadata{} - setStoredVGPUDevice(stored, &devices.VGPUDevice{ - Framework: devices.VGPUFrameworkMdev, - SysfsPath: "/sys/bus/mdev/devices/new-uuid", - MdevUUID: "new-uuid", +func (m *startRetentionNetworkManager) ReleaseAllocation(context.Context, *network.Allocation) error { + m.releaseCalls++ + return nil +} + +func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) { + t.Helper() + m := &manager{ + paths: paths.New(t.TempDir()), + imageManager: readyFixtureImageManager{name: "test-image"}, + instanceLocks: sync.Map{}, + bootMarkerScans: sync.Map{}, + createVGPU: func(_ context.Context, profileName, _ string) (*devices.VGPUDevice, error) { + device := testVendorVFIODevice(profileName) + return &device, nil + }, + destroyVGPU: destroy, + } + const id = "start-rollback" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + Image: "test-image", + GPUProfile: "NVIDIA L40S-2Q", + HypervisorType: hypervisor.TypeQEMU, + SocketPath: m.paths.InstanceSocket(id, "noop.sock"), + DataDir: m.paths.InstanceDir(id), + }})) + return m, id +} + +func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return nil + }) + networkManager := &startRetentionNetworkManager{config: network.NetworkConfig{ + IP: "192.0.2.20", + MAC: "02:00:00:00:00:20", + TAPDevice: "tap-new", + }} + m.networkManager = networkManager + + previousProgramStart := time.Now().Add(-time.Hour).UTC() + previousExitCode := 23 + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.NetworkEnabled = true + meta.IP = "192.0.2.10" + meta.MAC = "02:00:00:00:00:10" + meta.Entrypoint = []string{"old-entrypoint"} + meta.Cmd = []string{"old-command"} + meta.ProgramStartedAt = &previousProgramStart + meta.ExitCode = &previousExitCode + meta.ExitMessage = "previous exit" + require.NoError(t, m.saveMetadata(meta)) + + device := testVendorVFIODevice("NVIDIA L40S-2Q") + cause := errors.New("create verification and rollback failed") + m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { + return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} + } + + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{ + Entrypoint: []string{"new-entrypoint"}, + Cmd: []string{"new-command"}, + }) + require.ErrorIs(t, err, cause) + var pending *VGPUCleanupPendingError + require.ErrorAs(t, err, &pending) + assert.Equal(t, id, pending.InstanceID) + assert.True(t, pending.Retained) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, device.Framework, stored.GPUFramework) + assert.Equal(t, device.SysfsPath, stored.GPUDevicePath) + assert.NotNil(t, stored.GPUAssignedAt) + assert.Equal(t, []string{"old-entrypoint"}, stored.Entrypoint) + assert.Equal(t, []string{"old-command"}, stored.Cmd) + assert.Equal(t, previousProgramStart, *stored.ProgramStartedAt) + assert.Equal(t, previousExitCode, *stored.ExitCode) + assert.Equal(t, "previous exit", stored.ExitMessage) + assert.Equal(t, "192.0.2.10", stored.IP) + assert.Equal(t, "02:00:00:00:00:10", stored.MAC) + assert.Equal(t, 1, networkManager.releaseCalls) +} + +func TestStartReportsUnretainedVGPUWhenRetentionSaveFails(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return nil + }) + device := testVendorVFIODevice("NVIDIA L40S-2Q") + cause := errors.New("create verification and rollback failed") + m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} + } + + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.ErrorIs(t, err, cause) + var pending *VGPUCleanupPendingError + require.ErrorAs(t, err, &pending) + assert.Equal(t, id, pending.InstanceID) + assert.False(t, pending.Retained) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "retention save failed, so no assignment should be recorded") +} + +func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { + var destroyed []devices.VGPUAssignment + m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil }) - assert.Equal(t, devices.VGPUFrameworkMdev, stored.GPUFramework) - assert.Equal(t, "/sys/bus/mdev/devices/new-uuid", stored.GPUDevicePath) - assert.Equal(t, "new-uuid", stored.GPUMdevUUID) - clearStoredVGPUDevice(stored) + t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.Error(t, err) + + require.Len(t, destroyed, 1) + assert.Equal(t, devices.VGPUAssignment{ + Framework: devices.VGPUFrameworkVendorVFIO, + DevicePath: "/sys/bus/pci/devices/0000:82:00.4", + InstanceID: id, + }, destroyed[0]) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile) assert.Empty(t, stored.GPUFramework) assert.Empty(t, stored.GPUDevicePath) assert.Empty(t, stored.GPUMdevUUID) } + +func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return errors.New("destroy failed") + }) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + stalePID := os.Getpid() + meta.HypervisorPID = &stalePID + meta.HypervisorStartTime = 1 + meta.HypervisorBootID = "previous-boot" + require.NoError(t, m.saveMetadata(meta)) + + t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{Entrypoint: []string{"new-entrypoint"}}) + require.Error(t, err) + var pending *VGPUCleanupPendingError + require.ErrorAs(t, err, &pending, "a retained rollback assignment must surface as vgpu_cleanup_pending") + assert.Equal(t, id, pending.InstanceID) + assert.True(t, pending.Retained) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) + assert.NotNil(t, stored.GPUAssignedAt) + assert.Nil(t, stored.HypervisorPID) + assert.Zero(t, stored.HypervisorStartTime) + assert.Empty(t, stored.HypervisorBootID) + assert.Empty(t, stored.Entrypoint) +} + +func TestCleanupStartVGPUReportsRetentionWhenRollbackSaveFails(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + tests := []struct { + name string + assignmentSaved bool + wantPersisted bool + }{ + {name: "assignment save survived", assignmentSaved: true, wantPersisted: true}, + {name: "assignment never saved", assignmentSaved: false, wantPersisted: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return errors.New("destroy failed") + }) + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + assignedAt := time.Now().UTC() + + meta, err := m.loadMetadata(id) + require.NoError(t, err) + rollbackMeta := *meta + if tt.assignmentSaved { + setStoredVGPUDevice(&meta.StoredMetadata, &device, assignedAt) + require.NoError(t, m.saveMetadata(meta)) + } + + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + retained, persisted := m.cleanupStartVGPU(context.Background(), id, &device, assignedAt, rollbackMeta) + assert.True(t, retained) + assert.Equal(t, tt.wantPersisted, persisted) + }) + } +} + +func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, + } + const id = "failed-start" + require.NoError(t, m.ensureDirectories(id)) + + previousStart := time.Now().Add(-time.Hour).UTC() + exitCode := 1 + rollbackMeta := metadata{StoredMetadata: StoredMetadata{ + Id: id, + Name: "original name", + GPUProfile: "NVIDIA L40S-2Q", + Entrypoint: []string{"old-entrypoint"}, + StartedAt: &previousStart, + ExitCode: &exitCode, + ExitMessage: "previous exit", + }} + partial := metadata{StoredMetadata: StoredMetadata{Id: id, Name: "partial start"}} + assignedAt := time.Now().UTC() + device := &devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + setStoredVGPUDevice(&partial.StoredMetadata, device, assignedAt) + require.NoError(t, m.saveMetadata(&partial)) + + m.cleanupStartVGPU(context.Background(), id, device, assignedAt, rollbackMeta) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, rollbackMeta.StoredMetadata, stored.StoredMetadata) +} + +func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("invalid-instance")) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid-instance"), []byte("{"), 0o644)) + + _, err := m.vgpuAssignmentClaimedByLiveInstance("other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.Error(t, err) +} + +func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("legacy-claimant")) + pid := os.Getpid() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "legacy-claimant", + Name: "legacy-claimant", + GPUMdevUUID: "legacy-uuid", + HypervisorProcessIdentity: HypervisorProcessIdentity{ + HypervisorPID: &pid, + HypervisorStartTime: processStartTime(pid), + HypervisorBootID: hostBootID(), + }, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance("other-instance", "/sys/bus/mdev/devices/legacy-uuid") + require.NoError(t, err) + assert.True(t, claimed) +} + +func TestVGPUAssignmentClaimedByLiveInstanceLiveness(t *testing.T) { + t.Parallel() + + const devicePath = "/sys/bus/pci/devices/0000:82:00.4" + deadPID := 1 << 30 + require.False(t, ProcessExists(deadPID)) + recent := time.Now().UTC() + stale := recent.Add(-devices.VGPUAssignmentGracePeriod - time.Minute) + tests := []struct { + name string + assignedAt *time.Time + pid *int + wantErr string + }{ + {name: "recent without PID", assignedAt: &recent, wantErr: "no persisted hypervisor PID"}, + {name: "stale without PID", assignedAt: &stale}, + {name: "recent dead PID", assignedAt: &recent, pid: &deadPID, wantErr: "recorded hypervisor is not running"}, + {name: "legacy dead PID", pid: &deadPID}, + } + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + m := &manager{paths: paths.New(t.TempDir())} + id := fmt.Sprintf("claimant-%d", i) + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + GPUAssignedAt: tt.assignedAt, + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: tt.pid}, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance("requester", devicePath) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.False(t, claimed) + }) + } +} + +func TestReleaseStoredVGPUSkipsClaimScanForMdev(t *testing.T) { + t.Parallel() + + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, + } + require.NoError(t, m.ensureDirectories("invalid-instance")) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid-instance"), []byte("{"), 0o644)) + + stored := &StoredMetadata{ + Id: "mdev-instance", + GPUFramework: devices.VGPUFrameworkMdev, + GPUMdevUUID: "uuid-1", + GPUDevicePath: "/sys/bus/mdev/devices/uuid-1", + } + require.NoError(t, m.releaseStoredVGPU(context.Background(), stored), + "an unreadable metadata file must not block mdev releases") + assert.Empty(t, stored.GPUDevicePath) +} + +func TestReleaseStoredVGPURetainsRequesterOnAmbiguousClaim(t *testing.T) { + t.Parallel() + + const devicePath = "/sys/bus/pci/devices/0000:82:00.4" + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + t.Fatal("destroyVGPU must not be called for an ambiguous claim") + return nil + }, + } + require.NoError(t, m.ensureDirectories("ambiguous-claimant")) + assignedAt := time.Now().UTC() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "ambiguous-claimant", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + GPUAssignedAt: &assignedAt, + }})) + + stored := &StoredMetadata{ + Id: "requester", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + } + err := m.releaseStoredVGPU(context.Background(), stored) + require.Error(t, err) + assert.Contains(t, err.Error(), "ambiguous-claimant") + assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) + assert.Equal(t, devicePath, stored.GPUDevicePath) +} + +func TestStoredVGPUDevicePath(t *testing.T) { + t.Parallel() + + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", storedVGPUDevicePath(&StoredMetadata{ + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUMdevUUID: "legacy-uuid", + })) + assert.Equal(t, "/sys/bus/mdev/devices/legacy-uuid", storedVGPUDevicePath(&StoredMetadata{ + GPUMdevUUID: "legacy-uuid", + })) + assert.Empty(t, storedVGPUDevicePath(&StoredMetadata{})) +} diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index eb43eb45c..0a12662cf 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1006,7 +1006,7 @@ type GPUConfig struct { // GPUProfile Available vGPU profile type GPUProfile struct { - // Available Number of instances that can be created with this profile + // Available Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer. Available int `json:"available"` // FramebufferMb Frame buffer size in MB @@ -1018,6 +1018,9 @@ type GPUProfile struct { // GPUResourceStatus GPU resource status. Null if no GPUs available. type GPUResourceStatus struct { + // AllocatableSlots Free slots eligible for placement, matching admission control (excludes quarantined VFs; 0 while VF health state is unavailable) + AllocatableSlots int `json:"allocatable_slots"` + // Devices Physical GPUs (only in passthrough mode) Devices *[]PassthroughDevice `json:"devices,omitempty"` @@ -1027,10 +1030,13 @@ type GPUResourceStatus struct { // Profiles Available vGPU profiles (only in vGPU mode) Profiles *[]GPUProfile `json:"profiles,omitempty"` + // QuarantinedSlots VFs quarantined after guest driver init failures (vGPU mode only). May overlap used_slots until the affected instance releases its VF. + QuarantinedSlots int `json:"quarantined_slots"` + // TotalSlots Total slots (VFs for vGPU, physical GPUs for passthrough) TotalSlots int `json:"total_slots"` - // UsedSlots Slots currently in use + // UsedSlots Slots currently in use. Includes quarantined VFs that are still assigned, so this can overlap quarantined_slots. UsedSlots int `json:"used_slots"` } @@ -1354,7 +1360,10 @@ type InstanceHypervisor string // InstanceGPU GPU information attached to the instance type InstanceGPU struct { - // MdevUuid mdev device UUID + // DevicePath sysfs path of the assigned vGPU device + DevicePath *string `json:"device_path,omitempty"` + + // MdevUuid mdev device UUID (mdev hosts only) MdevUuid *string `json:"mdev_uuid,omitempty"` // Profile vGPU profile name @@ -19026,251 +19035,255 @@ var swaggerSpec = []string{ "JLVWYtGUYuKRzuH11nlS+sO1hpQW687I1X0sOmT69TTZ9iTD2d2s+KqI4cLW4OXpcxtn30dwuiDIzyXo", "107auYOasEveHzKbaOThFYAjDbAETJj4jFCBuKBTWu24ame9x5jdm1CmI65bU6f/YVMMnizcgCEeYJCW", "JyYE1hQA8YFe7Z50up3zomiUZUzVpXlTFN5aWpEysn4Zc+fs3U0jbzPBJzQEhAORQPapVdRcTOqrvcF5", - "b/uvJmpekx9IbJSZ6KGUx7UaT/b9dhfRy7N3Z01jKqouIX90S3NaAe9TRrsUKK7G1Wh91VahcadB3zNF", - "J6Uo/jwk2gKKxzifTIgYpQFb20/6OTIvFCgEpz9WxVstRrdVos8qmwNa9ARHlE03W69+wD5Xm8Y6mL+X", - "Z+/e2Co+TQDIequKSj8G+7iPfi3qXKGXZ+9kCXLUDxju6kB6DXldZ7OFpBFOTIumSAVlvr0NiLO1wHxW", - "fmgtkwGxOVwezR0EtDGfZjkcw/M3vZPX77fSmMy7lTFBXN2MJ0SPe9PjFnMHLlAmoVWYxLzJ8GEIQ7Y9", - "QN5aFSe49SJ55zWwOoornIxkwkMhPG/1QwQP0cb7n0x2sB5BF2WVrdS/+8g/Pn0/CZ4YAJRv6PYcOqxb", - "UCsHPKhK1hHJOtXpVToNHRWT2bQs8izXJuSX1Y3ml+vr4ZlGmvs9cslXNRu3lSuRydEy8IkuKgGZT40R", - "21pKJMmwwIoki1p4ZBXfnCw7mMk1iW6Q/fVCv/7J1LTIBRmpmSByxpNqWMJud7kuqoQQ4DmxpaDMnDw7", - "vOIoxeISLkYnV6OcmRWoRpDvroN7mSmV3WBSP799e2aUbUXEHCf1HAS55HA/JgleoDFRV4QwNxUsEfZD", - "UOs5nLKhHI9Qo4wIyqtr2NkN9HtuwpLRVOCIIPOVq6pst0TCrdl2KW0vAQDBKCJSNuzv9qr9tZ9O8qTd", - "HoeGtb22CHl0kw1+e3TmysoUZXvdMu8sr/IZET1z5Fz93tVbuyNXFzhyXTGDC7skMIwJVDyyiR9+XqYL", - "N4KKTvrzSi6kxxykn71n+4FTYJaqa875h1bCXv24h/zqKWZxqPyxifc3mfJTgM+CoGORgyuJxiYWySTv", - "G7HcT1sQBMeUESlracZRLpJOt9Ob2FkdbG0lPMIJIAju7W4/21od1bkynNdGL41iukrdczFOJgrGJbUa", - "+DWYdJUktnCWtTCImXVccz8Ae1oOH4SSx/pu8yQ8F08+GCxlTF/jSLkibWAhq3hAsX9sAee3Mh9oMDV1", - "V7So/dw/n4OgSzrDalYl/60l2ocwFQgx1DRirA+1dTRE/jEoUXERqsTFhbIJeGPiYhCL+9BF+DlE24qv", - "bPDMn+WT/f3d/XV8CJhN7ZjbcxeYqnm7mmJMvOW259c2ANFSVZnDHenVxUT1uqyhKc0Rl0hq9YLyjLAb", - "ref+3u7Ozdaz7UROXJRWjS+FEFqOTo+NTBRxpjBlRKCUKBxjhatMBkxLmstAqRdMUsjSmXy/mrU0hDP4", - "kCu3rVP1pZzhDSXr3jiE5hQzOtEM2b7p9yxneGf/yYEprBmTyd7+k36/f1Mgihcl8kSrrdgyMXMeJkVf", - "zj5vH+4Ab6LNXP7onB2+/VkzslwKc2ltyTFlB96/i3+WD+AP888xZWGcija1WOlkqQZrNTwst0DAJD5A", - "ZbltJ/e0qtcftg1DsDCA4wRR3ypBk3cH71bQOK2i+N8gVXdF6qoWV16zZNG4Nrcuq1oWHFdeOVU/+atF", - "aVX6cbW725m74B3bp8GDLqrOLju6b1U3WK4srbhUeSsjrCimmCTmr4gzwNkNVVasXJHuWYvCXHCN2Apc", - "RZf+j0Xv3o9H/kC8311hL+8nW2Lxww0jVFYKpH9blkPXcyEnjq45zGHbY3ErtC1na2HhgqHZD3wX3iaK", - "q9r76+l//f5/5NnTv2///ur9+/+ev/yv41/pf79Pzl5/FuTIakTAB4X1+2JIfqZEuw/n15aUTrGKAjY6", - "rf41rLB9YiwOKppBAU40JgdD1kOvqCLClHOr5SIOO2iDgKYEX2lxF2rVmDSwTf3xmXEw6o//cGLwp3ob", - "sc0RF3ZDCugPmY9jnmLKNodsyGxbyE1Egl6g/4pRhDNTk40ypPXfBRoLKKBnPT5l5130B86yT5tDZovm", - "G3DrDEMNskmRhMWcv9aOykSl2tdJgQJhnC5DVtzWBSSecfv1S/x8SpJ6Ck/DoqzW36zm9GwQAg+E9BK9", - "kVAjBlSQgrI1GRV5L+jZYHNZn1ujYxQ0tIL8rF/c5B8e5iFzcVPO4jGJaQR8xaXtzWxiZ5ExaSjNGvEy", - "wa8XsDdvTC5ZjHCuZpoXRTbPPeL8kpIubGkXTQQUeOiaL417fcaz3njRm/GswDzAwgSfYOOgrirZ/6dn", - "J9p7TwSd2J6CqeuaRAJCJxwZOzOTAVhYF5Ym9taU12Ba9JkT+7qp0yJNaQ4TWa1ywVz1CAI1JAFco6A+", - "EpLJv0dRQsHqJGc8T2I0A/w7pZsJQdh1BkUGCx5HMZnU/12NMNjZfwIarPv37k7rBFKzdKuoLE8COm3q", - "WF8Ljm3YJAzAiAcjZwhfExOkb0BbyArsFIrDf8+Ra6g8cQUjMd4pk9ImbTGCRHrJbJvBrCN7DCyCxwjb", - "09TmPlo6hZUkphYtmGAB+CxpgR3ywiRPvn11jhQRqUtn34j07sApMUARPSplbutiHR6dvtjsd4K4RxVX", - "FWzVyiSn6qAD0Ac2eKApJqK00eCUdNHJMSSv2mul1MUg2+AnLlBibsXyMjoA7IyquQebinonx1YATRZl", - "BIIRW4adTddiVr/eDtCbQgXExVCKNMSStlyT5WUCzdp4NJMKsdR6LWsV/GNW/bP3MSQ+QLlBw4sBu7Hx", - "/mpvc3RIUPqiqlnIbnwh+UEhjfYvb++/NMrxl5fRd28mo1sf8CibYRmi7pnv1YSXlvbddyNX2b1oDuGp", - "9DuSNHi2/uaKrnjXkCL6nqt8HoL13O9tb7/d3ru5+e6mALVVZCoPta7AqG0PLnsXIK0ByFWqRo2x3kg/", - "tpHdzi7y/hTNsGTfKXhYs45s7z5tY5SAXttGSfvx0XxihlRwKQdzVUT3GsCvS5okRoCRdMpwgp6jjfOT", - "l7+cvHq1iXro9evT+las+iK4P7fAqoVbANbRJHwFkI4qGfqoSOV7+/YVHK6EQDaEkcMvb49gu9a02ALR", - "1g3u5dk7cPxjOXJxlM2pg7hMvyXXVCq5DHLWKhz5cxB0zaftau67SZo2ytL7q2F4f67gxAZR6zbvAD/X", - "xZIvLecDQMs+ZM7e1wdruxKI9nPRZK2d4Y7AZBuvtBAQaw2gYL/pdrs9LOydDKeC7xJiW76E4xKqb43D", - "2u3QQDLpodQXD4nRyVlZeKl0Rrjma3N6vtPffvIMaoduD9ow9hRHK/o+PTxq3/lgx9wyB3h8EMUHoLDf", - "1mdlCduoIDi5wguovWeWdtgxF6an3XrH1iqSreJrluFub4duWxfjGvBrQZx1gUtylK4s/dEiW7COYZbm", - "FqExpUlCJYk4i2VVRp5hiWRmgElNCYxCgh8yGGAXFZWIQUpBOIpEXpoerXRt5f08s3QPZTgzzrQOADj8", - "v5CFRCkFJ2jRPYQ+SlQkpcRDtiFcAlORqQQVOGP9A6QDdF3YuR4aVVDqQ38wZHKWK83ENvvoiDOZp0RY", - "qywaU/AYbSKZG5UWxgursdAMU9KYiCHTrwWgT/8o1JODJ4PBYNDtFJrcrv73IERNd+r87FtoX5OCC+B7", - "zIL8AqqfyBnKWUxEUY6bGHKoh8jd0HH6mZi+7vN24pX9vJSrwgdzHQRwO2zfzwVUhaE26OcQHXoL5Xz/", - "9iJ6qxQgJ7/a5B/71egmEQwERTxPYq3xjfVtZwxyJLZmSEmU4c7mXSrRO1MKszp1G3qsOPo9J2KB3p+e", - "VsIeBJloHtBu4sAlGvaBZzfahp01NpK1o7mJe9mDn70PyNm6pOJJiF8cYNb3MLqcZEOhFcNWRXFeZV/T", - "WmUwi4Qys0+aaFZMsGbKj8l8lOchrUQ/crAk796dHFcoBeMn288Gz573no23n/T24sF2D2/vPunt7OPB", - "ZDd6uru9s7si46RFFtntE8OCqmkgYLgIDx+5MPVQ9HBTkkBNCLCBz1eUxfyqcs8EI1H93m2U67rul2PY", - "Ww8hmPmSYKmMnaCBZZzCbUoi3baJ/LaZikWJobBF8cnbwfbnmllgcA3M+K3ImfFfmtz6wlafegP2N6s6", - "ztvxVhiQyzBZt1p+5+0XbXCw//xg/3MXzWVJrBtjnZzucXObQq8c9nAtDcOlAnoGG2cJ7Fjhw5jPbdZG", - "p9spEkvgb7h1a0HLxeNW2VJNB7YbZiOr+HdD0vBJRTGAkBCDPRcfaJGgyDcd5woVqela1jhKeB4jz+hl", - "oLjA4XXiKQm6GfA/WVuYgdY0WQ9amQAsZ6iYQJlmxODo043YDOMD9BLehUc4NfqTHYSp2+H7uHC8MIEp", - "+ny5ro02s3rI51aRgW+0VoP0v2DaehmsbXR1E0YMOkC/cvimUKsYrxtZzeugzyy/XjfIbtjUX4cYAZ1Z", - "me4A/VTIcYUkaCW/DUnsnyPLsEqgls1Kurzd8Y6mlnLnvNTvbsesaKfbcQsFKeLLyeLvSqpfOn8+KYYi", - "tghO4CyXybi5oomFp4aZUKloJG2Wht7cJvnClhQi8choKU3BnybD02oyxUdOfHl/ijYAgfAvyFqQ9b82", - "i0DRyl2383zv+ZOnO8+ftMIZKge4XgY9gvzj5cGtFUijLB9ZI0TT1I/O3hkjQ2TU9yLI5P2pD+uQCa5Z", - "j565a9Dv/Hn/uQ+vFPN8nHgePIvFZtBcYcOCCGIFL2oIOPydJnM6mbDfP0aXO38XNN2+fiJ3xtsNsLGm", - "o7B968T34i8Zg8m4Z8oDhRFwgKCEbASJekMkzACdE4WAfnoIR6BHFGnDluQclJRd8SBh7e3u7j57ur/T", - "iq7s6LyDMwJrV+BStiPwjhi8iTbenJ+jLY/gTJsOSwGQvZnVMcPnDNnavoOqQNrfHuyGqKTh4i6pxrY9", - "TxuX/L3V0+yk7KJD9nOhwy2d8uBq7+4Onu7tP9tvd4ytHXYkrldzGJcbZJbHAtD7O78B0uTbwzMEmbcT", - "HFWNKC4U60ajUjcaFRRPMKDnNxjYs6dP9vd2d7bboZ2Fojssjl/lwFZ5V+DQBYgisBuBpVhmvd2m2yIk", - "ThkCe0OiBNP0MHK5DLXbx4Cbj4R5rdyENheDNf0vXVwtvm1lRSpsQyYTxogGXKCcFSU1+ut9n1/EhdnM", - "tc31sJ6rh/JfmF49C8tjSofdYikzQeaU5/ILNMSVSU6dJJyLG33bpLC8ITJPlPEzUonen34HPEXTGpKK", - "ZFUdylLjCvCiW07uRue5QiJhIm9arFa70WbrV02423Bqu6uQKyrcoBEyLNacK2froyyPcBLlUEQGF/up", - "ZwVoP5B7n2XJwgTRJwnnDEUzzMAbIWzJKjZFGM14EveDIaf6yWgSDF/gVyjhBuz4kpDM1lcxg9CfaRGG", - "zgna8CuLGVKq1fvcTw2TsRU0qtS4n4YLF2IZygorcs71emLFPRxe80nF5JjwqQSlUEF6QL8O/55hYaL+", - "MTP1guap0SUDkc2BIdaYeehGNTcpn1gF14ockNFtVhJHgkuJSEKnUJvm/WktUXhFclmRLrw+crI62Bak", - "azyHgasMrjjZuqxY6H4MJM58zg0JNAzJeStiEp1xMsUsh4orHiFbi3e/ddzhjEs1KgCgbjhYqUZQRiEX", - "pESJK9LbC3uQeyd4LzrWdpvlsgG+t/p6iarCTTUNsJmnBlc0vFrdggZDZLwMgbUSdauE8apjNt0EpK3E", - "3acSWqUePhjagOQSjy152PGbbaJRwiqr7mdJW7VVMV/tDc7b4qethks7w2p2wiY8ALJxAxels0TbsNCM", - "iJRCIREUE0ZJ7HTJwldpTV2QmZ1IguKc2JUz8qnAdsGxOd4AlMGcjYyyaY3X1ztsYx42Y1hdZQH6tS+2", - "iSuS4czVtyKHtTKBgRLhMoe1VbQllaOwO2u5YUGmeYIFqmMErhiyXKQJZZdtWpeLdMwTGiH9Qd0BPeFJ", - "wq9G+pH8Aeay2Wp2+oNRU0meczM4m4BnNqTWbzmFH/QsN2vpv2CJ2TLfbwE0S5swrWBI9k80IRZG7x2j", - "1x6hV2HI93YGTWnpDY1WEtKXIRhvyrktyQZPfC4DSXwrpRxXTIjEFhveiD1ZLk1dkxa3kgMfdS7A23l0", - "qhkan4cBcmT4dQ0BBI0JJNi4qS1zjRZssc1UgiUVcjlDf+fjqkG0bXxtoFDXBiuxKASZBAPpYUdXGqTN", - "G0tr4u3uTcAegK3qicJHN8RQWFfSrAxkauInb5aqe82IXTLq5mgqfbWoXOECLQqcANtre8CAeh22QGAw", - "wMFItYCKplBGZuEVF5RozIUA5GUt4XDmZgP4Jlrm0WvtAKbQ2xlZIEFSTNmQUVYYSQG1jCBG5kR46ahc", - "aCVrSuI++pun4gFWdZqphQVBB+P5dxLxK1aMccj8QerGc6nbOWTGsihyKJJfvqSbBa1PEwqkB4MTTAnq", - "8G8ngsiZP/dQrUgt411xETcW4Vkg9wrUdgEfK1L8kjCflRXNBFVD09DIfLUcLmcKvcJTq3+iSu1VVK+t", - "urq/XBIRFhKLKRWvtApd8Y6Kp5wYtBWAHoGaevYvw+ILuJEW4CJl8391TZY/nRWNV3+rveYBiDg830Nj", - "tg2aYCOTL1ML9ql60taGqkC+2SrYmGVfAtpwscquMElVEvAKhLS6J9ulvNWj8t1otiSJqr3vPdt/+qRl", - "hZbPctYZmKwv7Zqbpytccg07ddrG7/Ns/9nz57t7+893buRhcQkcDfvTlMTh7w/aINdKH9bkX//45/vT", - "mtdnH4KdBzcalEnhCA+pIY2jOqD3p//6xz/dqG49oBCjWYbibvDbN0bpJP5OukCBqguvnZNshX5/WDES", - "4ILNoA0ymRAwg47MuvXKwdTwNtpJwTjDEVWLACPHVyasvHilBindxh1UHWxI5DVtW/hRzblkPi6zOzdc", - "5+g/jW+4RgvPWhd6kvm4yQ/9ut6r8UKXXgs/xqFFiIEsaowvG7iL+VxhWYmc1n9HkODgUrmW01rMG6vh", - "bes5BxDFYuuZeaGAIVj0mjxpP/K3v7adnt+yYtapr/iHFeew+QjeyOobuJEDRt9ofQ5rjT/YC/B2X43G", - "fgm2lTXuKvXaylv35v22SNNdLkhQ3GA378/LTLzJh3XwXaBHOwa75GXb3QpJNFCTl3QSMKDxhPSKQD2b", - "kYJkbjyC+sxbPPdAqmR0ySeTKqjsfjMIOeDrQFaV6wUrpTWTLiLXzmZRR7A2YDrDzr4cdrQKMOxsp8NO", - "zW0VzFNM8fXIdlAFURmsQgUv88xrg5RuBuOER5emuhgUre6jAUoJZhLlDA5/zau2PVjtHep2Mm9vCgxu", - "YkKcltgWjGlMZnhOofSD9alMK4GY5JoqCQGj0M4BirmBVaqUVrUz1K+ZLMKDctJw6WC2sA3rBvV7nLmI", - "1vJdMPBNoKAr+0gE71pUAM2xX78+7ZoABgg9NAOrxDe6iZoRaAZZdFGrY1D+Ho4fHidkBOOu4+Kny+vo", - "J3+DZ1UQSZS0QNklOdSIAEU8Z6oOmJ+2U+Sq+VvLV1LOINjPhn8AAJrt3dblj0kEJ1Iun8Uqod+CuGt5", - "A3alQ4kDuyEShkMBvqSwr/iNdQjXB2CMDV5VZNOOH9dtvIQjqbgto1Wc6hG5jgiJ68ia4VfaxsrbL4Ox", - "8q+wBeMpChbbtyHeeXl2/bvLpIKxNq22H9PPOOsBDIjbUgvZYTD4LChMldAqGN8edsQohGMaeqFNajO5", - "Xr3Wv5JrBUDkcZ4YdLkw6VpWZS+jdSt+6xTCpgPNxfr6/ndQrs3Em9+qYJsNVX+Imm32rTup07a0O+dE", - "uXfPLRk1V+uvVFSpuLRcwL97pRpjY0ipi+wFj7bTzRoJ7s3CVhGLftsyGZLhlIwyQSb0egXxmBeMYlzF", - "DykPUpHBYIA8N1J8jfaeomiGhayNndHpTCWLagDOXgC06LOKGQqiCHOGwjY7X+6m+3A52s1up996SDg+", - "9zB4lmqHWJF0tAqg+qj0tlnrfIYXYMVpdBI+3d0bDHZ3BrdCqHbDusFyHZWf2NJ/1XaaUuq876yjvxKl", - "6rdQZDMv15O9EhSSootlkkoQnB5A4k2GI4ISMgE0uqKO93rPYr3r1YO3ApVFfino322U3Tfng6/Wpim6", - "suDebhod51ysgv34z9c4RBvYTLSEXRfIudvtDZ683d492H9ysL19F6jSxSI1ZXs8/bh99TTZwZO95Nni", - "6e/bs6fTnXQ3qIddUlOCpw2t/qLfbYyyKS/JKmhQhaWhDTuHjIh6oeB6gW1JEspITxYZUuvTFFfwAuN/", - "X3v+b2bnNzNYKTucVyfpixBYlYtToayHAbqyk1npu6jP5uR49SxulYFUH0iY3upDAfJqNxgoBbHd+UwI", - "hJy1vIbeeS+2vohWZsWtu4pCHnY46cFdbljxEHnXEBC8Wa+6wJcvuYDtdMoFVbN09W1RvFbgdUPc9Eep", - "4iqwUh+dTBlUCfd/LsLkfCVKf9zpdpKPe9UzY39vD7FloX4LArRb7UsFLcLIoAj96lWAV0rFQ5hIdq2r", - "6zH/sN3bfg5xCMnHvR8GvefViIOuWS1/+bbd25VfB23W0K+152o0bT+/UcS1W89VFPQLDVWKK+9lCwJs", - "abysyeyuDpdwW9ng8vHSHtcgcxoF0M+V9OzlNvKFppgkeBECgfcMtbKmPfpEhsZkSplsY7fdHRSG2/10", - "2OmjQ4vEDbpsWYG/0jzUXvfohKYpiamWMY3q35zBsNPSFlfXJW5WBMR9FZDW+mFx7fl6iIR1CVfrrsn+", - "Z+Tjfpb2207jXYXeAXY1p6ICWBe82EV0gjCrVQKlbI4TGttEekiMhHi1A4eIVpKs5QGylAOdnaSLplyh", - "MoW+pb0tZ812wWL85BrsrSswMwxB7HwRQJQCqYuuYl8nxygTPM6jMn80gUGXiB8ir2GhrRDy14fk3qV9", - "AxKzJ1yg9faNJoNGO/tk037XbJOaYJu3enuwfqvvxCjS7eRZvJ6HmZfacbAbQaSvSUEMmGiqy16TBL3J", - "fGjB0d/4K7is8xpbcqRFojxzDhZNU8uUFHC3gIshFNd7TBKir6nlRhBP4jJLgsqSi65nqdtPns2aXJzg", - "kVoeyC+EZFpXAfwj6C/FbBEcmKvvWdwlGwMHqy2Nw6tn6gLZ1aoO7ulaSaxxq3wTblOtAsPlazZvg5dy", - "6Zm/CzBtXzRbRkBxDL8ipL1pxtq3X7qwt0b78V2Y5R5SSHttXQ81fFSH3lvAkLv+y1hgLdZViXcv5J4P", - "kcVbqxk3Yb7Ws0B9q/Nh73+MlRmN+gdbP/zl/+59+M+gtbmmN0siejGZQKDRJVn0TJUfraP3q4inUGJA", - "C9NTSyoEp2BDAjRxexj98e4PCqax+BWnS1OACC2vRM/22gn95T+a45u8ZXwHfHItyX52BY67qFSquLuO", - "NlIipi6W3CWSbfaHDHLTLslCIq/wlxVpHKF+J4tPvAh0dGHEwD5h8ws0plBJUQ6Z1mpxFJFMaxO2lgw1", - "5cA5cB9BcOK3YwuQucRv65A08QQEvT9dgst9/e7tj6/f/Xo8en324tfDk9EvL/4bQjyueqaHuKdpb2//", - "iS0C7q/kdrAQxc3rKfTRqQ3Tt67+SQ4KLeB0SZTmKoegEHIdJbmkc+cgVMntKycsJ+vevhLBZ0LtKpWE", - "ohIsJHRCJwT8+nCd2KAaKh0xUgnV061xgzK0fGMbwhl2gJN6xe9DdSv0VoRXu9zY6qI/mbVjoQaENHDY", - "IeMVytwHtBcqAa/CxX54L6MNyBxxJV5d4uzmzUBRD4sGg5GHX7iSz+D5l6i2+W5lec05T3pavWkoSRC0", - "Jpu1CEbOQ1MmI6HT5HSYjgMyvDXtTukUB/wMIX/CF6mK6Qa0NmNqaf8by4OF8xiO6/UazLE0S1WrL1Az", - "EkjVa05zSLVUOypL/1eDZ3Jmc1epF1tXTVRNmdqy1WtDeBkxB9TwVdnK5Slz6Ig9+Gh9Eu5KvcqbmTeS", - "5r05depDTcFZsUBnemmuZkQQbyPggxIH/4ZLZvNyWqCwmOp/GRFlzKpL6tFSKbibJdooLD9uCYps42Vz", - "+Oo6B6f4uugBXClYLvkfYR5lnaXtlz8CJv0bV1uSTlwTMIyachdGYK9S0ao1cVS1vBk+VS3P27wfPHiW", - "V63gfk1nq0acZR8V0gzR498wVT9xAepgM+bJnQO5w+UfEwEYcHWY9lYY5zQl8YjnavX5t6Xr7ZVf1B8t", - "69c61RcDEUeVdN4mXuBQOcoxLK+0Xg4S5YKqxbleLxvMDWmQrmgsLCR0BD+XHUOhzk+fwGg8CSSMvCSM", - "CBpBGVR9HlPMQGNC70+9animMOISXiuIQK+PTqy5wUH+gvpIFZCei7s8PDvpdDtzIozK3Rn0d/sDOMwZ", - "YTijnYPObn+7P+iAVjWDKW5B6XqbP23zjQvF9SS2ktCP7iX9pcApUfDFbwEkAIg7tK+DCoKnnhKZYSqs", - "FpklgFBgCIbqrwHX312oB+ZW7pplb20zhTRjyH4h2Wu7uR9AUIazA9PcGQwssLmy1y/k7piEga2/2+jR", - "st9WUp1dogDM/ZKa52TLYuk/dTt7g+0bjWnVUODshjp+x7BN4iWgne/fcCFu1ekJM2l5NsnahkP5Jw4I", - "yT9rv33QeybzNMVi4RbMX62MyybBmEiE3btGj1MSRZpVQDGePnrNiHmOsELYRC6LnEENY/ehptDqKTBt", - "u00uQIp+5PHiiy1hpQ9no/hUZWf6uHxaoucvRzsFGS9vpH3kELYN1d4DAf2IiwLcD3ZS9gbP777TI84m", - "CY0U6hUEbOORqYSQnwTwwh32EBfo95wrjIpw/kd0pK3MOi7IrVteRVt/0PiTOd4JCZnBz4hIMTPJEead", - "NYd+6Tgbl0R5nFfeao7wobQH3FQOhMdcVCDIVY+of23VhcHl62gvgMBg+zTTix+Q8Pfu4YTbyRY1WB/y", - "yEHlS5RL8piOk3WxjUshJCjLvSTqa6H5wX1eWbaIwJ/wFD0WAn5JCgmv3K2lS2ErEzkzCnBQAnxTJiza", - "776rCn9vyydelAz4NXTTUM5CGb8qjhd95NbUKP1qARBLgsA84+Vr5UwP72s5YTv3ccJgxoWn6Ns19e2a", - "WnXKDbW4KcDB9E55CxvEjSwQfz77w42tD99sD+1tD60sD4xcWevC3/m4j2xEasRjguSM50mMxgQZvCMX", - "e6Kw6E8/IiyiGZ0TALWDIm15omiGBUSWpCjGChsfeqNhYqVZomhuSzfXc3GI5QLXcSwkGQEO36gJf7KM", - "QKSMkRjpTyx0XwknuFS325z9oIG9aLC8GtHVjEtS4Pkx5d3mkN4sjXYMzfaH7K0FetULCMHUjtdIkgBc", - "7Qr7D2cID5n94HvHQlwgmMRpybmwAMxAapApzbYsp7bpkY5kxENYO28Jw0z1ZEYiOqGRndYlWdh4zmCD", - "reou6QG7cb4/LRI20M5mGK8N4BnD4LzHxTNkKanqv2EQBB0leVw6uRyEEBZjnCTBwhzThI9xMjLrc0kC", - "PsGX8IZdFL++v/MmMR4TU6s9W6gZZ+bvfJwzlZu/x4JfSSKGnc3+kEEihl1rEndLARFdQSG3NOP6nAme", - "mj63zBC3/rgki0/9ITuMU8ocRcAnOJEckWv4DupbAWaG4V4N9GBOU9gPfpRLxVMf+dTRnRkmz1WWK5tR", - "IonqhlA/h0xx9IfDdvy09UfZ4ydwFhMcazrxXjFTAtm6adRyhPXsR/BqwN1OYAGGHX2RmjCPqcBMGdjO", - "ApwSTf0t3SiqI+hDullf4QgzlPHMVJYAopphTXKVNgCrAScJUnCU3LdacIedbJiPhd5Lx424ewYorXaM", - "KEOnP3qHabD3LHyeJIkECUWU/Nf5618R3Mp6D8xrZbiWSelgWmBAcQ6uU8fTXuBohoyjCooJDjs0HnYK", - "d268CWPNpQ2X6fXAp/iDHtoPppsujX/o93VTxl15gH77w7RyoM9Slhoc0GHnUxd5D6ZUzfJx8exDeEGb", - "4MvOK4wAbZhrbhM4CaaANOPd+OaKxCxG3N4CyQJhVHIgP3BlTBkWi1WJhIGltyvIJyaS0VuMP4YQuTjs", - "HAxd7OKw0x12CJvDbzbAcdj5FF4B67VsrlwH91nh3CyI6MlgsLkeCduub8Bn2cIx8IV1wEatqCi7qXfQ", - "wrD+ufwD/9b6Z+H6wUx3XkITGcXfGd8foQPCk9h9TTTggqiJ3ZhFJHFi93pDz/07D/RmRSRJ7ptAH4o8", - "C/dYgdT/qMgRNqs8RivN9w9McYP7ulQqZvuHod9HZz8PWM+t7ZzMXahzuE4JYNBYVRqZlxGW6BzG1DvX", - "yvcL+LVv/+t0P8BUvEj49OLAqO4o4VOUUGbzAbxAZS0e2LWEjwwMTfGdRaVxReI2jCTxr3/8EwZF2fRf", - "//inxXb/1z/+Ccd9y8CrQY3pixnBQo0JVhcH6BdCsh5O6Jy4yUAVWDInYoF2B9bmD4+QV+reSmlyyIbs", - "DVG5YF7ehKnXJm2D1lWg50NZTqSF8dEv0oktJmNiGwN2G3eWzVLe64nuBuAQYQbeBPSt6GgAsOSoKbRt", - "NdFO2GRq5lwxmtbDNJeC9dbzF0WulaHenhngDRkMLHHo3MEDO2m0cX7+YrOPQNsyVAEFg0B3KJuxakT/", - "G09az5MMR6kyFFhlw5sinOExTagzOTZUOzFHMMXRjDJSxhcXWOOuiQM3Us1jDs9OkA2E7MKrQ/b6fAtM", - "rIpEKhekazmBsAijZTk0bvNcoAfgX1RBdFjPvjtkE4IhT+jk2DABD4S7yAcsGmYA5AExrlRVKq91h8wg", - "yVrkYn3wUh6TBD6C/qdYkSu86KKi1q2rjpJgpRVi2dUvD5nBerVr0AOoEuQNsw/8zAyp5yJ5bc6WIJNE", - "q8YQgW/KfkPfGxMukI1w9qr8u+5MkqUZll60FEevz/X8pqAJcmMPhJZen7vd2OwiyVGUUKCGCLMhm0Ig", - "kAPv5ayyq0VC2QyLuBdxfQn4YE6XjF8lJJ428dgjn8juUJKp9BM4Tj/XyfWxCRez5QnoQ2wA6lZ77o7t", - "O+1cd7bFP5PvzhaCvIHzzlhwieE3ZnW/OfJaOPLC6+aceiHP2rFDYLy7iF/TxQMF/DraW15z88Rbsoew", - "6KENB20DXhEu0NnRCcJxLIiUm//e9j49U0Olpfyn70fNih8i9MSOhQsL+mftLVUCeSzs4I0dNcJuXvX6", - "uv79tlUpvtN40xV1eMor7+5vj1qnN7lGSqG3pLVvN8naYFsqIw5lBktq6YFolJBCfCnOqU9F66zKJoy3", - "uHJWikuWPZ8cuwN5f/Zl23XO6nfDPTDF4xpDfEBGWE219qtmPyZqflfsokObXmF+/rpIc3B/UtB9m6JD", - "ZP6Y1MW4tmyaCxqgk8YL9CVRBt7kLvV020Ng4udEuFNtBrowsy6mZT5FBqcFJgSWmNW674l5pZ3qa9r7", - "M2m+sDw3kVjskn8TUVoou+VarVJwT2wJ6LvTb6GHG6m3Xy5sxRJYYJHBijp2biewrG5guWDR5rfIlS9O", - "0SausVRihZs3iQtLtkFTKvSs+5LrDplfb1zLdFavpQxNEjqdWSdATCcQq6f8+t0wyp17GGVRJ1tgRWyI", - "4mPM+z3Ti2y9wHMiFHp9dGLW379St/6AoNX1qpJjXitv13dvXvUIi3hcOE+aZVL75AsrTIb+K7m893/q", - "HmE+K3XiQZPA+Bn7b4LJkYl/71P+v3Z+SuhYYLH4Xzs/4SSjjPyv3cMEKyLV5p0Ry+C+brr7VmAeMfFp", - "/YVWFw1YE5sCZOwagb94q6XM797/U4n9ZtI3EvyLdf0m+7eR/f3lWin+2624UwXA9PFAHq6C2EKrDY++", - "Qdrcg9HUUqQHaVPxIpWgNjMuFTx6fPnNNqicFhTnXxstrf/lgVx5fTjSPTnuwkJCRWmoaGHTB+/JF+DG", - "ce/Cre33/h0Bh+mYTnOeSz8zMcUqmhFps3YTUmXAj03sLq/nRsH7K6bSwX1eHfcuV3+j+zuS+Osbapi3", - "ceitk/ndW21lfvu+lvkNoqnNbLZlN7quJNNmQ6C1wzRtS8YV6NflAPDQuEK6CHqnFZVSXUCgQRwM2f/W", - "+sdviuD0ww8uhTIfDHaewO+EzT/84LIo2akjFcKUoLaC3uGvx+BFnUKgLBTZKxO26+MwNbuB9FxZgX87", - "Bal0JLfXkBwVftOQWmlI3nKt1pDsXtytilQtTXLvOpKjt9CCW0zxP6eW9Cd3j1Q0OJlPJjSihEGBF0hM", - "l0vxgEaT++YZuWVCMrP+SC+YqCKJtFYjC661RkIva0p/yWidbiPOO0dYKZJmCk0FjsgkT0xlBCRnuYr5", - "FXOw7zBBV0GIlvMJXe+uqZFrJJyEFq7+21bTLSp+3beq62ptP84sMJ7Z4rVWuSxFm2bt8mGJ9251yhZX", - "7f1rlY+ZxIz6trx0mdYQAmWMTAGrNDcpc8WXJQJaH719+8qlx2n1RLiiWIq7SliuSOiQ+ZWw+uhFWWLM", - "vOBa0OoDiW06LSQN2tpSMcFxQhmBeGIiQ5ls1fp1D3osvrwEHC7O10oCvudjacutPpwE/GCs4F5kzZNK", - "FWteGiT8un3FaXHyJpyaR8WvLAMKMJ6QrLeFc8V7NuF2a8YNClsYiPIswRHgUOrXDESaxTgwmIh+UwBc", - "IHiSEGGg77JcOXFryIrBUeYVpLeS2YVufpQzRZOLrgnnAfwSiTBbWPynIat0ZmU+yEOGHHsYoSCZGXGt", - "UqUeNOW5hLcgZdjvEuHkCi/kkNnMZfM5VPUVJDIokUnSRz9zAI1AeIop8xivKZf4nRyyCxonZGQxHy4Q", - "lUjOuFCEkRilfE5ktV+CRUKJgEkcYb1yEqV4AeBrBofSrA/PiAE4qyBLcP1vzGIKhfd0z8WUD4YMo53B", - "AKUEM2nzxCWewIVj20AwiMqAvkcY7Q2e269q+wYAwW75N/RpEoLMeYTHyQIRTcWAVKE2YQNTWwjTFBTW", - "2zehQpr9KuybtsJZZWOpdHUd4y7KWZkJD7b+nBWJ63q7VC4YzNN6AQkVxTVowT/GJMJ6PRmv9gOwizyK", - "chG6IPVWexVZ/x0FR29657BU4TzzBEwGEYlhzxlXMzjTHI7S5vcNVFUS1Z/jogkeEi4QRh5dlxYNEuXA", - "GjcApvCiLC/IXLngi83v3dnRx9cyAnf8DVDgY7mfgIj4ZFI5gOuvJnOAV+V3LJPwn/WcHrm6sj6Liyme", - "Mi4VjRwzrJeh/6YQtlYIV69skJonXFz6slWVfn/i4rKtBmbBT+njUsT8GX6Fjgg9PACafnh/BFjDjbKi", - "iebelbQ6fRWnFIQuqqQLdOYo4WyqT1Fplb93t4Gv1W0Y0Dh9mQrj7C4gfrQSMrI/mtK0ejK28Ce4GCLb", - "6kPzIt37PTijfuUK0TRLSEqgdG3PEJve7BIOCsr8U+mBIt2MV+pT5ecuG11QmviDrhOHgK7chm2A9L68", - "XUGmmvDpetDBonOHsBdAHRyyd9LAgV8Y19MFKniwFmgNxD+6mtFoBgiEoLfq9g1AIc6yiwJ8efMAvYSD", - "7GNQQ+cbBthf05rkCTHAgvM0vThYLs76/vQUPjLgg6YM68UBcgVZi/tD6rd8REE9iwRLhX61OIkbhTIO", - "O3qhsNY3i/ltWqzBEhx7yEK4g4xc2QbpBF14EIQXDfhYjt++4lP51biKypIGZi6KI6s6Am0SFneagjxo", - "Enb8bA8GIaTtlkiIZhh3DIS4NJhXfFqUU6iQMs6ytuRrhwlUPE/TFTSMNjxYNalinqu/SBUTIeBjS91N", - "xI02cGRLaeFLTagWRM8d7E0gv2Aok8E3Dy6VZqqdboewPO0c/Gb/NU/TTrdjx+Phot9AuF+DKFlvcDnk", - "Ru+MBxv5TSy/CSBkldl7iJC1m8Oq080S+Rvzwp/eW+hsdg9IhiAf1Iy4X5MI6o23avBhvEC2hJE9v4+R", - "AfwlihIuScXB83jAs6yhqyYzNhuK3Br39PDi3FUbahPBcm4/PXdffgW697pYETdm5KZ770EjyyN4zInA", - "cmk2Ey7qiEvrokm+ekL6cluyNNU2FPKNNm9uZWxFmFpPWGYR9oPYVJ/DueIpVjSCykfRjHPpkX0Bj2xq", - "lFnjcUGZYFoxWq7NILjQpHphzdAXVo04sCYzhP1Hto8+fG7zDsJfuEflFz95VoGC43ed6A/VAaA0u6Bk", - "gjKcS6KlujwlKFpEmiuaUlcERzMU4UzlgkAVP4JSymiapz7utd6xOQaMjovt9KKLxrlCCRZT0MrMQxds", - "E/E0JSwmYJ8bshnBc6pVSoESrAiLFj1JoPrvnKArLi4TjmMwMWQxBk8PVA8URFMggIinROEYKwyCzoU+", - "8SOTxHRRFAQ2aj0j1yU1xEMmcva9qWigm71wA71ABCC7qZwVhSMjHBMWBaGsz79uNvblbdHnRNUn+kCR", - "QbfipQ8ZKuTbXN1wvo4ookcWi82F3cY2bH6F0CubVdhq9ocjo3/PI23m6ub4QA6mYolXneKvw7NUEN1X", - "4116ePcRFyjOTXfeqQQy/7P6hAqG4gdbQWap2cbbOoaKCnnFMt+I52394f48uYUt7yvhhN1Gxb6pFlM5", - "6a+B5dpVvRXPfSAjprUl+Ta5h2PBLqLrwcQnLjwu91iMrZZhm6NZ8G2fOymBQfvi7BvbrrNtG/BwW7bt", - "bLNLLn2PkVPWgxjRMAe3ZtxGVm1NB/+m2Si12Xks88FZZOm5uDe2eFIwQsMaM7xIOI7/DEHCK/xHERfC", - "wF8AoMZjgl/1rIZ+egDY5soib12Xrfn+9HSziUsItZJHCPWIOYSXkqM/S+NlA+7rORGCxhalFB2dHttw", - "XSqRyFkfvU6pQoqjS0KyMqMFsgr7en4OCGS5oHwF8aPbIUyJRcYpU2tHUb56N4P5dKsy9PfMJy2e9zd3", - "eGt3OFj2Hx87Ay4DORtmAqs1U4XV2jqjlE24SI1chsc8161rHqSXSe+nQSqY0ITIhVQkNVGJkzyB4wa1", - "IWz9X/ud2eUuxOTqk2PS5TIiUiol5UwOmc0VyYjQfevPdftegFXQIaBwwV/PDJP8OoL39GBMvBpWTasG", - "kE1QV7Rz0NnCWbYVY4UbAsTs8D5jSD9BNB6Si3TMExqhhLJLiTYSemnUEzSXKNF/bK4M5xvBd1+6uvHt", - "T5Ze6RM24cHacYZmC2L+U2V1WbbmHJOPjq29JP5hcfwHNjrM1tbXTxYEJz2oR+yAe1CuaEI/GlanG6FS", - "0cikHOFi7d6fFky1P2SnRAn9DobUtiQxiAagXW5lgkdbw3ww2I0yCuhvuwQGBwyv+XEKPR6dvTNpqCTl", - "YtEdMv0PaPjt4Znx7k6wtSZ4A7WFk9HJ1us1Ac7nsEz/xhGCZoIr0QuCG/7NJXhzjJHGMyQbjijPVqlK", - "PPvTh7BaCe6bXeFx2hUA5KmYzUYB7OXQuMI2hDlP8lT/w/xxsg7XTOFo9h5e/WqkXTOctd24CT6KQ2nn", - "FBNT2/JBnB5mwR5rzKpeODcFEGIq0YDBW+BQ/Rmp+8ub7/11/ArdnXZFXd3Yr+Zs3ffNZ8fgEDb89Xgs", - "x9xQmpuJ4qutT1eYNluffkx4dCktFItvNtR6G+Cr6x9LPGzrIgQxATJDkYUwMkBZRHaHrGaANIg/EmGk", - "iEgpw8kWzNk0AsjezoqF55xCgnYEeSo9SWPATEoAvhvg7/RswFDlGvA8utJW1vLf8Z2RiqMxiXhKHNr5", - "Zkh1+xum6icuqtDlXwtffOutP0ACYgr29jVo7c09fhZ6+ym+hlDpOLcOZTeijZe8/NGYgroI9mbY2R3I", - "YaeLhp2ddNjRO3CEwYSKFdpHKWW5IrKPjo19C1JwnwyQJBFnsXSg686CtzuQTQm5hiwbsjufwHf3KfZY", - "qoKlfGM7CbEH/R7S30PSDtrwD5w9k3EXDl2MeK6Mud+eK/tWTBSYRzbv3VfrnZFvun0bTv43e3wrPAp2", - "WbNLb+sNZ89yOSPNJrdXppBRrsYA5u2Ki8oZ+jsfyy5i5MpYw4VU/SW+p78+Mx3cR6EB3dVNigzYuX+r", - "MNCiwkC5VmGwRhNgqa9kRx0GsZFcZ1woQHG0ufaGhkCTAOQIHuEEvT46GbJIsyIDLShIyoE7WTx0cwsf", - "/u0cvTh600XHUOgS/ZyPN/voNUsWrty48dEMmZHEDPOKMENjQ7UkDl3PZuxAPXcZLK47eKDK0eZkBDwr", - "bq9ckHi3MyM4Bonkj84rbjoLoA6/eaUPEAD/mi+Lbe+sFD46b4gSi97hRBGx3OypzZNiBWaGvaQdBJ0V", - "3Azwpe5QOuS1sk8jGxhojN2dTgAp49O3og93XyD1frxkJk7ElNsb54A0yiDJAMeLxxXLJGeoYI4hFuhf", - "10XZhKYsYcvLVioY0GVT5PdXZHJfybsq2PL/rqcLZvpoHU1ZZZ80ERflVtZ6el1y8MzAIVtHVYQzHFG1", - "6CKcJPaOsjdBEZHSK8TfsSD4MuZXrD9kb4pCLzahFx2dves6Ry2Kqbw0LVhfbB+9nhMh83ExOAQHzXiN", - "Yc1JPGSKowgnUZ5ocYNMJiSCXFyo3yIbfLnFUDp3eHbKToLFZryo9vzR1bgL0wTsXkkWdYrbMlu9JUiU", - "YJo2g49bQQ0CDiHUYKwb5QxRNklsSFUkuJTINtUjCZ3ScWIDhGQfvZ0RJHFKhixLMGNEoFyaqHg99F4m", - "iJS5SfDWDQBIr6GoLiqBBTPBlQ1NSDgX0kQTaAp/f4qkItkKMntjWj6FOd+RbGsatz09kJG6NoZmU4h9", - "BekNMZRiFlzTUZ64AMZ7DUU3A3poKfGxHPy3gk6nROhTgQ2TNeF45li75TSHvpKx3Fjv8rx4q129y6JV", - "LyvRy9hbCQw3KrG2487Nov4CnV/SRuxA++hmWcS/6I9a9l3NVg0Pwj76zFmGSnf+O1bJPPeSBNsasEoK", - "f2zmJG/klaNaSbRdD6vVOrP2LjNdW+NnPRhs1mNGy8KV9NkmhffrI4TB/aI83HeRtcdNWxW0q4pu2pDy", - "vx5N/6ugwLuB0X9glJNbwOh/VXn3gHP+cPgnwYP6UHn0Fd+zK7b7p0fCv6v0eQOHD3BsTenzhuvZ4NWV", - "itJ7+047Ncm2+GeS4G284w3kd7fs37T+FiqDt1jrXNCa4EmaqYULaLO+yjLoTNKPpN/gCC7iVu/OFXyL", - "kM4vRx6OThsDOv+ctfEfJGbUlg6kEp0cB4rOPzKMQf/MVS6WLX3r9LCIZnROmo3u1RNslygTpJfxDJwr", - "sVkwux7uLlNY9KcfkW3eYq7af0HtSYDqJzGKqSCRShamDqjmCKaP7yQSXGsC8JyLRXOUiDkiPwmeHtrZ", - "rLkP7ZmyxrAyzjBd9GKscG/uuM0KE9pnRHe6eErN8BBl6OWPaINcK2EqXKCJ1nwQnRRLSq4jQmIJNLnp", - "D3h70GDZpB/JaDpuM8oVtUpe21owKMql4qnb+5NjtAG1z6aE6b3Qov4EJNlM8DmNSVwZY2fOE7Oq2w0L", - "elO7qxYqisJ1Trkwg3sQGabNhTT9SLMqWyhCYsaUYRjc2qog1TNlkvh1f5gyF4Bj98iN4tsVZjW/Dafs", - "aEqEOpx2ERXnBuJ589s195ivOT8Zyt1pldvOheesNl63y49qmbZ0F4Ufity5+zVbv/96UnqofJTZPNZ0", - "Pi8U0iaz+ddFgoP7ux/u21z+/hGngL4kTvn2TOXQgG4xRDCvIKY7JnOS8CyFeujwbqfbyUXSOejMlMoO", - "trYg9nvGpTrYe/50t/Ppw6f/PwAA//91ZvGSme8BAA==", + "b/uvJmpekx9IbJSZ6KGUx7UaT/b9dhfRy7N3Z01jKqouIX90S3NaAe+zHO0yyZmp++TZVLCtx2gR1wwe", + "j+msj34kUvXIZMKFOrCBLOCQ8mq+4gUSJM41PfjQZJwhSccJnNGiVy3w61/0hAAIZJxPJkRUUQeeh2Rp", + "7+1RGjDu/aSfI/NCAXtw+mNVntZye1ut/axCDaC2T3BE2XSz9XYHDIK1aazDFXx59u6NLRvUhLisl7Io", + "LWTAlvvo16Kwll5qWaIq9QOWwho9mQBq/cJIJjwUjfKTIATBs7Kgpz574ME2GL1F1ACOAaOJM1dWEG1A", + "lFdMJPo9xwIzRRmJ0fuf5PdoYF3m739CJovGMiAq/cDGqqb4NLSljdlwZ7OFpBFOzLKY0h6U+VZKONKt", + "1Yyz8kNrzw0oG+Gico59oI35NMthAc/f9E5ev99KYzLvVsYE0YgznhA97k2Px84dJEOZuldhrfMmc5Gh", + "btmW7XhrVfC91ovkcbnA6nhU0ERw73+qEou5eEyqqjXXUkYVQG8B3vJGMUiDEN1Hp3hhtfoMOLjpykNA", + "xZOJAaAsOJsgCcESqlFL9P6n/towQcUVTprm8FY/tKdmQ09I76keZhdlFaKEk+QhP/ndPgkysHI+AacH", + "dFi3oPfRCQsfQhOLAthtBoZJSjplWtOQNhwlwqxYyaW9q3LyoJGijnXXqS5cZTrdADsKUUyId5rcumWh", + "e7k6Jr+sHhp+ub4io2mkud8jl/5X87JYzcbxN8gSdHExyHxq3CjWVidJhgVWJFnUAnSrCPtkOcSBXJPo", + "BvmHL/Trn0xVlVyQkZoJImc8qQbG7HaXK/NKCEKfE1uMzMzJ8wQpjlIsLuGUOc0O5cysQDWHYXcd4NBM", + "qewGk/r57dszY+5RRMxxUs+CkUshH8ckwQs0JuqKEOamgiXCfhB0PYtYNhSEEmqUEUF5dQ07u4F+z01g", + "PJoKHBFkvnJ1vQu2po9e26W0vQQgLKOISNmwv9ur9td+OsmTdnscGtb22jL40U02+O3RmStsVBSOdsu8", + "s7zKZ0T0zJFzFaRXb+2OXF1iy3XFDDLxkgQ5JlBzy6Ye+ZnBLuANaorpzyvZuB5zkH7+qO0HToFZqq45", + "5x9aqRv14x6K7Egxi0MFuE3GicFqmAKAG4S9ixx0ARqbaDhzJ3v3s02cEQTHlBEpa4nuUS6STrfTm9hZ", + "HWxtaX6fAIbl3u72s63VccUrA8pt/NwopqsMDi7KzsRhubRqAwAIk66SxBbOshYmWbOOa+4HYE/LAaxQ", + "dFvfbZ7I7zIaBoOlnP1rHClXJhBstBUfPPaPLSBNVwUZ3WBqKv9o3eu5fz4HwaCIDKtZlfy3lmgfAqUg", + "yFXTiLF/1dbREPnHoHTKRagWHBfKpoCOiYuCLe5DF2PqMJUr3trBM3+WT/b3d/fX8SFgNrVjbs9dYKrm", + "7WqSO/GW255f2wDE61VlDnekV5ez1euyhqY0R1wiqdULyjPCbrSe+3u7Ozdbz7YTOXFxgjW+FMIIOjo9", + "NjKR1iwxZUSglCgcY4WrTAaMm5rLQLEhTFLIE5t8v5q1NATU+KA/t62U9qXCMRqKJr5xGOEpZnQCSpJ5", + "0+9ZzvDO/pMDU9o1JpO9/Sf9fv+mUCgvSuyTVluxZaI2PVSUvpx93j7cAeJJm7n80Tk7fPuzZmS5FObS", + "2pJjyg68fxf/LB/AH+afY8rCSCltqgHTyVIV4GqAYm6hqEl8gMqC707uaRMw1uCdgHB1gGcK4g5Wwnbv", + "DmCwoHFarSNxg2TxFcnTWlx5zZJF49rcurBvWfJeeQV9fbtCi+K+9OPqgAtn/4R3bJ/G0lHUPV4OtbhV", + "5Wq5srjnUu23jLCinGeSmL8izgDpOVTbs3JFumctSsPBNWJrwBVd+j8WvXs/HvkD8X53peW8n2yRzw83", + "jJFaKZD+bVkOXc+FnDi65jCHjdHFrdC2oLIFJgwmBzzwXXibOMJq76+n//X7/5FnT/++/fur9+//e/7y", + "v45/pf/9Pjl7/VmgN6sxKR8UWPKLYUlC8FwFULItKZ1iFQVsdFr9a1hh+8RYHFQ0gxKwaEwOhqyHXlFF", + "hCkoWMuGHXbQBgFNCb7S4i5USzKJiJv64zPj4tYf/+HE4E/1NmKLUiDshhTgMzIfxzzFlG0O2ZDZtpCb", + "iAS9QP8VowhnpiogZUjrvws0FlDC0focy8676A+cZZ82hwyssuTawKtnGKrgTYo0QOYiBuyoTFy0fZ0U", + "OCQmRXXIitu6AGU0jud+WcGBkqSeRNawKKv1N6s5PRuE4CshwUlvJFQpAhWkoGxNRkXmFXo22FzW59bo", + "GAUNrSA/G5lhMmAP85C5uClr9pjENAK+4hJHZza1uMjZNZRmjXiZ4NcL2Js3JpsxRjhXM82LIou0EHF+", + "SUkXtrQL/lEIBYIvTYDHjGe98aI341mBuoGFCX/CJkSiqmT/n56daO89EXRiewqCJ2gSCQidcGTszEwO", + "amFdWJrYW1PghWnRZ07s66ZSkDTFYUxsv8oFc/VLCFQxBXiXgvpISCb/HkUJBauTnPE8idEMEBiVbiYE", + "otgZFDlUeBzFZFL/dzXGZWf/CWiw7t+7O61TmM3SraKyPAnotKljfS04tmGTMAAjHoycIXxNVJq+Aa1j", + "H+wUisN/z5FrqDxxBSMxnj6TVCltOYxEeumUm8G8N3sMLIbMCNvT1OY+WjqFlTS6Fi2YcBX4LGmBXvPC", + "pO++fXWOFBGpA1TYiPTuwCkxUCU9KmVuK7MdHp2+2Ox3gshbFZcWbNXKNLvqoAPgGzZ8pSkqp7TR4JR0", + "0ckxpE/ba6XUxSDf5ScuUGJuxfIyOgD0lqq5B5uajifHVgBNFmUMjBFbhp1N12JWv94O0JtCBcTFUIpE", + "2JK2XJPlZQLN2ohIk4yz1Hotbxr8Y1b9s/cxpN5AwUvDiwE9tPH+am9zdFhk+qKqWchufCH5YUmN9i9v", + "7780zvaXl9F3byajWy/0KJthGaLume/VhJeW9t13ZFfZvWgOIqv0O5I0eLb+5sr+eNeQIvqeq3weApbd", + "721vv93eu7n57qYQyVVsNA83sUBJbg9vfBcwwQHQX6pGjdkGSD+2uQXOLvL+FM2wZN8peFizjmzvPm1j", + "lIBe28bp+xH6fGKGVHApB7RWxJcbyLlLmiRGgJF0ynCCnqON85OXv5y8erWJeuj169P6Vqz6Irg/t0BL", + "hlsA1tGkHAawtioYEahIJn379hUcroRAPo6Rwy9vj6G81rTYAlPZDe7l2Ttw/GM5cpG8zcmruEwAJ9dU", + "KrkMs9cqIP5zMJzNp6VhrM0kTRs2xG8tEPTPFaTiIG7i5h0gOLtshqXlfABw44fMGv36gJVXQiF/Lp6x", + "tTPcEZxx45UWggKuQWTsN91utwcmvpPhVBCGQmzLl3BcSv+tkYC7HRpIZz60cXzo5Kws/VU6I1zztTk9", + "3+lvP3kG1Wu3B20Ye4qjFX2fHh6173ywY26ZAzw+iOIDUNhv67OyhG1UEJxc4QVUfzRLO+yYC9PTbr1j", + "axXJVvE1y4DLt8NXrotxDQjKIM66wCU5SlcWn2mRr1pH0UtzixGa0iShkkScxbIqI8+wRDIz0LimCEsh", + "wQ8ZDLCLilrYIKUgHEUiL02PVrq28n6eWbqHQrAZZ1oHgEoQv5CFRCkFJ2jRPYQ+SlSkRcVDtiFcCl2R", + "Kwc1YGP9AySkdG2qQ9yFqGEoNqM/GDI5y5VmYpt9dMSZzFMirFUWjSl4jDaRzI1KC+OF1VhohilpTMSQ", + "6dcC4Lt/FOrJwZPBYDDodgpNblf/exCipjt1fvYtuLRJAgf4R2ZhpgFXUuQM5SwmoigITww51EPkbug4", + "/UxUafd5O/HKfl7KVeGDuQ6Euh269OdC+sJQG/RziA69hXK+f3sRvVUSmpNfbfqZ/Wp0kwgGgiKeJ7HW", + "+Mb6tjMGORJbM6QkynDnIhPknSnGWp26DT1WHP2eE7FA709PK2EPgkw0D2g3ceASDfvAsxttw84aG8na", + "0dzEvewBIN8H6HFdUvEkxC8Ocex7GF1WvKHQimGrojivsq9prTKYkUOZ2SdNNCsmWKvHazBEysBIv1W5", + "kBML+GlFaZdTYQHICzyQ0qkvF3JrnMutLKJbNpdpC8BangFYy14wmz4m81Geh1Qj/cih87x7d3KMNuAX", + "ABs2CTKV7jF+sv1s8Ox579l4+0lvLx5s9/D27pPezj4eTHajp7vbO7srkopapFfePmMyqDEH4piLqPWR", + "i54PBTU35S7UZBMbj31FWcyvKtdfMEDW790G367rfjm0vvUQgilBCZbKmC8aONkpXPIk0m2bgHSbwlvU", + "3gobOp+8HWx/rvUHBtdwR7wVOTNuVQM6UbgQUm/A/mZVx3k7lg8Dcokv61bL77z9og0O9p8f7H/uornk", + "jXVjrJPTPW5uU0SYA+WuZYe4lFXPjuQMlB0rExmrvk0m6XQ7Rb4L/A3CQC2WunjcKomr6cB2w2xk1bXS", + "kE1/UtFXIFLFgDLGB1pScfoIlPgoMBu0CHSU8DxGni3OYNSBH+7E0110M+AWsyY6k0ZrkjEgM5JKm89H", + "mWbE4H/UjdjU+wP0Et6FRzg1ap0dhClo47vecLww8TL6fLmujZK1esjnVr+Cb7SyhfS/YNp6GazJdnUT", + "Rjo7QL9y+KbQ9hiv237N66BmLb9etxNvWPx2B6UCnVlR8wD9VIiXhYBqBdINSeyfI8uwSgSjzQqOhN3x", + "jqaWcuc8TIRux6xop9txCwXYCcsoCu9Kql86fz4phgLJCE7gLJdJ47miicVth5lQqWgkbfKI3twmscfm", + "ZpJ4ZJSnpphUk/pqFaziIydVvT9FGwDN+RdkDdv6X5tF/Grlrtt5vvf8ydOd509aAXCVA1wvGh9Bnvzy", + "4NbKyVGWj6xtpGnqR2fvjO0jMlaFIvbl/amPd5IJrlmPnrlr0O/8ef+5jzsW83yceI5FC1JoYI5hw4LQ", + "egUvaoiD/J0mczqZsN8/Rpc7fxc03b5+InfG2w14yqajsNntxA8uWLJRk3HP1M0KQ0MBQQnZiJ72hkiY", + "ATonCgH99BCOQL0p8qktyTmMNbviQcLa293dffZ0f6cVXdnReQdnBEa4wKVsR+AdMXgTbbw5P0dbHsGZ", + "Nh3ICEDeM6v6hs8ZskWvB1WBtL892A1RScPFXVKNbXueNi75e6s+2knZRYfk7UK1XDrlwdXe3R083dt/", + "tt/uGFvz8Ehcr+YwLmXJLI+tzODv/AZIk28PzxAkBE9wVLXtuAixG41K3WhUUFXEVAO4wcCePX2yv7e7", + "s90OBjAUdGIBLisHtsq7AocuQBSB3QgsxTLr7TbdFiFxyhDYGxIlmKaHkUuxqN0+BvV/JMxr5Sa0uRis", + "Br50cbX4tpVxqzBZmQQdIxpwgXJW1Jrpr3fJfhHPajPXNtfDeq4eSsthevUsXpWpqXeLpcwEmVOeyy/Q", + "EFcmZ3aScC5u9G2TwvKGyDxRxmZDJXp/+h3wFE1rSCqSVXUoS40rUL1uObkbnecKiYSJvGmxWu1Gm61f", + "NeFuw6ntrgLUqHCDRiy9WHOunK0P/jzCSZRDdSVc7KeeFYDCASRAliULE9ufJJwzFM0wAyeJ8CCw0Iwn", + "cT8YCaufjCbBqAp+hRJuUMAvCcls4SEzCP2ZFmHonKANv+SeIaVaIdz91DAZW1qmSo37abiiJ5ahZLUi", + "FV6vJ1bcA6g2n1QsoQmfSlAKFWQt9Ot1ETIsTDICZqaQ1jw1umQg4DowxBozD92o5iblE6vgWpEDEs3N", + "SuJIcOmBU70/reUvr8h5K7KY1wd0VgfbgnSNQzNwlRl4tNb19kL3YyCf53NuSKBhyBlcESrpjJMpZjmU", + "IvII2Rri+63DIWdcqlEB8nXDwUo1gvoiuSAlfGKRdV/Yg9w7wXvRsbbbLJeNO77V10tUFW6qaYDNPDW4", + "ouHV6hY0GCLjZZSzlcBqJdxcHczqJuiFZUEKKqFV6uHYoQ3IefHYkodJuNkmSCassup+lrRVWy721d7g", + "vC3O32pYvzOsZidswgPYHzfwnDpLtI1WzYhwUHsxYZTETpcsXKjW1AUJ44kkKM6JXTkjnwpsFxyb4w0+", + "K+ZsZJRNa7y+3mEb87AZw+ryI9CvfbFNuJMMJ9S+FTmslYlXlAiXqbWtgkCpHIXdWcsNCzLNEyyQRehs", + "M2S5SBPKLtu0LhfpmCc0QvqDul98wpOEX430I/kDzGWz1ez0B6OmWlXnZnA2L9BsSK3fcgo/6Flu1rKS", + "wRKzZb7fAsdom+ixYKT4TzQhFinxHaPXHqFX8fn3dgZN2fINjVby5JehQm/KuS3JBk98LgO5hSulHFdl", + "i8S2aIIRe7JcmoI/LW4lh8rrXIC38+hUE0c+D5rkyPDrGjAJGhPI+3FTW+YaLdhim6kEa43kcob+zsdV", + "g2jbsN9ABbsNVkJkCDIJxvfDjq40SJs3ltbE292bYFAAW9UThY9uCO2wrtZfGV/VxE/eLJW9mxG7ZNTN", + "0ZTAa1HSxcV/FPAFttf2OAb1AoWBeGVAqZFqAaV+ob7Swqu6KdGYCwGQ5FrC4czNBmBXtMyj19rhXqG3", + "M7JAgqSYsiGjrDCSApgaQYzMifCyZLnQStaUxH30N0/FAxD3NFMLWx0AjOffScSvWDHGIfMHqRvPpW7n", + "kBnLosgzVakfqpsFrU8TCmQtgxNMCSioSdUMTQSRM3/uoSKqWsa74iJurE61QO4VKHoEPlak+CVhPisr", + "mgmqhqahkflqOYrPVECGp1b/RJWixKhedHh1f7kkIiwkFlMqXmkVuuIdFU85MSAwgIgCxSbtX4bFFygo", + "LTBPyub/6posfzorGq/+VnvNwzVxuNOHxmwbNMFGJo2nFuxT9aStDVWBNLhVaDbLvgS04UKoXcWeqiTg", + "Vc5pdU+2y8SrJwu40WxJElV733u2//RJy9JFn+WsM+hdX9o1N09XuOQaduq0jd/n2f6z58939/af79zI", + "w+LyShr2pym3xN8ftEGulT6syb/+8c/3pzWvzz7EYA9uNCiTWRIeUkN2SXVA70//9Y9/ulHdekAhRrMM", + "Gd/gt2+M0kn8nXSBAlUXXjsn2Qr9/rBiJMAFm0EbBKC46ZyMzLr1ysHUYEDaScE4wxFViwAjx1cm2r14", + "pYa13cYdVB1sSOQ1bVtUVM25ZD4uk043XOfoP41vuEYLz1pXQJP5uMkP/breq/FCl14LP8ahRYiBLIrv", + "Lxu4i/lcYVkJ6NZ/R5B34TLMlrNtzBurUXfrqRAQxWIL/XmhgCHk+5o8aT/yt7+2nZ7fsmLWqa/4hxXn", + "sPkI3sjqG7iRA0bfaH1qbY0/2Avwdl+Nxn5twpXFHyuFDMtb9+b9tsgeXi6cUdxgN+/PS5i8yYd1TGCg", + "RzsGu+Rl290KSTRQk5cLEzCg8YT0vOIFpvCTzI1HUJ95CzMfyOCMLvlkUsW63W/GRgfYH0j2cr1gpbRm", + "0kXk2tks6sDaBuNn2NmXw45WAYad7XTYqbmtgumTKb4e2Q6q2C6DVWDlZfp7bZDSzWCc8OjSlN2Dau59", + "NEApwUyinMHhr3nVtgervUPdTubtTQENTkyI0xLbgjGNyQzPKVT3sD6VaSUQk1xTJSFgFNo5QDE3aE+V", + "msN2hvo1k9x4UE4aLh3MFrZh3aB+jzMX0Vq+Cwa+CVQ6Zh+J4F0LVqA59uvXp10TwAChh2ZglfhGN1Ez", + "As0giy5q5RXK38Pxw+OEjGDcdbj+dHkd/Zx08KwKIomSFr+7JIcaEaCI50zVcfzTdopcNa1s+UrKGQT7", + "2fAPwGWzvRsCQTGJ4ETK5bNYJfRbEHctb8CudChxYDdEwnAowJcU9hW/sQ7h+gCMscErF27a8eO6jZdw", + "JBW39eWKUz0i1xEhcR3wM/xK21h5+2UwVv4VthhBRSVv+zbEOy/Prn93CV4w1qbV9mP6GWc9QCdxW2qR", + "RAw0oMWqqRJaBXrcg7QYheBVQy+0ybgm16vX+ldyrQAfPc4TA3oXJl3LquxltG7Fb53Z2HSguSBr6zXe", + "QR1DE29+q0qGNlT9IYoZ2rfupIDh0u6cE+XePbdk1LhD1UIvFZeWC/h3r1RjbAwpdZG94NF2ulkjwb1Z", + "2CpiQXlb5mgynJJRJsiEXq8gHvOCUYyrsCblQSoyGAy+6EaKr9HeUxTNsJC1sTM6nalkUQ3A2QtgKX1W", + "lU9BFGHOUNhm58vddB8uR7vZ7fRbDwnH5x400FJJEyuSjlbhZh+V3jZrnc/wAqw4jU7Cp7t7g8HuzuBW", + "wNluWDdYrqPyE1sTs9pOU0qd95119FeiVP0WiiTr5ULLV4JCrnaxTFIJgtMDSLzJcERQQiYAklcktK73", + "LNa7Xj14K1DZLNqC/t1G2X1zPvhqyZyiK4s57qbRcc7FKgaR/3yNQ7SBzURLkHqBnLvd3uDJ2+3dg/0n", + "B9vbdwF2XSxSU7bH04/bV0+THTzZS54tnv6+PXs63Ul3g3rYJTWVgdrQ6i/63cYom/KSrGIZVVga2rBz", + "yIioV9CuV56XJKGM9GSRIbU+TXEFLzD+97Xn/2Z2fjODlbLDeXWSvgiBVbk4Fcp6GPwtO5mVvov6bE6O", + "V8/iVhlI9YGE6a0+FCCvdoOBChXbnc9EZshZy2vonfdi64toZVbcuqso5GGHkx7c5YYVD5F3DZjBm/Wq", + "C3z5kgvYTqdcUDVLV98WxWsFjDjETX+UKq7iPfXRyZRB+Xz/5yJMzlei9Medbif5uFc9M/b39shfFoG4", + "IEC71b5U0CKMLCFzkqxeBXilVDyEiWTXuroe8w/bve3nEIeQfNz7YdB7Xo046JrV8pdv271d+XXQZg39", + "EoCudNT28xtFXLv1XEVBv9BQAbvyXrbYxJbGy2Ll7upwCbeVDS4fL+1xDcmnUQD9XEnPXm4jX2iKSYIX", + "IWx6z1Ara9qjT2RoTKaUyTZ2291BYbjdT4edPjq0AOGgyype9OM3r2nFpxOapiSmWsY0qn9zBsNOS1tc", + "XZe4WW0S91VAWuuHxbXn6yES1iVcrbsm+5+Rj/tZ2m87jXcVegfY1ZyKChhi8GIX0QnCrFaglLI5Tmhs", + "E+khMRLi1Q4cUFtJspYHyFIOdHaSLppyhcoU+pb2tpw12wWL8ZNrsLeuwMwwBLHzRQBRCgAxuop9nRyj", + "TPA4j8r80QQGXSJ+iLwG0bZCyF8fknuX9g1IzJ5wgdbbN5oMGu3sk037XbNNaoJt3urtwfqtvhOjSLeT", + "Z/F6HmZeasfBboTcviYFMWCiqS57TRL0JvOhBUd/46/gss5rbMmRFonyzDlYNE0tU1LA3QIuhlBc7zFJ", + "iL6mlhtBPInLLAkqSy66nqVuP3k2a3JxgkdqeSC/EJJpXQXwj6C/FLNFcGCu7Ghxl2wMHNq3NA6vnilX", + "ZFerOrinayWxxq3yTbhNJRQMl6/ZvA1eyqVn/i4wvn3RbBkBxTH8ipD2prkEgP3Shb012o/vwiz3kELa", + "a+t6qMG2OlDhAh3d9V/GAmuxrkq8eyH3fIgs3lrNuAmKtp4F6ludD3v/Y6zMaNQ/2PrhL/9378N/Bq3N", + "Nb1ZEtGLyQQCjS7JomeKD2kdvV8FYoXKB1qYnlpSITgFGxKAnNvD6I93f1AwjcWvOF2aAkRoeZWDttdO", + "6C//0Rzf5C3jO+CTa0n2swuD3EUBVcXddbSREjF1seQukWyzP2SQm3ZJFhJ59cisSOMI9TtZfOJFoKML", + "Iwb2CZtfoDGFAo9yyLRWi6OIZFqbsCVuqKlSzoH7CIITvx1bF80lfluHpIknIOj96RKK7+t3b398/e7X", + "49Hrsxe/Hp6Mfnnx3xDicdUzPcQ9TXt7+09sbXJ/JbeD9TFuXuahj05tmL519U9yUGgBp0uiNFc5BIWQ", + "6yjJJZ07B6FKbl/QYTlZ9/YFEj4TAVipJBSVYJGqEzoh4NeH68QG1VDpiJFKKOpujRuUoeUb2xDOsAOc", + "1KvJHyqnobcivNrlxlYX/cmsHQs12KiBww4Zr1B9P6C9UAl4FS72w3sZbUDmiKs86xJnN2+G1XpYNBiM", + "PPzCBYYGz79EEdB3K6t+znnS0+pNQ6WEoDXZrEUwch6aMhkJnSanw3QckOGtaXdKpzjgZwj5E75IsU43", + "oLUZU0v731i1LJzHcFwvI2GOpVmqWtmDmpFAql5zmkOqpdoG4F1AFja5q9SLrasmqqZMbdmiuiG8jJgD", + "mPmqbOXylDl0xB58tD4Jd6Ve5c3MG0nz3pw69aGm4KxYoDO9NFczIoi3EfBBCc9/wyWzeTktUFhMUcKM", + "iDJm1SX1aKkU3M0SbRSWH7cERbbxsjl8dfmFU3xd9ACuFCyX/I8wj7L80/bLHwEq/40reUknrgkYRk25", + "CwPDV6lo1Zo4qlreDJ+qludt3g8ePMurVnC/prNVI86yjwpphujxb5iqn7gAdbAZ8+TO8eXh8o+JAAy4", + "Onp8K+h1mpJ4xHO1+vzbivr2yi/KopZldZ3qi4GIo0o6bxMvcKgc5RiWV1ovB4lyQdXiXK+XDeaGNEhX", + "yxYWEjqCn8uOoX7op09gNJ4EEkZeEkYEjaA6qz6PKWagMaH3p16RPlOvcQmvFUSg10cn1tzgIH9BfaQK", + "SM/FXR6enXS6nTkRRuXuDPq7/QEc5owwnNHOQWe3v90fdECrmsEUt6Civs2ftvnGheJ6EltJ6Ef3kv5S", + "4JQo+OK3ABIAxB3a10EFwVNPicwwFVaLzBJAKDAEQ/XXUG7AXagH5lbummVvbTOFNGPIfiHZa7u5H0BQ", + "hrMD09wZDCywubLXL+TumISBrb/b6NGy31ZSnV2iAPr+kprnZMti6T91O3uD7RuNadVQ4OyGOn7HsE3i", + "JaCd799wIW7V6QkzaXk2ydqGQ/knDgjJP2u/fdB7JvM0xWLhFsxfrYzLJsGYSITdu0aPUxJFmlVAjaA+", + "es2IeY6wQthELoucQWll96Gm0OopMG27TS5Ain7k8eKLLWGlD2ej+FRlZ/q4fFqi5y9HOwUZL2+kfeQQ", + "tg3V3gMB/YiLuuAPdlL2Bs/vvtMjziYJjRTqFQRs45GphJCfBPDCHfYQF+j3nCuMinD+R3Skrcw6Lsit", + "W15FW3/Q+JM53gkJmcHPiEgxM8kR5p01h37pOBuXRHmcV95qjvBPjjv2pnIgPOaiAkGuekT9a6suDC5f", + "R3sBBAbbp5le/ICEv3cPJ9xOtigN+5BHDgpyolySx3ScrIttXAohQVnuJVFfC80P7vPKskUE/oSn6LEQ", + "8EtSSHjlbi1dCluZyJlRgIMS4JsyYdF+911V+HtbPvGiZMCvoZuGchbK+FVxvOgjt6ZG6VcLgFgSBOYZ", + "L18rZ3p4X8sJ27mPEwYzLjxF366pb9fUqlNuqMVNAQ6md8pb2CBuZIH489kfbmx9+GZ7aG97aGV5YOTK", + "Whf+zsd9ZCNSIx4TJGc8T2I0JsjgHbnYE4VFf/oRYRHN6JwAqB0UacsTRTMsILIkRTFW2PjQGw0TK80S", + "RXNburmei0MsF7iOYyHJCHD4Rk34k2UEImWMxEh/YqH7SjjBpXLi5uwHDexFg+XViK5mXJICz48p7zaH", + "9GZptGNotj9kby3Qq15ACKZ2vEaSBOBqV9h/OEN4yOwH3zsW4gLBJE5LzoUFYAZSg0xptmU5tU2PdCQj", + "HsLaeUsYZqonMxLRCY3stC7JwsZzBhtsVXdJD9iN8/1pkbCBdjbDeG0AzxgG5z0uniFLSVX/DYMg6CjJ", + "49LJ5SCEsBjjJAkW5pgmfIyTkVmfSxLwCb6EN+yilA6X0pvEeExMCflsoWacmb/zcc5Ubv4eC34liRh2", + "NvtDBokYdq1J3C0FRHQFhdzSjOtzJnhq+twyQ9z645IsPvWH7DBOKXMUAZ/gRHJEruE7qG8FmBmGezXQ", + "gzlNYT/4US4VT33kU0d3Zpg8V1mubEaJJKobQv0cMsXRHw7b8dPWH2WPn8BZTHCs6cR7xUwJZOumUcsR", + "1rMfwasBdzuBBRh29EVqwjymAjNlYDsLcEo09bd0o6iOABVT6yscYYYynpnKEkBUM6xJrtIGYDXgJEEK", + "jpL7VgvusJMN87HQe+m4EXfPAKXVjhFl6PRH7zAN9p6Fz5MkkSChiJL/On/9K4JbWe+Bea0M1zIpHUwL", + "DCjOwXXqeNoLHM2QcVRBMcFhh8bDTuHOjTdhrLm04TK9HvgUf9BD+8F006XxD/2+bsq4Kw/Qb3+YVg70", + "WcpSgwM67HzqIu/BlKpZPi6efQgvaBN82XmFEaANc81tAifBFJBmvBvfXJGYxYjbWyBZIIxKDuQHrowp", + "w2KxKpEwsPR2BfnERDJ6i/HHECIXh52DoYtdHHa6ww5hc/jNBjgOO5/CK2C9ls2V6+A+K5ybBRE9GQw2", + "1yNh2/UN+CxbOAa+sA7YqBUVZTf1DloY1j+Xf+DfWv8sXD+Y6c5LaCKj+Dvj+yN0QHgSu6+JBlwQNbEb", + "s4gkTuxeb+i5f+eB3qyIJMl9E+hDkWfhHiuQ+h8VOcJmlcdopfn+gSlucF+XSsVs/zD0++js5wHrubWd", + "k7kLdQ7XKQEMGqtKI/MywhKdw5h651r5fgG/9u1/ne4HmIoXCZ9eHBjVHSV8ihLKbD6AF6isxQO7lvCR", + "gaEpvrOoNK5I3IaRJP71j3/CoCib/usf/7TY7v/6xz/huG8ZeDWoMX0xI1ioMcHq4gD9QkjWwwmdEzcZ", + "qAJL5kQs0O7A2vzhEfJK3VspTQ7ZkL0hKhfMy5sw9dqkbdC6CvR8KMuJtDA++kU6scVkTGxjwG7jzrJZ", + "yns90d0AHCLMwJuAvhUdDQCWHDWFtq0m2gmbTM2cK0bTepjmUrDeev6iyLUy1NszA7whg4ElDp07eGAn", + "jTbOz19s9hFoW4YqoGAQ6A5lM1aN6H/jSet5kuEoVYYCq2x4U4QzPKYJdSbHhmon5gimOJpRRsr44gJr", + "3DVx4Eaqeczh2QmygZBdeHXIXp9vgYlVkUjlgnQtJxAWYbQsh8Ztngv0APyLKogO69l3h2xCMOQJnRwb", + "JuCBcBf5gEXDDIA8IMaVqkrlte6QGSRZi1ysD17KY5LAR9D/FCtyhRddVNS6ddVREqy0Qiy7+uUhM1iv", + "dg16AFWCvGH2gZ+ZIfVcJK/N2RJkkmjVGCLwTdlv6HtjwgWyEc5elX/XnUmyNMPSi5bi6PW5nt8UNEFu", + "7IHQ0utztxubXSQ5ihIK1BBhNmRTCARy4L2cVXa1SCibYRH3Iq4vAR/M6ZLxq4TE0yYee+QT2R1KMpV+", + "Asfp5zq5PjbhYrY8AX2IDUDdas/dsX2nnevOtvhn8t3ZQpA3cN4ZCy4x/Mas7jdHXgtHXnjdnFMv5Fk7", + "dgiMdxfxa7p4oIBfR3vLa26eeEv2EBY9tOGgbcArwgU6OzpBOI4FkXLz39vep2dqqLSU//T9qFnxQ4Se", + "2LFwYUH/rL2lSiCPhR28saNG2M2rXl/Xv9+2KsV3Gm+6og5PeeXd/e1R6/Qm10gp9Ja09u0mWRtsS2XE", + "ocxgSS09EI0SUogvxTn1qWidVdmE8RZXzkpxybLnk2N3IO/Pvmy7zln9brgHpnhcY4gPyAirqdZ+1ezH", + "RM3vil10aNMrzM9fF2kO7k8Kum9TdIjMH5O6GNeWTXNBA3TSeIG+JMrAm9ylnm57CEz8nAh3qs1AF2bW", + "xbTMp8jgtMCEwBKzWvc9Ma+0U31Ne38mzReW5yYSi13ybyJKC2W3XKtVCu6JLQF9d/ot9HAj9fbLha1Y", + "AgssMlhRx87tBJbVDSwXLNr8FrnyxSnaxDWWSqxw8yZxYck2aEqFnnVfct0h8+uNa5nO6rWUoUlCpzPr", + "BIjpBGL1lF+/G0a5cw+jLOpkC6yIDVF8jHm/Z3qRrRd4ToRCr49OzPr7V+rWHxC0ul5Vcsxr5e367s2r", + "HmERjwvnSbNMap98YYXJ0H8ll/f+T90jzGelTjxoEhg/Y/9NMDky8e99yv/Xzk8JHQssFv9r5yecZJSR", + "/7V7mGBFpNq8M2IZ3NdNd98KzCMmPq2/0OqiAWtiU4CMXSPwF2+1lPnd+38qsd9M+kaCf7Gu32T/NrK/", + "v1wrxX+7FXeqAJg+HsjDVRBbaLXh0TdIm3swmlqK9CBtKl6kEtRmxqWCR48vv9kGldOC4vxro6X1vzyQ", + "K68PR7onx11YSKgoDRUtbPrgPfkC3DjuXbi1/d6/I+AwHdNpznPpZyamWEUzIm3WbkKqDPixid3l9dwo", + "eH/FVDq4z6vj3uXqb3R/RxJ/fUMN8zYOvXUyv3urrcxv39cyv0E0tZnNtuxG15Vk2mwItHaYpm3JuAL9", + "uhwAHhpXSBdB77SiUqoLCDSIgyH731r/+E0RnH74waVQ5oPBzhP4nbD5hx9cFiU7daRCmBLUVtA7/PUY", + "vKhTCJSFIntlwnZ9HKZmN5CeKyvwb6cglY7k9hqSo8JvGlIrDclbrtUakt2Lu1WRqqVJ7l1HcvQWWnCL", + "Kf7n1JL+5O6RigYn88mERpQwKPACielyKR7QaHLfPCO3TEhm1h/pBRNVJJHWamTBtdZI6GVN6S8ZrdNt", + "xHnnCCtF0kyhqcARmeSJqYyA5CxXMb9iDvYdJugqCNFyPqHr3TU1co2Ek9DC1X/barpFxa/7VnVdre3H", + "mQXGM1u81iqXpWjTrF0+LPHerU7Z4qq9f63yMZOYUd+Wly7TGkKgjJEpYJXmJmWu+LJEQOujt29fufQ4", + "rZ4IVxRLcVcJyxUJHTK/ElYfvShLjJkXXAtafSCxTaeFpEFbWyomOE4oIxBPTGQok61av+5Bj8WXl4DD", + "xflaScD3fCxtudWHk4AfjBXci6x5UqlizUuDhF+3rzgtTt6EU/Oo+JVlQAHGE5L1tnCueM8m3G7NuEFh", + "CwNRniU4AhxK/ZqBSLMYBwYT0W8KgAsETxIiDPRdlisnbg1ZMTjKvIL0VjK70M2PcqZoctE14TyAXyIR", + "ZguL/zRklc6szAd5yJBjDyMUJDMjrlWq1IOmPJfwFqQM+10inFzhhRwym7lsPoeqvoJEBiUySfroZw6g", + "EQhPMWUe4zXlEr+TQ3ZB44SMLObDBaISyRkXijASo5TPiaz2S7BIKBEwiSOsV06iFC8AfM3gUJr14Rkx", + "AGcVZAmu/41ZTKHwnu65mPLBkGG0MxiglGAmbZ64xBO4cGwbCAZRGdD3CKO9wXP7VW3fACDYLf+GPk1C", + "kDmP8DhZIKKpGJAq1CZsYGoLYZqCwnr7JlRIs1+FfdNWOKtsLJWurmPcRTkrM+HB1p+zInFdb5fKBYN5", + "Wi8goaK4Bi34x5hEWK8n49V+AHaRR1EuQhek3mqvIuu/o+DoTe8cliqcZ56AySAiMew542oGZ5rDUdr8", + "voGqSqL6c1w0wUPCBcLIo+vSokGiHFjjBsAUXpTlBZkrF3yx+b07O/r4Wkbgjr8BCnws9xMQEZ9MKgdw", + "/dVkDvCq/I5lEv6zntMjV1fWZ3ExxVPGpaKRY4b1MvTfFMLWCuHqlQ1S84SLS1+2qtLvT1xcttXALPgp", + "fVyKmD/Dr9ARoYcHQNMP748Aa7hRVjTR3LuSVqev4pSC0EWVdIHOHCWcTfUpKq3y9+428LW6DQMapy9T", + "YZzdBcSPVkJG9kdTmlZPxhb+BBdDZFt9aF6ke78HZ9SvXCGaZglJCZSu7Rli05tdwkFBmX8qPVCkm/FK", + "far83GWjC0oTf9B14hDQlduwDZDel7cryFQTPl0POlh07hD2AqiDQ/ZOGjjwC+N6ukAFD9YCrYH4R1cz", + "Gs0AgRD0Vt2+ASjEWXZRgC9vHqCXcJB9DGrofMMA+2takzwhBlhwnqYXB8vFWd+fnsJHBnzQlGG9OECu", + "IGtxf0j9lo8oqGeRYKnQrxYncaNQxmFHLxTW+mYxv02LNViCYw9ZCHeQkSvbIJ2gCw+C8KIBH8vx21d8", + "Kr8aV1FZ0sDMRXFkVUegTcLiTlOQB03Cjp/twSCEtN0SCdEM446BEJcG84pPi3IKFVLGWdaWfO0wgYrn", + "abqChtGGB6smVcxz9RepYiIEfGypu4m40QaObCktfKkJ1YLouYO9CeQXDGUy+ObBpdJMtdPtEJannYPf", + "7L/madrpdux4PFz0Gwj3axAl6w0uh9zonfFgI7+J5TcBhKwyew8RsnZzWHW6WSJ/Y17403sLnc3uAckQ", + "5IOaEfdrEkG98VYNPowXyJYwsuf3MTKAv0RRwiWpOHgeD3iWNXTVZMZmQ5Fb454eXpy7akNtIljO7afn", + "7suvQPdeFyvixozcdO89aGR5BI85EVguzWbCRR1xaV00yVdPSF9uS5am2oZCvtHmza2MrQhT6wnLLMJ+", + "EJvqczhXPMWKRlD5KJpxLj2yL+CRTY0yazwuKBNMK0bLtRkEF5pUL6wZ+sKqEQfWZIaw/8j20YfPbd5B", + "+Av3qPziJ88qUHD8rhP9oToAlGYXlExQhnNJtFSXpwRFi0hzRVPqiuBohiKcqVwQqOJHUEoZTfPUx73W", + "OzbHgNFxsZ1edNE4VyjBYgpamXnogm0inqaExQTsc0M2I3hOtUopUIIVYdGiJwlU/50TdMXFZcJxDCaG", + "LMbg6YHqgYJoCgQQ8ZQoHGOFQdC50Cd+ZJKYLoqCwEatZ+S6pIZ4yETOvjcVDXSzF26gF4gAZDeVs6Jw", + "ZIRjwqIglPX5183Gvrwt+pyo+kQfKDLoVrz0IUOFfJurG87XEUX0yGKxubDb2IbNrxB6ZbMKW83+cGT0", + "73mkzVzdHB/IwVQs8apT/HV4lgqi+2q8Sw/vPuICxbnpzjuVQOZ/Vp9QwVD8YCvILDXbeFvHUFEhr1jm", + "G/G8rT/cnye3sOV9JZyw26jYN9ViKif9NbBcu6q34rkPZMS0tiTfJvdwLNhFdD2Y+MSFx+Uei7HVMmxz", + "NAu+7XMnJTBoX5x9Y9t1tm0DHm7Ltp1tdsml7zFyynoQIxrm4NaM28iqreng3zQbpTY7j2U+OIssPRf3", + "xhZPCkZoWGOGFwnH8Z8hSHiF/yjiQhj4CwDUeEzwq57V0E8PANtcWeSt67I135+ebjZxCaFW8gihHjGH", + "8FJy9GdpvGzAfT0nQtDYopSio9NjG65LJRI566PXKVVIcXRJSFZmtEBWYV/PzwGBLBeUryB+dDuEKbHI", + "OGVq7SjKV+9mMJ9uVYb+nvmkxfP+5g5v7Q4Hy/7jY2fAZSBnw0xgtWaqsFpbZ5SyCRepkcvwmOe6dc2D", + "9DLp/TRIBROaELmQiqQmKnGSJ3DcoDaErf9rvzO73IWYXH1yTLpcRkRKpaScySGzuSIZEbpv/blu3wuw", + "CjoEFC7465lhkl9H8J4ejIlXw6pp1QCyCeqKdg46WzjLtmKscEOAmB3eZwzpJ4jGQ3KRjnlCI5RQdinR", + "RkIvjXqC5hIl+o/NleF8I/juS1c3vv3J0it9wiY8WDvO0GxBzH+qrC7L1pxj8tGxtZfEPyyO/8BGh9na", + "+vrJguCkB/WIHXAPyhVN6EfD6nQjVCoamZQjXKzd+9OCqfaH7JQood/BkNqWJAbRALTLrUzwaGuYDwa7", + "UUYB/W2XwOCA4TU/TqHHo7N3Jg2VpFwsukOm/wENvz08M97dCbbWBG+gtnAyOtl6vSbA+RyW6d84QtBM", + "cCV6QXDDv7kEb44x0niGZMMR5dkqVYlnf/oQVivBfbMrPE67AoA8FbPZKIC9HBpX2IYw50me6n+YP07W", + "4ZopHM3ew6tfjbRrhrO2GzfBR3Eo7ZxiYmpbPojTwyzYY41Z1QvnpgBCTCUaMHgLHKo/I3V/efO9v45f", + "obvTrqirG/vVnK37vvnsGBzChr8ej+WYG0pzM1F8tfXpCtNm69OPCY8upYVi8c2GWm8DfHX9Y4mHbV2E", + "ICZAZiiyEEYGKIvI7pDVDJAG8UcijBQRKWU42YI5m0YA2dtZsfCcU0jQjiBPpSdpDJhJCcB3A/ydng0Y", + "qlwDnkdX2spa/ju+M1JxNCYRT4lDO98MqW5/w1T9xEUVuvxr4YtvvfUHSEBMwd6+Bq29ucfPQm8/xdcQ", + "Kh3n1qHsRrTxkpc/GlNQF8HeDDu7AznsdNGws5MOO3oHjjCYULFC+yilLFdE9tGxsW9BCu6TAZIk4iyW", + "DnTdWfB2B7IpIdeQZUN25xP47j7FHktVsJRvbCch9qDfQ/p7SNpBG/6Bs2cy7sKhixHPlTH323Nl34qJ", + "AvPI5r37ar0z8k23b8PJ/2aPb4VHwS5rdultveHsWS5npNnk9soUMsrVGMC8XXFROUN/52PZRYxcGWu4", + "kKq/xPf012emg/soNKC7ukmRATv3bxUGWlQYKNcqDNZoAiz1leyowyA2kuuMCwUojjbX3tAQaBKAHMEj", + "nKDXRydDFmlWZKAFBUk5cCeLh25u4cO/naMXR2+66BgKXaKf8/FmH71mycKVGzc+miEzkphhXhFmaGyo", + "lsSh69mMHajnLoPFdQcPVDnanIyAZ8XtlQsS73ZmBMcgkfzRecVNZwHU4Tev9AEC4F/zZbHtnZXCR+cN", + "UWLRO5woIpabPbV5UqzAzLCXtIOgs4KbAb7UHUqHvFb2aWQDA42xu9MJIGV8+lb04e4LpN6Pl8zEiZhy", + "e+MckEYZJBngePG4YpnkDBXMMcQC/eu6KJvQlCVsedlKBQO6bIr8/opM7it5VwVb/t/1dMFMH62jKavs", + "kybiotzKWk+vSw6eGThk66iKcIYjqhZdhJPE3lH2JigiUnqF+DsWBF/G/Ir1h+xNUejFJvSio7N3Xeeo", + "RTGVl6YF64vto9dzImQ+LgaH4KAZrzGsOYmHTHEU4STKEy1ukMmERJCLC/VbZIMvtxhK5w7PTtlJsNiM", + "F9WeP7oad2GagN0ryaJOcVtmq7cEiRJM02bwcSuoQcAhhBqMdaOcIcomiQ2pigSXEtmmeiShUzpObICQ", + "7KO3M4IkTsmQZQlmjAiUSxMVr4feywSRMjcJ3roBAOk1FNVFJbBgJriyoQkJ50KaaAJN4e9PkVQkW0Fm", + "b0zLpzDnO5JtTeO2pwcyUtfG0GwKsa8gvSGGUsyCazrKExfAeK+h6GZADy0lPpaD/1bQ6ZQIfSqwYbIm", + "HM8ca7ec5tBXMpYb612eF2+1q3dZtOplJXoZeyuB4UYl1nbcuVnUX6DzS9qIHWgf3SyL+Bf9Ucu+q9mq", + "4UHYR585y1Dpzn/HKpnnXpJgWwNWSeGPzZzkjbxyVCuJtuthtVpn1t5lpmtr/KwHg816zGhZuJI+26Tw", + "fn2EMLhflIf7LrL2uGmrgnZV0U0bUv7Xo+l/FRR4NzD6D4xycgsY/a8q7x5wzh8O/yR4UB8qj77ie3bF", + "dv/0SPh3lT5v4PABjq0pfd5wPRu8ulJRem/faacm2Rb/TBK8jXe8gfzulv2b1t9CZfAWa50LWhM8STO1", + "cAFt1ldZBp1J+pH0GxzBRdzq3bmCbxHS+eXIw9FpY0Dnn7M2/oPEjNrSgVSik+NA0flHhjHon7nKxbKl", + "b50eFtGMzkmz0b16gu0SZYL0Mp6BcyU2C2bXw91lCov+9COyzVvMVfsvqD0JUP0kRjEVJFLJwtQB1RzB", + "9PGdRIJrTQCec7FojhIxR+QnwdNDO5s196E9U9YYVsYZpotejBXuzR23WWFC+4zoThdPqRkeogy9/BFt", + "kGslTIULNNGaD6KTYknJdURILIEmN/0Bbw8aLJv0IxlNx21GuaJWyWtbCwZFuVQ8dXt/cow2oPbZlDC9", + "F1rUn4Akmwk+pzGJK2PszHliVnW7YUFvanfVQkVRuM4pF2ZwDyLDtLmQph9pVmULRUjMmDIMg1tbFaR6", + "pkwSv+4PU+YCcOweuVF8u8Ks5rfhlB1NiVCH0y6i4txAPG9+u+Ye8zXnJ0O5O61y27nwnNXG63b5US3T", + "lu6i8EORO3e/Zuv3X09KD5WPMpvHms7nhULaZDb/ukhwcH/3w32by98/4hTQl8Qp356pHBrQLYYI5hXE", + "dMdkThKepVAPHd7tdDu5SDoHnZlS2cHWFsR+z7hUB3vPn+52Pn349P8HAAD//19ApuOy8gEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/paths/paths.go b/lib/paths/paths.go index 814dc1432..833a7a911 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -343,6 +343,11 @@ func (p *Paths) DeviceMetadata(id string) string { return filepath.Join(p.DeviceDir(id), "metadata.json") } +// VFHealthState returns the path to the persisted vGPU VF health file. +func (p *Paths) VFHealthState() string { + return filepath.Join(p.dataDir, "gpu", "vf-health.json") +} + // Volume path methods // VolumesDir returns the root volumes directory. diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 78788412e..2c7843532 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -1,46 +1,47 @@ package resources import ( + "context" + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/logger" ) // GPUResourceStatus represents the GPU resource status for the API response. // Returns nil if no GPU is available on the host. type GPUResourceStatus struct { - Mode string `json:"mode"` // "vgpu" or "passthrough" - TotalSlots int `json:"total_slots"` // VFs for vGPU, physical GPUs for passthrough - UsedSlots int `json:"used_slots"` // Slots currently in use - Profiles []devices.GPUProfile `json:"profiles,omitempty"` // vGPU mode only - Devices []devices.PassthroughDevice `json:"devices,omitempty"` // passthrough mode only + Mode string `json:"mode"` // "vgpu" or "passthrough" + TotalSlots int `json:"total_slots"` // VFs for vGPU, physical GPUs for passthrough + UsedSlots int `json:"used_slots"` // Slots currently in use, including assigned quarantined VFs + AllocatableSlots int `json:"allocatable_slots"` // Healthy free slots used by admission control + QuarantinedSlots int `json:"quarantined_slots"` // Quarantined VFs; may overlap UsedSlots + Profiles []devices.GPUProfile `json:"profiles,omitempty"` // vGPU mode only + Devices []devices.PassthroughDevice `json:"devices,omitempty"` // passthrough mode only } -// GetGPUStatus returns the current GPU resource status. -// Returns nil if no GPU is available or the mode is "none". -func GetGPUStatus() *GPUResourceStatus { - mode := devices.DetectHostGPUMode() - if mode == devices.GPUModeNone { - return nil +// GetGPUStatus returns the current GPU resource status and any error that +// prevents determining allocatable vGPU capacity. It returns nil if no GPU is +// available or the mode is "none". +func GetGPUStatus(ctx context.Context) (*GPUResourceStatus, error) { + framework, vfs, err := devices.DiscoverVGPU() + if err != nil { + // Only report passthrough once vGPU discovery confirms no vGPU + // framework. On a vGPU host a transient probe failure would otherwise + // expose the PFs/VFs as available passthrough slots while active vGPU + // assignments exist. + logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU state", "error", err) + return nil, nil } - - switch mode { - case devices.GPUModeVGPU: - return getVGPUStatus() - case devices.GPUModePassthrough: - return getPassthroughStatus() - default: - return nil + if framework != devices.VGPUFrameworkNone { + return getVGPUStatus(ctx, framework, vfs) } + return getPassthroughStatus(), nil } -// getVGPUStatus returns GPU status for vGPU mode (SR-IOV + mdev). -func getVGPUStatus() *GPUResourceStatus { - vfs, err := devices.DiscoverVFs() - if err != nil || len(vfs) == 0 { - return nil - } - - // Count used VFs (those with mdevs) +// getVGPUStatus returns GPU status for vGPU mode (SR-IOV). +func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []devices.VirtualFunction) (*GPUResourceStatus, error) { usedSlots := 0 + // Count used VFs (those with a vGPU assigned) for _, vf := range vfs { if vf.Allocated { usedSlots++ @@ -48,17 +49,25 @@ func getVGPUStatus() *GPUResourceStatus { } // Get available profiles (reuse VFs to avoid redundant discovery) - profiles, err := devices.ListGPUProfilesWithVFs(vfs) + profiles, err := devices.ListGPUProfilesWithVFs(framework, vfs) if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to list vGPU profiles; reporting none", "framework", framework, "error", err) profiles = nil } - - return &GPUResourceStatus{ - Mode: string(devices.GPUModeVGPU), - TotalSlots: len(vfs), - UsedSlots: usedSlots, - Profiles: profiles, + allocatableSlots, quarantinedSlots, err := devices.VGPUAvailability(framework, vfs) + status := &GPUResourceStatus{ + Mode: string(devices.GPUModeVGPU), + TotalSlots: len(vfs), + UsedSlots: usedSlots, + AllocatableSlots: allocatableSlots, + QuarantinedSlots: quarantinedSlots, + Profiles: profiles, + } + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to count allocatable vGPU slots; reporting none", "framework", framework, "error", err) + status.AllocatableSlots = 0 } + return status, err } // getPassthroughStatus returns GPU status for whole-GPU passthrough mode. @@ -93,9 +102,10 @@ func getPassthroughStatus() *GPUResourceStatus { } return &GPUResourceStatus{ - Mode: string(devices.GPUModePassthrough), - TotalSlots: len(passthroughDevices), - UsedSlots: usedSlots, - Devices: passthroughDevices, + Mode: string(devices.GPUModePassthrough), + TotalSlots: len(passthroughDevices), + UsedSlots: usedSlots, + AllocatableSlots: len(passthroughDevices) - usedSlots, + Devices: passthroughDevices, } } diff --git a/lib/resources/gpu_test.go b/lib/resources/gpu_test.go new file mode 100644 index 000000000..825a20260 --- /dev/null +++ b/lib/resources/gpu_test.go @@ -0,0 +1,88 @@ +package resources + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func initVFHealthForTest(t *testing.T, state []byte) { + t.Helper() + dataDir := t.TempDir() + if state != nil { + path := paths.New(dataDir).VFHealthState() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, state, 0o644)) + } + devices.NewManager(paths.New(dataDir)) + resetDir := t.TempDir() + t.Cleanup(func() { devices.NewManager(paths.New(resetDir)) }) +} + +func TestGetVGPUStatusFailsClosedWhenVFHealthIsUnavailable(t *testing.T) { + initVFHealthForTest(t, []byte("not json")) + + status, err := getVGPUStatus(context.Background(), devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + assert.Zero(t, status.AllocatableSlots) + assert.Zero(t, status.QuarantinedSlots) + require.ErrorContains(t, err, "VF health state unavailable") +} + +func TestGetVGPUStatusReportsQuarantinedSlots(t *testing.T) { + initVFHealthForTest(t, nil) + for _, instance := range []string{"instance-1", "instance-2"} { + _, err := devices.ReportVFInitFailure(devices.VFInitFailureReport{VFAddress: "0000:82:00.4", InstanceID: instance}) + require.NoError(t, err) + } + + status, err := getVGPUStatus(context.Background(), devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4"}, + {PCIAddress: "0000:82:00.5", Allocated: true}, + {PCIAddress: "0000:82:00.6"}, + }) + require.NoError(t, err) + assert.Equal(t, 3, status.TotalSlots) + assert.Equal(t, 1, status.UsedSlots) + assert.Equal(t, 1, status.AllocatableSlots) + assert.Equal(t, 1, status.QuarantinedSlots) +} + +func TestReserveAllocationUsesAllocatableGPUSlots(t *testing.T) { + status := &GPUResourceStatus{ + Mode: string(devices.GPUModeVGPU), + TotalSlots: 4, + UsedSlots: 1, + AllocatableSlots: 0, + } + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return status, nil }) + t.Cleanup(func() { setGPUStatusProvider(nil) }) + + mgr := NewManager(&config.Config{}, paths.New(t.TempDir())) + ctx := context.Background() + + err := mgr.ValidateAllocation(ctx, 0, 0, 0, 0, 0, 0, true) + require.ErrorContains(t, err, "no allocatable vgpu slots") + + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { + return status, errors.New("VF health state unavailable: read failed") + }) + err = mgr.ValidateAllocation(ctx, 0, 0, 0, 0, 0, 0, true) + require.ErrorContains(t, err, "vGPU placement is disabled: VF health state unavailable") + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return status, nil }) + + status.AllocatableSlots = 1 + require.NoError(t, mgr.ReserveAllocation(ctx, "pending-a", 0, 0, 0, 0, 0, 0, true)) + err = mgr.ReserveAllocation(ctx, "pending-b", 0, 0, 0, 0, 0, 0, true) + require.ErrorContains(t, err, "no allocatable vgpu slots") + + mgr.FinishAllocation("pending-a") + require.NoError(t, mgr.ReserveAllocation(ctx, "pending-b", 0, 0, 0, 0, 0, 0, true)) +} diff --git a/lib/resources/monitoring.go b/lib/resources/monitoring.go index 32f584193..d69b1e2a4 100644 --- a/lib/resources/monitoring.go +++ b/lib/resources/monitoring.go @@ -199,7 +199,7 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { gpuProfileSlots, err := meter.Int64ObservableGauge( "hypeman_resources_gpu_profile_slots", - metric.WithDescription("Available GPU slots by vGPU profile"), + metric.WithDescription("Virtual functions able to create each vGPU profile (best-effort snapshot)"), ) if err != nil { return err diff --git a/lib/resources/monitoring_test.go b/lib/resources/monitoring_test.go index ab6c3e4dd..39166856e 100644 --- a/lib/resources/monitoring_test.go +++ b/lib/resources/monitoring_test.go @@ -198,7 +198,7 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { mgr, _, _ := monitoringTestManager(t) originalProvider := currentGPUStatusProvider() - setGPUStatusProvider(func() *GPUResourceStatus { + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return &GPUResourceStatus{ Mode: "vgpu", TotalSlots: 8, @@ -207,7 +207,7 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { {Name: "L40S-1Q", Available: 5}, {Name: "L40S-2Q", Available: 2}, }, - } + }, nil }) defer func() { setGPUStatusProvider(originalProvider) diff --git a/lib/resources/resource.go b/lib/resources/resource.go index caaf5ba50..9c28fda0f 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -37,13 +37,13 @@ var ( gpuStatusProvider = GetGPUStatus ) -func currentGPUStatusProvider() func() *GPUResourceStatus { +func currentGPUStatusProvider() func(context.Context) (*GPUResourceStatus, error) { gpuStatusProviderMu.RLock() defer gpuStatusProviderMu.RUnlock() return gpuStatusProvider } -func setGPUStatusProvider(fn func() *GPUResourceStatus) { +func setGPUStatusProvider(fn func(context.Context) (*GPUResourceStatus, error)) { if fn == nil { fn = GetGPUStatus } @@ -427,7 +427,7 @@ func (m *Manager) GetFullStatus(ctx context.Context) (*FullResourceStatus, error } // Get GPU status - gpuStatus := currentGPUStatusProvider()() + gpuStatus, _ := currentGPUStatusProvider()(ctx) return &FullResourceStatus{ CPU: *cpuStatus, @@ -691,15 +691,18 @@ func (m *Manager) validateAllocationLocked(ctx context.Context, excludeID string // Check GPU if needed if req.GPUSlots > 0 { - gpuStatus := currentGPUStatusProvider()() + gpuStatus, gpuStatusErr := currentGPUStatusProvider()(ctx) if gpuStatus == nil { return fmt.Errorf("insufficient GPU: no GPU available on this host") } - availableSlots := gpuStatus.TotalSlots - gpuStatus.UsedSlots - pending.GPUSlots + availableSlots := gpuStatus.AllocatableSlots - pending.GPUSlots if availableSlots < req.GPUSlots { + if gpuStatusErr != nil { + return fmt.Errorf("insufficient GPU: vGPU placement is disabled: %w", gpuStatusErr) + } if availableSlots <= 0 { - return fmt.Errorf("insufficient GPU: all %d %s slots are in use", - gpuStatus.TotalSlots, gpuStatus.Mode) + return fmt.Errorf("insufficient GPU: no allocatable %s slots available (%d total, %d in use)", + gpuStatus.Mode, gpuStatus.TotalSlots, gpuStatus.UsedSlots) } return fmt.Errorf("insufficient GPU: requested %d %s slot(s), but only %d available", req.GPUSlots, gpuStatus.Mode, availableSlots) diff --git a/openapi.yaml b/openapi.yaml index a0b640ef4..cea2abe3f 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1757,8 +1757,12 @@ components: example: "L40S-1Q" mdev_uuid: type: string - description: mdev device UUID + description: mdev device UUID (mdev hosts only) example: "aa618089-8b16-4d01-a136-25a0f3c73123" + device_path: + type: string + description: sysfs path of the assigned vGPU device + example: "/sys/bus/pci/devices/0000:82:00.4" GPUProfile: type: object @@ -1775,7 +1779,7 @@ components: example: 1024 available: type: integer - description: Number of instances that can be created with this profile + description: "Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer." example: 59 PassthroughDevice: @@ -1796,7 +1800,7 @@ components: type: object description: GPU resource status. Null if no GPUs available. nullable: true - required: [mode, total_slots, used_slots] + required: [mode, total_slots, used_slots, allocatable_slots, quarantined_slots] properties: mode: type: string @@ -1809,8 +1813,16 @@ components: example: 64 used_slots: type: integer - description: Slots currently in use + description: Slots currently in use. Includes quarantined VFs that are still assigned, so this can overlap quarantined_slots. example: 5 + allocatable_slots: + type: integer + description: Free slots eligible for placement, matching admission control (excludes quarantined VFs; 0 while VF health state is unavailable) + example: 57 + quarantined_slots: + type: integer + description: VFs quarantined after guest driver init failures (vGPU mode only). May overlap used_slots until the affected instance releases its VF. + example: 2 profiles: type: array description: Available vGPU profiles (only in vGPU mode)