diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1b418ad1..79b079fb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -70,7 +70,7 @@ jobs: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} TLS_TEST_DOMAIN: "test.hypeman-development.com" TLS_ALLOWED_DOMAINS: '*.hypeman-development.com' - run: make test + run: make test TEST_TIMEOUT=20m test-darwin: runs-on: [self-hosted, macos, arm64] diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 50deb885..3c94b48e 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -447,6 +447,74 @@ func (s *ApiService) RestoreInstance(ctx context.Context, request oapi.RestoreIn return oapi.RestoreInstance200JSONResponse(instanceToOAPI(*result)), nil } +// ForkInstance forks an instance from stopped or standby into a new instance. +// The id parameter can be an instance ID, name, or ID prefix. +// Note: Resolution is handled by ResolveResource middleware. +func (s *ApiService) ForkInstance(ctx context.Context, request oapi.ForkInstanceRequestObject) (oapi.ForkInstanceResponseObject, error) { + inst := mw.GetResolvedInstance[instances.Instance](ctx) + if inst == nil { + return oapi.ForkInstance500JSONResponse{ + Code: "internal_error", + Message: "resource not resolved", + }, nil + } + log := logger.FromContext(ctx) + + if request.Body == nil { + return oapi.ForkInstance400JSONResponse{ + Code: "invalid_request", + Message: "request body is required", + }, nil + } + + targetState := instances.State("") + if request.Body.TargetState != nil { + targetState = instances.State(*request.Body.TargetState) + } + + result, err := s.InstanceManager.ForkInstance(ctx, inst.Id, instances.ForkInstanceRequest{ + Name: request.Body.Name, + FromRunning: request.Body.FromRunning != nil && *request.Body.FromRunning, + TargetState: targetState, + }) + if err != nil { + switch { + case errors.Is(err, instances.ErrNotFound): + return oapi.ForkInstance404JSONResponse{ + Code: "not_found", + Message: "instance not found", + }, nil + case errors.Is(err, instances.ErrInvalidState): + return oapi.ForkInstance409JSONResponse{ + Code: "invalid_state", + Message: err.Error(), + }, nil + case errors.Is(err, instances.ErrInvalidRequest): + return oapi.ForkInstance400JSONResponse{ + Code: "invalid_request", + Message: err.Error(), + }, nil + case errors.Is(err, instances.ErrAlreadyExists), errors.Is(err, network.ErrNameExists): + return oapi.ForkInstance409JSONResponse{ + Code: "name_conflict", + Message: err.Error(), + }, nil + case errors.Is(err, instances.ErrNotSupported): + return oapi.ForkInstance501JSONResponse{ + Code: "not_supported", + Message: err.Error(), + }, nil + default: + log.ErrorContext(ctx, "failed to fork instance", "error", err) + return oapi.ForkInstance500JSONResponse{ + Code: "internal_error", + Message: "failed to fork instance", + }, nil + } + } + return oapi.ForkInstance201JSONResponse(instanceToOAPI(*result)), nil +} + // StopInstance gracefully stops a running instance // The id parameter can be an instance ID, name, or ID prefix // Note: Resolution is handled by ResolveResource middleware diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 8c74cf89..06531ae0 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -2,6 +2,7 @@ package api import ( "context" + "fmt" "os" "testing" "time" @@ -9,6 +10,7 @@ import ( "github.com/c2h5oh/datasize" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/instances" + mw "github.com/kernel/hypeman/lib/middleware" "github.com/kernel/hypeman/lib/oapi" "github.com/kernel/hypeman/lib/paths" "github.com/kernel/hypeman/lib/system" @@ -137,6 +139,24 @@ type captureCreateManager struct { lastReq *instances.CreateInstanceRequest } +type captureForkManager struct { + instances.Manager + lastID string + lastReq *instances.ForkInstanceRequest + result *instances.Instance + err error +} + +func (m *captureForkManager) ForkInstance(ctx context.Context, id string, req instances.ForkInstanceRequest) (*instances.Instance, error) { + reqCopy := req + m.lastID = id + m.lastReq = &reqCopy + if m.err != nil { + return nil, m.err + } + return m.result, nil +} + func (m *captureCreateManager) CreateInstance(ctx context.Context, req instances.CreateInstanceRequest) (*instances.Instance, error) { reqCopy := req m.lastReq = &reqCopy @@ -190,6 +210,185 @@ func TestCreateInstance_OmittedHotplugSizeDefaultsToZero(t *testing.T) { assert.Equal(t, int64(0), int64(hotplugBytes), "response should report zero hotplug_size when omitted") } +func TestForkInstance_Success(t *testing.T) { + svc := newTestService(t) + + now := time.Now() + source := instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "src-instance", + Name: "src-instance", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateStopped, + } + + forked := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "forked-instance", + Name: "forked-instance", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateStopped, + } + + mockMgr := &captureForkManager{ + Manager: svc.InstanceManager, + result: forked, + } + svc.InstanceManager = mockMgr + + resp, err := svc.ForkInstance( + mw.WithResolvedInstance(ctx(), source.Id, source), + oapi.ForkInstanceRequestObject{ + Id: source.Id, + Body: &oapi.ForkInstanceRequest{ + Name: "forked-instance", + }, + }, + ) + require.NoError(t, err) + + created, ok := resp.(oapi.ForkInstance201JSONResponse) + require.True(t, ok, "expected 201 response") + assert.Equal(t, "forked-instance", created.Name) + assert.Equal(t, source.Id, mockMgr.lastID) + require.NotNil(t, mockMgr.lastReq) + assert.Equal(t, "forked-instance", mockMgr.lastReq.Name) + assert.False(t, mockMgr.lastReq.FromRunning) + assert.Equal(t, instances.State(""), mockMgr.lastReq.TargetState) +} + +func TestForkInstance_NotSupported(t *testing.T) { + svc := newTestService(t) + + source := instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "src-instance", + Name: "src-instance", + Image: "docker.io/library/alpine:latest", + CreatedAt: time.Now(), + HypervisorType: hypervisor.TypeQEMU, + }, + State: instances.StateStopped, + } + + mockMgr := &captureForkManager{ + Manager: svc.InstanceManager, + err: instances.ErrNotSupported, + } + svc.InstanceManager = mockMgr + + resp, err := svc.ForkInstance( + mw.WithResolvedInstance(ctx(), source.Id, source), + oapi.ForkInstanceRequestObject{ + Id: source.Id, + Body: &oapi.ForkInstanceRequest{ + Name: "forked-instance", + }, + }, + ) + require.NoError(t, err) + + notSupported, ok := resp.(oapi.ForkInstance501JSONResponse) + require.True(t, ok, "expected 501 response") + assert.Equal(t, "not_supported", notSupported.Code) +} + +func TestForkInstance_InvalidRequest(t *testing.T) { + svc := newTestService(t) + + source := instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "src-instance", + Name: "src-instance", + Image: "docker.io/library/alpine:latest", + CreatedAt: time.Now(), + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateStopped, + } + + mockMgr := &captureForkManager{ + Manager: svc.InstanceManager, + err: fmt.Errorf("%w: name is required", instances.ErrInvalidRequest), + } + svc.InstanceManager = mockMgr + + resp, err := svc.ForkInstance( + mw.WithResolvedInstance(ctx(), source.Id, source), + oapi.ForkInstanceRequestObject{ + Id: source.Id, + Body: &oapi.ForkInstanceRequest{ + Name: "", + }, + }, + ) + require.NoError(t, err) + + badReq, ok := resp.(oapi.ForkInstance400JSONResponse) + require.True(t, ok, "expected 400 response") + assert.Equal(t, "invalid_request", badReq.Code) +} + +func TestForkInstance_FromRunningFlagForwarded(t *testing.T) { + svc := newTestService(t) + + now := time.Now() + source := instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "src-instance", + Name: "src-instance", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateRunning, + } + + forked := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "forked-instance", + Name: "forked-instance", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateStandby, + } + + mockMgr := &captureForkManager{ + Manager: svc.InstanceManager, + result: forked, + } + svc.InstanceManager = mockMgr + + fromRunning := true + targetState := oapi.ForkTargetStateRunning + resp, err := svc.ForkInstance( + mw.WithResolvedInstance(ctx(), source.Id, source), + oapi.ForkInstanceRequestObject{ + Id: source.Id, + Body: &oapi.ForkInstanceRequest{ + Name: "forked-instance", + FromRunning: &fromRunning, + TargetState: &targetState, + }, + }, + ) + require.NoError(t, err) + + _, ok := resp.(oapi.ForkInstance201JSONResponse) + require.True(t, ok, "expected 201 response") + require.NotNil(t, mockMgr.lastReq) + assert.True(t, mockMgr.lastReq.FromRunning) + assert.Equal(t, instances.StateRunning, mockMgr.lastReq.TargetState) +} + func TestInstanceLifecycle_StopStart(t *testing.T) { // Require KVM access for VM creation if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index 9ea0c7c2..27e46346 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -84,6 +84,10 @@ func (m *mockInstanceManager) DeleteInstance(ctx context.Context, id string) err return nil } +func (m *mockInstanceManager) ForkInstance(ctx context.Context, id string, req instances.ForkInstanceRequest) (*instances.Instance, error) { + return nil, instances.ErrNotFound +} + func (m *mockInstanceManager) StandbyInstance(ctx context.Context, id string) (*instances.Instance, error) { return nil, nil } diff --git a/lib/forkvm/README.md b/lib/forkvm/README.md new file mode 100644 index 00000000..24b5cefa --- /dev/null +++ b/lib/forkvm/README.md @@ -0,0 +1,64 @@ +# VM Forking: Hypervisor Behavior + +This document describes hypervisor-specific fork behavior and how fork is made +to work across implementations. + +## Common fork model + +- **Stopped source**: clone VM data and start a new VM from copied state. +- **Standby source**: clone data + snapshot artifacts, then adapt snapshot + identity for the fork (paths, network, vsock behavior varies by hypervisor). +- **Running source**: transition source to standby, fork from that standby + snapshot, then restore the source. + +For networked forks, the fork gets a fresh host/guest identity (IP, MAC, TAP) +instead of reusing the source identity. + +## Cloud Hypervisor + +- Snapshot-based forks are supported by rewriting snapshot configuration before + restore. +- Path rewrites are constrained to exact source-directory matches or source-dir + path prefixes to avoid mutating unrelated values. +- Serial log path, vsock socket path, and network fields are updated for the + fork. +- Vsock CID is intentionally kept stable for snapshot restore compatibility. +- Running-source fork works by standby -> fork -> restore source, with source + and fork separated by rewritten runtime endpoints. + +## QEMU + +- Snapshot-based forks are supported by rewriting QEMU snapshot VM config. +- Rewrites are explicit and path-safe (source-dir exact/prefix replacement), + applied to disk/kernel/initrd/serial/vsock socket paths. +- Kernel arguments are left unchanged (not blanket-rewritten), to avoid + accidental mutation of non-path text. +- Network identity is updated in snapshot config for the fork. +- Vsock CID updates are supported for snapshot state, so running-source fork can + rotate source CID when needed to avoid CID collision after restore. + +## Firecracker + +- Firecracker snapshot restore supports **network overrides** but does not + expose a full snapshot-config rewrite surface for arbitrary embedded paths. +- To make standby/running fork work, fork preparation stores desired network + override data and source->target data-directory mapping. +- During restore, the source data path is temporarily aliased to the fork data + path so embedded snapshot paths resolve for the fork, then aliasing is + cleaned up. +- Network override fields are supplied at snapshot load to bind the fork to its + own TAP device. +- Vsock CID remains stable for snapshot-based flows. + +## VZ (Virtualization.framework) + +- Fork is not supported. +- Snapshot restore for Linux guests is not available in this mode, so standby + snapshot-based fork mechanics cannot be implemented. + +## Operational constraints + +- Writable attached volumes are rejected for fork to prevent concurrent + cross-VM writes to the same backing data. +- If a post-fork target-state transition fails, the partially created fork is + cleaned up rather than left orphaned. diff --git a/lib/forkvm/copy.go b/lib/forkvm/copy.go new file mode 100644 index 00000000..b942f8c0 --- /dev/null +++ b/lib/forkvm/copy.go @@ -0,0 +1,99 @@ +package forkvm + +import ( + "fmt" + "io" + "io/fs" + "os" + "path/filepath" +) + +// CopyGuestDirectory recursively copies a guest directory to a new destination. +// Runtime sockets are skipped because they are host-runtime artifacts. +func CopyGuestDirectory(srcDir, dstDir string) error { + srcInfo, err := os.Stat(srcDir) + if err != nil { + return fmt.Errorf("stat source directory: %w", err) + } + if !srcInfo.IsDir() { + return fmt.Errorf("source path is not a directory: %s", srcDir) + } + + if err := os.MkdirAll(dstDir, srcInfo.Mode().Perm()); err != nil { + return fmt.Errorf("create destination directory: %w", err) + } + + return filepath.WalkDir(srcDir, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + + relPath, err := filepath.Rel(srcDir, path) + if err != nil { + return fmt.Errorf("compute relative path: %w", err) + } + if relPath == "." { + return nil + } + + dstPath := filepath.Join(dstDir, relPath) + info, err := d.Info() + if err != nil { + return fmt.Errorf("stat source entry %s: %w", path, err) + } + + mode := info.Mode() + switch { + case mode.IsDir(): + if err := os.MkdirAll(dstPath, mode.Perm()); err != nil { + return fmt.Errorf("create destination directory %s: %w", dstPath, err) + } + return nil + + case mode.IsRegular(): + if err := copyRegularFile(path, dstPath, mode.Perm()); err != nil { + return fmt.Errorf("copy file %s: %w", path, err) + } + return nil + + case mode&os.ModeSymlink != 0: + target, err := os.Readlink(path) + if err != nil { + return fmt.Errorf("read symlink %s: %w", path, err) + } + if err := os.Symlink(target, dstPath); err != nil { + return fmt.Errorf("create symlink %s: %w", dstPath, err) + } + return nil + + case mode&os.ModeSocket != 0: + // Runtime socket; the forked instance will create its own. + return nil + + default: + return fmt.Errorf("unsupported file type %s (%s)", path, mode.String()) + } + }) +} + +func copyRegularFile(srcPath, dstPath string, perms fs.FileMode) error { + src, err := os.Open(srcPath) + if err != nil { + return err + } + defer src.Close() + + dst, err := os.OpenFile(dstPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, perms) + if err != nil { + return err + } + if _, err := io.Copy(dst, src); err != nil { + _ = dst.Close() + return err + } + if err := dst.Close(); err != nil { + return err + } + + return nil +} diff --git a/lib/forkvm/copy_test.go b/lib/forkvm/copy_test.go new file mode 100644 index 00000000..99e9226f --- /dev/null +++ b/lib/forkvm/copy_test.go @@ -0,0 +1,34 @@ +package forkvm + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCopyGuestDirectory(t *testing.T) { + src := filepath.Join(t.TempDir(), "src") + dst := filepath.Join(t.TempDir(), "dst") + + require.NoError(t, os.MkdirAll(filepath.Join(src, "logs"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(src, "metadata.json"), []byte(`{"id":"abc"}`), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(src, "logs", "app.log"), []byte("hello"), 0644)) + require.NoError(t, os.Symlink("metadata.json", filepath.Join(src, "meta-link"))) + + require.NoError(t, CopyGuestDirectory(src, dst)) + + assert.FileExists(t, filepath.Join(dst, "metadata.json")) + assert.FileExists(t, filepath.Join(dst, "logs", "app.log")) + assert.FileExists(t, filepath.Join(dst, "meta-link")) + + app, err := os.ReadFile(filepath.Join(dst, "logs", "app.log")) + require.NoError(t, err) + assert.Equal(t, "hello", string(app)) + + linkTarget, err := os.Readlink(filepath.Join(dst, "meta-link")) + require.NoError(t, err) + assert.Equal(t, "metadata.json", linkTarget) +} diff --git a/lib/hypervisor/cloudhypervisor/fork.go b/lib/hypervisor/cloudhypervisor/fork.go new file mode 100644 index 00000000..a2bbddbd --- /dev/null +++ b/lib/hypervisor/cloudhypervisor/fork.go @@ -0,0 +1,24 @@ +package cloudhypervisor + +import ( + "context" + + "github.com/kernel/hypeman/lib/hypervisor" +) + +// PrepareFork prepares cloud-hypervisor fork state by rewriting snapshot config +// when a snapshot path is provided. For stopped forks (no snapshot), this is a no-op. +func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareRequest) (hypervisor.ForkPrepareResult, error) { + _ = ctx + if req.SnapshotConfigPath == "" { + return hypervisor.ForkPrepareResult{}, nil + } + + if err := rewriteSnapshotConfigForFork(req.SnapshotConfigPath, req); err != nil { + return hypervisor.ForkPrepareResult{}, err + } + return hypervisor.ForkPrepareResult{ + // CH snapshot restore keeps CID stable; only socket/path-level rewrites are applied. + VsockCIDUpdated: false, + }, nil +} diff --git a/lib/hypervisor/cloudhypervisor/fork_snapshot.go b/lib/hypervisor/cloudhypervisor/fork_snapshot.go new file mode 100644 index 00000000..831c600e --- /dev/null +++ b/lib/hypervisor/cloudhypervisor/fork_snapshot.go @@ -0,0 +1,117 @@ +package cloudhypervisor + +import ( + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/kernel/hypeman/lib/hypervisor" +) + +// rewriteSnapshotConfigForFork rewrites Cloud Hypervisor snapshot config.json for a forked instance. +func rewriteSnapshotConfigForFork(configPath string, req hypervisor.ForkPrepareRequest) error { + data, err := os.ReadFile(configPath) + if err != nil { + return fmt.Errorf("read snapshot config: %w", err) + } + + var config map[string]any + if err := json.Unmarshal(data, &config); err != nil { + return fmt.Errorf("unmarshal snapshot config: %w", err) + } + + if req.SourceDataDir != "" && req.TargetDataDir != "" && req.SourceDataDir != req.TargetDataDir { + configAny := rewriteStringValues(config, func(s string) string { + if s == req.SourceDataDir || strings.HasPrefix(s, req.SourceDataDir+"/") { + return req.TargetDataDir + strings.TrimPrefix(s, req.SourceDataDir) + } + return s + }) + config = configAny.(map[string]any) + } + + updateVsockConfig(config, req.VsockCID, req.VsockSocket) + if req.SerialLogPath != "" { + updateSerialConfig(config, req.SerialLogPath) + } + if req.Network != nil { + updateNetworkConfig(config, req.Network) + } + + updated, err := json.MarshalIndent(config, "", " ") + if err != nil { + return fmt.Errorf("marshal snapshot config: %w", err) + } + + if err := os.WriteFile(configPath, updated, 0644); err != nil { + return fmt.Errorf("write snapshot config: %w", err) + } + + return nil +} + +func rewriteStringValues(value any, mapper func(string) string) any { + switch v := value.(type) { + case map[string]any: + out := make(map[string]any, len(v)) + for k, child := range v { + out[k] = rewriteStringValues(child, mapper) + } + return out + case []any: + out := make([]any, 0, len(v)) + for _, child := range v { + out = append(out, rewriteStringValues(child, mapper)) + } + return out + case string: + return mapper(v) + default: + return value + } +} + +func updateVsockConfig(config map[string]any, cid int64, socketPath string) { + _ = cid // Keep snapshot CID stable for CH restores; only rewrite socket path. + vsock, ok := config["vsock"].(map[string]any) + if !ok || vsock == nil { + return + } + if socketPath != "" { + vsock["socket"] = socketPath + } +} + +func updateSerialConfig(config map[string]any, logPath string) { + serial, ok := config["serial"].(map[string]any) + if !ok || serial == nil { + return + } + serial["file"] = logPath +} + +func updateNetworkConfig(config map[string]any, netCfg *hypervisor.ForkNetworkConfig) { + nets, ok := config["net"].([]any) + if !ok { + return + } + for _, netAny := range nets { + netMap, ok := netAny.(map[string]any) + if !ok || netMap == nil { + continue + } + if netCfg.TAPDevice != "" { + netMap["tap"] = netCfg.TAPDevice + } + if netCfg.IP != "" { + netMap["ip"] = netCfg.IP + } + if netCfg.MAC != "" { + netMap["mac"] = netCfg.MAC + } + if netCfg.Netmask != "" { + netMap["mask"] = netCfg.Netmask + } + } +} diff --git a/lib/hypervisor/cloudhypervisor/fork_snapshot_test.go b/lib/hypervisor/cloudhypervisor/fork_snapshot_test.go new file mode 100644 index 00000000..b25b1034 --- /dev/null +++ b/lib/hypervisor/cloudhypervisor/fork_snapshot_test.go @@ -0,0 +1,76 @@ +package cloudhypervisor + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRewriteSnapshotConfigForFork(t *testing.T) { + tmp := t.TempDir() + configPath := filepath.Join(tmp, "config.json") + + orig := map[string]any{ + "disks": []any{map[string]any{"path": "/src/guests/a/overlay.raw"}}, + "serial": map[string]any{"file": "/src/guests/a/logs/app.log"}, + "vsock": map[string]any{"cid": float64(100), "socket": "/src/guests/a/vsock.sock"}, + "metadata": map[string]any{ + "note": "keep-/src/guests/a-as-substring", + }, + "net": []any{map[string]any{ + "tap": "hype-old", + "ip": "10.0.0.10", + "mac": "02:00:00:00:00:01", + "mask": "255.255.255.0", + }}, + } + data, err := json.Marshal(orig) + require.NoError(t, err) + require.NoError(t, os.WriteFile(configPath, data, 0644)) + + err = rewriteSnapshotConfigForFork(configPath, hypervisor.ForkPrepareRequest{ + SourceDataDir: "/src/guests/a", + TargetDataDir: "/dst/guests/b", + VsockCID: 200, + VsockSocket: "/dst/guests/b/vsock.sock", + SerialLogPath: "/dst/guests/b/logs/app.log", + Network: &hypervisor.ForkNetworkConfig{ + TAPDevice: "hype-new", + IP: "10.0.0.20", + MAC: "02:00:00:00:00:02", + Netmask: "255.255.255.0", + }, + }) + require.NoError(t, err) + + updatedData, err := os.ReadFile(configPath) + require.NoError(t, err) + + var updated map[string]any + require.NoError(t, json.Unmarshal(updatedData, &updated)) + + disks := updated["disks"].([]any) + disk0 := disks[0].(map[string]any) + assert.Equal(t, "/dst/guests/b/overlay.raw", disk0["path"]) + + serial := updated["serial"].(map[string]any) + assert.Equal(t, "/dst/guests/b/logs/app.log", serial["file"]) + + vsock := updated["vsock"].(map[string]any) + assert.Equal(t, float64(100), vsock["cid"]) + assert.Equal(t, "/dst/guests/b/vsock.sock", vsock["socket"]) + + netCfg := updated["net"].([]any)[0].(map[string]any) + assert.Equal(t, "hype-new", netCfg["tap"]) + assert.Equal(t, "10.0.0.20", netCfg["ip"]) + assert.Equal(t, "02:00:00:00:00:02", netCfg["mac"]) + assert.Equal(t, "255.255.255.0", netCfg["mask"]) + + metadata := updated["metadata"].(map[string]any) + assert.Equal(t, "keep-/src/guests/a-as-substring", metadata["note"]) +} diff --git a/lib/hypervisor/firecracker/config.go b/lib/hypervisor/firecracker/config.go index 0491ed88..b9b60cb0 100644 --- a/lib/hypervisor/firecracker/config.go +++ b/lib/hypervisor/firecracker/config.go @@ -94,7 +94,8 @@ type instanceInfo struct { } type restoreMetadata struct { - NetworkOverrides []networkOverride `json:"network_overrides,omitempty"` + NetworkOverrides []networkOverride `json:"network_overrides,omitempty"` + SnapshotSourceDataDir string `json:"snapshot_source_data_dir,omitempty"` } func toBootSource(cfg hypervisor.VMConfig) bootSource { @@ -216,6 +217,10 @@ func saveRestoreMetadata(instanceDir string, networkConfigs []networkInterface) }) } + return saveRestoreMetadataState(instanceDir, &meta) +} + +func saveRestoreMetadataState(instanceDir string, meta *restoreMetadata) error { data, err := json.MarshalIndent(meta, "", " ") if err != nil { return fmt.Errorf("marshal firecracker restore metadata: %w", err) diff --git a/lib/hypervisor/firecracker/fork.go b/lib/hypervisor/firecracker/fork.go new file mode 100644 index 00000000..81936929 --- /dev/null +++ b/lib/hypervisor/firecracker/fork.go @@ -0,0 +1,64 @@ +package firecracker + +import ( + "context" + "path/filepath" + + "github.com/kernel/hypeman/lib/hypervisor" +) + +// PrepareFork updates Firecracker restore metadata for forked snapshots. +// Firecracker snapshot restore supports network overrides, but does not expose +// a public API for rewriting other snapshotted device paths. +// For standby/running forks, we persist source/target directory mapping so +// RestoreVM can temporarily alias source paths during snapshot load. +func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareRequest) (hypervisor.ForkPrepareResult, error) { + _ = ctx + if req.SnapshotConfigPath == "" { + return hypervisor.ForkPrepareResult{}, nil + } + + instanceDir := req.TargetDataDir + if instanceDir == "" { + // .../snapshots/snapshot-latest/config.json -> .../ + snapshotDir := filepath.Dir(req.SnapshotConfigPath) + instanceDir = filepath.Dir(filepath.Dir(snapshotDir)) + } + + meta, err := loadRestoreMetadata(instanceDir) + if err != nil { + return hypervisor.ForkPrepareResult{}, err + } + + changed := false + if req.Network != nil && req.Network.TAPDevice != "" { + if len(meta.NetworkOverrides) == 0 { + meta.NetworkOverrides = []networkOverride{{ + IfaceID: "eth0", + HostDevName: req.Network.TAPDevice, + }} + changed = true + } else { + for i := range meta.NetworkOverrides { + if meta.NetworkOverrides[i].HostDevName != req.Network.TAPDevice { + meta.NetworkOverrides[i].HostDevName = req.Network.TAPDevice + changed = true + } + } + } + } + if req.SourceDataDir != "" && req.TargetDataDir != "" && req.SourceDataDir != req.TargetDataDir { + if meta.SnapshotSourceDataDir != req.SourceDataDir { + meta.SnapshotSourceDataDir = req.SourceDataDir + changed = true + } + } + + if changed { + if err := saveRestoreMetadataState(instanceDir, meta); err != nil { + return hypervisor.ForkPrepareResult{}, err + } + } + + return hypervisor.ForkPrepareResult{}, nil +} diff --git a/lib/hypervisor/firecracker/fork_test.go b/lib/hypervisor/firecracker/fork_test.go new file mode 100644 index 00000000..52826d92 --- /dev/null +++ b/lib/hypervisor/firecracker/fork_test.go @@ -0,0 +1,43 @@ +package firecracker + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPrepareFork_NoSnapshotPathIsSupported(t *testing.T) { + starter := NewStarter() + result, err := starter.PrepareFork(context.Background(), hypervisor.ForkPrepareRequest{}) + require.NoError(t, err) + assert.False(t, result.VsockCIDUpdated) +} + +func TestPrepareFork_SnapshotRewritePersistsRestoreMetadata(t *testing.T) { + starter := NewStarter() + tmp := t.TempDir() + targetDir := filepath.Join(tmp, "target") + require.NoError(t, os.MkdirAll(targetDir, 0755)) + require.NoError(t, saveRestoreMetadata(targetDir, []networkInterface{{IfaceID: "eth0", HostDevName: "tap-old"}})) + + _, err := starter.PrepareFork(context.Background(), hypervisor.ForkPrepareRequest{ + SnapshotConfigPath: filepath.Join(targetDir, "snapshots", "snapshot-latest", "config.json"), + SourceDataDir: filepath.Join(tmp, "source"), + TargetDataDir: targetDir, + Network: &hypervisor.ForkNetworkConfig{ + TAPDevice: "tap-new", + }, + }) + require.NoError(t, err) + + meta, err := loadRestoreMetadata(targetDir) + require.NoError(t, err) + require.Len(t, meta.NetworkOverrides, 1) + assert.Equal(t, "tap-new", meta.NetworkOverrides[0].HostDevName) + assert.Equal(t, filepath.Join(tmp, "source"), meta.SnapshotSourceDataDir) +} diff --git a/lib/hypervisor/firecracker/process.go b/lib/hypervisor/firecracker/process.go index de7f5cea..14dd52c9 100644 --- a/lib/hypervisor/firecracker/process.go +++ b/lib/hypervisor/firecracker/process.go @@ -7,6 +7,8 @@ import ( "os" "os/exec" "path/filepath" + "strings" + "sync" "syscall" "time" @@ -37,6 +39,8 @@ func NewStarter() *Starter { var _ hypervisor.VMStarter = (*Starter)(nil) +var snapshotSourceAliasMu sync.Mutex + func (s *Starter) SocketName() string { return "fc.sock" } @@ -106,14 +110,91 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, if err != nil { return 0, nil, fmt.Errorf("load firecracker restore metadata: %w", err) } - if err := hv.loadSnapshot(ctx, snapshotPath, meta.NetworkOverrides); err != nil { + err = func() error { + snapshotSourceAliasMu.Lock() + defer snapshotSourceAliasMu.Unlock() + return withSnapshotSourceDirAlias(meta, filepath.Dir(socketPath), func() error { + return hv.loadSnapshot(ctx, snapshotPath, meta.NetworkOverrides) + }) + }() + if err != nil { return 0, nil, fmt.Errorf("load firecracker snapshot: %w", err) } + if meta.SnapshotSourceDataDir != "" { + meta.SnapshotSourceDataDir = "" + if err := saveRestoreMetadataState(filepath.Dir(socketPath), meta); err != nil { + return 0, nil, fmt.Errorf("clear firecracker snapshot source alias metadata: %w", err) + } + } cu.Release() return pid, hv, nil } +func withSnapshotSourceDirAlias(meta *restoreMetadata, targetDataDir string, run func() error) error { + if meta == nil || meta.SnapshotSourceDataDir == "" { + return run() + } + + sourceDataDir := filepath.Clean(meta.SnapshotSourceDataDir) + targetDataDir = filepath.Clean(targetDataDir) + if sourceDataDir == targetDataDir { + return run() + } + if rel, err := filepath.Rel(sourceDataDir, targetDataDir); err == nil && rel != "." && !strings.HasPrefix(rel, "..") { + return fmt.Errorf("invalid snapshot source alias: target data dir %q must not be nested under source data dir %q", targetDataDir, sourceDataDir) + } + if rel, err := filepath.Rel(targetDataDir, sourceDataDir); err == nil && rel != "." && !strings.HasPrefix(rel, "..") { + return fmt.Errorf("invalid snapshot source alias: source data dir %q must not be nested under target data dir %q", sourceDataDir, targetDataDir) + } + + if err := os.MkdirAll(filepath.Dir(sourceDataDir), 0755); err != nil { + return fmt.Errorf("ensure snapshot source parent dir: %w", err) + } + + backupDataDir := "" + if _, err := os.Stat(sourceDataDir); err == nil { + backupDataDir = fmt.Sprintf("%s.fork-bak.%d", sourceDataDir, time.Now().UnixNano()) + if err := os.Rename(sourceDataDir, backupDataDir); err != nil { + return fmt.Errorf("backup snapshot source data dir: %w", err) + } + } else if !os.IsNotExist(err) { + return fmt.Errorf("stat snapshot source data dir: %w", err) + } + + if err := os.Symlink(targetDataDir, sourceDataDir); err != nil { + if backupDataDir != "" { + _ = os.Rename(backupDataDir, sourceDataDir) + } + return fmt.Errorf("symlink snapshot source data dir: %w", err) + } + + runErr := run() + + removeErr := os.Remove(sourceDataDir) + restoreErr := error(nil) + if backupDataDir != "" { + restoreErr = os.Rename(backupDataDir, sourceDataDir) + } + + if runErr != nil { + if removeErr != nil { + return fmt.Errorf("%v; cleanup snapshot source symlink: %w", runErr, removeErr) + } + if restoreErr != nil { + return fmt.Errorf("%v; restore snapshot source data dir: %w", runErr, restoreErr) + } + return runErr + } + if removeErr != nil { + return fmt.Errorf("cleanup snapshot source symlink: %w", removeErr) + } + if restoreErr != nil { + return fmt.Errorf("restore snapshot source data dir: %w", restoreErr) + } + return nil +} + func (s *Starter) startProcess(_ context.Context, p *paths.Paths, version string, socketPath string) (int, error) { binaryPath, err := s.GetBinaryPath(p, version) if err != nil { diff --git a/lib/hypervisor/firecracker/process_test.go b/lib/hypervisor/firecracker/process_test.go new file mode 100644 index 00000000..df99180e --- /dev/null +++ b/lib/hypervisor/firecracker/process_test.go @@ -0,0 +1,80 @@ +package firecracker + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWithSnapshotSourceDirAlias_RestoresSourceDirOnSuccess(t *testing.T) { + tmp := t.TempDir() + sourceDir := filepath.Join(tmp, "source") + targetDir := filepath.Join(tmp, "target") + + require.NoError(t, os.MkdirAll(sourceDir, 0755)) + require.NoError(t, os.MkdirAll(targetDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "source-marker"), []byte("source"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(targetDir, "target-marker"), []byte("target"), 0644)) + + err := withSnapshotSourceDirAlias(&restoreMetadata{SnapshotSourceDataDir: sourceDir}, targetDir, func() error { + linkTarget, err := os.Readlink(sourceDir) + require.NoError(t, err) + assert.Equal(t, targetDir, linkTarget) + return os.WriteFile(filepath.Join(sourceDir, "via-source-path"), []byte("ok"), 0644) + }) + require.NoError(t, err) + + info, err := os.Lstat(sourceDir) + require.NoError(t, err) + assert.False(t, info.Mode()&os.ModeSymlink != 0, "source dir should be restored, not remain a symlink") + + _, err = os.Stat(filepath.Join(sourceDir, "source-marker")) + require.NoError(t, err) + + data, err := os.ReadFile(filepath.Join(targetDir, "via-source-path")) + require.NoError(t, err) + assert.Equal(t, "ok", string(data)) +} + +func TestWithSnapshotSourceDirAlias_RestoresSourceDirOnRunError(t *testing.T) { + tmp := t.TempDir() + sourceDir := filepath.Join(tmp, "source") + targetDir := filepath.Join(tmp, "target") + + require.NoError(t, os.MkdirAll(sourceDir, 0755)) + require.NoError(t, os.MkdirAll(targetDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "source-marker"), []byte("source"), 0644)) + + expectedErr := errors.New("boom") + err := withSnapshotSourceDirAlias(&restoreMetadata{SnapshotSourceDataDir: sourceDir}, targetDir, func() error { + return expectedErr + }) + require.Error(t, err) + assert.ErrorIs(t, err, expectedErr) + + info, statErr := os.Lstat(sourceDir) + require.NoError(t, statErr) + assert.False(t, info.Mode()&os.ModeSymlink != 0, "source dir should be restored, not remain a symlink") + + _, statErr = os.Stat(filepath.Join(sourceDir, "source-marker")) + require.NoError(t, statErr) +} + +func TestWithSnapshotSourceDirAlias_RejectsNestedPaths(t *testing.T) { + tmp := t.TempDir() + sourceDir := filepath.Join(tmp, "source") + targetDir := filepath.Join(sourceDir, "fork") + + require.NoError(t, os.MkdirAll(sourceDir, 0755)) + require.NoError(t, os.MkdirAll(targetDir, 0755)) + + err := withSnapshotSourceDirAlias(&restoreMetadata{SnapshotSourceDataDir: sourceDir}, targetDir, func() error { + return nil + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "must not be nested") +} diff --git a/lib/hypervisor/hypervisor.go b/lib/hypervisor/hypervisor.go index bb1603f4..0ea2fb8e 100644 --- a/lib/hypervisor/hypervisor.go +++ b/lib/hypervisor/hypervisor.go @@ -83,6 +83,43 @@ type VMStarter interface { // - QEMU: would start with -incoming or -loadvm flags (not yet implemented) // Returns the process ID and a Hypervisor client. The VM is in paused state after restore. RestoreVM(ctx context.Context, p *paths.Paths, version string, socketPath string, snapshotPath string) (pid int, hv Hypervisor, err error) + + // PrepareFork allows hypervisors to prepare forked instance state. + // For snapshot-based forks, implementations can rewrite snapshot config with + // fork identity (paths, vsock, network). Hypervisors that don't support fork + // should return ErrNotSupported. + PrepareFork(ctx context.Context, req ForkPrepareRequest) (ForkPrepareResult, error) +} + +// ForkNetworkConfig contains network identity fields for fork preparation. +type ForkNetworkConfig struct { + TAPDevice string + IP string + MAC string + Netmask string +} + +// ForkPrepareRequest contains hypervisor-specific fork preparation inputs. +type ForkPrepareRequest struct { + // SnapshotConfigPath is optional. When empty, implementations should only + // validate fork support and return without snapshot rewrites. + SnapshotConfigPath string + + SourceDataDir string + TargetDataDir string + + VsockCID int64 + VsockSocket string + + SerialLogPath string + Network *ForkNetworkConfig +} + +// ForkPrepareResult describes which optional fork rewrites were actually applied. +type ForkPrepareResult struct { + // VsockCIDUpdated indicates whether snapshot state was updated to use + // ForkPrepareRequest.VsockCID. + VsockCIDUpdated bool } // Hypervisor defines the interface for VM control operations. diff --git a/lib/hypervisor/qemu/fork.go b/lib/hypervisor/qemu/fork.go new file mode 100644 index 00000000..b7f8361f --- /dev/null +++ b/lib/hypervisor/qemu/fork.go @@ -0,0 +1,89 @@ +package qemu + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/kernel/hypeman/lib/hypervisor" +) + +// PrepareFork prepares QEMU fork state by rewriting snapshot VM config when a +// snapshot path is provided. For stopped forks (no snapshot), this is a no-op. +func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareRequest) (hypervisor.ForkPrepareResult, error) { + _ = ctx + if req.SnapshotConfigPath == "" { + return hypervisor.ForkPrepareResult{}, nil + } + + snapshotDir := filepath.Dir(req.SnapshotConfigPath) + cfg, err := loadVMConfig(snapshotDir) + if err != nil { + // The generic path points to CH's config.json; for QEMU, require qemu-config.json. + expectedPath := filepath.Join(snapshotDir, vmConfigFile) + if _, statErr := os.Stat(expectedPath); statErr != nil { + return hypervisor.ForkPrepareResult{}, fmt.Errorf("load qemu snapshot config %q: %w", expectedPath, err) + } + return hypervisor.ForkPrepareResult{}, fmt.Errorf("load qemu snapshot config: %w", err) + } + + if req.SourceDataDir != "" && req.TargetDataDir != "" && req.SourceDataDir != req.TargetDataDir { + cfg = rewriteQEMUConfigPaths(cfg, req.SourceDataDir, req.TargetDataDir) + } + + if req.VsockCID > 0 { + cfg.VsockCID = req.VsockCID + } + if req.VsockSocket != "" { + cfg.VsockSocket = req.VsockSocket + } + if req.SerialLogPath != "" { + cfg.SerialLogPath = req.SerialLogPath + } + + if req.Network != nil { + for i := range cfg.Networks { + if req.Network.TAPDevice != "" { + cfg.Networks[i].TAPDevice = req.Network.TAPDevice + } + if req.Network.MAC != "" { + cfg.Networks[i].MAC = req.Network.MAC + } + if req.Network.IP != "" { + cfg.Networks[i].IP = req.Network.IP + } + if req.Network.Netmask != "" { + cfg.Networks[i].Netmask = req.Network.Netmask + } + } + } + + if err := saveVMConfig(snapshotDir, cfg); err != nil { + return hypervisor.ForkPrepareResult{}, fmt.Errorf("write qemu snapshot config: %w", err) + } + return hypervisor.ForkPrepareResult{ + VsockCIDUpdated: req.VsockCID > 0, + }, nil +} + +func rewriteQEMUConfigPaths(cfg hypervisor.VMConfig, sourceDir, targetDir string) hypervisor.VMConfig { + replace := func(value string) string { + if value == sourceDir || strings.HasPrefix(value, sourceDir+"/") { + return targetDir + strings.TrimPrefix(value, sourceDir) + } + return value + } + + for i := range cfg.Disks { + cfg.Disks[i].Path = replace(cfg.Disks[i].Path) + } + + cfg.SerialLogPath = replace(cfg.SerialLogPath) + cfg.VsockSocket = replace(cfg.VsockSocket) + cfg.KernelPath = replace(cfg.KernelPath) + cfg.InitrdPath = replace(cfg.InitrdPath) + + return cfg +} diff --git a/lib/hypervisor/qemu/fork_test.go b/lib/hypervisor/qemu/fork_test.go new file mode 100644 index 00000000..f8bc721b --- /dev/null +++ b/lib/hypervisor/qemu/fork_test.go @@ -0,0 +1,83 @@ +package qemu + +import ( + "context" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPrepareFork_NoSnapshotPathIsNoOp(t *testing.T) { + starter := NewStarter() + result, err := starter.PrepareFork(context.Background(), hypervisor.ForkPrepareRequest{}) + require.NoError(t, err) + assert.False(t, result.VsockCIDUpdated) +} + +func TestPrepareFork_RewritesSnapshotConfig(t *testing.T) { + starter := NewStarter() + snapshotDir := t.TempDir() + + sourceDir := "/src/guest" + targetDir := "/dst/guest" + initial := hypervisor.VMConfig{ + VCPUs: 2, + MemoryBytes: 2 * 1024 * 1024 * 1024, + SerialLogPath: sourceDir + "/logs/app.log", + VsockCID: 12345, + VsockSocket: sourceDir + "/vsock/vsock.sock", + KernelPath: sourceDir + "/kernel/vmlinuz", + InitrdPath: sourceDir + "/kernel/initrd", + KernelArgs: "console=ttyS0 root=" + sourceDir + "/rootfs note=keep-" + sourceDir + "-as-substring", + Disks: []hypervisor.DiskConfig{ + {Path: sourceDir + "/overlay.raw"}, + {Path: "/volumes/volume-data.raw"}, + }, + Networks: []hypervisor.NetworkConfig{ + { + TAPDevice: "hype-oldtap", + IP: "10.100.10.10", + MAC: "02:00:00:aa:bb:cc", + Netmask: "255.255.0.0", + }, + }, + } + require.NoError(t, saveVMConfig(snapshotDir, initial)) + + result, err := starter.PrepareFork(context.Background(), hypervisor.ForkPrepareRequest{ + SnapshotConfigPath: filepath.Join(snapshotDir, "config.json"), + SourceDataDir: sourceDir, + TargetDataDir: targetDir, + VsockCID: 54321, + VsockSocket: targetDir + "/vsock/fork-vsock.sock", + SerialLogPath: targetDir + "/logs/fork-app.log", + Network: &hypervisor.ForkNetworkConfig{ + TAPDevice: "hype-newtap", + IP: "10.100.20.20", + MAC: "02:00:00:dd:ee:ff", + Netmask: "255.255.0.0", + }, + }) + require.NoError(t, err) + assert.True(t, result.VsockCIDUpdated) + + updated, err := loadVMConfig(snapshotDir) + require.NoError(t, err) + + assert.Equal(t, int64(54321), updated.VsockCID) + assert.Equal(t, targetDir+"/vsock/fork-vsock.sock", updated.VsockSocket) + assert.Equal(t, targetDir+"/logs/fork-app.log", updated.SerialLogPath) + assert.Equal(t, targetDir+"/kernel/vmlinuz", updated.KernelPath) + assert.Equal(t, targetDir+"/kernel/initrd", updated.InitrdPath) + assert.Equal(t, initial.KernelArgs, updated.KernelArgs) + assert.Equal(t, targetDir+"/overlay.raw", updated.Disks[0].Path) + assert.Equal(t, "/volumes/volume-data.raw", updated.Disks[1].Path, "non-instance paths should remain unchanged") + require.Len(t, updated.Networks, 1) + assert.Equal(t, "hype-newtap", updated.Networks[0].TAPDevice) + assert.Equal(t, "10.100.20.20", updated.Networks[0].IP) + assert.Equal(t, "02:00:00:dd:ee:ff", updated.Networks[0].MAC) + assert.Equal(t, "255.255.0.0", updated.Networks[0].Netmask) +} diff --git a/lib/hypervisor/vz/starter.go b/lib/hypervisor/vz/starter.go index e80260b0..cc8daf1b 100644 --- a/lib/hypervisor/vz/starter.go +++ b/lib/hypervisor/vz/starter.go @@ -217,6 +217,13 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, return 0, nil, hypervisor.ErrNotSupported } +// PrepareFork is not supported for vz. +func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareRequest) (hypervisor.ForkPrepareResult, error) { + _ = ctx + _ = req + return hypervisor.ForkPrepareResult{}, hypervisor.ErrNotSupported +} + func (s *Starter) waitForShim(ctx context.Context, socketPath string, timeout time.Duration) (*Client, error) { deadline := time.Now().Add(timeout) diff --git a/lib/instances/create.go b/lib/instances/create.go index ee026ab4..31c78574 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "path/filepath" - "regexp" "strings" "time" @@ -450,17 +449,8 @@ func (m *manager) createInstance( // validateCreateRequest validates the create instance request func validateCreateRequest(req CreateInstanceRequest) error { - if req.Name == "" { - return fmt.Errorf("name is required") - } - // Validate name format: lowercase letters, digits, dashes only - // No starting/ending with dashes, max 63 characters - if len(req.Name) > 63 { - return fmt.Errorf("name must be 63 characters or less") - } - namePattern := regexp.MustCompile(`^[a-z0-9]([a-z0-9-]*[a-z0-9])?$`) - if !namePattern.MatchString(req.Name) { - return fmt.Errorf("name must contain only lowercase letters, digits, and dashes; cannot start or end with a dash") + if err := validateInstanceName(req.Name); err != nil { + return err } if req.Image == "" { return fmt.Errorf("image is required") diff --git a/lib/instances/errors.go b/lib/instances/errors.go index 9925bb02..ef30ca4c 100644 --- a/lib/instances/errors.go +++ b/lib/instances/errors.go @@ -9,6 +9,9 @@ var ( // ErrInvalidState is returned when a state transition is not valid ErrInvalidState = errors.New("invalid state transition") + // ErrInvalidRequest is returned when request validation fails + ErrInvalidRequest = errors.New("invalid request") + // ErrAlreadyExists is returned when creating an instance that already exists ErrAlreadyExists = errors.New("instance already exists") @@ -20,4 +23,7 @@ var ( // ErrInsufficientResources is returned when resources (CPU, memory, network, GPU) are not available ErrInsufficientResources = errors.New("insufficient resources") + + // ErrNotSupported is returned when an operation is not supported for the instance hypervisor + ErrNotSupported = errors.New("operation not supported") ) diff --git a/lib/instances/firecracker_test.go b/lib/instances/firecracker_test.go index 8b1ede30..9579ea68 100644 --- a/lib/instances/firecracker_test.go +++ b/lib/instances/firecracker_test.go @@ -76,7 +76,7 @@ func createNginxImageAndWait(t *testing.T, ctx context.Context, imageManager ima }) require.NoError(t, err) - for i := 0; i < 60; i++ { + for i := 0; i < 180; i++ { img, err := imageManager.GetImage(ctx, nginxImage.Name) if err == nil && img.Status == images.StatusReady { return @@ -316,3 +316,68 @@ func TestFirecrackerNetworkLifecycle(t *testing.T) { _, err = mgr.networkManager.GetAllocation(ctx, inst.Id) require.Error(t, err, "network allocation should be removed on delete") } + +func TestFirecrackerForkFromRunningNetwork(t *testing.T) { + requireFirecrackerIntegrationPrereqs(t) + + mgr, tmpDir := setupTestManagerForFirecracker(t) + ctx := context.Background() + p := paths.New(tmpDir) + + imageManager, err := images.NewManager(p, 1, nil) + require.NoError(t, err) + createNginxImageAndWait(t, ctx, imageManager) + + systemManager := system.NewManager(p) + require.NoError(t, systemManager.EnsureSystemFiles(ctx)) + require.NoError(t, mgr.networkManager.Initialize(ctx, nil)) + + source, err := mgr.CreateInstance(ctx, CreateInstanceRequest{ + Name: "fc-fork-running-src", + Image: "docker.io/library/nginx:alpine", + Size: 2 * 1024 * 1024 * 1024, + HotplugSize: 256 * 1024 * 1024, + OverlaySize: 10 * 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: true, + Hypervisor: hypervisor.TypeFirecracker, + }) + require.NoError(t, err) + sourceID := source.Id + t.Cleanup(func() { _ = mgr.DeleteInstance(context.Background(), sourceID) }) + assert.NotEmpty(t, source.IP) + assert.NotEmpty(t, source.MAC) + + _, err = mgr.ForkInstance(ctx, sourceID, ForkInstanceRequest{Name: "fc-fork-running-no-flag"}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidState) + + forked, err := mgr.ForkInstance(ctx, sourceID, ForkInstanceRequest{ + Name: "fc-fork-running-copy", + FromRunning: true, + TargetState: StateRunning, + }) + require.NoError(t, err) + require.Equal(t, StateRunning, forked.State) + forkID := forked.Id + t.Cleanup(func() { _ = mgr.DeleteInstance(context.Background(), forkID) }) + assert.NotEmpty(t, forked.IP) + assert.NotEmpty(t, forked.MAC) + assert.Equal(t, mgr.paths.InstanceVsockSocket(forkID), forked.VsockSocket) + + forkMeta, err := mgr.loadMetadata(forkID) + require.NoError(t, err) + assert.Equal(t, mgr.paths.InstanceVsockSocket(forkID), forkMeta.StoredMetadata.VsockSocket) + + sourceAfterFork, err := mgr.GetInstance(ctx, sourceID) + require.NoError(t, err) + require.Equal(t, StateRunning, sourceAfterFork.State) + assert.NotEmpty(t, sourceAfterFork.IP) + assert.NotEmpty(t, sourceAfterFork.MAC) + + assertHostCanReachNginx(t, sourceAfterFork.IP, 80, 60*time.Second) + assertHostCanReachNginx(t, forked.IP, 80, 60*time.Second) + assertHostCanReachNginx(t, sourceAfterFork.IP, 80, 60*time.Second) + assert.NotEqual(t, sourceAfterFork.IP, forked.IP) + assert.NotEqual(t, sourceAfterFork.MAC, forked.MAC) +} diff --git a/lib/instances/fork.go b/lib/instances/fork.go new file mode 100644 index 00000000..ab619af7 --- /dev/null +++ b/lib/instances/fork.go @@ -0,0 +1,487 @@ +package instances + +import ( + "bytes" + "context" + "errors" + "fmt" + "hash/crc32" + "os" + "path/filepath" + "strings" + "time" + + "github.com/kernel/hypeman/lib/forkvm" + "github.com/kernel/hypeman/lib/guest" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/logger" + "github.com/kernel/hypeman/lib/network" + "github.com/nrednav/cuid2" + "gvisor.dev/gvisor/pkg/cleanup" +) + +// forkInstance creates a new instance by cloning a stopped or standby source +// instance. It returns the newly created fork and the requested final target +// state; callers apply remaining target state transitions outside the source lock. +func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceRequest) (*Instance, State, error) { + log := logger.FromContext(ctx) + log.InfoContext(ctx, "forking instance", "source_instance_id", id, "fork_name", req.Name) + + if err := validateForkRequest(req); err != nil { + return nil, "", err + } + + meta, err := m.loadMetadata(id) + if err != nil { + return nil, "", err + } + source := m.toInstance(ctx, meta) + targetState, err := resolveForkTargetState(req.TargetState, source.State) + if err != nil { + return nil, "", err + } + + switch source.State { + case StateRunning: + if !req.FromRunning { + return nil, "", fmt.Errorf("%w: cannot fork from state %s (set from_running=true to allow standby+restore flow)", ErrInvalidState, source.State) + } + + if err := m.validateForkSupport(ctx, source.HypervisorType); err != nil { + return nil, "", err + } + if err := ensureGuestAgentReadyForRunningFork(ctx, &source.StoredMetadata); err != nil { + return nil, "", err + } + + log.InfoContext(ctx, "fork from running requested; transitioning source to standby", + "source_instance_id", id, "hypervisor", source.HypervisorType) + if _, err := m.standbyInstance(ctx, id); err != nil { + return nil, "", fmt.Errorf("standby source instance: %w", err) + } + + forked, forkErr := m.forkInstanceFromStoppedOrStandby(ctx, id, req, true) + if forkErr == nil { + if err := m.rotateSourceVsockForRestore(ctx, id, forked.Id); err != nil { + forkErr = fmt.Errorf("prepare source snapshot for restore: %w", err) + if cleanupErr := m.cleanupForkInstanceOnError(ctx, forked.Id); cleanupErr != nil { + forkErr = fmt.Errorf("%v; additionally failed to cleanup forked instance %s: %v", forkErr, forked.Id, cleanupErr) + } + } + } + + // For Firecracker running-source forks, restoring the fork may temporarily alias + // the source data directory. Restore the fork while source remains standby and + // under lock, then restore the source. + if forkErr == nil && targetState == StateRunning { + restoredFork, err := m.applyForkTargetState(ctx, forked.Id, StateRunning) + if err != nil { + forkErr = fmt.Errorf("restore forked instance before source restore: %w", err) + if cleanupErr := m.cleanupForkInstanceOnError(ctx, forked.Id); cleanupErr != nil { + forkErr = fmt.Errorf("%v; additionally failed to cleanup forked instance %s: %v", forkErr, forked.Id, cleanupErr) + } + } else { + forked = restoredFork + } + } + + log.InfoContext(ctx, "restoring source instance after running fork", "source_instance_id", id) + _, restoreErr := m.restoreInstance(ctx, id) + + if restoreErr != nil { + if forkErr != nil { + return nil, "", fmt.Errorf("fork failed: %v; additionally failed to restore source instance: %w", forkErr, restoreErr) + } + return nil, "", fmt.Errorf("restore source instance after fork: %w", restoreErr) + } + if forkErr != nil { + return nil, "", forkErr + } + return forked, targetState, nil + case StateStopped, StateStandby: + forked, err := m.forkInstanceFromStoppedOrStandby(ctx, id, req, false) + if err != nil { + return nil, "", err + } + return forked, targetState, nil + default: + return nil, "", fmt.Errorf("%w: cannot fork from state %s (must be Stopped or Standby, or Running with from_running=true)", ErrInvalidState, source.State) + } +} + +func ensureGuestAgentReadyForRunningFork(ctx context.Context, source *StoredMetadata) error { + if source == nil || !source.NetworkEnabled || source.SkipGuestAgent { + return nil + } + + dialer, err := hypervisor.NewVsockDialer(source.HypervisorType, source.VsockSocket, source.VsockCID) + if err != nil { + return fmt.Errorf("create vsock dialer for running fork readiness check: %w", err) + } + + var stdout, stderr bytes.Buffer + exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ + Command: []string{"true"}, + Stdout: &stdout, + Stderr: &stderr, + WaitForAgent: 120 * time.Second, + }) + if err != nil { + return fmt.Errorf("wait for guest agent readiness before running fork: %w", err) + } + if exit.Code != 0 { + return fmt.Errorf( + "guest agent readiness probe failed before running fork (exit=%d, stdout=%q, stderr=%q)", + exit.Code, strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String()), + ) + } + return nil +} + +func (m *manager) rotateSourceVsockForRestore(ctx context.Context, sourceID, forkID string) error { + meta, err := m.loadMetadata(sourceID) + if err != nil { + return fmt.Errorf("reload source metadata: %w", err) + } + stored := &meta.StoredMetadata + + newCID := generateForkSourceVsockCID(sourceID, forkID, stored.VsockCID) + if newCID == stored.VsockCID { + return nil + } + + starter, err := m.getVMStarter(stored.HypervisorType) + if err != nil { + return fmt.Errorf("get vm starter: %w", err) + } + + prepareResult, err := starter.PrepareFork(ctx, hypervisor.ForkPrepareRequest{ + SnapshotConfigPath: m.paths.InstanceSnapshotConfig(sourceID), + VsockCID: newCID, + VsockSocket: stored.VsockSocket, + }) + if err != nil { + return fmt.Errorf("rewrite source snapshot vsock state: %w", err) + } + + if prepareResult.VsockCIDUpdated { + stored.VsockCID = newCID + if err := m.saveMetadata(meta); err != nil { + return fmt.Errorf("save source metadata: %w", err) + } + } + return nil +} + +func generateForkSourceVsockCID(sourceID, forkID string, current int64) int64 { + const cidRange = int64(4294967292) + seed := crc32.ChecksumIEEE([]byte(sourceID + ":" + forkID)) + cid := (int64(seed) % cidRange) + 3 + if cid == current { + cid = ((cid - 3 + 1) % cidRange) + 3 + } + return cid +} + +func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id string, req ForkInstanceRequest, supportValidated bool) (*Instance, error) { + log := logger.FromContext(ctx) + + meta, err := m.loadMetadata(id) + if err != nil { + return nil, err + } + + source := m.toInstance(ctx, meta) + stored := &meta.StoredMetadata + + switch source.State { + case StateStopped, StateStandby: + // allowed + default: + return nil, fmt.Errorf("%w: cannot fork from state %s (must be Stopped or Standby)", ErrInvalidState, source.State) + } + + if !supportValidated { + if err := m.validateForkSupport(ctx, stored.HypervisorType); err != nil { + return nil, err + } + } + if err := validateForkVolumeSafety(stored.Volumes); err != nil { + return nil, err + } + + existsByMetadata, err := m.instanceNameExists(req.Name) + if err != nil { + return nil, fmt.Errorf("check instance name availability: %w", err) + } + if existsByMetadata { + return nil, fmt.Errorf("%w: instance name '%s' already exists", ErrAlreadyExists, req.Name) + } + if stored.NetworkEnabled { + exists, err := m.networkManager.NameExists(ctx, req.Name, "") + if err != nil { + return nil, fmt.Errorf("check instance name availability: %w", err) + } + if exists { + return nil, fmt.Errorf("%w: instance name '%s' already exists in network", ErrAlreadyExists, req.Name) + } + } + + forkID := cuid2.Generate() + if _, err := m.loadMetadata(forkID); err == nil { + return nil, fmt.Errorf("%w: generated fork id already exists", ErrAlreadyExists) + } + + srcDir := m.paths.InstanceDir(id) + dstDir := m.paths.InstanceDir(forkID) + + cu := cleanup.Make(func() { + _ = os.RemoveAll(dstDir) + }) + defer cu.Clean() + + if err := forkvm.CopyGuestDirectory(srcDir, dstDir); err != nil { + return nil, fmt.Errorf("clone guest directory: %w", err) + } + + starter, err := m.getVMStarter(stored.HypervisorType) + if err != nil { + return nil, fmt.Errorf("get vm starter: %w", err) + } + + now := time.Now() + forkMeta := cloneStoredMetadataForFork(meta.StoredMetadata) + forkMeta.Id = forkID + forkMeta.Name = req.Name + forkMeta.CreatedAt = now + forkMeta.StartedAt = nil + forkMeta.StoppedAt = nil + forkMeta.HypervisorPID = nil + forkMeta.SocketPath = m.paths.InstanceSocket(forkID, starter.SocketName()) + forkMeta.DataDir = dstDir + forkMeta.VsockSocket = m.paths.InstanceVsockSocket(forkID) + forkMeta.ExitCode = nil + forkMeta.ExitMessage = "" + + // Keep the original CID for snapshot-based forks. + // Rewriting CID in restored memory snapshots is not reliable across + // hypervisors. + if source.State == StateStandby { + forkMeta.VsockCID = stored.VsockCID + } else { + forkMeta.VsockCID = generateVsockCID(forkID) + } + + if forkMeta.NetworkEnabled { + // Clear inherited network identity. For stopped instances this is regenerated on start, + // and for standby instances restore allocates if identity is empty. + forkMeta.IP = "" + forkMeta.MAC = "" + } + + if source.State == StateStandby { + snapshotConfigPath := m.paths.InstanceSnapshotConfig(forkID) + netCfg := (*hypervisor.ForkNetworkConfig)(nil) + if forkMeta.NetworkEnabled { + netCfg = &hypervisor.ForkNetworkConfig{TAPDevice: network.GenerateTAPName(forkID)} + } + if _, err := starter.PrepareFork(ctx, hypervisor.ForkPrepareRequest{ + SnapshotConfigPath: snapshotConfigPath, + SourceDataDir: stored.DataDir, + TargetDataDir: forkMeta.DataDir, + VsockCID: forkMeta.VsockCID, + VsockSocket: forkMeta.VsockSocket, + SerialLogPath: m.paths.InstanceAppLog(forkID), + Network: netCfg, + }); err != nil { + if errors.Is(err, hypervisor.ErrNotSupported) { + return nil, fmt.Errorf("%w: fork is not supported for hypervisor %s", ErrNotSupported, stored.HypervisorType) + } + return nil, fmt.Errorf("prepare fork snapshot state: %w", err) + } + } + + newMeta := &metadata{StoredMetadata: forkMeta} + if err := m.saveMetadata(newMeta); err != nil { + return nil, fmt.Errorf("save fork metadata: %w", err) + } + + cu.Release() + forked := m.toInstance(ctx, newMeta) + log.InfoContext(ctx, "instance forked successfully", + "source_instance_id", id, + "fork_instance_id", forked.Id, + "fork_name", forked.Name, + "state", forked.State) + return &forked, nil +} + +func (m *manager) validateForkSupport(ctx context.Context, hvType hypervisor.Type) error { + starter, err := m.getVMStarter(hvType) + if err != nil { + return fmt.Errorf("get vm starter: %w", err) + } + if _, err := starter.PrepareFork(ctx, hypervisor.ForkPrepareRequest{}); err != nil { + if errors.Is(err, hypervisor.ErrNotSupported) { + return fmt.Errorf("%w: fork is not supported for hypervisor %s", ErrNotSupported, hvType) + } + return fmt.Errorf("prepare fork state: %w", err) + } + return nil +} + +func validateForkRequest(req ForkInstanceRequest) error { + if err := validateInstanceName(req.Name); err != nil { + return fmt.Errorf("%w: %v", ErrInvalidRequest, err) + } + if req.TargetState != "" && req.TargetState != StateStopped && req.TargetState != StateStandby && req.TargetState != StateRunning { + return fmt.Errorf("%w: invalid fork target state %q (must be one of %s, %s, %s)", ErrInvalidRequest, req.TargetState, StateStopped, StateStandby, StateRunning) + } + return nil +} + +func validateForkVolumeSafety(volumes []VolumeAttachment) error { + for _, vol := range volumes { + if !vol.Readonly { + return fmt.Errorf("%w: cannot fork instance with writable volume %q mounted at %q; use readonly+overlay for safe concurrent forks", ErrNotSupported, vol.VolumeID, vol.MountPath) + } + } + return nil +} + +func (m *manager) instanceNameExists(name string) (bool, error) { + metaFiles, err := m.listMetadataFiles() + if err != nil { + return false, err + } + + for _, metaFile := range metaFiles { + id := filepath.Base(filepath.Dir(metaFile)) + meta, err := m.loadMetadata(id) + if err != nil { + continue + } + if meta.Name == name { + return true, nil + } + } + return false, nil +} + +func resolveForkTargetState(requested State, sourceState State) (State, error) { + if requested == "" { + switch sourceState { + case StateRunning, StateStandby, StateStopped: + return sourceState, nil + default: + return "", fmt.Errorf("%w: cannot derive fork target state from source state %s", ErrInvalidState, sourceState) + } + } + return requested, nil +} + +func (m *manager) applyForkTargetState(ctx context.Context, forkID string, target State) (*Instance, error) { + lock := m.getInstanceLock(forkID) + lock.Lock() + defer lock.Unlock() + + current, err := m.getInstance(ctx, forkID) + if err != nil { + return nil, err + } + if current.State == target { + return current, nil + } + + switch current.State { + case StateStopped: + switch target { + case StateRunning: + return m.startInstance(ctx, forkID, StartInstanceRequest{}) + case StateStandby: + if _, err := m.startInstance(ctx, forkID, StartInstanceRequest{}); err != nil { + return nil, fmt.Errorf("start forked instance for standby transition: %w", err) + } + return m.standbyInstance(ctx, forkID) + } + case StateStandby: + switch target { + case StateRunning: + return m.restoreInstance(ctx, forkID) + case StateStopped: + if err := os.RemoveAll(m.paths.InstanceSnapshotLatest(forkID)); err != nil { + return nil, fmt.Errorf("remove fork snapshot: %w", err) + } + return m.getInstance(ctx, forkID) + } + case StateRunning: + switch target { + case StateStandby: + return m.standbyInstance(ctx, forkID) + case StateStopped: + return m.stopInstance(ctx, forkID) + } + } + + return nil, fmt.Errorf("%w: cannot transition forked instance from %s to %s", ErrInvalidState, current.State, target) +} + +func (m *manager) cleanupForkInstanceOnError(ctx context.Context, forkID string) error { + lock := m.getInstanceLock(forkID) + lock.Lock() + defer lock.Unlock() + + err := m.deleteInstance(ctx, forkID) + if err == nil || errors.Is(err, ErrNotFound) { + m.instanceLocks.Delete(forkID) + return nil + } + return err +} + +func cloneStoredMetadataForFork(src StoredMetadata) StoredMetadata { + dst := src + + if src.Env != nil { + dst.Env = make(map[string]string, len(src.Env)) + for k, v := range src.Env { + dst.Env[k] = v + } + } + if src.Metadata != nil { + dst.Metadata = make(map[string]string, len(src.Metadata)) + for k, v := range src.Metadata { + dst.Metadata[k] = v + } + } + if src.Volumes != nil { + dst.Volumes = append([]VolumeAttachment(nil), src.Volumes...) + } + if src.Devices != nil { + dst.Devices = append([]string(nil), src.Devices...) + } + if src.Entrypoint != nil { + dst.Entrypoint = append([]string(nil), src.Entrypoint...) + } + if src.Cmd != nil { + dst.Cmd = append([]string(nil), src.Cmd...) + } + if src.HypervisorPID != nil { + pid := *src.HypervisorPID + dst.HypervisorPID = &pid + } + if src.StartedAt != nil { + startedAt := *src.StartedAt + dst.StartedAt = &startedAt + } + if src.StoppedAt != nil { + stoppedAt := *src.StoppedAt + dst.StoppedAt = &stoppedAt + } + if src.ExitCode != nil { + exitCode := *src.ExitCode + dst.ExitCode = &exitCode + } + + return dst +} diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go new file mode 100644 index 00000000..3e036cbf --- /dev/null +++ b/lib/instances/fork_test.go @@ -0,0 +1,496 @@ +package instances + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/kernel/hypeman/lib/guest" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestForkInstanceNotSupportedHypervisor(t *testing.T) { + manager, _ := setupTestManager(t) + ctx := context.Background() + if _, err := manager.getVMStarter(hypervisor.TypeVZ); err != nil { + t.Skip("vz starter not available on this platform") + } + + sourceID := "fork-vz-source" + require.NoError(t, manager.ensureDirectories(sourceID)) + + meta := &metadata{StoredMetadata: StoredMetadata{ + Id: sourceID, + Name: "fork-vz-source", + Image: "docker.io/library/alpine:latest", + CreatedAt: time.Now(), + HypervisorType: hypervisor.TypeVZ, + HypervisorVersion: "test", + SocketPath: paths.New(manager.paths.DataDir()).InstanceSocket(sourceID, "vz.sock"), + DataDir: paths.New(manager.paths.DataDir()).InstanceDir(sourceID), + VsockCID: 42, + VsockSocket: paths.New(manager.paths.DataDir()).InstanceVsockSocket(sourceID), + }} + require.NoError(t, manager.saveMetadata(meta)) + + _, err := manager.ForkInstance(ctx, sourceID, ForkInstanceRequest{Name: "fork-vz-copy"}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrNotSupported) +} + +func TestResolveForkTargetState_DefaultsToSourceState(t *testing.T) { + tests := []struct { + name string + source State + want State + }{ + {name: "running", source: StateRunning, want: StateRunning}, + {name: "standby", source: StateStandby, want: StateStandby}, + {name: "stopped", source: StateStopped, want: StateStopped}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := resolveForkTargetState("", tc.source) + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } +} + +func TestValidateForkRequest_InvalidTargetState(t *testing.T) { + err := validateForkRequest(ForkInstanceRequest{ + Name: "fork-invalid-target", + TargetState: State("Created"), + }) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidRequest) +} + +func TestValidateForkVolumeSafety(t *testing.T) { + err := validateForkVolumeSafety([]VolumeAttachment{ + {VolumeID: "vol-rw", MountPath: "/data", Readonly: false}, + }) + require.Error(t, err) + assert.ErrorIs(t, err, ErrNotSupported) + + err = validateForkVolumeSafety([]VolumeAttachment{ + {VolumeID: "vol-ro", MountPath: "/data", Readonly: true, Overlay: false}, + {VolumeID: "vol-cow", MountPath: "/tmp", Readonly: true, Overlay: true}, + }) + require.NoError(t, err) +} + +func TestCleanupForkInstanceOnError(t *testing.T) { + manager, _ := setupTestManager(t) + ctx := context.Background() + + forkID := "fork-cleanup-target" + require.NoError(t, manager.ensureDirectories(forkID)) + + meta := &metadata{StoredMetadata: StoredMetadata{ + Id: forkID, + Name: "fork-cleanup-target", + Image: "docker.io/library/alpine:latest", + CreatedAt: time.Now(), + HypervisorType: hypervisor.TypeCloudHypervisor, + HypervisorVersion: "test", + SocketPath: paths.New(manager.paths.DataDir()).InstanceSocket(forkID, "cloud-hypervisor.sock"), + DataDir: paths.New(manager.paths.DataDir()).InstanceDir(forkID), + VsockCID: 43, + VsockSocket: paths.New(manager.paths.DataDir()).InstanceVsockSocket(forkID), + }} + require.NoError(t, manager.saveMetadata(meta)) + + require.DirExists(t, meta.DataDir) + require.NoError(t, manager.cleanupForkInstanceOnError(ctx, forkID)) + assert.NoDirExists(t, meta.DataDir) + + _, err := manager.loadMetadata(forkID) + require.Error(t, err) + assert.ErrorIs(t, err, ErrNotFound) +} + +func TestForkInstance_CleansUpOnTargetTransitionError(t *testing.T) { + manager, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "fork-target-transition-source" + require.NoError(t, manager.ensureDirectories(sourceID)) + + now := time.Now() + meta := &metadata{StoredMetadata: StoredMetadata{ + Id: sourceID, + Name: sourceID, + Image: "docker.io/library/nonexistent:latest", + CreatedAt: now, + StoppedAt: &now, + HypervisorType: hypervisor.TypeCloudHypervisor, + HypervisorVersion: "test", + SocketPath: paths.New(manager.paths.DataDir()).InstanceSocket(sourceID, "cloud-hypervisor.sock"), + DataDir: paths.New(manager.paths.DataDir()).InstanceDir(sourceID), + VsockCID: 42, + VsockSocket: paths.New(manager.paths.DataDir()).InstanceVsockSocket(sourceID), + }} + require.NoError(t, manager.saveMetadata(meta)) + + _, err := manager.ForkInstance(ctx, sourceID, ForkInstanceRequest{ + Name: "fork-target-transition-copy", + TargetState: StateRunning, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "apply fork target state") + + entries, err := os.ReadDir(manager.paths.GuestsDir()) + require.NoError(t, err) + require.Len(t, entries, 1) + assert.Equal(t, sourceID, entries[0].Name()) +} + +func TestForkInstanceRejectsDuplicateNameForNonNetworkedSource(t *testing.T) { + manager, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "fork-duplicate-name-source" + require.NoError(t, manager.ensureDirectories(sourceID)) + + now := time.Now() + sourceMeta := &metadata{StoredMetadata: StoredMetadata{ + Id: sourceID, + Name: sourceID, + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + StoppedAt: &now, + HypervisorType: hypervisor.TypeCloudHypervisor, + HypervisorVersion: "test", + SocketPath: paths.New(manager.paths.DataDir()).InstanceSocket(sourceID, "cloud-hypervisor.sock"), + DataDir: paths.New(manager.paths.DataDir()).InstanceDir(sourceID), + VsockCID: 42, + VsockSocket: paths.New(manager.paths.DataDir()).InstanceVsockSocket(sourceID), + }} + require.NoError(t, manager.saveMetadata(sourceMeta)) + + existingID := "fork-duplicate-name-existing" + require.NoError(t, manager.ensureDirectories(existingID)) + existingMeta := &metadata{StoredMetadata: StoredMetadata{ + Id: existingID, + Name: "duplicate-name", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + StoppedAt: &now, + HypervisorType: hypervisor.TypeCloudHypervisor, + HypervisorVersion: "test", + SocketPath: paths.New(manager.paths.DataDir()).InstanceSocket(existingID, "cloud-hypervisor.sock"), + DataDir: paths.New(manager.paths.DataDir()).InstanceDir(existingID), + VsockCID: 43, + VsockSocket: paths.New(manager.paths.DataDir()).InstanceVsockSocket(existingID), + }} + require.NoError(t, manager.saveMetadata(existingMeta)) + + _, err := manager.ForkInstance(ctx, sourceID, ForkInstanceRequest{Name: "duplicate-name"}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrAlreadyExists) + assert.Contains(t, err.Error(), "already exists") +} + +func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { + startedAt := time.Now().Add(-2 * time.Minute) + stoppedAt := time.Now().Add(-1 * time.Minute) + pid := 1234 + exitCode := 17 + + src := StoredMetadata{ + Env: map[string]string{"A": "1"}, + Metadata: map[string]string{"m": "x"}, + Volumes: []VolumeAttachment{{VolumeID: "vol-1", MountPath: "/data"}}, + Devices: []string{"0000:01:00.0"}, + Entrypoint: []string{"/bin/sh", "-c"}, + Cmd: []string{"echo", "hello"}, + StartedAt: &startedAt, + StoppedAt: &stoppedAt, + HypervisorPID: &pid, + ExitCode: &exitCode, + } + + cloned := cloneStoredMetadataForFork(src) + require.Equal(t, src, cloned) + + cloned.Env["A"] = "2" + cloned.Metadata["m"] = "y" + cloned.Volumes[0].MountPath = "/mnt" + cloned.Devices[0] = "0000:02:00.0" + cloned.Entrypoint[0] = "/usr/bin/env" + cloned.Cmd[0] = "printf" + *cloned.HypervisorPID = 4321 + *cloned.ExitCode = 42 + now := time.Now() + *cloned.StartedAt = now + *cloned.StoppedAt = now + + require.Equal(t, "1", src.Env["A"]) + require.Equal(t, "x", src.Metadata["m"]) + require.Equal(t, "/data", src.Volumes[0].MountPath) + require.Equal(t, "0000:01:00.0", src.Devices[0]) + require.Equal(t, "/bin/sh", src.Entrypoint[0]) + require.Equal(t, "echo", src.Cmd[0]) + require.Equal(t, 1234, *src.HypervisorPID) + require.Equal(t, 17, *src.ExitCode) + require.Equal(t, startedAt, *src.StartedAt) + require.Equal(t, stoppedAt, *src.StoppedAt) +} + +func TestRotateSourceVsockForRestore_CloudHypervisorDoesNotPersistCIDRewrite(t *testing.T) { + manager, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "fork-rotate-ch-source" + forkID := "fork-rotate-ch-fork" + require.NoError(t, manager.ensureDirectories(sourceID)) + + snapshotConfigPath := manager.paths.InstanceSnapshotConfig(sourceID) + require.NoError(t, os.MkdirAll(filepath.Dir(snapshotConfigPath), 0755)) + require.NoError(t, os.WriteFile(snapshotConfigPath, []byte(`{"vsock":{"cid":100,"socket":"/tmp/vsock.sock"}}`), 0644)) + + meta := &metadata{StoredMetadata: StoredMetadata{ + Id: sourceID, + Name: sourceID, + CreatedAt: time.Now(), + HypervisorType: hypervisor.TypeCloudHypervisor, + SocketPath: manager.paths.InstanceSocket(sourceID, "cloud-hypervisor.sock"), + DataDir: manager.paths.InstanceDir(sourceID), + VsockCID: 100, + VsockSocket: manager.paths.InstanceVsockSocket(sourceID), + }} + require.NoError(t, manager.saveMetadata(meta)) + + expectedCID := generateForkSourceVsockCID(sourceID, forkID, meta.StoredMetadata.VsockCID) + require.NotEqual(t, meta.StoredMetadata.VsockCID, expectedCID) + + require.NoError(t, manager.rotateSourceVsockForRestore(ctx, sourceID, forkID)) + + updated, err := manager.loadMetadata(sourceID) + require.NoError(t, err) + assert.Equal(t, int64(100), updated.StoredMetadata.VsockCID) +} + +func TestRotateSourceVsockForRestore_QEMUPersistsCIDRewrite(t *testing.T) { + manager, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "fork-rotate-qemu-source" + forkID := "fork-rotate-qemu-fork" + require.NoError(t, manager.ensureDirectories(sourceID)) + + snapshotConfigPath := manager.paths.InstanceSnapshotConfig(sourceID) + snapshotDir := filepath.Dir(snapshotConfigPath) + require.NoError(t, os.MkdirAll(snapshotDir, 0755)) + require.NoError(t, os.WriteFile(snapshotConfigPath, []byte(`{}`), 0644)) + + qemuConfig, err := json.Marshal(hypervisor.VMConfig{VsockCID: 100, VsockSocket: manager.paths.InstanceVsockSocket(sourceID)}) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(snapshotDir, "qemu-config.json"), qemuConfig, 0644)) + + meta := &metadata{StoredMetadata: StoredMetadata{ + Id: sourceID, + Name: sourceID, + CreatedAt: time.Now(), + HypervisorType: hypervisor.TypeQEMU, + SocketPath: manager.paths.InstanceSocket(sourceID, "qemu.sock"), + DataDir: manager.paths.InstanceDir(sourceID), + VsockCID: 100, + VsockSocket: manager.paths.InstanceVsockSocket(sourceID), + }} + require.NoError(t, manager.saveMetadata(meta)) + + expectedCID := generateForkSourceVsockCID(sourceID, forkID, meta.StoredMetadata.VsockCID) + require.NoError(t, manager.rotateSourceVsockForRestore(ctx, sourceID, forkID)) + + updated, err := manager.loadMetadata(sourceID) + require.NoError(t, err) + assert.Equal(t, expectedCID, updated.StoredMetadata.VsockCID) +} + +func TestForkCloudHypervisorFromRunningNetwork(t *testing.T) { + if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { + t.Skip("/dev/kvm not available, skipping on this platform") + } + + manager, tmpDir := setupTestManager(t) + ctx := context.Background() + + imageManager, err := images.NewManager(paths.New(tmpDir), 1, nil) + require.NoError(t, err) + + t.Log("Ensuring nginx image...") + nginxImage, err := imageManager.CreateImage(ctx, images.CreateImageRequest{Name: "docker.io/library/nginx:alpine"}) + require.NoError(t, err) + + imageName := nginxImage.Name + for i := 0; i < 60; i++ { + img, err := imageManager.GetImage(ctx, imageName) + if err == nil && img.Status == images.StatusReady { + nginxImage = img + break + } + if err == nil && img.Status == images.StatusFailed { + t.Fatalf("image build failed: %s", *img.Error) + } + time.Sleep(1 * time.Second) + } + require.Equal(t, images.StatusReady, nginxImage.Status, "Image should be ready after 60 seconds") + + systemManager := manager.systemManager + require.NoError(t, systemManager.EnsureSystemFiles(ctx)) + + require.NoError(t, manager.networkManager.Initialize(ctx, nil)) + + source, err := manager.CreateInstance(ctx, CreateInstanceRequest{ + Name: "fork-running-src", + Image: "docker.io/library/nginx:alpine", + Size: 2 * 1024 * 1024 * 1024, + HotplugSize: 256 * 1024 * 1024, + OverlaySize: 10 * 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: true, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = manager.DeleteInstance(context.Background(), source.Id) }) + require.NoError(t, waitForVMReady(ctx, source.SocketPath, 5*time.Second)) + require.NoError(t, waitForLogMessage(ctx, manager, source.Id, "start worker processes", 15*time.Second)) + + assert.NotEmpty(t, source.IP) + assert.NotEmpty(t, source.MAC) + assertHostCanReachNginx(t, source.IP, 80, 60*time.Second) + + // Default behavior remains strict: running source requires explicit opt-in. + _, err = manager.ForkInstance(ctx, source.Id, ForkInstanceRequest{Name: "fork-running-no-flag"}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidState) + + // Fork from running (internally: standby source -> copy fork -> restore source). + forked, err := manager.ForkInstance(ctx, source.Id, ForkInstanceRequest{ + Name: "fork-running-copy", + FromRunning: true, + TargetState: StateRunning, + }) + require.NoError(t, err) + require.Equal(t, StateRunning, forked.State) + forkedID := forked.Id + t.Cleanup(func() { _ = manager.DeleteInstance(context.Background(), forkedID) }) + + // Source should be restored and still reachable by its private IP. + sourceAfterFork, err := manager.GetInstance(ctx, source.Id) + require.NoError(t, err) + require.Equal(t, StateRunning, sourceAfterFork.State) + require.NotEmpty(t, sourceAfterFork.IP) + assertHostCanReachNginx(t, sourceAfterFork.IP, 80, 60*time.Second) + + // Fork should already be running with target_state=Running. + require.NoError(t, waitForVMReady(ctx, forked.SocketPath, 5*time.Second)) + + assert.NotEmpty(t, forked.IP) + assert.NotEmpty(t, forked.MAC) + assert.NotEqual(t, sourceAfterFork.IP, forked.IP) + assert.NotEqual(t, sourceAfterFork.MAC, forked.MAC) + assertGuestHasOnlyExpectedIPv4(t, forked, forked.IP, 30*time.Second) + assertHostCanReachNginx(t, forked.IP, 80, 60*time.Second) + assertHostCanReachNginx(t, sourceAfterFork.IP, 80, 60*time.Second) +} + +func assertHostCanReachNginx(t *testing.T, ip string, port int, timeout time.Duration) { + t.Helper() + + deadline := time.Now().Add(timeout) + url := fmt.Sprintf("http://%s:%d/", ip, port) + client := &http.Client{Timeout: 3 * time.Second} + + var lastErr error + for time.Now().Before(deadline) { + resp, err := client.Get(url) + if err == nil { + body, readErr := io.ReadAll(resp.Body) + resp.Body.Close() + if readErr == nil && resp.StatusCode == http.StatusOK && strings.Contains(string(body), "Welcome to nginx!") { + return + } + if readErr != nil { + lastErr = fmt.Errorf("read body: %w", readErr) + } else { + lastErr = fmt.Errorf("status=%d body=%q", resp.StatusCode, string(body)) + } + } else { + lastErr = err + } + time.Sleep(500 * time.Millisecond) + } + + require.NoError(t, lastErr, "host should reach %s within %s", url, timeout) +} + +func assertGuestHasOnlyExpectedIPv4(t *testing.T, inst *Instance, expectedIP string, timeout time.Duration) { + t.Helper() + + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + output, exitCode, err := execInInstance(context.Background(), inst, "sh", "-c", "ip -4 -o addr show dev eth0 scope global | awk '{print $4}'") + if err == nil && exitCode == 0 { + var cidrs []string + for _, line := range strings.Split(strings.TrimSpace(output), "\n") { + if trimmed := strings.TrimSpace(line); trimmed != "" { + cidrs = append(cidrs, trimmed) + } + } + + if len(cidrs) == 1 && strings.HasPrefix(cidrs[0], expectedIP+"/") { + return + } + + lastErr = fmt.Errorf("expected only %s on eth0, got %v", expectedIP, cidrs) + } else if err != nil { + lastErr = err + } else { + lastErr = fmt.Errorf("ip addr command exit code %d, output=%q", exitCode, output) + } + + time.Sleep(500 * time.Millisecond) + } + + require.NoError(t, lastErr, "guest should expose only the fork IP on eth0 within %s", timeout) +} + +func execInInstance(ctx context.Context, inst *Instance, command ...string) (string, int, error) { + dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID) + if err != nil { + return "", -1, err + } + + var stdout, stderr bytes.Buffer + exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ + Command: command, + Stdout: &stdout, + Stderr: &stderr, + WaitForAgent: 30 * time.Second, + }) + if err != nil { + return "", -1, err + } + + output := stdout.String() + if stderr.Len() > 0 { + output += "\nSTDERR: " + stderr.String() + } + return output, exit.Code, nil +} diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 3c764d13..1d291fb1 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -25,6 +25,7 @@ type Manager interface { // Returns ErrAmbiguousName if prefix matches multiple instances. GetInstance(ctx context.Context, idOrName string) (*Instance, error) DeleteInstance(ctx context.Context, id string) error + ForkInstance(ctx context.Context, id string, req ForkInstanceRequest) (*Instance, error) StandbyInstance(ctx context.Context, id string) (*Instance, error) RestoreInstance(ctx context.Context, id string) (*Instance, error) StopInstance(ctx context.Context, id string) (*Instance, error) @@ -182,6 +183,26 @@ func (m *manager) DeleteInstance(ctx context.Context, id string) error { return err } +// ForkInstance creates a forked copy of an instance. +func (m *manager) ForkInstance(ctx context.Context, id string, req ForkInstanceRequest) (*Instance, error) { + lock := m.getInstanceLock(id) + lock.Lock() + forked, targetState, err := m.forkInstance(ctx, id, req) + lock.Unlock() + if err != nil { + return nil, err + } + + inst, err := m.applyForkTargetState(ctx, forked.Id, targetState) + if err != nil { + if cleanupErr := m.cleanupForkInstanceOnError(ctx, forked.Id); cleanupErr != nil { + return nil, fmt.Errorf("apply fork target state: %w; additionally failed to cleanup forked instance %s: %v", err, forked.Id, cleanupErr) + } + return nil, fmt.Errorf("apply fork target state: %w", err) + } + return inst, nil +} + // StandbyInstance puts an instance in standby (pause, snapshot, delete VMM) func (m *manager) StandbyInstance(ctx context.Context, id string) (*Instance, error) { lock := m.getInstanceLock(id) diff --git a/lib/instances/name_validation.go b/lib/instances/name_validation.go new file mode 100644 index 00000000..4818bc7e --- /dev/null +++ b/lib/instances/name_validation.go @@ -0,0 +1,21 @@ +package instances + +import ( + "fmt" + "regexp" +) + +var instanceNamePattern = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]*[a-z0-9])?$`) + +func validateInstanceName(name string) error { + if name == "" { + return fmt.Errorf("name is required") + } + if len(name) > 63 { + return fmt.Errorf("name must be 63 characters or less") + } + if !instanceNamePattern.MatchString(name) { + return fmt.Errorf("name must contain only lowercase letters, digits, and dashes; cannot start or end with a dash") + } + return nil +} diff --git a/lib/instances/qemu_test.go b/lib/instances/qemu_test.go index 5e2421f6..2937ba42 100644 --- a/lib/instances/qemu_test.go +++ b/lib/instances/qemu_test.go @@ -865,3 +865,93 @@ func TestQEMUStandbyAndRestore(t *testing.T) { t.Log("QEMU standby/restore test complete!") } + +func TestQEMUForkFromRunningNetwork(t *testing.T) { + if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { + t.Skip("/dev/kvm not available, skipping on this platform") + } + + starter := qemu.NewStarter() + if _, err := starter.GetBinaryPath(nil, ""); err != nil { + t.Fatalf("QEMU not available: %v", err) + } + + manager, tmpDir := setupTestManagerForQEMU(t) + ctx := context.Background() + p := paths.New(tmpDir) + + imageManager, err := images.NewManager(p, 1, nil) + require.NoError(t, err) + + t.Log("Ensuring nginx image...") + nginxImage, err := imageManager.CreateImage(ctx, images.CreateImageRequest{Name: "docker.io/library/nginx:alpine"}) + require.NoError(t, err) + + imageName := nginxImage.Name + for i := 0; i < 60; i++ { + img, err := imageManager.GetImage(ctx, imageName) + if err == nil && img.Status == images.StatusReady { + nginxImage = img + break + } + if err == nil && img.Status == images.StatusFailed { + t.Fatalf("image build failed: %s", *img.Error) + } + time.Sleep(1 * time.Second) + } + require.Equal(t, images.StatusReady, nginxImage.Status, "Image should be ready after 60 seconds") + + require.NoError(t, manager.systemManager.EnsureSystemFiles(ctx)) + require.NoError(t, manager.networkManager.Initialize(ctx, nil)) + + source, err := manager.CreateInstance(ctx, CreateInstanceRequest{ + Name: "qemu-fork-running-src", + Image: "docker.io/library/nginx:alpine", + Size: 2 * 1024 * 1024 * 1024, + HotplugSize: 256 * 1024 * 1024, + OverlaySize: 10 * 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: true, + Hypervisor: hypervisor.TypeQEMU, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = manager.DeleteInstance(context.Background(), source.Id) }) + require.NoError(t, waitForQEMUReady(ctx, source.SocketPath, 10*time.Second)) + require.NoError(t, waitForLogMessage(ctx, manager, source.Id, "start worker processes", 15*time.Second)) + + assert.NotEmpty(t, source.IP) + assert.NotEmpty(t, source.MAC) + assertHostCanReachNginx(t, source.IP, 80, 60*time.Second) + + _, err = manager.ForkInstance(ctx, source.Id, ForkInstanceRequest{Name: "qemu-fork-running-no-flag"}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidState) + + forked, err := manager.ForkInstance(ctx, source.Id, ForkInstanceRequest{ + Name: "qemu-fork-running-copy", + FromRunning: true, + TargetState: StateStandby, + }) + require.NoError(t, err) + require.Equal(t, StateStandby, forked.State) + forkedID := forked.Id + t.Cleanup(func() { _ = manager.DeleteInstance(context.Background(), forkedID) }) + + sourceAfterFork, err := manager.GetInstance(ctx, source.Id) + require.NoError(t, err) + require.Equal(t, StateRunning, sourceAfterFork.State) + require.NotEmpty(t, sourceAfterFork.IP) + assertHostCanReachNginx(t, sourceAfterFork.IP, 80, 60*time.Second) + + forked, err = manager.RestoreInstance(ctx, forkedID) + require.NoError(t, err) + require.Equal(t, StateRunning, forked.State) + require.NoError(t, waitForQEMUReady(ctx, forked.SocketPath, 10*time.Second)) + + assert.NotEmpty(t, forked.IP) + assert.NotEmpty(t, forked.MAC) + assert.NotEqual(t, sourceAfterFork.IP, forked.IP) + assert.NotEqual(t, sourceAfterFork.MAC, forked.MAC) + assertHostCanReachNginx(t, forked.IP, 80, 60*time.Second) + assertHostCanReachNginx(t, sourceAfterFork.IP, 80, 60*time.Second) +} diff --git a/lib/instances/restore.go b/lib/instances/restore.go index ed81fbfd..1ff09c55 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -1,13 +1,19 @@ package instances import ( + "bytes" "context" + "errors" "fmt" + "net" "os" + "strings" "time" + "github.com/kernel/hypeman/lib/guest" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" + "github.com/kernel/hypeman/lib/network" "go.opentelemetry.io/otel/trace" ) @@ -63,21 +69,100 @@ func (m *manager) restoreInstance( // 3. Get snapshot directory snapshotDir := m.paths.InstanceSnapshotLatest(id) + starter, err := m.getVMStarter(stored.HypervisorType) + if err != nil { + return nil, fmt.Errorf("get vm starter: %w", err) + } + + var allocatedNet *network.Allocation + releaseNetwork := func() { + if !stored.NetworkEnabled { + return + } + if allocatedNet != nil { + if err := m.networkManager.ReleaseAllocation(ctx, allocatedNet); err != nil { + log.WarnContext(ctx, "failed to release allocated network", "instance_id", id, "error", err) + } + return + } + netAlloc, _ := m.networkManager.GetAllocation(ctx, id) + if err := m.networkManager.ReleaseAllocation(ctx, netAlloc); err != nil { + log.WarnContext(ctx, "failed to release network", "instance_id", id, "error", err) + } + } - // 4. Recreate TAP device if network enabled + // 4. Recreate or allocate network if network enabled if stored.NetworkEnabled { var networkSpan trace.Span if m.metrics != nil && m.metrics.tracer != nil { ctx, networkSpan = m.metrics.tracer.Start(ctx, "RestoreNetwork") } - log.InfoContext(ctx, "recreating network for restore", "instance_id", id, "network", "default", - "download_bps", stored.NetworkBandwidthDownload, "upload_bps", stored.NetworkBandwidthUpload) - if err := m.networkManager.RecreateAllocation(ctx, id, stored.NetworkBandwidthDownload, stored.NetworkBandwidthUpload); err != nil { - if networkSpan != nil { - networkSpan.End() + // If IP/MAC is empty (forked standby flow), allocate a fresh identity and + // patch the copied snapshot config before restore. + if stored.IP == "" || stored.MAC == "" { + log.InfoContext(ctx, "allocating fresh network identity for standby restore", + "instance_id", id, "network", "default", + "download_bps", stored.NetworkBandwidthDownload, "upload_bps", stored.NetworkBandwidthUpload) + netConfig, err := m.networkManager.CreateAllocation(ctx, network.AllocateRequest{ + InstanceID: id, + InstanceName: stored.Name, + DownloadBps: stored.NetworkBandwidthDownload, + UploadBps: stored.NetworkBandwidthUpload, + UploadCeilBps: stored.NetworkBandwidthUpload * int64(m.networkManager.GetUploadBurstMultiplier()), + }) + if err != nil { + if networkSpan != nil { + networkSpan.End() + } + log.ErrorContext(ctx, "failed to allocate network", "instance_id", id, "error", err) + return nil, fmt.Errorf("allocate network: %w", err) + } + allocatedNet = &network.Allocation{ + InstanceID: id, + InstanceName: stored.Name, + Network: "default", + IP: netConfig.IP, + MAC: netConfig.MAC, + TAPDevice: netConfig.TAPDevice, + Gateway: netConfig.Gateway, + Netmask: netConfig.Netmask, + } + stored.IP = netConfig.IP + stored.MAC = netConfig.MAC + + if _, err := starter.PrepareFork(ctx, hypervisor.ForkPrepareRequest{ + SnapshotConfigPath: m.paths.InstanceSnapshotConfig(id), + VsockCID: stored.VsockCID, + VsockSocket: stored.VsockSocket, + Network: &hypervisor.ForkNetworkConfig{ + TAPDevice: netConfig.TAPDevice, + IP: netConfig.IP, + MAC: netConfig.MAC, + Netmask: netConfig.Netmask, + }, + }); err != nil { + if networkSpan != nil { + networkSpan.End() + } + if errors.Is(err, hypervisor.ErrNotSupported) { + log.ErrorContext(ctx, "forked standby network rewrite not supported for hypervisor", "instance_id", id, "hypervisor", stored.HypervisorType) + releaseNetwork() + return nil, fmt.Errorf("%w: standby fork restore network rewrite is not supported for hypervisor %s", ErrNotSupported, stored.HypervisorType) + } + log.ErrorContext(ctx, "failed to patch snapshot network identity", "instance_id", id, "error", err) + releaseNetwork() + return nil, fmt.Errorf("rewrite snapshot config: %w", err) + } + } else { + log.InfoContext(ctx, "recreating network for restore", "instance_id", id, "network", "default", + "download_bps", stored.NetworkBandwidthDownload, "upload_bps", stored.NetworkBandwidthUpload) + if err := m.networkManager.RecreateAllocation(ctx, id, stored.NetworkBandwidthDownload, stored.NetworkBandwidthUpload); err != nil { + if networkSpan != nil { + networkSpan.End() + } + log.ErrorContext(ctx, "failed to recreate network", "instance_id", id, "error", err) + return nil, fmt.Errorf("recreate network: %w", err) } - log.ErrorContext(ctx, "failed to recreate network", "instance_id", id, "error", err) - return nil, fmt.Errorf("recreate network: %w", err) } if networkSpan != nil { networkSpan.End() @@ -97,10 +182,7 @@ func (m *manager) restoreInstance( if err != nil { log.ErrorContext(ctx, "failed to restore from snapshot", "instance_id", id, "error", err) // Cleanup network on failure - if stored.NetworkEnabled { - netAlloc, _ := m.networkManager.GetAllocation(ctx, id) - m.networkManager.ReleaseAllocation(ctx, netAlloc) - } + releaseNetwork() return nil, err } @@ -120,16 +202,25 @@ func (m *manager) restoreInstance( log.ErrorContext(ctx, "failed to resume VM", "instance_id", id, "error", err) // Cleanup on failure hv.Shutdown(ctx) - if stored.NetworkEnabled { - netAlloc, _ := m.networkManager.GetAllocation(ctx, id) - m.networkManager.ReleaseAllocation(ctx, netAlloc) - } + releaseNetwork() return nil, fmt.Errorf("resume vm failed: %w", err) } if resumeSpan != nil { resumeSpan.End() } + // Forked standby restores may allocate a fresh identity while the guest memory snapshot + // still has the source VM's old IP configuration. Reconfigure guest networking after + // resume so host ingress to the new private IP works reliably. + if allocatedNet != nil && !stored.SkipGuestAgent { + if err := reconfigureGuestNetwork(ctx, stored, allocatedNet); err != nil { + log.ErrorContext(ctx, "failed to configure guest network after restore", "instance_id", id, "error", err) + _ = hv.Shutdown(ctx) + releaseNetwork() + return nil, fmt.Errorf("configure guest network after restore: %w", err) + } + } + // 8. Delete snapshot after successful restore log.InfoContext(ctx, "deleting snapshot after successful restore", "instance_id", id) os.RemoveAll(snapshotDir) // Best effort, ignore errors @@ -180,3 +271,48 @@ func (m *manager) restoreFromSnapshot( log.DebugContext(ctx, "VM restored from snapshot successfully", "instance_id", stored.Id, "pid", pid) return pid, hv, nil } + +func reconfigureGuestNetwork(ctx context.Context, stored *StoredMetadata, alloc *network.Allocation) error { + prefix, err := netmaskToPrefix(alloc.Netmask) + if err != nil { + return err + } + + dialer, err := hypervisor.NewVsockDialer(stored.HypervisorType, stored.VsockSocket, stored.VsockCID) + if err != nil { + return fmt.Errorf("create vsock dialer: %w", err) + } + + cmd := fmt.Sprintf( + "ip -4 addr flush dev eth0 scope global && ip addr add %s/%d dev eth0 && ip link set dev eth0 up && ip route replace default via %s dev eth0", + alloc.IP, prefix, alloc.Gateway, + ) + + var stdout, stderr bytes.Buffer + exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ + Command: []string{"sh", "-c", cmd}, + Stdout: &stdout, + Stderr: &stderr, + WaitForAgent: 120 * time.Second, + }) + if err != nil { + return fmt.Errorf("exec network reconfiguration command: %w", err) + } + if exit.Code != 0 { + return fmt.Errorf("network reconfiguration command failed (exit=%d, stdout=%q, stderr=%q)", exit.Code, strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String())) + } + + return nil +} + +func netmaskToPrefix(mask string) (int, error) { + ip := net.ParseIP(mask).To4() + if ip == nil { + return 0, fmt.Errorf("invalid netmask: %q", mask) + } + ones, bits := net.IPMask(ip).Size() + if bits != 32 { + return 0, fmt.Errorf("invalid netmask bits: %q", mask) + } + return ones, nil +} diff --git a/lib/instances/types.go b/lib/instances/types.go index ce688cf5..f7679285 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -175,6 +175,13 @@ type StartInstanceRequest struct { Cmd []string // Override cmd (nil = keep previous/image default) } +// ForkInstanceRequest is the domain request for forking an instance. +type ForkInstanceRequest struct { + Name string // Required: name for the new forked instance + FromRunning bool // Optional: allow forking from Running by auto standby/fork/restore + TargetState State // Optional: desired final state of forked instance (Stopped, Standby, Running). Empty means inherit source state. +} + // AttachVolumeRequest is the domain request for attaching a volume (used for API compatibility) type AttachVolumeRequest struct { MountPath string diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index fa06112e..8c754bb2 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -60,6 +60,13 @@ const ( Pci DeviceType = "pci" ) +// Defines values for ForkTargetState. +const ( + ForkTargetStateRunning ForkTargetState = "Running" + ForkTargetStateStandby ForkTargetState = "Standby" + ForkTargetStateStopped ForkTargetState = "Stopped" +) + // Defines values for GPUResourceStatusMode. const ( Passthrough GPUResourceStatusMode = "passthrough" @@ -90,13 +97,13 @@ const ( // Defines values for InstanceState. const ( - Created InstanceState = "Created" - Paused InstanceState = "Paused" - Running InstanceState = "Running" - Shutdown InstanceState = "Shutdown" - Standby InstanceState = "Standby" - Stopped InstanceState = "Stopped" - Unknown InstanceState = "Unknown" + InstanceStateCreated InstanceState = "Created" + InstanceStatePaused InstanceState = "Paused" + InstanceStateRunning InstanceState = "Running" + InstanceStateShutdown InstanceState = "Shutdown" + InstanceStateStandby InstanceState = "Standby" + InstanceStateStopped InstanceState = "Stopped" + InstanceStateUnknown InstanceState = "Unknown" ) // Defines values for GetInstanceLogsParamsSource. @@ -400,6 +407,22 @@ type ErrorDetail struct { Message *string `json:"message,omitempty"` } +// ForkInstanceRequest defines model for ForkInstanceRequest. +type ForkInstanceRequest struct { + // FromRunning Allow forking from a running source instance. + // When true and source is Running, the source is put into standby, forked, then restored back to Running. + FromRunning *bool `json:"from_running,omitempty"` + + // Name Name for the forked instance (lowercase letters, digits, and dashes only; cannot start or end with a dash) + Name string `json:"name"` + + // TargetState Target state for the forked instance after fork completes + TargetState *ForkTargetState `json:"target_state,omitempty"` +} + +// ForkTargetState Target state for the forked instance after fork completes +type ForkTargetState string + // GPUConfig GPU configuration for the instance type GPUConfig struct { // Profile vGPU profile name (e.g., "L40S-1Q"). Only used in vGPU mode. @@ -961,6 +984,9 @@ type CreateIngressJSONRequestBody = CreateIngressRequest // CreateInstanceJSONRequestBody defines body for CreateInstance for application/json ContentType. type CreateInstanceJSONRequestBody = CreateInstanceRequest +// ForkInstanceJSONRequestBody defines body for ForkInstance for application/json ContentType. +type ForkInstanceJSONRequestBody = ForkInstanceRequest + // StartInstanceJSONRequestBody defines body for StartInstance for application/json ContentType. type StartInstanceJSONRequestBody StartInstanceJSONBody @@ -1120,6 +1146,11 @@ type ClientInterface interface { // GetInstance request GetInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + // ForkInstanceWithBody request with any body + ForkInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + ForkInstance(ctx context.Context, id string, body ForkInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetInstanceLogs request GetInstanceLogs(ctx context.Context, id string, params *GetInstanceLogsParams, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -1496,6 +1527,30 @@ func (c *Client) GetInstance(ctx context.Context, id string, reqEditors ...Reque return c.Client.Do(req) } +func (c *Client) ForkInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewForkInstanceRequestWithBody(c.Server, id, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) ForkInstance(ctx context.Context, id string, body ForkInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewForkInstanceRequest(c.Server, id, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) GetInstanceLogs(ctx context.Context, id string, params *GetInstanceLogsParams, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewGetInstanceLogsRequest(c.Server, id, params) if err != nil { @@ -2524,6 +2579,53 @@ func NewGetInstanceRequest(server string, id string) (*http.Request, error) { return req, nil } +// NewForkInstanceRequest calls the generic ForkInstance builder with application/json body +func NewForkInstanceRequest(server string, id string, body ForkInstanceJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewForkInstanceRequestWithBody(server, id, "application/json", bodyReader) +} + +// NewForkInstanceRequestWithBody generates requests for ForkInstance with any type of body +func NewForkInstanceRequestWithBody(server string, id string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "id", runtime.ParamLocationPath, id) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/instances/%s/fork", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + // NewGetInstanceLogsRequest generates requests for GetInstanceLogs func NewGetInstanceLogsRequest(server string, id string, params *GetInstanceLogsParams) (*http.Request, error) { var err error @@ -3315,6 +3417,11 @@ type ClientWithResponsesInterface interface { // GetInstanceWithResponse request GetInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetInstanceResponse, error) + // ForkInstanceWithBodyWithResponse request with any body + ForkInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*ForkInstanceResponse, error) + + ForkInstanceWithResponse(ctx context.Context, id string, body ForkInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*ForkInstanceResponse, error) + // GetInstanceLogsWithResponse request GetInstanceLogsWithResponse(ctx context.Context, id string, params *GetInstanceLogsParams, reqEditors ...RequestEditorFn) (*GetInstanceLogsResponse, error) @@ -3925,6 +4032,33 @@ func (r GetInstanceResponse) StatusCode() int { return 0 } +type ForkInstanceResponse struct { + Body []byte + HTTPResponse *http.Response + JSON201 *Instance + JSON400 *Error + JSON404 *Error + JSON409 *Error + JSON500 *Error + JSON501 *Error +} + +// Status returns HTTPResponse.Status +func (r ForkInstanceResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r ForkInstanceResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type GetInstanceLogsResponse struct { Body []byte HTTPResponse *http.Response @@ -4532,6 +4666,23 @@ func (c *ClientWithResponses) GetInstanceWithResponse(ctx context.Context, id st return ParseGetInstanceResponse(rsp) } +// ForkInstanceWithBodyWithResponse request with arbitrary body returning *ForkInstanceResponse +func (c *ClientWithResponses) ForkInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*ForkInstanceResponse, error) { + rsp, err := c.ForkInstanceWithBody(ctx, id, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseForkInstanceResponse(rsp) +} + +func (c *ClientWithResponses) ForkInstanceWithResponse(ctx context.Context, id string, body ForkInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*ForkInstanceResponse, error) { + rsp, err := c.ForkInstance(ctx, id, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseForkInstanceResponse(rsp) +} + // GetInstanceLogsWithResponse request returning *GetInstanceLogsResponse func (c *ClientWithResponses) GetInstanceLogsWithResponse(ctx context.Context, id string, params *GetInstanceLogsParams, reqEditors ...RequestEditorFn) (*GetInstanceLogsResponse, error) { rsp, err := c.GetInstanceLogs(ctx, id, params, reqEditors...) @@ -5653,6 +5804,67 @@ func ParseGetInstanceResponse(rsp *http.Response) (*GetInstanceResponse, error) return response, nil } +// ParseForkInstanceResponse parses an HTTP response from a ForkInstanceWithResponse call +func ParseForkInstanceResponse(rsp *http.Response) (*ForkInstanceResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &ForkInstanceResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 201: + var dest Instance + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON201 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 409: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON409 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 501: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON501 = &dest + + } + + return response, nil +} + // ParseGetInstanceLogsResponse parses an HTTP response from a GetInstanceLogsWithResponse call func ParseGetInstanceLogsResponse(rsp *http.Response) (*GetInstanceLogsResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -6380,6 +6592,9 @@ type ServerInterface interface { // Get instance details // (GET /instances/{id}) GetInstance(w http.ResponseWriter, r *http.Request, id string) + // Fork an instance from stopped, standby, or running (with from_running=true) + // (POST /instances/{id}/fork) + ForkInstance(w http.ResponseWriter, r *http.Request, id string) // Stream instance logs (SSE) // (GET /instances/{id}/logs) GetInstanceLogs(w http.ResponseWriter, r *http.Request, id string, params GetInstanceLogsParams) @@ -6569,6 +6784,12 @@ func (_ Unimplemented) GetInstance(w http.ResponseWriter, r *http.Request, id st w.WriteHeader(http.StatusNotImplemented) } +// Fork an instance from stopped, standby, or running (with from_running=true) +// (POST /instances/{id}/fork) +func (_ Unimplemented) ForkInstance(w http.ResponseWriter, r *http.Request, id string) { + w.WriteHeader(http.StatusNotImplemented) +} + // Stream instance logs (SSE) // (GET /instances/{id}/logs) func (_ Unimplemented) GetInstanceLogs(w http.ResponseWriter, r *http.Request, id string, params GetInstanceLogsParams) { @@ -7275,6 +7496,37 @@ func (siw *ServerInterfaceWrapper) GetInstance(w http.ResponseWriter, r *http.Re handler.ServeHTTP(w, r) } +// ForkInstance operation middleware +func (siw *ServerInterfaceWrapper) ForkInstance(w http.ResponseWriter, r *http.Request) { + + var err error + + // ------------- Path parameter "id" ------------- + var id string + + err = runtime.BindStyledParameterWithOptions("simple", "id", chi.URLParam(r, "id"), &id, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "id", Err: err}) + return + } + + ctx := r.Context() + + ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) + + r = r.WithContext(ctx) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.ForkInstance(w, r, id) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + // GetInstanceLogs operation middleware func (siw *ServerInterfaceWrapper) GetInstanceLogs(w http.ResponseWriter, r *http.Request) { @@ -7992,6 +8244,9 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl r.Group(func(r chi.Router) { r.Get(options.BaseURL+"/instances/{id}", wrapper.GetInstance) }) + r.Group(func(r chi.Router) { + r.Post(options.BaseURL+"/instances/{id}/fork", wrapper.ForkInstance) + }) r.Group(func(r chi.Router) { r.Get(options.BaseURL+"/instances/{id}/logs", wrapper.GetInstanceLogs) }) @@ -8954,6 +9209,69 @@ func (response GetInstance500JSONResponse) VisitGetInstanceResponse(w http.Respo return json.NewEncoder(w).Encode(response) } +type ForkInstanceRequestObject struct { + Id string `json:"id"` + Body *ForkInstanceJSONRequestBody +} + +type ForkInstanceResponseObject interface { + VisitForkInstanceResponse(w http.ResponseWriter) error +} + +type ForkInstance201JSONResponse Instance + +func (response ForkInstance201JSONResponse) VisitForkInstanceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(201) + + return json.NewEncoder(w).Encode(response) +} + +type ForkInstance400JSONResponse Error + +func (response ForkInstance400JSONResponse) VisitForkInstanceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(400) + + return json.NewEncoder(w).Encode(response) +} + +type ForkInstance404JSONResponse Error + +func (response ForkInstance404JSONResponse) VisitForkInstanceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(404) + + return json.NewEncoder(w).Encode(response) +} + +type ForkInstance409JSONResponse Error + +func (response ForkInstance409JSONResponse) VisitForkInstanceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(409) + + return json.NewEncoder(w).Encode(response) +} + +type ForkInstance500JSONResponse Error + +func (response ForkInstance500JSONResponse) VisitForkInstanceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(500) + + return json.NewEncoder(w).Encode(response) +} + +type ForkInstance501JSONResponse Error + +func (response ForkInstance501JSONResponse) VisitForkInstanceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(501) + + return json.NewEncoder(w).Encode(response) +} + type GetInstanceLogsRequestObject struct { Id string `json:"id"` Params GetInstanceLogsParams @@ -9654,6 +9972,9 @@ type StrictServerInterface interface { // Get instance details // (GET /instances/{id}) GetInstance(ctx context.Context, request GetInstanceRequestObject) (GetInstanceResponseObject, error) + // Fork an instance from stopped, standby, or running (with from_running=true) + // (POST /instances/{id}/fork) + ForkInstance(ctx context.Context, request ForkInstanceRequestObject) (ForkInstanceResponseObject, error) // Stream instance logs (SSE) // (GET /instances/{id}/logs) GetInstanceLogs(ctx context.Context, request GetInstanceLogsRequestObject) (GetInstanceLogsResponseObject, error) @@ -10342,6 +10663,39 @@ func (sh *strictHandler) GetInstance(w http.ResponseWriter, r *http.Request, id } } +// ForkInstance operation middleware +func (sh *strictHandler) ForkInstance(w http.ResponseWriter, r *http.Request, id string) { + var request ForkInstanceRequestObject + + request.Id = id + + var body ForkInstanceJSONRequestBody + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + sh.options.RequestErrorHandlerFunc(w, r, fmt.Errorf("can't decode JSON body: %w", err)) + return + } + request.Body = &body + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.ForkInstance(ctx, request.(ForkInstanceRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "ForkInstance") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(ForkInstanceResponseObject); ok { + if err := validResponse.VisitForkInstanceResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + // GetInstanceLogs operation middleware func (sh *strictHandler) GetInstanceLogs(w http.ResponseWriter, r *http.Request, id string, params GetInstanceLogsParams) { var request GetInstanceLogsRequestObject @@ -10756,178 +11110,183 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x97XLbOLLoq6B4z9bKZyVZ/ojj+NTUKcdOPN6NE9849t6zo1wFIiEJYxLgAKASJZW/", - "+wD7iPMkp9AA+CVQovLhxDvZ2prQIgg0Gt2N7kaj+0MQ8iTljDAlg6MPgQxnJMHweKwUDmc3PM4S8pL8", - "lhGp9M+p4CkRihJolPCMqVGK1Uz/FREZCpoqyllwFFxiNUNvZ0QQNIdekJzxLI7QmCD4jkRBNyDvcJLG", - "JDgKthOmtiOscNAN1CLVP0klKJsGH7uBIDjiLF6YYSY4i1VwNMGxJN3asBe6a4Ql0p/04Ju8vzHnMcEs", - "+Ag9/pZRQaLg6JfyNF7njfn4VxIqPfjxHNMYj2NySuY0JMtoCDMhCFOjSNA5EcuoODHv4wUa84xFyLRD", - "HZbFMaITxDgjWxVksDmNqMaEbqKHDo6UyIgHMxHANKKRZwVOzpF5jc5PUWdG3lUH2X04Pgyau2Q4Icud", - "/pwlmPU0cjVYrn9oW+772b6vZ8qTJBtNBc/S5Z7PX1xcXCN4iViWjIko93i4m/dHmSJTInSHaUhHOIoE", - "kdI/f/eyDNtgMBgc4d2jwaA/8EE5JyziohGl5rUfpTuDiKzoshVKbf9LKH1+c356foxOuEi5wPDt0kg1", - "wi6jpzyvMtlUV8VH/48zGkfLVD/WPxMxokwqzBpo8Ny+1OjiE6RmBNnv0M0F6ky4QBEZZ9MpZdOtNvSu", - "BVZMFIlGWC0PB6Ai24ZyhhRNiFQ4SYNuMOEi0R8FEVakp9+0GlAQvGY43aLVYMuslpmVHCWyqXfXBFGG", - "EhrHVJKQs0iWx6BMHew3T6bEMEQI7pFQT/TPKCFS4ilBHS02texmSCqsMomoRBNMYxK1WiMfIZjJ/MrH", - "iEaEKTqhVf425NTD43Bnd88rOxI8JaOITu1OVO3+FH7XJKb7UQha+yeiGW3Rbh4wpCCT5fGeguiGQQSZ", - "EEE0jX/mcKngc8I0t+jx/gPGDf7PdrFFb9v9eRuQeVk0/9gNfstIRkYpl9RAuCS57BtNRoBqBF/4YYZX", - "q9a6RFFSYbGaP6DFF+BEA18r3FyZpnV5COLOdlPh7Eax92ROmEfxCTlT9kV1xs/4FMWUEWRbWPxqOacH", - "+CnmIOa+xNy6QYHSZYbWcH+CQDI/NPSm33UDwrJEIzPm0zI2ZwQLNSYVZDZsS7ajArpG9F9WWKK2/2BJ", - "RqulwiVljERIt7TMalqiTIL2uTR94IxbqkZzIqSXjwCsv1GFbIvGrmIe3k5oTEYzLGcGYhxFwIM4vqzM", - "xKOBVVRanGrB5joEzUAixdHVz8e7Dw6QHcCDQ8kzERoIlmdS+lp3b9oihcUYx7GXNprJbfN9d5lC/BRw", - "lTNG036SU6AjTCO9AruauvtukGZyZp5AHmuoYD/TYkCTV6yfX3smfQJCwmj+jXaQX697kZrFRtOYa5wu", - "UMbob1lFae6jc63/K6SFP41I1EUYXmgxjDPFe1PCiNByCk0ET0CDKim2qEP6034XDbWu19OabQ/v9gaD", - "3mAYVFXTeL83TTONCqwUERrA//8L7r0/7v1j0Hv0ungc9Xuv//IfPgJoq207Tc/Os+N4v4scsGUVvA7o", - "avV8hYbrkyJm+c4172+6eifnyxu8gT/i4S0Rfcq3YzoWWCy22ZSyd0cxVkSq6mxWt107P4BtxcTYVE99", - "w6nVDA4gt07M3xIRakkZE00gsquFJVWyi7C2WUHIIL2b/RcKMdM0azZ2LhBhEXpL1QxhaFfFQLLo4ZT2", - "qAE16AYJfveMsKmaBUcHe0v0qImxYx96r//T/bT1316SFFlMPMT4kmeKsimC12b3nVGJChioIsna7dZh", - "N4tBxUooOzef7eSQYCHwwr9qDrhVq2eMo8blCxOPJv1iToSgkdvRTi5OUSemt8SSJRIZQ8NsMNgLoQE8", - "EvtLyJMEs8j8ttVHLxKq9E6SFRuk8a70y0v4S0DCGYc9Po65nlCOvgYFwuHFGZqeJTp1ngmJrLULexoG", - "vxMs2dnl9baWKimWUs0Ez6azKlRWpG0GD5W3I8pH49QHE5W36Hz7BdICF8VUYycXsDuDwcXjbTkM9B8P", - "3B9bfXRqUAbg6/Xjwsp9OcOCgPYRIc7QyeU1wnHMQ2vPTbSSOKHTTJCoX3MjQO8+gidMiUXKqU/5rFFG", - "0XSZQHq94u0GdLA9pmxb6mXohZvhnbD5Z6hAT9icCs4SrYbOsaBablWcOh+C5y9On4yePL8JjjQTRVlo", - "PSSXL16+Co6CvcFgEPi0DE1Ba+TA2eX1CayUbj/jKo2z6UjS9x7RepzPDyUk4cKo/vYb1JlVJa/RjBAs", - "zjDYO3tsiGvnDOjKLUpEJbR2vZiOqxSze/bYRy2zRUrEnEqfzf9z/s6tfElOGsFUpW1JxJyInGiBivsl", - "vSuMeRb1SkN2gwkVJBRYk13QDX4jiVZA5u816RSwe77zm+KtNuk1uy+OU8pI4/bbDRKiMLihP51aryUR", - "vYhMqDY+bsmiN8dxRpDr2eKZ5GiuEnKaiZRL0z+eGqVVEZwER8FYY5JFXjr+Trb6t1zcxhxHvZ0vvNMz", - "onTfy1N8bl5U6dKH47r5yKK3NFKzUcTfMg2yZz+wb1DeON8U3umZ4Pj3f/7r5qLQa3fOxqndIXZ2H3zm", - "DlHbE3TXXps1n0iW+qdxnfoncXPx+z//5WbybSdBmKbPqHK+Y9xA1an8fUbUjIiSpuAWWP9kjA74HDl6", - "KQ1f8SuVD4OWmInPiYjxoiTkLUzBzgAkbQ0qQRXwl/1Oi+xbpD9eI/J1b06hOKsbQrsDv1D3AOWB6bHm", - "b7sHtYEkB2Rn98I+7i6D1ADRLU1HU63DjvA094utOqa7uqUpgi968IVZxjg2zBtlumc05lz1h+zvM8IQ", - "rB0sMHlHQpBT2vBHx5fnEr2lcQxWNAiC5W1syF6VRIFpLpX+r8hYF40zhQRJuCLIKsgwSAawQOMxQRnD", - "7hywP2RlrNgJ1unKouWWCEbi0YzgiAjZEjPmI2Q/akQOTHWCpSLCSOgsreLr9G8XV6hzumA4oSH6m+n1", - "gkdZTNBVlmoe3qpirztkqSBzwsB+0soQtePyCeKZ6vFJTwlCHIgJdJb7Iewh1fzs8toec8qt/pC9JBqx", - "hEUkApjdLiGRmmGFIs7+rDkWtstSt+Xxa0j383I3mIdpVsXybh3Dz+FwUc9nToXKcKxFVkWb9J41mlNs", - "j9VgDsnL1osVRTnBYVU9JGprgJqe4Uh7Waf225xGUWq2Odec6PuObnI/VphJxZPSAQ7q1FxUtOrMqgqP", - "OY97Wv8B1WB5f/fqLwbc5cPQZGG6MovSJCVH07HH76mFIWVoSqd4vFBVO2JnsLz0fkS7/n2obgoUMORB", - "opHiq49K6QS5tm1ORiCsYKT4aD6hnp7zTbPwyVGJwlpUgiVa3UUvDall3y56O6N6m5XIIQE4+OaibJf3", - "h6wHIucIneYD5N3mXWrJCv5X6KLDRQkICq50NF5sIYxuLvroVQ7tnyViWNE5cZETMyzRmBCGMlDPSATj", - "gzgtA5BJLcOoqn9uZZUJstgC9wO37/pIm0UJtnJfk3eCFQ3BfTumtfnAsZlZKD2SFgCsvOu02iVWHTC/", - "JFMqlagdL6POy6cne3t7j+r6wu6D3mCnt/Pg1c7gaKD//4/2J9FfPo7E19dxVV5Yh3hZopxcn5/uWuWk", - "Oo56v48fHb57h9WjA/pWPnqfjMX01z18J5EmfvF0WnjyUSfTZp8TfZqqfP77kpu8wT//yW73jYJc3EHf", - "qu3HzO6Vbvk1wmJ8h7P2aHDzwJW6EFx7vFua3NJ89K9aPygov+TmsKcoIfWeF51SeftYEHyrrUrP/qq3", - "Zzky+47f/5hpO2q8QOSdVs9IhATnaiKNn6OqpuzsP9w/3DvYPxwMPNEgy0TMQzoK9a7SCoAXJ+coxgsi", - "EHyDOmDoRWgc83GVeB/sHRw+HDza2W0LhzGT2uEh16LcV6hjMfIXF1no3lSA2t19eLC3tzc4ONjdbwWV", - "VfBaAeWUwYrq8HDv4f7O4e5+Kyz4zM4nLjqnHm0Q+VyNaRpTY2T3ZEpCOqEhgvgepD9AnQS2JZJbfFWe", - "HONoJKwa6N0PFKaxXOnhNIPZliaYK8liRdOYmHewIK00XZj5KfTk8x5TxogY5cFLG/RkY5rWesbcXPIm", - "qBKbVkHdBZWgWRQKESVxdGQ4dK2cg9UsAHvdRAd2Di2p4Rl/S0QvJnMSl4nAbEca2IQLgnI6MYtWmRVl", - "cxzTaERZmjV4RhtQ+TQToF+aThEe80wZUx0WrDwInMSCjTDR4rpdIEDhf18aWtuZGzr+UsEnNPZMA4xW", - "+9Zu6c4l9mx/cNXb+b/GLc/ihZEDlBlDN+ER6dfCX6F96+ldNsGUxx6jMnRLc8pdEx73aG7tOoxYozvE", - "DI0JstukceqC26QYpBDwj3wCcyJwQsbZZELEKPFYWk/1e2QaGB8UZejicVVoauHcVt26rCwO6FsTHNrQ", - "0XbY91hytWl0S9h87V+ul8REyzQFp+ilEraNjU/po+d5tDc6u7yWqHAneUy86vI2HqJezhZSGyemRxNr", - "RlnZMgPibC2GL4sPrQ3rEcaJVwA5RkCd+TTNgA2vXvbOX9xsJxGZdyswgQtoxmOi4d4q6VZzF6JSnPhW", - "jormTSqyIQzZloFKuMo5uDWSSvzqwY7iCscjGXPlgeaVfongJercPDWhCRqCLkorS6l/L2GhQt8HXo7R", - "Eqlp2CsYsG5rVxh8rdsjMdtWeXqVQX2s8jPBsbkbUqXnItrRLTy/rS40v13LvbYT37jn7rSwRTjFycWp", - "scxCzhSmjIj8oK569g1xO0E36GllIMIkAZ/o5L9Wn4M3+G5yclll/Z8sBZZ/Fcu/IXhSC7l4TiKUYEYn", - "RCobPFkZWc7w7oODIxO2HZHJ/oODfr+/aeDCkyJSodVSbJtz3VIMQ1/OPm8dvkJ8Qpu5fAguj1/9HBwF", - "25kU2zEPcbwtx5Qdlf7O/yxewIP5c0yZ9zy4VaQ/nSxF+FeWN9V7lvn9SM+EkTAnSA5a4lrfpH8nf65J", - "M6bvSYS8gXYKT5HWv4HiPi+i7jNi44sLWqoUE18+JmgRH0/frza3nWIEbeyYGVM0Lq4OLBvan3T5Q66M", - "pV2Ko00Jy6Nn49g8hZzNNVf4QmkrAty9W1qMt1zcUjYdRdRDnX83L1FEBQkVhM+s56FgG6fpelL0K3+5", - "TGt7LcAGBXp2l28uyT/F4Vod/cX0r7/9P3n58Ned357d3PzP/Oyvp8/p/9zEly/aH9l4Qk5Wx4N+06DO", - "lWdq4GWsBHO2JY8LrEKP4jPjUjVgzb5BiqNEf9xHJ2CgHQ1ZDz2jiggcH6FhgFPat8jshzwZBqhD3uFQ", - "ma8QZ0h3ZY+Ot/THlybsRn/8wdmAH+t9RPaMWFgk5+EcMhtHPMGUbQ3ZkNm+kJuIhEMb/RShEKcqE0Sv", - "iNY14wUaCxwWZ8PF4F30Aafpx60hA0uUvFNCzyDFQuXB424EWGgLlTkUss1JhCCuSlpLdsjy/QNMc92J", - "wmJKVD93IYKjpnYw04AUr5nBRTW24XDQ9awj0u30QsZUKsJQ7pWgEogXdVyQyuGgwv6Hg8P15485Da0g", - "P6Du5evajihb8IchYBjaCOPRTKl0ffgCyBvDI+jnV68uNRr0v1fIdVTgIl9iY4zhNI0pkeZUTcWgk9i4", - "oK3Ad3JmVrflhF6ZxvqzuEUYxhMYGL16doUUEQllRn53Qo3OCQ31/OB8h0qZaVKkGB2fXDzZ6re4bw64", - "zeFfsY6v8hnWjhGcc2vZwoQvCqe5xm8XnZ92tTplObRQtODc9CkXKDYCpuDrI3QtSTWKAZbKHPGYlYwX", - "hYfMSPVhsOV6TOuS4gi9zPU7nIOSX2opiMF1WfAldGsDW8yh7lLv3SqscFxt7Rcr2uAIFytknd6wFTeL", - "gtXs78E48DxnjYGdrXi77LTUg/lJo1j7r66B7G1qS24aXV8NSisFIeYB9u0j479GhPmyXfWOqlHjiQzS", - "r+35i7Mebi7QDEv2ZwUvazbEzt7DVve29ahtzzLKpxh8YkDKucpFuOU+eBPrd0vj2BxtSTplOEaPUOfq", - "/Oxv58+ebaEeevHior4Uq77wrU+LQHtH2meX1xC9juVIMpzKGVfNUS0YuTZ6plLJ5QDAVnEYqwP7f64E", - "33sjKre+YES+yBiDkJL6NO4i1v5bxm38G8X5B58Tpb8yrv5zg+Otiv2VYuMbRbovrrwq3c3PXzbK/auA", - "U4lX90mlsibiQvk+OUS9G1BPGNOx1IKXROj8srgOXLisXPe1OT3a7e8cHPZ3BoP+zqCNAy/B4YqxL45P", - "2g8+2DUujSM8PgqjIzL5DAeiJWyjMuL4LV5INHRK/TAwVkTJfCgJC6v4tzqcXb4J8GmB/3U1Zl1o/yah", - "/O1i9Ffk6biqZuhorRk++MdnJfMgbfWBK2jsvhpt4tomKORZHGnta6w5zxhzJLI2pySqSH4CzHrNbhl/", - "y6pTNx5Ozb+/ZUQs0M3FRcUfLsjE5oFoMXGepo3rwNONlmF3jYK+FppSuPxdhMjXJWFpB/riAfFl552L", - "zDFU18KJV+ih3oNuygy69dqvmFPN/RKR+SjLfOqVfuVibK+vz08rC47xwc7h4PBR73C8c9DbjwY7Pbyz", - "d9DbfYAHk73w4V5DxqT2gS6fHrtS5dDmmHZAPLgyzTWE6EjzUB58Ms4Uyq8ZauY80XoqKinEJoIbvAsv", - "jW6se4DdNdRv4kWuM6/8+BJrRnXfpvDX6i+uZpnSahB8I2eZQvovAFlPwdocq7swPH+EnnP4xkLa1Rtl", - "zXgxzTGLxovl5nVDp2NjeASRigsSwWBWgB2hp7nQysWeFXMdSeyjkaU21g3i+LaMS8TaGXa1gm5gsR50", - "A4PCoBs4zOhHM0N4AuCDbmAB8YbJlunG5+onOAYZVoTRZIrG9L1hOQ06lYqGxtbDsJpNbGfvIpJoZLbQ", - "psM4E5tht9n8I8fVNxeoAzdH/oKsKaj/2soP7sostL/7aP/RwcPdRwet4k4LANdL4xOIHFoGbq1oDtNs", - "5DLHNUz95PIaNh+9scksMba9nXthMWrBEWptjzJUpKIrBn/Uf1QOt414No5LviIbbw8xnW3yBjacVP1G", - "4zmdTNhv78Pb3V8FTXbeHcjdsdc4ygfya5LnZf/mktlFxj1z791vQwJBCdkYNPySSJgBuiIKAf30tMDS", - "O2oe8GNJzoUWW4x7CWt/b2/v8OGD3VZ0ZaErMc4I7L9lKC8sBCUWg5ao8/LqCm2XCM706aIgU0Gknpy5", - "B+PlM2TzjwwqAZLa9tjzUUmDwlJQje17njSi/MZqLHZSFukQt5RrM0tc7sX23t7g4f6Dwwft2NhaPCPx", - "brWEse3seb8gIaHzysp3wCf+6vgS6d7FBIdVDX9nd2//wcHDw42gUhtBpQRmMqFKbQTY4cODB/t7uzvt", - "ot99fm97r6PCsFXZ5WE6D1F4VsODimXR223aLXxa4nKw5Mr4zCLgsx7dt0k4b3GXj0rolZYiSVFHK1Fl", - "hbR0H22rjZ/BLyL1OE35aLW62DbSdnVg7SVWs3M24csHG5sYfDZcyTm8U634SMjUFxFGSeRkV275WV0K", - "AqBiSVCUEYs5oxsJbBGOzeFOitUMlFX4kLJpNfR7acA2ZpiBYfXNTRjXNmzjMZL+EJtXIgNcGc+yRLgI", - "tmnlJqdy5LcqljsWZJrFWKB6NPkKkOUiiSm7bdO7XCRjHtMQ6Q/q5vyExzF/O9Kv5E8wl61Ws9MfjIpz", - "5Zp5boCzUQVmQWrjFlP4Sc9yqxanBDv/tvl+GxKOt3HAeQ+bnmrjzQRcXzP6rkTo1WtQ+7uDprC0hk4r", - "AWnLwfqbynZLsj6Od3H0x3m6EM+hpjk2qlmwVT24Ml/fbOFcclUQ3rImgDrOp+eumVXxWrru1Wojbnc0", - "WvdeO2i2JQmro+8fPnh40PK+3Wep2itSMn+GYj1PVijUDSt10UZrO3xw+OjR3v6DR7sb6UfuoKNhfZoO", - "O8rrU8sKVNPZHgzgfxsBZY46/CA1HHdUAapk+PlkgD6uYN3iCkyD1b2qHEKxks7Mryrg7VTcFdrScUXl", - "KiXf65DJhIDjaGTw1iuAqYVktYIhxCkOqVp4LED8FqJUUN6kdpWjRe81YD0otX0jPFFEwGmEzMZFFEDH", - "DY7+01h2NVo4bH1tV2bjJivyRX1UY0OasK6o5qFo4SAwFOE7in+bIxO9xbLi1dfPoSJRt5RcsX78Y1q0", - "T3/taD3PgF0cp/uuI/mzXZeXv7acJaujoiTXMb5qC21mQa0RQMxYGwe7Z0f23HEK14dy1OSD3QA/7avR", - "uHyhfmXGgsrt+2LX3Xzcdmkhl78zO9jm45VO8Df5sH63GOjRwmBRXvTdrZCEj5rM+UpT4prE1QmqXT2m", - "pvKCvV+GSo1RhySpWrg7FM4y3drsvOc479BLjF846m3w6EvE3V+vDLT/N0mFVD5ic4OsPVxbWtPG6Fa/", - "unpaD18xNqFNBVENt6hdcJdqRYGRVcWsTFUpMPhsZPk0q1+F26CAVZOJX3COqxziKlits1xX+tNKMytB", - "0rw25nz1M6t9UenKfH0iyqz5tT5U25xRaQO4V88VYm4aCwr2nEWQQaxGQW6iL/sBVod9XOB3+QhgLWOJ", - "atkVzTxK2aPPHkP2gJcuZwSduC4AjHqezMefVwbNUdXyYqyqi+ZO8L2MZ+XPConWxFs14izG6K4uvaZF", - "FwkzQdXiSm8INjiNYEHEcWbIEHYKmAT8XAwO1xU+fgQzdeLRVs8II4KG6PjyHKgkwQyy7aKbCxTTCQkX", - "YUxstPnS2S4kS3hxct4z12TydIZQ1UQBQlwesePLc0hhZOuJBIP+bh8yUfOUMJzS4CjY6+9AkiaNBpji", - "NtxChEfriNJ8CDvZeWR33MemiUatTDmTBjm7g0GtPg0u0sRs/yqNh8Vsr62VQlMAbDneYikg0mkCFvyP", - "3WB/sLMRPGszu/iGvWY4UzMu6HsCYD7YEAmfNOg5M1a1S4tNbMOCZoOjX6rU+svrj6+7gcySBGsV0aCr", - "wFXKZZMKQyTCiJG39nrqr3zcR1fGJoFY86K0onEZkEiLJIwUFv3pe4RFOKNzMmRWEpssPVjAXZwEaQls", - "bkJUycwMbVbfsDCR6jGPFjXs5t1t6+56Lqq2QPDGhXvylJNpQwUfn3Q0ma1kyL0pvQjDTBWJkkxKq1sC", - "h5gT+s7bYavTeC08yjXtXHTn7pbfDwihyn4X+mn+ztWPqm4YWoemLIyzqNhVq3V7vFfvTf0Zm/nrlniU", - "kDNoYZFSjup22xfjETGxsulCzTgzz9k4Yyozz2PB30oi9CZnb+pYXGtbPK/7Z9Iq0gRuy5i7vXrMbQPi", - "9odbsvjYH7LjKHF3sW12XxxLblOimagHKlGeY3rImivG+ZXpE5vK1KQjctcMCzB5ptJM9ZGZCFH2ehE0", - "pxKlmZyRaMgURx+EydG4+Lj9oRjxI2inBEeaTkpNzJS2P9DoYxPUcoT17EdjV3axprMTQMAw0FrDMNDP", - "U4G1dprJGcIhxGboH8tL2jGMzQXs/Ft1DIeYoZSnWaz1KCAqk+mt0gdc1cRxjBSwkvtW6xOwkg3zsS5d", - "X34g6881DrgaG0GmoBIzDfYP/fwkSSiIzyz969WL5wi2KihvBc2K6wOAI8q0opFnVNaj94fsCQ5nyOgg", - "kGl0GNBoGBRljLYA1kwSowb0eqDE/AT13cwwXRr91O/rrox+dIR++WB6OdK8lCYjxW8JGwYfu6j0YkrV", - "LBvn7177EdrkFruqCALUMbJ/y12e1zMsbYNm38AsQtzK2niBMCokUNn6HVOGxaKpJhjPVHNokcktYJsV", - "63kwGGytPzqyU/VohpWGmhM+LilCu19MB7D6z7IOUKr/qXdcZhNHREbzuQMl5DGO3H3GH9rWGm3Lmokl", - "PQq+L4tkQ74xMaGsNWUIysQ5ZSjFAidEQUb3X/w0D1G8VP/tDnphJzJOkyrxdkvoqdtOr5cIe7+x/l5e", - "yQ5oYf8O6A/GLdL5wbiP7mpcHJtk0nlN4HtFjrBYjhC7fkPvjKjvgeIGdyVKXdbRb0i/94V+zohVwQqk", - "1aTZNpRxKHsR6rdNBMGJtL2YxtpsvAKYeleEKQSVX2Xf/uuMDwjkfxPz6ZsjZFAY27q30uaRzN3telO0", - "uISPTJqd/DubfSqcYTYlEnXM/vn7P//lanf+/s9/2dqdv//zX8Du27YSNXSXV519c4T+RkjawzGdEzcZ", - "CE4lcyIWaG9gC9fAK08uKzlkQ/aSqEwwmYd26XkBTkyHYAUwmA9lGZFIAgohy/zExhwZb57Hmna8bFB5", - "pxzdXTJ/7AxKE9C7oqMBOESmjCqKY2sKOTjg/lMBiJlzUB687phcclWvly+KvFOGensGwA0FjKna7OE7", - "U8jY9Ik6V1dPtvoI1H1DFRBXBnZD0Y21BPo/ZNJ6mWQkSlWgAJaNbCqlMW10a57aNnfh12xKcdrs2DSG", - "PNG2sZvMD7W7hZPTjzfn8PR5HU9d2v1mt+Onz9dX1LmVTfnl1tnR3jLObU2JAmXfwppEHZsOPM/6Uylc", - "8a2I/k4EcKneSS6FETe5hu7MwjnhbBLTUKGeg8VWyc2tniqB3Bdx8NJCjbCbV/0yRHmr2K7E9jVuGnmY", - "313uHrVBN9lGigsbBa392EnWkc4plSHX35aopRfi1OY8AiQWfFqmonW+nVP4Pd9yVirmed1qx5B35+Wx", - "Q2esvjfcgVA8rQnEbygIaxlVSlec7hM1X+er6EoMrXACfV+kObg7LeiuHUI+Mr9PHqGohjYtBWd5Fv4m", - "8rJ5+r/iQtsRPBO/IsJxtQHUZPIopmU+ReGMhLdmQrYS1iqN4NwVy/r6eoApNrDB7m/B/7HdtzAcC1yt", - "MhbPbXqXr2crwggbmYpf7vjREpgHyRCOMHaOVJM5BcsFC7f+UCeQd7Iz1CtX3SNOuszi2Dni50SoouRC", - "WZ5uf4DAlfV6suO2lbrI9ctnPcJCDpFKeZSNXyFxGda/rLZsFsxM5QeZtLGvAFWOMJqV0c9YfxNQhvJ0", - "nX/afWoTdv5p96lJ2fmnvWOTtHPrqxHL4K5E811rr/eY+LTySqtIA9Fk8qCv0/byVnei8NmCE5uofDmA", - "P7S+NlpfGV0rFb+89sdXVP1sSYVvc06QE5sP2/DKxZ/9wVS+u3U9WYosVcms+OJtDhkuijIGtsbe/QuQ", - "oznFleVvSx9qwZArtQNHuuenXVuhwtSVyGPx78ij6uC4cy3Rjnv37tTjZEynGc9kOcwfCpIQWVRvrgjg", - "+6a/Fttzowb7HVPp4C63jjtXUH/Q/VdSnesLaoS3Lfi8Rnl2rdaww1Maq1JmewmlP0w+WXNN6KXLU2uz", - "wW41BI25LMxtybiSBHw5mM0Hl6vBUCrLkGIqZB9dS6LRRNIX5oaHVIuYHA3Zf7tPflEEJ69/GuPwlrBo", - "mA0Guwf5O8Lmr3+SCm6KDtlFXu2eKUGJRFgQdPz8FA6mpnADt4+O47i4ClWHByWZtKXbXC2iNIZ8X0Zy", - "+NBXqi9RYLB1CYulO9+AAMiL4nASfLZYamlHFad27Q0pR6w/DKlWhlQJXasNqTx78te0pMwg38yUcvTm", - "Q7hNHPDDmLoLY0pmkwkNKWGqyPy1FOBkEwfewytGzJ7HlAITKvtxa2OqSGm+Wk+1xPstglLywe/ehnLp", - "Be9nqDQ3lyMiZ7UUm2Gz2fK90cPgboXz3Zsr95nEzsp1QP2GgbknFPPp+ltCeU/uSoznmtCQuaKhb4xQ", - "f4NyQkWKI0lirXK/ndFwBleG9G/Qv7lRhNP0TX5HeOsInUEocvnWMgzekURQHEOdAh6bEhtv5kny5mg5", - "U8vNxQV8ZG4LmZwsb46Qy86S85jUrcpXgPQsYiwVem4vNnX0ggvuih6+0fgszW/LXg4qrlMPme+iECNv", - "bYd0gt6U7gy9abg05IjwmV6lb8T53eYkGGYuiiMBiDNJHQiLGmw/jTX/daGdgTfhWMurSwaMr3xzaQmY", - "Z3yaJ+CokDJO07bka8EEKp4nyQoaRp1SGQ6pIp6pv0gVEWHqfFvqbiJu1MGh+UPhW1OVulJd0hR+8drr", - "5hq+F1WBqb3v6sWYv+ZJEphSlwn21X/5/Ctg9Q6XDUa9MqV7Xj/2jE1ucFWFfekKV23nsIWHIMeP17p8", - "aRr84TUXV6HpG5PhN7D0CigoFG5i0XgBa1uUvrpf91dgIYuZwX5n5+XlEfeukUdsxaw/PI8U9PEH55KQ", - "C2Fc07bs5f0JNCxZHCV270CdvaJ+XddZvTcXF1tNTGNq7TeyjPg+zOFP85XWSlwkkT+buqCRSzp2cnFa", - "1DUXGeujFwmFTGC3hKSQ14DyTCI4W+iX8ys3nQDkCZQJU2KRcsrUWiiKpl8HmI+flOnpjuWUDbX+w2/l", - "cMZ3/4SUKXaM8wms8tFqOaQaXSPOVVCpBjvmme59KVc0VEGSC6lIYvwkkywGJoKLLTaFCC5XeeoiqiTU", - "HOiCp7BU4WfIxmSi1ZCUCD22/hzSRhYmn8+bcKVwLjUvjej7PtwJkD4aLGismrBWK6WUpi5ztM9kzZNd", - "fzJIT8E/UK0yJVEnpremdCqaSxTrh62VDgZTgupLJ0j5dM7Ki6z5Lr4bms2J+Y8g4c5rYs0VEb53Yu2M", - "lJnFyR9YaL9Yk2vlmtiwCq/DXakab3/ILogSug0WBIU8jqH4ijGbtlPBw22oEBqmNDKlQgE4EHjNrxMY", - "8eTyGtqZFKPdIdN/LNeorAPqSl2eb79Y43I11Yn/je0xM8FVbOFf8B/etM1PYBp5SDawKE9XGUA8/cM7", - "DKwG98NbcD+9BXAEns+mMxU4BKVY2rLyfs+ArcW4/cE8nK8LpFA4nN240jjfh7ZrK2msG8ZN8F4wpZ1T", - "RExijrvnSZ4XO7mnly814twUQIkph4T4dwFTROmPRt1fPjyxjMeNghPvlLdc0pvvhrfueuezMLj4wDI+", - "7gubG0pzM4ESBGXvkyhXc1xpm7lie1BaNFctXZHJbrnWqcmxm/uQiiJZeVnF/pDldSRdjl9tXXWdaYUi", - "Km9ND9Z66iN/uU9j59man1AgI8RxaCpL5HUvTckH2WB9vSzVgv1q/FYM4lnovOCnzOsz3ieTw08TsHrl", - "ApBAcVadWnlD5Ma2uYtLAXYz2+BKgJvBjwsBLS4ElJDVptyUqd5ppZUtu1jcZZH0PZzu+C4U5ErJ17tO", - "8An79ZcjD0enjbv1j4sEd6YQFJeyz0/v/+2BMs9VZPS2tgp6tpZb2TW0ioMtilJBeq7CU2QQZvFhbI16", - "qbj+kL2aEfcXoi6ClUQoooKEKl4gyqAQl6v4+WeJBOfKvudi0VxSzrDIU8GTYzubNcZL69q3voOYjTPG", - "dD31PmmSJabWJ2Xo7DHU+hcmoBJNMI0hnNehlLwLCYkk0ORWvaauN8IyL567FsoVobF51bzQVDeb55ZY", - "B2eK96aE6bUoasmlgs9pVC+QXilO7IMWLMQvYKRN39O0ynprK04tM16VblFePM+WvCro061O8GObqOf4", - "hpuyXORIVJyjGIsp2fqxldznraTsTXL7RmVHaXcPrZ2DqaXf52vcQcudj3d7A+3m+/GJlHIi38OUHfPc", - "6Gu6+vZ9keDg7vaHu77ydnOPfehnxBm4petu0IHu0Ucwz3iIYxSROYl5CnX3TdugG2QitlXEj7a3Y91u", - "xqU6OhwcDoKPrz/+bwAAAP//OrJd9TvrAAA=", + "H4sIAAAAAAAC/+y9+3LbOLIw/ioo/s6plc9KsnyJ4/jU1CnHnni8Gyf+xbH3OzvKp0AkJGFMAhwAlKOk", + "8u8+wD7iPslXaAC8CZSoXJx4J1tbE5kEcWl0N7obffkQhDxJOSNMyeDoQyDDGUkw/DxWCoezGx5nCXlF", + "fs+IVPpxKnhKhKIEGiU8Y2qUYjXTf0VEhoKminIWHAWXWM3Q3YwIgubQC5IznsURGhME35Eo6AbkHU7S", + "mARHwXbC1HaEFQ66gVqk+pFUgrJp8LEbCIIjzuKFGWaCs1gFRxMcS9KtDXuhu0ZYIv1JD77J+xtzHhPM", + "go/Q4+8ZFSQKjn4tL+NN3piPfyOh0oMfzzGN8Tgmp2ROQ7IMhjATgjA1igSdE7EMihPzPl6gMc9YhEw7", + "1GFZHCM6QYwzslUBBpvTiGpI6CZ66OBIiYx4IBPBnEY08uzAyTkyr9H5KerMyLvqILuPx4dBc5cMJ2S5", + "01+yBLOeBq6elusf2pb7fr7v65nyJMlGU8GzdLnn85cXF9cIXiKWJWMiyj0e7ub9UabIlAjdYRrSEY4i", + "QaT0r9+9LM9tMBgMjvDu0WDQH/hmOScs4qIRpOa1H6Q7g4is6LIVSG3/SyB9cXN+en6MTrhIucDw7dJI", + "NcQug6e8rjLaVHfFh/9PMxpHy1g/1o+JGFEmFWYNOHhuX2pw8QlSM4Lsd+jmAnUmXKCIjLPplLLpVht8", + "1wwrJopEI6yWh4OpItuGcoYUTYhUOEmDbjDhItEfBRFWpKfftBpQELxmON2i1WDLpJaZnRwlsql31wRR", + "hhIax1SSkLNIlsegTB3sNy+mRDBECO7hUD/rxyghUuIpQR3NNjXvZkgqrDKJqEQTTGMStdojHyKYxfzG", + "x4hGhCk6oVX6NujUw+NwZ3fPyzsSPCWjiE7tSVTt/hSeaxTT/SgErf0L0YS2aLcOGFKQyfJ4z4B1wyCC", + "TIggGsc/c7hU8Dlhmlr0eP8B4wb/33ZxRG/b83kbgHlZNP/YDX7PSEZGKZfUzHCJc9k3Go0A1Ai+8M8Z", + "Xq3a6xJGSYXFavqAFl+AEs38WsHmyjSt80Ngd7abCmU3sr2f54R5BJ+QM2VfVFf8nE9RTBlBtoWFr+Zz", + "eoCfYg5s7kusrRsUIF0maD3vT2BI5kFDb/pdNyAsSzQwYz4tQ3NGsFBjUgFmw7FkOypm1wj+ywpJ1M4f", + "LMloNVe4pIyRCOmWllhNS5RJkD6Xlg+UcUvVaE6E9NIRTOuvVCHborGrmIe3ExqT0QzLmZkxjiKgQRxf", + "VlbikcAqIi1ONWNzHYJkIJHi6OqX491HB8gO4IGh5JkIzQyWV1L6Wndv2iKFxRjHsRc3mtFt83N3GUP8", + "GHCVE0bTeZJjoENMw70Cu5u6+26QZnJmfgE/1rOC80yzAY1esf79xrPoE2ASRvJv1IP8ct3L1Gw2msZc", + "w3SBMkZ/zypCcx+da/lfIc38aUSiLsLwQrNhnCnemxJGhOZTaCJ4AhJUSbBFHdKf9rtoqGW9npZse3i3", + "Nxj0BsOgKprG+71pmmlQYKWI0BP8v7/i3vvj3t8HvSdvip+jfu/Nn//DhwBtpW0n6dl1dhztd5GbbFkE", + "r090tXi+QsL1cRGzfeea9jfdvZPz5QPezD/i4S0Rfcq3YzoWWCy22ZSyd0cxVkSq6mpWt127PpjbioWx", + "qV76hkurKRyAbp2Y3xERak4ZE40gsquZJVWyi7DWWYHJIH2a/TcKMdM4aw52LhBhEbqjaoYwtKtCIFn0", + "cEp71Ew16AYJfvecsKmaBUcHe0v4qJGxY3/03vyXe7T1P16UFFlMPMj4imeKsimC1+b0nVGJijlQRZK1", + "x62DbhaDiJVQdm4+28lngoXAC/+uucmt2j2jHDVuX5h4JOmXcyIEjdyJdnJxijoxvSUWLZHIGBpmg8Fe", + "CA3gJ7FPQp4kmEXm2VYfvUyo0idJVhyQxrrSL2/hrwEJZxzO+DjmekE5+BoECAcXp2h6tujUWSYkstou", + "nGkY7E6wZWeX19uaq6RYSjUTPJvOqrOyLG2z+VB5O6J8NE59c6LyFp1vv0Sa4aKYaujkDHZnMLh4ui2H", + "gf7jkftjq49ODchg+nr/uLB8X86wICB9RIgzdHJ5jXAc89DqcxMtJE7oNBMk6tfMCNC7D+EJU2KRcuoT", + "PmuYUTRdRpBer3i7AR5sjynblnobeuFmcCds/hki0M9sTgVniRZD51hQzbcqRp0PwYuXpz+Pfn5xExxp", + "Ioqy0FpILl++eh0cBXuDwSDwSRkag9bwgbPL6xPYKd1+xlUaZ9ORpO89rPU4Xx9KSMKFEf3tN6gzq3Je", + "Ixkh2JxhsHf21CDXzhnglduUiEpo7XoxHVcxZvfsqQ9bZouUiDmVPp3/l/yd2/kSnzSMqYrbkog5ETnS", + "Ahb3S3JXGPMs6pWG7AYTKkgosEa7oBv8ThItgMzfa9Qp5u75zq+Ktzqk15y+OE4pI43HbzdIiMJghv50", + "bL2WRPQiMqFa+bgli94cxxlBrmcLZ5KDuYrIaSZSLk3/eGqEVkVwEhwFYw1JFnnx+Ds56u+4uI05jno7", + "X/ikZ0TpvpeX+MK8qOKlD8Z19ZFFdzRSs1HE75iesuc8sG9Q3jg/FN7pleD4X//4581FIdfunI1Te0Ls", + "7D76zBOidiborr06a76QLPUv4zr1L+Lm4l//+KdbybddBGEaP6PK/Y4xA1WX8rcZUTMiSpKC22D9yCgd", + "8Dly+FIavmJXKl8GLRETnxMR40WJyds5BTsD4LS1WQmqgL7sd5pl3yL98RqWr3tzAsVZXRHaHfiZumdS", + "njk91fRtz6A2M8knsrN7YX/uLk+pYUa3NB1NtQw7wtPcLrbqmu7qlqYIvujBF2Yb49gQb5TpntGYc9Uf", + "sr/NCEOwd7DB5B0JgU9pxR8dX55LdEfjGLRoYATLx9iQvS6xAtNcKv1fkbEuGmcKCZJwRZAVkGGQDOYC", + "jccEZQy7e8D+kJWhYhdYxysLllsiGIlHM4IjImRLyJiPkP2oETiw1AmWigjDobO0Cq/Tv15coc7pguGE", + "huivptcLHmUxQVdZqml4qwq97pClgswJA/1JC0PUjssniGeqxyc9JQhxU0ygs9wOYS+p5meX1/aaU271", + "h+wV0YAlLCIRzNmdEhKpGVYo4uxPmmLhuCx1Wx6/BnQ/LXeDeZhmVSjv1iH8Ai4X9XrmVKgMx5plVaRJ", + "712jucX2aA3mkrysvVhWlCMcVtVLorYKqOkZrrSXZWq/zmkEpWadc82Nvu/qJrdjhZlUPCld4KBOzURF", + "q8asKvOY87in5R8QDZbPd6/8Yqa7fBmaLExXZlOauORoOvbYPTUzpAxN6RSPF6qqR+wMlrfeD2jXvw/U", + "TY4CBj1INFJ89VUpnSDXts3NCLgVjBQfzSfU03N+aBY2OSpRWPNKsEiru+ilIbXk20V3M6qPWYkcEICC", + "by7Kenl/yHrAco7QaT5A3m3epeasYH+FLjpclCZBwZSOxosthNHNRR+9zmf7J4kYVnROnOfEDEs0JoSh", + "DMQzEsH4wE7LE8ik5mFU1T+3vMo4WWyB+YHbd32k1aIEW76v0TvBioZgvh3T2nrg2sxslB5JMwBWPnVa", + "nRKrLphfkSmVStSul1Hn1bOTvb29J3V5YfdRb7DT23n0emdwNND//3v7m+gv70fi6+u4yi+sQbzMUU6u", + "z093rXBSHUe938dPDt+9w+rJAb2TT94nYzH9bQ/fi6eJnz2dFpZ81Mm02udYn8Yqn/2+ZCZvsM9/stl9", + "IycXd9G36vgxq3utW34Ntxjf5ay9GtzccaXOBNde75YWt7Qe/VTLBwXml8wc9hYlpN77olMqb58Kgm+1", + "Vuk5X/XxLEfm3PHbHzOtR40XiLzT4hmJkOBcTaSxc1TFlJ39x/uHewf7h4OBxxtkGYl5SEehPlVaTeDl", + "yTmK8YIIBN+gDih6ERrHfFxF3kd7B4ePB092dtvOw6hJ7eCQS1HuK9SxEPmz8yx0byqT2t19fLC3tzc4", + "ONjdbzUrK+C1mpQTBiuiw+O9x/s7h7v7raDgUzt/dt45dW+DyGdqTNOYGiW7J1MS0gkNEfj3IP0B6iRw", + "LJFc46vS5BhHI2HFQO95oDCN5UoLpxnMtjTOXEkWK5rGxLyDDWkl6cLKT6Enn/WYMkbEKHde2qAn69O0", + "1jLm1pI3QRXftAroLqgEyaIQiCiJoyNDoWv5HOxmMbE3TXhg19ASG57zOyJ6MZmTuIwE5jjSk024ICjH", + "E7NplVVRNscxjUaUpVmDZbQBlM8yAfKl6RThMc+UUdVhw8qDwE0s6AgTza7bOQI84+J27V2ZPl1HImNM", + "d7NWyz6OY36nt/hWwwZOZozs184VoiTI5Sq1MTzY9xK9Ml8Yw0TxOM0UokxxrZSzaLzowkgkgnYMCSIV", + "B06Kw1stNdpu2kqMflnkhRZCnNnTjFfwznuy+fYmxuT2JQ2/CospUSOpsForsWhMeQ3tr6B56+v0+ofL", + "IgG8BJtTM5TxRBEBT50vDCk7plwpnqag0V0ZvAi6gd366o2Ie+iBRnEXtTTFs8vrTY3gqeATGnuWCwYc", + "+9aKt848/Hx/cNXb+f/NFRWLF+ZMpMwYfRIekX7NFRzatyP1s8vry6Y55X74qDy7pTXlZjoPieSWHwcR", + "a4AKMUNjgqzIaJAdTIjFIIWw88QnPEwETsg4m0yIGCUeq8Mz/R6ZBsYeSxm6eFoVILSg0lb1uKxsDuge", + "ExxaN+p20PdYNWrL6Jag+ca/Xa+I4XtNjlp6q4RtY321+uhFHvmAzi6vJSpMqx5zR3V7Gx0KLmcLqRV1", + "06Pxu6SsbKUA5GwtklwWH1p7jkcwSbyHsSME1JlP0wzI8OpV7/zlzXYSkXm3Micwh854TPS8t0rcYu7c", + "tQrvhwqTmDepiwYxZFsCKsEqp+DWQCrRqwc6iiscj2TMlWc2r/VLBC9R5+aZcdPRM+iitLKV+nkJChX8", + "PvBSjOZITcNewYB1u1OFwNeaABMjwpWXVxnURyq/EBybOKkqPheev27j+W11o/ntWuq1nfjGPXc35y1c", + "i04uTo0sFHKmMGVE5JfWVT8Q8GELukFPn1ERJgncD0z+e7VPSIMdM0eXVZawk6Ugi69iBWtwJNZMLp6T", + "CCWY0QmRyjoSV0aWM7z76ODIhDBEZLL/6KDf72/qxPNz4bXTaiu2jY9DyZ+nL2eftw9fwVenzVo+BJfH", + "r38JjoLtTIrtmIc43pZjyo5Kf+d/Fi/gh/lzTJnXN6JV1AudLEW7VLY31WeWeX6kV8JImCMkB41prZ2+", + "QXDXqBnT9yRCXqdThadaCjcY93nepZ8RJ1IEK6pSfEj5yqxFrAh9v9r05AQjaGPHzJiicRFGs2x0+qRA", + "KLnSr3zJpzwlLPckj2PzK+RsrqnC51ZeYeDu3dJm3Bn9cxRRD3b+zSqnERUkVOBKtp6Ggm2cputR0S/8", + "5TytbYiMdZD1nC7fnJN/yuVDdfSX07/8/n/k5ePfdn5/fnPzv/Ozv5y+oP97E1++bH996XG/Wu0b/U0d", + "nFfeL4PFveLY3BY9LrAKPYLPjEvVADX7BimOEv1xH52AgnY0ZD30nCoicHyEhgFOad8Csx/yZBigDnmH", + "Q2W+Qpwh3ZV1o9jSH18aS4T++IPTAT/W+4isv4SwQM5dm2Q2jniCKdsasiGzfSG3EAkXmPpXhEKcqkwQ", + "vSNa1owXaCxwWPhJFIN30Qecph+3hgw0UfJOCb2CFAuVB1K4EWCj7azMBaltTiIEPobSarJDlp8foJrr", + "TowdpZ8bK8BoWTM5NQDFq2ZwUfXzORx0PfuIdDu9kTGVijCUWyWoBORFHeewdTiokP/h4HD9XXyOQyvQ", + "D7B7OXWBQ8oW9GEQGIY2zHg0UyptYWTU/MbQCPrl9etLDQb97xVyHRWwyLfYKGM4TWNKpLlhVjHIJNZH", + "bivw2QTN7rZckDFmwWdxC5ekn2Fg9Pr5FVJEJJQZ/t0JNTgnNNTrg7tOKmWmUZFidHxy8fNWv0XuBYBt", + "Pv8V+/g6X2HtSs0Zt5psdjnGa/h20flpV4tTlkILQQt8CJ5xgWLDYAq6PkLXklQ9emCrzHWn2cl4UVjI", + "DFcfBluux7TOKY7Qq1y+w/lU8gCvAhlclwVdQrfWIm0cHJZ671bnCq4bVn+xrA3cGbBC9gIIjuJmVrCa", + "/D0QB5rnrNHJuRVtl42WejA/ahR7/9UlkL1NdclNI02qDpolh9w82KR9lMjXiLZY1qveUTVqvJ1E+rW9", + "i3Taw80FmmHJ/qTgZU2H2Nl73CqHgR617b1e+UaPT8yUcqpy3p75fZTxe72lcWyueSWdMhyjJ6hzdX72", + "1/Pnz7dQD718eVHfilVf+PanRdCJQ+2zy2uI5MByJBlO5YyrZg8vjFwbvVKp5LIzbKsbptVBLr9UAlG8", + "3sVbXzA6xV3LLS3jPuJOvqUP079RzEvwORErK2NMPjdQxIrYXylOpJGl+2IsqtzdPP6yER9fZTqV2A0f", + "VypLIs6t9ZPDNboB9bj0HUvNeEmEzi+L0PjCZOW6r63pyW5/5+CwvzMY9HcGbQx4CQ5XjH1xfNJ+8MGu", + "MWkc4fFRGB2RyWcYEC1iG5ERx3d4IdHQCfXDwGgRJfWhxCys4N/qcnY5KubTgmDqYsy6MJdNwlraxaus", + "yFlzVc1W01oyfPT3z0psQ9rKA9axwX412sS0TVDIszjS0tdYU55R5khkdU5JVJEICIj1mt0yfseqSzcW", + "Tk2/v2dELNDNxUXFHi7IxOZEabFwcIho2AeebrQNu2sE9LWzKYWO3Ee4SJ0Tlk6gLx4cUjbeOS81g3Ut", + "jHiFHOq96KbMgFvv/Yo11cwvEZmPsswnXulXzt/8+vr8tLLhGB/sHA4On/QOxzsHvf1osNPDO3sHvd1H", + "eDDZCx/vNWQPa+/o8um+K1UKbY7vAMCDKdOE5ERHmoZy55NxplDufqWJ80TLqagkEJtoBrAuWE8h3QOc", + "rqF+Ey9ymXnlx5dYE6r7NoW/Vn9xNcuUFoPgGznLFNJ/wZT1EqzOsboLQ/NH6AWHb4RzoWO8rryY5uAp", + "tdy8ruh0rA+Pc66DwSwDO0LPcqaVsz3L5jqS2J+Gl1q/T/Bp3TImEatn2N0qOW11AwPCoBs4yIBz17Kb", + "l52I12W8jDc+Uz/BMfCwwo0mUzSm7w3J6alTqWhodD0Mu9lEdjYul0Qjc4Q2XcYZ3wx7zOYfOaq+uUAd", + "iKL6M7KqoP5rK7+4K5PQ/u6T/ScHj3efHLTywS4muJ4bn4Dn0PLk1rLmMM1GLotiw9JPLq/h8NEHm8wS", + "o9vbtRcao2YcoZb2KENFWsZi8Cf9J2XX84hn47hkK7KxJ+Df3CaHZsNN1e80ntPJhP3+Przd/U3QZOfd", + "gdwde5WjfCC/JHletm8uqV1k3DM5IPw6JCCUkI0O9K+IhBWgK6IQ4E9PMyx9ouYOPxblnJu9hbgXsfb3", + "9vYOHz/abYVXdnYlwhmB/rc8yws7gxKJQUvUeXV1hbZLCGf6dF6QqSBSL87EhHnpDNlcPIOKg6TWPfZ8", + "WNIgsBRYY/ueJ40gv7ESi12UBTr4LeXSzBKVe6G9tzd4vP/o8FE7MrYaz0i8W81hbDt73y9ISOi8svMd", + "sIm/Pr5EuncxwWFVwt/Z3dt/dPD4cKNZqY1mpQRmMqFKbTSxw8cHj/b3dnfaRYL47N42xqlCsFXe5SE6", + "D1J4dsMDimXW2206LXxS4rKz5Er/zMLhs+7dt4k7bxHXSiX0SkuepKijhaiyQFqKzdxqY2fws0g9TlNu", + "Zi0utvW0Xe1Ye4nV7JxN+PLFxiYKn3VXcgbvVAs+ErJWRoRREjnelWt+VpYCB6hYEhRlxELOyEYCW4Bj", + "c7mTYjUDYRU+pGxadf1eGrCNGmbmsDqKGca1DdtYjKTfxea1yABWxrIsES6cbVqZyakc+bWK5Y4FmWYx", + "FqjuTb5iynKRxJTdtuldLpIxj2mI9Ad1dX7C45jfjfQr+ROsZavV6vQHo+Jeuaaem8lZrwKzIbVxiyX8", + "pFe5VfNTgpN/23y/Dcn32xjgvJdNz7TyZhyurxl9V0L0akjg/u6gyS2todOKQ9qys/6mvN2irI/inR/9", + "cZ46x3Opaa6NahpsVQ6urNe3WriXXOWEtywJoI6z6bmQyypcS6GPrQ7idlejdeu1m822JGF19P3DR48P", + "WsaefpaovSI9+WcI1vNkhUDdsFMXbaS2w0eHT57s7T96sruRfOQuOhr2p+myo7w/tQxZNZnt0QD+t9Gk", + "zFWHf0oN1x3VCVWyXX3yhD6uIN0iBKZB615VGqTYSafmVwXwdiLuCmnpuCJylRJRdshkQsBwNDJw6xWT", + "qblktZpDiFMcUrXwaID4DrxUUN6kFsrRovfaZD0gtX3baDzNuWQ2LrwAOm5w9F9Gs6vhwmHrEHaZjZu0", + "yJf1UY0Oady6opqFooWBwGCE7yr+LgcmusOyYtXXv0NFom4p0Wj9+se0aJ8K3uF6ng2+uE73hSP5M7+X", + "t7+2nSWtoyIk1yG+6ghtJkEtEYDPWBsDu+dE9sQ4hetdOWr8wR6An/bVaFxOLrEye0clE0Vx6m4+brsU", + "qcvfmRNs8/FKN/ibfFiPswd8tHOwIC/67lZQwodN5n6lKYlT4mpm1cLwqalCYuPLUKkx6pAkVQsXQ+E0", + "063N7nuO8w69yPiFvd4GT76E3/31Skf7f5O0YOUrNjfI2su1pT1t9G71i6undfcVoxPatChVd4tasgep", + "VhTbWVXYzVRYA4XPepZPs3oo3AbF3JpU/IJyXBUdV81tnea60p5WWllpJs17Y+5XP7PyHZWu5N0ngsyq", + "X+tdtc0dlVaAe/W8OSbSWFDQ5yyADGA1CHIVfdkOsNrt4wK/y0cAbRlLVMs0atZRyqR+9hSyB7xy+VPo", + "xHUB06jnjH36eSUBHVYtb8aqGoHuBt9LeJb/rOBoTbRVQ85ijO7qMoSadZEwE1QtrvSBYJ3TCBZEHGcG", + "DeGkgEXA42JwCFf4+BHU1IlHWj0jjAgaouPLc8CSBDPIPI1uLlBMJyRchDGx3uZLd7uQLOHlyXnPhMnk", + "qT2hwo8CgLiceseX55DOy9bWCQb93T5kZecpYTilwVGw19+BhGUaDLDEbYhChJ/WEKXpEE6y88ieuE9N", + "Ew1amXImDXB2B4NarSZcpEza/k0aC4s5XlsLhaYY3rK/xZJDpJME7PQ/doP9wc5G81mb5cg37DXDmZpx", + "Qd8TmOajDYHwSYOeM6NVuxTxxDYscDY4+rWKrb+++fimG8gsSbAWEQ24ClilXDaJMEQijBi5s+Gpv/Fx", + "H10ZnQR8zYsyo8ZkQCLNkjBSWPSn7xEW4YzOyZBZTmwyVmEBsTgJ0hzYREJU0cwMbXbfkDCR6imPFjXo", + "5t1t6+56zqu2APDGRazy9KtpQzUrH3c0Wd5kyL3p7QjDTBVJw0x6t1sCl5gT+s7bYavbeM08yvUdnXfn", + "7pbfDgiuyn4T+mn+ztVSqx4YWoamLIyzqDhVqzWsvKH3phaTzYJ3SzxCyBm0sEApe3W744vxiBhf2XSh", + "ZpyZ39k4Yyozv8eC30ki9CFnI3UsrG0KKIu6kGKUJhAtY2J79ZjbZorbH27J4mN/yI6jxMVi20zXOJbc", + "pgc0Xg9Uojzf+pA1V0/0C9MnNq2vSc1VziJlpskzlWaqj8xCiLLhRdAckl3JGYmGTHH0QZh8pYuP2x+K", + "ET+CdEpwpPGk1MQsafsDjT42zVqOsF79aOxKkNZkdgIAGAZaahgG+vdUYC2dZnKGcAi+GfpheUs7hrC5", + "gJN/qw7hEDOU8jSLtRwFSGWyHlb6gFBNHMdIASm5b7U8ATvZsB5r0vXlB7L2XGOAq5ERZAoqEdNg/9BP", + "T5KEgvjU0r9cvXyB4KiCUm/QrAgfABhRpgWNPLu4Hr0/ZD/jcIaMDAJZd4cBjYZBUdJrC+aaSWLEgF4P", + "hJifoNahGaZLo5/6fd2VkY+O0K8fTC9HmpbSZKT4LWHD4GMXlV5MqZpl4/zdGz9Am8xiVxVGgDqG92+5", + "4Hm9wtIxaM4NzCLELa+NFwijggOVtd8xZVgsmurj8Uw1uxaZ3AK2WbGfB4PB1vqrI7tUj2RYaagp4eOS", + "ILT7xWQAK/8sywClWrj6xGU2cURkJJ97EEKe4sjFM/6QttZIW1ZNLMlR8H2ZJRv0jYlxZa0JQ1Ay0QlD", + "KRY4IQqqG/zqx3nw4qX6b3fRCyeRMZpUkbdbAk9dd3qzhNj7jbUo86qOgAv794B/MG6R2hLGfXJf4+LY", + "JFbP62M/KHSEzXKI2PUremdEfQ8YN7gvVuoy8H5D/H0o+HNGrAhWAK3GzbahpEnZilCPNhEEJ9L2Yhpr", + "tfEK5tS7IkwhqIIs+/Zfp3yAI//bmE/fHiEDwtjWgJY2j2RubteHooUlfGTS7OTf2exT4QyzKZGoY87P", + "f/3jn66O7b/+8U9bx/Zf//gnkPu2rcoO3eUVmN8eob8SkvZwTOfELQacU8mciAXaG9giTvDKk8tKDtmQ", + "vSIqE0zmrl16XQAT06HNZavXQ1lGJJIAQqi4MLE+R8aa59GmHS0bUN4rRXeX1B+7gtIC9KnocAAukSmj", + "iuLYqkJuHhD/VEzErDkoD143TC6ZqtfzF0XeKYO9PTPBDRmMqWDuoTtT1Nv0iTpXVz9v9RGI+wYrwK8M", + "9IaiG6sJ9H/wpPU8yXCUKkMBKBveVEpj2mjWPLVt7sOu2ZTitNmwaRR5onVjt5gfYncLI6cfbs7g6bM6", + "nroSFM1mx09fr6/AeSud8svts8O9ZZjb+ioFyL6FNok6NjV+nvWnUsTlWyH9vTDgUu2fnAsjbnIN3ZuG", + "c8LZJKahQj03F1sxOtd6qgjyUNjBKztrhN266sEQ5aNiu+Lb13ho5G5+93l61Abd5BgpAjYKXPtxkqxD", + "nVMqQ66/LWFLL8SpzXkEQCzotIxF62w7p/A8P3JWCuZ5DXdHkPdn5bFDZ6x+NtwDUzytMcRvyAhrGVVK", + "IU4PCZuv81105bZWGIG+L9Qc3J8UdN8GIR+aPySLUFQDm+aCszwLfxN62Tz9X3Gj7QiehV8R4ajaTNRk", + "8iiWZT5F4YyEt2ZBtircKong3BWO+/pygCk2sMHpb6f/47hvoTgWsFqlLJ7b9C5fT1eEETZSFb/c9aNF", + "MA+QwR1h7AypJnMKlgsWbv2hbiDv5WSoV3F7QJR0mcWxM8TPiVBFyYUyP93+AI4r6+VkR20rZZHrV897", + "hIUcPJVyLxu/QOIyrH9ZadlsmFnKDzRpo18BqBxiNAujn7H/xqEM5ek6/3P3mU3Y+Z+7z0zKzv/cOzZJ", + "O7e+GrIM7os137f0+oCRTwuvtAo0YE0mD/o6aS9vdS8Cny04sYnIl0/wh9TXRuorg2ul4JfX/viKop8t", + "qfBt7glyZPNBG145/7M/mMh3v6Yni5GlKpkVW7zNIcNFUcbA1th7eA5yNMe4Mv9taUMtCHKldOBQ9/y0", + "aytUmLoSuS/+PVlU3TzuXUq0496/OfU4GdNpxjNZdvOHgiREFpXMKwz4ocmvxfHcKMF+x1g6uM+j494F", + "1B94/5VE5/qGGuZtCz6vEZ5dqzXk8IzGqpTZXkLpD5NP1oQJ5ZXhbTbYrQanMZeFuS0aV5KALzuz+ebl", + "ajCUyjKkmArZR9eSaDCR9KWJ8JBqEZOjIfsf98mviuDkzU9jHN4SFg2zwWD3IH9H2PzNT1JBpOiQXTi8", + "IUwJSiTCgqDjF6dwMTWFCNw+Oo7jIhSqPh+UZNKWbnO1iNIY8n0ZzuEDX6m+RAHB1iUslmK+AQCQF8XB", + "JPhsttRSjypu7dorUg5ZfyhSrRSpErhWK1J59uSvqUmZQb6ZKuXwzQdwmzjghzJ1H8qUzCYTGlLCVJH5", + "a8nBySYOfIAhRszex5QcEyrncWtlqkhpvlpOtcj7LZxS8sHvX4dy6QUfpqs0N8ERkdNaisOwWW353vBh", + "cL/M+f7VlYeMYmflOqB+xcDECU1s4jG/gPCMi9u2mGdjkOnXRMAvL52UV/gdyiZ6epDv4duLKHB4G79w", + "jTRVyeUeCLKOX9/UHdFBwurAJjbOFCg3hWjuqJrxzOS5GNmHJueUpgqbRRpEntD2+q3Zix79HgTQF1wh", + "mqQxSQjkpOoZbIIKQFmacpHXHaCyVHpkM/anyabsHGrSjdjyW129aywaL8Cml1cOAvP+8nZ5uWbMp+tj", + "K/PBXSChJ7hyyFyp5bdGFH6LciaLFEeSxCRU6G5GwxkEWupn0L+Jw8Rp+jbPrLB1hM6AUsu5HmDwjiSC", + "4hiqu/DYFCZ6O0+St0fL+a1uLi7gIxNjaTJZvT1CLqdVfkBI3aocOKlXEWOp0AsbDtrRmCS4KxX7Vp9C", + "pfVt2ZDKIgnFkPnCKxm5sx3SCXpbirR82xBq6Rjqc71L30he6janDjJrURwJAJzBTcKiBouZhpo/yHJn", + "4E3T2DLg00zjK8d7Lk3mOZ/maYsqqIzTtC362mkCFs+TZAUOo06peJFUEc/Un6WKiBDwscXuJuRGHRya", + "PxS+NbX8KzV5Tbksr5XTJC/xgkoz1VKVLfPXPEkCUyA4wb6qWZ8fOFvvcNnMpnemFB37Q9LeJO61yuxL", + "ga+1k8OWa2sWuV+ZBn94fc/VtfvGaPgN7GPFLChzogrsbVEw8GFF/cFGLsliplSgj0bcu0YasXUG//A0", + "UuDHH5xKQi6EudCzxYIfjnt2yU5TIvcOVCctqn52na3w5uJiq4lohFpJMuL7MCJ+mg2nVhgoifw1KASN", + "XKrGk4tTm9iRSq3l9dHLhEL+xFtCUsgGQ3kmEdzI9stZ6ZvuTfO084QpsUg5ZWrtLIqmX2cyHz8pP949", + "8ykboPKHP8rB/PDwmJQpEY/zBay62dJ8SDWaRpypoFJDe8wz3ftShn2oHScXUpHE2EkmWQxEBOGANvES", + "LtfG6yKqJFRq6cL9Sqku2pCNyUSLISkRemz9OSTbLVQ+nzXhSuGca14a1vd9mBMg6T5o0Fg1Qa1WgC5N", + "Xb59n8qalwj45Ck9A/tAtTafRJ2Y3pqC02guUax/bK00MJjCfV86rdSnU1ZemtKXLsTgbI7MfwQOd15j", + "a86A+uDY2hkpE4vjP7DRfrYm1/I1sWHtcge7Ug3z/pBdECV0GywICnkcQ8kqozZtp4KH21BXOUxpZAos", + "w+SA4TW/TmDEk8traGcSM3eHTP+xXNm3PlFXIPh8++Uak6up6f5vrI+ZBa4iC/+G/7CmbX5v3UhDsoFE", + "ebpKAeLpH95gYCW4H9aCh2ktAMehfDWdqcAhCMVylqmI3zG/ZcBWsN3+YH6cr3M/Uzic3biCYt+HtGvr", + "D60bxi3wQRClXVNETDqj+6dJnpeIeqAh6xpwbgkgxJQd6fyngCk990fD7i/vNlWG40ZuU/dKWy5V2HdD", + "W/d98tk5OK/qMjweCpkbTHMrgcItZeuTKNfAXambuRKlUJA5Fy1dad5uuUK0yUye25CK0oJ5Mdr+kOXV", + "d11mdK1ddZ1qhSIqb00PVnvqI3+RZKPn2UrJUFYoxHFo6vHk1YJNoRzZoH29KlXQ/mr0Vgzi2ei8TLLM", + "q9o+JJXDjxOwe+WyuYBxVpxaGVd3Y9vcRyiVPcw2CKRyK/gRRtUijKoErDZF+kzNY8utbLHaIgJQ0vdw", + "u+MLw8qFkq8XhPUJ5/WXQw+Hp42n9Y/wq3sTCIpUFuenDz/mqkxzFR69rbWCnq2AWTYNraJgC6JUkJ6r", + "ixcZgFl4GF2jXmCzP2SvZ8T9hajzYCURiqggoYoXiDIoX+jqJP9JIsG5su+5WDQX4jQk8kzw5NiuZo3y", + "0rpiuO8iZuM8W11PlWSaZImpkEwZOnuKOuSdEsahEk0wjcGd14GUvAsJiSTg5Fa9ErnXwzIvOb52litc", + "Y/Nao6GpCTnPNbEOzhTvTQnTe1FU4EwFn9PIhLc3lHT3zRY0xC+gpE3f07RKemvr9C0TXhVvUV5y1BYK", + "LPDT7U7w45ioV0aA/AJc5EBUnKMYiynZ+nGUPOSjpGxNcudG5URpF73bzsDU0u7zNSJ3c+Pj/cbt3nw/", + "NpFSJvkHmOhonit9TQHD3xcKDu7vfLjvQOGbB2xDPyNOwS0FCUMHukcfwjznIY5RROYk5mkCFdWgbdAN", + "MhEHR8FMqfRoezvW7WZcqqPDweEg+Pjm4/8LAAD//zctBtZ98wAA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/openapi.yaml b/openapi.yaml index d7974f8f..080499c2 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -216,6 +216,36 @@ components: description: Override image CMD (like docker run ). Omit to use image default. example: ["echo", "hello"] # Future: port_mappings, timeout_seconds + + ForkInstanceRequest: + type: object + required: [name] + properties: + name: + type: string + description: Name for the forked instance (lowercase letters, digits, and dashes only; cannot start or end with a dash) + pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ + maxLength: 63 + example: my-workload-1-fork + from_running: + type: boolean + description: | + Allow forking from a running source instance. + When true and source is Running, the source is put into standby, forked, then restored back to Running. + default: false + example: false + target_state: + $ref: "#/components/schemas/ForkTargetState" + description: | + Optional final state for the forked instance. + Default is the source instance state at fork time. + For example, forking from Running defaults the fork result to Running. + + ForkTargetState: + type: string + description: Target state for the forked instance after fork completes + enum: [Stopped, Standby, Running] + example: Running Instance: type: object @@ -1514,6 +1544,63 @@ paths: application/json: schema: $ref: "#/components/schemas/Error" + + /instances/{id}/fork: + post: + summary: Fork an instance from stopped, standby, or running (with from_running=true) + operationId: forkInstance + security: + - bearerAuth: [] + parameters: + - name: id + in: path + required: true + schema: + type: string + description: Source instance ID or name + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ForkInstanceRequest" + responses: + 201: + description: Forked instance created + content: + application/json: + schema: + $ref: "#/components/schemas/Instance" + 400: + description: Bad request - invalid fork request + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 404: + description: Source instance not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 409: + description: Conflict - invalid state (including running without from_running=true) or target name conflict + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 501: + description: Not implemented - fork not supported by this hypervisor + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 500: + description: Internal server error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" /instances/{id}/stop: post: