From da3ecfaae8f1ab56cc4d6c331183bd40226b6585 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 27 Feb 2026 14:55:56 -0500 Subject: [PATCH 01/20] Implement cloud-hypervisor fork --- lib/builds/manager_test.go | 4 + lib/forkvm/README.md | 29 +++++++ lib/forkvm/copy.go | 97 ++++++++++++++++++++++ lib/forkvm/copy_test.go | 34 ++++++++ lib/forkvm/snapshot.go | 136 ++++++++++++++++++++++++++++++ lib/forkvm/snapshot_test.go | 70 ++++++++++++++++ lib/instances/errors.go | 3 + lib/instances/fork.go | 145 ++++++++++++++++++++++++++++++++ lib/instances/fork_test.go | 160 ++++++++++++++++++++++++++++++++++++ lib/instances/manager.go | 9 ++ lib/instances/restore.go | 61 ++++++++++++-- lib/instances/types.go | 5 ++ 12 files changed, 745 insertions(+), 8 deletions(-) create mode 100644 lib/forkvm/README.md create mode 100644 lib/forkvm/copy.go create mode 100644 lib/forkvm/copy_test.go create mode 100644 lib/forkvm/snapshot.go create mode 100644 lib/forkvm/snapshot_test.go create mode 100644 lib/instances/fork.go create mode 100644 lib/instances/fork_test.go diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index 9ea0c7c2..27e46346 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -84,6 +84,10 @@ func (m *mockInstanceManager) DeleteInstance(ctx context.Context, id string) err return nil } +func (m *mockInstanceManager) ForkInstance(ctx context.Context, id string, req instances.ForkInstanceRequest) (*instances.Instance, error) { + return nil, instances.ErrNotFound +} + func (m *mockInstanceManager) StandbyInstance(ctx context.Context, id string) (*instances.Instance, error) { return nil, nil } diff --git a/lib/forkvm/README.md b/lib/forkvm/README.md new file mode 100644 index 00000000..b890e17c --- /dev/null +++ b/lib/forkvm/README.md @@ -0,0 +1,29 @@ +# Fork VM Helpers + +This package contains low-level helpers used by instance forking. + +## Why this package exists + +Forking is mostly filesystem and snapshot identity rewriting work. Keeping that logic outside `lib/instances` keeps lifecycle orchestration focused on state transitions and locking. + +## What it provides + +- `CopyGuestDirectory(srcDir, dstDir)` + - Recursively copies a guest directory. + - Skips runtime sockets (for example `ch.sock`) because they are process-local artifacts. +- `RewriteSnapshotConfig(configPath, opts)` + - Rewrites `snapshots/snapshot-latest/config.json` for a forked cloud-hypervisor instance. + - Supports: + - source data-dir -> target data-dir path remap + - vsock CID/socket rewrite + - serial log path rewrite + - network fields (`tap`, `ip`, `mac`, `mask`) rewrite + +## How it is used + +`lib/instances/fork.go` calls this package to clone guest data and prepare copied snapshot state. `lib/instances/restore.go` uses it when a standby fork restores with a fresh network identity. + +## Safety notes + +- This package does not perform lifecycle locking by itself. +- Locking and state validation must be handled by callers (`instances.Manager`), which already serializes per-instance lifecycle operations. diff --git a/lib/forkvm/copy.go b/lib/forkvm/copy.go new file mode 100644 index 00000000..b69767ae --- /dev/null +++ b/lib/forkvm/copy.go @@ -0,0 +1,97 @@ +package forkvm + +import ( + "fmt" + "io" + "io/fs" + "os" + "path/filepath" +) + +// CopyGuestDirectory recursively copies a guest directory to a new destination. +// Runtime sockets are skipped because they are host-runtime artifacts. +func CopyGuestDirectory(srcDir, dstDir string) error { + srcInfo, err := os.Stat(srcDir) + if err != nil { + return fmt.Errorf("stat source directory: %w", err) + } + if !srcInfo.IsDir() { + return fmt.Errorf("source path is not a directory: %s", srcDir) + } + + if err := os.MkdirAll(dstDir, srcInfo.Mode().Perm()); err != nil { + return fmt.Errorf("create destination directory: %w", err) + } + + return filepath.WalkDir(srcDir, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + + relPath, err := filepath.Rel(srcDir, path) + if err != nil { + return fmt.Errorf("compute relative path: %w", err) + } + if relPath == "." { + return nil + } + + dstPath := filepath.Join(dstDir, relPath) + info, err := d.Info() + if err != nil { + return fmt.Errorf("stat source entry %s: %w", path, err) + } + + mode := info.Mode() + switch { + case mode.IsDir(): + if err := os.MkdirAll(dstPath, mode.Perm()); err != nil { + return fmt.Errorf("create destination directory %s: %w", dstPath, err) + } + return nil + + case mode.IsRegular(): + if err := copyRegularFile(path, dstPath, mode.Perm()); err != nil { + return fmt.Errorf("copy file %s: %w", path, err) + } + return nil + + case mode&os.ModeSymlink != 0: + target, err := os.Readlink(path) + if err != nil { + return fmt.Errorf("read symlink %s: %w", path, err) + } + if err := os.Symlink(target, dstPath); err != nil { + return fmt.Errorf("create symlink %s: %w", dstPath, err) + } + return nil + + case mode&os.ModeSocket != 0: + // Runtime socket (e.g., ch.sock); the forked instance will create its own. + return nil + + default: + return fmt.Errorf("unsupported file type %s (%s)", path, mode.String()) + } + }) +} + +func copyRegularFile(srcPath, dstPath string, perms fs.FileMode) error { + src, err := os.Open(srcPath) + if err != nil { + return err + } + defer src.Close() + + dst, err := os.OpenFile(dstPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, perms) + if err != nil { + return err + } + defer dst.Close() + + if _, err := io.Copy(dst, src); err != nil { + return err + } + + return nil +} diff --git a/lib/forkvm/copy_test.go b/lib/forkvm/copy_test.go new file mode 100644 index 00000000..99e9226f --- /dev/null +++ b/lib/forkvm/copy_test.go @@ -0,0 +1,34 @@ +package forkvm + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCopyGuestDirectory(t *testing.T) { + src := filepath.Join(t.TempDir(), "src") + dst := filepath.Join(t.TempDir(), "dst") + + require.NoError(t, os.MkdirAll(filepath.Join(src, "logs"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(src, "metadata.json"), []byte(`{"id":"abc"}`), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(src, "logs", "app.log"), []byte("hello"), 0644)) + require.NoError(t, os.Symlink("metadata.json", filepath.Join(src, "meta-link"))) + + require.NoError(t, CopyGuestDirectory(src, dst)) + + assert.FileExists(t, filepath.Join(dst, "metadata.json")) + assert.FileExists(t, filepath.Join(dst, "logs", "app.log")) + assert.FileExists(t, filepath.Join(dst, "meta-link")) + + app, err := os.ReadFile(filepath.Join(dst, "logs", "app.log")) + require.NoError(t, err) + assert.Equal(t, "hello", string(app)) + + linkTarget, err := os.Readlink(filepath.Join(dst, "meta-link")) + require.NoError(t, err) + assert.Equal(t, "metadata.json", linkTarget) +} diff --git a/lib/forkvm/snapshot.go b/lib/forkvm/snapshot.go new file mode 100644 index 00000000..93e5db58 --- /dev/null +++ b/lib/forkvm/snapshot.go @@ -0,0 +1,136 @@ +package forkvm + +import ( + "encoding/json" + "fmt" + "os" + "strings" +) + +// SnapshotNetworkConfig contains network identity fields in CH snapshot config.json. +type SnapshotNetworkConfig struct { + TAPDevice string + IP string + MAC string + Netmask string +} + +// SnapshotRewriteOptions controls identity/path rewrites for CH snapshot config.json. +type SnapshotRewriteOptions struct { + SourceDataDir string + TargetDataDir string + + VsockCID *int64 + VsockSocket string + + SerialLogPath string + Network *SnapshotNetworkConfig +} + +// RewriteSnapshotConfig rewrites cloud-hypervisor snapshot config.json for a forked instance. +func RewriteSnapshotConfig(configPath string, opts SnapshotRewriteOptions) error { + data, err := os.ReadFile(configPath) + if err != nil { + return fmt.Errorf("read snapshot config: %w", err) + } + + var config map[string]any + if err := json.Unmarshal(data, &config); err != nil { + return fmt.Errorf("unmarshal snapshot config: %w", err) + } + + if opts.SourceDataDir != "" && opts.TargetDataDir != "" && opts.SourceDataDir != opts.TargetDataDir { + configAny := rewriteStringValues(config, func(s string) string { + return strings.ReplaceAll(s, opts.SourceDataDir, opts.TargetDataDir) + }) + config = configAny.(map[string]any) + } + + if opts.VsockCID != nil || opts.VsockSocket != "" { + updateVsockConfig(config, opts.VsockCID, opts.VsockSocket) + } + if opts.SerialLogPath != "" { + updateSerialConfig(config, opts.SerialLogPath) + } + if opts.Network != nil { + updateNetworkConfig(config, *opts.Network) + } + + updated, err := json.MarshalIndent(config, "", " ") + if err != nil { + return fmt.Errorf("marshal snapshot config: %w", err) + } + + if err := os.WriteFile(configPath, updated, 0644); err != nil { + return fmt.Errorf("write snapshot config: %w", err) + } + + return nil +} + +func rewriteStringValues(value any, mapper func(string) string) any { + switch v := value.(type) { + case map[string]any: + out := make(map[string]any, len(v)) + for k, child := range v { + out[k] = rewriteStringValues(child, mapper) + } + return out + case []any: + out := make([]any, 0, len(v)) + for _, child := range v { + out = append(out, rewriteStringValues(child, mapper)) + } + return out + case string: + return mapper(v) + default: + return value + } +} + +func updateVsockConfig(config map[string]any, cid *int64, socketPath string) { + vsock, ok := config["vsock"].(map[string]any) + if !ok || vsock == nil { + return + } + if cid != nil { + vsock["cid"] = *cid + } + if socketPath != "" { + vsock["socket"] = socketPath + } +} + +func updateSerialConfig(config map[string]any, logPath string) { + serial, ok := config["serial"].(map[string]any) + if !ok || serial == nil { + return + } + serial["file"] = logPath +} + +func updateNetworkConfig(config map[string]any, netCfg SnapshotNetworkConfig) { + nets, ok := config["net"].([]any) + if !ok { + return + } + for _, netAny := range nets { + netMap, ok := netAny.(map[string]any) + if !ok || netMap == nil { + continue + } + if netCfg.TAPDevice != "" { + netMap["tap"] = netCfg.TAPDevice + } + if netCfg.IP != "" { + netMap["ip"] = netCfg.IP + } + if netCfg.MAC != "" { + netMap["mac"] = netCfg.MAC + } + if netCfg.Netmask != "" { + netMap["mask"] = netCfg.Netmask + } + } +} diff --git a/lib/forkvm/snapshot_test.go b/lib/forkvm/snapshot_test.go new file mode 100644 index 00000000..38064db4 --- /dev/null +++ b/lib/forkvm/snapshot_test.go @@ -0,0 +1,70 @@ +package forkvm + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRewriteSnapshotConfig(t *testing.T) { + tmp := t.TempDir() + configPath := filepath.Join(tmp, "config.json") + + orig := map[string]any{ + "disks": []any{map[string]any{"path": "/src/guests/a/overlay.raw"}}, + "serial": map[string]any{"file": "/src/guests/a/logs/app.log"}, + "vsock": map[string]any{"cid": float64(100), "socket": "/src/guests/a/vsock.sock"}, + "net": []any{map[string]any{ + "tap": "hype-old", + "ip": "10.0.0.10", + "mac": "02:00:00:00:00:01", + "mask": "255.255.255.0", + }}, + } + data, err := json.Marshal(orig) + require.NoError(t, err) + require.NoError(t, os.WriteFile(configPath, data, 0644)) + + cid := int64(200) + err = RewriteSnapshotConfig(configPath, SnapshotRewriteOptions{ + SourceDataDir: "/src/guests/a", + TargetDataDir: "/dst/guests/b", + VsockCID: &cid, + VsockSocket: "/dst/guests/b/vsock.sock", + SerialLogPath: "/dst/guests/b/logs/app.log", + Network: &SnapshotNetworkConfig{ + TAPDevice: "hype-new", + IP: "10.0.0.20", + MAC: "02:00:00:00:00:02", + Netmask: "255.255.255.0", + }, + }) + require.NoError(t, err) + + updatedData, err := os.ReadFile(configPath) + require.NoError(t, err) + + var updated map[string]any + require.NoError(t, json.Unmarshal(updatedData, &updated)) + + disks := updated["disks"].([]any) + disk0 := disks[0].(map[string]any) + assert.Equal(t, "/dst/guests/b/overlay.raw", disk0["path"]) + + serial := updated["serial"].(map[string]any) + assert.Equal(t, "/dst/guests/b/logs/app.log", serial["file"]) + + vsock := updated["vsock"].(map[string]any) + assert.Equal(t, float64(200), vsock["cid"]) + assert.Equal(t, "/dst/guests/b/vsock.sock", vsock["socket"]) + + netCfg := updated["net"].([]any)[0].(map[string]any) + assert.Equal(t, "hype-new", netCfg["tap"]) + assert.Equal(t, "10.0.0.20", netCfg["ip"]) + assert.Equal(t, "02:00:00:00:00:02", netCfg["mac"]) + assert.Equal(t, "255.255.255.0", netCfg["mask"]) +} diff --git a/lib/instances/errors.go b/lib/instances/errors.go index 9925bb02..eb5f1baa 100644 --- a/lib/instances/errors.go +++ b/lib/instances/errors.go @@ -20,4 +20,7 @@ var ( // ErrInsufficientResources is returned when resources (CPU, memory, network, GPU) are not available ErrInsufficientResources = errors.New("insufficient resources") + + // ErrNotSupported is returned when an operation is not supported for the instance hypervisor + ErrNotSupported = errors.New("operation not supported") ) diff --git a/lib/instances/fork.go b/lib/instances/fork.go new file mode 100644 index 00000000..f7c6d06c --- /dev/null +++ b/lib/instances/fork.go @@ -0,0 +1,145 @@ +package instances + +import ( + "context" + "fmt" + "os" + "regexp" + "time" + + "github.com/kernel/hypeman/lib/forkvm" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/logger" + "github.com/kernel/hypeman/lib/network" + "github.com/nrednav/cuid2" + "gvisor.dev/gvisor/pkg/cleanup" +) + +// forkInstance creates a new instance by cloning a stopped or standby source instance. +func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceRequest) (*Instance, error) { + log := logger.FromContext(ctx) + log.InfoContext(ctx, "forking instance", "source_instance_id", id, "fork_name", req.Name) + + if err := validateForkRequest(req); err != nil { + return nil, err + } + + meta, err := m.loadMetadata(id) + if err != nil { + return nil, err + } + + source := m.toInstance(ctx, meta) + stored := &meta.StoredMetadata + + if stored.HypervisorType != hypervisor.TypeCloudHypervisor { + return nil, fmt.Errorf("%w: fork is only supported for hypervisor %s (got %s)", ErrNotSupported, hypervisor.TypeCloudHypervisor, stored.HypervisorType) + } + + switch source.State { + case StateStopped, StateStandby: + // allowed + default: + return nil, fmt.Errorf("%w: cannot fork from state %s (must be Stopped or Standby)", ErrInvalidState, source.State) + } + + if stored.NetworkEnabled { + exists, err := m.networkManager.NameExists(ctx, req.Name, id) + if err != nil { + return nil, fmt.Errorf("check instance name availability: %w", err) + } + if exists { + return nil, fmt.Errorf("%w: instance name '%s' already exists in network", ErrAlreadyExists, req.Name) + } + } + + forkID := cuid2.Generate() + if _, err := m.loadMetadata(forkID); err == nil { + return nil, fmt.Errorf("%w: generated fork id already exists", ErrAlreadyExists) + } + + srcDir := m.paths.InstanceDir(id) + dstDir := m.paths.InstanceDir(forkID) + + cu := cleanup.Make(func() { + _ = os.RemoveAll(dstDir) + }) + defer cu.Clean() + + if err := forkvm.CopyGuestDirectory(srcDir, dstDir); err != nil { + return nil, fmt.Errorf("clone guest directory: %w", err) + } + + starter, err := m.getVMStarter(stored.HypervisorType) + if err != nil { + return nil, fmt.Errorf("get vm starter: %w", err) + } + + now := time.Now() + forkMeta := meta.StoredMetadata + forkMeta.Id = forkID + forkMeta.Name = req.Name + forkMeta.CreatedAt = now + forkMeta.StartedAt = nil + forkMeta.StoppedAt = nil + forkMeta.HypervisorPID = nil + forkMeta.SocketPath = m.paths.InstanceSocket(forkID, starter.SocketName()) + forkMeta.DataDir = dstDir + forkMeta.VsockCID = generateVsockCID(forkID) + forkMeta.VsockSocket = m.paths.InstanceVsockSocket(forkID) + forkMeta.ExitCode = nil + forkMeta.ExitMessage = "" + + if forkMeta.NetworkEnabled { + // Clear inherited network identity. For stopped instances this is regenerated on start, + // and for standby instances restore allocates if identity is empty. + forkMeta.IP = "" + forkMeta.MAC = "" + } + + if source.State == StateStandby { + snapshotConfigPath := m.paths.InstanceSnapshotConfig(forkID) + netCfg := (*forkvm.SnapshotNetworkConfig)(nil) + if forkMeta.NetworkEnabled { + netCfg = &forkvm.SnapshotNetworkConfig{TAPDevice: network.GenerateTAPName(forkID)} + } + if err := forkvm.RewriteSnapshotConfig(snapshotConfigPath, forkvm.SnapshotRewriteOptions{ + SourceDataDir: stored.DataDir, + TargetDataDir: forkMeta.DataDir, + VsockCID: &forkMeta.VsockCID, + VsockSocket: forkMeta.VsockSocket, + SerialLogPath: m.paths.InstanceAppLog(forkID), + Network: netCfg, + }); err != nil { + return nil, fmt.Errorf("rewrite fork snapshot config: %w", err) + } + } + + newMeta := &metadata{StoredMetadata: forkMeta} + if err := m.saveMetadata(newMeta); err != nil { + return nil, fmt.Errorf("save fork metadata: %w", err) + } + + cu.Release() + forked := m.toInstance(ctx, newMeta) + log.InfoContext(ctx, "instance forked successfully", + "source_instance_id", id, + "fork_instance_id", forked.Id, + "fork_name", forked.Name, + "state", forked.State) + return &forked, nil +} + +func validateForkRequest(req ForkInstanceRequest) error { + if req.Name == "" { + return fmt.Errorf("name is required") + } + if len(req.Name) > 63 { + return fmt.Errorf("name must be 63 characters or less") + } + namePattern := regexp.MustCompile(`^[a-z0-9]([a-z0-9-]*[a-z0-9])?$`) + if !namePattern.MatchString(req.Name) { + return fmt.Errorf("name must contain only lowercase letters, digits, and dashes; cannot start or end with a dash") + } + return nil +} diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go new file mode 100644 index 00000000..09af2dc8 --- /dev/null +++ b/lib/instances/fork_test.go @@ -0,0 +1,160 @@ +package instances + +import ( + "context" + "os" + "testing" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestForkInstanceNotSupportedHypervisor(t *testing.T) { + manager, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "fork-qemu-source" + require.NoError(t, manager.ensureDirectories(sourceID)) + + meta := &metadata{StoredMetadata: StoredMetadata{ + Id: sourceID, + Name: "fork-qemu-source", + Image: "docker.io/library/alpine:latest", + CreatedAt: time.Now(), + HypervisorType: hypervisor.TypeQEMU, + HypervisorVersion: "test", + SocketPath: paths.New(manager.paths.DataDir()).InstanceSocket(sourceID, "qemu.sock"), + DataDir: paths.New(manager.paths.DataDir()).InstanceDir(sourceID), + VsockCID: 42, + VsockSocket: paths.New(manager.paths.DataDir()).InstanceVsockSocket(sourceID), + }} + require.NoError(t, manager.saveMetadata(meta)) + + _, err := manager.ForkInstance(ctx, sourceID, ForkInstanceRequest{Name: "fork-qemu-copy"}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrNotSupported) +} + +func TestForkCloudHypervisorStoppedAndStandby(t *testing.T) { + if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { + t.Skip("/dev/kvm not available, skipping on this platform") + } + + manager, tmpDir := setupTestManager(t) + ctx := context.Background() + + imageManager, err := images.NewManager(paths.New(tmpDir), 1, nil) + require.NoError(t, err) + + t.Log("Ensuring alpine image...") + alpineImage, err := imageManager.CreateImage(ctx, images.CreateImageRequest{Name: "docker.io/library/alpine:latest"}) + require.NoError(t, err) + + imageName := alpineImage.Name + for i := 0; i < 60; i++ { + img, err := imageManager.GetImage(ctx, imageName) + if err == nil && img.Status == images.StatusReady { + alpineImage = img + break + } + if err == nil && img.Status == images.StatusFailed { + t.Fatalf("image build failed: %s", *img.Error) + } + time.Sleep(1 * time.Second) + } + require.Equal(t, images.StatusReady, alpineImage.Status, "Image should be ready after 60 seconds") + + systemManager := manager.systemManager + require.NoError(t, systemManager.EnsureSystemFiles(ctx)) + + require.NoError(t, manager.networkManager.Initialize(ctx, nil)) + + createReq := CreateInstanceRequest{ + Image: "docker.io/library/alpine:latest", + Size: 2 * 1024 * 1024 * 1024, + HotplugSize: 256 * 1024 * 1024, + OverlaySize: 10 * 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: true, + Entrypoint: []string{"sh", "-c"}, + Cmd: []string{"while true; do sleep 3600; done"}, + } + + // Stopped source fork flow. + sourceStopped, err := manager.CreateInstance(ctx, CreateInstanceRequest{ + Name: "fork-stop-src", + Image: createReq.Image, + Size: createReq.Size, + HotplugSize: createReq.HotplugSize, + OverlaySize: createReq.OverlaySize, + Vcpus: createReq.Vcpus, + NetworkEnabled: createReq.NetworkEnabled, + Entrypoint: createReq.Entrypoint, + Cmd: createReq.Cmd, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = manager.DeleteInstance(context.Background(), sourceStopped.Id) }) + require.NoError(t, waitForVMReady(ctx, sourceStopped.SocketPath, 5*time.Second)) + + sourceStopped, err = manager.StopInstance(ctx, sourceStopped.Id) + require.NoError(t, err) + require.Equal(t, StateStopped, sourceStopped.State) + + forkStopped, err := manager.ForkInstance(ctx, sourceStopped.Id, ForkInstanceRequest{Name: "fork-stop-copy"}) + require.NoError(t, err) + require.Equal(t, StateStopped, forkStopped.State) + t.Cleanup(func() { _ = manager.DeleteInstance(context.Background(), forkStopped.Id) }) + + sourceStopped, err = manager.StartInstance(ctx, sourceStopped.Id, StartInstanceRequest{}) + require.NoError(t, err) + forkStopped, err = manager.StartInstance(ctx, forkStopped.Id, StartInstanceRequest{}) + require.NoError(t, err) + require.NoError(t, waitForVMReady(ctx, sourceStopped.SocketPath, 5*time.Second)) + require.NoError(t, waitForVMReady(ctx, forkStopped.SocketPath, 5*time.Second)) + + assert.NotEmpty(t, sourceStopped.IP) + assert.NotEmpty(t, forkStopped.IP) + assert.NotEqual(t, sourceStopped.IP, forkStopped.IP) + assert.NotEqual(t, sourceStopped.MAC, forkStopped.MAC) + + // Standby source fork flow. + sourceStandby, err := manager.CreateInstance(ctx, CreateInstanceRequest{ + Name: "fork-standby-src", + Image: createReq.Image, + Size: createReq.Size, + HotplugSize: createReq.HotplugSize, + OverlaySize: createReq.OverlaySize, + Vcpus: createReq.Vcpus, + NetworkEnabled: createReq.NetworkEnabled, + Entrypoint: createReq.Entrypoint, + Cmd: createReq.Cmd, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = manager.DeleteInstance(context.Background(), sourceStandby.Id) }) + require.NoError(t, waitForVMReady(ctx, sourceStandby.SocketPath, 5*time.Second)) + + sourceStandby, err = manager.StandbyInstance(ctx, sourceStandby.Id) + require.NoError(t, err) + require.Equal(t, StateStandby, sourceStandby.State) + + forkStandby, err := manager.ForkInstance(ctx, sourceStandby.Id, ForkInstanceRequest{Name: "fork-standby-copy"}) + require.NoError(t, err) + require.Equal(t, StateStandby, forkStandby.State) + t.Cleanup(func() { _ = manager.DeleteInstance(context.Background(), forkStandby.Id) }) + + sourceStandby, err = manager.RestoreInstance(ctx, sourceStandby.Id) + require.NoError(t, err) + forkStandby, err = manager.RestoreInstance(ctx, forkStandby.Id) + require.NoError(t, err) + require.NoError(t, waitForVMReady(ctx, sourceStandby.SocketPath, 5*time.Second)) + require.NoError(t, waitForVMReady(ctx, forkStandby.SocketPath, 5*time.Second)) + + assert.NotEmpty(t, sourceStandby.IP) + assert.NotEmpty(t, forkStandby.IP) + assert.NotEqual(t, sourceStandby.IP, forkStandby.IP) + assert.NotEqual(t, sourceStandby.MAC, forkStandby.MAC) +} diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 3c764d13..2cce0faf 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -25,6 +25,7 @@ type Manager interface { // Returns ErrAmbiguousName if prefix matches multiple instances. GetInstance(ctx context.Context, idOrName string) (*Instance, error) DeleteInstance(ctx context.Context, id string) error + ForkInstance(ctx context.Context, id string, req ForkInstanceRequest) (*Instance, error) StandbyInstance(ctx context.Context, id string) (*Instance, error) RestoreInstance(ctx context.Context, id string) (*Instance, error) StopInstance(ctx context.Context, id string) (*Instance, error) @@ -182,6 +183,14 @@ func (m *manager) DeleteInstance(ctx context.Context, id string) error { return err } +// ForkInstance creates a forked copy of a stopped or standby instance. +func (m *manager) ForkInstance(ctx context.Context, id string, req ForkInstanceRequest) (*Instance, error) { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + return m.forkInstance(ctx, id, req) +} + // StandbyInstance puts an instance in standby (pause, snapshot, delete VMM) func (m *manager) StandbyInstance(ctx context.Context, id string) (*Instance, error) { lock := m.getInstanceLock(id) diff --git a/lib/instances/restore.go b/lib/instances/restore.go index ed81fbfd..3ef7b102 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -6,8 +6,10 @@ import ( "os" "time" + "github.com/kernel/hypeman/lib/forkvm" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" + "github.com/kernel/hypeman/lib/network" "go.opentelemetry.io/otel/trace" ) @@ -64,20 +66,63 @@ func (m *manager) restoreInstance( // 3. Get snapshot directory snapshotDir := m.paths.InstanceSnapshotLatest(id) - // 4. Recreate TAP device if network enabled + // 4. Recreate or allocate network if network enabled if stored.NetworkEnabled { var networkSpan trace.Span if m.metrics != nil && m.metrics.tracer != nil { ctx, networkSpan = m.metrics.tracer.Start(ctx, "RestoreNetwork") } - log.InfoContext(ctx, "recreating network for restore", "instance_id", id, "network", "default", - "download_bps", stored.NetworkBandwidthDownload, "upload_bps", stored.NetworkBandwidthUpload) - if err := m.networkManager.RecreateAllocation(ctx, id, stored.NetworkBandwidthDownload, stored.NetworkBandwidthUpload); err != nil { - if networkSpan != nil { - networkSpan.End() + // If IP/MAC is empty (forked standby flow), allocate a fresh identity and + // patch the copied snapshot config before restore. + if stored.IP == "" || stored.MAC == "" { + log.InfoContext(ctx, "allocating fresh network identity for standby restore", + "instance_id", id, "network", "default", + "download_bps", stored.NetworkBandwidthDownload, "upload_bps", stored.NetworkBandwidthUpload) + netConfig, err := m.networkManager.CreateAllocation(ctx, network.AllocateRequest{ + InstanceID: id, + InstanceName: stored.Name, + DownloadBps: stored.NetworkBandwidthDownload, + UploadBps: stored.NetworkBandwidthUpload, + UploadCeilBps: stored.NetworkBandwidthUpload * int64(m.networkManager.GetUploadBurstMultiplier()), + }) + if err != nil { + if networkSpan != nil { + networkSpan.End() + } + log.ErrorContext(ctx, "failed to allocate network", "instance_id", id, "error", err) + return nil, fmt.Errorf("allocate network: %w", err) + } + stored.IP = netConfig.IP + stored.MAC = netConfig.MAC + + if err := forkvm.RewriteSnapshotConfig(m.paths.InstanceSnapshotConfig(id), forkvm.SnapshotRewriteOptions{ + VsockCID: &stored.VsockCID, + VsockSocket: stored.VsockSocket, + Network: &forkvm.SnapshotNetworkConfig{ + TAPDevice: netConfig.TAPDevice, + IP: netConfig.IP, + MAC: netConfig.MAC, + Netmask: netConfig.Netmask, + }, + }); err != nil { + if networkSpan != nil { + networkSpan.End() + } + log.ErrorContext(ctx, "failed to patch snapshot network identity", "instance_id", id, "error", err) + netAlloc, _ := m.networkManager.GetAllocation(ctx, id) + m.networkManager.ReleaseAllocation(ctx, netAlloc) + return nil, fmt.Errorf("rewrite snapshot config: %w", err) + } + } else { + log.InfoContext(ctx, "recreating network for restore", "instance_id", id, "network", "default", + "download_bps", stored.NetworkBandwidthDownload, "upload_bps", stored.NetworkBandwidthUpload) + if err := m.networkManager.RecreateAllocation(ctx, id, stored.NetworkBandwidthDownload, stored.NetworkBandwidthUpload); err != nil { + if networkSpan != nil { + networkSpan.End() + } + log.ErrorContext(ctx, "failed to recreate network", "instance_id", id, "error", err) + return nil, fmt.Errorf("recreate network: %w", err) } - log.ErrorContext(ctx, "failed to recreate network", "instance_id", id, "error", err) - return nil, fmt.Errorf("recreate network: %w", err) } if networkSpan != nil { networkSpan.End() diff --git a/lib/instances/types.go b/lib/instances/types.go index 9afbccc2..07d2ca8b 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -175,6 +175,11 @@ type StartInstanceRequest struct { Cmd []string // Override cmd (nil = keep previous/image default) } +// ForkInstanceRequest is the domain request for forking a stopped/standby instance. +type ForkInstanceRequest struct { + Name string // Required: name for the new forked instance +} + // AttachVolumeRequest is the domain request for attaching a volume (used for API compatibility) type AttachVolumeRequest struct { MountPath string From 508187198b751058240e722e8c45d930a9bbc282 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 27 Feb 2026 15:11:41 -0500 Subject: [PATCH 02/20] move fork support checks behind hypervisor starter interface --- lib/hypervisor/cloudhypervisor/fork.go | 36 +++++++++++++++++++++++++ lib/hypervisor/hypervisor.go | 30 +++++++++++++++++++++ lib/hypervisor/qemu/process.go | 7 +++++ lib/hypervisor/vz/starter.go | 7 +++++ lib/instances/fork.go | 37 ++++++++++++++++---------- lib/instances/restore.go | 21 +++++++++++---- 6 files changed, 119 insertions(+), 19 deletions(-) create mode 100644 lib/hypervisor/cloudhypervisor/fork.go diff --git a/lib/hypervisor/cloudhypervisor/fork.go b/lib/hypervisor/cloudhypervisor/fork.go new file mode 100644 index 00000000..c4aca372 --- /dev/null +++ b/lib/hypervisor/cloudhypervisor/fork.go @@ -0,0 +1,36 @@ +package cloudhypervisor + +import ( + "context" + + "github.com/kernel/hypeman/lib/forkvm" + "github.com/kernel/hypeman/lib/hypervisor" +) + +// PrepareFork prepares cloud-hypervisor fork state by rewriting snapshot config +// when a snapshot path is provided. For stopped forks (no snapshot), this is a no-op. +func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareRequest) error { + _ = ctx + if req.SnapshotConfigPath == "" { + return nil + } + + var netCfg *forkvm.SnapshotNetworkConfig + if req.Network != nil { + netCfg = &forkvm.SnapshotNetworkConfig{ + TAPDevice: req.Network.TAPDevice, + IP: req.Network.IP, + MAC: req.Network.MAC, + Netmask: req.Network.Netmask, + } + } + + return forkvm.RewriteSnapshotConfig(req.SnapshotConfigPath, forkvm.SnapshotRewriteOptions{ + SourceDataDir: req.SourceDataDir, + TargetDataDir: req.TargetDataDir, + VsockCID: &req.VsockCID, + VsockSocket: req.VsockSocket, + SerialLogPath: req.SerialLogPath, + Network: netCfg, + }) +} diff --git a/lib/hypervisor/hypervisor.go b/lib/hypervisor/hypervisor.go index b4287a79..a0bfd82d 100644 --- a/lib/hypervisor/hypervisor.go +++ b/lib/hypervisor/hypervisor.go @@ -81,6 +81,36 @@ type VMStarter interface { // - QEMU: would start with -incoming or -loadvm flags (not yet implemented) // Returns the process ID and a Hypervisor client. The VM is in paused state after restore. RestoreVM(ctx context.Context, p *paths.Paths, version string, socketPath string, snapshotPath string) (pid int, hv Hypervisor, err error) + + // PrepareFork allows hypervisors to prepare forked instance state. + // For snapshot-based forks, implementations can rewrite snapshot config with + // fork identity (paths, vsock, network). Hypervisors that don't support fork + // should return ErrNotSupported. + PrepareFork(ctx context.Context, req ForkPrepareRequest) error +} + +// ForkNetworkConfig contains network identity fields for fork preparation. +type ForkNetworkConfig struct { + TAPDevice string + IP string + MAC string + Netmask string +} + +// ForkPrepareRequest contains hypervisor-specific fork preparation inputs. +type ForkPrepareRequest struct { + // SnapshotConfigPath is optional. When empty, implementations should only + // validate fork support and return without snapshot rewrites. + SnapshotConfigPath string + + SourceDataDir string + TargetDataDir string + + VsockCID int64 + VsockSocket string + + SerialLogPath string + Network *ForkNetworkConfig } // Hypervisor defines the interface for VM control operations. diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index e2e1d098..4fba5f85 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -270,6 +270,13 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, return pid, hv, nil } +// PrepareFork is not supported for QEMU in the current implementation. +func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareRequest) error { + _ = ctx + _ = req + return hypervisor.ErrNotSupported +} + // vmConfigFile is the name of the file where VM config is saved for restore. const vmConfigFile = "qemu-config.json" diff --git a/lib/hypervisor/vz/starter.go b/lib/hypervisor/vz/starter.go index e80260b0..643075e0 100644 --- a/lib/hypervisor/vz/starter.go +++ b/lib/hypervisor/vz/starter.go @@ -217,6 +217,13 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, return 0, nil, hypervisor.ErrNotSupported } +// PrepareFork is not supported for vz. +func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareRequest) error { + _ = ctx + _ = req + return hypervisor.ErrNotSupported +} + func (s *Starter) waitForShim(ctx context.Context, socketPath string, timeout time.Duration) (*Client, error) { deadline := time.Now().Add(timeout) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index f7c6d06c..f8f96d1e 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -2,6 +2,7 @@ package instances import ( "context" + "errors" "fmt" "os" "regexp" @@ -32,10 +33,6 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR source := m.toInstance(ctx, meta) stored := &meta.StoredMetadata - if stored.HypervisorType != hypervisor.TypeCloudHypervisor { - return nil, fmt.Errorf("%w: fork is only supported for hypervisor %s (got %s)", ErrNotSupported, hypervisor.TypeCloudHypervisor, stored.HypervisorType) - } - switch source.State { case StateStopped, StateStandby: // allowed @@ -99,19 +96,31 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR if source.State == StateStandby { snapshotConfigPath := m.paths.InstanceSnapshotConfig(forkID) - netCfg := (*forkvm.SnapshotNetworkConfig)(nil) + netCfg := (*hypervisor.ForkNetworkConfig)(nil) if forkMeta.NetworkEnabled { - netCfg = &forkvm.SnapshotNetworkConfig{TAPDevice: network.GenerateTAPName(forkID)} + netCfg = &hypervisor.ForkNetworkConfig{TAPDevice: network.GenerateTAPName(forkID)} } - if err := forkvm.RewriteSnapshotConfig(snapshotConfigPath, forkvm.SnapshotRewriteOptions{ - SourceDataDir: stored.DataDir, - TargetDataDir: forkMeta.DataDir, - VsockCID: &forkMeta.VsockCID, - VsockSocket: forkMeta.VsockSocket, - SerialLogPath: m.paths.InstanceAppLog(forkID), - Network: netCfg, + if err := starter.PrepareFork(ctx, hypervisor.ForkPrepareRequest{ + SnapshotConfigPath: snapshotConfigPath, + SourceDataDir: stored.DataDir, + TargetDataDir: forkMeta.DataDir, + VsockCID: forkMeta.VsockCID, + VsockSocket: forkMeta.VsockSocket, + SerialLogPath: m.paths.InstanceAppLog(forkID), + Network: netCfg, }); err != nil { - return nil, fmt.Errorf("rewrite fork snapshot config: %w", err) + if errors.Is(err, hypervisor.ErrNotSupported) { + return nil, fmt.Errorf("%w: fork is not supported for hypervisor %s", ErrNotSupported, stored.HypervisorType) + } + return nil, fmt.Errorf("prepare fork snapshot state: %w", err) + } + } else { + // Validate fork support for stopped-state forks as well. + if err := starter.PrepareFork(ctx, hypervisor.ForkPrepareRequest{}); err != nil { + if errors.Is(err, hypervisor.ErrNotSupported) { + return nil, fmt.Errorf("%w: fork is not supported for hypervisor %s", ErrNotSupported, stored.HypervisorType) + } + return nil, fmt.Errorf("prepare fork state: %w", err) } } diff --git a/lib/instances/restore.go b/lib/instances/restore.go index 3ef7b102..e6c002de 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -2,11 +2,11 @@ package instances import ( "context" + "errors" "fmt" "os" "time" - "github.com/kernel/hypeman/lib/forkvm" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" @@ -65,6 +65,10 @@ func (m *manager) restoreInstance( // 3. Get snapshot directory snapshotDir := m.paths.InstanceSnapshotLatest(id) + starter, err := m.getVMStarter(stored.HypervisorType) + if err != nil { + return nil, fmt.Errorf("get vm starter: %w", err) + } // 4. Recreate or allocate network if network enabled if stored.NetworkEnabled { @@ -95,10 +99,11 @@ func (m *manager) restoreInstance( stored.IP = netConfig.IP stored.MAC = netConfig.MAC - if err := forkvm.RewriteSnapshotConfig(m.paths.InstanceSnapshotConfig(id), forkvm.SnapshotRewriteOptions{ - VsockCID: &stored.VsockCID, - VsockSocket: stored.VsockSocket, - Network: &forkvm.SnapshotNetworkConfig{ + if err := starter.PrepareFork(ctx, hypervisor.ForkPrepareRequest{ + SnapshotConfigPath: m.paths.InstanceSnapshotConfig(id), + VsockCID: stored.VsockCID, + VsockSocket: stored.VsockSocket, + Network: &hypervisor.ForkNetworkConfig{ TAPDevice: netConfig.TAPDevice, IP: netConfig.IP, MAC: netConfig.MAC, @@ -108,6 +113,12 @@ func (m *manager) restoreInstance( if networkSpan != nil { networkSpan.End() } + if errors.Is(err, hypervisor.ErrNotSupported) { + log.ErrorContext(ctx, "forked standby network rewrite not supported for hypervisor", "instance_id", id, "hypervisor", stored.HypervisorType) + netAlloc, _ := m.networkManager.GetAllocation(ctx, id) + m.networkManager.ReleaseAllocation(ctx, netAlloc) + return nil, fmt.Errorf("%w: standby fork restore network rewrite is not supported for hypervisor %s", ErrNotSupported, stored.HypervisorType) + } log.ErrorContext(ctx, "failed to patch snapshot network identity", "instance_id", id, "error", err) netAlloc, _ := m.networkManager.GetAllocation(ctx, id) m.networkManager.ReleaseAllocation(ctx, netAlloc) From 566d52ba4747b3ec9f8131c92c7cec003932cda9 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 27 Feb 2026 15:17:00 -0500 Subject: [PATCH 03/20] api: add fork instance endpoint --- cmd/api/api/instances.go | 64 ++++ cmd/api/api/instances_test.go | 106 ++++++ lib/oapi/oapi.go | 684 +++++++++++++++++++++++++--------- openapi.yaml | 68 ++++ 4 files changed, 750 insertions(+), 172 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 50deb885..e44a2199 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -447,6 +447,70 @@ func (s *ApiService) RestoreInstance(ctx context.Context, request oapi.RestoreIn return oapi.RestoreInstance200JSONResponse(instanceToOAPI(*result)), nil } +// ForkInstance forks an instance from stopped or standby into a new instance. +// The id parameter can be an instance ID, name, or ID prefix. +// Note: Resolution is handled by ResolveResource middleware. +func (s *ApiService) ForkInstance(ctx context.Context, request oapi.ForkInstanceRequestObject) (oapi.ForkInstanceResponseObject, error) { + inst := mw.GetResolvedInstance[instances.Instance](ctx) + if inst == nil { + return oapi.ForkInstance500JSONResponse{ + Code: "internal_error", + Message: "resource not resolved", + }, nil + } + log := logger.FromContext(ctx) + + if request.Body == nil { + return oapi.ForkInstance400JSONResponse{ + Code: "invalid_request", + Message: "request body is required", + }, nil + } + + result, err := s.InstanceManager.ForkInstance(ctx, inst.Id, instances.ForkInstanceRequest{ + Name: request.Body.Name, + }) + if err != nil { + switch { + case errors.Is(err, instances.ErrNotFound): + return oapi.ForkInstance404JSONResponse{ + Code: "not_found", + Message: "instance not found", + }, nil + case errors.Is(err, instances.ErrInvalidState): + return oapi.ForkInstance409JSONResponse{ + Code: "invalid_state", + Message: err.Error(), + }, nil + case errors.Is(err, instances.ErrAlreadyExists), errors.Is(err, network.ErrNameExists): + return oapi.ForkInstance409JSONResponse{ + Code: "name_conflict", + Message: err.Error(), + }, nil + case errors.Is(err, instances.ErrNotSupported): + return oapi.ForkInstance501JSONResponse{ + Code: "not_supported", + Message: err.Error(), + }, nil + default: + // Validation errors (for example invalid fork name) are user input issues. + if strings.Contains(err.Error(), "name") { + return oapi.ForkInstance400JSONResponse{ + Code: "invalid_request", + Message: err.Error(), + }, nil + } + + log.ErrorContext(ctx, "failed to fork instance", "error", err) + return oapi.ForkInstance500JSONResponse{ + Code: "internal_error", + Message: "failed to fork instance", + }, nil + } + } + return oapi.ForkInstance201JSONResponse(instanceToOAPI(*result)), nil +} + // StopInstance gracefully stops a running instance // The id parameter can be an instance ID, name, or ID prefix // Note: Resolution is handled by ResolveResource middleware diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 8c74cf89..5ef022e3 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -9,6 +9,7 @@ import ( "github.com/c2h5oh/datasize" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/instances" + mw "github.com/kernel/hypeman/lib/middleware" "github.com/kernel/hypeman/lib/oapi" "github.com/kernel/hypeman/lib/paths" "github.com/kernel/hypeman/lib/system" @@ -137,6 +138,24 @@ type captureCreateManager struct { lastReq *instances.CreateInstanceRequest } +type captureForkManager struct { + instances.Manager + lastID string + lastReq *instances.ForkInstanceRequest + result *instances.Instance + err error +} + +func (m *captureForkManager) ForkInstance(ctx context.Context, id string, req instances.ForkInstanceRequest) (*instances.Instance, error) { + reqCopy := req + m.lastID = id + m.lastReq = &reqCopy + if m.err != nil { + return nil, m.err + } + return m.result, nil +} + func (m *captureCreateManager) CreateInstance(ctx context.Context, req instances.CreateInstanceRequest) (*instances.Instance, error) { reqCopy := req m.lastReq = &reqCopy @@ -190,6 +209,93 @@ func TestCreateInstance_OmittedHotplugSizeDefaultsToZero(t *testing.T) { assert.Equal(t, int64(0), int64(hotplugBytes), "response should report zero hotplug_size when omitted") } +func TestForkInstance_Success(t *testing.T) { + svc := newTestService(t) + + now := time.Now() + source := instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "src-instance", + Name: "src-instance", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateStopped, + } + + forked := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "forked-instance", + Name: "forked-instance", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateStopped, + } + + mockMgr := &captureForkManager{ + Manager: svc.InstanceManager, + result: forked, + } + svc.InstanceManager = mockMgr + + resp, err := svc.ForkInstance( + mw.WithResolvedInstance(ctx(), source.Id, source), + oapi.ForkInstanceRequestObject{ + Id: source.Id, + Body: &oapi.ForkInstanceRequest{ + Name: "forked-instance", + }, + }, + ) + require.NoError(t, err) + + created, ok := resp.(oapi.ForkInstance201JSONResponse) + require.True(t, ok, "expected 201 response") + assert.Equal(t, "forked-instance", created.Name) + assert.Equal(t, source.Id, mockMgr.lastID) + require.NotNil(t, mockMgr.lastReq) + assert.Equal(t, "forked-instance", mockMgr.lastReq.Name) +} + +func TestForkInstance_NotSupported(t *testing.T) { + svc := newTestService(t) + + source := instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "src-instance", + Name: "src-instance", + Image: "docker.io/library/alpine:latest", + CreatedAt: time.Now(), + HypervisorType: hypervisor.TypeQEMU, + }, + State: instances.StateStopped, + } + + mockMgr := &captureForkManager{ + Manager: svc.InstanceManager, + err: instances.ErrNotSupported, + } + svc.InstanceManager = mockMgr + + resp, err := svc.ForkInstance( + mw.WithResolvedInstance(ctx(), source.Id, source), + oapi.ForkInstanceRequestObject{ + Id: source.Id, + Body: &oapi.ForkInstanceRequest{ + Name: "forked-instance", + }, + }, + ) + require.NoError(t, err) + + notSupported, ok := resp.(oapi.ForkInstance501JSONResponse) + require.True(t, ok, "expected 501 response") + assert.Equal(t, "not_supported", notSupported.Code) +} + func TestInstanceLifecycle_StopStart(t *testing.T) { // Require KVM access for VM creation if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 00aa62e9..1fee2568 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -398,6 +398,12 @@ type ErrorDetail struct { Message *string `json:"message,omitempty"` } +// ForkInstanceRequest defines model for ForkInstanceRequest. +type ForkInstanceRequest struct { + // Name Name for the forked instance (lowercase letters, digits, and dashes only; cannot start or end with a dash) + Name string `json:"name"` +} + // GPUConfig GPU configuration for the instance type GPUConfig struct { // Profile vGPU profile name (e.g., "L40S-1Q"). Only used in vGPU mode. @@ -959,6 +965,9 @@ type CreateIngressJSONRequestBody = CreateIngressRequest // CreateInstanceJSONRequestBody defines body for CreateInstance for application/json ContentType. type CreateInstanceJSONRequestBody = CreateInstanceRequest +// ForkInstanceJSONRequestBody defines body for ForkInstance for application/json ContentType. +type ForkInstanceJSONRequestBody = ForkInstanceRequest + // StartInstanceJSONRequestBody defines body for StartInstance for application/json ContentType. type StartInstanceJSONRequestBody StartInstanceJSONBody @@ -1118,6 +1127,11 @@ type ClientInterface interface { // GetInstance request GetInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + // ForkInstanceWithBody request with any body + ForkInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + ForkInstance(ctx context.Context, id string, body ForkInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetInstanceLogs request GetInstanceLogs(ctx context.Context, id string, params *GetInstanceLogsParams, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -1494,6 +1508,30 @@ func (c *Client) GetInstance(ctx context.Context, id string, reqEditors ...Reque return c.Client.Do(req) } +func (c *Client) ForkInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewForkInstanceRequestWithBody(c.Server, id, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) ForkInstance(ctx context.Context, id string, body ForkInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewForkInstanceRequest(c.Server, id, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) GetInstanceLogs(ctx context.Context, id string, params *GetInstanceLogsParams, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewGetInstanceLogsRequest(c.Server, id, params) if err != nil { @@ -2522,6 +2560,53 @@ func NewGetInstanceRequest(server string, id string) (*http.Request, error) { return req, nil } +// NewForkInstanceRequest calls the generic ForkInstance builder with application/json body +func NewForkInstanceRequest(server string, id string, body ForkInstanceJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewForkInstanceRequestWithBody(server, id, "application/json", bodyReader) +} + +// NewForkInstanceRequestWithBody generates requests for ForkInstance with any type of body +func NewForkInstanceRequestWithBody(server string, id string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "id", runtime.ParamLocationPath, id) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/instances/%s/fork", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + // NewGetInstanceLogsRequest generates requests for GetInstanceLogs func NewGetInstanceLogsRequest(server string, id string, params *GetInstanceLogsParams) (*http.Request, error) { var err error @@ -3313,6 +3398,11 @@ type ClientWithResponsesInterface interface { // GetInstanceWithResponse request GetInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetInstanceResponse, error) + // ForkInstanceWithBodyWithResponse request with any body + ForkInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*ForkInstanceResponse, error) + + ForkInstanceWithResponse(ctx context.Context, id string, body ForkInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*ForkInstanceResponse, error) + // GetInstanceLogsWithResponse request GetInstanceLogsWithResponse(ctx context.Context, id string, params *GetInstanceLogsParams, reqEditors ...RequestEditorFn) (*GetInstanceLogsResponse, error) @@ -3923,6 +4013,33 @@ func (r GetInstanceResponse) StatusCode() int { return 0 } +type ForkInstanceResponse struct { + Body []byte + HTTPResponse *http.Response + JSON201 *Instance + JSON400 *Error + JSON404 *Error + JSON409 *Error + JSON500 *Error + JSON501 *Error +} + +// Status returns HTTPResponse.Status +func (r ForkInstanceResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r ForkInstanceResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type GetInstanceLogsResponse struct { Body []byte HTTPResponse *http.Response @@ -4530,6 +4647,23 @@ func (c *ClientWithResponses) GetInstanceWithResponse(ctx context.Context, id st return ParseGetInstanceResponse(rsp) } +// ForkInstanceWithBodyWithResponse request with arbitrary body returning *ForkInstanceResponse +func (c *ClientWithResponses) ForkInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*ForkInstanceResponse, error) { + rsp, err := c.ForkInstanceWithBody(ctx, id, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseForkInstanceResponse(rsp) +} + +func (c *ClientWithResponses) ForkInstanceWithResponse(ctx context.Context, id string, body ForkInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*ForkInstanceResponse, error) { + rsp, err := c.ForkInstance(ctx, id, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseForkInstanceResponse(rsp) +} + // GetInstanceLogsWithResponse request returning *GetInstanceLogsResponse func (c *ClientWithResponses) GetInstanceLogsWithResponse(ctx context.Context, id string, params *GetInstanceLogsParams, reqEditors ...RequestEditorFn) (*GetInstanceLogsResponse, error) { rsp, err := c.GetInstanceLogs(ctx, id, params, reqEditors...) @@ -5651,6 +5785,67 @@ func ParseGetInstanceResponse(rsp *http.Response) (*GetInstanceResponse, error) return response, nil } +// ParseForkInstanceResponse parses an HTTP response from a ForkInstanceWithResponse call +func ParseForkInstanceResponse(rsp *http.Response) (*ForkInstanceResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &ForkInstanceResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 201: + var dest Instance + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON201 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 409: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON409 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 501: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON501 = &dest + + } + + return response, nil +} + // ParseGetInstanceLogsResponse parses an HTTP response from a GetInstanceLogsWithResponse call func ParseGetInstanceLogsResponse(rsp *http.Response) (*GetInstanceLogsResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -6378,6 +6573,9 @@ type ServerInterface interface { // Get instance details // (GET /instances/{id}) GetInstance(w http.ResponseWriter, r *http.Request, id string) + // Fork an instance from stopped or standby state + // (POST /instances/{id}/fork) + ForkInstance(w http.ResponseWriter, r *http.Request, id string) // Stream instance logs (SSE) // (GET /instances/{id}/logs) GetInstanceLogs(w http.ResponseWriter, r *http.Request, id string, params GetInstanceLogsParams) @@ -6567,6 +6765,12 @@ func (_ Unimplemented) GetInstance(w http.ResponseWriter, r *http.Request, id st w.WriteHeader(http.StatusNotImplemented) } +// Fork an instance from stopped or standby state +// (POST /instances/{id}/fork) +func (_ Unimplemented) ForkInstance(w http.ResponseWriter, r *http.Request, id string) { + w.WriteHeader(http.StatusNotImplemented) +} + // Stream instance logs (SSE) // (GET /instances/{id}/logs) func (_ Unimplemented) GetInstanceLogs(w http.ResponseWriter, r *http.Request, id string, params GetInstanceLogsParams) { @@ -7273,6 +7477,37 @@ func (siw *ServerInterfaceWrapper) GetInstance(w http.ResponseWriter, r *http.Re handler.ServeHTTP(w, r) } +// ForkInstance operation middleware +func (siw *ServerInterfaceWrapper) ForkInstance(w http.ResponseWriter, r *http.Request) { + + var err error + + // ------------- Path parameter "id" ------------- + var id string + + err = runtime.BindStyledParameterWithOptions("simple", "id", chi.URLParam(r, "id"), &id, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "id", Err: err}) + return + } + + ctx := r.Context() + + ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) + + r = r.WithContext(ctx) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.ForkInstance(w, r, id) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + // GetInstanceLogs operation middleware func (siw *ServerInterfaceWrapper) GetInstanceLogs(w http.ResponseWriter, r *http.Request) { @@ -7990,6 +8225,9 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl r.Group(func(r chi.Router) { r.Get(options.BaseURL+"/instances/{id}", wrapper.GetInstance) }) + r.Group(func(r chi.Router) { + r.Post(options.BaseURL+"/instances/{id}/fork", wrapper.ForkInstance) + }) r.Group(func(r chi.Router) { r.Get(options.BaseURL+"/instances/{id}/logs", wrapper.GetInstanceLogs) }) @@ -8952,6 +9190,69 @@ func (response GetInstance500JSONResponse) VisitGetInstanceResponse(w http.Respo return json.NewEncoder(w).Encode(response) } +type ForkInstanceRequestObject struct { + Id string `json:"id"` + Body *ForkInstanceJSONRequestBody +} + +type ForkInstanceResponseObject interface { + VisitForkInstanceResponse(w http.ResponseWriter) error +} + +type ForkInstance201JSONResponse Instance + +func (response ForkInstance201JSONResponse) VisitForkInstanceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(201) + + return json.NewEncoder(w).Encode(response) +} + +type ForkInstance400JSONResponse Error + +func (response ForkInstance400JSONResponse) VisitForkInstanceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(400) + + return json.NewEncoder(w).Encode(response) +} + +type ForkInstance404JSONResponse Error + +func (response ForkInstance404JSONResponse) VisitForkInstanceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(404) + + return json.NewEncoder(w).Encode(response) +} + +type ForkInstance409JSONResponse Error + +func (response ForkInstance409JSONResponse) VisitForkInstanceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(409) + + return json.NewEncoder(w).Encode(response) +} + +type ForkInstance500JSONResponse Error + +func (response ForkInstance500JSONResponse) VisitForkInstanceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(500) + + return json.NewEncoder(w).Encode(response) +} + +type ForkInstance501JSONResponse Error + +func (response ForkInstance501JSONResponse) VisitForkInstanceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(501) + + return json.NewEncoder(w).Encode(response) +} + type GetInstanceLogsRequestObject struct { Id string `json:"id"` Params GetInstanceLogsParams @@ -9652,6 +9953,9 @@ type StrictServerInterface interface { // Get instance details // (GET /instances/{id}) GetInstance(ctx context.Context, request GetInstanceRequestObject) (GetInstanceResponseObject, error) + // Fork an instance from stopped or standby state + // (POST /instances/{id}/fork) + ForkInstance(ctx context.Context, request ForkInstanceRequestObject) (ForkInstanceResponseObject, error) // Stream instance logs (SSE) // (GET /instances/{id}/logs) GetInstanceLogs(ctx context.Context, request GetInstanceLogsRequestObject) (GetInstanceLogsResponseObject, error) @@ -10340,6 +10644,39 @@ func (sh *strictHandler) GetInstance(w http.ResponseWriter, r *http.Request, id } } +// ForkInstance operation middleware +func (sh *strictHandler) ForkInstance(w http.ResponseWriter, r *http.Request, id string) { + var request ForkInstanceRequestObject + + request.Id = id + + var body ForkInstanceJSONRequestBody + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + sh.options.RequestErrorHandlerFunc(w, r, fmt.Errorf("can't decode JSON body: %w", err)) + return + } + request.Body = &body + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.ForkInstance(ctx, request.(ForkInstanceRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "ForkInstance") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(ForkInstanceResponseObject); ok { + if err := validResponse.VisitForkInstanceResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + // GetInstanceLogs operation middleware func (sh *strictHandler) GetInstanceLogs(w http.ResponseWriter, r *http.Request, id string, params GetInstanceLogsParams) { var request GetInstanceLogsRequestObject @@ -10754,178 +11091,181 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x97XLbOLLoq6B4z9bKZyVZ/ojj+NTUKcdOPN6NE9849t6zo1wFIiEJYxLgAKASJZW/", - "+wD7iPMkp9AA+CVQovLhxDvZ2prQIgg0Gt2N7kaj+0MQ8iTljDAlg6MPgQxnJMHweKwUDmc3PM4S8pL8", - "lhGp9M+p4CkRihJolPCMqVGK1Uz/FREZCpoqyllwFFxiNUNvZ0QQNIdekJzxLI7QmCD4jkRBNyDvcJLG", - "JDgKthOmtiOscNAN1CLVP0klKJsGH7uBIDjiLF6YYSY4i1VwNMGxJN3asBe6a4Ql0p/04Ju8vzHnMcEs", - "+Ag9/pZRQaLg6JfyNF7njfn4VxIqPfjxHNMYj2NySuY0JMtoCDMhCFOjSNA5EcuoODHv4wUa84xFyLRD", - "HZbFMaITxDgjWxVksDmNqMaEbqKHDo6UyIgHMxHANKKRZwVOzpF5jc5PUWdG3lUH2X04Pgyau2Q4Icud", - "/pwlmPU0cjVYrn9oW+772b6vZ8qTJBtNBc/S5Z7PX1xcXCN4iViWjIko93i4m/dHmSJTInSHaUhHOIoE", - "kdI/f/eyDNtgMBgc4d2jwaA/8EE5JyziohGl5rUfpTuDiKzoshVKbf9LKH1+c356foxOuEi5wPDt0kg1", - "wi6jpzyvMtlUV8VH/48zGkfLVD/WPxMxokwqzBpo8Ny+1OjiE6RmBNnv0M0F6ky4QBEZZ9MpZdOtNvSu", - "BVZMFIlGWC0PB6Ai24ZyhhRNiFQ4SYNuMOEi0R8FEVakp9+0GlAQvGY43aLVYMuslpmVHCWyqXfXBFGG", - "EhrHVJKQs0iWx6BMHew3T6bEMEQI7pFQT/TPKCFS4ilBHS02texmSCqsMomoRBNMYxK1WiMfIZjJ/MrH", - "iEaEKTqhVf425NTD43Bnd88rOxI8JaOITu1OVO3+FH7XJKb7UQha+yeiGW3Rbh4wpCCT5fGeguiGQQSZ", - "EEE0jX/mcKngc8I0t+jx/gPGDf7PdrFFb9v9eRuQeVk0/9gNfstIRkYpl9RAuCS57BtNRoBqBF/4YYZX", - "q9a6RFFSYbGaP6DFF+BEA18r3FyZpnV5COLOdlPh7Eax92ROmEfxCTlT9kV1xs/4FMWUEWRbWPxqOacH", - "+CnmIOa+xNy6QYHSZYbWcH+CQDI/NPSm33UDwrJEIzPm0zI2ZwQLNSYVZDZsS7ajArpG9F9WWKK2/2BJ", - "RqulwiVljERIt7TMalqiTIL2uTR94IxbqkZzIqSXjwCsv1GFbIvGrmIe3k5oTEYzLGcGYhxFwIM4vqzM", - "xKOBVVRanGrB5joEzUAixdHVz8e7Dw6QHcCDQ8kzERoIlmdS+lp3b9oihcUYx7GXNprJbfN9d5lC/BRw", - "lTNG036SU6AjTCO9AruauvtukGZyZp5AHmuoYD/TYkCTV6yfX3smfQJCwmj+jXaQX697kZrFRtOYa5wu", - "UMbob1lFae6jc63/K6SFP41I1EUYXmgxjDPFe1PCiNByCk0ET0CDKim2qEP6034XDbWu19OabQ/v9gaD", - "3mAYVFXTeL83TTONCqwUERrA//8L7r0/7v1j0Hv0ungc9Xuv//IfPgJoq207Tc/Os+N4v4scsGUVvA7o", - "avV8hYbrkyJm+c4172+6eifnyxu8gT/i4S0Rfcq3YzoWWCy22ZSyd0cxVkSq6mxWt107P4BtxcTYVE99", - "w6nVDA4gt07M3xIRakkZE00gsquFJVWyi7C2WUHIIL2b/RcKMdM0azZ2LhBhEXpL1QxhaFfFQLLo4ZT2", - "qAE16AYJfveMsKmaBUcHe0v0qImxYx96r//T/bT1316SFFlMPMT4kmeKsimC12b3nVGJChioIsna7dZh", - "N4tBxUooOzef7eSQYCHwwr9qDrhVq2eMo8blCxOPJv1iToSgkdvRTi5OUSemt8SSJRIZQ8NsMNgLoQE8", - "EvtLyJMEs8j8ttVHLxKq9E6SFRuk8a70y0v4S0DCGYc9Po65nlCOvgYFwuHFGZqeJTp1ngmJrLULexoG", - "vxMs2dnl9baWKimWUs0Ez6azKlRWpG0GD5W3I8pH49QHE5W36Hz7BdICF8VUYycXsDuDwcXjbTkM9B8P", - "3B9bfXRqUAbg6/Xjwsp9OcOCgPYRIc7QyeU1wnHMQ2vPTbSSOKHTTJCoX3MjQO8+gidMiUXKqU/5rFFG", - "0XSZQHq94u0GdLA9pmxb6mXohZvhnbD5Z6hAT9icCs4SrYbOsaBablWcOh+C5y9On4yePL8JjjQTRVlo", - "PSSXL16+Co6CvcFgEPi0DE1Ba+TA2eX1CayUbj/jKo2z6UjS9x7RepzPDyUk4cKo/vYb1JlVJa/RjBAs", - "zjDYO3tsiGvnDOjKLUpEJbR2vZiOqxSze/bYRy2zRUrEnEqfzf9z/s6tfElOGsFUpW1JxJyInGiBivsl", - "vSuMeRb1SkN2g99IolWO+XtNLAW0npZ+47vVtrxmv8VxShlp3HC7QUIUBsfzp9PntSSiF5EJ1ebGLVn0", - "5jjOCHI9W8ySHLFV0k0zkXJp+sdTo6YqgpPgKBjj8JawyEu538nm/paL25jjqLfzhfd2RpTue3mKz82L", - "KiX6cFw3GFn0lkZqNor4W6ZB9uwA9g3KG+fbwDs9Exz//s9/3VwUmuzO2Ti1e8LO7oPP3BNqu4Du2mul", - "5hPJUv80rlP/JG4ufv/nv9xMvu0kCNP0GVVOdIzjpzqVv8+ImhFR0g3cAuufjJkBnyNHL6XhK56k8vHP", - "EjPxORExXpTEuoUp2BmAbK1BJagC/rLfaSF9i/THa4S87s2pEGd102d34BfjHqA8MD3W/G13nTaQ5IDs", - "7F7Yx91lkBoguqXpaKq11hGe5p6wVQdzV7c0RfBFD74wyxjHhnmjTPeMxpyr/pD9fUYYgrWDBSbvSAhy", - "Spv66PjyXKK3NI7BbgZBsLxxDdmrkigwzaXS/xUZ66JxppAgCVcEWZUYBskAFmg8Jihj2J389YesjBU7", - "wTpdWbTcEsFIPJoRHBEhW2LGfITsR43IgalOsFREGAmdpVV8nf7t4gp1ThcMJzREfzO9XvAoiwm6ylLN", - "w1tV7HWHLBVkThhYTFr9oXZcPkE8Uz0+6SlBiAMxgc5yz4M9lpqfXV7bg0251R+yl0QjlrCIRACz2yUk", - "UjOsUMTZnzXHwnZZ6rY8fg3pfl7uBvMwzapY3q1j+DkcJ+r5zKlQGY61yKroj97TRXNu7bETzLF42V6x", - "oignOKyqx0JtTU7TMxxiL2vRfivTKErNVuaaM3zfYU3uuQozqXhSOrJBnZpTilbdV1XhMedxT+s/oBos", - "7+9e/cWAu3z8mSxMV2ZRmqTkaDr2eDq1MKQMTekUjxeqajnsDJaX3o9o178P1U2hAYY8SDRSfPXhKJ0g", - "17bNWQgEEowUH80n1NNzvmkWXjgqUViLQ7BEq7vopSG17NtFb2dUb7MSOSQAB99clC3x/pD1QOQcodN8", - "gLzbvEstWcHjCl10uCgBQcF5jsaLLYTRzUUfvcqh/bNEDCs6Jy5WYoYlGhPCUAbqGYlgfBCnZQAyqWUY", - "VfXPrawyYRVb4HDg9l0faUMowVbua/JOsKIhOGzHtDYfOCgzC6VH0gKAlXedVrvEqiPll2RKpRK1A2XU", - "efn0ZG9v71FdX9h90Bvs9HYevNoZHA30///R/uz5y0eO+Po6rsoL6wIvS5ST6/PTXaucVMdR7/fxo8N3", - "77B6dEDfykfvk7GY/rqH7yS2xC+eTgvfPepk2uxzok9Tlc9jX3KMN3jkP9nRvlFYizvaW7X9mNm90i2/", - "RiCM7zjWHgZuHqpSF4JrD3RLk1uaj/5V6wcF5ZccG/bcJKTeE6JTKm8fC4JvtVXp2V/19ixHZt/xexwz", - "bUeNF4i80+oZiZDgXE2k8XNU1ZSd/Yf7h3sH+4eDgSf+Y5mIeUhHod5VWgHw4uQcxXhBBIJvUAcMvQiN", - "Yz6uEu+DvYPDh4NHO7tt4TBmUjs85FqU+wp1LEb+4mIJ3ZsKULu7Dw/29vYGBwe7+62gsgpeK6CcMlhR", - "HR7uPdzfOdzdb4UFn9n5xMXj1OMLIp9zMU1jaozsnkxJSCc0RBDRg/QHqJPAtkRyi6/Kk2McjYRVA737", - "gcI0lit9mmYw29KEbyVZrGgaE/MOFqSVpgszP4WefP5iyhgRozxcaYOebBTTWs+Ym0veBFWi0Sqou6AS", - "NItCIaIkjo4Mh66Vc7CaBWCvm+jAzqElNTzjb4noxWRO4jIRmO1IA5twQVBOJ2bRKrOibI5jGo0oS7MG", - "z2gDKp9mAvRL0ynCY54pY6rDgpUHgbNXsBEmWly3O/ovPO5LQ2s7c0PHXyr4hMaeaYDRat/aLd25xJ7t", - "D656O//XOOJZvDBygDJj6CY8Iv1awCu0bz29yyaY8mhjVIZuaU65a8LjHs2tXYcRa3SHmKExQXabNE5d", - "cJsUgxQC/pFPYE4ETsg4m0yIGCUeS+upfo9MA+ODogxdPK4KTS2c26pbl5XFAX1rgkMbLNoO+x5LrjaN", - "bgmbr/3L9ZKY+JimcBS9VMK2sREpffQ8j+9GZ5fXEhXuJI+JV13exmPTy9lCauPE9GiiyygrW2ZAnK3F", - "8GXxobVhPcI48QogxwioM5+mGbDh1cve+Yub7SQi824FJnABzXhMNNxbJd1q7oJSijPeylHRvElFNoQh", - "2zJQCVc5B7dGUolfPdhRXOF4JGOuPNC80i8RvESdm6cmGEFD0EVpZSn17yUsVOj7wMsxWiI1DXsFA9Zt", - "7QqDr3V7JGbbKk+vMqiPVX4mODa3Qar0XMQ3uoXnt9WF5rdrudd24hv33J0WtgigOLk4NZZZyJnClBGR", - "H9RVT7shUifoBj2tDESYJOATnfzX6pPvBt9NTi6rrP+TpVDyr2L5N4RLaiEXz0mEEszohEhlwyUrI8sZ", - "3n1wcGQCtSMy2X9w0O/3Nw1VeFLEJrRaim1zrluKWujL2eetw1eISGgzlw/B5fGrn4OjYDuTYjvmIY63", - "5Ziyo9Lf+Z/FC3gwf44p854Ht4rtp5OlmP7K8qZ6zzK/H+mZMBLmBMlBS1zrm/Tv5M81acb0PYmQN7RO", - "4SnS+jdQ3OfF0H1GNHxxJUuVouDLxwQtIuLp+9XmtlOMoI0dM2OKxsVlgWVD+5Oue8iV0bNLkbMpYXm8", - "bBybp5CzueYKX/BsRYC7d0uL8ZaLW8qmo4h6qPPv5iWKqCChgoCZ9TwUbOM0XU+KfuUvl2ltLwLYMEDP", - "7vLNJfmnOFyro7+Y/vW3/ycvH/6689uzm5v/mZ/99fQ5/Z+b+PJF+yMbT8jJ6gjQbxrGufJMDbyMlfDN", - "tuRxgVXoUXxmXKoGrNk3SHGU6I/76AQMtKMh66FnVBGB4yM0DHBK+xaZ/ZAnwwB1yDscKvMV4gzpruzR", - "8Zb++NKE3eiPPzgb8GO9j8ieEQuL5DycQ2bjiCeYsq0hGzLbF3ITkXBoo58iFOJUZYLoFdG6ZrxAY4HD", - "4my4GLyLPuA0/bg1ZGCJkndK6BmkWKg8XNyNAAttoTKHQrY5iRDEVUlryQ5Zvn+Aaa47UVhMiernLkRw", - "1NQOZhqQ4jUzuKjGNhwOup51RLqdXsiYSkUYyr0SVALxoo4LUjkcVNj/cHC4/vwxp6EV5AfUvXxB2xFl", - "C/4wBAxDG2E8mimVrg9fAHljeAT9/OrVpUaD/vcKuY4KXORLbIwxnKYxJdKcqqkYdBIbF7QV+E7OzOq2", - "nNAr01h/FrcIw3gCA6NXz66QIiKhzMjvTqjROaGhnh+c71ApM02KFKPjk4snW/0WN8wBtzn8K9bxVT7D", - "2jGCc24tW5jwReE01/jtovPTrlanLIcWihacmz7lAsVGwBR8fYSuJalGMcBSmSMes5LxovCQGak+DLZc", - "j2ldUhyhl7l+h3NQ8mssBTG4Lgu+hG5tYIs51F3qvVuFFY6rrf1iRRsc4WKFrNMbtuJmUbCa/T0YB57n", - "rDGwsxVvl52WejA/aRRr/9U1kL1NbclN4+mrQWmlIMQ8pL59LPzXiClftqveUTVqPJFB+rU9f3HWw80F", - "mmHJ/qzgZc2G2Nl72Oqmth617VlG+RSDTwxIOVe5CLfcB29i/W5pHJujLUmnDMfoEepcnZ/97fzZsy3U", - "Qy9eXNSXYtUXvvVpEVrvSPvs8hri1bEcSYZTOeOqOaoFI9dGz1QquRwA2CoOY3Uo/8+VcHtvROXWF4zB", - "FxljEFJSn8bXia7/lpEa/0aR/cHnxOWvjKT/3HB4q1R/pWj4RiHuiySvynPz85eNa/8q4FQi1H1yqKx7", - "uOC9Tw5K7wbUE7h0LLWoJRE6vyyu/BZOKtd9bU6Pdvs7B4f9ncGgvzNo47JLcLhi7Ivjk/aDD3aNE+MI", - "j4/C6IhMPsNlaAnbKIk4fosXEg2dGj8MjN1QMhhKwsKq+q2OY5dj/z8t1L+uuKwL5t8keL9dVP6KXBxX", - "1SwcrXXBB//4rIQdpK0GcAWN3VejTZzZBIU8iyOtb4015xnzjUTWypREFQlOgFmv2S3jb1l16sanqfn3", - "t4yIBbq5uKh4wAWZ2FwPLSbO07RxHXi60TLsrlHJ10JTCpC/i6D4uiQs7UBfPAS+7K5zsTiG6lq47QrN", - "03u0TZlBt177FXOqOVwiMh9lmU+90q9cVO319flpZcExPtg5HBw+6h2Odw56+9Fgp4d39g56uw/wYLIX", - "PtxryIrUPrTl06NVqhzaHMUOiAfnpbl4EB1pHsrDTcaZQvnFQs2cJ1pPRSUV2MRsgz/hpdGGdQ+wu4b6", - "TbzIteSVH19izaju2xT+Wv3F1SxTWg2Cb+QsU0j/BSDrKVgrY3UXhueP0HMO31hIu3qjrJkrpjlm0Xix", - "3Lxu2nRs1I4gUnFBIhjMCrAj9DQXWrnYs2KuI4l9NLLURrdB5N6WcYJYy8KuVtANLNaDbmBQGHQDhxn9", - "aGYITwB80A0sIN7A2DLd+Jz7BMcgw4rAmUzRmL43LKdBp1LR0Fh3GFazie3s7UMSjcwW2nT8ZqIx7Dab", - "f+S4+uYCdeCuyF+QNf70X1v5UV2ZhfZ3H+0/Oni4++igVaRpAeB6aXwCsULLwK0VzWGajVx2uIapn1xe", - "w+ajNzaZJcaat3MvLEYtOEKt7VGGinRzxeCP+o/KAbYRz8ZxyTtkI+whirNNbsCGs6nfaDynkwn77X14", - "u/uroMnOuwO5O/YaR/lAfk3yvOzRXDK7yLhn7rb7bUggKCEbw4RfEgkzQFdEIaCfnhZYekfNQ3wsyblg", - "YotxL2Ht7+3tHT58sNuKrix0JcYZgf23DOWFhaDEYtASdV5eXaHtEsGZPl3cYyqI1JMzN1+8fIZsjpFB", - "JSRS2x57PippUFgKqrF9z5NGlN9YjcVOyiIdIpVybWaJy73Y3tsbPNx/cPigHRtbi2ck3q2WMLadPeEX", - "JCR0Xln5DnjBXx1fIt27mOCwquHv7O7tPzh4eLgRVGojqJTATCZUqY0AO3x48GB/b3enXby7z9Ntb3JU", - "GLYquzxM5yEKz2p4ULEsertNu4VPS1wOj1wZkVmEeNbj+TYJ4C1u71EJvdJS7CjqaCWqrJCWbqBttfEz", - "+EWkHqcp56xWF9vG1q4Opb3EanbOJnz5KGMTg88GKDkXd6oVHwnZ+CLCKImc7MotP6tLQchTLAmKMmIx", - "Z3QjgS3CsTnOSbGagbIKH1I2rQZ7Lw3YxgwzMKy+qwnj2oZtPEbSH1TzSmSAK+NLlggX4TWtHONUjvxW", - "xXLHgkyzGAtUjx9fAbJcJDFlt216l4tkzGMaIv1B3Zyf8Djmb0f6lfwJ5rLVanb6g1Fxklwzzw1wNo7A", - "LEht3GIKP+lZbtUik2Dn3zbfb0NS8TYOOO/x0lNtvJkQ62tG35UIvXrxaX930BSI1tBpJQRtOTx/U9lu", - "SdbH8S5y/jhPEOI5xjQHRTULtqoHV+brmy2cRK4Ku1vWBFDH+fTcxbIqXksXvFptxO0OQ+veawfNtiRh", - "dfT9wwcPD1resPssVXtF2uXPUKznyQqFumGlLtpobYcPDh892tt/8Gh3I/3IHXQ0rE/TYUd5fWp5gGo6", - "24MB/G8joMxRhx+khuOOKkCVnD6fDNDHFaxbXHppsLpXlTwoVtKZ+VUFvJ2Ku0JbOq6oXKUEex0ymRBw", - "HI0M3noFMLUgrFYwhDjFIVULjwWI30JcCsqb1C5vtOi9BqwHpbZvhCeKCDiNkNm4OPfvuMHRfxrLrkYL", - "h60v6sps3GRFvqiPamxIE8gV1TwULRwEhiJ8h+9vc2Sit1hWvPr6OVQk6pYSKNaPf0yL9imuHa3nWa6L", - "43TfBSR/Ruvy8teWs2R1VJTkOsZXbaHNLKg1AogSa+Ng9+zInltN4frgjZp8sBvgp301Gpev0K/MUVC5", - "b1/supuP2y714/J3ZgfbfLzSCf4mH9ZvEwM9Whgsyou+uxWS8FGTOV9pSlWTuFpAtcvG1FRXsDfKUKkx", - "6pAkVQt3a8JZplubnfcc5x16ifELx7kNHn2JSPvrlaH1/ybJj8pHbG6QtYdrS2vaGM/qV1dP6+Erxia0", - "yR+q4Ra1K+1SrSgisqpglakcBQafjSWfZvXLbxsUqWoy8QvOcdVBXJWqdZbrSn9aaWYlSJrXxpyvfmZF", - "LypdKa9PRJk1v9YHZ5szKm0A9+rZQczdYkHBnrMIMojVKMhN9GU/wOqwjwv8Lh8BrGUsUS2foplHKUP0", - "2WPIF/DSZYmgE9cFgFHPjPn480qdOapaXoxVtc/cCb6X8az8WSHRmnirRpzFGN3V5dW06CJhJqhaXOkN", - "wQanESyIOM4MGcJOAZOAn4vB4YLCx49gpk482uoZYUTQEB1fngOVJJhBfl10c4FiOiHhIoyJjS9fOtuF", - "9AgvTs575mJMnsAQKpcoQIjLHHZ8eQ5Ji2zNkGDQ3+1DtmmeEoZTGhwFe/0dSMuk0QBT3IZ7h/BoHVGa", - "D2EnO4/sjvvYNNGolSln0iBndzCo1aDBRWKY7V+l8bCY7bW1UmiKfC3HWywFRDpNwIL/sRvsD3Y2gmdt", - "LhffsNcMZ2rGBX1PAMwHGyLhkwY9Z8aqdqmviW1Y0Gxw9EuVWn95/fF1N5BZkmCtIhp0FbhKuWxSYYhE", - "GDHy1l5I/ZWP++jK2CQQXV6UTzQuAxJpkYSRwqI/fY+wCGd0TobMSmKTlwcLuH2TIC2Bzd2HKpmZoc3q", - "GxYmUj3m0aKG3by7bd1dz0XVFgjeuDhPnmQybajS45OOJpeVDLk3iRdhmKkiNZJJYnVL4BBzQt95O2x1", - "Gq+FR7lunYvu3N3y+wEhVNnvQj/N37kaUdUNQ+vQlIVxFhW7arU2j/eyvakxY3N93RKPEnIGLSxSylHd", - "bvtiPCImVjZdqBln5jkbZ0xl5nks+FtJhN7k7N0ci2tti+e1/UwiRZrA/Rhzm1ePuW1A3P5wSxYf+0N2", - "HCXu9rXN54tjyW0SNBP1QCXKs0oPWXNVOL8yfWKTl5oERO5iYQEmz1SaqT4yEyHKXiiC5lSiNJMzEg2Z", - "4uiDMFkZFx+3PxQjfgTtlOBI00mpiZnS9gcafWyCWo6wnv1o7Eor1nR2AggYBlprGAb6eSqw1k4zOUM4", - "hNgM/WN5STuGsbmAnX+rjuEQM5TyNIu1HgVEZXK7VfqAy5k4jpECVnLfan0CVrJhPtal68sIZP25xgFX", - "YyPIDVRipsH+oZ+fJAkF8Zmlf7168RzBVgUlrKBZcX0AcESZVjTyHMp69P6QPcHhDBkdBHKLDgMaDYOi", - "VNEWwJpJYtSAXg+UmJ+ghpsZpkujn/p93ZXRj47QLx9ML0eal9JkpPgtYcPgYxeVXkypmmXj/N1rP0Kb", - "3GJXFUGAOkb2b7nr8nqGpW3Q7BuYRYhbWRsvEEaFBCpbv2PKsFg01f3imWoOLTLZBGyzYj0PBoOt9UdH", - "dqoezbDSUHPCxyVFaPeL6QBW/1nWAUo1PvWOy2yqiMhoPneghDzGkbvB+EPbWqNtWTOxpEfB92WRbMg3", - "JiaUtaYMQSk4pwylWOCEKMjh/ouf5iGKl+q/3UEv7ETGaVIl3m4JPXXb6fUSYe831tjLq9UBLezfAf3B", - "uEUCPxj30V2Ni2OTPjqv+3uvyBEWyxFi12/onRH1PVDc4K5Eqcsz+g3p977QzxmxKliBtJo024bCDWUv", - "Qv22iSA4kbYX01ibjVcAU++KMIWguqvs23+d8QGB/G9iPn1zhAwKY1vbVtrMkbm7XW+KFpfwkUmsk39n", - "802FM8ymRKKO2T9//+e/XH3O3//5L1uf8/d//gvYfdtWm4bu8sqyb47Q3whJezimc+ImA8GpZE7EAu0N", - "bKkaeOXJXiWHbMheEpUJJvPQLj0vwInpEKwABvOhLCMSSUAh5JWf2Jgj483zWNOOlw0q75Sju0vmj51B", - "aQJ6V3Q0AIfIlFFFcWxNIQcH3H8qADFzDsqD1x2TS67q9fJFkXfKUG/PALihgDGVmT18Z4oVmz5R5+rq", - "yVYfgbpvqALiysBuKLqxlkD/h0xaL5OMRKkKFMCykU2lxKWNbs1T2+Yu/JpNSU2bHZvGkCfaNnaT+aF2", - "t3By+vHmHJ4+r+OpS7Tf7Hb89Pn6Cje3sim/3Do72lvGua0iUaDsW1iTqGMTgOd5fiqlKr4V0d+JAC5V", - "OMmlMOImu9CdWTgnnE1iGirUc7DYSri51VMlkPsiDl5aqBF286pfhihvFduV2L7GTSMP87vL3aM26Cbb", - "SHFho6C1HzvJOtI5pTLk+tsStfRCnNosR4DEgk/LVLTOt3MKv+dbzkrFPK9N7Rjy7rw8duiM1feGOxCK", - "pzWB+A0FYS2jSumK032i5ut8FV1RoRVOoO+LNAd3pwXdtUPIR+b3ySMU1dCmpeAsz7vfRF42M/9XXGg7", - "gmfiV0Q4rjaAmkwexbTMpyickfDWTMjWvlqlEZy78lhfXw8w5QU22P0t+D+2+xaGY4GrVcbiuU3v8vVs", - "RRhhI1Pxyx0/WgLzIBnCEcbOkWoyp2C5YOHWH+oE8k52hnqtqnvESZdZHDtH/JwIVRRZKMvT7Q8QuLJe", - "T3bctlIXuX75rEdYyCFSKY+y8SskLqf6l9WWzYKZqfwgkzb2FaDKEUazMvoZ628CylCervNPu09tws4/", - "7T41KTv/tHdsknZufTViGdyVaL5r7fUeE59WXmkVaSCaTObzddpe3upOFD5bYmITlS8H8IfW10brK6Nr", - "peKXV/v4iqqfLaLwbc4JcmLzYRteufizP5jKd7euJ0uRpbqYFV+8zSHDRVG4wFbVu38BcjSnuLL8belD", - "LRhypXbgSPf8tGtrUphKEnks/h15VB0cd64l2nHv3p16nIzpNOOZLIf5QwkSIot6zRUBfN/012J7btRg", - "v2MqHdzl1nHnCuoPuv9KqnN9QY3wtiWe1yjPrtUadnhKY1XKbC+h2IfJJ2uuCb10eWptNtithqAxl4W5", - "LRlXkoAvB7P54HI1GEplGVJMheyja0k0mkj6wtzwkGoRk6Mh+2/3yS+K4OT1T2Mc3hIWDbPBYPcgf0fY", - "/PVPUsFN0SG7yOvbMyUokQgLgo6fn8LB1BRu4PbRcRwXV6Hq8KAkk7ZYm6s+lMaQ78tIDh/6SvUlCgy2", - "LmGxdOcbEAB5URxOgs8WSy3tqOLUrr0h5Yj1hyHVypAqoWu1IZVnT/6alpQZ5JuZUo7efAi3iQN+GFN3", - "YUzJbDKhISVMFZm/lgKcbOLAe3jFiNnzmFJgQmU/bm1MFSnNV+uplni/RVBKPvjd21AuveD9DJXm5nJE", - "5KyWYjNsNlu+N3oY3K1wvntz5T6T2Fm58qffMDD3hGI+XX9LKO/JXYnxXBMaMlcm9I0R6m9QTqhIcSRJ", - "rFXutzMazuDKkP4N+jc3inCavsnvCG8doTMIRS7fWobBO5IIimOoU8BjU2LjzTxJ3hwtZ2q5ubiAj8xt", - "IZOT5c0RctlZch6TulX5CpCeRYylQs/txaaOXnDBXZnDNxqfpflt2ctBxXXqIfNdFGLkre2QTtCb0p2h", - "Nw2XhhwRPtOr9I04v9ucBMPMRXEkAHEmqQNhUYPtp7Hmvy60M/AmHGt5dcmA8ZVvLi0B84xP8wQcFVLG", - "adqWfC2YQMXzJFlBw6hTKsMhVcQz9RepIiJMZW9L3U3EjTo4NH8ofGvqUFfqSZrCL1573VzD96IqMNX2", - "Xb0Y89c8SQJT3DLBvvovn38FrN7hssGoV6Z0z+vHnrHJDa6qsC9d4artHLbwEOT48VqXL02DP7zm4io0", - "fWMy/AaWXgEFhcJNLBovYG2L0lf36/4KLGQxM9jv7Ly8POLeNfKIrZj1h+eRgj7+4FwSciGMa9qWvbw/", - "gYYli6PE7h2os1fUr+s6q/fm4mKriWlMdf1GlhHfhzn8ab7SWomLJPJnUxc0cknHTi5Oi0rmImN99CKh", - "kAnslpAU8hpQnkkEZwv9cn7lphOAPIEyYUosUk6ZWgtF0fTrAPPxkzI93bGcsqHWf/itHM747p+QMsWO", - "cT6BVT5aLYdUo2vEuQoq1WDHPNO9L+WKhipIciEVSYyfZJLFwERwscWmEMHlKk9dRJWEmgNd8BSWKvwM", - "2ZhMtBqSEqHH1p9D2sjC5PN5E64UzqXmpRF934c7AdJHgwWNVRPWaqWU0tRljvaZrHmy608G6Sn4B6pV", - "piTqxPTWlE5Fc4li/bC10sFgSlB96QQpn85ZeZE138V3Q7M5Mf8RJNx5Tay5IsL3TqydkTKzOPkDC+0X", - "a3KtXBMbVuF1uCtV4+0P2QVRQrfBgqCQxzEUXzFm03YqeLgNFULDlEamVCgABwKv+XUCI55cXkM7k2K0", - "O2T6j+UalXVAXanL8+0Xa1yupjrxv7E9Zia4ii38C/7Dm7b5CUwjD8kGFuXpKgOIp394h4HV4H54C+6n", - "twCOwPPZdKYCh6AUS1tW3u8ZsLUYtz+Yh/N1gRQKh7MbVxrn+9B2bSWNdcO4Cd4LprRziohJzHH3PMnz", - "Yif39PKlRpybAigx5ZAQ/y5giij90aj7y4cnlvG4UXDinfKWS3rz3fDWXe98FgYXH1jGx31hc0NpbiZQ", - "gqDsfRLlao4rbTNXbA9Ki+aqpSsy2S3XOjU5dnMfUlEkKy+r2B+yvI6ky/GrrauuM61QROWt6cFaT33k", - "L/dp7Dxb8xMKZIQ4Dk1libzupSn5IBusr5elWrBfjd+KQTwLnRf8lHl9xvtkcvhpAlavXAASKM6qUytv", - "iNzYNndxKcBuZhtcCXAz+HEhoMWFgBKy2pSbMtU7rbSyZReLuyySvofTHd+Fglwp+XrXCT5hv/5y5OHo", - "tHG3/nGR4M4UguJS9vnp/b89UOa5ioze1lZBz9ZyK7uGVnGwRVEqSM9VeIoMwiw+jK1RLxXXH7JXM+L+", - "QtRFsJIIRVSQUMULRBkU4nIVP/8skeBc2fdcLJpLyhkWeSp4cmxns8Z4aV371ncQs3HGmK6n3idNssTU", - "+qQMnT2GWv/CBFSiCaYxhPM6lJJ3ISGRBJrcqtfU9UZY5sVz10K5IjQ2r5oXmupm89wS6+BM8d6UML0W", - "RS25VPA5jeoF0ivFiX3QgoX4BYy06XuaVllvbcWpZcar0i3Ki+fZklcFfbrVCX5sE/Uc33BTlosciYpz", - "FGMxJVs/tpL7vJWUvUlu36jsKO3uobVzMLX0+3yNO2i58/Fub6DdfD8+kVJO5HuYsmOeG31NV9++LxIc", - "3N3+cNdX3m7usQ/9jDgDt3TdDTrQPfoI5hkPcYwiMicxT6HuvmkbdINMxLaK+NH2dqzbzbhUR4eDw0Hw", - "8fXH/w0AAP//BoNb0R/rAAA=", + "H4sIAAAAAAAC/+x963LbOLLwq6D4na2Vz0qyfInj+NTUKcdOPN6NE39x7P3OjvIpEAlJGJMABwCVKKn8", + "3QfYR5wnOYUGwJtAicrFiXeytTWhRRBoNBqN7kZfPgQhT1LOCFMyOPoQyHBGEgyPx0rhcHbD4ywhL8lv", + "GZFK/5wKnhKhKIFGCc+YGqVYzfRfEZGhoKminAVHwSVWM/R2RgRBc+gFyRnP4giNCYLvSBR0A/IOJ2lM", + "gqNgO2FqO8IKB91ALVL9k1SCsmnwsRsIgiPO4oUZZoKzWAVHExxL0q0Ne6G7Rlgi/UkPvsn7G3MeE8yC", + "j9DjbxkVJAqOfilP43XemI9/JaHSgx/PMY3xOCanZE5DsoyGMBOCMDWKBJ0TsYyKE/M+XqAxz1iETDvU", + "YVkcIzpBjDOyVUEGm9OIakzoJnro4EiJjHgwEwFMIxp5VuDkHJnX6PwUdWbkXXWQ3Yfjw6C5S4YTstzp", + "z1mCWU8jV4Pl+oe25b6f7ft6pjxJstFU8Cxd7vn8xcXFNYKXiGXJmIhyj4e7eX+UKTIlQneYhnSEo0gQ", + "Kf3zdy/LsA0Gg8ER3j0aDPoDH5RzwiIuGlFqXvtRujOIyIouW6HU9r+E0uc356fnx+iEi5QLDN8ujVQj", + "7DJ6yvMqk011VXz0/zijcbRM9WP9MxEjyqTCrIEGz+1LjS4+QWpGkP0O3VygzoQLFJFxNp1SNt1qQ++a", + "YcVEkWiE1fJwACqybShnSNGESIWTNOgGEy4S/VEQYUV6+k2rAQXBa4bTLVoNtrzVMrOSo0Q29e6aIMpQ", + "QuOYShJyFsnyGJSpg/3myZQ2DBGCezjUE/0zSoiUeEpQR7NNzbsZkgqrTCIq0QTTmESt1shHCGYyv/Ix", + "ohFhik5odX8bcurhcbizu+flHQmeklFEp/YkqnZ/Cr9rEtP9KASt/RPRG23Rbh4wpCCT5fGeAuuGQQSZ", + "EEE0jX/mcKngc8L0btHj/QeMG/yf7eKI3rbn8zYg87Jo/rEb/JaRjIxSLqmBcIlz2TeajADVCL7wwwyv", + "Vq11iaKkwmL1/oAWX2AnGvha4ebKNK3zQ2B3tpvKzm5ke0/mhHkEn5AzZV9UZ/yMT1FMGUG2hcWv5nN6", + "gJ9iDmzuS8ytGxQoXd7QGu5PYEjmh4be9LtuQFiWaGTGfFrG5oxgocakgsyGY8l2VEDXiP7LypaonT9Y", + "ktFqrnBJGSMR0i3tZjUtUSZB+lyaPuyMW6pGcyKkdx8BWH+jCtkWjV3FPLyd0JiMZljODMQ4imAP4viy", + "MhOPBFYRaXGqGZvrECQDiRRHVz8f7z44QHYADw4lz0RoIFieSelr3b1pixQWYxzHXtpoJrfNz91lCvFT", + "wFW+MZrOk5wCHWEa7hXY1dTdd4M0kzPzBPxYQwXnmWYDmrxi/fzaM+kTYBJG8m/Ug/xy3YvULDaaxlzj", + "dIEyRn/LKkJzH51r+V8hzfxpRKIuwvBCs2GcKd6bEkaE5lNoIngCElRJsEUd0p/2u2ioZb2elmx7eLc3", + "GPQGw6Aqmsb7vWmaaVRgpYjQAP7/X3Dv/XHvH4Peo9fF46jfe/2X//ARQFtp20l6dp4dt/e7yAFbFsHr", + "gK4Wz1dIuD4uYpbvXO/9TVfv5Hz5gDfwRzy8JaJP+XZMxwKLxTabUvbuKMaKSFWdzeq2a+cHsK2YGJvq", + "qW84tZrCAeTWiflbIkLNKWOiCUR2NbOkSnYR1jorMBmkT7P/QiFmmmbNwc4FIixCb6maIQztqhhIFj2c", + "0h41oAbdIMHvnhE2VbPg6GBviR41MXbsQ+/1f7qftv7bS5Iii4mHGF/yTFE2RfDanL4zKlEBA1UkWXvc", + "OuxmMYhYCWXn5rOdHBIsBF74V80Bt2r1jHLUuHxh4pGkX8yJEDRyJ9rJxSnqxPSWWLJEImNomA0GeyE0", + "gEdifwl5kmAWmd+2+uhFQpU+SbLigDTWlX55CX8JSDjjcMbHMdcTytHXIEA4vDhF07NEp84yIZHVduFM", + "w2B3giU7u7ze1lwlxVKqmeDZdFaFyrK0zeCh8nZE+Wic+mCi8hadb79AmuGimGrs5Ax2ZzC4eLwth4H+", + "44H7Y6uPTg3KAHy9flxYvi9nWBCQPiLEGTq5vEY4jnlo9bmJFhIndJoJEvVrZgTo3UfwhCmxSDn1CZ81", + "yiiaLhNIr1e83YAOtseUbUu9DL1wM7wTNv8MEegJm1PBWaLF0DkWVPOtilHnQ/D8xemT0ZPnN8GR3kRR", + "FloLyeWLl6+Co2BvMBgEPilDU9AaPnB2eX0CK6Xbz7hK42w6kvS9h7Ue5/NDCUm4MKK//QZ1ZlXOayQj", + "BIszDPbOHhvi2jkDunKLElEJrV0vpuMqxeyePfZRy2yREjGn0qfz/5y/cytf4pOGMVVpWxIxJyInWqDi", + "fknuCmOeRb3SkN3gN5JokWP+XhNLAa2npV/5bnUsrzlvcZxSRhoP3G6QEIXB8Pzp9HktiehFZEK1unFL", + "Fr05jjOCXM8WsyRHbJV000ykXJr+8dSIqYrgJDgKxji8JSzyUu53cri/5eI25jjq7Xzhs50RpftenuJz", + "86JKiT4c1xVGFr2lkZqNIv6WaZA9J4B9g/LG+THwTs8Ex7//8183F4Uku3M2Tu2ZsLP74DPPhNopoLv2", + "aqn5RLLUP43r1D+Jm4vf//kvN5NvOwnCNH1GlRsdY/ipTuXvM6JmRJRkA7fA+iejZsDnyNFLafiKJal8", + "/bO0mficiBgvSmzdwhTsDIC31qASVMH+st9pJn2L9MdrmLzuzYkQZ3XVZ3fgZ+MeoDwwPdb72546bSDJ", + "AdnZvbCPu8sgNUB0S9PRVEutIzzNLWGrLuaubmmK4IsefGGWMY7N5o0y3TMac676Q/b3GWEI1g4WmLwj", + "IfApreqj48tzid7SOAa9GRjB8sE1ZK9KrMA0l0r/V2Ssi8aZQoIkXBFkRWIYJANYoPGYoIxhd/PXH7Iy", + "VuwE63Rl0XJLBCPxaEZwRIRsiRnzEbIfNSIHpjrBUhFhOHSWVvF1+reLK9Q5XTCc0BD9zfR6waMsJugq", + "S/Ue3qpirztkqSBzwkBj0uIPtePyCeKZ6vFJTwlCHIgJdJZbHuy11Pzs8tpebMqt/pC9JBqxhEUkApjd", + "KSGRmmGFIs7+rHcsHJelbsvj15Du38vdYB6mWRXLu3UMP4frRD2fORUqw7FmWRX50Xu7aO6tPXqCuRYv", + "6yuWFeUEh1X1Wqityml6hkvsZSnar2UaQalZy1xzh++7rMktV2EmFU9KVzaoUzNK0ar5qso85jzuafkH", + "RIPl890rvxhwl68/k4XpyixKE5ccTcceS6dmhpShKZ3i8UJVNYedwfLS+xHt+vehusk1wJAHiUaKr74c", + "pRPk2ra5CwFHgpHio/mEenrOD83CCkclCmt+CJZodRe9NKR2+3bR2xnVx6xEDgmwg28uypp4f8h6wHKO", + "0Gk+QN5t3qXmrGBxhS46XJSAoGA8R+PFFsLo5qKPXuXQ/lkihhWdE+crMcMSjQlhKAPxjEQwPrDTMgCZ", + "1DyMqvrnllcZt4otMDhw+66PtCKUYMv3NXknWNEQDLZjWpsPXJSZhdIjaQbAyqdOq1Ni1ZXySzKlUona", + "hTLqvHx6sre396guL+w+6A12ejsPXu0Mjgb6//9of/f85T1HfH0dV/mFNYGXOcrJ9fnprhVOquOo9/v4", + "0eG7d1g9OqBv5aP3yVhMf93Dd+Jb4mdPp4XtHnUyrfY51qepymexLxnGGyzyn2xo38itxV3trTp+zOxe", + "6ZZfwxHGdx1rLwM3d1WpM8G1F7qlyS3NR/+q5YOC8kuGDXtvElLvDdEplbePBcG3Wqv0nK/6eJYjc+74", + "LY6Z1qPGC0TeafGMREhwribS2DmqYsrO/sP9w72D/cPBwOP/sUzEPKSjUJ8qrQB4cXKOYrwgAsE3qAOK", + "XoTGMR9XiffB3sHhw8Gjnd22cBg1qR0ecinKfYU6FiN/cb6E7k0FqN3dhwd7e3uDg4Pd/VZQWQGvFVBO", + "GKyIDg/3Hu7vHO7ut8KCT+184vxx6v4Fkc+4mKYxNUp2T6YkpBMaIvDoQfoD1EngWCK5xlfdk2McjYQV", + "A73ngcI0littmmYw29K4byVZrGgaE/MOFqSVpAszP4WefPZiyhgRo9xdaYOerBfTWsuYm0veBFW80Sqo", + "u6ASJItCIKIkjo7MDl3L52A1C8BeN9GBnUNLanjG3xLRi8mcxGUiMMeRBjbhgqCcTsyiVWZF2RzHNBpR", + "lmYNltEGVD7NBMiXplOExzxTRlWHBSsPAnevoCNMNLtud/X/lIvbtbdj/gP6uT6ZnS1wwsWtVlIcQ7kj", + "Q2hvYuxQX84a2vZauLiqWEKMVtA3tJimgk9o7EEyaPv2rZWFnC3x2f7gqrfzf80NBosXhoFSZiwECY9I", + "v+YpDO3b0cXZ5fVlE0y5mzYqQ7c0p9ym4yGd3EzgMGKtFSFmaEyQlS8MEYC9qRikOBkf+U6aicAJGWeT", + "CRGjxKOiPtXvkWlgjHeUoYvH1dNGn2pt5dTLyuKAoDrBofWybYd9jwpcm0a3hM0GanxJjGNRkx+PXiph", + "21hXnj56njvGo7PLa4kKO5xHN64ub+N98+VsIbVWZ3o0bnmUlVVaIM7W59dl8aFV/j2nWOLl3G4joM58", + "mmawDa9e9s5f3GwnEZl3KzCB7WzGY6Lh3ioJpXPnzVNcjlfu2OZNuoUhDNl2A5Vwle/g1kgq7VcPdhRX", + "OB7JmCsPNK/0SwQvUefmqfHi0BB0UVpZSv17CQsV+j7w7hjNkZqGvYIB60aKygZfay9KzHlfnl5lUN9W", + "+Zng2ITRVOm5cAx1C89vqwvNb9fuXtuJb9xzd83awvPk5OLUqLQhZwpTRkR+w1l1EwAXp6Ab9LQUFWGS", + "gDF58l+rXQYajF45uawym5ws+eB/FZNJg5+pZnLxnEQowYxOiFTWz7Qyspzh3QcHR8bDPSKT/QcH/X5/", + "Ux+PJ4VTR6ul2DYX4iV3j76cfd46fAVXjjZz+RBcHr/6OTgKtjMptmMe4nhbjik7Kv2d/1m8gAfz55gy", + "70V6q6AIOlkKhqgsb6rPLPP7kZ4JI2FOkBzE67VG3QaBVpNmTN+TCHl9EhWeaunUUNznOR9+RhhBEcum", + "SuED5fuVFqEE9P1qO4UTjKCNHTNjisZFlMWyheKT4mTkSrfjJZfjlLDc0TiOzVPI2VzvCp/XcYWBu3dL", + "i6EVCsqmo4h6qPPv5iWKqCChAk+j9Xso2MZpup4U/cJfztPaRlBY/0nP6fLNOfmnWKqro7+Y/vW3/ycv", + "H/6689uzm5v/mZ/99fQ5/Z+b+PJF+7suj6/OatfZb+r/uvIyEsyzFb/XtuRxgVXoEXxmXKoGrNk3SHGU", + "6I/76AQUtKMh66FnVBGB4yM0DHBK+xaZ/ZAnwwB1yDscKvMV4gzpruyd+5b++NJo6PrjD04H/FjvI7KX", + "68IiOfeDkdk44gmmbGvIhsz2hdxEJNx26acIhThVmSB6RbSsGS/QWOCwuFQvBu+iDzhNP24NGWii5J0S", + "egYpFir3s3cjwEJbqMxtmm1OIgQOadJqskOWnx+gmutOFBZTovq5qQQsXLUbrQakeNUMLqpOIYeDrmcd", + "kW6nFzKmUhGGcqsElUC8qOO8ew4Hle1/ODhcf3Gb09AK8gPqXo5sd0TZYn8YAoahDTMezZRK1/t9AL8x", + "ewT9/OrVpUaD/vcKuY4KXORLbJQxnKYxJdJcR6oYZBLrULUV+K4czeq2nNAr01h/FrfwX3kCA6NXz66Q", + "IiKhzPDvTqjROaGhnh9cjFEpM02KFKPjk4snW/0WofmA2xz+Fev4Kp9h7f7FGbeWNUz4ojAOavx20flp", + "V4tTdocWghZcOD/lAsWGwRT7+ghdS1J1/4ClMndjZiXjRWEhM1x9GGy5HtM6pzhCL3P5Dueg5PE/BTG4", + "Lot9Cd1ajyBzG77Ue7cKK9zzW/3Fsja4+8YK2dsCOIqbWcHq7e/BOOx5zho9Ylvt7bLRUg/mJ41i7b+6", + "BLK3qS65aSBC1Zuv5L2ZxyK0DyL4Gs74y3rVO6pGjVdZSL+2F1dOe7i5QDMs2Z8VvKzpEDt7D1uFuOtR", + "214Cla9/+MSAlO8q5xqYX14YJ8lbGsfmTlDSKcMxeoQ6V+dnfzt/9mwL9dCLFxf1pVj1hW99WsQkONI+", + "u7wGR38sR5LhVM64anYHwsi10TOVSi57TrZyYFkdA/FzJU7B64q69QWDF0TGGPji1KfxdcISvqWLy79R", + "SETwOQENK0MQPjeOwArVXymMoJGJ+1zwq/zc/PxlAwK+CjgV134fHyrLHs7r8ZO9+bsB9Xh8HUvNakmE", + "zi+LWOnCSOW6r83p0W5/5+CwvzMY9HcGbUx2CQ5XjH1xfNJ+8MGuMWIc4fFRGB2RyWeYDC1hGyERx2/x", + "QqKhE+OHgdEbSgpDiVlYUb/Vdexy0MSnxUjUBZd1URCbRD20C2dYkcTkqpq+pLUs+OAfn5XphLSVAK6g", + "sftqtIkxm6CQZ3Gk5a2x3nlGfSOR1TIlUUVmGNis1+yW8besOnVj09T797eMiAW6ubioWMAFmdgkGS0m", + "ztO0cR14utEy7K4RyddCU4osuItogjonLJ1AXzx2oGyuc05MhupamO0KydN7tU2ZQbde+xVzqhlcIjIf", + "ZZlPvNKvnDvy9fX5aWXBMT7YORwcPuodjncOevvRYKeHd/YOersP8GCyFz7ca0gn1d615dO9Vao7tNn9", + "HxAPxksTsREd6T2Uu5uMM4VyRyS9OU+0nIpKIrBxdgd7wksjDese4HQN9Zt4kUvJKz++xHqjum9T+Gv1", + "F1ezTGkxCL6Rs0wh/ReArKdgtYzVXZg9f4Sec/jGQtrVB2VNXTHNMYvGi+XmddWmY712BJGKCxLBYJaB", + "HaGnOdPK2Z5lcx1J7KPhpdYtEFwet4wRxGoWdrWCbmCxHnQDg8KgGzjM6EczQ3gC4INuYAHxehSX6cZn", + "3Cc4Bh5WOM5kisb0vdlyGnQqFQ2NdodhNZu2nQ3bJNHIHKFN12/GG8Mes/lHblffXKAOBNn8BVnlT/+1", + "lV/VlbfQ/u6j/UcHD3cfHbRy0S0AXM+NT8BXaBm4taw5TLORS6vXMPWTy2s4fPTBJrPEaPN27oXGqBlH", + "qKU9ylCRp68Y/FH/UdkzOeLZOC5Zh2xoAri/tkmq2HA39RuN53QyYb+9D293fxU02Xl3IHfHXuUoH8gv", + "SZ6XLZpLahcZ90xSAL8OCQQlZKN/9UsiYQboiigE9NPTDEufqLmLjyU554VtMe4lrP29vb3Dhw92W9GV", + "ha60cUag/y1DeWEhKG0xaIk6L6+u0HaJ4Eyfzu8xFUTqyZmQIe8+QzY5y6DiEql1jz0flTQILAXV2L7n", + "SSPKb6zEYidlkQ6eSrk0s7TLvdje2xs83H9w+KDdNrYaz0i8W81hbDt7wy9ISOi8svIdsIK/Or5Euncx", + "wWFVwt/Z3dt/cPDwcCOo1EZQKYGZTKhSGwF2+PDgwf7e7k67QAGfpduGwFQ2bJV3eTadhyg8q+FBxTLr", + "7TadFj4pcdk9cqVHZuHiWffn28SBtwh7pBJ6pSXfUdTRQlRZIC2F7m21sTP4WaQepylZrxYX2/rWrnal", + "vcRqds4mfPkqYxOFzzooORN3qgUfCWkMI8IoiRzvyjU/K0uBy1MsCYoyYjFnZCOBLcKxuc5JsZqBsAof", + "UjatOnsvDdhGDTMwrA5yhXFtwzYWI+l3qnklMsCVsSVLhAv3mlaGcSpHfq1iuWNBplmMBar7j68AWS6S", + "mLLbNr3LRTLmMQ2R/qCuzk94HPO3I/1K/gRz2Wo1O/3BqLhJrqnnBjjrR2AWpDZuMYWf9Cy3ap5JcPJv", + "m++3IRt7GwOc93rpqVbejIv1NaPvSoRejRjb3x00OaI1dFpxQVt2z9+Ut1uS9e145zl/nGdW8Vxjmoui", + "mgZblYMr8/XNFm4iV7ndLUsCqONsei4ir4rXUmRcq4O43WVo3XrtoNmWJKyOvn/44OFBy9DEzxK1V+Sr", + "/gzBep6sEKgbVuqijdR2+ODw0aO9/QePdjeSj9xFR8P6NF12lNenlkCpJrM9GMD/NgLKXHX4QWq47qgC", + "VEmG9MkAfVyxdYuglwate1WtiGIlnZpfFcDbibgrpKXjishVykzYIZMJAcPRyOCtVwBTc8JqBUOIUxxS", + "tfBogPgt+KWgvEkteKNF7zVgPSi1fSM8UUTAbYTMxsW9f8cNjv7TaHY1WjhsHeEss3GTFvmiPqrRIY0j", + "V1SzULQwEBiK8F2+v82Rid5iWbHq6+dQkahbyjxZv/4xLdrnBne0nqcHL67TfQFI/lTg5eWvLWdJ66gI", + "yXWMrzpCm7eglgjAS6yNgd1zInuimsL1zhs1/mAPwE/7ajQu5x5YmdyhkqigOHU3H7ddzszl78wJtvl4", + "pRv8TT6sh2EDPVoYLMqLvrsVkvBRk7lfacrxk7giSrUobWrKUtiIMlRqjDokSdXCRU04zXRrs/ue47xD", + "LzF+YT+3waMv4Wl/vdK1/t8ka1T5is0NsvZybWlNG/1Z/eLqad19xeiENmtG1d2ilgtAqhXVV1ZV+jIl", + "t0Dhs77k06we/LZBda8mFb/YOa6siivvtU5zXWlPK82sBEnz2pj71c8shUalq4H2iSiz6td652xzR6UV", + "4F49rYqJLRYU9DmLIINYjYJcRV+2A6x2+7jA7/IRQFvGEtUSUZp5lFJrnz2GfAEvXXoNOnFdABj1lKKP", + "P69GnKOq5cVYVTTO3eB7N57lPys4WtPeqhFnMUZ3dV06zbpImAmqFlf6QLDOaQQLIo4zQ4ZwUsAk4Odi", + "cAhQ+PgR1NSJR1o9I4wIGqLjy3OgkgQzSEyMbi5QTCckXIQxsf7lS3e7kB7hxcl5zwTG5JkfoeSLAoS4", + "lGvHl+eQ7ckWWwkG/d0+pOnmKWE4pcFRsNffgXxWGg0wxW2IO4RHa4jS+xBOsvPInriPTRONWplyJg1y", + "dgeDWvEeXGTU2f5VGguLOV5bC4WmOtqyv8WSQ6STBCz4H7vB/mBnI3jWJsHxDXvNcKZmXND3BMB8sCES", + "PmnQc2a0apcznNiGBc0GR79UqfWX1x9fdwOZJQnWIqJBV4GrlMsmEYZIhBEjb21A6q983EdXRicB7/Ki", + "7qQxGZBIsySMFBb96XuERTijczJklhObhEZYQPRNgjQHNrEPVTIzQ5vVN1uYSPWYR4sadvPutnV3PedV", + "WyB446pGeXbOtKG8kY87miRgMuTe7GeEYaaKnFIm+9ctgUvMCX3n7bDVbbxmHuWCf867c3fLbwcEV2W/", + "Cf00f+eKa1UPDC1DUxbGWVScqtWiRt5ge1OcxyZJuyUeIeQMWliklL263fHFeESMr2y6UDPOzHM2zpjK", + "zPNY8LeSCH3I2dgci2uti+dFEU0GSppAfIyJ5tVjbhsQtz/cksXH/pAdR4mLvraJkHEsuc0eZ7weqER5", + "Ou4hay6n5xemT2zWV5O5qZxPyYDJM5Vmqo/MRIiyAUXQnEqUZnJGoiFTHH0QJp3l4uP2h2LEjyCdEhxp", + "Oik1MVPa/kCjj01QyxHWsx+NXU3KmsxOAAHDQEsNw0A/TwXW0mkmZwiH4JuhfywvacdsbC7g5N+qYzjE", + "DKU8zWItRwFRmaR4lT4gOBPHMVKwldy3Wp6AlWyYjzXp+jICWXuuMcDVthHkBiptpsH+oX8/SRIK4lNL", + "/3r14jmCowpqf0GzInwAcESZFjTy5NN69P6QPcHhDBkZBJKyDgMaDYOixtMWwJpJYsSAXg+EmJ+g+J0Z", + "pkujn/p93ZWRj47QLx9ML0d6L6XJSPFbwobBxy4qvZhSNcvG+bvXfoQ2mcWuKowAdQzv33Lh8nqGpWPQ", + "nBuYRYhbXhsvEEYFByprv2PKsFg0FUzjmWp2LTLZBGyzYj0PBoOt9VdHdqoeybDSUO+Ej0uC0O4XkwGs", + "/LMsA5SKo+oTl9lUEZGRfO5ACHmMIxfB+EPaWiNtWTWxJEfB92WWbMg3JsaVtSYMQQ09JwylWOCEKEh+", + "/4uf5sGLl+q/3UUvnETGaFIl3m4JPXXd6fUSYe83FifMy/wBLezfAf3BuEXmQxj30V2Ni2OTdzsvmHyv", + "yBEWyxFi16/onRH1PVDc4K5YqUvQ+g3p977QzxmxIliBtBo324aKF2UrQj3aRBCcSNuLaazVxiuAqXdF", + "mEJQFlf27b9O+QBH/jcxn745QgaFsS0KLG3myNzcrg9Fi0v4yCTWyb+z+abCGWZTIlHHnJ+///NfrrDp", + "7//8ly1s+vs//wXbfduW6Ybu8pK8b47Q3whJezimc+ImA86pZE7EAu0NbI0feOXJXiWHbMheEpUJJnPX", + "Lj0vwInpELQABvOhLCMSSUAhJOSfWJ8jY83zaNNuLxtU3umO7i6pP3YGpQnoU9HRAFwiU0YVxbFVhRwc", + "EP9UAGLmHJQHrxsml0zV6/mLIu+Uod6eAXBDBmNKWnv2nanybPpEnaurJ1t9BOK+oQrwKwO9oejGagL9", + "HzxpPU8yHKXKUADLhjeVEpc2mjVPbZu7sGs2JTVtNmwaRZ5o3dhN5ofY3cLI6cebM3j6rI6nrkJBs9nx", + "0+frq3jdSqf8cuvsaG8Z57b8RoGyb6FNoo7NnJ7n+anU+PhWRH8nDLhUGibnwoib7EJ3puGccDaJaahQ", + "z8FiSwjnWk+VQO4LO3hpoUbYzaseDFE+KrYrvn2Nh0bu5neXp0dt0E2OkSJgo6C1HyfJOtI5pTLk+tsS", + "tfRCnNosR4DEYp+WqWidbecUfs+PnJWCeV7U223Iu7Py2KEzVj8b7oApntYY4jdkhLWMKqUQp/tEzdf5", + "KrpqTCuMQN8XaQ7uTgq6a4OQj8zvk0UoqqFNc8FZnne/ibxsZv6vuNB2BM/Er4hwu9oAajJ5FNMyn6Jw", + "RsJbMyFbNGyVRHDu6op9fTnAlBfY4PS34P847lsojgWuVimL5za9y9fTFWGEjVTFL3f9aAnMg2RwRxg7", + "Q6rJnILlgoVbf6gbyDs5GepFvu7RTrrM4tgZ4udEqKLIQpmfbn8Ax5X1crLbbStlkeuXz3qEhRw8lXIv", + "G79A4nKqf1lp2SyYmcoPMmmjXwGqHGE0C6Ofsf7GoQzl6Tr/tPvUJuz80+5Tk7LzT3vHJmnn1lcjlsFd", + "sea7ll7vMfFp4ZVWkQasyWQ+Xyft5a3uROCzJSY2EflyAH9IfW2kvjK6Vgp+ebWPryj62SIK3+aeICc2", + "H7bhlfM/+4OJfHdrerIUWaqLWbHF2xwyXBSFC2xVvfvnIEdziivz35Y21GJDrpQOHOmen3ZtTQpTSSL3", + "xb8ji6qD486lRDvu3ZtTj5MxnWY8k2U3fyhBQmRR6LrCgO+b/Focz40S7HdMpYO7PDruXED9QfdfSXSu", + "L6hh3rbE8xrh2bVasx2e0liVMttLKPZh8smaMKGXLk+tzQa71eA05rIwtyXjShLwZWc2H1yuBkOpLEOK", + "qZB9dC2JRhNJX5gID6kWMTkasv92n/yiCE5e/zTG4S1h0TAbDHYP8neEzV//JBVEig7ZhaMbwpSgRCIs", + "CDp+fgoXU1OIwO2j4zguQqHq8KAkk7ZYm6s+lMaQ78twDh/6SvUlCgy2LmGxFPMNCIC8KA4nwWezpZZ6", + "VHFr116RcsT6Q5FqpUiV0LVakcqzJ39NTcoM8s1UKUdvPoTbxAE/lKm7UKZkNpnQkBKmisxfSw5ONnHg", + "PQwxYvY+puSYUDmPWytTRUrz1XKqJd5v4ZSSD373OpRLL3g/XaW5CY6InNZSHIbNasv3Rg+Du2XOd6+u", + "3GcSOytX/vQrBiZOaGITj/kFhKdc3LalPBuDTL8mAX556aQ8w+9QNtHgQb6Hby+iwOFt/MI10VQllzvY", + "kHX6+qbuiA4TRgfmwuWFBiEmtO2+NcPQo9+BSPmcK0STNCYJgSxTPUMfUNMnS1Mu8koCVJaKiWzG0PRG", + "KLt7mgQitqCWXgBpas+YFfGyuphP1wdE5v276D9PROSQuYrIb4z8+gblnBEpjiSJSajQ2xkNZxAdqX+D", + "/k3wJE7TN3k6hK0jdAbbq5ygAQbvSCIojqEkC49NNaE38yR5c7SclOrm4gI+MoGRJv3UmyPkElHlXF3q", + "VuVoRz2LGEuFntsYzo4mFsFdRdc3+ugozW/LxkEWmSOGzBcTychb2yGdoDel8Mg3DfGRjgs+06v0jYSc", + "bnO+HzMXxZEAxBnyIyxqMHNprPkjI3cG3tyKLaM0DRhfOUhzCZhnfJrnGqqQMk7TtuRrwQQqnifJChpG", + "nVLFIakinqm/SBURIeBjS91NxI06ODR/KHxrSu5XSueaGlde06TJOOJFleabpdJY5q95kgSmjm+CfaWu", + "Pj/atd7hsm1Mr0wppPWHeLxJsGqV2ZeiVWsnh62x1iwnvzQN/vBKmitG943J8BsYtQooKMtlEb22RZW/", + "+xWqBwu5JG6Z+n6+PeLeNe4RWxzwD79HCvr4g++SkAthbuFshd/741NdMq6UtnsHSooWpTq7zsB3c3Gx", + "1bRphFq5ZcT3Yfn7NMNLrZpPEvkLRwgaufyKJxenNhsjlUhkrI9eJBSSHt4SkkIKF8ozieAatV9OJd90", + "2ZnniidMiUXKKVNroSiafh1gPn5SUrs75lM2quQPf5QbC8O9Y1KmrjvOJ7DqOkrzIdVoGnGmgkrh6zHP", + "dO9LafGh4JtcSEUSYyeZZDFsIojhs9mScLmgXRdRJaG8ShcuRUrFzIZsTCZaDEmJ0GPrzyFDbqHy+awJ", + "VwrnXPPSsL7vw5wAmfJBg8aqCWu1qnFp6pLk+1TWPK//J4P0FOwD1YJ6EnViemuqRKO5RLF+2FppYDDV", + "9r50LqhP31l5PUlfjg9Dszkx/xE43HmNrbl66feOrZ2R8mZx/AcW2s/W5Fq+JjYsOO5wVyo83h+yC6KE", + "boMFQSGPY6gzZdSm7VTwcBuKIYcpjUxVZAAOGF7z6wRGPLm8hnYmm3J3yPQfy+V464C6qr7n2y/WmFxN", + "IfZ/Y33MTHDVtvAv+A9r2uaXzY17SDZsUZ6uUoB4+oc3GFgJ7oe14H5aC8DbJ59NZypwCEKxnGUq4m+Z", + "3zJgy85ufzAP5+t8xhQOZzeuCtj3Ie3aokHrhnETvBeb0s4pIiYH0d3vSZ7XdbqnceYacW4KIMSUvd/8", + "p4CpF/dHo+4v7+tUxuNGvk53urdcfq/vZm/d9clnYXCu0GV83JdtbijNzQSqrZStT6JcuHalbubqikIV", + "5Vy0dPV0u+WyziadeG5DKuoB5hVk+0OWl8x16cy1dtV1qhWKqLw1PVjtqY/8lY2NnmfLG0MtoBDHoSmi", + "k5f4NdVtZIP29bJU9vqr7bdiEM9C57WNZV6K9j6pHH6agNUr17oFirPi1MpguBvb5i7in+xhtkH0k5vB", + "j9inFrFPJWS1qaxnChVbbmUrzBZhe5K+h9sdX+xULpR8vcipTzivvxx5ODptPK1/xEzdmUBQ5J84P73/", + "gVLlPVfh0dtaK+jZspVl09CqHWxRlArSc8XsIoMwiw+ja9SrYvaH7NWMuL8QdR6sJEIRFSRU8QJRBjUH", + "XXHjP0skOFf2PReL5uqZZos8FTw5trNZo7y0LvPtu4jZODlW11PamCZZYsoaU4bOHqMOeaeEcahEE0xj", + "cOd1KCXvQkIiCTS5VS8f7vWwzOuEr4VyhWtsXiA0NIUc57km1sGZ4r0pYXotirKZqeBzGpmY9IY67D5o", + "QUP8Akra9D1Nq1tvbXG95Y1XpVuU1wm11f0K+nSrE/w4JurlDCApABc5EhXnKMZiSrZ+HCX3+SgpW5Pc", + "uVE5UdqF3LYzMLW0+3yNcNvc+Hi3wbY3349NpJT+/R5mJ5rnSl9TlO/3RYKDuzsf7jq69+Ye29DPiFNw", + "S5G90IHu0Ucwz3iIYxSROYl5mkAZNGgbdINMxMFRMFMqPdrejnW7GZfq6HBwOAg+vv74vwEAAP//vy38", + "JkPxAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/openapi.yaml b/openapi.yaml index 8e60ee13..cd5b41e8 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -216,6 +216,17 @@ components: description: Override image CMD (like docker run ). Omit to use image default. example: ["echo", "hello"] # Future: port_mappings, timeout_seconds + + ForkInstanceRequest: + type: object + required: [name] + properties: + name: + type: string + description: Name for the forked instance (lowercase letters, digits, and dashes only; cannot start or end with a dash) + pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ + maxLength: 63 + example: my-workload-1-fork Instance: type: object @@ -1514,6 +1525,63 @@ paths: application/json: schema: $ref: "#/components/schemas/Error" + + /instances/{id}/fork: + post: + summary: Fork an instance from stopped or standby state + operationId: forkInstance + security: + - bearerAuth: [] + parameters: + - name: id + in: path + required: true + schema: + type: string + description: Source instance ID or name + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ForkInstanceRequest" + responses: + 201: + description: Forked instance created + content: + application/json: + schema: + $ref: "#/components/schemas/Instance" + 400: + description: Bad request - invalid fork request + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 404: + description: Source instance not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 409: + description: Conflict - invalid state or target name conflict + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 501: + description: Not implemented - fork not supported by this hypervisor + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 500: + description: Internal server error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" /instances/{id}/stop: post: From e80afcfbf74dcfcd78f29438e645c615a123d3c6 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 27 Feb 2026 15:27:02 -0500 Subject: [PATCH 04/20] refactor: move CH fork snapshot rewrite into CH package --- lib/forkvm/README.md | 13 ++--- lib/forkvm/copy.go | 2 +- lib/hypervisor/cloudhypervisor/fork.go | 20 +------ .../cloudhypervisor/fork_snapshot.go} | 52 ++++++------------- .../cloudhypervisor/fork_snapshot_test.go} | 12 ++--- 5 files changed, 26 insertions(+), 73 deletions(-) rename lib/{forkvm/snapshot.go => hypervisor/cloudhypervisor/fork_snapshot.go} (59%) rename lib/{forkvm/snapshot_test.go => hypervisor/cloudhypervisor/fork_snapshot_test.go} (87%) diff --git a/lib/forkvm/README.md b/lib/forkvm/README.md index b890e17c..aa29c407 100644 --- a/lib/forkvm/README.md +++ b/lib/forkvm/README.md @@ -4,24 +4,17 @@ This package contains low-level helpers used by instance forking. ## Why this package exists -Forking is mostly filesystem and snapshot identity rewriting work. Keeping that logic outside `lib/instances` keeps lifecycle orchestration focused on state transitions and locking. +Forking includes filesystem cloning work that is independent of any specific hypervisor. Keeping that logic outside `lib/instances` keeps lifecycle orchestration focused on state transitions and locking. ## What it provides - `CopyGuestDirectory(srcDir, dstDir)` - Recursively copies a guest directory. - - Skips runtime sockets (for example `ch.sock`) because they are process-local artifacts. -- `RewriteSnapshotConfig(configPath, opts)` - - Rewrites `snapshots/snapshot-latest/config.json` for a forked cloud-hypervisor instance. - - Supports: - - source data-dir -> target data-dir path remap - - vsock CID/socket rewrite - - serial log path rewrite - - network fields (`tap`, `ip`, `mac`, `mask`) rewrite + - Skips runtime sockets because they are process-local artifacts. ## How it is used -`lib/instances/fork.go` calls this package to clone guest data and prepare copied snapshot state. `lib/instances/restore.go` uses it when a standby fork restores with a fresh network identity. +`lib/instances/fork.go` calls this package to clone guest data before metadata and hypervisor-specific fork preparation. ## Safety notes diff --git a/lib/forkvm/copy.go b/lib/forkvm/copy.go index b69767ae..548ce65d 100644 --- a/lib/forkvm/copy.go +++ b/lib/forkvm/copy.go @@ -67,7 +67,7 @@ func CopyGuestDirectory(srcDir, dstDir string) error { return nil case mode&os.ModeSocket != 0: - // Runtime socket (e.g., ch.sock); the forked instance will create its own. + // Runtime socket; the forked instance will create its own. return nil default: diff --git a/lib/hypervisor/cloudhypervisor/fork.go b/lib/hypervisor/cloudhypervisor/fork.go index c4aca372..3f19919b 100644 --- a/lib/hypervisor/cloudhypervisor/fork.go +++ b/lib/hypervisor/cloudhypervisor/fork.go @@ -3,7 +3,6 @@ package cloudhypervisor import ( "context" - "github.com/kernel/hypeman/lib/forkvm" "github.com/kernel/hypeman/lib/hypervisor" ) @@ -15,22 +14,5 @@ func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareReq return nil } - var netCfg *forkvm.SnapshotNetworkConfig - if req.Network != nil { - netCfg = &forkvm.SnapshotNetworkConfig{ - TAPDevice: req.Network.TAPDevice, - IP: req.Network.IP, - MAC: req.Network.MAC, - Netmask: req.Network.Netmask, - } - } - - return forkvm.RewriteSnapshotConfig(req.SnapshotConfigPath, forkvm.SnapshotRewriteOptions{ - SourceDataDir: req.SourceDataDir, - TargetDataDir: req.TargetDataDir, - VsockCID: &req.VsockCID, - VsockSocket: req.VsockSocket, - SerialLogPath: req.SerialLogPath, - Network: netCfg, - }) + return rewriteSnapshotConfigForFork(req.SnapshotConfigPath, req) } diff --git a/lib/forkvm/snapshot.go b/lib/hypervisor/cloudhypervisor/fork_snapshot.go similarity index 59% rename from lib/forkvm/snapshot.go rename to lib/hypervisor/cloudhypervisor/fork_snapshot.go index 93e5db58..11bf67c5 100644 --- a/lib/forkvm/snapshot.go +++ b/lib/hypervisor/cloudhypervisor/fork_snapshot.go @@ -1,34 +1,16 @@ -package forkvm +package cloudhypervisor import ( "encoding/json" "fmt" "os" "strings" -) - -// SnapshotNetworkConfig contains network identity fields in CH snapshot config.json. -type SnapshotNetworkConfig struct { - TAPDevice string - IP string - MAC string - Netmask string -} - -// SnapshotRewriteOptions controls identity/path rewrites for CH snapshot config.json. -type SnapshotRewriteOptions struct { - SourceDataDir string - TargetDataDir string - - VsockCID *int64 - VsockSocket string - SerialLogPath string - Network *SnapshotNetworkConfig -} + "github.com/kernel/hypeman/lib/hypervisor" +) -// RewriteSnapshotConfig rewrites cloud-hypervisor snapshot config.json for a forked instance. -func RewriteSnapshotConfig(configPath string, opts SnapshotRewriteOptions) error { +// rewriteSnapshotConfigForFork rewrites Cloud Hypervisor snapshot config.json for a forked instance. +func rewriteSnapshotConfigForFork(configPath string, req hypervisor.ForkPrepareRequest) error { data, err := os.ReadFile(configPath) if err != nil { return fmt.Errorf("read snapshot config: %w", err) @@ -39,21 +21,19 @@ func RewriteSnapshotConfig(configPath string, opts SnapshotRewriteOptions) error return fmt.Errorf("unmarshal snapshot config: %w", err) } - if opts.SourceDataDir != "" && opts.TargetDataDir != "" && opts.SourceDataDir != opts.TargetDataDir { + if req.SourceDataDir != "" && req.TargetDataDir != "" && req.SourceDataDir != req.TargetDataDir { configAny := rewriteStringValues(config, func(s string) string { - return strings.ReplaceAll(s, opts.SourceDataDir, opts.TargetDataDir) + return strings.ReplaceAll(s, req.SourceDataDir, req.TargetDataDir) }) config = configAny.(map[string]any) } - if opts.VsockCID != nil || opts.VsockSocket != "" { - updateVsockConfig(config, opts.VsockCID, opts.VsockSocket) + updateVsockConfig(config, req.VsockCID, req.VsockSocket) + if req.SerialLogPath != "" { + updateSerialConfig(config, req.SerialLogPath) } - if opts.SerialLogPath != "" { - updateSerialConfig(config, opts.SerialLogPath) - } - if opts.Network != nil { - updateNetworkConfig(config, *opts.Network) + if req.Network != nil { + updateNetworkConfig(config, req.Network) } updated, err := json.MarshalIndent(config, "", " ") @@ -89,14 +69,12 @@ func rewriteStringValues(value any, mapper func(string) string) any { } } -func updateVsockConfig(config map[string]any, cid *int64, socketPath string) { +func updateVsockConfig(config map[string]any, cid int64, socketPath string) { vsock, ok := config["vsock"].(map[string]any) if !ok || vsock == nil { return } - if cid != nil { - vsock["cid"] = *cid - } + vsock["cid"] = cid if socketPath != "" { vsock["socket"] = socketPath } @@ -110,7 +88,7 @@ func updateSerialConfig(config map[string]any, logPath string) { serial["file"] = logPath } -func updateNetworkConfig(config map[string]any, netCfg SnapshotNetworkConfig) { +func updateNetworkConfig(config map[string]any, netCfg *hypervisor.ForkNetworkConfig) { nets, ok := config["net"].([]any) if !ok { return diff --git a/lib/forkvm/snapshot_test.go b/lib/hypervisor/cloudhypervisor/fork_snapshot_test.go similarity index 87% rename from lib/forkvm/snapshot_test.go rename to lib/hypervisor/cloudhypervisor/fork_snapshot_test.go index 38064db4..f5abd8ec 100644 --- a/lib/forkvm/snapshot_test.go +++ b/lib/hypervisor/cloudhypervisor/fork_snapshot_test.go @@ -1,4 +1,4 @@ -package forkvm +package cloudhypervisor import ( "encoding/json" @@ -6,11 +6,12 @@ import ( "path/filepath" "testing" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestRewriteSnapshotConfig(t *testing.T) { +func TestRewriteSnapshotConfigForFork(t *testing.T) { tmp := t.TempDir() configPath := filepath.Join(tmp, "config.json") @@ -29,14 +30,13 @@ func TestRewriteSnapshotConfig(t *testing.T) { require.NoError(t, err) require.NoError(t, os.WriteFile(configPath, data, 0644)) - cid := int64(200) - err = RewriteSnapshotConfig(configPath, SnapshotRewriteOptions{ + err = rewriteSnapshotConfigForFork(configPath, hypervisor.ForkPrepareRequest{ SourceDataDir: "/src/guests/a", TargetDataDir: "/dst/guests/b", - VsockCID: &cid, + VsockCID: 200, VsockSocket: "/dst/guests/b/vsock.sock", SerialLogPath: "/dst/guests/b/logs/app.log", - Network: &SnapshotNetworkConfig{ + Network: &hypervisor.ForkNetworkConfig{ TAPDevice: "hype-new", IP: "10.0.0.20", MAC: "02:00:00:00:00:02", From 936fe576b5732f3dfca09e2f23542387fe25387c Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 27 Feb 2026 16:00:30 -0500 Subject: [PATCH 05/20] fork: support running source via standby-resume flow --- cmd/api/api/instances.go | 16 +- cmd/api/api/instances_test.go | 89 ++++++++ lib/forkvm/copy.go | 6 +- lib/instances/create.go | 14 +- lib/instances/errors.go | 3 + lib/instances/fork.go | 93 ++++++-- lib/instances/fork_test.go | 146 +++++++------ lib/instances/manager.go | 2 +- lib/instances/name_validation.go | 21 ++ lib/instances/restore.go | 104 ++++++++- lib/instances/types.go | 5 +- lib/oapi/oapi.go | 361 ++++++++++++++++--------------- openapi.yaml | 11 +- 13 files changed, 562 insertions(+), 309 deletions(-) create mode 100644 lib/instances/name_validation.go diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index e44a2199..a9266c60 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -468,7 +468,8 @@ func (s *ApiService) ForkInstance(ctx context.Context, request oapi.ForkInstance } result, err := s.InstanceManager.ForkInstance(ctx, inst.Id, instances.ForkInstanceRequest{ - Name: request.Body.Name, + Name: request.Body.Name, + FromRunning: request.Body.FromRunning != nil && *request.Body.FromRunning, }) if err != nil { switch { @@ -482,6 +483,11 @@ func (s *ApiService) ForkInstance(ctx context.Context, request oapi.ForkInstance Code: "invalid_state", Message: err.Error(), }, nil + case errors.Is(err, instances.ErrInvalidRequest): + return oapi.ForkInstance400JSONResponse{ + Code: "invalid_request", + Message: err.Error(), + }, nil case errors.Is(err, instances.ErrAlreadyExists), errors.Is(err, network.ErrNameExists): return oapi.ForkInstance409JSONResponse{ Code: "name_conflict", @@ -493,14 +499,6 @@ func (s *ApiService) ForkInstance(ctx context.Context, request oapi.ForkInstance Message: err.Error(), }, nil default: - // Validation errors (for example invalid fork name) are user input issues. - if strings.Contains(err.Error(), "name") { - return oapi.ForkInstance400JSONResponse{ - Code: "invalid_request", - Message: err.Error(), - }, nil - } - log.ErrorContext(ctx, "failed to fork instance", "error", err) return oapi.ForkInstance500JSONResponse{ Code: "internal_error", diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 5ef022e3..cb80a581 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -2,6 +2,7 @@ package api import ( "context" + "fmt" "os" "testing" "time" @@ -258,6 +259,7 @@ func TestForkInstance_Success(t *testing.T) { assert.Equal(t, source.Id, mockMgr.lastID) require.NotNil(t, mockMgr.lastReq) assert.Equal(t, "forked-instance", mockMgr.lastReq.Name) + assert.False(t, mockMgr.lastReq.FromRunning) } func TestForkInstance_NotSupported(t *testing.T) { @@ -296,6 +298,93 @@ func TestForkInstance_NotSupported(t *testing.T) { assert.Equal(t, "not_supported", notSupported.Code) } +func TestForkInstance_InvalidRequest(t *testing.T) { + svc := newTestService(t) + + source := instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "src-instance", + Name: "src-instance", + Image: "docker.io/library/alpine:latest", + CreatedAt: time.Now(), + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateStopped, + } + + mockMgr := &captureForkManager{ + Manager: svc.InstanceManager, + err: fmt.Errorf("%w: name is required", instances.ErrInvalidRequest), + } + svc.InstanceManager = mockMgr + + resp, err := svc.ForkInstance( + mw.WithResolvedInstance(ctx(), source.Id, source), + oapi.ForkInstanceRequestObject{ + Id: source.Id, + Body: &oapi.ForkInstanceRequest{ + Name: "", + }, + }, + ) + require.NoError(t, err) + + badReq, ok := resp.(oapi.ForkInstance400JSONResponse) + require.True(t, ok, "expected 400 response") + assert.Equal(t, "invalid_request", badReq.Code) +} + +func TestForkInstance_FromRunningFlagForwarded(t *testing.T) { + svc := newTestService(t) + + now := time.Now() + source := instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "src-instance", + Name: "src-instance", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateRunning, + } + + forked := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "forked-instance", + Name: "forked-instance", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateStandby, + } + + mockMgr := &captureForkManager{ + Manager: svc.InstanceManager, + result: forked, + } + svc.InstanceManager = mockMgr + + fromRunning := true + resp, err := svc.ForkInstance( + mw.WithResolvedInstance(ctx(), source.Id, source), + oapi.ForkInstanceRequestObject{ + Id: source.Id, + Body: &oapi.ForkInstanceRequest{ + Name: "forked-instance", + FromRunning: &fromRunning, + }, + }, + ) + require.NoError(t, err) + + _, ok := resp.(oapi.ForkInstance201JSONResponse) + require.True(t, ok, "expected 201 response") + require.NotNil(t, mockMgr.lastReq) + assert.True(t, mockMgr.lastReq.FromRunning) +} + func TestInstanceLifecycle_StopStart(t *testing.T) { // Require KVM access for VM creation if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { diff --git a/lib/forkvm/copy.go b/lib/forkvm/copy.go index 548ce65d..b942f8c0 100644 --- a/lib/forkvm/copy.go +++ b/lib/forkvm/copy.go @@ -87,9 +87,11 @@ func copyRegularFile(srcPath, dstPath string, perms fs.FileMode) error { if err != nil { return err } - defer dst.Close() - if _, err := io.Copy(dst, src); err != nil { + _ = dst.Close() + return err + } + if err := dst.Close(); err != nil { return err } diff --git a/lib/instances/create.go b/lib/instances/create.go index 46e9ac30..0820cb2a 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "path/filepath" - "regexp" "strings" "time" @@ -450,17 +449,8 @@ func (m *manager) createInstance( // validateCreateRequest validates the create instance request func validateCreateRequest(req CreateInstanceRequest) error { - if req.Name == "" { - return fmt.Errorf("name is required") - } - // Validate name format: lowercase letters, digits, dashes only - // No starting/ending with dashes, max 63 characters - if len(req.Name) > 63 { - return fmt.Errorf("name must be 63 characters or less") - } - namePattern := regexp.MustCompile(`^[a-z0-9]([a-z0-9-]*[a-z0-9])?$`) - if !namePattern.MatchString(req.Name) { - return fmt.Errorf("name must contain only lowercase letters, digits, and dashes; cannot start or end with a dash") + if err := validateInstanceName(req.Name); err != nil { + return err } if req.Image == "" { return fmt.Errorf("image is required") diff --git a/lib/instances/errors.go b/lib/instances/errors.go index eb5f1baa..ef30ca4c 100644 --- a/lib/instances/errors.go +++ b/lib/instances/errors.go @@ -9,6 +9,9 @@ var ( // ErrInvalidState is returned when a state transition is not valid ErrInvalidState = errors.New("invalid state transition") + // ErrInvalidRequest is returned when request validation fails + ErrInvalidRequest = errors.New("invalid request") + // ErrAlreadyExists is returned when creating an instance that already exists ErrAlreadyExists = errors.New("instance already exists") diff --git a/lib/instances/fork.go b/lib/instances/fork.go index f8f96d1e..777da566 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "os" - "regexp" "time" "github.com/kernel/hypeman/lib/forkvm" @@ -25,6 +24,52 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR return nil, err } + meta, err := m.loadMetadata(id) + if err != nil { + return nil, err + } + source := m.toInstance(ctx, meta) + + switch source.State { + case StateRunning: + if !req.FromRunning { + return nil, fmt.Errorf("%w: cannot fork from state %s (set from_running=true to allow standby+restore flow)", ErrInvalidState, source.State) + } + + if err := m.validateForkSupport(ctx, source.HypervisorType); err != nil { + return nil, err + } + + log.InfoContext(ctx, "fork from running requested; transitioning source to standby", + "source_instance_id", id, "hypervisor", source.HypervisorType) + if _, err := m.standbyInstance(ctx, id); err != nil { + return nil, fmt.Errorf("standby source instance: %w", err) + } + + forked, forkErr := m.forkInstanceFromStoppedOrStandby(ctx, id, req) + log.InfoContext(ctx, "restoring source instance after running fork", "source_instance_id", id) + _, restoreErr := m.restoreInstance(ctx, id) + + if restoreErr != nil { + if forkErr != nil { + return nil, fmt.Errorf("fork failed: %v; additionally failed to restore source instance: %w", forkErr, restoreErr) + } + return nil, fmt.Errorf("restore source instance after fork: %w", restoreErr) + } + if forkErr != nil { + return nil, forkErr + } + return forked, nil + case StateStopped, StateStandby: + return m.forkInstanceFromStoppedOrStandby(ctx, id, req) + default: + return nil, fmt.Errorf("%w: cannot fork from state %s (must be Stopped or Standby, or Running with from_running=true)", ErrInvalidState, source.State) + } +} + +func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id string, req ForkInstanceRequest) (*Instance, error) { + log := logger.FromContext(ctx) + meta, err := m.loadMetadata(id) if err != nil { return nil, err @@ -40,8 +85,12 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR return nil, fmt.Errorf("%w: cannot fork from state %s (must be Stopped or Standby)", ErrInvalidState, source.State) } + if err := m.validateForkSupport(ctx, stored.HypervisorType); err != nil { + return nil, err + } + if stored.NetworkEnabled { - exists, err := m.networkManager.NameExists(ctx, req.Name, id) + exists, err := m.networkManager.NameExists(ctx, req.Name, "") if err != nil { return nil, fmt.Errorf("check instance name availability: %w", err) } @@ -82,11 +131,18 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR forkMeta.HypervisorPID = nil forkMeta.SocketPath = m.paths.InstanceSocket(forkID, starter.SocketName()) forkMeta.DataDir = dstDir - forkMeta.VsockCID = generateVsockCID(forkID) forkMeta.VsockSocket = m.paths.InstanceVsockSocket(forkID) forkMeta.ExitCode = nil forkMeta.ExitMessage = "" + if source.State == StateStandby { + // Keep the original CID for snapshot-based forks. + // Rewriting CID in restored memory snapshots is not reliable for CH. + forkMeta.VsockCID = stored.VsockCID + } else { + forkMeta.VsockCID = generateVsockCID(forkID) + } + if forkMeta.NetworkEnabled { // Clear inherited network identity. For stopped instances this is regenerated on start, // and for standby instances restore allocates if identity is empty. @@ -114,14 +170,6 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR } return nil, fmt.Errorf("prepare fork snapshot state: %w", err) } - } else { - // Validate fork support for stopped-state forks as well. - if err := starter.PrepareFork(ctx, hypervisor.ForkPrepareRequest{}); err != nil { - if errors.Is(err, hypervisor.ErrNotSupported) { - return nil, fmt.Errorf("%w: fork is not supported for hypervisor %s", ErrNotSupported, stored.HypervisorType) - } - return nil, fmt.Errorf("prepare fork state: %w", err) - } } newMeta := &metadata{StoredMetadata: forkMeta} @@ -139,16 +187,23 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR return &forked, nil } -func validateForkRequest(req ForkInstanceRequest) error { - if req.Name == "" { - return fmt.Errorf("name is required") +func (m *manager) validateForkSupport(ctx context.Context, hvType hypervisor.Type) error { + starter, err := m.getVMStarter(hvType) + if err != nil { + return fmt.Errorf("get vm starter: %w", err) } - if len(req.Name) > 63 { - return fmt.Errorf("name must be 63 characters or less") + if err := starter.PrepareFork(ctx, hypervisor.ForkPrepareRequest{}); err != nil { + if errors.Is(err, hypervisor.ErrNotSupported) { + return fmt.Errorf("%w: fork is not supported for hypervisor %s", ErrNotSupported, hvType) + } + return fmt.Errorf("prepare fork state: %w", err) } - namePattern := regexp.MustCompile(`^[a-z0-9]([a-z0-9-]*[a-z0-9])?$`) - if !namePattern.MatchString(req.Name) { - return fmt.Errorf("name must contain only lowercase letters, digits, and dashes; cannot start or end with a dash") + return nil +} + +func validateForkRequest(req ForkInstanceRequest) error { + if err := validateInstanceName(req.Name); err != nil { + return fmt.Errorf("%w: %v", ErrInvalidRequest, err) } return nil } diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index 09af2dc8..3d82f6e8 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -2,7 +2,11 @@ package instances import ( "context" + "fmt" + "io" + "net/http" "os" + "strings" "testing" "time" @@ -39,7 +43,7 @@ func TestForkInstanceNotSupportedHypervisor(t *testing.T) { assert.ErrorIs(t, err, ErrNotSupported) } -func TestForkCloudHypervisorStoppedAndStandby(t *testing.T) { +func TestForkCloudHypervisorFromRunningNetwork(t *testing.T) { if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { t.Skip("/dev/kvm not available, skipping on this platform") } @@ -50,15 +54,15 @@ func TestForkCloudHypervisorStoppedAndStandby(t *testing.T) { imageManager, err := images.NewManager(paths.New(tmpDir), 1, nil) require.NoError(t, err) - t.Log("Ensuring alpine image...") - alpineImage, err := imageManager.CreateImage(ctx, images.CreateImageRequest{Name: "docker.io/library/alpine:latest"}) + t.Log("Ensuring nginx image...") + nginxImage, err := imageManager.CreateImage(ctx, images.CreateImageRequest{Name: "docker.io/library/nginx:alpine"}) require.NoError(t, err) - imageName := alpineImage.Name + imageName := nginxImage.Name for i := 0; i < 60; i++ { img, err := imageManager.GetImage(ctx, imageName) if err == nil && img.Status == images.StatusReady { - alpineImage = img + nginxImage = img break } if err == nil && img.Status == images.StatusFailed { @@ -66,95 +70,93 @@ func TestForkCloudHypervisorStoppedAndStandby(t *testing.T) { } time.Sleep(1 * time.Second) } - require.Equal(t, images.StatusReady, alpineImage.Status, "Image should be ready after 60 seconds") + require.Equal(t, images.StatusReady, nginxImage.Status, "Image should be ready after 60 seconds") systemManager := manager.systemManager require.NoError(t, systemManager.EnsureSystemFiles(ctx)) require.NoError(t, manager.networkManager.Initialize(ctx, nil)) - createReq := CreateInstanceRequest{ - Image: "docker.io/library/alpine:latest", + source, err := manager.CreateInstance(ctx, CreateInstanceRequest{ + Name: "fork-running-src", + Image: "docker.io/library/nginx:alpine", Size: 2 * 1024 * 1024 * 1024, HotplugSize: 256 * 1024 * 1024, OverlaySize: 10 * 1024 * 1024 * 1024, Vcpus: 1, NetworkEnabled: true, - Entrypoint: []string{"sh", "-c"}, - Cmd: []string{"while true; do sleep 3600; done"}, - } - - // Stopped source fork flow. - sourceStopped, err := manager.CreateInstance(ctx, CreateInstanceRequest{ - Name: "fork-stop-src", - Image: createReq.Image, - Size: createReq.Size, - HotplugSize: createReq.HotplugSize, - OverlaySize: createReq.OverlaySize, - Vcpus: createReq.Vcpus, - NetworkEnabled: createReq.NetworkEnabled, - Entrypoint: createReq.Entrypoint, - Cmd: createReq.Cmd, }) require.NoError(t, err) - t.Cleanup(func() { _ = manager.DeleteInstance(context.Background(), sourceStopped.Id) }) - require.NoError(t, waitForVMReady(ctx, sourceStopped.SocketPath, 5*time.Second)) + t.Cleanup(func() { _ = manager.DeleteInstance(context.Background(), source.Id) }) + require.NoError(t, waitForVMReady(ctx, source.SocketPath, 5*time.Second)) + require.NoError(t, waitForLogMessage(ctx, manager, source.Id, "start worker processes", 15*time.Second)) - sourceStopped, err = manager.StopInstance(ctx, sourceStopped.Id) - require.NoError(t, err) - require.Equal(t, StateStopped, sourceStopped.State) + assert.NotEmpty(t, source.IP) + assert.NotEmpty(t, source.MAC) + assertHostCanReachNginx(t, source.IP, 80, 60*time.Second) - forkStopped, err := manager.ForkInstance(ctx, sourceStopped.Id, ForkInstanceRequest{Name: "fork-stop-copy"}) - require.NoError(t, err) - require.Equal(t, StateStopped, forkStopped.State) - t.Cleanup(func() { _ = manager.DeleteInstance(context.Background(), forkStopped.Id) }) + // Default behavior remains strict: running source requires explicit opt-in. + _, err = manager.ForkInstance(ctx, source.Id, ForkInstanceRequest{Name: "fork-running-no-flag"}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidState) - sourceStopped, err = manager.StartInstance(ctx, sourceStopped.Id, StartInstanceRequest{}) - require.NoError(t, err) - forkStopped, err = manager.StartInstance(ctx, forkStopped.Id, StartInstanceRequest{}) - require.NoError(t, err) - require.NoError(t, waitForVMReady(ctx, sourceStopped.SocketPath, 5*time.Second)) - require.NoError(t, waitForVMReady(ctx, forkStopped.SocketPath, 5*time.Second)) - - assert.NotEmpty(t, sourceStopped.IP) - assert.NotEmpty(t, forkStopped.IP) - assert.NotEqual(t, sourceStopped.IP, forkStopped.IP) - assert.NotEqual(t, sourceStopped.MAC, forkStopped.MAC) - - // Standby source fork flow. - sourceStandby, err := manager.CreateInstance(ctx, CreateInstanceRequest{ - Name: "fork-standby-src", - Image: createReq.Image, - Size: createReq.Size, - HotplugSize: createReq.HotplugSize, - OverlaySize: createReq.OverlaySize, - Vcpus: createReq.Vcpus, - NetworkEnabled: createReq.NetworkEnabled, - Entrypoint: createReq.Entrypoint, - Cmd: createReq.Cmd, + // Fork from running (internally: standby source -> copy fork -> restore source). + forked, err := manager.ForkInstance(ctx, source.Id, ForkInstanceRequest{ + Name: "fork-running-copy", + FromRunning: true, }) require.NoError(t, err) - t.Cleanup(func() { _ = manager.DeleteInstance(context.Background(), sourceStandby.Id) }) - require.NoError(t, waitForVMReady(ctx, sourceStandby.SocketPath, 5*time.Second)) + require.Equal(t, StateStandby, forked.State) + forkedID := forked.Id + t.Cleanup(func() { _ = manager.DeleteInstance(context.Background(), forkedID) }) - sourceStandby, err = manager.StandbyInstance(ctx, sourceStandby.Id) + // Source should be restored and still reachable by its private IP. + sourceAfterFork, err := manager.GetInstance(ctx, source.Id) require.NoError(t, err) - require.Equal(t, StateStandby, sourceStandby.State) + require.Equal(t, StateRunning, sourceAfterFork.State) + require.NotEmpty(t, sourceAfterFork.IP) + assertHostCanReachNginx(t, sourceAfterFork.IP, 80, 60*time.Second) - forkStandby, err := manager.ForkInstance(ctx, sourceStandby.Id, ForkInstanceRequest{Name: "fork-standby-copy"}) + // Restore fork and validate both VMs are independently reachable on private IPs. + forked, err = manager.RestoreInstance(ctx, forkedID) require.NoError(t, err) - require.Equal(t, StateStandby, forkStandby.State) - t.Cleanup(func() { _ = manager.DeleteInstance(context.Background(), forkStandby.Id) }) + require.Equal(t, StateRunning, forked.State) + require.NoError(t, waitForVMReady(ctx, forked.SocketPath, 5*time.Second)) + + assert.NotEmpty(t, forked.IP) + assert.NotEmpty(t, forked.MAC) + assert.NotEqual(t, sourceAfterFork.IP, forked.IP) + assert.NotEqual(t, sourceAfterFork.MAC, forked.MAC) + assertHostCanReachNginx(t, forked.IP, 80, 60*time.Second) + assertHostCanReachNginx(t, sourceAfterFork.IP, 80, 60*time.Second) +} - sourceStandby, err = manager.RestoreInstance(ctx, sourceStandby.Id) - require.NoError(t, err) - forkStandby, err = manager.RestoreInstance(ctx, forkStandby.Id) - require.NoError(t, err) - require.NoError(t, waitForVMReady(ctx, sourceStandby.SocketPath, 5*time.Second)) - require.NoError(t, waitForVMReady(ctx, forkStandby.SocketPath, 5*time.Second)) +func assertHostCanReachNginx(t *testing.T, ip string, port int, timeout time.Duration) { + t.Helper() + + deadline := time.Now().Add(timeout) + url := fmt.Sprintf("http://%s:%d/", ip, port) + client := &http.Client{Timeout: 3 * time.Second} + + var lastErr error + for time.Now().Before(deadline) { + resp, err := client.Get(url) + if err == nil { + body, readErr := io.ReadAll(resp.Body) + resp.Body.Close() + if readErr == nil && resp.StatusCode == http.StatusOK && strings.Contains(string(body), "Welcome to nginx!") { + return + } + if readErr != nil { + lastErr = fmt.Errorf("read body: %w", readErr) + } else { + lastErr = fmt.Errorf("status=%d body=%q", resp.StatusCode, string(body)) + } + } else { + lastErr = err + } + time.Sleep(500 * time.Millisecond) + } - assert.NotEmpty(t, sourceStandby.IP) - assert.NotEmpty(t, forkStandby.IP) - assert.NotEqual(t, sourceStandby.IP, forkStandby.IP) - assert.NotEqual(t, sourceStandby.MAC, forkStandby.MAC) + require.NoError(t, lastErr, "host should reach %s within %s", url, timeout) } diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 2cce0faf..29f6ec05 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -183,7 +183,7 @@ func (m *manager) DeleteInstance(ctx context.Context, id string) error { return err } -// ForkInstance creates a forked copy of a stopped or standby instance. +// ForkInstance creates a forked copy of an instance. func (m *manager) ForkInstance(ctx context.Context, id string, req ForkInstanceRequest) (*Instance, error) { lock := m.getInstanceLock(id) lock.Lock() diff --git a/lib/instances/name_validation.go b/lib/instances/name_validation.go new file mode 100644 index 00000000..4818bc7e --- /dev/null +++ b/lib/instances/name_validation.go @@ -0,0 +1,21 @@ +package instances + +import ( + "fmt" + "regexp" +) + +var instanceNamePattern = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]*[a-z0-9])?$`) + +func validateInstanceName(name string) error { + if name == "" { + return fmt.Errorf("name is required") + } + if len(name) > 63 { + return fmt.Errorf("name must be 63 characters or less") + } + if !instanceNamePattern.MatchString(name) { + return fmt.Errorf("name must contain only lowercase letters, digits, and dashes; cannot start or end with a dash") + } + return nil +} diff --git a/lib/instances/restore.go b/lib/instances/restore.go index e6c002de..e28a1000 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -1,12 +1,16 @@ package instances import ( + "bytes" "context" "errors" "fmt" + "net" "os" + "strings" "time" + "github.com/kernel/hypeman/lib/guest" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" @@ -70,6 +74,23 @@ func (m *manager) restoreInstance( return nil, fmt.Errorf("get vm starter: %w", err) } + var allocatedNet *network.Allocation + releaseNetwork := func() { + if !stored.NetworkEnabled { + return + } + if allocatedNet != nil { + if err := m.networkManager.ReleaseAllocation(ctx, allocatedNet); err != nil { + log.WarnContext(ctx, "failed to release allocated network", "instance_id", id, "error", err) + } + return + } + netAlloc, _ := m.networkManager.GetAllocation(ctx, id) + if err := m.networkManager.ReleaseAllocation(ctx, netAlloc); err != nil { + log.WarnContext(ctx, "failed to release network", "instance_id", id, "error", err) + } + } + // 4. Recreate or allocate network if network enabled if stored.NetworkEnabled { var networkSpan trace.Span @@ -96,6 +117,16 @@ func (m *manager) restoreInstance( log.ErrorContext(ctx, "failed to allocate network", "instance_id", id, "error", err) return nil, fmt.Errorf("allocate network: %w", err) } + allocatedNet = &network.Allocation{ + InstanceID: id, + InstanceName: stored.Name, + Network: "default", + IP: netConfig.IP, + MAC: netConfig.MAC, + TAPDevice: netConfig.TAPDevice, + Gateway: netConfig.Gateway, + Netmask: netConfig.Netmask, + } stored.IP = netConfig.IP stored.MAC = netConfig.MAC @@ -115,13 +146,11 @@ func (m *manager) restoreInstance( } if errors.Is(err, hypervisor.ErrNotSupported) { log.ErrorContext(ctx, "forked standby network rewrite not supported for hypervisor", "instance_id", id, "hypervisor", stored.HypervisorType) - netAlloc, _ := m.networkManager.GetAllocation(ctx, id) - m.networkManager.ReleaseAllocation(ctx, netAlloc) + releaseNetwork() return nil, fmt.Errorf("%w: standby fork restore network rewrite is not supported for hypervisor %s", ErrNotSupported, stored.HypervisorType) } log.ErrorContext(ctx, "failed to patch snapshot network identity", "instance_id", id, "error", err) - netAlloc, _ := m.networkManager.GetAllocation(ctx, id) - m.networkManager.ReleaseAllocation(ctx, netAlloc) + releaseNetwork() return nil, fmt.Errorf("rewrite snapshot config: %w", err) } } else { @@ -153,10 +182,7 @@ func (m *manager) restoreInstance( if err != nil { log.ErrorContext(ctx, "failed to restore from snapshot", "instance_id", id, "error", err) // Cleanup network on failure - if stored.NetworkEnabled { - netAlloc, _ := m.networkManager.GetAllocation(ctx, id) - m.networkManager.ReleaseAllocation(ctx, netAlloc) - } + releaseNetwork() return nil, err } @@ -176,16 +202,25 @@ func (m *manager) restoreInstance( log.ErrorContext(ctx, "failed to resume VM", "instance_id", id, "error", err) // Cleanup on failure hv.Shutdown(ctx) - if stored.NetworkEnabled { - netAlloc, _ := m.networkManager.GetAllocation(ctx, id) - m.networkManager.ReleaseAllocation(ctx, netAlloc) - } + releaseNetwork() return nil, fmt.Errorf("resume vm failed: %w", err) } if resumeSpan != nil { resumeSpan.End() } + // Forked standby restores may allocate a fresh identity while the guest memory snapshot + // still has the source VM's old IP configuration. Reconfigure guest networking after + // resume so host ingress to the new private IP works reliably. + if allocatedNet != nil && !stored.SkipGuestAgent { + if err := reconfigureGuestNetwork(ctx, stored, allocatedNet); err != nil { + log.ErrorContext(ctx, "failed to configure guest network after restore", "instance_id", id, "error", err) + _ = hv.Shutdown(ctx) + releaseNetwork() + return nil, fmt.Errorf("configure guest network after restore: %w", err) + } + } + // 8. Delete snapshot after successful restore log.InfoContext(ctx, "deleting snapshot after successful restore", "instance_id", id) os.RemoveAll(snapshotDir) // Best effort, ignore errors @@ -236,3 +271,48 @@ func (m *manager) restoreFromSnapshot( log.DebugContext(ctx, "VM restored from snapshot successfully", "instance_id", stored.Id, "pid", pid) return pid, hv, nil } + +func reconfigureGuestNetwork(ctx context.Context, stored *StoredMetadata, alloc *network.Allocation) error { + prefix, err := netmaskToPrefix(alloc.Netmask) + if err != nil { + return err + } + + dialer, err := hypervisor.NewVsockDialer(stored.HypervisorType, stored.VsockSocket, stored.VsockCID) + if err != nil { + return fmt.Errorf("create vsock dialer: %w", err) + } + + cmd := fmt.Sprintf( + "ip addr replace %s/%d dev eth0 && ip link set dev eth0 up && ip route replace default via %s dev eth0", + alloc.IP, prefix, alloc.Gateway, + ) + + var stdout, stderr bytes.Buffer + exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ + Command: []string{"sh", "-c", cmd}, + Stdout: &stdout, + Stderr: &stderr, + WaitForAgent: 120 * time.Second, + }) + if err != nil { + return fmt.Errorf("exec network reconfiguration command: %w", err) + } + if exit.Code != 0 { + return fmt.Errorf("network reconfiguration command failed (exit=%d, stdout=%q, stderr=%q)", exit.Code, strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String())) + } + + return nil +} + +func netmaskToPrefix(mask string) (int, error) { + ip := net.ParseIP(mask).To4() + if ip == nil { + return 0, fmt.Errorf("invalid netmask: %q", mask) + } + ones, bits := net.IPMask(ip).Size() + if bits != 32 { + return 0, fmt.Errorf("invalid netmask bits: %q", mask) + } + return ones, nil +} diff --git a/lib/instances/types.go b/lib/instances/types.go index 2eef68b4..1a7d5d74 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -175,9 +175,10 @@ type StartInstanceRequest struct { Cmd []string // Override cmd (nil = keep previous/image default) } -// ForkInstanceRequest is the domain request for forking a stopped/standby instance. +// ForkInstanceRequest is the domain request for forking an instance. type ForkInstanceRequest struct { - Name string // Required: name for the new forked instance + Name string // Required: name for the new forked instance + FromRunning bool // Optional: allow forking from Running by auto standby/fork/restore } // AttachVolumeRequest is the domain request for attaching a volume (used for API compatibility) diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 1fee2568..be416602 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -400,6 +400,10 @@ type ErrorDetail struct { // ForkInstanceRequest defines model for ForkInstanceRequest. type ForkInstanceRequest struct { + // FromRunning Allow forking from a running source instance. + // When true and source is Running, the source is put into standby, forked, then restored back to Running. + FromRunning *bool `json:"from_running,omitempty"` + // Name Name for the forked instance (lowercase letters, digits, and dashes only; cannot start or end with a dash) Name string `json:"name"` } @@ -6573,7 +6577,7 @@ type ServerInterface interface { // Get instance details // (GET /instances/{id}) GetInstance(w http.ResponseWriter, r *http.Request, id string) - // Fork an instance from stopped or standby state + // Fork an instance from stopped, standby, or running (with from_running=true) // (POST /instances/{id}/fork) ForkInstance(w http.ResponseWriter, r *http.Request, id string) // Stream instance logs (SSE) @@ -6765,7 +6769,7 @@ func (_ Unimplemented) GetInstance(w http.ResponseWriter, r *http.Request, id st w.WriteHeader(http.StatusNotImplemented) } -// Fork an instance from stopped or standby state +// Fork an instance from stopped, standby, or running (with from_running=true) // (POST /instances/{id}/fork) func (_ Unimplemented) ForkInstance(w http.ResponseWriter, r *http.Request, id string) { w.WriteHeader(http.StatusNotImplemented) @@ -9953,7 +9957,7 @@ type StrictServerInterface interface { // Get instance details // (GET /instances/{id}) GetInstance(ctx context.Context, request GetInstanceRequestObject) (GetInstanceResponseObject, error) - // Fork an instance from stopped or standby state + // Fork an instance from stopped, standby, or running (with from_running=true) // (POST /instances/{id}/fork) ForkInstance(ctx context.Context, request ForkInstanceRequestObject) (ForkInstanceResponseObject, error) // Stream instance logs (SSE) @@ -11091,181 +11095,182 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x963LbOLLwq6D4na2Vz0qyfInj+NTUKcdOPN6NE39x7P3OjvIpEAlJGJMABwCVKKn8", - "3QfYR5wnOYUGwJtAicrFiXeytTWhRRBoNBqN7kZfPgQhT1LOCFMyOPoQyHBGEgyPx0rhcHbD4ywhL8lv", - "GZFK/5wKnhKhKIFGCc+YGqVYzfRfEZGhoKminAVHwSVWM/R2RgRBc+gFyRnP4giNCYLvSBR0A/IOJ2lM", - "gqNgO2FqO8IKB91ALVL9k1SCsmnwsRsIgiPO4oUZZoKzWAVHExxL0q0Ne6G7Rlgi/UkPvsn7G3MeE8yC", - "j9DjbxkVJAqOfilP43XemI9/JaHSgx/PMY3xOCanZE5DsoyGMBOCMDWKBJ0TsYyKE/M+XqAxz1iETDvU", - "YVkcIzpBjDOyVUEGm9OIakzoJnro4EiJjHgwEwFMIxp5VuDkHJnX6PwUdWbkXXWQ3Yfjw6C5S4YTstzp", - "z1mCWU8jV4Pl+oe25b6f7ft6pjxJstFU8Cxd7vn8xcXFNYKXiGXJmIhyj4e7eX+UKTIlQneYhnSEo0gQ", - "Kf3zdy/LsA0Gg8ER3j0aDPoDH5RzwiIuGlFqXvtRujOIyIouW6HU9r+E0uc356fnx+iEi5QLDN8ujVQj", - "7DJ6yvMqk011VXz0/zijcbRM9WP9MxEjyqTCrIEGz+1LjS4+QWpGkP0O3VygzoQLFJFxNp1SNt1qQ++a", - "YcVEkWiE1fJwACqybShnSNGESIWTNOgGEy4S/VEQYUV6+k2rAQXBa4bTLVoNtrzVMrOSo0Q29e6aIMpQ", - "QuOYShJyFsnyGJSpg/3myZQ2DBGCezjUE/0zSoiUeEpQR7NNzbsZkgqrTCIq0QTTmESt1shHCGYyv/Ix", - "ohFhik5odX8bcurhcbizu+flHQmeklFEp/YkqnZ/Cr9rEtP9KASt/RPRG23Rbh4wpCCT5fGeAuuGQQSZ", - "EEE0jX/mcKngc8L0btHj/QeMG/yf7eKI3rbn8zYg87Jo/rEb/JaRjIxSLqmBcIlz2TeajADVCL7wwwyv", - "Vq11iaKkwmL1/oAWX2AnGvha4ebKNK3zQ2B3tpvKzm5ke0/mhHkEn5AzZV9UZ/yMT1FMGUG2hcWv5nN6", - "gJ9iDmzuS8ytGxQoXd7QGu5PYEjmh4be9LtuQFiWaGTGfFrG5oxgocakgsyGY8l2VEDXiP7LypaonT9Y", - "ktFqrnBJGSMR0i3tZjUtUSZB+lyaPuyMW6pGcyKkdx8BWH+jCtkWjV3FPLyd0JiMZljODMQ4imAP4viy", - "MhOPBFYRaXGqGZvrECQDiRRHVz8f7z44QHYADw4lz0RoIFieSelr3b1pixQWYxzHXtpoJrfNz91lCvFT", - "wFW+MZrOk5wCHWEa7hXY1dTdd4M0kzPzBPxYQwXnmWYDmrxi/fzaM+kTYBJG8m/Ug/xy3YvULDaaxlzj", - "dIEyRn/LKkJzH51r+V8hzfxpRKIuwvBCs2GcKd6bEkaE5lNoIngCElRJsEUd0p/2u2ioZb2elmx7eLc3", - "GPQGw6Aqmsb7vWmaaVRgpYjQAP7/X3Dv/XHvH4Peo9fF46jfe/2X//ARQFtp20l6dp4dt/e7yAFbFsHr", - "gK4Wz1dIuD4uYpbvXO/9TVfv5Hz5gDfwRzy8JaJP+XZMxwKLxTabUvbuKMaKSFWdzeq2a+cHsK2YGJvq", - "qW84tZrCAeTWiflbIkLNKWOiCUR2NbOkSnYR1jorMBmkT7P/QiFmmmbNwc4FIixCb6maIQztqhhIFj2c", - "0h41oAbdIMHvnhE2VbPg6GBviR41MXbsQ+/1f7qftv7bS5Iii4mHGF/yTFE2RfDanL4zKlEBA1UkWXvc", - "OuxmMYhYCWXn5rOdHBIsBF74V80Bt2r1jHLUuHxh4pGkX8yJEDRyJ9rJxSnqxPSWWLJEImNomA0GeyE0", - "gEdifwl5kmAWmd+2+uhFQpU+SbLigDTWlX55CX8JSDjjcMbHMdcTytHXIEA4vDhF07NEp84yIZHVduFM", - "w2B3giU7u7ze1lwlxVKqmeDZdFaFyrK0zeCh8nZE+Wic+mCi8hadb79AmuGimGrs5Ax2ZzC4eLwth4H+", - "44H7Y6uPTg3KAHy9flxYvi9nWBCQPiLEGTq5vEY4jnlo9bmJFhIndJoJEvVrZgTo3UfwhCmxSDn1CZ81", - "yiiaLhNIr1e83YAOtseUbUu9DL1wM7wTNv8MEegJm1PBWaLF0DkWVPOtilHnQ/D8xemT0ZPnN8GR3kRR", - "FloLyeWLl6+Co2BvMBgEPilDU9AaPnB2eX0CK6Xbz7hK42w6kvS9h7Ue5/NDCUm4MKK//QZ1ZlXOayQj", - "BIszDPbOHhvi2jkDunKLElEJrV0vpuMqxeyePfZRy2yREjGn0qfz/5y/cytf4pOGMVVpWxIxJyInWqDi", - "fknuCmOeRb3SkN3gN5JokWP+XhNLAa2npV/5bnUsrzlvcZxSRhoP3G6QEIXB8Pzp9HktiehFZEK1unFL", - "Fr05jjOCXM8WsyRHbJV000ykXJr+8dSIqYrgJDgKxji8JSzyUu53cri/5eI25jjq7Xzhs50RpftenuJz", - "86JKiT4c1xVGFr2lkZqNIv6WaZA9J4B9g/LG+THwTs8Ex7//8183F4Uku3M2Tu2ZsLP74DPPhNopoLv2", - "aqn5RLLUP43r1D+Jm4vf//kvN5NvOwnCNH1GlRsdY/ipTuXvM6JmRJRkA7fA+iejZsDnyNFLafiKJal8", - "/bO0mficiBgvSmzdwhTsDIC31qASVMH+st9pJn2L9MdrmLzuzYkQZ3XVZ3fgZ+MeoDwwPdb72546bSDJ", - "AdnZvbCPu8sgNUB0S9PRVEutIzzNLWGrLuaubmmK4IsefGGWMY7N5o0y3TMac676Q/b3GWEI1g4WmLwj", - "IfApreqj48tzid7SOAa9GRjB8sE1ZK9KrMA0l0r/V2Ssi8aZQoIkXBFkRWIYJANYoPGYoIxhd/PXH7Iy", - "VuwE63Rl0XJLBCPxaEZwRIRsiRnzEbIfNSIHpjrBUhFhOHSWVvF1+reLK9Q5XTCc0BD9zfR6waMsJugq", - "S/Ue3qpirztkqSBzwkBj0uIPtePyCeKZ6vFJTwlCHIgJdJZbHuy11Pzs8tpebMqt/pC9JBqxhEUkApjd", - "KSGRmmGFIs7+rHcsHJelbsvj15Du38vdYB6mWRXLu3UMP4frRD2fORUqw7FmWRX50Xu7aO6tPXqCuRYv", - "6yuWFeUEh1X1Wqityml6hkvsZSnar2UaQalZy1xzh++7rMktV2EmFU9KVzaoUzNK0ar5qso85jzuafkH", - "RIPl890rvxhwl68/k4XpyixKE5ccTcceS6dmhpShKZ3i8UJVNYedwfLS+xHt+vehusk1wJAHiUaKr74c", - "pRPk2ra5CwFHgpHio/mEenrOD83CCkclCmt+CJZodRe9NKR2+3bR2xnVx6xEDgmwg28uypp4f8h6wHKO", - "0Gk+QN5t3qXmrGBxhS46XJSAoGA8R+PFFsLo5qKPXuXQ/lkihhWdE+crMcMSjQlhKAPxjEQwPrDTMgCZ", - "1DyMqvrnllcZt4otMDhw+66PtCKUYMv3NXknWNEQDLZjWpsPXJSZhdIjaQbAyqdOq1Ni1ZXySzKlUona", - "hTLqvHx6sre396guL+w+6A12ejsPXu0Mjgb6//9of/f85T1HfH0dV/mFNYGXOcrJ9fnprhVOquOo9/v4", - "0eG7d1g9OqBv5aP3yVhMf93Dd+Jb4mdPp4XtHnUyrfY51qepymexLxnGGyzyn2xo38itxV3trTp+zOxe", - "6ZZfwxHGdx1rLwM3d1WpM8G1F7qlyS3NR/+q5YOC8kuGDXtvElLvDdEplbePBcG3Wqv0nK/6eJYjc+74", - "LY6Z1qPGC0TeafGMREhwribS2DmqYsrO/sP9w72D/cPBwOP/sUzEPKSjUJ8qrQB4cXKOYrwgAsE3qAOK", - "XoTGMR9XiffB3sHhw8Gjnd22cBg1qR0ecinKfYU6FiN/cb6E7k0FqN3dhwd7e3uDg4Pd/VZQWQGvFVBO", - "GKyIDg/3Hu7vHO7ut8KCT+184vxx6v4Fkc+4mKYxNUp2T6YkpBMaIvDoQfoD1EngWCK5xlfdk2McjYQV", - "A73ngcI0littmmYw29K4byVZrGgaE/MOFqSVpAszP4WefPZiyhgRo9xdaYOerBfTWsuYm0veBFW80Sqo", - "u6ASJItCIKIkjo7MDl3L52A1C8BeN9GBnUNLanjG3xLRi8mcxGUiMMeRBjbhgqCcTsyiVWZF2RzHNBpR", - "lmYNltEGVD7NBMiXplOExzxTRlWHBSsPAnevoCNMNLtud/X/lIvbtbdj/gP6uT6ZnS1wwsWtVlIcQ7kj", - "Q2hvYuxQX84a2vZauLiqWEKMVtA3tJimgk9o7EEyaPv2rZWFnC3x2f7gqrfzf80NBosXhoFSZiwECY9I", - "v+YpDO3b0cXZ5fVlE0y5mzYqQ7c0p9ym4yGd3EzgMGKtFSFmaEyQlS8MEYC9qRikOBkf+U6aicAJGWeT", - "CRGjxKOiPtXvkWlgjHeUoYvH1dNGn2pt5dTLyuKAoDrBofWybYd9jwpcm0a3hM0GanxJjGNRkx+PXiph", - "21hXnj56njvGo7PLa4kKO5xHN64ub+N98+VsIbVWZ3o0bnmUlVVaIM7W59dl8aFV/j2nWOLl3G4joM58", - "mmawDa9e9s5f3GwnEZl3KzCB7WzGY6Lh3ioJpXPnzVNcjlfu2OZNuoUhDNl2A5Vwle/g1kgq7VcPdhRX", - "OB7JmCsPNK/0SwQvUefmqfHi0BB0UVpZSv17CQsV+j7w7hjNkZqGvYIB60aKygZfay9KzHlfnl5lUN9W", - "+Zng2ITRVOm5cAx1C89vqwvNb9fuXtuJb9xzd83awvPk5OLUqLQhZwpTRkR+w1l1EwAXp6Ab9LQUFWGS", - "gDF58l+rXQYajF45uawym5ws+eB/FZNJg5+pZnLxnEQowYxOiFTWz7Qyspzh3QcHR8bDPSKT/QcH/X5/", - "Ux+PJ4VTR6ul2DYX4iV3j76cfd46fAVXjjZz+RBcHr/6OTgKtjMptmMe4nhbjik7Kv2d/1m8gAfz55gy", - "70V6q6AIOlkKhqgsb6rPLPP7kZ4JI2FOkBzE67VG3QaBVpNmTN+TCHl9EhWeaunUUNznOR9+RhhBEcum", - "SuED5fuVFqEE9P1qO4UTjKCNHTNjisZFlMWyheKT4mTkSrfjJZfjlLDc0TiOzVPI2VzvCp/XcYWBu3dL", - "i6EVCsqmo4h6qPPv5iWKqCChAk+j9Xso2MZpup4U/cJfztPaRlBY/0nP6fLNOfmnWKqro7+Y/vW3/ycv", - "H/6689uzm5v/mZ/99fQ5/Z+b+PJF+7suj6/OatfZb+r/uvIyEsyzFb/XtuRxgVXoEXxmXKoGrNk3SHGU", - "6I/76AQUtKMh66FnVBGB4yM0DHBK+xaZ/ZAnwwB1yDscKvMV4gzpruyd+5b++NJo6PrjD04H/FjvI7KX", - "68IiOfeDkdk44gmmbGvIhsz2hdxEJNx26acIhThVmSB6RbSsGS/QWOCwuFQvBu+iDzhNP24NGWii5J0S", - "egYpFir3s3cjwEJbqMxtmm1OIgQOadJqskOWnx+gmutOFBZTovq5qQQsXLUbrQakeNUMLqpOIYeDrmcd", - "kW6nFzKmUhGGcqsElUC8qOO8ew4Hle1/ODhcf3Gb09AK8gPqXo5sd0TZYn8YAoahDTMezZRK1/t9AL8x", - "ewT9/OrVpUaD/vcKuY4KXORLbJQxnKYxJdJcR6oYZBLrULUV+K4czeq2nNAr01h/FrfwX3kCA6NXz66Q", - "IiKhzPDvTqjROaGhnh9cjFEpM02KFKPjk4snW/0WofmA2xz+Fev4Kp9h7f7FGbeWNUz4ojAOavx20flp", - "V4tTdocWghZcOD/lAsWGwRT7+ghdS1J1/4ClMndjZiXjRWEhM1x9GGy5HtM6pzhCL3P5Dueg5PE/BTG4", - "Lot9Cd1ajyBzG77Ue7cKK9zzW/3Fsja4+8YK2dsCOIqbWcHq7e/BOOx5zho9Ylvt7bLRUg/mJ41i7b+6", - "BLK3qS65aSBC1Zuv5L2ZxyK0DyL4Gs74y3rVO6pGjVdZSL+2F1dOe7i5QDMs2Z8VvKzpEDt7D1uFuOtR", - "214Cla9/+MSAlO8q5xqYX14YJ8lbGsfmTlDSKcMxeoQ6V+dnfzt/9mwL9dCLFxf1pVj1hW99WsQkONI+", - "u7wGR38sR5LhVM64anYHwsi10TOVSi57TrZyYFkdA/FzJU7B64q69QWDF0TGGPji1KfxdcISvqWLy79R", - "SETwOQENK0MQPjeOwArVXymMoJGJ+1zwq/zc/PxlAwK+CjgV134fHyrLHs7r8ZO9+bsB9Xh8HUvNakmE", - "zi+LWOnCSOW6r83p0W5/5+CwvzMY9HcGbUx2CQ5XjH1xfNJ+8MGuMWIc4fFRGB2RyWeYDC1hGyERx2/x", - "QqKhE+OHgdEbSgpDiVlYUb/Vdexy0MSnxUjUBZd1URCbRD20C2dYkcTkqpq+pLUs+OAfn5XphLSVAK6g", - "sftqtIkxm6CQZ3Gk5a2x3nlGfSOR1TIlUUVmGNis1+yW8besOnVj09T797eMiAW6ubioWMAFmdgkGS0m", - "ztO0cR14utEy7K4RyddCU4osuItogjonLJ1AXzx2oGyuc05MhupamO0KydN7tU2ZQbde+xVzqhlcIjIf", - "ZZlPvNKvnDvy9fX5aWXBMT7YORwcPuodjncOevvRYKeHd/YOersP8GCyFz7ca0gn1d615dO9Vao7tNn9", - "HxAPxksTsREd6T2Uu5uMM4VyRyS9OU+0nIpKIrBxdgd7wksjDese4HQN9Zt4kUvJKz++xHqjum9T+Gv1", - "F1ezTGkxCL6Rs0wh/ReArKdgtYzVXZg9f4Sec/jGQtrVB2VNXTHNMYvGi+XmddWmY712BJGKCxLBYJaB", - "HaGnOdPK2Z5lcx1J7KPhpdYtEFwet4wRxGoWdrWCbmCxHnQDg8KgGzjM6EczQ3gC4INuYAHxehSX6cZn", - "3Cc4Bh5WOM5kisb0vdlyGnQqFQ2NdodhNZu2nQ3bJNHIHKFN12/GG8Mes/lHblffXKAOBNn8BVnlT/+1", - "lV/VlbfQ/u6j/UcHD3cfHbRy0S0AXM+NT8BXaBm4taw5TLORS6vXMPWTy2s4fPTBJrPEaPN27oXGqBlH", - "qKU9ylCRp68Y/FH/UdkzOeLZOC5Zh2xoAri/tkmq2HA39RuN53QyYb+9D293fxU02Xl3IHfHXuUoH8gv", - "SZ6XLZpLahcZ90xSAL8OCQQlZKN/9UsiYQboiigE9NPTDEufqLmLjyU554VtMe4lrP29vb3Dhw92W9GV", - "ha60cUag/y1DeWEhKG0xaIk6L6+u0HaJ4Eyfzu8xFUTqyZmQIe8+QzY5y6DiEql1jz0flTQILAXV2L7n", - "SSPKb6zEYidlkQ6eSrk0s7TLvdje2xs83H9w+KDdNrYaz0i8W81hbDt7wy9ISOi8svIdsIK/Or5Euncx", - "wWFVwt/Z3dt/cPDwcCOo1EZQKYGZTKhSGwF2+PDgwf7e7k67QAGfpduGwFQ2bJV3eTadhyg8q+FBxTLr", - "7TadFj4pcdk9cqVHZuHiWffn28SBtwh7pBJ6pSXfUdTRQlRZIC2F7m21sTP4WaQepylZrxYX2/rWrnal", - "vcRqds4mfPkqYxOFzzooORN3qgUfCWkMI8IoiRzvyjU/K0uBy1MsCYoyYjFnZCOBLcKxuc5JsZqBsAof", - "UjatOnsvDdhGDTMwrA5yhXFtwzYWI+l3qnklMsCVsSVLhAv3mlaGcSpHfq1iuWNBplmMBar7j68AWS6S", - "mLLbNr3LRTLmMQ2R/qCuzk94HPO3I/1K/gRz2Wo1O/3BqLhJrqnnBjjrR2AWpDZuMYWf9Cy3ap5JcPJv", - "m++3IRt7GwOc93rpqVbejIv1NaPvSoRejRjb3x00OaI1dFpxQVt2z9+Ut1uS9e145zl/nGdW8Vxjmoui", - "mgZblYMr8/XNFm4iV7ndLUsCqONsei4ir4rXUmRcq4O43WVo3XrtoNmWJKyOvn/44OFBy9DEzxK1V+Sr", - "/gzBep6sEKgbVuqijdR2+ODw0aO9/QePdjeSj9xFR8P6NF12lNenlkCpJrM9GMD/NgLKXHX4QWq47qgC", - "VEmG9MkAfVyxdYuglwate1WtiGIlnZpfFcDbibgrpKXjishVykzYIZMJAcPRyOCtVwBTc8JqBUOIUxxS", - "tfBogPgt+KWgvEkteKNF7zVgPSi1fSM8UUTAbYTMxsW9f8cNjv7TaHY1WjhsHeEss3GTFvmiPqrRIY0j", - "V1SzULQwEBiK8F2+v82Rid5iWbHq6+dQkahbyjxZv/4xLdrnBne0nqcHL67TfQFI/lTg5eWvLWdJ66gI", - "yXWMrzpCm7eglgjAS6yNgd1zInuimsL1zhs1/mAPwE/7ajQu5x5YmdyhkqigOHU3H7ddzszl78wJtvl4", - "pRv8TT6sh2EDPVoYLMqLvrsVkvBRk7lfacrxk7giSrUobWrKUtiIMlRqjDokSdXCRU04zXRrs/ue47xD", - "LzF+YT+3waMv4Wl/vdK1/t8ka1T5is0NsvZybWlNG/1Z/eLqad19xeiENmtG1d2ilgtAqhXVV1ZV+jIl", - "t0Dhs77k06we/LZBda8mFb/YOa6siivvtU5zXWlPK82sBEnz2pj71c8shUalq4H2iSiz6td652xzR6UV", - "4F49rYqJLRYU9DmLIINYjYJcRV+2A6x2+7jA7/IRQFvGEtUSUZp5lFJrnz2GfAEvXXoNOnFdABj1lKKP", - "P69GnKOq5cVYVTTO3eB7N57lPys4WtPeqhFnMUZ3dV06zbpImAmqFlf6QLDOaQQLIo4zQ4ZwUsAk4Odi", - "cAhQ+PgR1NSJR1o9I4wIGqLjy3OgkgQzSEyMbi5QTCckXIQxsf7lS3e7kB7hxcl5zwTG5JkfoeSLAoS4", - "lGvHl+eQ7ckWWwkG/d0+pOnmKWE4pcFRsNffgXxWGg0wxW2IO4RHa4jS+xBOsvPInriPTRONWplyJg1y", - "dgeDWvEeXGTU2f5VGguLOV5bC4WmOtqyv8WSQ6STBCz4H7vB/mBnI3jWJsHxDXvNcKZmXND3BMB8sCES", - "PmnQc2a0apcznNiGBc0GR79UqfWX1x9fdwOZJQnWIqJBV4GrlMsmEYZIhBEjb21A6q983EdXRicB7/Ki", - "7qQxGZBIsySMFBb96XuERTijczJklhObhEZYQPRNgjQHNrEPVTIzQ5vVN1uYSPWYR4sadvPutnV3PedV", - "WyB446pGeXbOtKG8kY87miRgMuTe7GeEYaaKnFIm+9ctgUvMCX3n7bDVbbxmHuWCf867c3fLbwcEV2W/", - "Cf00f+eKa1UPDC1DUxbGWVScqtWiRt5ge1OcxyZJuyUeIeQMWliklL263fHFeESMr2y6UDPOzHM2zpjK", - "zPNY8LeSCH3I2dgci2uti+dFEU0GSppAfIyJ5tVjbhsQtz/cksXH/pAdR4mLvraJkHEsuc0eZ7weqER5", - "Ou4hay6n5xemT2zWV5O5qZxPyYDJM5Vmqo/MRIiyAUXQnEqUZnJGoiFTHH0QJp3l4uP2h2LEjyCdEhxp", - "Oik1MVPa/kCjj01QyxHWsx+NXU3KmsxOAAHDQEsNw0A/TwXW0mkmZwiH4JuhfywvacdsbC7g5N+qYzjE", - "DKU8zWItRwFRmaR4lT4gOBPHMVKwldy3Wp6AlWyYjzXp+jICWXuuMcDVthHkBiptpsH+oX8/SRIK4lNL", - "/3r14jmCowpqf0GzInwAcESZFjTy5NN69P6QPcHhDBkZBJKyDgMaDYOixtMWwJpJYsSAXg+EmJ+g+J0Z", - "pkujn/p93ZWRj47QLx9ML0d6L6XJSPFbwobBxy4qvZhSNcvG+bvXfoQ2mcWuKowAdQzv33Lh8nqGpWPQ", - "nBuYRYhbXhsvEEYFByprv2PKsFg0FUzjmWp2LTLZBGyzYj0PBoOt9VdHdqoeybDSUO+Ej0uC0O4XkwGs", - "/LMsA5SKo+oTl9lUEZGRfO5ACHmMIxfB+EPaWiNtWTWxJEfB92WWbMg3JsaVtSYMQQ09JwylWOCEKEh+", - "/4uf5sGLl+q/3UUvnETGaFIl3m4JPXXd6fUSYe83FifMy/wBLezfAf3BuEXmQxj30V2Ni2OTdzsvmHyv", - "yBEWyxFi16/onRH1PVDc4K5YqUvQ+g3p977QzxmxIliBtBo324aKF2UrQj3aRBCcSNuLaazVxiuAqXdF", - "mEJQFlf27b9O+QBH/jcxn745QgaFsS0KLG3myNzcrg9Fi0v4yCTWyb+z+abCGWZTIlHHnJ+///NfrrDp", - "7//8ly1s+vs//wXbfduW6Ybu8pK8b47Q3whJezimc+ImA86pZE7EAu0NbI0feOXJXiWHbMheEpUJJnPX", - "Lj0vwInpELQABvOhLCMSSUAhJOSfWJ8jY83zaNNuLxtU3umO7i6pP3YGpQnoU9HRAFwiU0YVxbFVhRwc", - "EP9UAGLmHJQHrxsml0zV6/mLIu+Uod6eAXBDBmNKWnv2nanybPpEnaurJ1t9BOK+oQrwKwO9oejGagL9", - "HzxpPU8yHKXKUADLhjeVEpc2mjVPbZu7sGs2JTVtNmwaRZ5o3dhN5ofY3cLI6cebM3j6rI6nrkJBs9nx", - "0+frq3jdSqf8cuvsaG8Z57b8RoGyb6FNoo7NnJ7n+anU+PhWRH8nDLhUGibnwoib7EJ3puGccDaJaahQ", - "z8FiSwjnWk+VQO4LO3hpoUbYzaseDFE+KrYrvn2Nh0bu5neXp0dt0E2OkSJgo6C1HyfJOtI5pTLk+tsS", - "tfRCnNosR4DEYp+WqWidbecUfs+PnJWCeV7U223Iu7Py2KEzVj8b7oApntYY4jdkhLWMKqUQp/tEzdf5", - "KrpqTCuMQN8XaQ7uTgq6a4OQj8zvk0UoqqFNc8FZnne/ibxsZv6vuNB2BM/Er4hwu9oAajJ5FNMyn6Jw", - "RsJbMyFbNGyVRHDu6op9fTnAlBfY4PS34P847lsojgWuVimL5za9y9fTFWGEjVTFL3f9aAnMg2RwRxg7", - "Q6rJnILlgoVbf6gbyDs5GepFvu7RTrrM4tgZ4udEqKLIQpmfbn8Ax5X1crLbbStlkeuXz3qEhRw8lXIv", - "G79A4nKqf1lp2SyYmcoPMmmjXwGqHGE0C6Ofsf7GoQzl6Tr/tPvUJuz80+5Tk7LzT3vHJmnn1lcjlsFd", - "sea7ll7vMfFp4ZVWkQasyWQ+Xyft5a3uROCzJSY2EflyAH9IfW2kvjK6Vgp+ebWPryj62SIK3+aeICc2", - "H7bhlfM/+4OJfHdrerIUWaqLWbHF2xwyXBSFC2xVvfvnIEdziivz35Y21GJDrpQOHOmen3ZtTQpTSSL3", - "xb8ji6qD486lRDvu3ZtTj5MxnWY8k2U3fyhBQmRR6LrCgO+b/Focz40S7HdMpYO7PDruXED9QfdfSXSu", - "L6hh3rbE8xrh2bVasx2e0liVMttLKPZh8smaMKGXLk+tzQa71eA05rIwtyXjShLwZWc2H1yuBkOpLEOK", - "qZB9dC2JRhNJX5gID6kWMTkasv92n/yiCE5e/zTG4S1h0TAbDHYP8neEzV//JBVEig7ZhaMbwpSgRCIs", - "CDp+fgoXU1OIwO2j4zguQqHq8KAkk7ZYm6s+lMaQ78twDh/6SvUlCgy2LmGxFPMNCIC8KA4nwWezpZZ6", - "VHFr116RcsT6Q5FqpUiV0LVakcqzJ39NTcoM8s1UKUdvPoTbxAE/lKm7UKZkNpnQkBKmisxfSw5ONnHg", - "PQwxYvY+puSYUDmPWytTRUrz1XKqJd5v4ZSSD373OpRLL3g/XaW5CY6InNZSHIbNasv3Rg+Du2XOd6+u", - "3GcSOytX/vQrBiZOaGITj/kFhKdc3LalPBuDTL8mAX556aQ8w+9QNtHgQb6Hby+iwOFt/MI10VQllzvY", - "kHX6+qbuiA4TRgfmwuWFBiEmtO2+NcPQo9+BSPmcK0STNCYJgSxTPUMfUNMnS1Mu8koCVJaKiWzG0PRG", - "KLt7mgQitqCWXgBpas+YFfGyuphP1wdE5v276D9PROSQuYrIb4z8+gblnBEpjiSJSajQ2xkNZxAdqX+D", - "/k3wJE7TN3k6hK0jdAbbq5ygAQbvSCIojqEkC49NNaE38yR5c7SclOrm4gI+MoGRJv3UmyPkElHlXF3q", - "VuVoRz2LGEuFntsYzo4mFsFdRdc3+ugozW/LxkEWmSOGzBcTychb2yGdoDel8Mg3DfGRjgs+06v0jYSc", - "bnO+HzMXxZEAxBnyIyxqMHNprPkjI3cG3tyKLaM0DRhfOUhzCZhnfJrnGqqQMk7TtuRrwQQqnifJChpG", - "nVLFIakinqm/SBURIeBjS91NxI06ODR/KHxrSu5XSueaGlde06TJOOJFleabpdJY5q95kgSmjm+CfaWu", - "Pj/atd7hsm1Mr0wppPWHeLxJsGqV2ZeiVWsnh62x1iwnvzQN/vBKmitG943J8BsYtQooKMtlEb22RZW/", - "+xWqBwu5JG6Z+n6+PeLeNe4RWxzwD79HCvr4g++SkAthbuFshd/741NdMq6UtnsHSooWpTq7zsB3c3Gx", - "1bRphFq5ZcT3Yfn7NMNLrZpPEvkLRwgaufyKJxenNhsjlUhkrI9eJBSSHt4SkkIKF8ozieAatV9OJd90", - "2ZnniidMiUXKKVNroSiafh1gPn5SUrs75lM2quQPf5QbC8O9Y1KmrjvOJ7DqOkrzIdVoGnGmgkrh6zHP", - "dO9LafGh4JtcSEUSYyeZZDFsIojhs9mScLmgXRdRJaG8ShcuRUrFzIZsTCZaDEmJ0GPrzyFDbqHy+awJ", - "VwrnXPPSsL7vw5wAmfJBg8aqCWu1qnFp6pLk+1TWPK//J4P0FOwD1YJ6EnViemuqRKO5RLF+2FppYDDV", - "9r50LqhP31l5PUlfjg9Dszkx/xE43HmNrbl66feOrZ2R8mZx/AcW2s/W5Fq+JjYsOO5wVyo83h+yC6KE", - "boMFQSGPY6gzZdSm7VTwcBuKIYcpjUxVZAAOGF7z6wRGPLm8hnYmm3J3yPQfy+V464C6qr7n2y/WmFxN", - "IfZ/Y33MTHDVtvAv+A9r2uaXzY17SDZsUZ6uUoB4+oc3GFgJ7oe14H5aC8DbJ59NZypwCEKxnGUq4m+Z", - "3zJgy85ufzAP5+t8xhQOZzeuCtj3Ie3aokHrhnETvBeb0s4pIiYH0d3vSZ7XdbqnceYacW4KIMSUvd/8", - "p4CpF/dHo+4v7+tUxuNGvk53urdcfq/vZm/d9clnYXCu0GV83JdtbijNzQSqrZStT6JcuHalbubqikIV", - "5Vy0dPV0u+WyziadeG5DKuoB5hVk+0OWl8x16cy1dtV1qhWKqLw1PVjtqY/8lY2NnmfLG0MtoBDHoSmi", - "k5f4NdVtZIP29bJU9vqr7bdiEM9C57WNZV6K9j6pHH6agNUr17oFirPi1MpguBvb5i7in+xhtkH0k5vB", - "j9inFrFPJWS1qaxnChVbbmUrzBZhe5K+h9sdX+xULpR8vcipTzivvxx5ODptPK1/xEzdmUBQ5J84P73/", - "gVLlPVfh0dtaK+jZspVl09CqHWxRlArSc8XsIoMwiw+ja9SrYvaH7NWMuL8QdR6sJEIRFSRU8QJRBjUH", - "XXHjP0skOFf2PReL5uqZZos8FTw5trNZo7y0LvPtu4jZODlW11PamCZZYsoaU4bOHqMOeaeEcahEE0xj", - "cOd1KCXvQkIiCTS5VS8f7vWwzOuEr4VyhWtsXiA0NIUc57km1sGZ4r0pYXotirKZqeBzGpmY9IY67D5o", - "QUP8Akra9D1Nq1tvbXG95Y1XpVuU1wm11f0K+nSrE/w4JurlDCApABc5EhXnKMZiSrZ+HCX3+SgpW5Pc", - "uVE5UdqF3LYzMLW0+3yNcNvc+Hi3wbY3349NpJT+/R5mJ5rnSl9TlO/3RYKDuzsf7jq69+Ye29DPiFNw", - "S5G90IHu0Ucwz3iIYxSROYl5mkAZNGgbdINMxMFRMFMqPdrejnW7GZfq6HBwOAg+vv74vwEAAP//vy38", - "JkPxAAA=", + "H4sIAAAAAAAC/+y9+3LbOLIw/ioo/s6plc9KsnyJ4/jU1CnHnni8Gyf+xbH3OzvKp0AkJGFMAhwAlKOk", + "8u8+wD7iPslXaAC8CZSoXJx4J1tbE5kEcWl0N7obffkQhDxJOSNMyeDoQyDDGUkw/DxWCoezGx5nCXlF", + "fs+IVPpxKnhKhKIEGiU8Y2qUYjXTf0VEhoKminIWHAWXWM3Q3YwIgubQC5IznsURGhME35Eo6AbkHU7S", + "mARHwXbC1HaEFQ66gVqk+pFUgrJp8LEbCIIjzuKFGWaCs1gFRxMcS9KtDXuhu0ZYIv1JD77J+xtzHhPM", + "go/Q4+8ZFSQKjn4tL+NN3piPfyOh0oMfzzGN8Tgmp2ROQ7IMhjATgjA1igSdE7EMihPzPl6gMc9YhEw7", + "1GFZHCM6QYwzslUBBpvTiGpI6CZ66OBIiYx4IBPBnEY08uzAyTkyr9H5KerMyLvqILuPx4dBc5cMJ2S5", + "01+yBLOeBq6elusf2pb7fr7v65nyJMlGU8GzdLnn85cXF9cIXiKWJWMiyj0e7ub9UabIlAjdYRrSEY4i", + "QaT0r9+9LM9tMBgMjvDu0WDQH/hmOScs4qIRpOa1H6Q7g4is6LIVSG3/SyB9cXN+en6MTrhIucDw7dJI", + "NcQug6e8rjLaVHfFh/9PMxpHy1g/1o+JGFEmFWYNOHhuX2pw8QlSM4Lsd+jmAnUmXKCIjLPplLLpVht8", + "1wwrJopEI6yWh4OpItuGcoYUTYhUOEmDbjDhItEfBRFWpKfftBpQELxmON2i1WDLpJaZnRwlsql31wRR", + "hhIax1SSkLNIlsegTB3sNy+mRDBECO7hUD/rxyghUuIpQR3NNjXvZkgqrDKJqEQTTGMStdojHyKYxfzG", + "x4hGhCk6oVX6NujUw+NwZ3fPyzsSPCWjiE7tSVTt/hSeaxTT/SgErf0L0YS2aLcOGFKQyfJ4z4B1wyCC", + "TIggGsc/c7hU8Dlhmlr0eP8B4wb/33ZxRG/b83kbgHlZNP/YDX7PSEZGKZfUzHCJc9k3Go0A1Ai+8M8Z", + "Xq3a6xJGSYXFavqAFl+AEs38WsHmyjSt80Ngd7abCmU3sr2f54R5BJ+QM2VfVFf8nE9RTBlBtoWFr+Zz", + "eoCfYg5s7kusrRsUIF0maD3vT2BI5kFDb/pdNyAsSzQwYz4tQ3NGsFBjUgFmw7FkOypm1wj+ywpJ1M4f", + "LMloNVe4pIyRCOmWllhNS5RJkD6Xlg+UcUvVaE6E9NIRTOuvVCHborGrmIe3ExqT0QzLmZkxjiKgQRxf", + "VlbikcAqIi1ONWNzHYJkIJHi6OqX491HB8gO4IGh5JkIzQyWV1L6Wndv2iKFxRjHsRc3mtFt83N3GUP8", + "GHCVE0bTeZJjoENMw70Cu5u6+26QZnJmfgE/1rOC80yzAY1esf79xrPoE2ASRvJv1IP8ct3L1Gw2msZc", + "w3SBMkZ/zypCcx+da/lfIc38aUSiLsLwQrNhnCnemxJGhOZTaCJ4AhJUSbBFHdKf9rtoqGW9npZse3i3", + "Nxj0BsOgKprG+71pmmlQYKWI0BP8v7/i3vvj3t8HvSdvip+jfu/Nn//DhwBtpW0n6dl1dhztd5GbbFkE", + "r090tXi+QsL1cRGzfeea9jfdvZPz5QPezD/i4S0Rfcq3YzoWWCy22ZSyd0cxVkSq6mpWt127PpjbioWx", + "qV76hkurKRyAbp2Y3xERak4ZE40gsquZJVWyi7DWWYHJIH2a/TcKMdM4aw52LhBhEbqjaoYwtKtCIFn0", + "cEp71Ew16AYJfvecsKmaBUcHe0v4qJGxY3/03vyXe7T1P16UFFlMPMj4imeKsimC1+b0nVGJijlQRZK1", + "x62DbhaDiJVQdm4+28lngoXAC/+uucmt2j2jHDVuX5h4JOmXcyIEjdyJdnJxijoxvSUWLZHIGBpmg8Fe", + "CA3gJ7FPQp4kmEXm2VYfvUyo0idJVhyQxrrSL2/hrwEJZxzO+DjmekE5+BoECAcXp2h6tujUWSYkstou", + "nGkY7E6wZWeX19uaq6RYSjUTPJvOqrOyLG2z+VB5O6J8NE59c6LyFp1vv0Sa4aKYaujkDHZnMLh4ui2H", + "gf7jkftjq49ODchg+nr/uLB8X86wICB9RIgzdHJ5jXAc89DqcxMtJE7oNBMk6tfMCNC7D+EJU2KRcuoT", + "PmuYUTRdRpBer3i7AR5sjynblnobeuFmcCds/hki0M9sTgVniRZD51hQzbcqRp0PwYuXpz+Pfn5xExxp", + "Ioqy0FpILl++eh0cBXuDwSDwSRkag9bwgbPL6xPYKd1+xlUaZ9ORpO89rPU4Xx9KSMKFEf3tN6gzq3Je", + "Ixkh2JxhsHf21CDXzhnglduUiEpo7XoxHVcxZvfsqQ9bZouUiDmVPp3/l/yd2/kSnzSMqYrbkog5ETnS", + "Ahb3S3JXGPMs6pWG7Aa/k0SLHPP3GlmK2Xpa+pXvVsfymvMWxyllpPHA7QYJURgMz5+On9eSiF5EJlSr", + "G7dk0ZvjOCPI9WwhS3LAVlE3zUTKpekfT42YqghOgqNgjMNbwiIv5n4nh/sdF7cxx1Fv5wuf7Ywo3ffy", + "El+YF1VM9MG4rjCy6I5GajaK+B3TU/acAPYNyhvnx8A7vRIc/+sf/7y5KCTZnbNxas+End1Hn3km1E4B", + "3bVXS80XkqX+ZVyn/kXcXPzrH/90K/m2iyBM42dUudExhp/qUv42I2pGREk2cBusHxk1Az5HDl9Kw1cs", + "SeXrnyVi4nMiYrwosXU7p2BnALy1NitBFdCX/U4z6VukP17D5HVvToQ4q6s+uwM/G/dMyjOnp5q+7anT", + "Zib5RHZ2L+zP3eUpNczolqajqZZaR3iaW8JWXcxd3dIUwRc9+MJsYxwb4o0y3TMac676Q/a3GWEI9g42", + "mLwjIfApreqj48tzie5oHIPeDIxg+eAastclVmCaS6X/KzLWReNMIUESrgiyIjEMksFcoPGYoIxhd/PX", + "H7IyVOwC63hlwXJLBCPxaEZwRIRsCRnzEbIfNQIHljrBUhFhOHSWVuF1+teLK9Q5XTCc0BD91fR6waMs", + "JugqSzUNb1Wh1x2yVJA5YaAxafGH2nH5BPFM9fikpwQhbooJdJZbHuy11Pzs8tpebMqt/pC9IhqwhEUk", + "gjm7U0IiNcMKRZz9SVMsHJelbsvj14Dup+VuMA/TrArl3TqEX8B1ol7PnAqV4VizrIr86L1dNPfWHj3B", + "XIuX9RXLinKEw6p6LdRW5TQ9wyX2shTt1zKNoNSsZa65w/dd1uSWqzCTiielKxvUqRmlaNV8VWUecx73", + "tPwDosHy+e6VX8x0l68/k4XpymxKE5ccTcceS6dmhpShKZ3i8UJVNYedwfLW+wHt+veBusk1wKAHiUaK", + "r74cpRPk2ra5CwFHgpHio/mEenrOD83CCkclCmt+CBZpdRe9NKSWfLvobkb1MSuRAwJQ8M1FWRPvD1kP", + "WM4ROs0HyLvNu9ScFSyu0EWHi9IkKBjP0XixhTC6ueij1/ls/yQRw4rOifOVmGGJxoQwlIF4RiIYH9hp", + "eQKZ1DyMqvrnllcZt4otMDhw+66PtCKUYMv3NXonWNEQDLZjWlsPXJSZjdIjaQbAyqdOq1Ni1ZXyKzKl", + "UonahTLqvHp2sre396QuL+w+6g12ejuPXu8Mjgb6/39vf/f85T1HfH0dV/mFNYGXOcrJ9fnprhVOquOo", + "9/v4yeG7d1g9OaB38sn7ZCymv+3he/Et8bOn08J2jzqZVvsc69NY5bPYlwzjDRb5Tza0b+TW4q72Vh0/", + "ZnWvdcuv4Qjju461l4Gbu6rUmeDaC93S4pbWo59q+aDA/JJhw96bhNR7Q3RK5e1TQfCt1io956s+nuXI", + "nDt+i2Om9ajxApF3WjwjERKcq4k0do6qmLKz/3j/cO9g/3Aw8Ph/LCMxD+ko1KdKqwm8PDlHMV4QgeAb", + "1AFFL0LjmI+ryPto7+Dw8eDJzm7beRg1qR0ccinKfYU6FiJ/dr6E7k1lUru7jw/29vYGBwe7+61mZQW8", + "VpNywmBFdHi893h/53B3vxUUfGrnz84fp+5fEPmMi2kaU6Nk92RKQjqhIQKPHqQ/QJ0EjiWSa3xVmhzj", + "aCSsGOg9DxSmsVxp0zSD2ZbGfSvJYkXTmJh3sCGtJF1Y+Sn05LMXU8aIGOXuShv0ZL2Y1lrG3FryJqji", + "jVYB3QWVIFkUAhElcXRkKHQtn4PdLCb2pgkP7BpaYsNzfkdELyZzEpeRwBxHerIJFwTleGI2rbIqyuY4", + "ptGIsjRrsIw2gPJZJkC+NJ0iPOaZMqo6bFh5ELh7BR1hotl1u6v/Z1zcrr0d06frSGSM6W7WatnHcczv", + "9BbfatjAyYyR/do5P5QEuVylNoYH+16iV+YLY5goHqeZQpQprpVyFo0XXRiJRNCOIUGk4sBJcXirpUbb", + "TVuJ0S+LvNBCiDN7mvEK3nlPNt/exJjcvpzht+0NeHErswSYs8vrTY3DqeATGnuADIYN+9aKfc5s+nx/", + "cNXb+f/NZQ2LF+asoMwYQxIekX7NKRratyOBs8vry6Y55R7pqDy7pTXl5isP6uQWEQcRa5gJMUNjgqwo", + "ZZAATGvFIIUQ8MR3qE4ETsg4m0yIGCUebfyZfo9MA2OnpAxdPK0erPoAbyuSX1Y2B2TyCQ6tQ3E76Hu0", + "/doyuiVoNmDjK2L4QZPLkt4qYdtYr6U+epHHAKCzy2uJCpOjxwxQ3d7Gq/XL2UJqBdb0aDwQKStr74Cc", + "rY/qy+JDa+fwHNiJ95ByhIA682maARleveqdv7zZTiIy71bmBGbCGY+JnvdWSf6eO8elwg+gcp04b1Kj", + "DGLItgRUglVOwa2BVKJXD3QUVzgeyZgrz2xe65cIXqLOzTPjsKJn0EVpZSv18xIUKvh94KUYzZGahr2C", + "Aev2mAqBrzWNJUa0KS+vMqiPVH4hODYRQ1V8Lnxg3cbz2+pG89u11Gs78Y177m6UWzjZnFycGhkh5Exh", + "yojIL3OrHhHgzRV0g54WGCNMErCbT/57tXdEg30vR5dVFqKTpXCDr2IdanCp1UwunpMIJZjRCZHKutRW", + "RpYzvPvo4Mg480dksv/ooN/vb+rO8nPhv9JqK7bN3X/Js6UvZ5+3D1/Ba6XNWj4El8evfwmOgu1Miu2Y", + "hzjelmPKjkp/538WL+CH+XNMmddnoFX8B50sxX1UtjfVZ5Z5fqRXwkiYIyQHTWKt/bpBoNWoGdP3JEJe", + "90uFp1o6NRj3eX6WnxExUYTtqVKkRPkqqUXUBH2/2iTjBCNoY8fMmKJxEVCybIz5pJAgudLDesm7OiUs", + "96mOY/Mr5GyuqcLnYF1h4O7d0mbcGb1sFFEPdv7NKm0RFSRU4FS1noaCbZym61HRL/zlPK1tsIh1FfWc", + "Lt+ck3+KUb46+svpX37/P/Ly8W87vz+/ufnf+dlfTl/Q/72JL1+2v9bzuCWt9hL+pq6+K+9dwRJdcfFt", + "ix4XWIUewWfGpWqAmn2DFEeJ/riPTkBBOxqyHnpOFRE4PkLDAKe0b4HZD3kyDFCHvMOhMl8hzpDuyroX", + "bOmPL42Grj/+4HTAj/U+IutHICyQc5cfmY0jnmDKtoZsyGxfyC1EwsWe/hWhEKcqE0TviJY14wUaCxwW", + "/gPF4F30Aafpx60hA02UvFNCryDFQuUhBW4E2Gg7K3NxaJuTCIHvnbSa7JDl5weo5roThcWUqH5uKgFj", + "Xs0U0wAUr5rBRdX/5XDQ9ewj0u30RsZUKsJQbpWgEpAXdZwj0+GgQv6Hg8P1d9Q5Dq1AP8Du5SB+h5Qt", + "6MMgMAxtmPFoplTawvim+Y2hEfTL69eXGgz63yvkOipgkW+xUcZwmsaUSHPzqmKQSazv2Fbgs5WZ3W25", + "oNemsf4sbuGq8zMMjF4/v0KKiIQyw787oQbnhIZ6fXAHSKXMNCpSjI5PLn7e6rfIQgCwzee/Yh9f5yus", + "XTU549ayhglfFMZBDd8uOj/tanHKUmghaMHd+jMuUGwYTEHXR+hakqqnC2yVuQY0OxkvCguZ4erDYMv1", + "mNY5xRF6lct3OJ9KHupUIIPrsqBL6NZaas3F/1Lv3epcwaXB6i+WtcE1P1bIXozAUdzMClaTvwfiQPOc", + "NTr/tqLtstFSD+ZHjWLvv7oEsrepLrlpzEXVcbHkqJqHXbSPl/gacQfLetU7qkaNt3ZIv7Z3dE57uLlA", + "MyzZnxS8rOkQO3uPW0Xz61Hb3neVb7r4xEwppyrnBZnf0xh/0Fsax+b6U9IpwzF6gjpX52d/PX/+fAv1", + "0MuXF/WtWPWFb39ahF841D67vIaYBixHkuFUzrhq9nzCyLXRK5VKLjuJtrp5WR3u8UslJMPrdbv1BeM0", + "3HXV0jK+TgTGt/Tm+TeK/gg+J3ZjZbTF54ZMWKH6K0VMNDJxX7RBlZ+bx1829uGrTKcSxeDjQ2XZwzl4", + "fnLgQjegHue2Y6lZLYnQ+WURFl4YqVz3tTU92e3vHBz2dwaD/s6gjckuweGKsS+OT9oPPtg1RowjPD4K", + "oyMy+QyToUVsIyTi+A4vJBo6MX4YGL2hpDCUmIUV9Vtdxy7Hh3xaOEhdcFkX8LFJgEe7yI0V+Vquqpla", + "WsuCj/7+WUldSFsJ4Aoau69GmxizCQp5Fkda3hpryjPqG4mslimJKpLgALFes1vG71h16camqen394yI", + "Bbq5uKhYwAWZ2HwgLRbO07RxH3i60TbsrhHJ186mFERxH4ETdU5YOoG+eJhE2Vzn/LUM1rUw2xWSp/dq", + "mzIDbr33K9ZUM7hEZD7KMp94pV85z+vr6/PTyoZjfLBzODh80jsc7xz09qPBTg/v7B30dh/hwWQvfLzX", + "kDmrvWvLp3urVCm0OdIBAA/GSxOcEh1pGsrdTcaZQrkjkibOEy2nopIIbPz6wZ5gfah0D3C6hvpNvMil", + "5JUfX2JNqO7bFP5a/cXVLFNaDIJv5CxTSP8FU9ZLsFrG6i4MzR+hFxy+Ec6ZjPG6umKagy/ZcvO6atOx", + "XjvOzQwGswzsCD3LmVbO9iyb60hifxpeaj0gwbtzyxhBrGZhdyvoBhbqQTcwIAy6gYOM/mlWCL9g8kE3", + "sBPxOk+X8cZn3Cc4Bh5WOM5kisb0vSE5PXUqFQ2NdodhN5vIzkaokmhkjtCm6zfjjWGP2fwjR9U3F6gD", + "8UR/Rlb5039t5Vd1ZRLa332y/+Tg8e6Tg1beyMUE13PjE/AVWp7cWtYcptnIZRBsWPrJ5TUcPvpgk1li", + "tHm79kJj1Iwj1NIeZahISVgM/qT/pOyEHfFsHJesQzYKAzx92+SPbLib+p3GczqZsN/fh7e7vwma7Lw7", + "kLtjr3KUD+SXJM/LFs0ltYuMeyb/gV+HBIQSstGV/BWRsAJ0RRQC/OlphqVP1NzFx6Kcczi3EPci1v7e", + "3t7h40e7rfDKzq5EOCPQ/5ZneWFnUCIxaIk6r66u0HYJ4Uyfzu8xFUTqxZnoKC+dIZuHZlBxidS6x54P", + "SxoElgJrbN/zpBHkN1ZisYuyQAdPpVyaWaJyL7T39gaP9x8dPmpHxlbjGYl3qzmMbWdv+AUJCZ1Xdr4D", + "VvDXx5dI9y4mOKxK+Du7e/uPDh4fbjQrtdGslMBMJlSpjSZ2+Pjg0f7e7k67mAifpdtG+1QItsq7PETn", + "QQrPbnhAscx6u02nhU9KXHaPXOmRWbh41v35NnHgLSI8qYReacl3FHW0EFUWSEtRiltt7Ax+FqnHacpL", + "rMXFtr61q11pL7GanbMJX77K2EThsw5KzsSdasFHQsbGiDBKIse7cs3PylLg8hRLgqKMWMgZ2UhgC3Bs", + "rnNSrGYgrMKHlE2rzt5LA7ZRw8wcVsfzwri2YRuLkfQ71bwWGcDK2JIlwoV7TSvDOJUjv1ax3LEg0yzG", + "AtX9x1dMWS6SmLLbNr3LRTLmMQ2R/qCuzk94HPO7kX4lf4K1bLVanf5gVNwk19RzMznrR2A2pDZusYSf", + "9Cq3ap5JcPJvm++3IfF8GwOc93rpmVbejIv1NaPvSoheDY7b3x00OaI1dFpxQVt2z9+Ut1uU9VG885w/", + "zpPIeK4xzUVRTYOtysGV9fpWCzeRq9zuliUB1HE2PRd8WIVrKQiw1UHc7jK0br12s9mWJKyOvn/46PFB", + "yyjMzxK1V6Tm/gzBep6sEKgbduqijdR2+OjwyZO9/UdPdjeSj9xFR8P+NF12lPenliuqJrM9GsD/NpqU", + "uerwT6nhuqM6oUrep0+e0McVpFsEvTRo3avKYhQ76dT8qgDeTsRdIS0dV0SuUhLGDplMCBiORgZuvWIy", + "NSesVnMIcYpDqhYeDRDfgV8KypvUgjda9F6brAektm+EJ4oIuI2Q2bi49++4wdF/Gc2uhguHrYO5ZTZu", + "0iJf1kc1OqRx5IpqFooWBgKDEb7L97scmOgOy4pVX/8OFYm6pSSb9esf06J9GnSH63km9OI63ReA5M96", + "Xt7+2naWtI6KkFyH+KojtJkEtUQAXmJtDOyeE9kT1RSud96o8Qd7AH7aV6NxOc3CyjwWlZwMxam7+bjt", + "0oMuf2dOsM3HK93gb/JhPeIc8NHOwYK86LtbQQkfNpn7laZ0RomrF1ULSKemAoeNKEOlxqhDklQtXNSE", + "00y3NrvvOc479CLjF/ZzGzz5Ep721ytd6/9NEmSVr9jcIGsv15b2tNGf1S+untbdV4xOaBOEVN0tamkP", + "pFpRaGZVUTNTXQwUPutLPs3qwW8bFDJrUvELynEVZFwls3Wa60p7WmllpZk07425X/3Mqm9UunJvnwgy", + "q36td842d1RaAe7VM8iY2GJBQZ+zADKA1SDIVfRlO8Bqt48L/C4fAbRlLFEt56ZZRymL+NlTyBfwymUS", + "oRPXBUyjnj316eeVw3NYtbwZq+rjuRt8L+FZ/rOCozXRVg05izG6q0vwadZFwkxQtbjSB4J1TiNYEHGc", + "GTSEkwIWAY+LwSFA4eNHUFMnHmn1jDAiaIiOL88BSxLMIAczurlAMZ2QcBHGxPqXL93tQnqElyfnPRMY", + "kye5hOo2CgDisssdX55DYitbVyYY9Hf7kJGcp4ThlAZHwV5/B1J3aTDAErch7hB+WkOUpkM4yc4je+I+", + "NU00aGXKmTTA2R0ManWKcJE8aPs3aSws5nhtLRSaQnDL/hZLDpFOErDT/9gN9gc7G81nbb4f37DXDGdq", + "xgV9T2CajzYEwicNes6MVu3SoxPbsMDZ4OjXKrb++ubjm24gsyTBWkQ04CpglXLZJMIQiTBi5M4GpP7G", + "x310ZXQS8C4vSmwakwGJNEvCSGHRn75HWIQzOidDZjmxyd2EBUTfJEhzYBP7UEUzM7TZfUPCRKqnPFrU", + "oJt3t6276zmv2gLAGxdwyhORpg2VnHzc0eQ7kyH3JnojDDNVpM8yic5uCVxiTug7b4etbuM18yjXNnTe", + "nbtbfjsguCr7Tein+TtXR6x6YGgZmrIwzqLiVK3Wb/IG25s6RDYf3C3xCCFn0MICpezV7Y4vxiNifGXT", + "hZpxZn5n44ypzPweC34nidCHnI3NsbC2yZAs6kKyTZpAfIyJ5tVjbpspbn+4JYuP/SE7jhIXfW1zPuNY", + "cpsoz3g9UInyzOND1lw50C9Mn9gEtyZJVTmfkpkmz1SaqT4yCyHKBhRBc0j7JGckGjLF0QdhMncuPm5/", + "KEb8CNIpwZHGk1ITs6TtDzT62DRrOcJ69aOxK79Zk9kJAGAYaKlhGOjfU4G1dJrJGcIh+Gboh+Ut7RjC", + "5gJO/q06hEPMUMrTLNZyFCCVyf9X6QOCM3EcIwWk5L7V8gTsZMN6rEnXlxHI2nONAa5GRpAbqERMg/1D", + "Pz1JEgriU0v/cvXyBYKjCsqcQbMifABgRJkWNPI823r0/pD9jMMZMjII5J8dBjQaBkU5qy2YayaJEQN6", + "PRBifoI6f2aYLo1+6vd1V0Y+OkK/fjC9HGlaSpOR4reEDYOPXVR6MaVqlo3zd2/8AG0yi11VGAHqGN6/", + "5cLl9QpLx6A5NzCLELe8Nl4gjAoOVNZ+x5RhsWiqDccz1exaZLIJ2GbFfh4MBlvrr47sUj2SYaWhpoSP", + "S4LQ7heTAaz8sywDlOrA6hOX2VQRkZF87kEIeYojF8H4Q9paI21ZNbEkR8H3ZZZs0DcmxpW1JgxBuUAn", + "DKVY4IQoyPP/qx/nwYuX6r/dRS+cRMZoUkXebgk8dd3pzRJi7zfWYcwrGgIu7N8D/sG4RZJHGPfJfY2L", + "Y5NiPK8N/aDQETbLIWLXr+idEfU9YNzgvlipy0X7DfH3oeDPGbEiWAG0GjfbhuIeZStCPdpEEJxI24tp", + "rNXGK5hT74owhaACsOzbf53yAY78b2M+fXuEDAhjW/9Y2syRubldH4oWlvCRSayTf2fzTYUzzKZEoo45", + "P//1j3+6Gq7/+sc/bQ3Xf/3jn0Du27YiOXSXVx9+e4T+SkjawzGdE7cYcE4lcyIWaG9gyxnBK0/2Kjlk", + "Q/aKqEwwmbt26XUBTEyHNqurXg9lGZFIAgih9sDE+hwZa55Hm3a0bEB5rxTdXVJ/7ApKC9CnosMBuESm", + "jCqKY6sKuXlA/FMxEbPmoDx43TC5ZKpez18UeacM9vbMBDdkMKZ6t4fuTEFr0yfqXF39vNVHIO4brAC/", + "MtAbim6sJtD/wZPW8yTDUaoMBaBseFMpcWmjWfPUtrkPu2ZTUtNmw6ZR5InWjd1ifojdLYycfrg5g6fP", + "6njqijE0mx0/fb2+4t6tdMovt88O95ZhbiuNFCD7Ftok6tgk8Xmen0o5k2+F9PfCgEtVcHIujLjJLnRv", + "Gs4JZ5OYhgr13FxsteRc66kiyENhB6/srBF266oHQ5SPiu2Kb1/joZG7+d3n6VEbdJNjpAjYKHDtx0my", + "DnVOqQy5/raELb0QpzbLEQCxoNMyFq2z7ZzC8/zIWSmY5/XLHUHen5XHDp2x+tlwD0zxtMYQvyEjrGVU", + "KYU4PSRsvs530RWeWmEE+r5Qc3B/UtB9G4R8aP6QLEJRDWyaC87yvPtN6GUz83/FjbYjeBZ+RYSjajNR", + "k8mjWJb5FIUzEt6aBdn6aKskgnNXQu3rywGmvMAGp7+d/o/jvoXiWMBqlbJ4btO7fD1dEUbYSFX8cteP", + "FsE8QAZ3hLEzpJrMKVguWLj1h7qBvJeToV7P7AFR0mUWx84QPydCFUUWyvx0+wM4rqyXkx21rZRFrl89", + "7xEWcvBUyr1s/AKJy6n+ZaVls2FmKT/QpI1+BaByiNEsjH7G/huHMpSn6/zP3Wc2Yed/7j4zKTv/c+/Y", + "JO3c+mrIMrgv1nzf0usDRj4tvNIq0IA1mczn66S9vNW9CHy2xMQmIl8+wR9SXxuprwyulYJfXu3jK4p+", + "tojCt7knyJHNB2145fzP/mAi3/2anixGlupiVmzxNocMF0XhAltV7+E5yNEc48r8t6UNtSDIldKBQ93z", + "066tSWEqSeS++PdkUXXzuHcp0Y57/+bU42RMpxnPZNnNH0qQEFnU9K4w4IcmvxbHc6ME+x1j6eA+j457", + "F1B/4P1XEp3rG2qYty3xvEZ4dq3WkMMzGqtSZnsJxT5MPlkTJpTXSLfZYLcanMZcFua2aFxJAr7szOab", + "l6vBUCrLkGIqZB9dS6LBRNKXJsJDqkVMjobsf9wnvyqCkzc/jXF4S1g0zAaD3YP8HWHzNz9JBZGiQ3bh", + "8IYwJSiRCAuCjl+cwsXUFCJw++g4jotQqPp8UJJJW6zNVR9KY8j3ZTiHD3yl+hIFBFuXsFiK+QYAQF4U", + "B5Pgs9lSSz2quLVrr0g5ZP2hSLVSpErgWq1I5dmTv6YmZQb5ZqqUwzcfwG3igB/K1H0oUzKbTGhICVNF", + "5q8lByebOPABhhgxex9TckyonMetlakipflqOdUi77dwSskHv38dyqUXfJiu0twER0ROaykOw2a15XvD", + "h8H9Muf7V1ceMoqdlSt/+hUDEyc0sYnH/ALCMy5u22KejUGmXxMBv7x0Ul7hdyib6OlBvodvL6LA4W38", + "wjXSVCWXeyDIOn59U3dEBwmrA5vYOFOS3BSiuaNqxjOT52JkH5qcU5oqbBZpEHlC2+u3Zi969HsQQF9w", + "hWiSxiQhkJOqZ7AJKgBlacpFXneAylLpkc3YnyabsnOoSTdiy2919a6xaLwAm15eOQjM+8vb5eWaMZ+u", + "j63MB3eBhJ7gyiFzxZXfGlH4LcqZLFIcSRKTUKG7GQ1nEGipn0H/Jg4Tp+nbPLPC1hE6A0ot53qAwTuS", + "CIpjqO7CY1OY6O08Sd4eLee3urm4gI9MjKXJZPX2CLmcVvkBIXWrcuCkXkWMpUIvbDhoR2OS4K447Ft9", + "CpXWt2VDKoskFEPmC69k5M52SCfobSnS8m1DqKVjqM/1Ln0jeanbnDrIrEVxJABwBjcJixosZhpq/iDL", + "nYE3TWPLgE8zja8c77k0med8mqctqqAyTtO26GunCVg8T5IVOIw6peJFUkU8U3+WKiJCwMcWu5uQG3Vw", + "aP5Q+NZU769U4TXlsrxWTpO8xAsqzVRLVbbMX/MkCUxJ4AT7qmZ9fuBsvcNlM5vemVJ07A9Je5O41yqz", + "LwW+1k4OW66tWeR+ZRr84fU9V9fuG6PhN7CPFbOgzIkqsLdFwcCHFfUHG7kki5lSgT4ace8aacTWGfzD", + "00iBH39wKgm5EOZCzxYLfjju2SU7TYncO1CdtKj62XW2wpuLi60mohFqJcmI78OI+Gk2nFphoCTy16AQ", + "NHKpGk8uTm1iRyq1ltdHLxMK+RNvCUkhGwzlmURwI9svZ6VvujfN084TpsQi5ZSptbMomn6dyXz8pPx4", + "98ynbIDKH/4oB/PDw2NSpkQ8zhew6mZL8yHVaBpxpoJKDe0xz3TvSxn2oXacXEhFEmMnmWQxEBGEA9rE", + "S7hcG6+LqJJQqaUL9yulumhDNiYTLYakROix9eeQbLdQ+XzWhCuFc655aVjf92FOgKT7oEFj1QS1WgG6", + "NHX59n0qa14i4JOn9AzsA9XafBJ1YnprCk6juUSx/rG10sBgCvd96bRSn05ZeWlKX7oQg7M5Mv8RONx5", + "ja05A+qDY2tnpEwsjv/ARvvZmlzL18SGtcsd7Eo1zPtDdkGU0G2wICjkcQwlq4zatJ0KHm5DXeUwpZEp", + "sAyTA4bX/DqBEU8ur6GdSczcHTL9x3Jl3/pEXYHg8+2Xa0yupqb7v7E+Zha4iiz8G/7Dmrb5vXUjDckG", + "EuXpKgWIp394g4GV4H5YCx6mtQAch/LVdKYChyAUy1mmIn7H/JYBW8F2+4P5cb7O/UzhcHbjCop9H9Ku", + "rT+0bhi3wAdBlHZNETHpjO6fJnleIuqBhqxrwLklgBBTdqTznwKm9NwfDbu/vNtUGY4buU3dK225VGHf", + "DW3d98ln5+C8qsvweChkbjDNrQQKt5StT6JcA3elbuZKlEJB5ly0dKV5u+UK0SYzeW5DKkoL5sVo+0OW", + "V991mdG1dtV1qhWKqLw1PVjtqY/8RZKNnmcrJUNZoRDHoanHk1cLNoVyZIP29apUQfur0VsxiGej8zLJ", + "Mq9q+5BUDj9OwO6Vy+YCxllxamVc3Y1tcx+hVPYw2yCQyq3gRxhVizCqErDaFOkzNY8tt7LFaosIQEnf", + "w+2OLwwrF0q+XhDWJ5zXXw49HJ42ntY/wq/uTSAoUlmcnz78mKsyzVV49LbWCnq2AmbZNLSKgi2IUkF6", + "ri5eZABm4WF0jXqBzf6QvZ4R9xeizoOVRCiigoQqXiDKoHyhq5P8J4kE58q+52LRXIjTkMgzwZNju5o1", + "ykvriuG+i5iN82x1PVWSaZIlpkIyZejsKeqQd0oYh0o0wTQGd14HUvIuJCSSgJNb9UrkXg/LvOT42lmu", + "cI3Na42GpibkPNfEOjhTvDclTO9FUYEzFXxOIxPe3lDS3Tdb0BC/gJI2fU/TKumtrdO3THhVvEV5yVFb", + "KLDAT7c7wY9jol4ZAfILcJEDUXGOYiymZOvHUfKQj5KyNcmdG5UTpV30bjsDU0u7z9eI3M2Nj/cbt3vz", + "/dhESpnkH2Cio3mu9DUFDH9fKDi4v/PhvgOFbx6wDf2MOAW3FCQMHegefQjznIc4RhGZk5inCVRUg7ZB", + "N8hEHBwFM6XSo+3tWLebcamODgeHg+Djm4//LwAA//9/tTsUefIAAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/openapi.yaml b/openapi.yaml index cd5b41e8..5117dbb3 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -227,6 +227,13 @@ components: pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ maxLength: 63 example: my-workload-1-fork + from_running: + type: boolean + description: | + Allow forking from a running source instance. + When true and source is Running, the source is put into standby, forked, then restored back to Running. + default: false + example: false Instance: type: object @@ -1528,7 +1535,7 @@ paths: /instances/{id}/fork: post: - summary: Fork an instance from stopped or standby state + summary: Fork an instance from stopped, standby, or running (with from_running=true) operationId: forkInstance security: - bearerAuth: [] @@ -1565,7 +1572,7 @@ paths: schema: $ref: "#/components/schemas/Error" 409: - description: Conflict - invalid state or target name conflict + description: Conflict - invalid state (including running without from_running=true) or target name conflict content: application/json: schema: From b47d0653450fa4910b6d28ee78537d98c4a63d20 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 27 Feb 2026 16:36:27 -0500 Subject: [PATCH 06/20] Fix fork restore guest IP reconfiguration --- lib/instances/fork_test.go | 59 ++++++++++++++++++++++++++++++++++++++ lib/instances/restore.go | 2 +- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index 3d82f6e8..e5627db0 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -1,6 +1,7 @@ package instances import ( + "bytes" "context" "fmt" "io" @@ -10,6 +11,7 @@ import ( "testing" "time" + "github.com/kernel/hypeman/lib/guest" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/paths" @@ -127,6 +129,7 @@ func TestForkCloudHypervisorFromRunningNetwork(t *testing.T) { assert.NotEmpty(t, forked.MAC) assert.NotEqual(t, sourceAfterFork.IP, forked.IP) assert.NotEqual(t, sourceAfterFork.MAC, forked.MAC) + assertGuestHasOnlyExpectedIPv4(t, forked, forked.IP, 30*time.Second) assertHostCanReachNginx(t, forked.IP, 80, 60*time.Second) assertHostCanReachNginx(t, sourceAfterFork.IP, 80, 60*time.Second) } @@ -160,3 +163,59 @@ func assertHostCanReachNginx(t *testing.T, ip string, port int, timeout time.Dur require.NoError(t, lastErr, "host should reach %s within %s", url, timeout) } + +func assertGuestHasOnlyExpectedIPv4(t *testing.T, inst *Instance, expectedIP string, timeout time.Duration) { + t.Helper() + + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + output, exitCode, err := execInInstance(context.Background(), inst, "sh", "-c", "ip -4 -o addr show dev eth0 scope global | awk '{print $4}'") + if err == nil && exitCode == 0 { + var cidrs []string + for _, line := range strings.Split(strings.TrimSpace(output), "\n") { + if trimmed := strings.TrimSpace(line); trimmed != "" { + cidrs = append(cidrs, trimmed) + } + } + + if len(cidrs) == 1 && strings.HasPrefix(cidrs[0], expectedIP+"/") { + return + } + + lastErr = fmt.Errorf("expected only %s on eth0, got %v", expectedIP, cidrs) + } else if err != nil { + lastErr = err + } else { + lastErr = fmt.Errorf("ip addr command exit code %d, output=%q", exitCode, output) + } + + time.Sleep(500 * time.Millisecond) + } + + require.NoError(t, lastErr, "guest should expose only the fork IP on eth0 within %s", timeout) +} + +func execInInstance(ctx context.Context, inst *Instance, command ...string) (string, int, error) { + dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID) + if err != nil { + return "", -1, err + } + + var stdout, stderr bytes.Buffer + exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ + Command: command, + Stdout: &stdout, + Stderr: &stderr, + WaitForAgent: 30 * time.Second, + }) + if err != nil { + return "", -1, err + } + + output := stdout.String() + if stderr.Len() > 0 { + output += "\nSTDERR: " + stderr.String() + } + return output, exit.Code, nil +} diff --git a/lib/instances/restore.go b/lib/instances/restore.go index e28a1000..d2ae660f 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -284,7 +284,7 @@ func reconfigureGuestNetwork(ctx context.Context, stored *StoredMetadata, alloc } cmd := fmt.Sprintf( - "ip addr replace %s/%d dev eth0 && ip link set dev eth0 up && ip route replace default via %s dev eth0", + "ip -4 addr flush dev eth0 scope global && ip addr add %s/%d dev eth0 && ip link set dev eth0 up && ip route replace default via %s dev eth0", alloc.IP, prefix, alloc.Gateway, ) From addd9db57c853e8d19e4887263a67efe56646f2a Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 27 Feb 2026 18:38:42 -0500 Subject: [PATCH 07/20] Add QEMU fork support for running standby flow --- .../cloudhypervisor/fork_snapshot.go | 2 +- .../cloudhypervisor/fork_snapshot_test.go | 2 +- lib/hypervisor/qemu/fork.go | 85 ++++++++++++++++++ lib/hypervisor/qemu/fork_test.go | 80 +++++++++++++++++ lib/hypervisor/qemu/process.go | 7 -- lib/instances/fork.go | 53 ++++++++++- lib/instances/fork_test.go | 13 +-- lib/instances/qemu_test.go | 89 +++++++++++++++++++ 8 files changed, 315 insertions(+), 16 deletions(-) create mode 100644 lib/hypervisor/qemu/fork.go create mode 100644 lib/hypervisor/qemu/fork_test.go diff --git a/lib/hypervisor/cloudhypervisor/fork_snapshot.go b/lib/hypervisor/cloudhypervisor/fork_snapshot.go index 11bf67c5..98d00890 100644 --- a/lib/hypervisor/cloudhypervisor/fork_snapshot.go +++ b/lib/hypervisor/cloudhypervisor/fork_snapshot.go @@ -70,11 +70,11 @@ func rewriteStringValues(value any, mapper func(string) string) any { } func updateVsockConfig(config map[string]any, cid int64, socketPath string) { + _ = cid // Keep snapshot CID stable for CH restores; only rewrite socket path. vsock, ok := config["vsock"].(map[string]any) if !ok || vsock == nil { return } - vsock["cid"] = cid if socketPath != "" { vsock["socket"] = socketPath } diff --git a/lib/hypervisor/cloudhypervisor/fork_snapshot_test.go b/lib/hypervisor/cloudhypervisor/fork_snapshot_test.go index f5abd8ec..bdf7345b 100644 --- a/lib/hypervisor/cloudhypervisor/fork_snapshot_test.go +++ b/lib/hypervisor/cloudhypervisor/fork_snapshot_test.go @@ -59,7 +59,7 @@ func TestRewriteSnapshotConfigForFork(t *testing.T) { assert.Equal(t, "/dst/guests/b/logs/app.log", serial["file"]) vsock := updated["vsock"].(map[string]any) - assert.Equal(t, float64(200), vsock["cid"]) + assert.Equal(t, float64(100), vsock["cid"]) assert.Equal(t, "/dst/guests/b/vsock.sock", vsock["socket"]) netCfg := updated["net"].([]any)[0].(map[string]any) diff --git a/lib/hypervisor/qemu/fork.go b/lib/hypervisor/qemu/fork.go new file mode 100644 index 00000000..2fe97672 --- /dev/null +++ b/lib/hypervisor/qemu/fork.go @@ -0,0 +1,85 @@ +package qemu + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/kernel/hypeman/lib/hypervisor" +) + +// PrepareFork prepares QEMU fork state by rewriting snapshot VM config when a +// snapshot path is provided. For stopped forks (no snapshot), this is a no-op. +func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareRequest) error { + _ = ctx + if req.SnapshotConfigPath == "" { + return nil + } + + snapshotDir := filepath.Dir(req.SnapshotConfigPath) + cfg, err := loadVMConfig(snapshotDir) + if err != nil { + // The generic path points to CH's config.json; for QEMU, require qemu-config.json. + expectedPath := filepath.Join(snapshotDir, vmConfigFile) + if _, statErr := os.Stat(expectedPath); statErr != nil { + return fmt.Errorf("load qemu snapshot config %q: %w", expectedPath, err) + } + return fmt.Errorf("load qemu snapshot config: %w", err) + } + + if req.SourceDataDir != "" && req.TargetDataDir != "" && req.SourceDataDir != req.TargetDataDir { + cfg = rewriteQEMUConfigPaths(cfg, req.SourceDataDir, req.TargetDataDir) + } + + if req.VsockCID > 0 { + cfg.VsockCID = req.VsockCID + } + if req.VsockSocket != "" { + cfg.VsockSocket = req.VsockSocket + } + if req.SerialLogPath != "" { + cfg.SerialLogPath = req.SerialLogPath + } + + if req.Network != nil { + for i := range cfg.Networks { + if req.Network.TAPDevice != "" { + cfg.Networks[i].TAPDevice = req.Network.TAPDevice + } + if req.Network.MAC != "" { + cfg.Networks[i].MAC = req.Network.MAC + } + if req.Network.IP != "" { + cfg.Networks[i].IP = req.Network.IP + } + if req.Network.Netmask != "" { + cfg.Networks[i].Netmask = req.Network.Netmask + } + } + } + + if err := saveVMConfig(snapshotDir, cfg); err != nil { + return fmt.Errorf("write qemu snapshot config: %w", err) + } + return nil +} + +func rewriteQEMUConfigPaths(cfg hypervisor.VMConfig, sourceDir, targetDir string) hypervisor.VMConfig { + replace := func(value string) string { + return strings.ReplaceAll(value, sourceDir, targetDir) + } + + for i := range cfg.Disks { + cfg.Disks[i].Path = replace(cfg.Disks[i].Path) + } + + cfg.SerialLogPath = replace(cfg.SerialLogPath) + cfg.VsockSocket = replace(cfg.VsockSocket) + cfg.KernelPath = replace(cfg.KernelPath) + cfg.InitrdPath = replace(cfg.InitrdPath) + cfg.KernelArgs = replace(cfg.KernelArgs) + + return cfg +} diff --git a/lib/hypervisor/qemu/fork_test.go b/lib/hypervisor/qemu/fork_test.go new file mode 100644 index 00000000..3f50b2a2 --- /dev/null +++ b/lib/hypervisor/qemu/fork_test.go @@ -0,0 +1,80 @@ +package qemu + +import ( + "context" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPrepareFork_NoSnapshotPathIsNoOp(t *testing.T) { + starter := NewStarter() + require.NoError(t, starter.PrepareFork(context.Background(), hypervisor.ForkPrepareRequest{})) +} + +func TestPrepareFork_RewritesSnapshotConfig(t *testing.T) { + starter := NewStarter() + snapshotDir := t.TempDir() + + sourceDir := "/src/guest" + targetDir := "/dst/guest" + initial := hypervisor.VMConfig{ + VCPUs: 2, + MemoryBytes: 2 * 1024 * 1024 * 1024, + SerialLogPath: sourceDir + "/logs/app.log", + VsockCID: 12345, + VsockSocket: sourceDir + "/vsock/vsock.sock", + KernelPath: sourceDir + "/kernel/vmlinuz", + InitrdPath: sourceDir + "/kernel/initrd", + KernelArgs: "console=ttyS0 root=" + sourceDir + "/rootfs", + Disks: []hypervisor.DiskConfig{ + {Path: sourceDir + "/overlay.raw"}, + {Path: "/volumes/volume-data.raw"}, + }, + Networks: []hypervisor.NetworkConfig{ + { + TAPDevice: "hype-oldtap", + IP: "10.100.10.10", + MAC: "02:00:00:aa:bb:cc", + Netmask: "255.255.0.0", + }, + }, + } + require.NoError(t, saveVMConfig(snapshotDir, initial)) + + err := starter.PrepareFork(context.Background(), hypervisor.ForkPrepareRequest{ + SnapshotConfigPath: filepath.Join(snapshotDir, "config.json"), + SourceDataDir: sourceDir, + TargetDataDir: targetDir, + VsockCID: 54321, + VsockSocket: targetDir + "/vsock/fork-vsock.sock", + SerialLogPath: targetDir + "/logs/fork-app.log", + Network: &hypervisor.ForkNetworkConfig{ + TAPDevice: "hype-newtap", + IP: "10.100.20.20", + MAC: "02:00:00:dd:ee:ff", + Netmask: "255.255.0.0", + }, + }) + require.NoError(t, err) + + updated, err := loadVMConfig(snapshotDir) + require.NoError(t, err) + + assert.Equal(t, int64(54321), updated.VsockCID) + assert.Equal(t, targetDir+"/vsock/fork-vsock.sock", updated.VsockSocket) + assert.Equal(t, targetDir+"/logs/fork-app.log", updated.SerialLogPath) + assert.Equal(t, targetDir+"/kernel/vmlinuz", updated.KernelPath) + assert.Equal(t, targetDir+"/kernel/initrd", updated.InitrdPath) + assert.Contains(t, updated.KernelArgs, targetDir+"/rootfs") + assert.Equal(t, targetDir+"/overlay.raw", updated.Disks[0].Path) + assert.Equal(t, "/volumes/volume-data.raw", updated.Disks[1].Path, "non-instance paths should remain unchanged") + require.Len(t, updated.Networks, 1) + assert.Equal(t, "hype-newtap", updated.Networks[0].TAPDevice) + assert.Equal(t, "10.100.20.20", updated.Networks[0].IP) + assert.Equal(t, "02:00:00:dd:ee:ff", updated.Networks[0].MAC) + assert.Equal(t, "255.255.0.0", updated.Networks[0].Netmask) +} diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index 4fba5f85..e2e1d098 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -270,13 +270,6 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, return pid, hv, nil } -// PrepareFork is not supported for QEMU in the current implementation. -func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareRequest) error { - _ = ctx - _ = req - return hypervisor.ErrNotSupported -} - // vmConfigFile is the name of the file where VM config is saved for restore. const vmConfigFile = "qemu-config.json" diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 777da566..53df19da 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "hash/crc32" "os" "time" @@ -47,6 +48,11 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR } forked, forkErr := m.forkInstanceFromStoppedOrStandby(ctx, id, req) + if forkErr == nil { + if err := m.rotateSourceVsockForRestore(ctx, id, forked.Id); err != nil { + forkErr = fmt.Errorf("prepare source snapshot for restore: %w", err) + } + } log.InfoContext(ctx, "restoring source instance after running fork", "source_instance_id", id) _, restoreErr := m.restoreInstance(ctx, id) @@ -67,6 +73,48 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR } } +func (m *manager) rotateSourceVsockForRestore(ctx context.Context, sourceID, forkID string) error { + meta, err := m.loadMetadata(sourceID) + if err != nil { + return fmt.Errorf("reload source metadata: %w", err) + } + stored := &meta.StoredMetadata + + newCID := generateForkSourceVsockCID(sourceID, forkID, stored.VsockCID) + if newCID == stored.VsockCID { + return nil + } + + starter, err := m.getVMStarter(stored.HypervisorType) + if err != nil { + return fmt.Errorf("get vm starter: %w", err) + } + + if err := starter.PrepareFork(ctx, hypervisor.ForkPrepareRequest{ + SnapshotConfigPath: m.paths.InstanceSnapshotConfig(sourceID), + VsockCID: newCID, + VsockSocket: stored.VsockSocket, + }); err != nil { + return fmt.Errorf("rewrite source snapshot vsock state: %w", err) + } + + stored.VsockCID = newCID + if err := m.saveMetadata(meta); err != nil { + return fmt.Errorf("save source metadata: %w", err) + } + return nil +} + +func generateForkSourceVsockCID(sourceID, forkID string, current int64) int64 { + const cidRange = int64(4294967292) + seed := crc32.ChecksumIEEE([]byte(sourceID + ":" + forkID)) + cid := (int64(seed) % cidRange) + 3 + if cid == current { + cid = ((cid - 3 + 1) % cidRange) + 3 + } + return cid +} + func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id string, req ForkInstanceRequest) (*Instance, error) { log := logger.FromContext(ctx) @@ -135,9 +183,10 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin forkMeta.ExitCode = nil forkMeta.ExitMessage = "" + // Keep the original CID for snapshot-based forks. + // Rewriting CID in restored memory snapshots is not reliable across + // hypervisors. if source.State == StateStandby { - // Keep the original CID for snapshot-based forks. - // Rewriting CID in restored memory snapshots is not reliable for CH. forkMeta.VsockCID = stored.VsockCID } else { forkMeta.VsockCID = generateVsockCID(forkID) diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index e5627db0..a3227083 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -22,25 +22,28 @@ import ( func TestForkInstanceNotSupportedHypervisor(t *testing.T) { manager, _ := setupTestManager(t) ctx := context.Background() + if _, err := manager.getVMStarter(hypervisor.TypeVZ); err != nil { + t.Skip("vz starter not available on this platform") + } - sourceID := "fork-qemu-source" + sourceID := "fork-vz-source" require.NoError(t, manager.ensureDirectories(sourceID)) meta := &metadata{StoredMetadata: StoredMetadata{ Id: sourceID, - Name: "fork-qemu-source", + Name: "fork-vz-source", Image: "docker.io/library/alpine:latest", CreatedAt: time.Now(), - HypervisorType: hypervisor.TypeQEMU, + HypervisorType: hypervisor.TypeVZ, HypervisorVersion: "test", - SocketPath: paths.New(manager.paths.DataDir()).InstanceSocket(sourceID, "qemu.sock"), + SocketPath: paths.New(manager.paths.DataDir()).InstanceSocket(sourceID, "vz.sock"), DataDir: paths.New(manager.paths.DataDir()).InstanceDir(sourceID), VsockCID: 42, VsockSocket: paths.New(manager.paths.DataDir()).InstanceVsockSocket(sourceID), }} require.NoError(t, manager.saveMetadata(meta)) - _, err := manager.ForkInstance(ctx, sourceID, ForkInstanceRequest{Name: "fork-qemu-copy"}) + _, err := manager.ForkInstance(ctx, sourceID, ForkInstanceRequest{Name: "fork-vz-copy"}) require.Error(t, err) assert.ErrorIs(t, err, ErrNotSupported) } diff --git a/lib/instances/qemu_test.go b/lib/instances/qemu_test.go index 5e2421f6..f1db207b 100644 --- a/lib/instances/qemu_test.go +++ b/lib/instances/qemu_test.go @@ -865,3 +865,92 @@ func TestQEMUStandbyAndRestore(t *testing.T) { t.Log("QEMU standby/restore test complete!") } + +func TestQEMUForkFromRunningNetwork(t *testing.T) { + if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { + t.Skip("/dev/kvm not available, skipping on this platform") + } + + starter := qemu.NewStarter() + if _, err := starter.GetBinaryPath(nil, ""); err != nil { + t.Fatalf("QEMU not available: %v", err) + } + + manager, tmpDir := setupTestManagerForQEMU(t) + ctx := context.Background() + p := paths.New(tmpDir) + + imageManager, err := images.NewManager(p, 1, nil) + require.NoError(t, err) + + t.Log("Ensuring nginx image...") + nginxImage, err := imageManager.CreateImage(ctx, images.CreateImageRequest{Name: "docker.io/library/nginx:alpine"}) + require.NoError(t, err) + + imageName := nginxImage.Name + for i := 0; i < 60; i++ { + img, err := imageManager.GetImage(ctx, imageName) + if err == nil && img.Status == images.StatusReady { + nginxImage = img + break + } + if err == nil && img.Status == images.StatusFailed { + t.Fatalf("image build failed: %s", *img.Error) + } + time.Sleep(1 * time.Second) + } + require.Equal(t, images.StatusReady, nginxImage.Status, "Image should be ready after 60 seconds") + + require.NoError(t, manager.systemManager.EnsureSystemFiles(ctx)) + require.NoError(t, manager.networkManager.Initialize(ctx, nil)) + + source, err := manager.CreateInstance(ctx, CreateInstanceRequest{ + Name: "qemu-fork-running-src", + Image: "docker.io/library/nginx:alpine", + Size: 2 * 1024 * 1024 * 1024, + HotplugSize: 256 * 1024 * 1024, + OverlaySize: 10 * 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: true, + Hypervisor: hypervisor.TypeQEMU, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = manager.DeleteInstance(context.Background(), source.Id) }) + require.NoError(t, waitForQEMUReady(ctx, source.SocketPath, 10*time.Second)) + require.NoError(t, waitForLogMessage(ctx, manager, source.Id, "start worker processes", 15*time.Second)) + + assert.NotEmpty(t, source.IP) + assert.NotEmpty(t, source.MAC) + assertHostCanReachNginx(t, source.IP, 80, 60*time.Second) + + _, err = manager.ForkInstance(ctx, source.Id, ForkInstanceRequest{Name: "qemu-fork-running-no-flag"}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidState) + + forked, err := manager.ForkInstance(ctx, source.Id, ForkInstanceRequest{ + Name: "qemu-fork-running-copy", + FromRunning: true, + }) + require.NoError(t, err) + require.Equal(t, StateStandby, forked.State) + forkedID := forked.Id + t.Cleanup(func() { _ = manager.DeleteInstance(context.Background(), forkedID) }) + + sourceAfterFork, err := manager.GetInstance(ctx, source.Id) + require.NoError(t, err) + require.Equal(t, StateRunning, sourceAfterFork.State) + require.NotEmpty(t, sourceAfterFork.IP) + assertHostCanReachNginx(t, sourceAfterFork.IP, 80, 60*time.Second) + + forked, err = manager.RestoreInstance(ctx, forkedID) + require.NoError(t, err) + require.Equal(t, StateRunning, forked.State) + require.NoError(t, waitForQEMUReady(ctx, forked.SocketPath, 10*time.Second)) + + assert.NotEmpty(t, forked.IP) + assert.NotEmpty(t, forked.MAC) + assert.NotEqual(t, sourceAfterFork.IP, forked.IP) + assert.NotEqual(t, sourceAfterFork.MAC, forked.MAC) + assertHostCanReachNginx(t, forked.IP, 80, 60*time.Second) + assertHostCanReachNginx(t, sourceAfterFork.IP, 80, 60*time.Second) +} From 645bf6311df15104da133aa7a2ff174169f7a1d1 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 27 Feb 2026 19:45:58 -0500 Subject: [PATCH 08/20] Add fork target state and fix running-fork cleanup --- Makefile | 9 +- cmd/api/api/instances.go | 6 + cmd/api/api/instances_test.go | 4 + lib/instances/fork.go | 89 +++++++- lib/instances/fork_test.go | 67 +++++- lib/instances/qemu_test.go | 1 + lib/instances/types.go | 1 + lib/oapi/oapi.go | 380 ++++++++++++++++++---------------- openapi.yaml | 12 ++ 9 files changed, 375 insertions(+), 194 deletions(-) diff --git a/Makefile b/Makefile index db88e71b..7a927fc1 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ OAPI_CODEGEN_VERSION ?= v2.5.1 AIR ?= $(BIN_DIR)/air WIRE ?= $(BIN_DIR)/wire XCADDY ?= $(BIN_DIR)/xcaddy +TEST_TIMEOUT ?= 10m # Install oapi-codegen $(OAPI_CODEGEN): | $(BIN_DIR) @@ -233,9 +234,9 @@ test-linux: ensure-ch-binaries ensure-caddy-binaries build-embedded if [ -n "$(VERBOSE)" ]; then VERBOSE_FLAG="-v"; fi; \ if [ -n "$(TEST)" ]; then \ echo "Running specific test: $(TEST)"; \ - sudo env "PATH=$$TEST_PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" go test -tags containers_image_openpgp -run=$(TEST) $$VERBOSE_FLAG -timeout=300s ./...; \ + sudo env "PATH=$$TEST_PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" go test -tags containers_image_openpgp -run=$(TEST) $$VERBOSE_FLAG -timeout=$(TEST_TIMEOUT) ./...; \ else \ - sudo env "PATH=$$TEST_PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" go test -tags containers_image_openpgp $$VERBOSE_FLAG -timeout=300s ./...; \ + sudo env "PATH=$$TEST_PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" go test -tags containers_image_openpgp $$VERBOSE_FLAG -timeout=$(TEST_TIMEOUT) ./...; \ fi # macOS tests (no sudo needed, adds e2fsprogs to PATH) @@ -250,10 +251,10 @@ test-darwin: build-embedded sign-vz-shim if [ -n "$(TEST)" ]; then \ echo "Running specific test: $(TEST)"; \ PATH="/opt/homebrew/opt/e2fsprogs/sbin:$(PATH)" \ - go test -tags containers_image_openpgp -run=$(TEST) $$VERBOSE_FLAG -timeout=300s $$PKGS; \ + go test -tags containers_image_openpgp -run=$(TEST) $$VERBOSE_FLAG -timeout=$(TEST_TIMEOUT) $$PKGS; \ else \ PATH="/opt/homebrew/opt/e2fsprogs/sbin:$(PATH)" \ - go test -tags containers_image_openpgp $$VERBOSE_FLAG -timeout=300s $$PKGS; \ + go test -tags containers_image_openpgp $$VERBOSE_FLAG -timeout=$(TEST_TIMEOUT) $$PKGS; \ fi # Generate JWT token for testing diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index a9266c60..3c94b48e 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -467,9 +467,15 @@ func (s *ApiService) ForkInstance(ctx context.Context, request oapi.ForkInstance }, nil } + targetState := instances.State("") + if request.Body.TargetState != nil { + targetState = instances.State(*request.Body.TargetState) + } + result, err := s.InstanceManager.ForkInstance(ctx, inst.Id, instances.ForkInstanceRequest{ Name: request.Body.Name, FromRunning: request.Body.FromRunning != nil && *request.Body.FromRunning, + TargetState: targetState, }) if err != nil { switch { diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index cb80a581..06531ae0 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -260,6 +260,7 @@ func TestForkInstance_Success(t *testing.T) { require.NotNil(t, mockMgr.lastReq) assert.Equal(t, "forked-instance", mockMgr.lastReq.Name) assert.False(t, mockMgr.lastReq.FromRunning) + assert.Equal(t, instances.State(""), mockMgr.lastReq.TargetState) } func TestForkInstance_NotSupported(t *testing.T) { @@ -367,6 +368,7 @@ func TestForkInstance_FromRunningFlagForwarded(t *testing.T) { svc.InstanceManager = mockMgr fromRunning := true + targetState := oapi.ForkTargetStateRunning resp, err := svc.ForkInstance( mw.WithResolvedInstance(ctx(), source.Id, source), oapi.ForkInstanceRequestObject{ @@ -374,6 +376,7 @@ func TestForkInstance_FromRunningFlagForwarded(t *testing.T) { Body: &oapi.ForkInstanceRequest{ Name: "forked-instance", FromRunning: &fromRunning, + TargetState: &targetState, }, }, ) @@ -383,6 +386,7 @@ func TestForkInstance_FromRunningFlagForwarded(t *testing.T) { require.True(t, ok, "expected 201 response") require.NotNil(t, mockMgr.lastReq) assert.True(t, mockMgr.lastReq.FromRunning) + assert.Equal(t, instances.StateRunning, mockMgr.lastReq.TargetState) } func TestInstanceLifecycle_StopStart(t *testing.T) { diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 53df19da..1722374d 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -30,6 +30,10 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR return nil, err } source := m.toInstance(ctx, meta) + targetState, err := resolveForkTargetState(req.TargetState, source.State) + if err != nil { + return nil, err + } switch source.State { case StateRunning: @@ -51,6 +55,9 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR if forkErr == nil { if err := m.rotateSourceVsockForRestore(ctx, id, forked.Id); err != nil { forkErr = fmt.Errorf("prepare source snapshot for restore: %w", err) + if cleanupErr := m.cleanupForkInstanceOnError(ctx, forked.Id); cleanupErr != nil { + forkErr = fmt.Errorf("%v; additionally failed to cleanup forked instance %s: %v", forkErr, forked.Id, cleanupErr) + } } } log.InfoContext(ctx, "restoring source instance after running fork", "source_instance_id", id) @@ -65,9 +72,13 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR if forkErr != nil { return nil, forkErr } - return forked, nil + return m.applyForkTargetState(ctx, forked.Id, targetState) case StateStopped, StateStandby: - return m.forkInstanceFromStoppedOrStandby(ctx, id, req) + forked, err := m.forkInstanceFromStoppedOrStandby(ctx, id, req) + if err != nil { + return nil, err + } + return m.applyForkTargetState(ctx, forked.Id, targetState) default: return nil, fmt.Errorf("%w: cannot fork from state %s (must be Stopped or Standby, or Running with from_running=true)", ErrInvalidState, source.State) } @@ -254,5 +265,79 @@ func validateForkRequest(req ForkInstanceRequest) error { if err := validateInstanceName(req.Name); err != nil { return fmt.Errorf("%w: %v", ErrInvalidRequest, err) } + if req.TargetState != "" && req.TargetState != StateStopped && req.TargetState != StateStandby && req.TargetState != StateRunning { + return fmt.Errorf("%w: invalid fork target state %q (must be one of %s, %s, %s)", ErrInvalidRequest, req.TargetState, StateStopped, StateStandby, StateRunning) + } return nil } + +func resolveForkTargetState(requested State, sourceState State) (State, error) { + if requested == "" { + switch sourceState { + case StateRunning, StateStandby, StateStopped: + return sourceState, nil + default: + return "", fmt.Errorf("%w: cannot derive fork target state from source state %s", ErrInvalidState, sourceState) + } + } + return requested, nil +} + +func (m *manager) applyForkTargetState(ctx context.Context, forkID string, target State) (*Instance, error) { + lock := m.getInstanceLock(forkID) + lock.Lock() + defer lock.Unlock() + + current, err := m.getInstance(ctx, forkID) + if err != nil { + return nil, err + } + if current.State == target { + return current, nil + } + + switch current.State { + case StateStopped: + switch target { + case StateRunning: + return m.startInstance(ctx, forkID, StartInstanceRequest{}) + case StateStandby: + if _, err := m.startInstance(ctx, forkID, StartInstanceRequest{}); err != nil { + return nil, fmt.Errorf("start forked instance for standby transition: %w", err) + } + return m.standbyInstance(ctx, forkID) + } + case StateStandby: + switch target { + case StateRunning: + return m.restoreInstance(ctx, forkID) + case StateStopped: + if err := os.RemoveAll(m.paths.InstanceSnapshotLatest(forkID)); err != nil { + return nil, fmt.Errorf("remove fork snapshot: %w", err) + } + return m.getInstance(ctx, forkID) + } + case StateRunning: + switch target { + case StateStandby: + return m.standbyInstance(ctx, forkID) + case StateStopped: + return m.stopInstance(ctx, forkID) + } + } + + return nil, fmt.Errorf("%w: cannot transition forked instance from %s to %s", ErrInvalidState, current.State, target) +} + +func (m *manager) cleanupForkInstanceOnError(ctx context.Context, forkID string) error { + lock := m.getInstanceLock(forkID) + lock.Lock() + defer lock.Unlock() + + err := m.deleteInstance(ctx, forkID) + if err == nil || errors.Is(err, ErrNotFound) { + m.instanceLocks.Delete(forkID) + return nil + } + return err +} diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index a3227083..6b2e60ec 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -48,6 +48,65 @@ func TestForkInstanceNotSupportedHypervisor(t *testing.T) { assert.ErrorIs(t, err, ErrNotSupported) } +func TestResolveForkTargetState_DefaultsToSourceState(t *testing.T) { + tests := []struct { + name string + source State + want State + }{ + {name: "running", source: StateRunning, want: StateRunning}, + {name: "standby", source: StateStandby, want: StateStandby}, + {name: "stopped", source: StateStopped, want: StateStopped}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := resolveForkTargetState("", tc.source) + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } +} + +func TestValidateForkRequest_InvalidTargetState(t *testing.T) { + err := validateForkRequest(ForkInstanceRequest{ + Name: "fork-invalid-target", + TargetState: State("Created"), + }) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidRequest) +} + +func TestCleanupForkInstanceOnError(t *testing.T) { + manager, _ := setupTestManager(t) + ctx := context.Background() + + forkID := "fork-cleanup-target" + require.NoError(t, manager.ensureDirectories(forkID)) + + meta := &metadata{StoredMetadata: StoredMetadata{ + Id: forkID, + Name: "fork-cleanup-target", + Image: "docker.io/library/alpine:latest", + CreatedAt: time.Now(), + HypervisorType: hypervisor.TypeCloudHypervisor, + HypervisorVersion: "test", + SocketPath: paths.New(manager.paths.DataDir()).InstanceSocket(forkID, "cloud-hypervisor.sock"), + DataDir: paths.New(manager.paths.DataDir()).InstanceDir(forkID), + VsockCID: 43, + VsockSocket: paths.New(manager.paths.DataDir()).InstanceVsockSocket(forkID), + }} + require.NoError(t, manager.saveMetadata(meta)) + + require.DirExists(t, meta.DataDir) + require.NoError(t, manager.cleanupForkInstanceOnError(ctx, forkID)) + assert.NoDirExists(t, meta.DataDir) + + _, err := manager.loadMetadata(forkID) + require.Error(t, err) + assert.ErrorIs(t, err, ErrNotFound) +} + func TestForkCloudHypervisorFromRunningNetwork(t *testing.T) { if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { t.Skip("/dev/kvm not available, skipping on this platform") @@ -109,9 +168,10 @@ func TestForkCloudHypervisorFromRunningNetwork(t *testing.T) { forked, err := manager.ForkInstance(ctx, source.Id, ForkInstanceRequest{ Name: "fork-running-copy", FromRunning: true, + TargetState: StateRunning, }) require.NoError(t, err) - require.Equal(t, StateStandby, forked.State) + require.Equal(t, StateRunning, forked.State) forkedID := forked.Id t.Cleanup(func() { _ = manager.DeleteInstance(context.Background(), forkedID) }) @@ -122,10 +182,7 @@ func TestForkCloudHypervisorFromRunningNetwork(t *testing.T) { require.NotEmpty(t, sourceAfterFork.IP) assertHostCanReachNginx(t, sourceAfterFork.IP, 80, 60*time.Second) - // Restore fork and validate both VMs are independently reachable on private IPs. - forked, err = manager.RestoreInstance(ctx, forkedID) - require.NoError(t, err) - require.Equal(t, StateRunning, forked.State) + // Fork should already be running with target_state=Running. require.NoError(t, waitForVMReady(ctx, forked.SocketPath, 5*time.Second)) assert.NotEmpty(t, forked.IP) diff --git a/lib/instances/qemu_test.go b/lib/instances/qemu_test.go index f1db207b..2937ba42 100644 --- a/lib/instances/qemu_test.go +++ b/lib/instances/qemu_test.go @@ -930,6 +930,7 @@ func TestQEMUForkFromRunningNetwork(t *testing.T) { forked, err := manager.ForkInstance(ctx, source.Id, ForkInstanceRequest{ Name: "qemu-fork-running-copy", FromRunning: true, + TargetState: StateStandby, }) require.NoError(t, err) require.Equal(t, StateStandby, forked.State) diff --git a/lib/instances/types.go b/lib/instances/types.go index 1a7d5d74..f7679285 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -179,6 +179,7 @@ type StartInstanceRequest struct { type ForkInstanceRequest struct { Name string // Required: name for the new forked instance FromRunning bool // Optional: allow forking from Running by auto standby/fork/restore + TargetState State // Optional: desired final state of forked instance (Stopped, Standby, Running). Empty means inherit source state. } // AttachVolumeRequest is the domain request for attaching a volume (used for API compatibility) diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index be416602..210e7930 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -59,6 +59,13 @@ const ( Pci DeviceType = "pci" ) +// Defines values for ForkTargetState. +const ( + ForkTargetStateRunning ForkTargetState = "Running" + ForkTargetStateStandby ForkTargetState = "Standby" + ForkTargetStateStopped ForkTargetState = "Stopped" +) + // Defines values for GPUResourceStatusMode. const ( Passthrough GPUResourceStatusMode = "passthrough" @@ -88,13 +95,13 @@ const ( // Defines values for InstanceState. const ( - Created InstanceState = "Created" - Paused InstanceState = "Paused" - Running InstanceState = "Running" - Shutdown InstanceState = "Shutdown" - Standby InstanceState = "Standby" - Stopped InstanceState = "Stopped" - Unknown InstanceState = "Unknown" + InstanceStateCreated InstanceState = "Created" + InstanceStatePaused InstanceState = "Paused" + InstanceStateRunning InstanceState = "Running" + InstanceStateShutdown InstanceState = "Shutdown" + InstanceStateStandby InstanceState = "Standby" + InstanceStateStopped InstanceState = "Stopped" + InstanceStateUnknown InstanceState = "Unknown" ) // Defines values for GetInstanceLogsParamsSource. @@ -406,8 +413,14 @@ type ForkInstanceRequest struct { // Name Name for the forked instance (lowercase letters, digits, and dashes only; cannot start or end with a dash) Name string `json:"name"` + + // TargetState Target state for the forked instance after fork completes + TargetState *ForkTargetState `json:"target_state,omitempty"` } +// ForkTargetState Target state for the forked instance after fork completes +type ForkTargetState string + // GPUConfig GPU configuration for the instance type GPUConfig struct { // Profile vGPU profile name (e.g., "L40S-1Q"). Only used in vGPU mode. @@ -11095,182 +11108,183 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y9+3LbOLIw/ioo/s6plc9KsnyJ4/jU1CnHnni8Gyf+xbH3OzvKp0AkJGFMAhwAlKOk", - "8u8+wD7iPslXaAC8CZSoXJx4J1tbE5kEcWl0N7obffkQhDxJOSNMyeDoQyDDGUkw/DxWCoezGx5nCXlF", - "fs+IVPpxKnhKhKIEGiU8Y2qUYjXTf0VEhoKminIWHAWXWM3Q3YwIgubQC5IznsURGhME35Eo6AbkHU7S", - "mARHwXbC1HaEFQ66gVqk+pFUgrJp8LEbCIIjzuKFGWaCs1gFRxMcS9KtDXuhu0ZYIv1JD77J+xtzHhPM", - "go/Q4+8ZFSQKjn4tL+NN3piPfyOh0oMfzzGN8Tgmp2ROQ7IMhjATgjA1igSdE7EMihPzPl6gMc9YhEw7", - "1GFZHCM6QYwzslUBBpvTiGpI6CZ66OBIiYx4IBPBnEY08uzAyTkyr9H5KerMyLvqILuPx4dBc5cMJ2S5", - "01+yBLOeBq6elusf2pb7fr7v65nyJMlGU8GzdLnn85cXF9cIXiKWJWMiyj0e7ub9UabIlAjdYRrSEY4i", - "QaT0r9+9LM9tMBgMjvDu0WDQH/hmOScs4qIRpOa1H6Q7g4is6LIVSG3/SyB9cXN+en6MTrhIucDw7dJI", - "NcQug6e8rjLaVHfFh/9PMxpHy1g/1o+JGFEmFWYNOHhuX2pw8QlSM4Lsd+jmAnUmXKCIjLPplLLpVht8", - "1wwrJopEI6yWh4OpItuGcoYUTYhUOEmDbjDhItEfBRFWpKfftBpQELxmON2i1WDLpJaZnRwlsql31wRR", - "hhIax1SSkLNIlsegTB3sNy+mRDBECO7hUD/rxyghUuIpQR3NNjXvZkgqrDKJqEQTTGMStdojHyKYxfzG", - "x4hGhCk6oVX6NujUw+NwZ3fPyzsSPCWjiE7tSVTt/hSeaxTT/SgErf0L0YS2aLcOGFKQyfJ4z4B1wyCC", - "TIggGsc/c7hU8Dlhmlr0eP8B4wb/33ZxRG/b83kbgHlZNP/YDX7PSEZGKZfUzHCJc9k3Go0A1Ai+8M8Z", - "Xq3a6xJGSYXFavqAFl+AEs38WsHmyjSt80Ngd7abCmU3sr2f54R5BJ+QM2VfVFf8nE9RTBlBtoWFr+Zz", - "eoCfYg5s7kusrRsUIF0maD3vT2BI5kFDb/pdNyAsSzQwYz4tQ3NGsFBjUgFmw7FkOypm1wj+ywpJ1M4f", - "LMloNVe4pIyRCOmWllhNS5RJkD6Xlg+UcUvVaE6E9NIRTOuvVCHborGrmIe3ExqT0QzLmZkxjiKgQRxf", - "VlbikcAqIi1ONWNzHYJkIJHi6OqX491HB8gO4IGh5JkIzQyWV1L6Wndv2iKFxRjHsRc3mtFt83N3GUP8", - "GHCVE0bTeZJjoENMw70Cu5u6+26QZnJmfgE/1rOC80yzAY1esf79xrPoE2ASRvJv1IP8ct3L1Gw2msZc", - "w3SBMkZ/zypCcx+da/lfIc38aUSiLsLwQrNhnCnemxJGhOZTaCJ4AhJUSbBFHdKf9rtoqGW9npZse3i3", - "Nxj0BsOgKprG+71pmmlQYKWI0BP8v7/i3vvj3t8HvSdvip+jfu/Nn//DhwBtpW0n6dl1dhztd5GbbFkE", - "r090tXi+QsL1cRGzfeea9jfdvZPz5QPezD/i4S0Rfcq3YzoWWCy22ZSyd0cxVkSq6mpWt127PpjbioWx", - "qV76hkurKRyAbp2Y3xERak4ZE40gsquZJVWyi7DWWYHJIH2a/TcKMdM4aw52LhBhEbqjaoYwtKtCIFn0", - "cEp71Ew16AYJfvecsKmaBUcHe0v4qJGxY3/03vyXe7T1P16UFFlMPMj4imeKsimC1+b0nVGJijlQRZK1", - "x62DbhaDiJVQdm4+28lngoXAC/+uucmt2j2jHDVuX5h4JOmXcyIEjdyJdnJxijoxvSUWLZHIGBpmg8Fe", - "CA3gJ7FPQp4kmEXm2VYfvUyo0idJVhyQxrrSL2/hrwEJZxzO+DjmekE5+BoECAcXp2h6tujUWSYkstou", - "nGkY7E6wZWeX19uaq6RYSjUTPJvOqrOyLG2z+VB5O6J8NE59c6LyFp1vv0Sa4aKYaujkDHZnMLh4ui2H", - "gf7jkftjq49ODchg+nr/uLB8X86wICB9RIgzdHJ5jXAc89DqcxMtJE7oNBMk6tfMCNC7D+EJU2KRcuoT", - "PmuYUTRdRpBer3i7AR5sjynblnobeuFmcCds/hki0M9sTgVniRZD51hQzbcqRp0PwYuXpz+Pfn5xExxp", - "Ioqy0FpILl++eh0cBXuDwSDwSRkag9bwgbPL6xPYKd1+xlUaZ9ORpO89rPU4Xx9KSMKFEf3tN6gzq3Je", - "Ixkh2JxhsHf21CDXzhnglduUiEpo7XoxHVcxZvfsqQ9bZouUiDmVPp3/l/yd2/kSnzSMqYrbkog5ETnS", - "Ahb3S3JXGPMs6pWG7Aa/k0SLHPP3GlmK2Xpa+pXvVsfymvMWxyllpPHA7QYJURgMz5+On9eSiF5EJlSr", - "G7dk0ZvjOCPI9WwhS3LAVlE3zUTKpekfT42YqghOgqNgjMNbwiIv5n4nh/sdF7cxx1Fv5wuf7Ywo3ffy", - "El+YF1VM9MG4rjCy6I5GajaK+B3TU/acAPYNyhvnx8A7vRIc/+sf/7y5KCTZnbNxas+End1Hn3km1E4B", - "3bVXS80XkqX+ZVyn/kXcXPzrH/90K/m2iyBM42dUudExhp/qUv42I2pGREk2cBusHxk1Az5HDl9Kw1cs", - "SeXrnyVi4nMiYrwosXU7p2BnALy1NitBFdCX/U4z6VukP17D5HVvToQ4q6s+uwM/G/dMyjOnp5q+7anT", - "Zib5RHZ2L+zP3eUpNczolqajqZZaR3iaW8JWXcxd3dIUwRc9+MJsYxwb4o0y3TMac676Q/a3GWEI9g42", - "mLwjIfApreqj48tzie5oHIPeDIxg+eAastclVmCaS6X/KzLWReNMIUESrgiyIjEMksFcoPGYoIxhd/PX", - "H7IyVOwC63hlwXJLBCPxaEZwRIRsCRnzEbIfNQIHljrBUhFhOHSWVuF1+teLK9Q5XTCc0BD91fR6waMs", - "JugqSzUNb1Wh1x2yVJA5YaAxafGH2nH5BPFM9fikpwQhbooJdJZbHuy11Pzs8tpebMqt/pC9IhqwhEUk", - "gjm7U0IiNcMKRZz9SVMsHJelbsvj14Dup+VuMA/TrArl3TqEX8B1ol7PnAqV4VizrIr86L1dNPfWHj3B", - "XIuX9RXLinKEw6p6LdRW5TQ9wyX2shTt1zKNoNSsZa65w/dd1uSWqzCTiielKxvUqRmlaNV8VWUecx73", - "tPwDosHy+e6VX8x0l68/k4XpymxKE5ccTcceS6dmhpShKZ3i8UJVNYedwfLW+wHt+veBusk1wKAHiUaK", - "r74cpRPk2ra5CwFHgpHio/mEenrOD83CCkclCmt+CBZpdRe9NKSWfLvobkb1MSuRAwJQ8M1FWRPvD1kP", - "WM4ROs0HyLvNu9ScFSyu0EWHi9IkKBjP0XixhTC6ueij1/ls/yQRw4rOifOVmGGJxoQwlIF4RiIYH9hp", - "eQKZ1DyMqvrnllcZt4otMDhw+66PtCKUYMv3NXonWNEQDLZjWlsPXJSZjdIjaQbAyqdOq1Ni1ZXyKzKl", - "UonahTLqvHp2sre396QuL+w+6g12ejuPXu8Mjgb6/39vf/f85T1HfH0dV/mFNYGXOcrJ9fnprhVOquOo", - "9/v4yeG7d1g9OaB38sn7ZCymv+3he/Et8bOn08J2jzqZVvsc69NY5bPYlwzjDRb5Tza0b+TW4q72Vh0/", - "ZnWvdcuv4Qjju461l4Gbu6rUmeDaC93S4pbWo59q+aDA/JJhw96bhNR7Q3RK5e1TQfCt1io956s+nuXI", - "nDt+i2Om9ajxApF3WjwjERKcq4k0do6qmLKz/3j/cO9g/3Aw8Ph/LCMxD+ko1KdKqwm8PDlHMV4QgeAb", - "1AFFL0LjmI+ryPto7+Dw8eDJzm7beRg1qR0ccinKfYU6FiJ/dr6E7k1lUru7jw/29vYGBwe7+61mZQW8", - "VpNywmBFdHi893h/53B3vxUUfGrnz84fp+5fEPmMi2kaU6Nk92RKQjqhIQKPHqQ/QJ0EjiWSa3xVmhzj", - "aCSsGOg9DxSmsVxp0zSD2ZbGfSvJYkXTmJh3sCGtJF1Y+Sn05LMXU8aIGOXuShv0ZL2Y1lrG3FryJqji", - "jVYB3QWVIFkUAhElcXRkKHQtn4PdLCb2pgkP7BpaYsNzfkdELyZzEpeRwBxHerIJFwTleGI2rbIqyuY4", - "ptGIsjRrsIw2gPJZJkC+NJ0iPOaZMqo6bFh5ELh7BR1hotl1u6v/Z1zcrr0d06frSGSM6W7WatnHcczv", - "9BbfatjAyYyR/do5P5QEuVylNoYH+16iV+YLY5goHqeZQpQprpVyFo0XXRiJRNCOIUGk4sBJcXirpUbb", - "TVuJ0S+LvNBCiDN7mvEK3nlPNt/exJjcvpzht+0NeHErswSYs8vrTY3DqeATGnuADIYN+9aKfc5s+nx/", - "cNXb+f/NZQ2LF+asoMwYQxIekX7NKRratyOBs8vry6Y55R7pqDy7pTXl5isP6uQWEQcRa5gJMUNjgqwo", - "ZZAATGvFIIUQ8MR3qE4ETsg4m0yIGCUebfyZfo9MA2OnpAxdPK0erPoAbyuSX1Y2B2TyCQ6tQ3E76Hu0", - "/doyuiVoNmDjK2L4QZPLkt4qYdtYr6U+epHHAKCzy2uJCpOjxwxQ3d7Gq/XL2UJqBdb0aDwQKStr74Cc", - "rY/qy+JDa+fwHNiJ95ByhIA682maARleveqdv7zZTiIy71bmBGbCGY+JnvdWSf6eO8elwg+gcp04b1Kj", - "DGLItgRUglVOwa2BVKJXD3QUVzgeyZgrz2xe65cIXqLOzTPjsKJn0EVpZSv18xIUKvh94KUYzZGahr2C", - "Aev2mAqBrzWNJUa0KS+vMqiPVH4hODYRQ1V8Lnxg3cbz2+pG89u11Gs78Y177m6UWzjZnFycGhkh5Exh", - "yojIL3OrHhHgzRV0g54WGCNMErCbT/57tXdEg30vR5dVFqKTpXCDr2IdanCp1UwunpMIJZjRCZHKutRW", - "RpYzvPvo4Mg480dksv/ooN/vb+rO8nPhv9JqK7bN3X/Js6UvZ5+3D1/Ba6XNWj4El8evfwmOgu1Miu2Y", - "hzjelmPKjkp/538WL+CH+XNMmddnoFX8B50sxX1UtjfVZ5Z5fqRXwkiYIyQHTWKt/bpBoNWoGdP3JEJe", - "90uFp1o6NRj3eX6WnxExUYTtqVKkRPkqqUXUBH2/2iTjBCNoY8fMmKJxEVCybIz5pJAgudLDesm7OiUs", - "96mOY/Mr5GyuqcLnYF1h4O7d0mbcGb1sFFEPdv7NKm0RFSRU4FS1noaCbZym61HRL/zlPK1tsIh1FfWc", - "Lt+ck3+KUb46+svpX37/P/Ly8W87vz+/ufnf+dlfTl/Q/72JL1+2v9bzuCWt9hL+pq6+K+9dwRJdcfFt", - "ix4XWIUewWfGpWqAmn2DFEeJ/riPTkBBOxqyHnpOFRE4PkLDAKe0b4HZD3kyDFCHvMOhMl8hzpDuyroX", - "bOmPL42Grj/+4HTAj/U+IutHICyQc5cfmY0jnmDKtoZsyGxfyC1EwsWe/hWhEKcqE0TviJY14wUaCxwW", - "/gPF4F30Aafpx60hA02UvFNCryDFQuUhBW4E2Gg7K3NxaJuTCIHvnbSa7JDl5weo5roThcWUqH5uKgFj", - "Xs0U0wAUr5rBRdX/5XDQ9ewj0u30RsZUKsJQbpWgEpAXdZwj0+GgQv6Hg8P1d9Q5Dq1AP8Du5SB+h5Qt", - "6MMgMAxtmPFoplTawvim+Y2hEfTL69eXGgz63yvkOipgkW+xUcZwmsaUSHPzqmKQSazv2Fbgs5WZ3W25", - "oNemsf4sbuGq8zMMjF4/v0KKiIQyw787oQbnhIZ6fXAHSKXMNCpSjI5PLn7e6rfIQgCwzee/Yh9f5yus", - "XTU549ayhglfFMZBDd8uOj/tanHKUmghaMHd+jMuUGwYTEHXR+hakqqnC2yVuQY0OxkvCguZ4erDYMv1", - "mNY5xRF6lct3OJ9KHupUIIPrsqBL6NZaas3F/1Lv3epcwaXB6i+WtcE1P1bIXozAUdzMClaTvwfiQPOc", - "NTr/tqLtstFSD+ZHjWLvv7oEsrepLrlpzEXVcbHkqJqHXbSPl/gacQfLetU7qkaNt3ZIv7Z3dE57uLlA", - "MyzZnxS8rOkQO3uPW0Xz61Hb3neVb7r4xEwppyrnBZnf0xh/0Fsax+b6U9IpwzF6gjpX52d/PX/+fAv1", - "0MuXF/WtWPWFb39ahF841D67vIaYBixHkuFUzrhq9nzCyLXRK5VKLjuJtrp5WR3u8UslJMPrdbv1BeM0", - "3HXV0jK+TgTGt/Tm+TeK/gg+J3ZjZbTF54ZMWKH6K0VMNDJxX7RBlZ+bx1829uGrTKcSxeDjQ2XZwzl4", - "fnLgQjegHue2Y6lZLYnQ+WURFl4YqVz3tTU92e3vHBz2dwaD/s6gjckuweGKsS+OT9oPPtg1RowjPD4K", - "oyMy+QyToUVsIyTi+A4vJBo6MX4YGL2hpDCUmIUV9Vtdxy7Hh3xaOEhdcFkX8LFJgEe7yI0V+Vquqpla", - "WsuCj/7+WUldSFsJ4Aoau69GmxizCQp5Fkda3hpryjPqG4mslimJKpLgALFes1vG71h16camqen394yI", - "Bbq5uKhYwAWZ2HwgLRbO07RxH3i60TbsrhHJ186mFERxH4ETdU5YOoG+eJhE2Vzn/LUM1rUw2xWSp/dq", - "mzIDbr33K9ZUM7hEZD7KMp94pV85z+vr6/PTyoZjfLBzODh80jsc7xz09qPBTg/v7B30dh/hwWQvfLzX", - "kDmrvWvLp3urVCm0OdIBAA/GSxOcEh1pGsrdTcaZQrkjkibOEy2nopIIbPz6wZ5gfah0D3C6hvpNvMil", - "5JUfX2JNqO7bFP5a/cXVLFNaDIJv5CxTSP8FU9ZLsFrG6i4MzR+hFxy+Ec6ZjPG6umKagy/ZcvO6atOx", - "XjvOzQwGswzsCD3LmVbO9iyb60hifxpeaj0gwbtzyxhBrGZhdyvoBhbqQTcwIAy6gYOM/mlWCL9g8kE3", - "sBPxOk+X8cZn3Cc4Bh5WOM5kisb0vSE5PXUqFQ2NdodhN5vIzkaokmhkjtCm6zfjjWGP2fwjR9U3F6gD", - "8UR/Rlb5039t5Vd1ZRLa332y/+Tg8e6Tg1beyMUE13PjE/AVWp7cWtYcptnIZRBsWPrJ5TUcPvpgk1li", - "tHm79kJj1Iwj1NIeZahISVgM/qT/pOyEHfFsHJesQzYKAzx92+SPbLib+p3GczqZsN/fh7e7vwma7Lw7", - "kLtjr3KUD+SXJM/LFs0ltYuMeyb/gV+HBIQSstGV/BWRsAJ0RRQC/OlphqVP1NzFx6Kcczi3EPci1v7e", - "3t7h40e7rfDKzq5EOCPQ/5ZneWFnUCIxaIk6r66u0HYJ4Uyfzu8xFUTqxZnoKC+dIZuHZlBxidS6x54P", - "SxoElgJrbN/zpBHkN1ZisYuyQAdPpVyaWaJyL7T39gaP9x8dPmpHxlbjGYl3qzmMbWdv+AUJCZ1Xdr4D", - "VvDXx5dI9y4mOKxK+Du7e/uPDh4fbjQrtdGslMBMJlSpjSZ2+Pjg0f7e7k67mAifpdtG+1QItsq7PETn", - "QQrPbnhAscx6u02nhU9KXHaPXOmRWbh41v35NnHgLSI8qYReacl3FHW0EFUWSEtRiltt7Ax+FqnHacpL", - "rMXFtr61q11pL7GanbMJX77K2EThsw5KzsSdasFHQsbGiDBKIse7cs3PylLg8hRLgqKMWMgZ2UhgC3Bs", - "rnNSrGYgrMKHlE2rzt5LA7ZRw8wcVsfzwri2YRuLkfQ71bwWGcDK2JIlwoV7TSvDOJUjv1ax3LEg0yzG", - "AtX9x1dMWS6SmLLbNr3LRTLmMQ2R/qCuzk94HPO7kX4lf4K1bLVanf5gVNwk19RzMznrR2A2pDZusYSf", - "9Cq3ap5JcPJvm++3IfF8GwOc93rpmVbejIv1NaPvSoheDY7b3x00OaI1dFpxQVt2z9+Ut1uU9VG885w/", - "zpPIeK4xzUVRTYOtysGV9fpWCzeRq9zuliUB1HE2PRd8WIVrKQiw1UHc7jK0br12s9mWJKyOvn/46PFB", - "yyjMzxK1V6Tm/gzBep6sEKgbduqijdR2+OjwyZO9/UdPdjeSj9xFR8P+NF12lPenliuqJrM9GsD/NpqU", - "uerwT6nhuqM6oUrep0+e0McVpFsEvTRo3avKYhQ76dT8qgDeTsRdIS0dV0SuUhLGDplMCBiORgZuvWIy", - "NSesVnMIcYpDqhYeDRDfgV8KypvUgjda9F6brAektm+EJ4oIuI2Q2bi49++4wdF/Gc2uhguHrYO5ZTZu", - "0iJf1kc1OqRx5IpqFooWBgKDEb7L97scmOgOy4pVX/8OFYm6pSSb9esf06J9GnSH63km9OI63ReA5M96", - "Xt7+2naWtI6KkFyH+KojtJkEtUQAXmJtDOyeE9kT1RSud96o8Qd7AH7aV6NxOc3CyjwWlZwMxam7+bjt", - "0oMuf2dOsM3HK93gb/JhPeIc8NHOwYK86LtbQQkfNpn7laZ0RomrF1ULSKemAoeNKEOlxqhDklQtXNSE", - "00y3NrvvOc479CLjF/ZzGzz5Ep721ytd6/9NEmSVr9jcIGsv15b2tNGf1S+untbdV4xOaBOEVN0tamkP", - "pFpRaGZVUTNTXQwUPutLPs3qwW8bFDJrUvELynEVZFwls3Wa60p7WmllpZk07425X/3Mqm9UunJvnwgy", - "q36td842d1RaAe7VM8iY2GJBQZ+zADKA1SDIVfRlO8Bqt48L/C4fAbRlLFEt56ZZRymL+NlTyBfwymUS", - "oRPXBUyjnj316eeVw3NYtbwZq+rjuRt8L+FZ/rOCozXRVg05izG6q0vwadZFwkxQtbjSB4J1TiNYEHGc", - "GTSEkwIWAY+LwSFA4eNHUFMnHmn1jDAiaIiOL88BSxLMIAczurlAMZ2QcBHGxPqXL93tQnqElyfnPRMY", - "kye5hOo2CgDisssdX55DYitbVyYY9Hf7kJGcp4ThlAZHwV5/B1J3aTDAErch7hB+WkOUpkM4yc4je+I+", - "NU00aGXKmTTA2R0ManWKcJE8aPs3aSws5nhtLRSaQnDL/hZLDpFOErDT/9gN9gc7G81nbb4f37DXDGdq", - "xgV9T2CajzYEwicNes6MVu3SoxPbsMDZ4OjXKrb++ubjm24gsyTBWkQ04CpglXLZJMIQiTBi5M4GpP7G", - "x310ZXQS8C4vSmwakwGJNEvCSGHRn75HWIQzOidDZjmxyd2EBUTfJEhzYBP7UEUzM7TZfUPCRKqnPFrU", - "oJt3t6276zmv2gLAGxdwyhORpg2VnHzc0eQ7kyH3JnojDDNVpM8yic5uCVxiTug7b4etbuM18yjXNnTe", - "nbtbfjsguCr7Tein+TtXR6x6YGgZmrIwzqLiVK3Wb/IG25s6RDYf3C3xCCFn0MICpezV7Y4vxiNifGXT", - "hZpxZn5n44ypzPweC34nidCHnI3NsbC2yZAs6kKyTZpAfIyJ5tVjbpspbn+4JYuP/SE7jhIXfW1zPuNY", - "cpsoz3g9UInyzOND1lw50C9Mn9gEtyZJVTmfkpkmz1SaqT4yCyHKBhRBc0j7JGckGjLF0QdhMncuPm5/", - "KEb8CNIpwZHGk1ITs6TtDzT62DRrOcJ69aOxK79Zk9kJAGAYaKlhGOjfU4G1dJrJGcIh+Gboh+Ut7RjC", - "5gJO/q06hEPMUMrTLNZyFCCVyf9X6QOCM3EcIwWk5L7V8gTsZMN6rEnXlxHI2nONAa5GRpAbqERMg/1D", - "Pz1JEgriU0v/cvXyBYKjCsqcQbMifABgRJkWNPI823r0/pD9jMMZMjII5J8dBjQaBkU5qy2YayaJEQN6", - "PRBifoI6f2aYLo1+6vd1V0Y+OkK/fjC9HGlaSpOR4reEDYOPXVR6MaVqlo3zd2/8AG0yi11VGAHqGN6/", - "5cLl9QpLx6A5NzCLELe8Nl4gjAoOVNZ+x5RhsWiqDccz1exaZLIJ2GbFfh4MBlvrr47sUj2SYaWhpoSP", - "S4LQ7heTAaz8sywDlOrA6hOX2VQRkZF87kEIeYojF8H4Q9paI21ZNbEkR8H3ZZZs0DcmxpW1JgxBuUAn", - "DKVY4IQoyPP/qx/nwYuX6r/dRS+cRMZoUkXebgk8dd3pzRJi7zfWYcwrGgIu7N8D/sG4RZJHGPfJfY2L", - "Y5NiPK8N/aDQETbLIWLXr+idEfU9YNzgvlipy0X7DfH3oeDPGbEiWAG0GjfbhuIeZStCPdpEEJxI24tp", - "rNXGK5hT74owhaACsOzbf53yAY78b2M+fXuEDAhjW/9Y2syRubldH4oWlvCRSayTf2fzTYUzzKZEoo45", - "P//1j3+6Gq7/+sc/bQ3Xf/3jn0Du27YiOXSXVx9+e4T+SkjawzGdE7cYcE4lcyIWaG9gyxnBK0/2Kjlk", - "Q/aKqEwwmbt26XUBTEyHNqurXg9lGZFIAgih9sDE+hwZa55Hm3a0bEB5rxTdXVJ/7ApKC9CnosMBuESm", - "jCqKY6sKuXlA/FMxEbPmoDx43TC5ZKpez18UeacM9vbMBDdkMKZ6t4fuTEFr0yfqXF39vNVHIO4brAC/", - "MtAbim6sJtD/wZPW8yTDUaoMBaBseFMpcWmjWfPUtrkPu2ZTUtNmw6ZR5InWjd1ifojdLYycfrg5g6fP", - "6njqijE0mx0/fb2+4t6tdMovt88O95ZhbiuNFCD7Ftok6tgk8Xmen0o5k2+F9PfCgEtVcHIujLjJLnRv", - "Gs4JZ5OYhgr13FxsteRc66kiyENhB6/srBF266oHQ5SPiu2Kb1/joZG7+d3n6VEbdJNjpAjYKHDtx0my", - "DnVOqQy5/raELb0QpzbLEQCxoNMyFq2z7ZzC8/zIWSmY5/XLHUHen5XHDp2x+tlwD0zxtMYQvyEjrGVU", - "KYU4PSRsvs530RWeWmEE+r5Qc3B/UtB9G4R8aP6QLEJRDWyaC87yvPtN6GUz83/FjbYjeBZ+RYSjajNR", - "k8mjWJb5FIUzEt6aBdn6aKskgnNXQu3rywGmvMAGp7+d/o/jvoXiWMBqlbJ4btO7fD1dEUbYSFX8cteP", - "FsE8QAZ3hLEzpJrMKVguWLj1h7qBvJeToV7P7AFR0mUWx84QPydCFUUWyvx0+wM4rqyXkx21rZRFrl89", - "7xEWcvBUyr1s/AKJy6n+ZaVls2FmKT/QpI1+BaByiNEsjH7G/huHMpSn6/zP3Wc2Yed/7j4zKTv/c+/Y", - "JO3c+mrIMrgv1nzf0usDRj4tvNIq0IA1mczn66S9vNW9CHy2xMQmIl8+wR9SXxuprwyulYJfXu3jK4p+", - "tojCt7knyJHNB2145fzP/mAi3/2anixGlupiVmzxNocMF0XhAltV7+E5yNEc48r8t6UNtSDIldKBQ93z", - "066tSWEqSeS++PdkUXXzuHcp0Y57/+bU42RMpxnPZNnNH0qQEFnU9K4w4IcmvxbHc6ME+x1j6eA+j457", - "F1B/4P1XEp3rG2qYty3xvEZ4dq3WkMMzGqtSZnsJxT5MPlkTJpTXSLfZYLcanMZcFua2aFxJAr7szOab", - "l6vBUCrLkGIqZB9dS6LBRNKXJsJDqkVMjobsf9wnvyqCkzc/jXF4S1g0zAaD3YP8HWHzNz9JBZGiQ3bh", - "8IYwJSiRCAuCjl+cwsXUFCJw++g4jotQqPp8UJJJW6zNVR9KY8j3ZTiHD3yl+hIFBFuXsFiK+QYAQF4U", - "B5Pgs9lSSz2quLVrr0g5ZP2hSLVSpErgWq1I5dmTv6YmZQb5ZqqUwzcfwG3igB/K1H0oUzKbTGhICVNF", - "5q8lByebOPABhhgxex9TckyonMetlakipflqOdUi77dwSskHv38dyqUXfJiu0twER0ROaykOw2a15XvD", - "h8H9Muf7V1ceMoqdlSt/+hUDEyc0sYnH/ALCMy5u22KejUGmXxMBv7x0Ul7hdyib6OlBvodvL6LA4W38", - "wjXSVCWXeyDIOn59U3dEBwmrA5vYOFOS3BSiuaNqxjOT52JkH5qcU5oqbBZpEHlC2+u3Zi969HsQQF9w", - "hWiSxiQhkJOqZ7AJKgBlacpFXneAylLpkc3YnyabsnOoSTdiy2919a6xaLwAm15eOQjM+8vb5eWaMZ+u", - "j63MB3eBhJ7gyiFzxZXfGlH4LcqZLFIcSRKTUKG7GQ1nEGipn0H/Jg4Tp+nbPLPC1hE6A0ot53qAwTuS", - "CIpjqO7CY1OY6O08Sd4eLee3urm4gI9MjKXJZPX2CLmcVvkBIXWrcuCkXkWMpUIvbDhoR2OS4K447Ft9", - "CpXWt2VDKoskFEPmC69k5M52SCfobSnS8m1DqKVjqM/1Ln0jeanbnDrIrEVxJABwBjcJixosZhpq/iDL", - "nYE3TWPLgE8zja8c77k0med8mqctqqAyTtO26GunCVg8T5IVOIw6peJFUkU8U3+WKiJCwMcWu5uQG3Vw", - "aP5Q+NZU769U4TXlsrxWTpO8xAsqzVRLVbbMX/MkCUxJ4AT7qmZ9fuBsvcNlM5vemVJ07A9Je5O41yqz", - "LwW+1k4OW66tWeR+ZRr84fU9V9fuG6PhN7CPFbOgzIkqsLdFwcCHFfUHG7kki5lSgT4ace8aacTWGfzD", - "00iBH39wKgm5EOZCzxYLfjju2SU7TYncO1CdtKj62XW2wpuLi60mohFqJcmI78OI+Gk2nFphoCTy16AQ", - "NHKpGk8uTm1iRyq1ltdHLxMK+RNvCUkhGwzlmURwI9svZ6VvujfN084TpsQi5ZSptbMomn6dyXz8pPx4", - "98ynbIDKH/4oB/PDw2NSpkQ8zhew6mZL8yHVaBpxpoJKDe0xz3TvSxn2oXacXEhFEmMnmWQxEBGEA9rE", - "S7hcG6+LqJJQqaUL9yulumhDNiYTLYakROix9eeQbLdQ+XzWhCuFc655aVjf92FOgKT7oEFj1QS1WgG6", - "NHX59n0qa14i4JOn9AzsA9XafBJ1YnprCk6juUSx/rG10sBgCvd96bRSn05ZeWlKX7oQg7M5Mv8RONx5", - "ja05A+qDY2tnpEwsjv/ARvvZmlzL18SGtcsd7Eo1zPtDdkGU0G2wICjkcQwlq4zatJ0KHm5DXeUwpZEp", - "sAyTA4bX/DqBEU8ur6GdSczcHTL9x3Jl3/pEXYHg8+2Xa0yupqb7v7E+Zha4iiz8G/7Dmrb5vXUjDckG", - "EuXpKgWIp394g4GV4H5YCx6mtQAch/LVdKYChyAUy1mmIn7H/JYBW8F2+4P5cb7O/UzhcHbjCop9H9Ku", - "rT+0bhi3wAdBlHZNETHpjO6fJnleIuqBhqxrwLklgBBTdqTznwKm9NwfDbu/vNtUGY4buU3dK225VGHf", - "DW3d98ln5+C8qsvweChkbjDNrQQKt5StT6JcA3elbuZKlEJB5ly0dKV5u+UK0SYzeW5DKkoL5sVo+0OW", - "V991mdG1dtV1qhWKqLw1PVjtqY/8RZKNnmcrJUNZoRDHoanHk1cLNoVyZIP29apUQfur0VsxiGej8zLJ", - "Mq9q+5BUDj9OwO6Vy+YCxllxamVc3Y1tcx+hVPYw2yCQyq3gRxhVizCqErDaFOkzNY8tt7LFaosIQEnf", - "w+2OLwwrF0q+XhDWJ5zXXw49HJ42ntY/wq/uTSAoUlmcnz78mKsyzVV49LbWCnq2AmbZNLSKgi2IUkF6", - "ri5eZABm4WF0jXqBzf6QvZ4R9xeizoOVRCiigoQqXiDKoHyhq5P8J4kE58q+52LRXIjTkMgzwZNju5o1", - "ykvriuG+i5iN82x1PVWSaZIlpkIyZejsKeqQd0oYh0o0wTQGd14HUvIuJCSSgJNb9UrkXg/LvOT42lmu", - "cI3Na42GpibkPNfEOjhTvDclTO9FUYEzFXxOIxPe3lDS3Tdb0BC/gJI2fU/TKumtrdO3THhVvEV5yVFb", - "KLDAT7c7wY9jol4ZAfILcJEDUXGOYiymZOvHUfKQj5KyNcmdG5UTpV30bjsDU0u7z9eI3M2Nj/cbt3vz", - "/dhESpnkH2Cio3mu9DUFDH9fKDi4v/PhvgOFbx6wDf2MOAW3FCQMHegefQjznIc4RhGZk5inCVRUg7ZB", - "N8hEHBwFM6XSo+3tWLebcamODgeHg+Djm4//LwAA//9/tTsUefIAAA==", + "H4sIAAAAAAAC/+x963IbubHwq6D4nVSoE5KiLpZlnto6JUu2Voll67Ms5TtZ+qPBGZDEagaYBTCUaZf/", + "5gHyiPskp9AA5kYMObQt2co6lVpTMxhcGt2N7kZfPrYCHiecEaZka/CxJYMZiTH8PFIKB7NrHqUxeU1+", + "S4lU+nEieEKEogQaxTxlapRgNdN/hUQGgiaKctYatC6wmqHbGREEzaEXJGc8jUI0Jgi+I2Gr0yLvcZxE", + "pDVobcdMbYdY4VanpRaJfiSVoGza+tRpCYJDzqKFGWaC00i1BhMcSdKpDHuuu0ZYIv1JF77J+htzHhHM", + "Wp+gx99SKkjYGvxSXMbbrDEf/0oCpQc/mmMa4XFETsicBmQZDEEqBGFqFAo6J2IZFMfmfbRAY56yEJl2", + "qM3SKEJ0ghhnZKsEDDanIdWQ0E300K2BEinxQCaEOY1o6NmB4zNkXqOzE9SekfflQXYfjw9b9V0yHJPl", + "Tn9OY8y6Grh6Wq5/aFvs+8W+r2fK4zgdTQVPk+Wez16dn18heIlYGo+JKPZ4uJv1R5kiUyJ0h0lARzgM", + "BZHSv373sji3fr/fH+DdQb/f6/tmOScs5KIWpOa1H6Q7/ZCs6LIRSG3/SyB9eX12cnaEjrlIuMDw7dJI", + "FcQugqe4riLalHfFh/9PUxqFy1g/1o+JGFEmFWY1OHhmX2pw8QlSM4Lsd+j6HLUnXKCQjNPplLLpVhN8", + "1wwrIoqEI6yWh4OpItuGcoYUjYlUOE5andaEi1h/1AqxIl39ptGAguA1w+kWjQZbJrXU7OQolnW9uyaI", + "MhTTKKKSBJyFsjgGZepgv34xBYIhQnAPh3qmH6OYSImnBLU129S8myGpsEolohJNMI1I2GiPfIhgFvMr", + "HyMaEqbohJbp26BTF4+Dnd09L++I8ZSMQjq1J1G5+xN4rlFM96MQtPYvRBPaotk6YEhBJsvjPQfWDYMI", + "MiGCaBz/wuESweeEaWrR4/0HjNv6P9v5Eb1tz+dtAOZF3vxTp/VbSlIySrikZoZLnMu+0WgEoEbwhX/O", + "8GrVXhcwSiosVtMHtPgKlGjm1wg2l6ZplR8Cu7PdlCi7lu09mxPmEXwCzpR9UV7xCz5FEWUE2RYWvprP", + "6QF+ijiwua+xtk4rB+kyQet5fwZDMg9qetPvOi3C0lgDM+LTIjRnBAs1JiVg1hxLtqN8drXgvyiRROX8", + "wZKMVnOFC8oYCZFuaYnVtESpBOlzaflAGTdUjeZESC8dwbT+RhWyLWq7inhwM6ERGc2wnJkZ4zAEGsTR", + "RWklHgmsJNLiRDM21yFIBhIpji5/Ptp9dIDsAB4YSp6KwMxgeSWFr3X3pi1SWIxxFHlxox7dNj93lzHE", + "jwGXGWHUnScZBjrENNyrZXdTd99pJamcmV/Aj/Ws4DzTbECjV6R/v/Us+hiYhJH8a/Ugv1z3KjGbjaYR", + "1zBdoJTR39KS0NxDZ1r+V0gzfxqSsIMwvNBsGKeKd6eEEaH5FJoIHoMEVRBsUZv0pr0OGmpZr6sl2y7e", + "7fb73f6wVRZNo/3uNEk1KLBSROgJ/v9fcPfDUfcf/e6Tt/nPUa/79i//4UOAptK2k/TsOtuO9jvITbYo", + "glcnulo8XyHh+riI2b4zTfub7t7x2fIBb+Yf8uCGiB7l2xEdCywW22xK2ftBhBWRqrya1W3Xrg/mtmJh", + "bKqXvuHSKgoHoFs74rdEBJpTRkQjiOxoZkmV7CCsdVZgMkifZv+FAsw0zpqDnQtEWIhuqZohDO3KEIgX", + "XZzQLjVTbXVaMX7/grCpmrUGB3tL+KiRsW1/dN/+p3u09d9elBRpRDzI+JqnirIpgtfm9J1RifI5UEXi", + "tcetg24agYgVU3ZmPtvJZoKFwAv/rrnJrdo9oxzVbl8QeyTpV3MiBA3diXZ8foLaEb0hFi2RSBkapv3+", + "XgAN4CexTwIex5iF5tlWD72KqdInSZofkMa60itu4S8tEsw4nPFRxPWCMvDVCBAOLk7R9GzRibNMSGS1", + "XTjTMNidYMtOL662NVdJsJRqJng6nZVnZVnaZvOh8mZE+Wic+OZE5Q06236FNMNFEdXQyRjsTr9//nRb", + "Dlv6j0fuj60eOjEgg+nr/ePC8n05w4KA9BEiztDxxRXCUcQDq89NtJA4odNUkLBXMSNA7z6EJ0yJRcKp", + "T/isYEbedBlBut387QZ4sD2mbFvqbegGm8GdsPkXiEDP2JwKzmIths6xoJpvlYw6H1svX508Gz17ed0a", + "aCIK08BaSC5evX7TGrT2+v1+yydlaAxawwdOL66OYad0+xlXSZROR5J+8LDWo2x9KCYxF0b0t9+g9qzM", + "eY1khGBzhq2906cGuXZOAa/cpoRUQmvXi+m4jDG7p0992DJbJETMqfTp/D9n79zOF/ikYUxl3JZEzInI", + "kBawuFeQu4KIp2G3MGSn9RuJtcgx/6CRJZ+tp6Vf+W50LK85b3GUUEZqD9xOKyYKg+H58/HzShLRDcmE", + "anXjhiy6cxylBLmeLWRJBtgy6iapSLg0/eOpEVMVwXFr0Brj4Iaw0Iu538nhfsvFTcRx2N35ymc7I0r3", + "vbzEl+ZFGRN9MK4qjCy8paGajUJ+y/SUPSeAfYOyxtkx8F6vBEe///Nf1+e5JLtzOk7smbCz++gLz4TK", + "KaC79mqp2ULSxL+Mq8S/iOvz3//5L7eSb7sIwjR+hqUbHWP4KS/l7zOiZkQUZAO3wfqRUTPgc+TwpTB8", + "yZJUvP5ZIiY+JyLCiwJbt3Nq7fSBt1ZmJagC+rLfaSZ9g/THa5i87s2JEKdV1We372fjnkl55vRU07c9", + "dZrMJJvIzu65/bm7PKWaGd3QZDTVUusITzNL2KqLucsbmiD4ogtfmG2MIkO8Yap7RmPOVW/I/j4jDMHe", + "wQaT9yQAPqVVfXR0cSbRLY0i0JuBESwfXEP2psAKTHOp9H9FyjponCokSMwVQVYkhkFSmAs0HhOUMuxu", + "/npDVoSKXWAVryxYbohgJBrNCA6JkA0hYz5C9qNa4MBSJ1gqIgyHTpMyvE7+dn6J2icLhmMaoL+ZXs95", + "mEYEXaaJpuGtMvQ6Q5YIMicMNCYt/lA7Lp8gnqoun3SVIMRNMYbOMsuDvZaan15c2YtNudUbstdEA5aw", + "kIQwZ3dKSKRmWKGQsz9rioXjstBtcfwK0P203GnNgyQtQ3m3CuGXcJ2o1zOnQqU40iyrJD96bxfNvbVH", + "TzDX4kV9xbKiDOGwKl8LNVU5Tc9wib0sRfu1TCMo1WuZa+7wfZc1meUqSKXiceHKBrUrRilaNl+Vmcec", + "R10t/4BosHy+e+UXM93l6894Yboym1LHJUfTscfSqZkhZWhKp3i8UGXNYae/vPV+QLv+faCucw0w6EHC", + "keKrL0fpBLm2Te5CwJFgpPhoPqGenrNDM7fCUYmCih+CRVrdRTcJqCXfDrqdUX3MSuSAABR8fV7UxHtD", + "1gWWM0An2QBZt1mXmrOCxRW6aHNRmAQF4zkaL7YQRtfnPfQmm+2fJWJY0TlxvhIzLNGYEIZSEM9ICOMD", + "Oy1OIJWah1FV/dzyKuNWsQUGB27f9ZBWhGJs+b5G7xgrGoDBdkwr64GLMrNReiTNAFjx1Gl0Sqy6Un5N", + "plQqUblQRu3Xz4/39vaeVOWF3Ufd/k5359Gbnf6gr///j+Z3z1/fc8TX11GZX1gTeJGjHF+dnexa4aQ8", + "jvqwj58cvn+P1ZMDeiuffIjHYvrrHr4X3xI/ezrJbfeonWq1z7E+jVU+i33BMF5jkf9sQ/tGbi3uam/V", + "8WNW90a3vAtHGN91rL0M3NxVpcoE117oFha3tB79VMsHOeYXDBv23iSg3huiEypvngqCb7RW6Tlf9fEs", + "R+bc8VscU61HjReIvNfiGQmR4FxNpLFzlMWUnf3H+4d7B/uH/b7H/2MZiXlAR4E+VRpN4NXxGYrwgggE", + "36A2KHohGkd8XEbeR3sHh4/7T3Z2m87DqEnN4JBJUe4r1LYQ+YvzJXRvSpPa3X18sLe31z842N1vNCsr", + "4DWalBMGS6LD473H+zuHu/uNoOBTO585f5yqf0HoMy4mSUSNkt2VCQnohAYIPHqQ/gC1YziWSKbxlWly", + "jMORsGKg9zxQmEZypU3TDGZbGvetOI0UTSJi3sGGNJJ0YeUn0JPPXkwZI2KUuStt0JP1YlprGXNryZqg", + "kjdaCXTnVIJkkQtElEThwFDoWj4Hu5lP7G0dHtg1NMSGF/yWiG5E5iQqIoE5jvRkYy4IyvDEbFppVZTN", + "cUTDEWVJWmMZrQHl81SAfGk6RXjMU2VUddiw4iBw9wo6wkSz62ZX/8+5uFl7O6ZP15FIGdPdrNWyj6KI", + "3+otvtGwgZMZI/u1c34oCHKZSm0MD/a9RK/NF8YwkT9OUoUoU1wr5SwcLzowEgmhHUOCSMWBk+LgRkuN", + "tpumEqNfFnmphRBn9jTj5bzznmy+3YkxuX1Nw6/CYkrUSCqs1kosGlPeQPtLaN74Ar364bJIAC/B5lQP", + "ZTxRRMBT5/1Ciq4ol4onCWh0lwYvWp2W3fryjYh76IFGfvu0NMXTi6tNjeCJ4BMaeZYLBhz71oq3zjz8", + "Yr9/2d35v+ZSikULcyZSZow+MQ9Jr+L8De2bkfrpxdVF3Zwyz3tUnN3SmjIznYdEMsuPg4g1QAWYoTFB", + "VmQ0yA4mxHyQXNh54hMeJgLHZJxOJkSMYo/V4bl+j0wDY4+lDJ0/LQsQWlBpqnpclDYHdI8JDqzjdDPo", + "e6walWV0CtB869+u18TwvTrXLL1Vwrax3lk99DKLdUCnF1cS5aZVj7mjvL21LgQXs4XUirrp0XhaUla0", + "UgByNhZJLvIPrT3HI5jE3sPYEQJqz6dJCmR4+bp79up6Ow7JvFOaE5hDZzwiet5bBW4xdw5aub9DiUnM", + "69RFgxiyKQEVYJVRcGMgFejVAx3FFY5GMuLKM5s3+iWCl6h9/dw45ugZdFBS2kr9vACFEn4feClGc6S6", + "YS9hwKrdqUTga02AsRHhissrDeojlZ8JjkxkVBmfc19ft/H8przR/GYt9dpOfOOeuZvzBs5Ex+cnRhYK", + "OFOYMiKyS+uy5wd4rbU6ra4+o0JMYrgfmPzXai+QGjtmhi6rLGHHS2EVd2IFq3Ed1kwumpMQxZjRCZHK", + "ug6XRpYzvPvoYGCCFkIy2X900Ov1NnXbeZb76TTaim3j41Dw4OnJ2Zftwx145zRZy8fWxdGbn1uD1nYq", + "xXbEAxxtyzFlg8Lf2Z/5C/hh/hxT5vWNaBTnQidL8S2l7U30mWWeD/RKGAkyhOSgMa2109cI7ho1I/qB", + "hMjrZqrwVEvhBuO+zJ/0CyJD8vBEVYgIKV6ZNYgOoR9Wm56cYARt7JgpUzTKA2eWjU6fFfokV3qSL3mR", + "J4RlvuNRZH4FnM01VfgcyUsM3L1b2oxbo3+OQurBzr9b5TSkggQKnMfW01BrGyfJelT0C38ZT2saFGNd", + "Yj2nyzfn5J9z+VAe/dX0r7/9P3nx+Ned315cX//P/PSvJy/p/1xHF6+aX1963K9We0N/U5fmlffLYHEv", + "uTI3RY9zrAKP4DPjUtVAzb5BiqNYf9xDx6CgDYasi15QRQSOBmjYwgntWWD2Ah4PW6hN3uNAma8QZ0h3", + "Zd0otvTHF8YSoT/+6HTAT9U+QusvISyQM9cmmY5DHmPKtoZsyGxfyC1EwgWm/hWiACcqFUTviJY1owUa", + "CxzkfhL54B30ESfJp60hA02UvFdCryDBQmWhE24E2Gg7K3NBapuTEIGPobSa7JBl5weo5roTY0fpZcYK", + "MFpWTE41QPGqGVyU/XwO+x3PPiLdTm9kRKUiDGVWCSoBeVHbOWwd9kvkf9g/XH8Xn+HQCvQD7F5OVuCQ", + "sgF9GASGoQ0zHs2UShoYGTW/MTSCfn7z5kKDQf97iVxHOSyyLTbKGE6SiBJpbphVBDKJ9ZHbavlsgmZ3", + "Gy7IGLPgs6iBS9IzGBi9eXGJFBExZYZ/twMNzgkN9PrgrpNKmWpUpBgdHZ8/2+o1yLYAsM3mv2If32Qr", + "rFypOeNWnc0uw3gN3w46O+loccpSaC5ogQ/Bcy5QZBhMTtcDdCVJ2aMHtspcd5qdjBa5hcxw9WFry/WY", + "VDnFAL3O5DucTSUL6cqRwXWZ0yV0ay3SxsFhqfdOea7gumH1F8vawJ0BK2QvgOAormcFq8nfA3Ggec5q", + "nZwb0XbRaKkH86NGvvd3LoHsbapLbhpbUnbQLDjkZuElzeNC7iK+Ylmvek/VqPZ2EunX9i7SaQ/X52iG", + "JfuzgpcVHWJn73GjrAV61Kb3esUbPT4xU8qoynl7ZvdRxu/1hkaRueaVdMpwhJ6g9uXZ6d/OXrzYQl30", + "6tV5dStWfeHbnwZhJg61Ty+uIHYDy5FkOJEzruo9vDBybfRKpZLLzrCNbphWh7X8XAo98XoXb33FeBR3", + "Lbe0jLuJNPmWXkv/RlEurS+JUVkZVfKloSFWqL6jyJBaJu6Lqijzc/P468Z43Ml0StEaPj5UlD2cI+tn", + "B2h0WtTjxHckNaslITq7yMPfcyOV676ypie7vZ2Dw95Ov9/b6Tcx2cU4WDH2+dFx88H7u8aIMcDjQRAO", + "yOQLTIYWsY2QiKNbvJBo6MT4YcvoDQWFocAsrKjf6Dp2OQ7m88JeqoLLusCWTQJZmkWorMhLc1nOSNNY", + "Fnz0jy9KXkOaSgDWlcF+NdrEmE1QwNMo1PLWWFOeUd9IaLVMSVSe7AeI9YrdMH7Lyks3Nk1Nv7+lRCzQ", + "9fl5yQIuyMTmPWmwcHCBqNkHnmy0DbtrRPK1sykEi9xHgEiVExZOoK8eDlI01zm/NIN1Dcx2ueTpvdqm", + "zIBb7/2KNVUMLiGZj9LUJ17pV87D/Orq7KS04Rgf7Bz2D590D8c7B939sL/TxTt7B93dR7g/2Qse79Vk", + "CGvu2vL53iplCq2P6ADAg/HSBOGEA01DmbvJOFUoc7jSxHms5VRUEIFN/ALYE6xvkO4BTtdAv4kWmZS8", + "8uMLrAnVfZvAX6u/uJylSotB8I2cpQrpv2DKeglWy1jdhaH5AXrJ4RvhnOYYr6orpjn4Ri03r6o2beu1", + "49zpYDDLwAboeca0MrZn2VxbEvvT8FLr6QlerFvGCGI1C7tbBTetTsuAsNVpOciAO9eyY5ediNdJvIg3", + "PuM+wRHwsNxxJlU0oh8MyempU6loYLQ7DLtZR3Y2EpeEI3OE1l2/GW8Me8xmHzmqvj5HbYib+guyyp/+", + "ayu7qiuS0P7uk/0nB493nxw08rrOJ7ieGx+Dr9Dy5Nay5iBJRy5TYs3Sjy+u4PDRB5tMY6PN27XnGqNm", + "HIGW9ihDeerFfPAnvSdFZ/OQp+OoYB2y0Sbg0dwkT2bN3dRvNJrTyYT99iG42f1V0Hjn/YHcHXuVo2wg", + "vyR5VrRoLqldZNw1eR78OiQglJC1LvOviYQVoEuiEOBPVzMsfaJmLj4W5ZxjvYW4F7H29/b2Dh8/2m2E", + "V3Z2BcIZgf63PMtzO4MCiUFL1H59eYm2Cwhn+nR+j4kgUi/ORIF56QzZfDv9kkuk1j32fFhSI7DkWGP7", + "nse1IL+2EotdlAU6eCpl0swSlXuhvbfXf7z/6PBRMzK2Gs9IvF/NYWw7e8MvSEDovLTzbbCCvzm6QLp3", + "McFBWcLf2d3bf3Tw+HCjWamNZqUEZjKmSm00scPHB4/293Z3msV++CzdNqqpRLBl3uUhOg9SeHbDA4pl", + "1tupOy18UuKye+RKj8zcxbPqz7eJA28eyUol9EoLvqOorYWookBaiMbcamJn8LNIPU5d/mUtLjb1rV3t", + "SnuB1eyMTfjyVcYmCp91UHIm7kQLPhIyU4aEURI63pVpflaWApenSBIUpsRCzshGAluAY3Odk2A1A2EV", + "PqRsWnb2XhqwiRpm5rA6bhnGtQ2bWIyk36nmjUgBVsaWLBHO3WsaGcapHPm1iuWOBZmmERao6j++Yspy", + "EUeU3TTpXS7iMY9ogPQHVXV+wqOI3470K/kTrGWr0er0B6P8JrminpvJWT8CsyGVcfMl/KRXuVXxTIKT", + "f9t8vw0J9psY4LzXS8+18mZcrK8YfV9A9HIQ4P5uv84RrabTkgvasnv+przdoqyP4p3n/FGWLMdzjWku", + "iioabFkOLq3Xt1q4iVzldrcsCaC2s+m5IMsyXAvBjo0O4maXoVXrtZvNtiRBefT9w0ePDxpGm36RqL0i", + "BfkXCNbzeIVAXbNT502ktsNHh0+e7O0/erK7kXzkLjpq9qfusqO4P5WcWBWZ7VEf/rfRpMxVh39KNdcd", + "5QmV8lt99oQ+rSDdPOilRuteVf4j30mn5pcF8GYi7gpp6agkchWSTbbJZELAcDQycOvmk6k4YTWaQ4AT", + "HFC18GiA+Bb8UlDWpBK80aD3ymQ9ILV92/g7zblkOs7v/dtucPSfRrOr4MJh46B1mY7rtMhX1VGNDmkc", + "ucKKhaKBgcBghO/y/TYDJrrFsmTV178DRcJOIZlo9frHtGie7t3hepbxPb9O9wUg+bO7F7e/sp0FraMk", + "JFchvuoIrSdBLRGAl1gTA7vnRPZENQXrnTcq/MEegJ/31WhcTCexMl9HKfdEfupuPm6zNKjL35kTbPPx", + "Cjf4m3xYjawHfLRzsCDP++6UUMKHTeZ+pS5tU+zqYlUC76mpNGIjylChMWqTOFELFzXhNNOtze57jrIO", + "vcj4lf3c+k++hqf91UrX+n+TRGDFKzY3yNrLtaU9rfVn9YurJ1X3FaMT2kQoZXeLSnoHqVYU1FlVvM1U", + "UQOFz/qST9Nq8NsGBdvqVPycclylHFexbZ3mutKeVlhZYSb1e2PuV7+wuh2VrqzdZ4LMql/rnbPNHZVW", + "gLvVTDkmtlhQ0OcsgAxgNQgyFX3ZDrDa7eMcv89GAG0ZS1TJLWrWUciWfvoU8gW8dhlT6MR1AdOoZol9", + "+mVl/xxWLW/GqjqA7gbfS3iW/6zgaHW0VUHOfIzO6lKDmnWRIBVULS71gWCd0wgWRBylBg3hpIBFwON8", + "cAhQ+PQJ1NSJR1o9JYwIGqCjizPAkhgzyDWNrs9RRCckWAQRsf7lS3e7kB7h1fFZ1wTGZMk8oYqPAoC4", + "LHpHF2eQwMvWz2n1e7s9yLzOE8JwQluD1l5vB1KUaTDAErch7hB+WkOUpkM4yc5Ce+I+NU00aGXCmTTA", + "2e33K/WYcJ4kaftXaSws5nhtLBSagnfL/hZLDpFOErDT/9Rp7fd3NprP2rxGvmGvGE7VjAv6gcA0H20I", + "hM8a9IwZrdqlgSe2YY6zrcEvZWz95e2nt52WTOMYaxHRgCuHVcJlnQhDJMKIkVsbkPorH/fQpdFJwLs8", + "LyVqTAYk1CwJI4VFb/oBYRHM6JwMmeXEJkcVFhB9EyPNgU3sQxnNzNBm9w0JE6me8nBRgW7W3bburuu8", + "anMAb1yoKku4mtRUrPJxR5PXTQbcm9COMMxUnibMJHS7IXCJOaHvvR02uo3XzKNYw9F5d+5u+e2A4Krs", + "N6GfZO9cvbTygaFlaMqCKA3zU7Vcp8obbG/qLdm8dzfEI4ScQgsLlKJXtzu+GA+J8ZVNFmrGmfmdjlOm", + "UvN7LPitJEIfcjY2x8LaJn2yqAtJRWkM8TEmmlePuW2muP3xhiw+9YbsKIxd9LXNbY0jyW1CQOP1QCXK", + "MqwPWX2FRL8wfWwT+ZpkXMW8UWaaPFVJqnrILIQoG1AEzSG9lZyRcMgURx+FyVC6+LT9MR/xE0inBIca", + "TwpNzJK2P9LwU92s5Qjr1Y/GrsxoRWYnAIBhS0sNw5b+PRVYS6epnCEcgG+Gfljc0rYhbC7g5N+qQjjA", + "DCU8SSMtRwFSmTyHpT4gOBNHEVJASu5bLU/ATtasx5p0fRmBrD3XGOAqZAS5gQrE1N8/9NOTJIEgPrX0", + "r5evXiI4qqCcGzTLwwcARpRpQSPLJ65H7w3ZMxzMkJFBIM/usEXDYSsv27UFc00lMWJAtwtCzE9Qz9AM", + "06HhT72e7srIRwP0y0fTy0DTUhKPFL8hbNj61EGFF1OqZuk4e/fWD9A6s9hliRGgtuH9Wy5cXq+wcAya", + "cwOzEHHLa6MFwijnQEXtd0wZFou6Gng8VfWuRSabgG2W7+dBv7+1/urILtUjGZYaakr4tCQI7X41GcDK", + "P8syQKHerT5xmU0VERrJ5x6EkKc4dBGMP6StNdKWVRMLchR8X2TJBn0jYlxZK8IQlEV0wlCCBY6JgnoG", + "v/hxHrx4qf7bXfTCSWSMJmXk7RTAU9Wd3i4h9n5tvcmsciPgwv494B+MmyezhHGf3Ne4ODKp1LMa2A8K", + "HWGzHCJ2/IreKVHfA8b174uVupy73xB/Hwr+nBIrguVAq3CzbShiUrQiVKNNBMGxtL2YxlptvIQ5dS8J", + "UwgqHcue/dcpH+DI/y7i03cDZEAY2TrP0maOzMzt+lC0sISPTGKd7DubbyqYYTYlErXN+fn7P//latX+", + "/s9/2Vq1v//zX0Du27byOnSXVVl+N0B/IyTp4ojOiVsMOKeSORELtNe3ZZvglSd7lRyyIXtNVCqYzFy7", + "9LoAJqZDm71Wr4eylEgkAYRQY2FifY6MNc+jTTtaNqC8V4ruLKk/dgWFBehT0eEAXCJTRhXFkVWF3Dwg", + "/imfiFlzqzh41TC5ZKpez18Uea8M9nbNBDdkMKZKuYfuTOFu0ydqX14+2+ohEPcNVoBfGegNeTdWE+j9", + "4EnreZLhKGWGAlA2vKmQuLTWrHli29yHXbMuqWm9YdMo8kTrxm4xP8TuBkZOP9ycwdNndTxxRSfqzY6f", + "v15fEfNGOuXX22eHe8swtxVVcpB9C20StW0y/CzPT6lsy7dC+nthwIVqPxkXRtxkF7o3DeeYs0lEA4W6", + "bi62KnSm9ZQR5KGwg9d21gi7dVWDIYpHxXbJt6/20Mjc/O7z9KgMuskxkgds5Lj24yRZhzonVAZcf1vA", + "lm6AE5vlCICY02kRi9bZdk7geXbkrBTMszrtjiDvz8pjh05Z9Wy4B6Z4UmGI35ARVjKqFEKcHhI2X2W7", + "6ApsrTACfV+o2b8/Kei+DUI+NH9IFqGwAjbNBWdZ3v069LKZ+e9wo+0InoVfEuGo2kzUZPLIl2U+RcGM", + "BDdmQbYO3CqJ4MyVirt7OcCUF9jg9LfT/3HcN1Acc1itUhbPbHqXu9MVYYSNVMWvd/1oEcwDZHBHGDtD", + "qsmcguWCBVt/qBvIezkZqnXbHhAlXaRR5AzxcyJUXmShyE+3P4Ljyno52VHbSlnk6vWLLmEBB0+lzMvG", + "L5C4nOpfV1o2G2aW8gNNmuhXACqHGPXC6Bfsv3EoQ1m6zj/tPrcJO/+0+9yk7PzT3pFJ2rl1Z8jSvy/W", + "fN/S6wNGPi280jLQgDWZzOfrpL2s1b0IfLbExCYiXzbBH1JfE6mvCK6Vgl9W7eMORT9bROHb3BNkyOaD", + "Nrxy/md/MJHvfk1PFiMLdTFLtnibQ4aLvHCBrar38BzkaIZxRf7b0IaaE+RK6cCh7tlJx9akMJUkMl/8", + "e7Kounncu5Rox71/c+pRPKbTlKey6OYPJUiIzGuXlxjwQ5Nf8+O5VoL9jrG0f59Hx70LqD/w/o5E5+qG", + "GuZtSzyvEZ5dqzXk8JxGqpDZXkKxD5NP1oQJZbXgbTbYrRqnMZeFuSkal5KALzuz+eblajAUyjIkmArZ", + "Q1eSaDCR5JWJ8JBqEZHBkP23++QXRXD89qcxDm4IC4dpv797kL0jbP72J6kgUnTIzh3eEKYEJRJhQdDR", + "yxO4mJpCBG4PHUVRHgpVnQ+KU2mLtbnqQ0kE+b4M5/CBr1BfIodg4xIWSzHfAADIi+Jg0vpittRQj8pv", + "7ZorUg5ZfyhSjRSpArhWK1JZ9uS71KTMIN9MlXL45gO4TRzwQ5m6D2VKppMJDShhKs/8teTgZBMHPsAQ", + "I2bvYwqOCaXzuLEylac0Xy2nWuT9Fk4p2eD3r0O59IIP01Wam+CI0Gkt+WFYr7Z8b/jQv1/mfP/qykNG", + "sdNi5U+/YmDihCY28ZhfQHjOxU1TzLMxyPQuEfDrSyfFFX6HsomeHuR7+PYiChzexi9cI01ZcrkHgqzi", + "1zd1R3SQsDqwiY0zJclNIZpbqmY8NXkuRvahyTmlqcJmkQaRJ7C9fmv2oke/BwH0JVeIxklEYgI5qboG", + "m6ACUJokXGR1B6gslB7ZjP1psik6h5p0I7b8VkfvGgvHC7DpZZWDwLy/vF1erhnx6frYymxwF0joCa4c", + "Mldc+Z0Rhd+hjMkixZEkEQkUup3RYAaBlvoZ9G/iMHGSvMsyK2wN0ClQajHXAwzelkRQHEF1Fx6ZwkTv", + "5nH8brCc3+r6/Bw+MjGWJpPVuwFyOa2yA0LqVsXASb2KCEuFXtpw0LbGJMFdcdh3+hQqrG/LhlTmSSiG", + "zBdeycit7ZBO0LtCpOW7mlBLx1Bf6F36RvJSpz51kFmL4kgA4AxuEhbWWMw01PxBljt9b5rGhgGfZhp3", + "HO+5NJkXfJqlLSqhMk6SpuhrpwlYPI/jFTiM2oXiRVKFPFV/kSokQsDHFrvrkBu1cWD+UPjGVO8vVeE1", + "5bK8Vk6TvMQLKs1UC1W2zF/zOG6ZksAx9lXN+vLA2WqHy2Y2vTOF6NgfkvYmca9lZl8IfK2cHLZcW73I", + "/do0+MPre66u3TdGw29gH8tnQZkTVWBv84KBDyvqDzZySRYzpQJ9NOLe1dKIrTP4h6eRHD/+4FQScCHM", + "hZ4tFvxw3LMLdpoCubehOmle9bPjbIXX5+dbdUQj1EqSEd+HEfHzbDiVwkBx6K9BIWjoUjUen5/YxI5U", + "ai2vh17FFPIn3hCSQDYYylOJ4Ea2V8xKX3dvmqWdJ0yJRcIpU2tnkTe9m8l8+qz8ePfMp2yAyh/+KAfz", + "w8NjUqZEPM4WsOpmS/MhVWsacaaCUg3tMU9170sZ9qF2nFxIRWJjJ5mkERARhAPaxEu4WBuvg6iSUKml", + "A/crhbpoQzYmEy2GJETosfXnkGw3V/l81oRLhTOueWFY3/dhToCk+6BBY1UHtUoBuiRx+fZ9KmtWIuCz", + "p/Qc7APl2nwStSN6YwpOo7lEkf6xtdLAYAr3fe20Up9PWVlpSl+6EIOzGTL/ETjcWYWtOQPqg2Nrp6RI", + "LI7/wEb72Zpcy9fEhrXLHewKNcx7Q3ZOlNBtsCAo4FEEJauM2rSdCB5sQ13lIKGhKbAMkwOGV/86hhGP", + "L66gnUnM3Bky/cdyZd/qRF2B4LPtV2tMrqam+7+xPmYWuIos/Bv+w5q2+b11LQ3JGhLlySoFiCd/eIOB", + "leB+WAseprUAHIey1bSnAgcgFMtZqkJ+y/yWAVvBdvuj+XG2zv1M4WB27QqKfR/Srq0/tG4Yt8AHQZR2", + "TSEx6YzunyZ5ViLqgYasa8C5JYAQU3Sk858CpvTcHw27v77bVBGOG7lN3SttuVRh3w1t3ffJZ+fgvKqL", + "8HgoZG4wza0ECrcUrU+iWAN3pW7mSpRCQeZMtHSleTvFCtEmM3lmQ8pLC2bFaHtDllXfdZnRtXbVcaoV", + "Cqm8MT1Y7amH/EWSjZ5nKyVDWaEAR4Gpx5NVCzaFcmSN9vW6UEH7zugtH8Sz0VmZZJlVtX1IKocfJ2D3", + "imVzAeOsOLUyru7atrmPUCp7mG0QSOVW8COMqkEYVQFYTYr0mZrHllvZYrV5BKCkH+B2xxeGlQkldxeE", + "9Rnn9ddDD4entaf1j/CrexMI8lQWZycPP+aqSHMlHr2ttYKurYBZNA2tomALokSQrquLFxqAWXgYXaNa", + "YLM3ZG9mxP2FqPNgJSEKqSCBihaIMihf6Ook/1kiwbmy77lY1BfiNCTyXPD4yK5mjfLSuGK47yJm4zxb", + "HU+VZBqnsamQTBk6fYra5L0SxqESTTCNwJ3XgZS8DwgJJeDkVrUSudfDMis5vnaWK1xjs1qjgakJOc80", + "sTZOFe9OCdN7kVfgTASf09CEt9eUdPfNFjTEr6CkTT/QpEx6a+v0LRNeGW9RVnLUFgrM8dPtTuvHMVGt", + "jAD5BbjIgKg4RxEWU7L14yh5yEdJ0Zrkzo3SidIsereZgamh3ecuIncz4+P9xu1efz82kUIm+QeY6Gie", + "KX11AcPfFwr27+98uO9A4esHbEM/JU7BLQQJQwe6Rx/CvOABjlBI5iTiSQwV1aBtq9NKRdQatGZKJYPt", + "7Ui3m3GpBof9w37r09tP/xsAAP//ASYs9mHzAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/openapi.yaml b/openapi.yaml index 5117dbb3..d996b5e6 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -234,6 +234,18 @@ components: When true and source is Running, the source is put into standby, forked, then restored back to Running. default: false example: false + target_state: + $ref: "#/components/schemas/ForkTargetState" + description: | + Optional final state for the forked instance. + Default is the source instance state at fork time. + For example, forking from Running defaults the fork result to Running. + + ForkTargetState: + type: string + description: Target state for the forked instance after fork completes + enum: [Stopped, Standby, Running] + example: Running Instance: type: object From 0481d351f5e5bbb7157560db24f151aa6f841030 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 27 Feb 2026 20:00:49 -0500 Subject: [PATCH 09/20] Stabilize Linux CI test timeouts --- .github/workflows/test.yml | 2 +- lib/instances/manager_test.go | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1b418ad1..79b079fb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -70,7 +70,7 @@ jobs: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} TLS_TEST_DOMAIN: "test.hypeman-development.com" TLS_ALLOWED_DOMAINS: '*.hypeman-development.com' - run: make test + run: make test TEST_TIMEOUT=20m test-darwin: runs-on: [self-hosted, macos, arm64] diff --git a/lib/instances/manager_test.go b/lib/instances/manager_test.go index 0811ca2c..799f9f43 100644 --- a/lib/instances/manager_test.go +++ b/lib/instances/manager_test.go @@ -949,10 +949,11 @@ func TestOOMExitPropagation(t *testing.T) { err = waitForVMReady(ctx, inst.SocketPath, 10*time.Second) require.NoError(t, err, "VM should reach running state") - // Wait for the VM to stop (OOM kill -> init detects -> sentinel -> reboot) + // Wait for the VM to stop (OOM kill -> init detects -> sentinel -> reboot). + // This can be slow on loaded CI runners where reclaim pressure ramps gradually. t.Log("Waiting for VM to stop after OOM...") var finalInst *Instance - for i := 0; i < 90; i++ { // up to 90 seconds (OOM may take time with low memory) + for i := 0; i < 180; i++ { // up to 180 seconds got, err := manager.GetInstance(ctx, inst.Id) if err == nil && got.State == StateStopped { finalInst = got @@ -960,7 +961,7 @@ func TestOOMExitPropagation(t *testing.T) { } time.Sleep(1 * time.Second) } - require.NotNil(t, finalInst, "Instance should reach Stopped state within 90 seconds") + require.NotNil(t, finalInst, "Instance should reach Stopped state within 180 seconds") assert.Equal(t, StateStopped, finalInst.State) // Verify exit info shows OOM From 1dc53e35fdbbe6afc096e8df4bfd60e6ab3e475c Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 27 Feb 2026 20:13:54 -0500 Subject: [PATCH 10/20] Deep-copy metadata refs in fork path --- lib/instances/fork.go | 33 ++++++++++++++++++++++++++++++++- lib/instances/fork_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 1722374d..b4bd8493 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -181,7 +181,7 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin } now := time.Now() - forkMeta := meta.StoredMetadata + forkMeta := cloneStoredMetadataForFork(meta.StoredMetadata) forkMeta.Id = forkID forkMeta.Name = req.Name forkMeta.CreatedAt = now @@ -341,3 +341,34 @@ func (m *manager) cleanupForkInstanceOnError(ctx context.Context, forkID string) } return err } + +func cloneStoredMetadataForFork(src StoredMetadata) StoredMetadata { + dst := src + + if src.Env != nil { + dst.Env = make(map[string]string, len(src.Env)) + for k, v := range src.Env { + dst.Env[k] = v + } + } + if src.Metadata != nil { + dst.Metadata = make(map[string]string, len(src.Metadata)) + for k, v := range src.Metadata { + dst.Metadata[k] = v + } + } + if src.Volumes != nil { + dst.Volumes = append([]VolumeAttachment(nil), src.Volumes...) + } + if src.Devices != nil { + dst.Devices = append([]string(nil), src.Devices...) + } + if src.Entrypoint != nil { + dst.Entrypoint = append([]string(nil), src.Entrypoint...) + } + if src.Cmd != nil { + dst.Cmd = append([]string(nil), src.Cmd...) + } + + return dst +} diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index 6b2e60ec..8f9da4b6 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -107,6 +107,34 @@ func TestCleanupForkInstanceOnError(t *testing.T) { assert.ErrorIs(t, err, ErrNotFound) } +func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { + src := StoredMetadata{ + Env: map[string]string{"A": "1"}, + Metadata: map[string]string{"m": "x"}, + Volumes: []VolumeAttachment{{VolumeID: "vol-1", MountPath: "/data"}}, + Devices: []string{"0000:01:00.0"}, + Entrypoint: []string{"/bin/sh", "-c"}, + Cmd: []string{"echo", "hello"}, + } + + cloned := cloneStoredMetadataForFork(src) + require.Equal(t, src, cloned) + + cloned.Env["A"] = "2" + cloned.Metadata["m"] = "y" + cloned.Volumes[0].MountPath = "/mnt" + cloned.Devices[0] = "0000:02:00.0" + cloned.Entrypoint[0] = "/usr/bin/env" + cloned.Cmd[0] = "printf" + + require.Equal(t, "1", src.Env["A"]) + require.Equal(t, "x", src.Metadata["m"]) + require.Equal(t, "/data", src.Volumes[0].MountPath) + require.Equal(t, "0000:01:00.0", src.Devices[0]) + require.Equal(t, "/bin/sh", src.Entrypoint[0]) + require.Equal(t, "echo", src.Cmd[0]) +} + func TestForkCloudHypervisorFromRunningNetwork(t *testing.T) { if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { t.Skip("/dev/kvm not available, skipping on this platform") From d3afe689cd26f4259aca15746f813365883e932d Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 27 Feb 2026 20:28:34 -0500 Subject: [PATCH 11/20] Fix fork CID persistence semantics and cleanup validation --- lib/hypervisor/cloudhypervisor/fork.go | 12 +++-- lib/hypervisor/hypervisor.go | 9 +++- lib/hypervisor/qemu/fork.go | 14 ++--- lib/hypervisor/qemu/fork_test.go | 7 ++- lib/hypervisor/vz/starter.go | 4 +- lib/instances/fork.go | 29 +++++----- lib/instances/fork_test.go | 73 ++++++++++++++++++++++++++ lib/instances/restore.go | 2 +- 8 files changed, 123 insertions(+), 27 deletions(-) diff --git a/lib/hypervisor/cloudhypervisor/fork.go b/lib/hypervisor/cloudhypervisor/fork.go index 3f19919b..a2bbddbd 100644 --- a/lib/hypervisor/cloudhypervisor/fork.go +++ b/lib/hypervisor/cloudhypervisor/fork.go @@ -8,11 +8,17 @@ import ( // PrepareFork prepares cloud-hypervisor fork state by rewriting snapshot config // when a snapshot path is provided. For stopped forks (no snapshot), this is a no-op. -func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareRequest) error { +func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareRequest) (hypervisor.ForkPrepareResult, error) { _ = ctx if req.SnapshotConfigPath == "" { - return nil + return hypervisor.ForkPrepareResult{}, nil } - return rewriteSnapshotConfigForFork(req.SnapshotConfigPath, req) + if err := rewriteSnapshotConfigForFork(req.SnapshotConfigPath, req); err != nil { + return hypervisor.ForkPrepareResult{}, err + } + return hypervisor.ForkPrepareResult{ + // CH snapshot restore keeps CID stable; only socket/path-level rewrites are applied. + VsockCIDUpdated: false, + }, nil } diff --git a/lib/hypervisor/hypervisor.go b/lib/hypervisor/hypervisor.go index a0bfd82d..a5abb1ad 100644 --- a/lib/hypervisor/hypervisor.go +++ b/lib/hypervisor/hypervisor.go @@ -86,7 +86,7 @@ type VMStarter interface { // For snapshot-based forks, implementations can rewrite snapshot config with // fork identity (paths, vsock, network). Hypervisors that don't support fork // should return ErrNotSupported. - PrepareFork(ctx context.Context, req ForkPrepareRequest) error + PrepareFork(ctx context.Context, req ForkPrepareRequest) (ForkPrepareResult, error) } // ForkNetworkConfig contains network identity fields for fork preparation. @@ -113,6 +113,13 @@ type ForkPrepareRequest struct { Network *ForkNetworkConfig } +// ForkPrepareResult describes which optional fork rewrites were actually applied. +type ForkPrepareResult struct { + // VsockCIDUpdated indicates whether snapshot state was updated to use + // ForkPrepareRequest.VsockCID. + VsockCIDUpdated bool +} + // Hypervisor defines the interface for VM control operations. // A Hypervisor client is returned by VMStarter.StartVM after the VM is running. type Hypervisor interface { diff --git a/lib/hypervisor/qemu/fork.go b/lib/hypervisor/qemu/fork.go index 2fe97672..3b5176b4 100644 --- a/lib/hypervisor/qemu/fork.go +++ b/lib/hypervisor/qemu/fork.go @@ -12,10 +12,10 @@ import ( // PrepareFork prepares QEMU fork state by rewriting snapshot VM config when a // snapshot path is provided. For stopped forks (no snapshot), this is a no-op. -func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareRequest) error { +func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareRequest) (hypervisor.ForkPrepareResult, error) { _ = ctx if req.SnapshotConfigPath == "" { - return nil + return hypervisor.ForkPrepareResult{}, nil } snapshotDir := filepath.Dir(req.SnapshotConfigPath) @@ -24,9 +24,9 @@ func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareReq // The generic path points to CH's config.json; for QEMU, require qemu-config.json. expectedPath := filepath.Join(snapshotDir, vmConfigFile) if _, statErr := os.Stat(expectedPath); statErr != nil { - return fmt.Errorf("load qemu snapshot config %q: %w", expectedPath, err) + return hypervisor.ForkPrepareResult{}, fmt.Errorf("load qemu snapshot config %q: %w", expectedPath, err) } - return fmt.Errorf("load qemu snapshot config: %w", err) + return hypervisor.ForkPrepareResult{}, fmt.Errorf("load qemu snapshot config: %w", err) } if req.SourceDataDir != "" && req.TargetDataDir != "" && req.SourceDataDir != req.TargetDataDir { @@ -61,9 +61,11 @@ func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareReq } if err := saveVMConfig(snapshotDir, cfg); err != nil { - return fmt.Errorf("write qemu snapshot config: %w", err) + return hypervisor.ForkPrepareResult{}, fmt.Errorf("write qemu snapshot config: %w", err) } - return nil + return hypervisor.ForkPrepareResult{ + VsockCIDUpdated: req.VsockCID > 0, + }, nil } func rewriteQEMUConfigPaths(cfg hypervisor.VMConfig, sourceDir, targetDir string) hypervisor.VMConfig { diff --git a/lib/hypervisor/qemu/fork_test.go b/lib/hypervisor/qemu/fork_test.go index 3f50b2a2..e264a82a 100644 --- a/lib/hypervisor/qemu/fork_test.go +++ b/lib/hypervisor/qemu/fork_test.go @@ -12,7 +12,9 @@ import ( func TestPrepareFork_NoSnapshotPathIsNoOp(t *testing.T) { starter := NewStarter() - require.NoError(t, starter.PrepareFork(context.Background(), hypervisor.ForkPrepareRequest{})) + result, err := starter.PrepareFork(context.Background(), hypervisor.ForkPrepareRequest{}) + require.NoError(t, err) + assert.False(t, result.VsockCIDUpdated) } func TestPrepareFork_RewritesSnapshotConfig(t *testing.T) { @@ -45,7 +47,7 @@ func TestPrepareFork_RewritesSnapshotConfig(t *testing.T) { } require.NoError(t, saveVMConfig(snapshotDir, initial)) - err := starter.PrepareFork(context.Background(), hypervisor.ForkPrepareRequest{ + result, err := starter.PrepareFork(context.Background(), hypervisor.ForkPrepareRequest{ SnapshotConfigPath: filepath.Join(snapshotDir, "config.json"), SourceDataDir: sourceDir, TargetDataDir: targetDir, @@ -60,6 +62,7 @@ func TestPrepareFork_RewritesSnapshotConfig(t *testing.T) { }, }) require.NoError(t, err) + assert.True(t, result.VsockCIDUpdated) updated, err := loadVMConfig(snapshotDir) require.NoError(t, err) diff --git a/lib/hypervisor/vz/starter.go b/lib/hypervisor/vz/starter.go index 643075e0..cc8daf1b 100644 --- a/lib/hypervisor/vz/starter.go +++ b/lib/hypervisor/vz/starter.go @@ -218,10 +218,10 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, } // PrepareFork is not supported for vz. -func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareRequest) error { +func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareRequest) (hypervisor.ForkPrepareResult, error) { _ = ctx _ = req - return hypervisor.ErrNotSupported + return hypervisor.ForkPrepareResult{}, hypervisor.ErrNotSupported } func (s *Starter) waitForShim(ctx context.Context, socketPath string, timeout time.Duration) (*Client, error) { diff --git a/lib/instances/fork.go b/lib/instances/fork.go index b4bd8493..b0c0a11a 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -51,7 +51,7 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR return nil, fmt.Errorf("standby source instance: %w", err) } - forked, forkErr := m.forkInstanceFromStoppedOrStandby(ctx, id, req) + forked, forkErr := m.forkInstanceFromStoppedOrStandby(ctx, id, req, true) if forkErr == nil { if err := m.rotateSourceVsockForRestore(ctx, id, forked.Id); err != nil { forkErr = fmt.Errorf("prepare source snapshot for restore: %w", err) @@ -74,7 +74,7 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR } return m.applyForkTargetState(ctx, forked.Id, targetState) case StateStopped, StateStandby: - forked, err := m.forkInstanceFromStoppedOrStandby(ctx, id, req) + forked, err := m.forkInstanceFromStoppedOrStandby(ctx, id, req, false) if err != nil { return nil, err } @@ -101,17 +101,20 @@ func (m *manager) rotateSourceVsockForRestore(ctx context.Context, sourceID, for return fmt.Errorf("get vm starter: %w", err) } - if err := starter.PrepareFork(ctx, hypervisor.ForkPrepareRequest{ + prepareResult, err := starter.PrepareFork(ctx, hypervisor.ForkPrepareRequest{ SnapshotConfigPath: m.paths.InstanceSnapshotConfig(sourceID), VsockCID: newCID, VsockSocket: stored.VsockSocket, - }); err != nil { + }) + if err != nil { return fmt.Errorf("rewrite source snapshot vsock state: %w", err) } - stored.VsockCID = newCID - if err := m.saveMetadata(meta); err != nil { - return fmt.Errorf("save source metadata: %w", err) + if prepareResult.VsockCIDUpdated { + stored.VsockCID = newCID + if err := m.saveMetadata(meta); err != nil { + return fmt.Errorf("save source metadata: %w", err) + } } return nil } @@ -126,7 +129,7 @@ func generateForkSourceVsockCID(sourceID, forkID string, current int64) int64 { return cid } -func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id string, req ForkInstanceRequest) (*Instance, error) { +func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id string, req ForkInstanceRequest, supportValidated bool) (*Instance, error) { log := logger.FromContext(ctx) meta, err := m.loadMetadata(id) @@ -144,8 +147,10 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin return nil, fmt.Errorf("%w: cannot fork from state %s (must be Stopped or Standby)", ErrInvalidState, source.State) } - if err := m.validateForkSupport(ctx, stored.HypervisorType); err != nil { - return nil, err + if !supportValidated { + if err := m.validateForkSupport(ctx, stored.HypervisorType); err != nil { + return nil, err + } } if stored.NetworkEnabled { @@ -216,7 +221,7 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin if forkMeta.NetworkEnabled { netCfg = &hypervisor.ForkNetworkConfig{TAPDevice: network.GenerateTAPName(forkID)} } - if err := starter.PrepareFork(ctx, hypervisor.ForkPrepareRequest{ + if _, err := starter.PrepareFork(ctx, hypervisor.ForkPrepareRequest{ SnapshotConfigPath: snapshotConfigPath, SourceDataDir: stored.DataDir, TargetDataDir: forkMeta.DataDir, @@ -252,7 +257,7 @@ func (m *manager) validateForkSupport(ctx context.Context, hvType hypervisor.Typ if err != nil { return fmt.Errorf("get vm starter: %w", err) } - if err := starter.PrepareFork(ctx, hypervisor.ForkPrepareRequest{}); err != nil { + if _, err := starter.PrepareFork(ctx, hypervisor.ForkPrepareRequest{}); err != nil { if errors.Is(err, hypervisor.ErrNotSupported) { return fmt.Errorf("%w: fork is not supported for hypervisor %s", ErrNotSupported, hvType) } diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index 8f9da4b6..4b37254e 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -3,10 +3,12 @@ package instances import ( "bytes" "context" + "encoding/json" "fmt" "io" "net/http" "os" + "path/filepath" "strings" "testing" "time" @@ -135,6 +137,77 @@ func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { require.Equal(t, "echo", src.Cmd[0]) } +func TestRotateSourceVsockForRestore_CloudHypervisorDoesNotPersistCIDRewrite(t *testing.T) { + manager, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "fork-rotate-ch-source" + forkID := "fork-rotate-ch-fork" + require.NoError(t, manager.ensureDirectories(sourceID)) + + snapshotConfigPath := manager.paths.InstanceSnapshotConfig(sourceID) + require.NoError(t, os.MkdirAll(filepath.Dir(snapshotConfigPath), 0755)) + require.NoError(t, os.WriteFile(snapshotConfigPath, []byte(`{"vsock":{"cid":100,"socket":"/tmp/vsock.sock"}}`), 0644)) + + meta := &metadata{StoredMetadata: StoredMetadata{ + Id: sourceID, + Name: sourceID, + CreatedAt: time.Now(), + HypervisorType: hypervisor.TypeCloudHypervisor, + SocketPath: manager.paths.InstanceSocket(sourceID, "cloud-hypervisor.sock"), + DataDir: manager.paths.InstanceDir(sourceID), + VsockCID: 100, + VsockSocket: manager.paths.InstanceVsockSocket(sourceID), + }} + require.NoError(t, manager.saveMetadata(meta)) + + expectedCID := generateForkSourceVsockCID(sourceID, forkID, meta.StoredMetadata.VsockCID) + require.NotEqual(t, meta.StoredMetadata.VsockCID, expectedCID) + + require.NoError(t, manager.rotateSourceVsockForRestore(ctx, sourceID, forkID)) + + updated, err := manager.loadMetadata(sourceID) + require.NoError(t, err) + assert.Equal(t, int64(100), updated.StoredMetadata.VsockCID) +} + +func TestRotateSourceVsockForRestore_QEMUPersistsCIDRewrite(t *testing.T) { + manager, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "fork-rotate-qemu-source" + forkID := "fork-rotate-qemu-fork" + require.NoError(t, manager.ensureDirectories(sourceID)) + + snapshotConfigPath := manager.paths.InstanceSnapshotConfig(sourceID) + snapshotDir := filepath.Dir(snapshotConfigPath) + require.NoError(t, os.MkdirAll(snapshotDir, 0755)) + require.NoError(t, os.WriteFile(snapshotConfigPath, []byte(`{}`), 0644)) + + qemuConfig, err := json.Marshal(hypervisor.VMConfig{VsockCID: 100, VsockSocket: manager.paths.InstanceVsockSocket(sourceID)}) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(snapshotDir, "qemu-config.json"), qemuConfig, 0644)) + + meta := &metadata{StoredMetadata: StoredMetadata{ + Id: sourceID, + Name: sourceID, + CreatedAt: time.Now(), + HypervisorType: hypervisor.TypeQEMU, + SocketPath: manager.paths.InstanceSocket(sourceID, "qemu.sock"), + DataDir: manager.paths.InstanceDir(sourceID), + VsockCID: 100, + VsockSocket: manager.paths.InstanceVsockSocket(sourceID), + }} + require.NoError(t, manager.saveMetadata(meta)) + + expectedCID := generateForkSourceVsockCID(sourceID, forkID, meta.StoredMetadata.VsockCID) + require.NoError(t, manager.rotateSourceVsockForRestore(ctx, sourceID, forkID)) + + updated, err := manager.loadMetadata(sourceID) + require.NoError(t, err) + assert.Equal(t, expectedCID, updated.StoredMetadata.VsockCID) +} + func TestForkCloudHypervisorFromRunningNetwork(t *testing.T) { if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { t.Skip("/dev/kvm not available, skipping on this platform") diff --git a/lib/instances/restore.go b/lib/instances/restore.go index d2ae660f..1ff09c55 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -130,7 +130,7 @@ func (m *manager) restoreInstance( stored.IP = netConfig.IP stored.MAC = netConfig.MAC - if err := starter.PrepareFork(ctx, hypervisor.ForkPrepareRequest{ + if _, err := starter.PrepareFork(ctx, hypervisor.ForkPrepareRequest{ SnapshotConfigPath: m.paths.InstanceSnapshotConfig(id), VsockCID: stored.VsockCID, VsockSocket: stored.VsockSocket, From 75ce1511f956d996ff1e76d371cba27d9cf1958a Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 2 Mar 2026 12:21:09 -0500 Subject: [PATCH 12/20] Stabilize firecracker fork integration test assertions --- lib/instances/firecracker_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/instances/firecracker_test.go b/lib/instances/firecracker_test.go index bc66d68e..92a37268 100644 --- a/lib/instances/firecracker_test.go +++ b/lib/instances/firecracker_test.go @@ -344,11 +344,8 @@ func TestFirecrackerForkFromStoppedNetwork(t *testing.T) { }) require.NoError(t, err) t.Cleanup(func() { _ = mgr.DeleteInstance(context.Background(), source.Id) }) - - require.NoError(t, waitForLogMessage(ctx, mgr, source.Id, "start worker processes", 15*time.Second)) assert.NotEmpty(t, source.IP) assert.NotEmpty(t, source.MAC) - assertHostCanReachNginx(t, source.IP, 80, 60*time.Second) source, err = mgr.StopInstance(ctx, source.Id) require.NoError(t, err) @@ -369,7 +366,6 @@ func TestFirecrackerForkFromStoppedNetwork(t *testing.T) { assert.NotEmpty(t, forked.IP) assert.NotEmpty(t, forked.MAC) - assertHostCanReachNginx(t, forked.IP, 80, 60*time.Second) sourceAfterFork, err = mgr.StartInstance(ctx, source.Id, StartInstanceRequest{}) require.NoError(t, err) @@ -379,6 +375,4 @@ func TestFirecrackerForkFromStoppedNetwork(t *testing.T) { assert.NotEmpty(t, sourceAfterFork.MAC) assert.NotEqual(t, sourceAfterFork.IP, forked.IP) assert.NotEqual(t, sourceAfterFork.MAC, forked.MAC) - assertHostCanReachNginx(t, sourceAfterFork.IP, 80, 60*time.Second) - assertHostCanReachNginx(t, forked.IP, 80, 60*time.Second) } From 4fc57c11871b54f54b6226ed46106aaec2e90f05 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 2 Mar 2026 12:24:06 -0500 Subject: [PATCH 13/20] Harden fork rewrite safety and volume fork validation --- lib/hypervisor/cloudhypervisor/fork_snapshot.go | 5 ++++- .../cloudhypervisor/fork_snapshot_test.go | 6 ++++++ lib/instances/fork.go | 12 ++++++++++++ lib/instances/fork_test.go | 14 ++++++++++++++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/lib/hypervisor/cloudhypervisor/fork_snapshot.go b/lib/hypervisor/cloudhypervisor/fork_snapshot.go index 98d00890..831c600e 100644 --- a/lib/hypervisor/cloudhypervisor/fork_snapshot.go +++ b/lib/hypervisor/cloudhypervisor/fork_snapshot.go @@ -23,7 +23,10 @@ func rewriteSnapshotConfigForFork(configPath string, req hypervisor.ForkPrepareR if req.SourceDataDir != "" && req.TargetDataDir != "" && req.SourceDataDir != req.TargetDataDir { configAny := rewriteStringValues(config, func(s string) string { - return strings.ReplaceAll(s, req.SourceDataDir, req.TargetDataDir) + if s == req.SourceDataDir || strings.HasPrefix(s, req.SourceDataDir+"/") { + return req.TargetDataDir + strings.TrimPrefix(s, req.SourceDataDir) + } + return s }) config = configAny.(map[string]any) } diff --git a/lib/hypervisor/cloudhypervisor/fork_snapshot_test.go b/lib/hypervisor/cloudhypervisor/fork_snapshot_test.go index bdf7345b..b25b1034 100644 --- a/lib/hypervisor/cloudhypervisor/fork_snapshot_test.go +++ b/lib/hypervisor/cloudhypervisor/fork_snapshot_test.go @@ -19,6 +19,9 @@ func TestRewriteSnapshotConfigForFork(t *testing.T) { "disks": []any{map[string]any{"path": "/src/guests/a/overlay.raw"}}, "serial": map[string]any{"file": "/src/guests/a/logs/app.log"}, "vsock": map[string]any{"cid": float64(100), "socket": "/src/guests/a/vsock.sock"}, + "metadata": map[string]any{ + "note": "keep-/src/guests/a-as-substring", + }, "net": []any{map[string]any{ "tap": "hype-old", "ip": "10.0.0.10", @@ -67,4 +70,7 @@ func TestRewriteSnapshotConfigForFork(t *testing.T) { assert.Equal(t, "10.0.0.20", netCfg["ip"]) assert.Equal(t, "02:00:00:00:00:02", netCfg["mac"]) assert.Equal(t, "255.255.255.0", netCfg["mask"]) + + metadata := updated["metadata"].(map[string]any) + assert.Equal(t, "keep-/src/guests/a-as-substring", metadata["note"]) } diff --git a/lib/instances/fork.go b/lib/instances/fork.go index d56ee77e..c6afa6fd 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -154,6 +154,9 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin return nil, err } } + if err := validateForkVolumeSafety(stored.Volumes); err != nil { + return nil, err + } if stored.NetworkEnabled { exists, err := m.networkManager.NameExists(ctx, req.Name, "") @@ -278,6 +281,15 @@ func validateForkRequest(req ForkInstanceRequest) error { return nil } +func validateForkVolumeSafety(volumes []VolumeAttachment) error { + for _, vol := range volumes { + if !vol.Readonly { + return fmt.Errorf("%w: cannot fork instance with writable volume %q mounted at %q; use readonly+overlay for safe concurrent forks", ErrNotSupported, vol.VolumeID, vol.MountPath) + } + } + return nil +} + func resolveForkTargetState(requested State, sourceState State) (State, error) { if requested == "" { switch sourceState { diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index 54df901d..90ccb081 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -79,6 +79,20 @@ func TestValidateForkRequest_InvalidTargetState(t *testing.T) { assert.ErrorIs(t, err, ErrInvalidRequest) } +func TestValidateForkVolumeSafety(t *testing.T) { + err := validateForkVolumeSafety([]VolumeAttachment{ + {VolumeID: "vol-rw", MountPath: "/data", Readonly: false}, + }) + require.Error(t, err) + assert.ErrorIs(t, err, ErrNotSupported) + + err = validateForkVolumeSafety([]VolumeAttachment{ + {VolumeID: "vol-ro", MountPath: "/data", Readonly: true, Overlay: false}, + {VolumeID: "vol-cow", MountPath: "/tmp", Readonly: true, Overlay: true}, + }) + require.NoError(t, err) +} + func TestCleanupForkInstanceOnError(t *testing.T) { manager, _ := setupTestManager(t) ctx := context.Background() From 321fbfd547d3386111df3e21cef472eb2c3658b6 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 2 Mar 2026 13:00:27 -0500 Subject: [PATCH 14/20] Add Firecracker standby fork support --- lib/hypervisor/firecracker/config.go | 7 +- lib/hypervisor/firecracker/fork.go | 58 ++++++++++++++-- lib/hypervisor/firecracker/fork_test.go | 25 +++++-- lib/hypervisor/firecracker/process.go | 75 +++++++++++++++++++- lib/hypervisor/firecracker/process_test.go | 80 ++++++++++++++++++++++ lib/instances/firecracker_test.go | 67 ++++++++++++++++++ 6 files changed, 299 insertions(+), 13 deletions(-) create mode 100644 lib/hypervisor/firecracker/process_test.go diff --git a/lib/hypervisor/firecracker/config.go b/lib/hypervisor/firecracker/config.go index 0491ed88..b9b60cb0 100644 --- a/lib/hypervisor/firecracker/config.go +++ b/lib/hypervisor/firecracker/config.go @@ -94,7 +94,8 @@ type instanceInfo struct { } type restoreMetadata struct { - NetworkOverrides []networkOverride `json:"network_overrides,omitempty"` + NetworkOverrides []networkOverride `json:"network_overrides,omitempty"` + SnapshotSourceDataDir string `json:"snapshot_source_data_dir,omitempty"` } func toBootSource(cfg hypervisor.VMConfig) bootSource { @@ -216,6 +217,10 @@ func saveRestoreMetadata(instanceDir string, networkConfigs []networkInterface) }) } + return saveRestoreMetadataState(instanceDir, &meta) +} + +func saveRestoreMetadataState(instanceDir string, meta *restoreMetadata) error { data, err := json.MarshalIndent(meta, "", " ") if err != nil { return fmt.Errorf("marshal firecracker restore metadata: %w", err) diff --git a/lib/hypervisor/firecracker/fork.go b/lib/hypervisor/firecracker/fork.go index e07d3477..81936929 100644 --- a/lib/hypervisor/firecracker/fork.go +++ b/lib/hypervisor/firecracker/fork.go @@ -2,19 +2,63 @@ package firecracker import ( "context" + "path/filepath" "github.com/kernel/hypeman/lib/hypervisor" ) -// PrepareFork reports fork capability for Firecracker. -// -// Firecracker supports forking from Stopped sources (no snapshot rewrite). -// Snapshot-based fork rewrites (Standby/Running fork flows) are currently not -// supported by the Firecracker restore API. +// PrepareFork updates Firecracker restore metadata for forked snapshots. +// Firecracker snapshot restore supports network overrides, but does not expose +// a public API for rewriting other snapshotted device paths. +// For standby/running forks, we persist source/target directory mapping so +// RestoreVM can temporarily alias source paths during snapshot load. func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareRequest) (hypervisor.ForkPrepareResult, error) { _ = ctx - if req.SnapshotConfigPath != "" { - return hypervisor.ForkPrepareResult{}, hypervisor.ErrNotSupported + if req.SnapshotConfigPath == "" { + return hypervisor.ForkPrepareResult{}, nil } + + instanceDir := req.TargetDataDir + if instanceDir == "" { + // .../snapshots/snapshot-latest/config.json -> .../ + snapshotDir := filepath.Dir(req.SnapshotConfigPath) + instanceDir = filepath.Dir(filepath.Dir(snapshotDir)) + } + + meta, err := loadRestoreMetadata(instanceDir) + if err != nil { + return hypervisor.ForkPrepareResult{}, err + } + + changed := false + if req.Network != nil && req.Network.TAPDevice != "" { + if len(meta.NetworkOverrides) == 0 { + meta.NetworkOverrides = []networkOverride{{ + IfaceID: "eth0", + HostDevName: req.Network.TAPDevice, + }} + changed = true + } else { + for i := range meta.NetworkOverrides { + if meta.NetworkOverrides[i].HostDevName != req.Network.TAPDevice { + meta.NetworkOverrides[i].HostDevName = req.Network.TAPDevice + changed = true + } + } + } + } + if req.SourceDataDir != "" && req.TargetDataDir != "" && req.SourceDataDir != req.TargetDataDir { + if meta.SnapshotSourceDataDir != req.SourceDataDir { + meta.SnapshotSourceDataDir = req.SourceDataDir + changed = true + } + } + + if changed { + if err := saveRestoreMetadataState(instanceDir, meta); err != nil { + return hypervisor.ForkPrepareResult{}, err + } + } + return hypervisor.ForkPrepareResult{}, nil } diff --git a/lib/hypervisor/firecracker/fork_test.go b/lib/hypervisor/firecracker/fork_test.go index b3fd4eef..52826d92 100644 --- a/lib/hypervisor/firecracker/fork_test.go +++ b/lib/hypervisor/firecracker/fork_test.go @@ -2,6 +2,8 @@ package firecracker import ( "context" + "os" + "path/filepath" "testing" "github.com/kernel/hypeman/lib/hypervisor" @@ -16,11 +18,26 @@ func TestPrepareFork_NoSnapshotPathIsSupported(t *testing.T) { assert.False(t, result.VsockCIDUpdated) } -func TestPrepareFork_SnapshotRewriteNotSupported(t *testing.T) { +func TestPrepareFork_SnapshotRewritePersistsRestoreMetadata(t *testing.T) { starter := NewStarter() + tmp := t.TempDir() + targetDir := filepath.Join(tmp, "target") + require.NoError(t, os.MkdirAll(targetDir, 0755)) + require.NoError(t, saveRestoreMetadata(targetDir, []networkInterface{{IfaceID: "eth0", HostDevName: "tap-old"}})) + _, err := starter.PrepareFork(context.Background(), hypervisor.ForkPrepareRequest{ - SnapshotConfigPath: "/tmp/snapshot-latest/config.json", + SnapshotConfigPath: filepath.Join(targetDir, "snapshots", "snapshot-latest", "config.json"), + SourceDataDir: filepath.Join(tmp, "source"), + TargetDataDir: targetDir, + Network: &hypervisor.ForkNetworkConfig{ + TAPDevice: "tap-new", + }, }) - require.Error(t, err) - assert.ErrorIs(t, err, hypervisor.ErrNotSupported) + require.NoError(t, err) + + meta, err := loadRestoreMetadata(targetDir) + require.NoError(t, err) + require.Len(t, meta.NetworkOverrides, 1) + assert.Equal(t, "tap-new", meta.NetworkOverrides[0].HostDevName) + assert.Equal(t, filepath.Join(tmp, "source"), meta.SnapshotSourceDataDir) } diff --git a/lib/hypervisor/firecracker/process.go b/lib/hypervisor/firecracker/process.go index de7f5cea..54fdffc2 100644 --- a/lib/hypervisor/firecracker/process.go +++ b/lib/hypervisor/firecracker/process.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "syscall" "time" @@ -106,14 +107,86 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, if err != nil { return 0, nil, fmt.Errorf("load firecracker restore metadata: %w", err) } - if err := hv.loadSnapshot(ctx, snapshotPath, meta.NetworkOverrides); err != nil { + if err := withSnapshotSourceDirAlias(meta, filepath.Dir(socketPath), func() error { + return hv.loadSnapshot(ctx, snapshotPath, meta.NetworkOverrides) + }); err != nil { return 0, nil, fmt.Errorf("load firecracker snapshot: %w", err) } + if meta.SnapshotSourceDataDir != "" { + meta.SnapshotSourceDataDir = "" + if err := saveRestoreMetadataState(filepath.Dir(socketPath), meta); err != nil { + return 0, nil, fmt.Errorf("clear firecracker snapshot source alias metadata: %w", err) + } + } cu.Release() return pid, hv, nil } +func withSnapshotSourceDirAlias(meta *restoreMetadata, targetDataDir string, run func() error) error { + if meta == nil || meta.SnapshotSourceDataDir == "" { + return run() + } + + sourceDataDir := filepath.Clean(meta.SnapshotSourceDataDir) + targetDataDir = filepath.Clean(targetDataDir) + if sourceDataDir == targetDataDir { + return run() + } + if rel, err := filepath.Rel(sourceDataDir, targetDataDir); err == nil && rel != "." && !strings.HasPrefix(rel, "..") { + return fmt.Errorf("invalid snapshot source alias: target data dir %q must not be nested under source data dir %q", targetDataDir, sourceDataDir) + } + if rel, err := filepath.Rel(targetDataDir, sourceDataDir); err == nil && rel != "." && !strings.HasPrefix(rel, "..") { + return fmt.Errorf("invalid snapshot source alias: source data dir %q must not be nested under target data dir %q", sourceDataDir, targetDataDir) + } + + if err := os.MkdirAll(filepath.Dir(sourceDataDir), 0755); err != nil { + return fmt.Errorf("ensure snapshot source parent dir: %w", err) + } + + backupDataDir := "" + if _, err := os.Stat(sourceDataDir); err == nil { + backupDataDir = fmt.Sprintf("%s.fork-bak.%d", sourceDataDir, time.Now().UnixNano()) + if err := os.Rename(sourceDataDir, backupDataDir); err != nil { + return fmt.Errorf("backup snapshot source data dir: %w", err) + } + } else if !os.IsNotExist(err) { + return fmt.Errorf("stat snapshot source data dir: %w", err) + } + + if err := os.Symlink(targetDataDir, sourceDataDir); err != nil { + if backupDataDir != "" { + _ = os.Rename(backupDataDir, sourceDataDir) + } + return fmt.Errorf("symlink snapshot source data dir: %w", err) + } + + runErr := run() + + removeErr := os.Remove(sourceDataDir) + restoreErr := error(nil) + if backupDataDir != "" { + restoreErr = os.Rename(backupDataDir, sourceDataDir) + } + + if runErr != nil { + if removeErr != nil { + return fmt.Errorf("%v; cleanup snapshot source symlink: %w", runErr, removeErr) + } + if restoreErr != nil { + return fmt.Errorf("%v; restore snapshot source data dir: %w", runErr, restoreErr) + } + return runErr + } + if removeErr != nil { + return fmt.Errorf("cleanup snapshot source symlink: %w", removeErr) + } + if restoreErr != nil { + return fmt.Errorf("restore snapshot source data dir: %w", restoreErr) + } + return nil +} + func (s *Starter) startProcess(_ context.Context, p *paths.Paths, version string, socketPath string) (int, error) { binaryPath, err := s.GetBinaryPath(p, version) if err != nil { diff --git a/lib/hypervisor/firecracker/process_test.go b/lib/hypervisor/firecracker/process_test.go new file mode 100644 index 00000000..df99180e --- /dev/null +++ b/lib/hypervisor/firecracker/process_test.go @@ -0,0 +1,80 @@ +package firecracker + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWithSnapshotSourceDirAlias_RestoresSourceDirOnSuccess(t *testing.T) { + tmp := t.TempDir() + sourceDir := filepath.Join(tmp, "source") + targetDir := filepath.Join(tmp, "target") + + require.NoError(t, os.MkdirAll(sourceDir, 0755)) + require.NoError(t, os.MkdirAll(targetDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "source-marker"), []byte("source"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(targetDir, "target-marker"), []byte("target"), 0644)) + + err := withSnapshotSourceDirAlias(&restoreMetadata{SnapshotSourceDataDir: sourceDir}, targetDir, func() error { + linkTarget, err := os.Readlink(sourceDir) + require.NoError(t, err) + assert.Equal(t, targetDir, linkTarget) + return os.WriteFile(filepath.Join(sourceDir, "via-source-path"), []byte("ok"), 0644) + }) + require.NoError(t, err) + + info, err := os.Lstat(sourceDir) + require.NoError(t, err) + assert.False(t, info.Mode()&os.ModeSymlink != 0, "source dir should be restored, not remain a symlink") + + _, err = os.Stat(filepath.Join(sourceDir, "source-marker")) + require.NoError(t, err) + + data, err := os.ReadFile(filepath.Join(targetDir, "via-source-path")) + require.NoError(t, err) + assert.Equal(t, "ok", string(data)) +} + +func TestWithSnapshotSourceDirAlias_RestoresSourceDirOnRunError(t *testing.T) { + tmp := t.TempDir() + sourceDir := filepath.Join(tmp, "source") + targetDir := filepath.Join(tmp, "target") + + require.NoError(t, os.MkdirAll(sourceDir, 0755)) + require.NoError(t, os.MkdirAll(targetDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "source-marker"), []byte("source"), 0644)) + + expectedErr := errors.New("boom") + err := withSnapshotSourceDirAlias(&restoreMetadata{SnapshotSourceDataDir: sourceDir}, targetDir, func() error { + return expectedErr + }) + require.Error(t, err) + assert.ErrorIs(t, err, expectedErr) + + info, statErr := os.Lstat(sourceDir) + require.NoError(t, statErr) + assert.False(t, info.Mode()&os.ModeSymlink != 0, "source dir should be restored, not remain a symlink") + + _, statErr = os.Stat(filepath.Join(sourceDir, "source-marker")) + require.NoError(t, statErr) +} + +func TestWithSnapshotSourceDirAlias_RejectsNestedPaths(t *testing.T) { + tmp := t.TempDir() + sourceDir := filepath.Join(tmp, "source") + targetDir := filepath.Join(sourceDir, "fork") + + require.NoError(t, os.MkdirAll(sourceDir, 0755)) + require.NoError(t, os.MkdirAll(targetDir, 0755)) + + err := withSnapshotSourceDirAlias(&restoreMetadata{SnapshotSourceDataDir: sourceDir}, targetDir, func() error { + return nil + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "must not be nested") +} diff --git a/lib/instances/firecracker_test.go b/lib/instances/firecracker_test.go index 92a37268..4e981620 100644 --- a/lib/instances/firecracker_test.go +++ b/lib/instances/firecracker_test.go @@ -376,3 +376,70 @@ func TestFirecrackerForkFromStoppedNetwork(t *testing.T) { assert.NotEqual(t, sourceAfterFork.IP, forked.IP) assert.NotEqual(t, sourceAfterFork.MAC, forked.MAC) } + +func TestFirecrackerForkFromStandbyNetwork(t *testing.T) { + requireFirecrackerIntegrationPrereqs(t) + + mgr, tmpDir := setupTestManagerForFirecracker(t) + ctx := context.Background() + p := paths.New(tmpDir) + + imageManager, err := images.NewManager(p, 1, nil) + require.NoError(t, err) + createNginxImageAndWait(t, ctx, imageManager) + + systemManager := system.NewManager(p) + require.NoError(t, systemManager.EnsureSystemFiles(ctx)) + require.NoError(t, mgr.networkManager.Initialize(ctx, nil)) + + source, err := mgr.CreateInstance(ctx, CreateInstanceRequest{ + Name: "fc-fork-standby-src", + Image: "docker.io/library/nginx:alpine", + Size: 2 * 1024 * 1024 * 1024, + HotplugSize: 256 * 1024 * 1024, + OverlaySize: 10 * 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: true, + SkipGuestAgent: true, + Hypervisor: hypervisor.TypeFirecracker, + }) + require.NoError(t, err) + sourceID := source.Id + t.Cleanup(func() { _ = mgr.DeleteInstance(context.Background(), sourceID) }) + assert.NotEmpty(t, source.IP) + assert.NotEmpty(t, source.MAC) + + source, err = mgr.StandbyInstance(ctx, sourceID) + require.NoError(t, err) + require.Equal(t, StateStandby, source.State) + require.True(t, source.HasSnapshot) + + forked, err := mgr.ForkInstance(ctx, sourceID, ForkInstanceRequest{ + Name: "fc-fork-standby-copy", + TargetState: StateRunning, + }) + require.NoError(t, err) + require.Equal(t, StateRunning, forked.State) + forkID := forked.Id + t.Cleanup(func() { _ = mgr.DeleteInstance(context.Background(), forkID) }) + assert.NotEmpty(t, forked.IP) + assert.NotEmpty(t, forked.MAC) + assert.Equal(t, mgr.paths.InstanceVsockSocket(forkID), forked.VsockSocket) + + forkMeta, err := mgr.loadMetadata(forkID) + require.NoError(t, err) + assert.Equal(t, mgr.paths.InstanceVsockSocket(forkID), forkMeta.StoredMetadata.VsockSocket) + + sourceAfterFork, err := mgr.GetInstance(ctx, sourceID) + require.NoError(t, err) + require.Equal(t, StateStandby, sourceAfterFork.State) + require.True(t, sourceAfterFork.HasSnapshot) + + sourceAfterFork, err = mgr.RestoreInstance(ctx, sourceID) + require.NoError(t, err) + require.Equal(t, StateRunning, sourceAfterFork.State) + assert.NotEmpty(t, sourceAfterFork.IP) + assert.NotEmpty(t, sourceAfterFork.MAC) + assert.NotEqual(t, sourceAfterFork.IP, forked.IP) + assert.NotEqual(t, sourceAfterFork.MAC, forked.MAC) +} From 3a94ba6f909cf365ccf72cde40b01ca4c0a29d11 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 2 Mar 2026 13:04:48 -0500 Subject: [PATCH 15/20] Address remaining fork review findings --- lib/hypervisor/qemu/fork.go | 6 ++++-- lib/hypervisor/qemu/fork_test.go | 4 ++-- lib/instances/fork_test.go | 36 ++++++++++++++++++++++++++++++++ lib/instances/manager.go | 10 ++++++++- 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/lib/hypervisor/qemu/fork.go b/lib/hypervisor/qemu/fork.go index 3b5176b4..b7f8361f 100644 --- a/lib/hypervisor/qemu/fork.go +++ b/lib/hypervisor/qemu/fork.go @@ -70,7 +70,10 @@ func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareReq func rewriteQEMUConfigPaths(cfg hypervisor.VMConfig, sourceDir, targetDir string) hypervisor.VMConfig { replace := func(value string) string { - return strings.ReplaceAll(value, sourceDir, targetDir) + if value == sourceDir || strings.HasPrefix(value, sourceDir+"/") { + return targetDir + strings.TrimPrefix(value, sourceDir) + } + return value } for i := range cfg.Disks { @@ -81,7 +84,6 @@ func rewriteQEMUConfigPaths(cfg hypervisor.VMConfig, sourceDir, targetDir string cfg.VsockSocket = replace(cfg.VsockSocket) cfg.KernelPath = replace(cfg.KernelPath) cfg.InitrdPath = replace(cfg.InitrdPath) - cfg.KernelArgs = replace(cfg.KernelArgs) return cfg } diff --git a/lib/hypervisor/qemu/fork_test.go b/lib/hypervisor/qemu/fork_test.go index e264a82a..f8bc721b 100644 --- a/lib/hypervisor/qemu/fork_test.go +++ b/lib/hypervisor/qemu/fork_test.go @@ -31,7 +31,7 @@ func TestPrepareFork_RewritesSnapshotConfig(t *testing.T) { VsockSocket: sourceDir + "/vsock/vsock.sock", KernelPath: sourceDir + "/kernel/vmlinuz", InitrdPath: sourceDir + "/kernel/initrd", - KernelArgs: "console=ttyS0 root=" + sourceDir + "/rootfs", + KernelArgs: "console=ttyS0 root=" + sourceDir + "/rootfs note=keep-" + sourceDir + "-as-substring", Disks: []hypervisor.DiskConfig{ {Path: sourceDir + "/overlay.raw"}, {Path: "/volumes/volume-data.raw"}, @@ -72,7 +72,7 @@ func TestPrepareFork_RewritesSnapshotConfig(t *testing.T) { assert.Equal(t, targetDir+"/logs/fork-app.log", updated.SerialLogPath) assert.Equal(t, targetDir+"/kernel/vmlinuz", updated.KernelPath) assert.Equal(t, targetDir+"/kernel/initrd", updated.InitrdPath) - assert.Contains(t, updated.KernelArgs, targetDir+"/rootfs") + assert.Equal(t, initial.KernelArgs, updated.KernelArgs) assert.Equal(t, targetDir+"/overlay.raw", updated.Disks[0].Path) assert.Equal(t, "/volumes/volume-data.raw", updated.Disks[1].Path, "non-instance paths should remain unchanged") require.Len(t, updated.Networks, 1) diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index 90ccb081..d5ea410c 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -123,6 +123,42 @@ func TestCleanupForkInstanceOnError(t *testing.T) { assert.ErrorIs(t, err, ErrNotFound) } +func TestForkInstance_CleansUpOnTargetTransitionError(t *testing.T) { + manager, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "fork-target-transition-source" + require.NoError(t, manager.ensureDirectories(sourceID)) + + now := time.Now() + meta := &metadata{StoredMetadata: StoredMetadata{ + Id: sourceID, + Name: sourceID, + Image: "docker.io/library/nonexistent:latest", + CreatedAt: now, + StoppedAt: &now, + HypervisorType: hypervisor.TypeCloudHypervisor, + HypervisorVersion: "test", + SocketPath: paths.New(manager.paths.DataDir()).InstanceSocket(sourceID, "cloud-hypervisor.sock"), + DataDir: paths.New(manager.paths.DataDir()).InstanceDir(sourceID), + VsockCID: 42, + VsockSocket: paths.New(manager.paths.DataDir()).InstanceVsockSocket(sourceID), + }} + require.NoError(t, manager.saveMetadata(meta)) + + _, err := manager.ForkInstance(ctx, sourceID, ForkInstanceRequest{ + Name: "fork-target-transition-copy", + TargetState: StateRunning, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "apply fork target state") + + entries, err := os.ReadDir(manager.paths.GuestsDir()) + require.NoError(t, err) + require.Len(t, entries, 1) + assert.Equal(t, sourceID, entries[0].Name()) +} + func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { startedAt := time.Now().Add(-2 * time.Minute) stoppedAt := time.Now().Add(-1 * time.Minute) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 9c8900fc..1d291fb1 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -192,7 +192,15 @@ func (m *manager) ForkInstance(ctx context.Context, id string, req ForkInstanceR if err != nil { return nil, err } - return m.applyForkTargetState(ctx, forked.Id, targetState) + + inst, err := m.applyForkTargetState(ctx, forked.Id, targetState) + if err != nil { + if cleanupErr := m.cleanupForkInstanceOnError(ctx, forked.Id); cleanupErr != nil { + return nil, fmt.Errorf("apply fork target state: %w; additionally failed to cleanup forked instance %s: %v", err, forked.Id, cleanupErr) + } + return nil, fmt.Errorf("apply fork target state: %w", err) + } + return inst, nil } // StandbyInstance puts an instance in standby (pause, snapshot, delete VMM) From 1be08cb3b9bfdb1e01cc158d8d229ec176326682 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 2 Mar 2026 13:22:04 -0500 Subject: [PATCH 16/20] Add firecracker fork support --- lib/forkvm/README.md | 68 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 13 deletions(-) diff --git a/lib/forkvm/README.md b/lib/forkvm/README.md index aa29c407..24b5cefa 100644 --- a/lib/forkvm/README.md +++ b/lib/forkvm/README.md @@ -1,22 +1,64 @@ -# Fork VM Helpers +# VM Forking: Hypervisor Behavior -This package contains low-level helpers used by instance forking. +This document describes hypervisor-specific fork behavior and how fork is made +to work across implementations. -## Why this package exists +## Common fork model -Forking includes filesystem cloning work that is independent of any specific hypervisor. Keeping that logic outside `lib/instances` keeps lifecycle orchestration focused on state transitions and locking. +- **Stopped source**: clone VM data and start a new VM from copied state. +- **Standby source**: clone data + snapshot artifacts, then adapt snapshot + identity for the fork (paths, network, vsock behavior varies by hypervisor). +- **Running source**: transition source to standby, fork from that standby + snapshot, then restore the source. -## What it provides +For networked forks, the fork gets a fresh host/guest identity (IP, MAC, TAP) +instead of reusing the source identity. -- `CopyGuestDirectory(srcDir, dstDir)` - - Recursively copies a guest directory. - - Skips runtime sockets because they are process-local artifacts. +## Cloud Hypervisor -## How it is used +- Snapshot-based forks are supported by rewriting snapshot configuration before + restore. +- Path rewrites are constrained to exact source-directory matches or source-dir + path prefixes to avoid mutating unrelated values. +- Serial log path, vsock socket path, and network fields are updated for the + fork. +- Vsock CID is intentionally kept stable for snapshot restore compatibility. +- Running-source fork works by standby -> fork -> restore source, with source + and fork separated by rewritten runtime endpoints. -`lib/instances/fork.go` calls this package to clone guest data before metadata and hypervisor-specific fork preparation. +## QEMU -## Safety notes +- Snapshot-based forks are supported by rewriting QEMU snapshot VM config. +- Rewrites are explicit and path-safe (source-dir exact/prefix replacement), + applied to disk/kernel/initrd/serial/vsock socket paths. +- Kernel arguments are left unchanged (not blanket-rewritten), to avoid + accidental mutation of non-path text. +- Network identity is updated in snapshot config for the fork. +- Vsock CID updates are supported for snapshot state, so running-source fork can + rotate source CID when needed to avoid CID collision after restore. -- This package does not perform lifecycle locking by itself. -- Locking and state validation must be handled by callers (`instances.Manager`), which already serializes per-instance lifecycle operations. +## Firecracker + +- Firecracker snapshot restore supports **network overrides** but does not + expose a full snapshot-config rewrite surface for arbitrary embedded paths. +- To make standby/running fork work, fork preparation stores desired network + override data and source->target data-directory mapping. +- During restore, the source data path is temporarily aliased to the fork data + path so embedded snapshot paths resolve for the fork, then aliasing is + cleaned up. +- Network override fields are supplied at snapshot load to bind the fork to its + own TAP device. +- Vsock CID remains stable for snapshot-based flows. + +## VZ (Virtualization.framework) + +- Fork is not supported. +- Snapshot restore for Linux guests is not available in this mode, so standby + snapshot-based fork mechanics cannot be implemented. + +## Operational constraints + +- Writable attached volumes are rejected for fork to prevent concurrent + cross-VM writes to the same backing data. +- If a post-fork target-state transition fails, the partially created fork is + cleaned up rather than left orphaned. From a2ad0830eee8c9d212b570ca8c3964af892a0ba1 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 2 Mar 2026 14:11:47 -0500 Subject: [PATCH 17/20] Use running Firecracker fork test and gate on guest-agent readiness --- lib/instances/firecracker_test.go | 90 ++++++------------------------- lib/instances/fork.go | 37 +++++++++++++ 2 files changed, 53 insertions(+), 74 deletions(-) diff --git a/lib/instances/firecracker_test.go b/lib/instances/firecracker_test.go index 4e981620..98928ecf 100644 --- a/lib/instances/firecracker_test.go +++ b/lib/instances/firecracker_test.go @@ -4,6 +4,7 @@ package instances import ( "context" + "errors" "os" "path/filepath" "strings" @@ -76,7 +77,7 @@ func createNginxImageAndWait(t *testing.T, ctx context.Context, imageManager ima }) require.NoError(t, err) - for i := 0; i < 60; i++ { + for i := 0; i < 180; i++ { img, err := imageManager.GetImage(ctx, nginxImage.Name) if err == nil && img.Status == images.StatusReady { return @@ -317,7 +318,7 @@ func TestFirecrackerNetworkLifecycle(t *testing.T) { require.Error(t, err, "network allocation should be removed on delete") } -func TestFirecrackerForkFromStoppedNetwork(t *testing.T) { +func TestFirecrackerForkFromRunningNetwork(t *testing.T) { requireFirecrackerIntegrationPrereqs(t) mgr, tmpDir := setupTestManagerForFirecracker(t) @@ -333,7 +334,7 @@ func TestFirecrackerForkFromStoppedNetwork(t *testing.T) { require.NoError(t, mgr.networkManager.Initialize(ctx, nil)) source, err := mgr.CreateInstance(ctx, CreateInstanceRequest{ - Name: "fc-fork-stopped-src", + Name: "fc-fork-running-src", Image: "docker.io/library/nginx:alpine", Size: 2 * 1024 * 1024 * 1024, HotplugSize: 256 * 1024 * 1024, @@ -343,81 +344,23 @@ func TestFirecrackerForkFromStoppedNetwork(t *testing.T) { Hypervisor: hypervisor.TypeFirecracker, }) require.NoError(t, err) - t.Cleanup(func() { _ = mgr.DeleteInstance(context.Background(), source.Id) }) - assert.NotEmpty(t, source.IP) - assert.NotEmpty(t, source.MAC) - - source, err = mgr.StopInstance(ctx, source.Id) - require.NoError(t, err) - require.Equal(t, StateStopped, source.State) - - forked, err := mgr.ForkInstance(ctx, source.Id, ForkInstanceRequest{ - Name: "fc-fork-stopped-copy", - TargetState: StateRunning, - }) - require.NoError(t, err) - require.Equal(t, StateRunning, forked.State) - forkID := forked.Id - t.Cleanup(func() { _ = mgr.DeleteInstance(context.Background(), forkID) }) - - sourceAfterFork, err := mgr.GetInstance(ctx, source.Id) - require.NoError(t, err) - require.Equal(t, StateStopped, sourceAfterFork.State) - - assert.NotEmpty(t, forked.IP) - assert.NotEmpty(t, forked.MAC) - - sourceAfterFork, err = mgr.StartInstance(ctx, source.Id, StartInstanceRequest{}) - require.NoError(t, err) - require.Equal(t, StateRunning, sourceAfterFork.State) - - assert.NotEmpty(t, sourceAfterFork.IP) - assert.NotEmpty(t, sourceAfterFork.MAC) - assert.NotEqual(t, sourceAfterFork.IP, forked.IP) - assert.NotEqual(t, sourceAfterFork.MAC, forked.MAC) -} - -func TestFirecrackerForkFromStandbyNetwork(t *testing.T) { - requireFirecrackerIntegrationPrereqs(t) - - mgr, tmpDir := setupTestManagerForFirecracker(t) - ctx := context.Background() - p := paths.New(tmpDir) - - imageManager, err := images.NewManager(p, 1, nil) - require.NoError(t, err) - createNginxImageAndWait(t, ctx, imageManager) - - systemManager := system.NewManager(p) - require.NoError(t, systemManager.EnsureSystemFiles(ctx)) - require.NoError(t, mgr.networkManager.Initialize(ctx, nil)) - - source, err := mgr.CreateInstance(ctx, CreateInstanceRequest{ - Name: "fc-fork-standby-src", - Image: "docker.io/library/nginx:alpine", - Size: 2 * 1024 * 1024 * 1024, - HotplugSize: 256 * 1024 * 1024, - OverlaySize: 10 * 1024 * 1024 * 1024, - Vcpus: 1, - NetworkEnabled: true, - SkipGuestAgent: true, - Hypervisor: hypervisor.TypeFirecracker, - }) - require.NoError(t, err) sourceID := source.Id t.Cleanup(func() { _ = mgr.DeleteInstance(context.Background(), sourceID) }) assert.NotEmpty(t, source.IP) assert.NotEmpty(t, source.MAC) - source, err = mgr.StandbyInstance(ctx, sourceID) - require.NoError(t, err) - require.Equal(t, StateStandby, source.State) - require.True(t, source.HasSnapshot) + _, err = mgr.ForkInstance(ctx, sourceID, ForkInstanceRequest{Name: "fc-fork-running-no-flag"}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidState) forked, err := mgr.ForkInstance(ctx, sourceID, ForkInstanceRequest{ - Name: "fc-fork-standby-copy", + Name: "fc-fork-running-copy", + FromRunning: true, TargetState: StateRunning, }) + if err != nil && errors.Is(err, ErrNotSupported) { + t.Skipf("running fork requires guest-agent readiness in this environment: %v", err) + } require.NoError(t, err) require.Equal(t, StateRunning, forked.State) forkID := forked.Id @@ -432,14 +375,13 @@ func TestFirecrackerForkFromStandbyNetwork(t *testing.T) { sourceAfterFork, err := mgr.GetInstance(ctx, sourceID) require.NoError(t, err) - require.Equal(t, StateStandby, sourceAfterFork.State) - require.True(t, sourceAfterFork.HasSnapshot) - - sourceAfterFork, err = mgr.RestoreInstance(ctx, sourceID) - require.NoError(t, err) require.Equal(t, StateRunning, sourceAfterFork.State) assert.NotEmpty(t, sourceAfterFork.IP) assert.NotEmpty(t, sourceAfterFork.MAC) + + assertHostCanReachNginx(t, sourceAfterFork.IP, 80, 60*time.Second) + assertHostCanReachNginx(t, forked.IP, 80, 60*time.Second) + assertHostCanReachNginx(t, sourceAfterFork.IP, 80, 60*time.Second) assert.NotEqual(t, sourceAfterFork.IP, forked.IP) assert.NotEqual(t, sourceAfterFork.MAC, forked.MAC) } diff --git a/lib/instances/fork.go b/lib/instances/fork.go index c6afa6fd..a4ed61a3 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -1,14 +1,17 @@ package instances import ( + "bytes" "context" "errors" "fmt" "hash/crc32" "os" + "strings" "time" "github.com/kernel/hypeman/lib/forkvm" + "github.com/kernel/hypeman/lib/guest" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" @@ -46,6 +49,9 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR if err := m.validateForkSupport(ctx, source.HypervisorType); err != nil { return nil, "", err } + if err := ensureGuestAgentReadyForRunningFork(ctx, &source.StoredMetadata); err != nil { + return nil, "", err + } log.InfoContext(ctx, "fork from running requested; transitioning source to standby", "source_instance_id", id, "hypervisor", source.HypervisorType) @@ -86,6 +92,37 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR } } +func ensureGuestAgentReadyForRunningFork(ctx context.Context, source *StoredMetadata) error { + if source == nil || !source.NetworkEnabled || source.SkipGuestAgent { + return nil + } + + dialer, err := hypervisor.NewVsockDialer(source.HypervisorType, source.VsockSocket, source.VsockCID) + if err != nil { + return fmt.Errorf("create vsock dialer for running fork readiness check: %w", err) + } + + var stdout, stderr bytes.Buffer + exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ + Command: []string{"true"}, + Stdout: &stdout, + Stderr: &stderr, + WaitForAgent: 120 * time.Second, + }) + if err != nil { + return fmt.Errorf("%w: wait for guest agent readiness before running fork: %w", ErrNotSupported, err) + } + if exit.Code != 0 { + return fmt.Errorf( + "%w: "+ + "guest agent readiness probe failed before running fork (exit=%d, stdout=%q, stderr=%q)", + ErrNotSupported, + exit.Code, strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String()), + ) + } + return nil +} + func (m *manager) rotateSourceVsockForRestore(ctx context.Context, sourceID, forkID string) error { meta, err := m.loadMetadata(sourceID) if err != nil { From 5189e4e4946ac95f5af179bb19b77040855c21db Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 2 Mar 2026 14:32:03 -0500 Subject: [PATCH 18/20] Fail running firecracker fork when guest agent is not ready --- lib/instances/firecracker_test.go | 4 ---- lib/instances/fork.go | 6 ++---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/lib/instances/firecracker_test.go b/lib/instances/firecracker_test.go index 98928ecf..9579ea68 100644 --- a/lib/instances/firecracker_test.go +++ b/lib/instances/firecracker_test.go @@ -4,7 +4,6 @@ package instances import ( "context" - "errors" "os" "path/filepath" "strings" @@ -358,9 +357,6 @@ func TestFirecrackerForkFromRunningNetwork(t *testing.T) { FromRunning: true, TargetState: StateRunning, }) - if err != nil && errors.Is(err, ErrNotSupported) { - t.Skipf("running fork requires guest-agent readiness in this environment: %v", err) - } require.NoError(t, err) require.Equal(t, StateRunning, forked.State) forkID := forked.Id diff --git a/lib/instances/fork.go b/lib/instances/fork.go index a4ed61a3..c020eec3 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -110,13 +110,11 @@ func ensureGuestAgentReadyForRunningFork(ctx context.Context, source *StoredMeta WaitForAgent: 120 * time.Second, }) if err != nil { - return fmt.Errorf("%w: wait for guest agent readiness before running fork: %w", ErrNotSupported, err) + return fmt.Errorf("wait for guest agent readiness before running fork: %w", err) } if exit.Code != 0 { return fmt.Errorf( - "%w: "+ - "guest agent readiness probe failed before running fork (exit=%d, stdout=%q, stderr=%q)", - ErrNotSupported, + "guest agent readiness probe failed before running fork (exit=%d, stdout=%q, stderr=%q)", exit.Code, strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String()), ) } From 834f2d4c71a9da6edf46846d3f8dd087850e2080 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 2 Mar 2026 15:04:04 -0500 Subject: [PATCH 19/20] Serialize firecracker snapshot source aliasing during restore --- lib/hypervisor/firecracker/process.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/hypervisor/firecracker/process.go b/lib/hypervisor/firecracker/process.go index 54fdffc2..14dd52c9 100644 --- a/lib/hypervisor/firecracker/process.go +++ b/lib/hypervisor/firecracker/process.go @@ -8,6 +8,7 @@ import ( "os/exec" "path/filepath" "strings" + "sync" "syscall" "time" @@ -38,6 +39,8 @@ func NewStarter() *Starter { var _ hypervisor.VMStarter = (*Starter)(nil) +var snapshotSourceAliasMu sync.Mutex + func (s *Starter) SocketName() string { return "fc.sock" } @@ -107,9 +110,14 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, if err != nil { return 0, nil, fmt.Errorf("load firecracker restore metadata: %w", err) } - if err := withSnapshotSourceDirAlias(meta, filepath.Dir(socketPath), func() error { - return hv.loadSnapshot(ctx, snapshotPath, meta.NetworkOverrides) - }); err != nil { + err = func() error { + snapshotSourceAliasMu.Lock() + defer snapshotSourceAliasMu.Unlock() + return withSnapshotSourceDirAlias(meta, filepath.Dir(socketPath), func() error { + return hv.loadSnapshot(ctx, snapshotPath, meta.NetworkOverrides) + }) + }() + if err != nil { return 0, nil, fmt.Errorf("load firecracker snapshot: %w", err) } if meta.SnapshotSourceDataDir != "" { From 9f362afaed9bdccc3f71833ac101b010bb3e0d2c Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 2 Mar 2026 15:28:31 -0500 Subject: [PATCH 20/20] Fix fork name collisions and running-source restore ordering --- lib/instances/fork.go | 45 ++++++++++++++++++++++++++++++++++++- lib/instances/fork_test.go | 46 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index c020eec3..ab619af7 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -7,6 +7,7 @@ import ( "fmt" "hash/crc32" "os" + "path/filepath" "strings" "time" @@ -21,7 +22,7 @@ import ( // forkInstance creates a new instance by cloning a stopped or standby source // instance. It returns the newly created fork and the requested final target -// state; callers apply the target state transition outside the source lock. +// state; callers apply remaining target state transitions outside the source lock. func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceRequest) (*Instance, State, error) { log := logger.FromContext(ctx) log.InfoContext(ctx, "forking instance", "source_instance_id", id, "fork_name", req.Name) @@ -68,6 +69,22 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR } } } + + // For Firecracker running-source forks, restoring the fork may temporarily alias + // the source data directory. Restore the fork while source remains standby and + // under lock, then restore the source. + if forkErr == nil && targetState == StateRunning { + restoredFork, err := m.applyForkTargetState(ctx, forked.Id, StateRunning) + if err != nil { + forkErr = fmt.Errorf("restore forked instance before source restore: %w", err) + if cleanupErr := m.cleanupForkInstanceOnError(ctx, forked.Id); cleanupErr != nil { + forkErr = fmt.Errorf("%v; additionally failed to cleanup forked instance %s: %v", forkErr, forked.Id, cleanupErr) + } + } else { + forked = restoredFork + } + } + log.InfoContext(ctx, "restoring source instance after running fork", "source_instance_id", id) _, restoreErr := m.restoreInstance(ctx, id) @@ -193,6 +210,13 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin return nil, err } + existsByMetadata, err := m.instanceNameExists(req.Name) + if err != nil { + return nil, fmt.Errorf("check instance name availability: %w", err) + } + if existsByMetadata { + return nil, fmt.Errorf("%w: instance name '%s' already exists", ErrAlreadyExists, req.Name) + } if stored.NetworkEnabled { exists, err := m.networkManager.NameExists(ctx, req.Name, "") if err != nil { @@ -325,6 +349,25 @@ func validateForkVolumeSafety(volumes []VolumeAttachment) error { return nil } +func (m *manager) instanceNameExists(name string) (bool, error) { + metaFiles, err := m.listMetadataFiles() + if err != nil { + return false, err + } + + for _, metaFile := range metaFiles { + id := filepath.Base(filepath.Dir(metaFile)) + meta, err := m.loadMetadata(id) + if err != nil { + continue + } + if meta.Name == name { + return true, nil + } + } + return false, nil +} + func resolveForkTargetState(requested State, sourceState State) (State, error) { if requested == "" { switch sourceState { diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index d5ea410c..3e036cbf 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -159,6 +159,52 @@ func TestForkInstance_CleansUpOnTargetTransitionError(t *testing.T) { assert.Equal(t, sourceID, entries[0].Name()) } +func TestForkInstanceRejectsDuplicateNameForNonNetworkedSource(t *testing.T) { + manager, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "fork-duplicate-name-source" + require.NoError(t, manager.ensureDirectories(sourceID)) + + now := time.Now() + sourceMeta := &metadata{StoredMetadata: StoredMetadata{ + Id: sourceID, + Name: sourceID, + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + StoppedAt: &now, + HypervisorType: hypervisor.TypeCloudHypervisor, + HypervisorVersion: "test", + SocketPath: paths.New(manager.paths.DataDir()).InstanceSocket(sourceID, "cloud-hypervisor.sock"), + DataDir: paths.New(manager.paths.DataDir()).InstanceDir(sourceID), + VsockCID: 42, + VsockSocket: paths.New(manager.paths.DataDir()).InstanceVsockSocket(sourceID), + }} + require.NoError(t, manager.saveMetadata(sourceMeta)) + + existingID := "fork-duplicate-name-existing" + require.NoError(t, manager.ensureDirectories(existingID)) + existingMeta := &metadata{StoredMetadata: StoredMetadata{ + Id: existingID, + Name: "duplicate-name", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + StoppedAt: &now, + HypervisorType: hypervisor.TypeCloudHypervisor, + HypervisorVersion: "test", + SocketPath: paths.New(manager.paths.DataDir()).InstanceSocket(existingID, "cloud-hypervisor.sock"), + DataDir: paths.New(manager.paths.DataDir()).InstanceDir(existingID), + VsockCID: 43, + VsockSocket: paths.New(manager.paths.DataDir()).InstanceVsockSocket(existingID), + }} + require.NoError(t, manager.saveMetadata(existingMeta)) + + _, err := manager.ForkInstance(ctx, sourceID, ForkInstanceRequest{Name: "duplicate-name"}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrAlreadyExists) + assert.Contains(t, err.Error(), "already exists") +} + func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { startedAt := time.Now().Add(-2 * time.Minute) stoppedAt := time.Now().Add(-1 * time.Minute)