fix: complete reset recovery safety

This commit is contained in:
proxyctl coordinator
2026-07-17 11:25:09 +08:00
parent 9e2418f45e
commit 4cf9731db4
9 changed files with 339 additions and 52 deletions
+70 -32
View File
@@ -23,6 +23,7 @@ type recorder struct{ events []string }
type fakeRunner struct {
recorder *recorder
show string
active string
fail map[string]error
}
@@ -36,7 +37,7 @@ func (r *fakeRunner) Run(_ context.Context, name string, args ...string) ([]byte
return []byte(r.show), nil
}
if strings.Contains(command, " is-active ") {
return []byte("active\n"), nil
return []byte(r.active), nil
}
return nil, nil
}
@@ -73,7 +74,7 @@ func fixture(t *testing.T) (*dockerdaemon.Adapter, *recorder, string, string, *f
daemonPath := filepath.Join(directory, "etc", "docker", "daemon.json")
recoveryPath := filepath.Join(directory, "state", "docker-daemon.json")
record := &recorder{}
runner := &fakeRunner{recorder: record, show: "LoadState=loaded\nFragmentPath=/lib/systemd/system/docker.service\n", fail: map[string]error{}}
runner := &fakeRunner{recorder: record, show: "LoadState=loaded\nFragmentPath=/lib/systemd/system/docker.service\n", active: "active\n", fail: map[string]error{}}
adapter := dockerdaemon.New(dockerdaemon.Configuration{DaemonConfigPath: daemonPath, RecoveryPath: recoveryPath, TrustedRoot: directory}, recordingFS{recorder: record}, runner)
return adapter, record, daemonPath, recoveryPath, runner
}
@@ -429,9 +430,75 @@ func TestDockerDaemonDisableAndResetRejectDriftWithoutMutatingDaemonOrRecovery(t
}
}
func TestDockerDaemonResetCleansAlreadyRestoredRecoveryWithoutDaemonWriteOrSystemctl(t *testing.T) {
func TestDockerDaemonResetAlreadyRestoredReloadsRestartsAndInspectsBeforeMetadataCleanup(t *testing.T) {
a, record, daemonPath, recoveryPath, _ := fixture(t)
original := []byte("{\n \"log-level\": \"debug\",\n \"proxies\": {\"http-proxy\": \"http://prior.example:3128\"}\n}\n")
prepareAlreadyRestoredDaemon(t, a, daemonPath, original)
record.events = nil
reset := app.NewMaintenance(profile.State{Profiles: []profile.Profile{*activeProfile()}, ActiveProfile: activeProfile().Name}, []target.Target{a}, immediateLock{}, nil).Reset(context.Background(), true)
if reset.Err != nil || len(reset.Targets) != 1 || reset.Targets[0].Err != nil || reset.Targets[0].Inspection.State != target.External {
t.Fatalf("already-restored reset = %#v", reset)
}
assertSubsequence(t, record.events, []string{"systemctl daemon-reload", "systemctl restart docker.service", "systemctl is-active docker.service", "remove docker-daemon.json"})
if contains(record.events, "write daemon.json") {
t.Fatalf("already-restored reset rewrote daemon config: %#v", record.events)
}
if got, err := os.ReadFile(daemonPath); err != nil || !reflect.DeepEqual(got, original) {
t.Fatalf("already-restored daemon configuration changed to %q: %v", got, err)
}
if _, err := os.Stat(recoveryPath); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("recovery metadata remains: %v", err)
}
}
func TestDockerDaemonAlreadyRestoredPreservesMetadataAndReportsFreshDriftOnEveryServiceFailure(t *testing.T) {
failures := []struct {
name string
command string
active string
}{
{name: "daemon reload command", command: "systemctl daemon-reload"},
{name: "restart command", command: "systemctl restart docker.service"},
{name: "active inspection command", command: "systemctl is-active docker.service"},
{name: "inactive service state", active: "failed\n"},
}
for _, failure := range failures {
t.Run(failure.name, func(t *testing.T) {
a, record, daemonPath, recoveryPath, runner := fixture(t)
original := []byte("{\n \"log-level\": \"debug\"\n}\n")
prepareAlreadyRestoredDaemon(t, a, daemonPath, original)
metadataBefore, err := os.ReadFile(recoveryPath)
if err != nil {
t.Fatal(err)
}
if failure.command != "" {
runner.fail[failure.command] = errors.New("service failure")
}
if failure.active != "" {
runner.active = failure.active
}
record.events = nil
reset := app.NewMaintenance(profile.State{Profiles: []profile.Profile{*activeProfile()}, ActiveProfile: activeProfile().Name}, []target.Target{a}, immediateLock{}, nil).Reset(context.Background(), true)
if reset.Err == nil || len(reset.Targets) != 1 || reset.Targets[0].Inspection.State != target.Drifted {
t.Fatalf("failed already-restored reset = %#v", reset)
}
if contains(record.events, "write daemon.json") || contains(record.events, "remove docker-daemon.json") {
t.Fatalf("service failure mutated configuration or metadata: %#v", record.events)
}
if got, err := os.ReadFile(daemonPath); err != nil || !reflect.DeepEqual(got, original) {
t.Fatalf("daemon config changed to %q: %v", got, err)
}
if got, err := os.ReadFile(recoveryPath); err != nil || !reflect.DeepEqual(got, metadataBefore) {
t.Fatalf("metadata changed to %q: %v", got, err)
}
})
}
}
func prepareAlreadyRestoredDaemon(t *testing.T, a *dockerdaemon.Adapter, daemonPath string, original []byte) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(daemonPath), 0o700); err != nil {
t.Fatal(err)
}
@@ -444,35 +511,6 @@ func TestDockerDaemonResetCleansAlreadyRestoredRecoveryWithoutDaemonWriteOrSyste
if err := os.WriteFile(daemonPath, original, 0o640); err != nil {
t.Fatal(err)
}
record.events = nil
reset := app.NewMaintenance(profile.State{Profiles: []profile.Profile{*activeProfile()}, ActiveProfile: activeProfile().Name}, []target.Target{a}, immediateLock{}, nil).Reset(context.Background(), true)
if reset.Err != nil || len(reset.Targets) != 1 || reset.Targets[0].Err != nil || reset.Targets[0].Inspection.State != target.External {
t.Fatalf("already-restored reset = %#v", reset)
}
for _, event := range record.events {
if event == "write daemon.json" || strings.Contains(event, "daemon-reload") || strings.Contains(event, " restart ") {
t.Fatalf("already-restored reset changed daemon or systemd state: %#v", record.events)
}
}
if !contains(record.events, "remove docker-daemon.json") {
t.Fatalf("already-restored reset did not clean recovery metadata: %#v", record.events)
}
if got, err := os.ReadFile(daemonPath); err != nil || !reflect.DeepEqual(got, original) {
t.Fatalf("already-restored daemon configuration changed to %q: %v", got, err)
}
if info, err := os.Stat(daemonPath); err != nil || info.Mode().Perm() != 0o640 {
t.Fatalf("already-restored daemon metadata changed: info=%v err=%v", info, err)
}
if _, err := os.Stat(recoveryPath); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("recovery metadata remains: %v", err)
}
record.events = nil
retry := a.SetDesired(context.Background(), false, nil)
if retry.Err != nil || retry.Inspection.State != target.External || len(mutationEvents(record.events)) != 0 {
t.Fatalf("already-restored cleanup retry = %#v events=%#v", retry, record.events)
}
}
func TestDockerDaemonPreservesAndRestoresExistingConfiguration(t *testing.T) {