fix: harden operations and configuration recovery

This commit is contained in:
proxyctl coordinator
2026-08-30 11:13:45 +08:00
parent e34bf7d989
commit 8928986830
27 changed files with 542 additions and 125 deletions
+76 -51
View File
@@ -2,9 +2,10 @@ package app
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"strings"
"sync"
"time"
@@ -28,11 +29,21 @@ const (
)
type Intent struct {
Kind IntentKind
Name string
Profile profile.Profile
Confirm bool
TargetID string
Kind IntentKind
Name string
Profile profile.Profile
Confirm bool
ConfirmationToken string
TargetID string
}
func confirmationToken(parts ...string) string {
hash := sha256.New()
for _, part := range parts {
_, _ = hash.Write([]byte(part))
_, _ = hash.Write([]byte{0})
}
return hex.EncodeToString(hash.Sum(nil))
}
// TargetActionKind identifies a Target action independently of display labels.
@@ -299,7 +310,7 @@ type targetDetectionUpdate struct {
complete bool
}
func (a *Application) beginTargetDetection() (Snapshot, <-chan targetDetectionUpdate) {
func (a *Application) beginTargetDetection(ctx context.Context) (Snapshot, <-chan targetDetectionUpdate) {
a.stateMu.Lock()
a.detectionGeneration++
generation := a.detectionGeneration
@@ -316,7 +327,7 @@ func (a *Application) beginTargetDetection() (Snapshot, <-chan targetDetectionUp
for i, adapter := range a.targets {
adapter, id := adapter, a.targetIDs[i]
go func() {
outcomes <- targetDetectionOutcome{targetID: id, detection: detectTarget(adapter, active)}
outcomes <- targetDetectionOutcome{targetID: id, detection: detectTarget(ctx, adapter, active)}
}()
}
go func() {
@@ -351,28 +362,30 @@ func (a *Application) beginTargetDetection() (Snapshot, <-chan targetDetectionUp
return checking, updates
}
func detectTarget(adapter target.Target, active *profile.Profile) target.Detection {
func detectTarget(ctx context.Context, adapter target.Target, active *profile.Profile) target.Detection {
if detector, ok := adapter.(target.Detector); ok {
return detector.Detect(context.Background(), active)
return detector.Detect(ctx, active)
}
return target.Detection{
Availability: adapter.Availability(context.Background()),
Inspection: adapter.Inspect(context.Background(), active),
Availability: adapter.Availability(ctx),
Inspection: adapter.Inspect(ctx, active),
}
}
func (a *Application) Dispatch(intent Intent) Result {
return a.dispatch(intent, nil)
ctx, cancel := context.WithTimeout(context.Background(), operationTimeout)
defer cancel()
return a.dispatch(ctx, intent, nil)
}
func (a *Application) dispatch(intent Intent, publish func(Result)) Result {
func (a *Application) dispatch(ctx context.Context, intent Intent, publish func(Result)) Result {
// Serialize local mutation workflows even when tests intentionally omit the
// cross-process MutationLock.
a.dispatchMu.Lock()
defer a.dispatchMu.Unlock()
if a.mutationLock != nil {
unlock, err := a.mutationLock.Lock(context.Background())
unlock, err := a.mutationLock.Lock(ctx)
if err != nil {
return Result{Snapshot: a.Snapshot(), Err: fmt.Errorf("lock proxyctl mutation: %w", err)}
}
@@ -388,16 +401,16 @@ func (a *Application) dispatch(intent Intent, publish func(Result)) Result {
}
a.replaceState(fresh)
if intent.Kind == ActivateProfile {
a.refreshInspections()
a.refreshInspections(ctx)
}
if isTargetMutationIntent(intent.Kind) {
return a.setTargetDesired(intent, publish)
return a.setTargetDesired(ctx, intent, publish)
}
if intent.Kind == RefreshTargets {
return a.refreshTargets(publish)
return a.refreshTargets(ctx, publish)
}
if intent.Kind == ActivateProfile {
return a.activateProfile(intent, publish)
return a.activateProfile(ctx, intent, publish)
}
var notice string
next, err := a.store.Mutate(func(current profile.State) (profile.State, error) {
@@ -416,7 +429,7 @@ func (a *Application) dispatch(intent Intent, publish func(Result)) Result {
})
if err == nil {
a.replaceState(next)
a.refreshInspections()
a.refreshInspections(ctx)
} else {
var committed *profile.CommitError
if errors.As(err, &committed) {
@@ -434,7 +447,7 @@ func (a *Application) dispatch(intent Intent, publish func(Result)) Result {
return Result{Snapshot: a.Snapshot(), Notice: notice, Err: err}
}
func (a *Application) activateProfile(intent Intent, publish func(Result)) Result {
func (a *Application) activateProfile(ctx context.Context, intent Intent, publish func(Result)) Result {
state := a.stateCopy()
if index(state, intent.Name) < 0 {
return Result{Snapshot: a.Snapshot(), Err: fmt.Errorf("Profile %q does not exist", intent.Name)}
@@ -446,23 +459,28 @@ func (a *Application) activateProfile(intent Intent, publish func(Result)) Resul
inspections := a.inspectionsCopy()
var enabled []int
var warnings []string
confirmationScope := []string{"activate-profile", intent.Name}
for i, adapter := range a.targets {
id := a.targetIDs[i]
if inspections[id].State != target.Enabled {
continue
}
enabled = append(enabled, i)
confirmationScope = append(confirmationScope, id)
if warning := adapter.Confirmation(true); warning != "" {
warnings = append(warnings, a.targetDescriptions[i].Name+": "+warning)
detail := a.targetDescriptions[i].Name + ": " + warning
warnings = append(warnings, detail)
confirmationScope = append(confirmationScope, detail)
} else {
confirmationScope = append(confirmationScope, "")
}
}
if len(warnings) > 0 && !intent.Confirm {
message := "Changing the Active Profile requires confirmation:\n" + strings.Join(warnings, "\n")
token := confirmationToken(confirmationScope...)
if len(warnings) > 0 && (!intent.Confirm || intent.ConfirmationToken != token) {
confirmation := &Confirmation{
Title: "Confirm Active Profile change", Summary: "Changing the Active Profile has high-impact Target effects.",
Details: warnings, ConfirmLabel: "Change Profile", CancelLabel: "Cancel", HighImpact: true,
Details: warnings, ConfirmLabel: "Change Profile", CancelLabel: "Cancel", HighImpact: true, Token: token,
}
_ = message
return Result{Snapshot: a.Snapshot(), ConfirmationRequest: confirmation,
Status: OperationStatus{Summary: confirmation.Title}}
}
@@ -489,7 +507,7 @@ func (a *Application) activateProfile(intent Intent, publish func(Result)) Resul
// without the state lock, then publish all observations atomically.
observed := make(map[string]target.Inspection, len(a.targets))
for i, adapter := range a.targets {
observed[a.targetIDs[i]] = adapter.Inspect(context.Background(), active)
observed[a.targetIDs[i]] = adapter.Inspect(ctx, active)
}
a.replaceInspections(observed)
@@ -505,16 +523,25 @@ func (a *Application) activateProfile(intent Intent, publish func(Result)) Resul
publish(Result{Snapshot: a.Snapshot(), Progress: []TargetProgress{{TargetID: id, Phase: TargetProgressRunning, State: a.inspection(id).State}}, Notice: "Updating " + name + " Target",
Status: OperationStatus{Summary: "Updating " + name + " Target", TargetID: id}})
}
operation := adapter.SetDesired(context.Background(), true, active)
operation := adapter.SetDesired(ctx, true, active)
inspection := operation.Inspection
var safeErr error
if operation.Err != nil {
// A failure-returned inspection may predate a partial write. Always
// inspect explicitly against the new Active Profile before publishing.
inspection = adapter.Inspect(context.Background(), active)
inspection = adapter.Inspect(ctx, active)
safeErr = fmt.Errorf("%s (fresh inspection: %s — %s)", profile.RedactText(operation.Err.Error()), inspection.State, profile.RedactText(inspection.Evidence))
}
a.replaceInspection(id, inspection)
if a.logger != nil {
outcome, detail := "success", inspection.Evidence
if safeErr != nil {
outcome, detail = "failure", safeErr.Error()
}
if logErr := a.logger.Write(LogEntry{Time: time.Now(), Operation: "switch-Active-Profile", Target: id, Result: outcome, Detail: RedactLogDetail(detail)}); logErr != nil {
safeErr = errors.Join(safeErr, fmt.Errorf("write %s Target operation log: %w", name, logErr))
}
}
progressPhase := TargetProgressSucceeded
if safeErr != nil {
progressPhase = TargetProgressFailed
@@ -528,13 +555,6 @@ func (a *Application) activateProfile(intent Intent, publish func(Result)) Resul
}
publish(Result{Snapshot: a.Snapshot(), Progress: []TargetProgress{item}, Notice: name + " Target update completed", Status: status, Err: safeErr})
}
if a.logger != nil {
outcome, detail := "success", inspection.Evidence
if safeErr != nil {
outcome, detail = "failure", safeErr.Error()
}
_ = a.logger.Write(LogEntry{Time: time.Now(), Operation: "switch-Active-Profile", Target: id, Result: outcome, Detail: RedactLogDetail(detail)})
}
if safeErr != nil {
failures = append(failures, fmt.Errorf("%s Target: %w", name, safeErr))
}
@@ -545,7 +565,7 @@ func (a *Application) activateProfile(intent Intent, publish func(Result)) Resul
for i, adapter := range a.targets {
id := a.targetIDs[i]
if !updated[id] {
remaining[id] = adapter.Inspect(context.Background(), active)
remaining[id] = adapter.Inspect(ctx, active)
}
}
a.mergeInspections(remaining)
@@ -670,8 +690,8 @@ func targetByID(snapshot Snapshot, id string) (TargetSnapshot, string) {
return TargetSnapshot{ID: id}, id
}
func (a *Application) refreshTargets(publish func(Result)) Result {
checking, updates := a.beginTargetDetection()
func (a *Application) refreshTargets(ctx context.Context, publish func(Result)) Result {
checking, updates := a.beginTargetDetection(ctx)
if publish != nil {
progress := make([]TargetProgress, 0, len(checking.Targets))
for _, item := range checking.Targets {
@@ -714,11 +734,11 @@ func (a *Application) refreshTargets(publish func(Result)) Result {
return result
}
func (a *Application) refreshInspections() {
func (a *Application) refreshInspections(ctx context.Context) {
active := a.activeProfile()
inspections := make(map[string]target.Inspection, len(a.targets))
for i, adapter := range a.targets {
inspections[a.targetIDs[i]] = adapter.Inspect(context.Background(), active)
inspections[a.targetIDs[i]] = adapter.Inspect(ctx, active)
}
a.replaceInspections(inspections)
}
@@ -769,7 +789,7 @@ func validateTargetAction(intent Intent, selected TargetSnapshot) (targetActionP
return policy, nil
}
func (a *Application) setTargetDesired(intent Intent, publish func(Result)) Result {
func (a *Application) setTargetDesired(ctx context.Context, intent Intent, publish func(Result)) Result {
selectedIndex := -1
for i, id := range a.targetIDs {
if id == intent.TargetID {
@@ -785,27 +805,30 @@ func (a *Application) setTargetDesired(intent Intent, publish func(Result)) Resu
// This targeted check runs inside dispatch serialization and immediately
// before action validation. Adapter calls happen without stateMu; the full
// outcome is then published under one lock before policy is derived.
detection := detectTarget(selected, active)
detection := detectTarget(ctx, selected, active)
snapshot := a.publishTargetDetection(intent.TargetID, detection)
selectedSnapshot := snapshot.Targets[selectedIndex]
policy, err := validateTargetAction(intent, selectedSnapshot)
if err != nil {
return Result{Snapshot: snapshot, Err: err}
}
if warning := selected.Confirmation(policy.desired); warning != "" && !intent.Confirm {
confirmation := &Confirmation{Title: "Confirm " + a.targetDescriptions[selectedIndex].Name + " Target change", Summary: warning,
ConfirmLabel: "Continue", CancelLabel: "Cancel", HighImpact: true}
return Result{Snapshot: a.Snapshot(), ConfirmationRequest: confirmation,
Status: OperationStatus{Summary: confirmation.Title, TargetID: intent.TargetID}}
if warning := selected.Confirmation(policy.desired); warning != "" {
token := confirmationToken("target", intent.TargetID, fmt.Sprint(intent.Kind), fmt.Sprint(selectedSnapshot.State), fmt.Sprint(policy.desired), warning)
if !intent.Confirm || intent.ConfirmationToken != token {
confirmation := &Confirmation{Title: "Confirm " + a.targetDescriptions[selectedIndex].Name + " Target change", Summary: warning,
ConfirmLabel: "Continue", CancelLabel: "Cancel", HighImpact: true, Token: token}
return Result{Snapshot: a.Snapshot(), ConfirmationRequest: confirmation,
Status: OperationStatus{Summary: confirmation.Title, TargetID: intent.TargetID}}
}
}
if publish != nil {
publish(Result{Snapshot: snapshot, Progress: []TargetProgress{{TargetID: intent.TargetID, Phase: TargetProgressRunning, State: selectedSnapshot.State}},
Status: OperationStatus{Summary: "Updating " + a.targetDescriptions[selectedIndex].Name + " Target", TargetID: intent.TargetID}})
}
result := selected.SetDesired(context.Background(), policy.desired, active)
result := selected.SetDesired(ctx, policy.desired, active)
inspection := result.Inspection
if result.Err != nil {
inspection = selected.Inspect(context.Background(), active)
inspection = selected.Inspect(ctx, active)
}
a.replaceInspection(intent.TargetID, inspection)
outcome, detail := "success", inspection.Evidence
@@ -813,7 +836,9 @@ func (a *Application) setTargetDesired(intent Intent, publish func(Result)) Resu
outcome, detail = "failure", result.Err.Error()
}
if a.logger != nil {
_ = a.logger.Write(LogEntry{Time: time.Now(), Operation: policy.operation, Target: intent.TargetID, Result: outcome, Detail: RedactLogDetail(detail)})
if logErr := a.logger.Write(LogEntry{Time: time.Now(), Operation: policy.operation, Target: intent.TargetID, Result: outcome, Detail: RedactLogDetail(detail)}); logErr != nil {
result.Err = errors.Join(result.Err, fmt.Errorf("write Target operation log: %w", logErr))
}
}
notice := ""
progressPhase := TargetProgressSucceeded