fix(service): explain sidecar failures and clear recovered notices

This commit is contained in:
wonfen
2026-10-03 10:21:54 +08:00
parent 8c1fb3ac56
commit f958dccb25
24 changed files with 284 additions and 84 deletions
+15 -11
View File
@@ -34,7 +34,10 @@ async fn execute_service_operation_sync(status: ServiceStatus, error_code: &str)
SERVICE_MANAGER.handle_service_status(status).await
};
#[cfg(windows)]
return finish_service_installation(result, error_code, || manager.continue_with_sidecar()).await;
return finish_service_installation(result, error_code, |reason| async move {
manager.continue_with_sidecar(Some(&reason)).await
})
.await;
#[cfg(not(windows))]
result
@@ -49,7 +52,7 @@ async fn finish_service_installation<Fallback, FallbackFuture>(
fallback: Fallback,
) -> CmdResult<ServiceInstallOutcome>
where
Fallback: FnOnce() -> FallbackFuture,
Fallback: FnOnce(String) -> FallbackFuture,
FallbackFuture: std::future::Future<Output = anyhow::Result<()>>,
{
let Err(error) = installation else {
@@ -72,12 +75,9 @@ where
return Err(proxy_aware_coded_error(&error, error_code));
}
let reason = reasons.join("\n");
fallback().await.map_err(|error| {
proxy_aware_coded_error(
&error.context(format!("Service core rejected: {reason}; Sidecar fallback failed")),
"SERVICE_SIDECAR_FAILED",
)
})?;
fallback(reason.clone())
.await
.map_err(|error| proxy_aware_coded_error(&error, "SERVICE_SIDECAR_FAILED"))?;
Ok(ServiceInstallOutcome::Sidecar { reason })
}
@@ -107,11 +107,16 @@ pub async fn repair_service() -> CmdResult<ServiceInstallOutcome> {
#[tauri::command]
pub async fn continue_with_sidecar() -> CmdResult {
crate::core::CoreManager::global()
.continue_with_sidecar()
.continue_with_sidecar(None)
.await
.map_err(|error| proxy_aware_coded_error(&error, "SERVICE_SIDECAR_FAILED"))
}
#[tauri::command]
pub fn get_sidecar_failure() -> crate::core::notification::SidecarFailureSnapshot {
crate::core::notification::sidecar_failure_snapshot()
}
#[tauri::command]
pub fn take_service_fallback_notice() -> Option<crate::core::service::ServiceFallbackNotice> {
crate::core::service::take_service_fallback_notice()
@@ -175,7 +180,7 @@ mod tests {
None => anyhow::anyhow!("{reason}"),
}
.context("install service failed");
let result = finish_service_installation(Err(error), "SERVICE_INSTALL_FAILED", || async {
let result = finish_service_installation(Err(error), "SERVICE_INSTALL_FAILED", |_| async {
fallback_called.set(true);
anyhow::ensure!(fallback_ready, "another core is running");
Ok(())
@@ -201,7 +206,6 @@ mod tests {
})
);
if rejected {
assert!(error.detail.contains(reason));
assert!(error.detail.contains("another core is running"));
}
}
+74 -53
View File
@@ -334,6 +334,18 @@ enum HandoffOutcome {
Failed,
}
fn report_sidecar_failure(error: anyhow::Error, service_rejection: Option<&str>) -> anyhow::Error {
let error = match service_rejection {
Some(reason) => error.context(format!("Service core rejected: {reason}; Sidecar fallback failed")),
None => error,
};
// Proxy failures have their own pending state and recovery dialog.
if SysproxyFailure::from_chain(&error).is_none() {
crate::core::notification::record_sidecar_failure(format!("{error:#}").into());
}
error
}
impl CoreManager {
async fn rollback_failed_start(&self) {
proxy_control::stop_guard().await;
@@ -364,9 +376,12 @@ impl CoreManager {
}
#[tracing::instrument(skip_all, level = "info", fields(status = tracing::field::Empty, tun_disabled = false, readiness_generation = tracing::field::Empty))]
pub async fn continue_with_sidecar(&self) -> Result<()> {
pub async fn continue_with_sidecar(&self, service_rejection: Option<&str>) -> Result<()> {
if !self.try_start_config_update() {
anyhow::bail!("configuration update is already running");
return Err(report_sidecar_failure(
anyhow::anyhow!("configuration update is already running"),
service_rejection,
));
}
defer! {
self.finish_config_update();
@@ -375,61 +390,66 @@ impl CoreManager {
// draft without claiming it, so without this it can commit another transaction's staged patch.
let config_write = Config::lock_config_write().await;
let _life = self.lifecycle_lock.lock().await;
let status = SERVICE_MANAGER.current().await;
tracing::Span::current().record("status", tracing::field::debug(&status));
let mode = self.get_running_mode();
if !can_allow_sidecar_for_session(&mode, &status) {
anyhow::bail!("Sidecar continuation is not allowed from {mode:?} / {status:?}");
}
#[cfg(target_os = "windows")]
if matches!(*mode, RunningMode::NotRunning) {
clash_verge_service_ipc::execution::check_sidecar_available().await?;
}
if !matches!(status, ServiceStatus::SidecarAllowed) {
SERVICE_MANAGER.allow_sidecar_for_session()?;
}
// Settling on Sidecar is what makes the verdict final, so ask only once it is recorded.
// Elevation alone carries TUN on Sidecar; a Sidecar that cannot must write it off.
let prepared = async {
let tun_enabled = Config::verge().await.latest_arc().enable_tun_mode.unwrap_or(false);
if crate::core::runstate::RUN_STATE
.state()
.tun_should_be_disabled(tun_enabled)
{
tracing::Span::current().record("tun_disabled", true);
Config::disable_tun_and_persist().await?;
}
Config::generate().await
}
.await;
drop(config_write);
if let Err(error) = prepared {
SERVICE_MANAGER.withdraw_sidecar_allowance();
return Err(error);
}
// Drop the guard last so an earlier failure cannot leave a running Sidecar unguarded.
proxy_control::stop_guard().await;
// Record failures before releasing the lifecycle lock, including preparation and rollback errors.
let result = async {
self.start_core_inner().await?;
if !matches!(*self.get_running_mode(), RunningMode::Sidecar)
|| self.current_core_readiness_generation().is_none()
{
anyhow::bail!("Sidecar did not become ready");
let status = SERVICE_MANAGER.current().await;
tracing::Span::current().record("status", tracing::field::debug(&status));
let mode = self.get_running_mode();
if !can_allow_sidecar_for_session(&mode, &status) {
anyhow::bail!("Sidecar continuation is not allowed from {mode:?} / {status:?}");
}
tracing::Span::current().record("readiness_generation", self.current_core_readiness_generation());
self.apply_proxy_after_start().await
#[cfg(target_os = "windows")]
if matches!(*mode, RunningMode::NotRunning) {
clash_verge_service_ipc::execution::check_sidecar_available().await?;
}
if !matches!(status, ServiceStatus::SidecarAllowed) {
SERVICE_MANAGER.allow_sidecar_for_session()?;
}
// Settling on Sidecar is what makes the verdict final, so ask only once it is recorded.
// Elevation alone carries TUN on Sidecar; a Sidecar that cannot must write it off.
let prepared = async {
let tun_enabled = Config::verge().await.latest_arc().enable_tun_mode.unwrap_or(false);
if crate::core::runstate::RUN_STATE
.state()
.tun_should_be_disabled(tun_enabled)
{
tracing::Span::current().record("tun_disabled", true);
Config::disable_tun_and_persist().await?;
}
Config::generate().await
}
.await;
drop(config_write);
if let Err(error) = prepared {
SERVICE_MANAGER.withdraw_sidecar_allowance();
return Err(error);
}
// Drop the guard last so an earlier failure cannot leave a running Sidecar unguarded.
proxy_control::stop_guard().await;
let result = async {
self.start_core_inner().await?;
if !matches!(*self.get_running_mode(), RunningMode::Sidecar)
|| self.current_core_readiness_generation().is_none()
{
anyhow::bail!("Sidecar did not become ready");
}
tracing::Span::current().record("readiness_generation", self.current_core_readiness_generation());
self.apply_proxy_after_start().await
}
.await;
if let Err(error) = result {
// Revoke only the failed Sidecar allowance; its startup says nothing about Service health.
SERVICE_MANAGER.withdraw_sidecar_allowance();
if let Err(cleanup_error) = self.rollback_failed_sidecar_transition().await {
return Err(proxy_control::rollback_failure(error, cleanup_error)
.context("failed to clear the proxy before rolling back"));
}
return Err(error);
}
Ok(())
}
.await;
if let Err(error) = result {
// Revoke only the failed Sidecar allowance; its startup says nothing about Service health.
SERVICE_MANAGER.withdraw_sidecar_allowance();
if let Err(cleanup_error) = self.rollback_failed_sidecar_transition().await {
return Err(proxy_control::rollback_failure(error, cleanup_error)
.context("failed to clear the proxy before rolling back"));
}
return Err(error);
}
Ok(())
result.map_err(|error| report_sidecar_failure(error, service_rejection))
}
#[tracing::instrument(skip_all, level = "info", fields(readiness_generation = tracing::field::Empty))]
@@ -572,6 +592,7 @@ impl CoreManager {
}
// tell the window when a background apply lands
Handle::refresh_verge();
crate::core::notification::retire_sidecar_failure();
Ok(())
}
+40
View File
@@ -231,6 +231,46 @@ pub fn what_was_asked() -> FailedOperation {
static PENDING_FAILURES: Lazy<FailureTable> = Lazy::new(FailureTable::default);
/// A removal has a revision too, so a delayed read cannot restore a resolved failure.
#[derive(Debug, Default, Clone, serde::Serialize)]
pub struct SidecarFailureSnapshot {
pub revision: u64,
pub detail: Option<String>,
}
impl SidecarFailureSnapshot {
fn record(&mut self, detail: String) {
self.revision += 1;
self.detail = Some(detail);
}
fn recover(&mut self) -> bool {
if self.detail.take().is_none() {
return false;
}
self.revision += 1;
true
}
}
static SIDECAR_FAILURE: Lazy<Mutex<SidecarFailureSnapshot>> = Lazy::new(Mutex::default);
pub fn sidecar_failure_snapshot() -> SidecarFailureSnapshot {
SIDECAR_FAILURE.lock().clone()
}
pub fn record_sidecar_failure(detail: String) {
SIDECAR_FAILURE.lock().record(detail);
notify_pending_failures_changed();
}
pub fn retire_sidecar_failure() {
let changed = SIDECAR_FAILURE.lock().recover();
if changed {
notify_pending_failures_changed();
}
}
pub fn record_failure(operation: FailedOperation, code: &str, detail: impl Into<String>) {
PENDING_FAILURES.record(operation, code, detail.into());
notify_pending_failures_changed();
+1
View File
@@ -143,6 +143,7 @@ mod app_init {
cmd::upgrade_clash_core,
cmd::get_runtime_state,
cmd::get_pending_failures,
cmd::get_sidecar_failure,
cmd::entry_lightweight_mode,
cmd::install_service,
cmd::uninstall_service,