refactor(utils): migrate remaining bounded retry loops

This commit is contained in:
Tunglies
2026-10-01 14:34:16 +08:00
committed by GitHub
parent 9d1b400299
commit 5375ce360f
11 changed files with 478 additions and 304 deletions
+43 -35
View File
@@ -285,48 +285,56 @@ impl CoreManager {
anyhow::bail!("core startup blocked after mixed proxy port fallback failure: {reason}");
}
let mut retries = 0;
loop {
match self.start_core().await {
Ok(()) => {
crate::config::Config::notify_startup_mixed_port_fallback();
return Ok(!matches!(*self.get_running_mode(), RunningMode::NotRunning));
}
Err(start_error) if retries < MAX_PORT_FALLBACK_RETRIES => {
if !matches!(*self.get_running_mode(), RunningMode::NotRunning) {
use crate::utils::retry::{RetryError, RetryPolicy, retry};
retry(
RetryPolicy::fixed(
std::num::NonZeroUsize::MIN.saturating_add(MAX_PORT_FALLBACK_RETRIES),
std::time::Duration::ZERO,
),
|retries| async move {
match self.start_core().await {
Ok(()) => {
crate::config::Config::notify_startup_mixed_port_fallback();
return Err(start_error);
Ok(!matches!(*self.get_running_mode(), RunningMode::NotRunning))
}
match crate::config::Config::resolve_startup_mixed_port().await {
Ok(true) => {
retries += 1;
tracing::Span::current().record("retries", retries);
logging!(
warn,
Type::Core,
"Retrying core startup after mixed proxy port fallback ({}/{}): {start_error:#}",
retries,
MAX_PORT_FALLBACK_RETRIES
);
}
Ok(false) => {
Err(start_error) if retries < MAX_PORT_FALLBACK_RETRIES => {
if !matches!(*self.get_running_mode(), RunningMode::NotRunning) {
crate::config::Config::notify_startup_mixed_port_fallback();
return Err(start_error);
return Err(RetryError::Stop(start_error));
}
Err(fallback_error) => {
crate::config::Config::block_startup_core(&fallback_error);
return Err(start_error.context(format!(
"the mixed proxy port fallback did not rescue core startup: {fallback_error:#}"
)));
match crate::config::Config::resolve_startup_mixed_port().await {
Ok(true) => {
let retries = retries + 1;
tracing::Span::current().record("retries", retries);
logging!(
warn,
Type::Core,
"Retrying core startup after mixed proxy port fallback ({}/{}): {start_error:#}",
retries,
MAX_PORT_FALLBACK_RETRIES
);
Err(RetryError::Retry(start_error))
}
Ok(false) => {
crate::config::Config::notify_startup_mixed_port_fallback();
Err(RetryError::Stop(start_error))
}
Err(fallback_error) => {
crate::config::Config::block_startup_core(&fallback_error);
Err(RetryError::Stop(start_error.context(format!(
"the mixed proxy port fallback did not rescue core startup: {fallback_error:#}"
))))
}
}
}
Err(error) => {
crate::config::Config::notify_startup_mixed_port_fallback();
Err(RetryError::Stop(error))
}
}
Err(error) => {
crate::config::Config::notify_startup_mixed_port_fallback();
return Err(error);
}
}
}
},
)
.await
}
}
+29 -31
View File
@@ -1,6 +1,7 @@
#[cfg(test)]
use super::claim_core_readiness_generation;
use super::{CoreManager, PROFILE_SELECTIONS_PENDING_COMMIT, RunningMode};
use crate::utils::retry::{RetryError, RetryPolicy, retry};
use crate::{
AsyncHandler,
config::Config,
@@ -11,6 +12,7 @@ use crate::{
use anyhow::{Context as _, Result};
use clash_verge_logging::Type;
use log::Level;
use std::num::NonZeroUsize;
use std::path::Path;
use tauri_plugin_mihomo::MihomoExt as _;
use tauri_plugin_shell::ShellExt as _;
@@ -41,14 +43,14 @@ async fn retry_service_start<Start, StartFuture>(
mut start: Start,
) -> Result<()>
where
Start: FnMut() -> StartFuture,
StartFuture: std::future::Future<Output = Result<()>>,
Start: FnMut() -> StartFuture + Send,
StartFuture: std::future::Future<Output = Result<()>> + Send,
{
let mut last_error = None;
for attempt in 0..attempts {
match start().await {
Ok(()) => return Ok(()),
Err(error) => {
let attempts = NonZeroUsize::new(attempts).ok_or_else(|| anyhow::anyhow!("service start failed"))?;
retry(RetryPolicy::fixed(attempts, retry_delay), |attempt| {
let future = start();
async move {
future.await.map_err(|error| {
logging!(
warn,
Type::Core,
@@ -60,16 +62,14 @@ where
.downcast_ref::<service::ServiceStartRefusal>()
.is_some_and(|refusal| service::StageRequest::is_about_the_bundle(refusal.code))
{
return Err(error);
RetryError::Stop(error)
} else {
RetryError::Retry(error)
}
last_error = Some(error);
if attempt + 1 < attempts {
tokio::time::sleep(retry_delay).await;
}
}
})
}
}
Err(last_error.unwrap_or_else(|| anyhow::anyhow!("service start failed")))
})
.await
}
impl CoreManager {
@@ -95,14 +95,16 @@ async fn poll_sidecar_readiness<F, Fut>(
mut probe: F,
) -> Result<()>
where
F: FnMut() -> Fut,
Fut: std::future::Future<Output = Result<()>>,
F: FnMut() -> Fut + Send,
Fut: std::future::Future<Output = Result<()>> + Send,
{
let mut last_error = None;
for attempt in 0..max_attempts {
match probe().await {
Ok(()) => return Ok(()),
Err(error) => {
let attempts = NonZeroUsize::new(max_attempts)
.context("sidecar readiness was configured with no attempts")
.context("Mihomo API did not become ready")?;
retry(RetryPolicy::fixed(attempts, retry_delay), |attempt| {
let future = probe();
async move {
future.await.map_err(|error| {
logging!(
debug,
Type::Core,
@@ -110,16 +112,12 @@ where
attempt + 1,
max_attempts
);
last_error = Some(error);
}
RetryError::Retry(error)
})
}
if attempt + 1 < max_attempts {
tokio::time::sleep(retry_delay).await;
}
}
Err(last_error
.unwrap_or_else(|| anyhow::anyhow!("sidecar readiness was configured with no attempts"))
.context("Mihomo API did not become ready"))
})
.await
.context("Mihomo API did not become ready")
}
fn should_clear_terminated_sidecar(running_mode: &RunningMode, current_pid: Option<u32>, terminated_pid: u32) -> bool {
+25 -16
View File
@@ -522,25 +522,34 @@ pub async fn clear() -> Result<()> {
/// A failed system call is usually a transient RPC hiccup, so give it a few tries.
async fn clear_with_retry() -> Result<()> {
for attempt in 1..CLEAR_ATTEMPTS {
match clear_inner().await {
Err(error)
use crate::utils::retry::{RetryError, RetryPolicy, retry};
retry(
RetryPolicy::fixed(
std::num::NonZeroUsize::MIN.saturating_add((CLEAR_ATTEMPTS - 1) as usize),
CLEAR_RETRY_DELAY,
),
|attempt| async move {
clear_inner().await.map_err(|error| {
if matches!(
SysproxyFailure::from_chain(&error),
Some(SysproxyFailure::SystemCallFailed)
) =>
{
logging!(
warn,
Type::Core,
"clearing the system proxy failed (attempt {attempt}/{CLEAR_ATTEMPTS}); retrying: {error:#}"
);
tokio::time::sleep(CLEAR_RETRY_DELAY).await;
}
other => return other,
}
}
clear_inner().await
) {
if attempt + 1 < CLEAR_ATTEMPTS as usize {
logging!(
warn,
Type::Core,
"clearing the system proxy failed (attempt {}/{CLEAR_ATTEMPTS}); retrying: {error:#}",
attempt + 1
);
}
RetryError::Retry(error)
} else {
RetryError::Stop(error)
}
})
},
)
.await
}
#[tracing::instrument(skip_all, level = "info", fields(route = tracing::field::Empty))]
+13 -12
View File
@@ -130,20 +130,21 @@ impl<E: RunStateEnv> RunStateStore<E> {
/// Retries transport failures but stops at the first readable reply, including a rejection.
pub async fn await_ready(&self, attempts: usize, interval: Duration) -> Result<RunState, ReadyWaitError> {
let mut last_error = None;
for attempt in 0..attempts {
use crate::utils::retry::{RetryError, RetryPolicy, retry};
let attempts = std::num::NonZeroUsize::new(attempts).ok_or_else(|| {
ReadyWaitError::Unreachable(anyhow::anyhow!(
"service readiness wait was configured with no attempts"
))
})?;
retry(RetryPolicy::fixed(attempts, interval), |_| async {
match self.env.probe_service_version().await {
Ok(reply) => return self.record_reply(&reply).map_err(ReadyWaitError::Rejected),
Err(error) => last_error = Some(error),
Ok(reply) => self
.record_reply(&reply)
.map_err(|error| RetryError::Stop(ReadyWaitError::Rejected(error))),
Err(error) => Err(RetryError::Retry(ReadyWaitError::Unreachable(error))),
}
if attempt + 1 < attempts {
tokio::time::sleep(interval).await;
}
}
Err(ReadyWaitError::Unreachable(last_error.unwrap_or_else(|| {
anyhow::anyhow!("service readiness wait was configured with no attempts")
})))
})
.await
}
fn record_reply(&self, reply: &ServiceVersionReply) -> Result<RunState> {
+109 -81
View File
@@ -315,35 +315,40 @@ static SERVICE_CORE_STAGING_GENERATION: AtomicU64 = AtomicU64::new(0);
#[cfg(any(all(target_os = "macos", feature = "verge-dev"), test))]
fn create_service_core_staging_file(directory: &Path, core_name: &std::ffi::OsStr) -> Result<(PathBuf, std::fs::File)> {
for _ in 0..32 {
let generation = SERVICE_CORE_STAGING_GENERATION.fetch_add(1, Ordering::Relaxed);
let temporary_name = format!(
".{}.{}.{generation}.tmp",
core_name.to_string_lossy(),
std::process::id()
);
let temporary_path = directory.join(temporary_name);
match std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&temporary_path)
{
Ok(file) => return Ok((temporary_path, file)),
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {}
Err(error) => {
return Err(error).with_context(|| {
use crate::utils::retry::{RetryError, RetryPolicy, retry_sync};
retry_sync(
RetryPolicy::fixed(
std::num::NonZeroUsize::MIN.saturating_add(31),
std::time::Duration::ZERO,
),
|_| {
let generation = SERVICE_CORE_STAGING_GENERATION.fetch_add(1, Ordering::Relaxed);
let temporary_name = format!(
".{}.{}.{generation}.tmp",
core_name.to_string_lossy(),
std::process::id()
);
let temporary_path = directory.join(temporary_name);
match std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&temporary_path)
{
Ok(file) => Ok((temporary_path, file)),
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
Err(RetryError::Retry(anyhow::anyhow!(
"failed to create a unique temporary development Service core in {}",
directory.display()
)))
}
Err(error) => Err(RetryError::Stop(anyhow::Error::from(error).context({
format!(
"failed to create temporary development Service core {}",
temporary_path.display()
)
});
}))),
}
}
}
bail!(
"failed to create a unique temporary development Service core in {}",
directory.display()
},
)
}
@@ -1014,38 +1019,47 @@ pub(crate) fn request_runtime_provider_sync(delay: Duration) {
}
AsyncHandler::spawn(move || async move {
tokio::time::sleep(delay).await;
for attempt in 1..=RUNTIME_PROVIDER_SYNC_ATTEMPTS {
let outcome = {
let _serial = PROVIDER_SYNC_SERIAL.lock().await;
if attempt == 1 {
PROVIDER_SYNC_QUEUED.store(false, Ordering::Release);
use crate::utils::retry::{RetryError, RetryPolicy, retry};
let _ = retry(
RetryPolicy::fixed(
std::num::NonZeroUsize::MIN.saturating_add((RUNTIME_PROVIDER_SYNC_ATTEMPTS - 1) as usize),
constants::timing::RUNTIME_PROVIDER_SYNC_RETRY_DELAY,
),
|index| async move {
let attempt = index as u32 + 1;
let outcome = {
let _serial = PROVIDER_SYNC_SERIAL.lock().await;
if attempt == 1 {
PROVIDER_SYNC_QUEUED.store(false, Ordering::Release);
}
sync_runtime_providers_by_service().await
};
match outcome {
Ok(ProviderSync { pending: 0, .. }) => Ok(()),
Ok(ProviderSync { pending, .. }) if attempt < RUNTIME_PROVIDER_SYNC_ATTEMPTS => {
logging!(
info,
Type::Service,
"{pending} provider caches are not ready yet; retrying"
);
Err(RetryError::Retry(()))
}
Ok(ProviderSync { pending, .. }) => {
logging!(warn, Type::Service, "{pending} provider caches were not synced");
Ok(())
}
Err(error) => {
logging!(
warn,
Type::Service,
"failed to sync provider caches from the service: {error:#}"
);
Ok(())
}
}
sync_runtime_providers_by_service().await
};
match outcome {
Ok(ProviderSync { pending: 0, .. }) => return,
Ok(ProviderSync { pending, .. }) if attempt < RUNTIME_PROVIDER_SYNC_ATTEMPTS => {
logging!(
info,
Type::Service,
"{pending} provider caches are not ready yet; retrying"
);
}
Ok(ProviderSync { pending, .. }) => {
logging!(warn, Type::Service, "{pending} provider caches were not synced");
return;
}
Err(error) => {
logging!(
warn,
Type::Service,
"failed to sync provider caches from the service: {error:#}"
);
return;
}
}
tokio::time::sleep(constants::timing::RUNTIME_PROVIDER_SYNC_RETRY_DELAY).await;
}
},
)
.await;
});
}
@@ -1353,21 +1367,29 @@ async fn create_sync_temp(target: &Path) -> Result<(PathBuf, tokio::fs::File)> {
.file_name()
.map(|name| name.to_string_lossy())
.unwrap_or_default();
for _ in 0..8 {
let sequence = SYNC_TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed);
let temp = target.with_file_name(format!(".{name}.sync-{}-{sequence}.tmp", std::process::id()));
match tokio::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&temp)
.await
{
Ok(file) => return Ok((temp, file)),
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {}
Err(error) => return Err(error).with_context(|| format!("failed to create {}", temp.display())),
}
}
bail!("no free temporary name beside {}", target.display())
use crate::utils::retry::{RetryError, RetryPolicy, retry};
retry(
RetryPolicy::fixed(std::num::NonZeroUsize::MIN.saturating_add(7), std::time::Duration::ZERO),
|_| async {
let sequence = SYNC_TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed);
let temp = target.with_file_name(format!(".{name}.sync-{}-{sequence}.tmp", std::process::id()));
match tokio::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&temp)
.await
{
Ok(file) => Ok((temp, file)),
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => Err(RetryError::Retry(
anyhow::anyhow!("no free temporary name beside {}", target.display()),
)),
Err(error) => Err(RetryError::Stop(
anyhow::anyhow!(error).context(format!("failed to create {}", temp.display())),
)),
}
},
)
.await
}
async fn same_contents(temp: &Path, target: &Path) -> bool {
@@ -1734,22 +1756,28 @@ async fn recover_after_owner_loss_while_locked(reason: OwnerRecoveryReason) {
}
async fn clear_proxy_after_owner_loss() {
let mut last_error = None;
for attempt in 1..=3 {
match proxy_control::clear().await {
Ok(()) => return,
Err(error) => {
use crate::utils::retry::{RetryError, RetryPolicy, retry};
let result = retry(
RetryPolicy::fixed(
std::num::NonZeroUsize::MIN.saturating_add(2),
Duration::from_millis(100),
),
|index| async move {
let attempt = index + 1;
proxy_control::clear().await.map_err(|error| {
logging!(
warn,
Type::Service,
"proxy clear attempt {attempt}/3 after owner loss failed: {error:#}"
);
last_error = Some(error);
tokio::time::sleep(Duration::from_millis(100)).await;
}
}
}
if let Some(error) = last_error {
RetryError::Retry(error)
})
},
)
.await;
if let Err(error) = result {
// The existing recovery path also waits after its final failed clear.
tokio::time::sleep(Duration::from_millis(100)).await;
logging!(
error,
Type::Service,
+8 -6
View File
@@ -388,12 +388,14 @@ impl SilentUpdater {
// The new webview may not accept evaluation immediately.
std::thread::spawn(move || {
for i in 0..10 {
std::thread::sleep(std::time::Duration::from_millis(100 * (i + 1)));
if window.eval(&js).is_ok() {
return;
}
}
use crate::utils::retry::{RetryError, RetryPolicy, retry_sync};
let _ = retry_sync(
RetryPolicy::fixed(std::num::NonZeroUsize::MIN.saturating_add(9), std::time::Duration::ZERO),
|index| {
std::thread::sleep(std::time::Duration::from_millis(100 * (index as u64 + 1)));
window.eval(&js).map_err(RetryError::Retry)
},
);
});
logging!(info, Type::System, "Update splash window shown");