gitea-adminandClaude Fable 5.1
83dc7b5ef9
The first ping after `herdr update --handoff` was sent exactly once. A server still binding its socket turned into "更新/热切换未完成" even though the update had succeeded. Retry connection errors for up to 20 seconds before giving up; a healthy server still answers immediately. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
180 lines
9.7 KiB
Python
180 lines
9.7 KiB
Python
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from herdr_hub.cli import Endpoint
|
|
from herdr_hub.control import ControlError, HerdrControl
|
|
from herdr_hub.restart import update_and_handoff
|
|
|
|
|
|
class HandoffTests(unittest.TestCase):
|
|
def setUp(self):
|
|
identity = patch("herdr_hub.restart.binary_identity", return_value=(1, 1, 1, 1))
|
|
identity.start()
|
|
self.addCleanup(identity.stop)
|
|
|
|
def test_failed_update_never_sends_handoff(self):
|
|
with patch("herdr_hub.restart.subprocess.check_output", return_value="status: running\nsocket: /tmp/fake.sock\n"), patch(
|
|
"herdr_hub.restart.request", return_value={"capabilities": {"live_handoff": True}}
|
|
) as call, patch("herdr_hub.restart.subprocess.run", side_effect=subprocess.CalledProcessError(1, "update")), contextlib.redirect_stdout(io.StringIO()):
|
|
with self.assertRaises(subprocess.CalledProcessError):
|
|
update_and_handoff("/fake/herdr", "named")
|
|
self.assertEqual([c.args[1] for c in call.call_args_list], ["ping"])
|
|
|
|
def test_native_082_update_uses_handoff_and_does_not_repeat_it(self):
|
|
old = {"version": "0.8.2", "capabilities": {"live_handoff": True}}
|
|
with patch("herdr_hub.restart.subprocess.check_output", return_value="status: running\nsocket: /tmp/fake.sock\n"), patch(
|
|
"herdr_hub.restart.request", side_effect=[old, {"version": "0.9.1"}]
|
|
) as call, patch("herdr_hub.restart.subprocess.run") as run, patch(
|
|
"herdr_hub.restart.installed_version", return_value="0.9.1"
|
|
), contextlib.redirect_stdout(io.StringIO()):
|
|
update_and_handoff("/fake/herdr", "named")
|
|
self.assertEqual(run.call_args.args[0], ["/fake/herdr", "--session", "named", "update", "--handoff"])
|
|
self.assertEqual([c.args[1] for c in call.call_args_list], ["ping", "ping"])
|
|
|
|
def test_native_handoff_failure_after_install_is_not_replayed(self):
|
|
old = {"version": "0.8.2", "capabilities": {"live_handoff": True}}
|
|
with patch("herdr_hub.restart.subprocess.check_output", return_value="status: running\nsocket: /tmp/fake.sock\n"), patch(
|
|
"herdr_hub.restart.request", side_effect=[old, old]
|
|
) as call, patch("herdr_hub.restart.subprocess.run"), patch(
|
|
"herdr_hub.restart.installed_version", return_value="0.9.1"
|
|
), patch("herdr_hub.restart.binary_identity", side_effect=[(1, 1, 1, 1), (1, 2, 1, 1)]), contextlib.redirect_stdout(io.StringIO()):
|
|
with self.assertRaisesRegex(RuntimeError, "不会重复发送热切换"):
|
|
update_and_handoff("/fake/herdr", "named")
|
|
self.assertEqual([c.args[1] for c in call.call_args_list], ["ping", "ping"])
|
|
|
|
def test_unverified_handoff_is_failure_not_success(self):
|
|
old = {"version": "0.9.0", "capabilities": {"live_handoff": True}}
|
|
with patch("herdr_hub.restart.subprocess.check_output", return_value="status: running\nsocket: /tmp/fake.sock\n"), patch(
|
|
"herdr_hub.restart.request", side_effect=[old, old, {}, old]
|
|
) as call, patch("herdr_hub.restart.subprocess.run"), patch(
|
|
"herdr_hub.restart.installed_version", return_value="0.9.1"
|
|
), patch("herdr_hub.restart.time.monotonic", side_effect=[0, 21]), contextlib.redirect_stdout(io.StringIO()):
|
|
with self.assertRaisesRegex(RuntimeError, "未确认新服务器就绪"):
|
|
update_and_handoff("/fake/herdr", "named")
|
|
self.assertEqual([c.args[1] for c in call.call_args_list].count("server.live_handoff"), 1)
|
|
|
|
def test_server_restarting_after_native_update_is_polled_not_failed(self):
|
|
old = {"version": "0.9.0", "capabilities": {"live_handoff": True}}
|
|
with patch("herdr_hub.restart.subprocess.check_output", return_value="status: running\nsocket: /tmp/fake.sock\n"), patch(
|
|
"herdr_hub.restart.request",
|
|
side_effect=[old, ConnectionRefusedError(), FileNotFoundError(), {"version": "0.9.1"}],
|
|
) as call, patch("herdr_hub.restart.subprocess.run"), patch(
|
|
"herdr_hub.restart.installed_version", return_value="0.9.1"
|
|
), patch("herdr_hub.restart.time.sleep"), patch(
|
|
"herdr_hub.restart.time.monotonic", side_effect=[0, 0.2]
|
|
), contextlib.redirect_stdout(io.StringIO()):
|
|
update_and_handoff("/fake/herdr", "named")
|
|
self.assertEqual([c.args[1] for c in call.call_args_list], ["ping"] * 4)
|
|
|
|
def test_server_unreachable_after_update_is_reported_without_restart(self):
|
|
old = {"version": "0.9.0", "capabilities": {"live_handoff": True}}
|
|
with patch("herdr_hub.restart.subprocess.check_output", return_value="status: running\nsocket: /tmp/fake.sock\n"), patch(
|
|
"herdr_hub.restart.request",
|
|
side_effect=[old, ConnectionRefusedError(), ConnectionRefusedError()],
|
|
) as call, patch("herdr_hub.restart.subprocess.run"), patch(
|
|
"herdr_hub.restart.installed_version", return_value="0.9.1"
|
|
), patch("herdr_hub.restart.time.sleep"), patch(
|
|
"herdr_hub.restart.time.monotonic", side_effect=[0, 25]
|
|
), contextlib.redirect_stdout(io.StringIO()):
|
|
with self.assertRaisesRegex(RuntimeError, "无法连接 Herdr 服务器"):
|
|
update_and_handoff("/fake/herdr", "named")
|
|
self.assertNotIn("server.live_handoff", [c.args[1] for c in call.call_args_list])
|
|
|
|
def test_real_helper_updates_old_server_even_when_binary_already_latest(self):
|
|
self._run_real_helper()
|
|
|
|
def test_real_helper_failure_does_not_stop_or_retry_server(self):
|
|
self._run_real_helper(fail_handoff=True)
|
|
|
|
def _run_real_helper(self, fail_handoff=False):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
socket_path = root / "api.sock"
|
|
record = root / "commands.jsonl"
|
|
binary = root / "herdr"
|
|
binary.write_text(
|
|
f"#!{sys.executable}\n"
|
|
"import json, os, sys\n"
|
|
f"with open({str(record)!r}, 'a') as f: f.write(json.dumps(sys.argv[1:])+'\\n')\n"
|
|
"if os.environ.get('HERDR_ENV') == '1': sys.exit(8)\n"
|
|
"if sys.argv[1:] == ['--version']: print('herdr 0.9.1')\n"
|
|
"elif sys.argv[1:] == ['--session', 'named', 'status', 'server']:\n"
|
|
f" print('status: running\\nsocket: {socket_path}')\n"
|
|
"elif sys.argv[1:] == ['--session', 'named', 'update', '--handoff']:\n"
|
|
" print('already up to date (0.9.1)', flush=True)\n"
|
|
"else: sys.exit(9)\n"
|
|
)
|
|
binary.chmod(0o755)
|
|
methods = []
|
|
done = threading.Event()
|
|
errors = []
|
|
server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
server.bind(str(socket_path))
|
|
server.listen()
|
|
server.settimeout(.1)
|
|
def serve():
|
|
version = "0.9.0"
|
|
try:
|
|
while not done.is_set():
|
|
try:
|
|
connection, _ = server.accept()
|
|
except socket.timeout:
|
|
continue
|
|
with connection, connection.makefile("rb") as stream:
|
|
message = json.loads(stream.readline())
|
|
methods.append(message)
|
|
response = {"id": message["id"]}
|
|
if message["method"] == "server.live_handoff":
|
|
if fail_handoff:
|
|
response["error"] = {"message": "handoff refused"}
|
|
else:
|
|
version = "0.9.1"
|
|
response["result"] = {}
|
|
elif message["method"] == "ping":
|
|
response["result"] = {"version": version, "capabilities": {"live_handoff": True}}
|
|
else:
|
|
raise AssertionError(message)
|
|
connection.sendall((json.dumps(response) + "\n").encode())
|
|
except Exception as error:
|
|
errors.append(error)
|
|
thread = threading.Thread(target=serve)
|
|
thread.start()
|
|
output = []
|
|
try:
|
|
with patch.dict(os.environ, {"HERDR_ENV": "1", "HERDR_SESSION": "wrong"}):
|
|
control = HerdrControl(Endpoint.local("named"), str(binary))
|
|
if fail_handoff:
|
|
with self.assertRaisesRegex(ControlError, "handoff refused"):
|
|
control.update_and_restart(output.append)
|
|
else:
|
|
result = control.update_and_restart(output.append)
|
|
self.assertIn("服务器已确认使用 0.9.1", result)
|
|
finally:
|
|
done.set()
|
|
thread.join(3)
|
|
server.close()
|
|
self.assertFalse(errors)
|
|
handoffs = [m for m in methods if m["method"] == "server.live_handoff"]
|
|
self.assertEqual(len(handoffs), 1)
|
|
self.assertEqual(handoffs[0]["params"]["import_exe"], str(binary.resolve()))
|
|
self.assertEqual(handoffs[0]["params"]["expected_version"], "0.9.1")
|
|
commands = [json.loads(line) for line in record.read_text().splitlines()]
|
|
self.assertEqual(commands.count(["--session", "named", "update", "--handoff"]), 1)
|
|
self.assertFalse(any("stop" in command for command in commands))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|