From 3f11734ce4097626a6ad7306f8fab76dbbc8fd8c Mon Sep 17 00:00:00 2001 From: Donald Date: Sat, 3 Oct 2026 03:00:09 +0000 Subject: [PATCH] Decode streamed update output at character boundaries The update popup kept the last 64 KiB of raw bytes and decoded the whole buffer on every refresh. Cutting the buffer, or reading a chunk that ended inside a multi-byte sequence, made the UTF-8 decode fail and sent everything through the GB18030 fallback, garbling the display. Clip the retained tail at a UTF-8 boundary and decode still-growing streams with an incremental decoder that holds back an incomplete trailing sequence instead of treating it as corruption. Co-Authored-By: Claude Fable 5.1 --- herdr_hub/control.py | 13 +++++++++---- herdr_hub/remotes.py | 27 ++++++++++++++++++++++++++- tests/test_remotes.py | 25 +++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 5 deletions(-) diff --git a/herdr_hub/control.py b/herdr_hub/control.py index 898a34d..b372729 100644 --- a/herdr_hub/control.py +++ b/herdr_hub/control.py @@ -18,7 +18,7 @@ import uuid import unicodedata from .remotes import ( - RemoteEntry, decode_remote_output, parse_remote_session_result, + RemoteEntry, clip_utf8_tail, decode_remote_output, parse_remote_session_result, remote_control_command, remote_session_command, ) @@ -177,13 +177,18 @@ class HerdrControl: chunk = os.pread(output.fileno(), 65536, position) if chunk: position += len(chunk) - captured = (captured + chunk)[-65536:] - progress(decode_remote_output(captured).replace("\r", "\n")) + # Bound memory at a character boundary; the + # chunk may also end mid-sequence, so decode + # it as a still-growing stream. + captured = clip_utf8_tail(captured + chunk, 65536) + progress( + decode_remote_output(captured, partial=True).replace("\r", "\n") + ) last_output = time.monotonic() now = time.monotonic() if not finished and now - last_notice >= 1: progress( - decode_remote_output(captured).replace("\r", "\n") + decode_remote_output(captured, partial=True).replace("\r", "\n") + f"\n已执行 {int(now - started)} 秒 · {int(now - last_output)} 秒无新输出" ) last_notice = now diff --git a/herdr_hub/remotes.py b/herdr_hub/remotes.py index 53daedc..6af8f2f 100644 --- a/herdr_hub/remotes.py +++ b/herdr_hub/remotes.py @@ -1,5 +1,6 @@ from __future__ import annotations +import codecs from collections import deque from dataclasses import dataclass import fcntl @@ -491,9 +492,33 @@ def stop_remote_session( raise RemoteSessionStopError(" ".join(detail.splitlines())) -def decode_remote_output(raw: bytes) -> str: +def clip_utf8_tail(raw: bytes, limit: int) -> bytes: + """Keep the last ``limit`` bytes without starting inside a UTF-8 sequence.""" + if len(raw) <= limit: + return raw + clipped = raw[-limit:] + skip = 0 + while skip < min(3, len(clipped)) and 0x80 <= clipped[skip] <= 0xBF: + skip += 1 + return clipped[skip:] + + +def decode_remote_output(raw: bytes, *, partial: bool = False) -> str: + """Decode UTF-8 or Windows code page 936 output. + + ``partial`` marks a prefix of a stream that is still being written. An + incomplete trailing multi-byte sequence is then held back instead of being + treated as corruption, which would otherwise push the whole buffer through + the GB18030 fallback and garble every character of a progress display. + """ if not raw: return "" + if partial: + for encoding in ("utf-8", "gb18030"): + try: + return codecs.getincrementaldecoder(encoding)().decode(raw, final=False) + except UnicodeDecodeError: + continue try: return raw.decode("utf-8") except UnicodeDecodeError: diff --git a/tests/test_remotes.py b/tests/test_remotes.py index c7936b8..62a285d 100644 --- a/tests/test_remotes.py +++ b/tests/test_remotes.py @@ -15,6 +15,7 @@ from herdr_hub.remotes import ( RemoteQueryPool, RemoteRegistry, RemoteTargetError, + clip_utf8_tail, delete_remote_session, decode_remote_output, parse_remote_session_result, @@ -295,5 +296,29 @@ class RemoteSshTests(unittest.TestCase): self.assertIn("timed out", results[0].error or "") +class RemoteOutputDecodingTests(unittest.TestCase): + def test_partial_streams_hold_back_incomplete_sequences(self) -> None: + text = "更新目标 Session：default\n正在热切换…" + raw = text.encode("utf-8") + # A chunk boundary inside the last character must not garble the + # characters already received. + self.assertEqual(decode_remote_output(raw[:-1], partial=True), text[:-1]) + self.assertEqual(decode_remote_output(raw, partial=True), text) + gbk = "错误：下载失败".encode("gb18030") + self.assertEqual(decode_remote_output(gbk[:-1], partial=True), "错误：下载失") + # Final decoding keeps its strict behaviour. + self.assertEqual(decode_remote_output(raw), text) + self.assertEqual(decode_remote_output(gbk), "错误：下载失败") + + def test_clip_utf8_tail_never_starts_inside_a_character(self) -> None: + raw = "abc更新目标".encode("utf-8") + self.assertEqual(clip_utf8_tail(raw, len(raw)), raw) + # Cutting one byte into "更" drops the rest of that character. + clipped = clip_utf8_tail(raw, len(raw) - 4) + self.assertEqual(clipped.decode("utf-8"), "新目标") + self.assertEqual(clip_utf8_tail(raw, 3).decode("utf-8"), "标") + self.assertEqual(clip_utf8_tail(b"", 10), b"") + + if __name__ == "__main__": unittest.main()