842 lines
35 KiB
Python
842 lines
35 KiB
Python
#!/usr/bin/env python3
|
|
"""Prepare and execute a guarded batch publication through cms_api.py."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any, NoReturn
|
|
|
|
import cms_api as cms
|
|
import cms_batch_common as runtime
|
|
|
|
|
|
PACKET_FORMAT = "bjd-cms-publish-batch/v2"
|
|
BATCH_VERIFIER_VERSION = "2"
|
|
PACKET_MAX_AGE = dt.timedelta(hours=24)
|
|
PACKET_FUTURE_SKEW = dt.timedelta(minutes=5)
|
|
MAX_FILE_BYTES = 1024 * 1024
|
|
MAX_ITEMS = 50
|
|
MAX_AFFECTED_PAGES = 20
|
|
ITEM_PATH = re.compile(
|
|
r"^/api/ai/(news|products|series|industries|application-cases|downloads|"
|
|
r"about-culture-items|recruitment-cards|welfare-items|staff-activities)/([1-9][0-9]*)$"
|
|
)
|
|
RESOURCE_SCHEMA = {
|
|
"news": "news",
|
|
"products": "product",
|
|
"series": "series",
|
|
"industries": "industry",
|
|
"application-cases": "application_case",
|
|
"downloads": "download",
|
|
"about-culture-items": "about_culture_item",
|
|
"recruitment-cards": "recruitment-cards",
|
|
"welfare-items": "welfare-items",
|
|
"staff-activities": "staff-activities",
|
|
}
|
|
EXPECTED_CACHE_TARGETS = {
|
|
("thinkphp_file_cache", "runtime/cache"),
|
|
("cms_frontend_template_cache", "runtime/cms/temp"),
|
|
}
|
|
|
|
|
|
class BatchError(Exception):
|
|
def __init__(self, message: str, **details: Any) -> None:
|
|
super().__init__(message)
|
|
self.details = details
|
|
|
|
|
|
def fail(message: str, code: int = 2) -> NoReturn:
|
|
print(message, file=sys.stderr)
|
|
raise SystemExit(code)
|
|
|
|
|
|
def canonical_json(value: Any) -> str:
|
|
return runtime.canonical_json(value)
|
|
|
|
|
|
def packet_hash(packet: dict[str, Any]) -> str:
|
|
return runtime.review_hash(packet)
|
|
|
|
|
|
def verifier_identity() -> dict[str, str]:
|
|
public_fetch_version = getattr(cms, "PUBLIC_FETCH_CONTRACT_VERSION", "")
|
|
if not isinstance(public_fetch_version, str) or not public_fetch_version:
|
|
raise BatchError("CMS public-fetch verifier does not expose a contract version")
|
|
return {
|
|
"batch": BATCH_VERIFIER_VERSION,
|
|
"public_fetch": public_fetch_version,
|
|
}
|
|
|
|
|
|
def load_json_file(path_text: str, label: str) -> dict[str, Any]:
|
|
path = Path(path_text).expanduser()
|
|
if path_text == "-":
|
|
raw = sys.stdin.buffer.read(MAX_FILE_BYTES + 1)
|
|
else:
|
|
if not path.is_file() or path.is_symlink():
|
|
raise BatchError(f"{label} must be a regular non-symlink file")
|
|
if path.stat().st_size > MAX_FILE_BYTES:
|
|
raise BatchError(f"{label} exceeds the 1 MiB limit")
|
|
raw = path.read_bytes()
|
|
if len(raw) > MAX_FILE_BYTES:
|
|
raise BatchError(f"{label} exceeds the 1 MiB limit")
|
|
try:
|
|
value = json.loads(raw.decode("utf-8"))
|
|
except (UnicodeDecodeError, json.JSONDecodeError):
|
|
raise BatchError(f"{label} must contain one UTF-8 JSON object") from None
|
|
if not isinstance(value, dict):
|
|
raise BatchError(f"{label} must contain one JSON object")
|
|
return value
|
|
|
|
|
|
def write_packet(path_text: str, packet: dict[str, Any]) -> None:
|
|
try:
|
|
runtime.write_secure_json(path_text, packet, label="packet")
|
|
except ValueError as exc:
|
|
raise BatchError(str(exc)) from None
|
|
|
|
|
|
def require_keys(value: dict[str, Any], allowed: set[str], label: str) -> None:
|
|
unknown = sorted(set(value) - allowed)
|
|
if unknown:
|
|
raise BatchError(f"{label} contains unknown field: {unknown[0]}")
|
|
|
|
|
|
def validate_text(value: Any, label: str, maximum: int = 2000) -> str:
|
|
if not isinstance(value, str) or not value or len(value) > maximum or cms.CONTROL.search(value):
|
|
raise BatchError(f"{label} must be a non-empty control-free string of at most {maximum} characters")
|
|
return value
|
|
|
|
|
|
def parse_item_path(path: Any) -> tuple[str, str, int]:
|
|
path = validate_text(path, "item path", 512)
|
|
match = ITEM_PATH.fullmatch(path)
|
|
if not match:
|
|
raise BatchError("item path is not a supported routine-content detail endpoint")
|
|
resource = match.group(1)
|
|
return path, resource, int(match.group(2))
|
|
|
|
|
|
def validate_affected_pages(value: Any, site: str) -> list[dict[str, Any]]:
|
|
if value is None:
|
|
return []
|
|
if not isinstance(value, list) or len(value) > MAX_AFFECTED_PAGES:
|
|
raise BatchError(f"affected_pages must be an array of at most {MAX_AFFECTED_PAGES} items")
|
|
pages: list[dict[str, Any]] = []
|
|
for index, page in enumerate(value):
|
|
if not isinstance(page, dict):
|
|
raise BatchError(f"affected_pages[{index}] must be an object")
|
|
require_keys(page, {"url", "expected_text", "expected_url", "label"}, f"affected_pages[{index}]")
|
|
url = validate_text(page.get("url"), f"affected_pages[{index}].url", 4096)
|
|
expected_text = page.get("expected_text")
|
|
expected_url = page.get("expected_url")
|
|
if expected_text is not None:
|
|
expected_text = validate_text(expected_text, f"affected_pages[{index}].expected_text")
|
|
if expected_url is not None:
|
|
expected_url = validate_text(expected_url, f"affected_pages[{index}].expected_url", 4096)
|
|
if not cms._url_matches_site(expected_url, site):
|
|
raise BatchError(f"affected_pages[{index}].expected_url does not match site={site}")
|
|
if expected_text is None and expected_url is None:
|
|
raise BatchError(f"affected_pages[{index}] requires expected_text, expected_url, or both")
|
|
label = page.get("label", url)
|
|
label = validate_text(label, f"affected_pages[{index}].label", 200)
|
|
if not cms._url_matches_site(url, site):
|
|
raise BatchError(f"affected_pages[{index}].url does not match site={site}")
|
|
pages.append({
|
|
"url": url,
|
|
"expected_text": expected_text,
|
|
"expected_url": expected_url,
|
|
"label": label,
|
|
})
|
|
return pages
|
|
|
|
|
|
def validate_plan(plan: dict[str, Any]) -> dict[str, Any]:
|
|
require_keys(plan, {"site", "items", "affected_pages", "ui_changed"}, "plan")
|
|
site = plan.get("site")
|
|
if site not in {"cn", "en", "jp"}:
|
|
raise BatchError("plan.site must be cn, en, or jp")
|
|
raw_items = plan.get("items")
|
|
if not isinstance(raw_items, list) or not raw_items or len(raw_items) > MAX_ITEMS:
|
|
raise BatchError(f"plan.items must contain 1-{MAX_ITEMS} items")
|
|
items: list[dict[str, Any]] = []
|
|
seen: set[str] = set()
|
|
for index, item in enumerate(raw_items):
|
|
if not isinstance(item, dict):
|
|
raise BatchError(f"items[{index}] must be an object")
|
|
require_keys(item, {"path"}, f"items[{index}]")
|
|
path, resource, item_id = parse_item_path(item.get("path"))
|
|
if path in seen:
|
|
raise BatchError(f"duplicate item path: {path}")
|
|
seen.add(path)
|
|
items.append({"path": path, "resource": resource, "schema_type": RESOURCE_SCHEMA[resource], "id": item_id})
|
|
ui_changed = plan.get("ui_changed", False)
|
|
if not isinstance(ui_changed, bool):
|
|
raise BatchError("plan.ui_changed must be true or false")
|
|
return {
|
|
"site": site,
|
|
"items": items,
|
|
"affected_pages": validate_affected_pages(plan.get("affected_pages"), site),
|
|
"ui_changed": ui_changed,
|
|
}
|
|
|
|
|
|
def api_call(arguments: list[str]) -> dict[str, Any]:
|
|
response = cms.request_command(arguments, emit=False)
|
|
if not isinstance(response, dict):
|
|
raise BatchError("CMS helper returned an invalid response")
|
|
return response
|
|
|
|
|
|
def data_of(response: dict[str, Any], label: str) -> dict[str, Any]:
|
|
data = response.get("data")
|
|
if response.get("ok") is not True or response.get("errors") or not isinstance(data, dict):
|
|
raise BatchError(f"{label} returned an invalid envelope")
|
|
return data
|
|
|
|
|
|
def fetch_schema(schema_type: str, site: str) -> None:
|
|
data = data_of(api_call(["GET", f"/api/ai/content/schema/{schema_type}"]), "schema")
|
|
schema = data.get("types", {}).get(schema_type) if isinstance(data.get("types"), dict) else None
|
|
if not isinstance(schema, dict) or not isinstance(schema.get("writable_fields"), list):
|
|
raise BatchError(f"schema did not expose type={schema_type}")
|
|
site_schema = schema.get("sites", {}).get(site) if isinstance(schema.get("sites"), dict) else None
|
|
if not isinstance(site_schema, dict):
|
|
raise BatchError(f"schema did not expose site={site} for type={schema_type}")
|
|
|
|
|
|
def media_counts(detail: dict[str, Any]) -> dict[str, int]:
|
|
return {
|
|
field: len(detail[field]) if isinstance(detail.get(field), list) else 0
|
|
for field in ("thumb", "image", "bq", "down", "file")
|
|
}
|
|
|
|
|
|
def read_detail(item: dict[str, Any], site: str) -> dict[str, Any]:
|
|
data = data_of(api_call(["GET", item["path"], "--query", f"site={site}"]), "detail")
|
|
if data.get("site") != site or data.get("id") != item["id"]:
|
|
raise BatchError("detail returned an unexpected site or item ID", path=item["path"])
|
|
version = data.get("version")
|
|
if not isinstance(version, str) or not cms.CONTENT_VERSION.fullmatch(version):
|
|
raise BatchError("detail omitted a canonical content version", path=item["path"])
|
|
title = data.get("title")
|
|
url = data.get("url")
|
|
if not isinstance(title, str) or not title or not isinstance(url, str) or not url:
|
|
raise BatchError("batch publication requires a titled item with a public URL", path=item["path"])
|
|
if not cms._url_matches_site(url, site):
|
|
raise BatchError("detail public URL does not match the selected site", path=item["path"])
|
|
return data
|
|
|
|
|
|
def prepare_batch(
|
|
plan: dict[str, Any],
|
|
progress: runtime.ProgressCallback | None = None,
|
|
) -> dict[str, Any]:
|
|
normalized = validate_plan(plan)
|
|
cms.reset_http_trace()
|
|
origin = cms._settings()[0]
|
|
total = len(normalized["items"])
|
|
runtime.emit_progress(progress, "preflight", 0, total)
|
|
for schema_type in sorted({item["schema_type"] for item in normalized["items"]}):
|
|
fetch_schema(schema_type, normalized["site"])
|
|
reviewed: list[dict[str, Any]] = []
|
|
for index, item in enumerate(normalized["items"], start=1):
|
|
detail = read_detail(item, normalized["site"])
|
|
if detail.get("published") is not False:
|
|
raise BatchError("prepare requires unpublished drafts", path=item["path"])
|
|
reviewed.append({
|
|
**item,
|
|
"expected_version": detail["version"],
|
|
"title": detail["title"],
|
|
"url": detail["url"],
|
|
"catid": detail.get("catid"),
|
|
"media_counts": media_counts(detail),
|
|
})
|
|
runtime.emit_progress(progress, "validate", index, total, item["id"])
|
|
for page in normalized["affected_pages"]:
|
|
try:
|
|
cms.fetch_public_url(page["url"], normalized["site"])
|
|
except cms.ClientError as exc:
|
|
raise BatchError(
|
|
"affected page is not reachable during prepare",
|
|
label=page["label"],
|
|
url=page["url"],
|
|
reason=str(exc),
|
|
) from None
|
|
packet: dict[str, Any] = {
|
|
"format": PACKET_FORMAT,
|
|
"verifier": verifier_identity(),
|
|
"origin": origin,
|
|
"site": normalized["site"],
|
|
"prepared_at": dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat(),
|
|
"items": reviewed,
|
|
"affected_pages": normalized["affected_pages"],
|
|
"ui_changed": normalized["ui_changed"],
|
|
}
|
|
packet["review_hash"] = packet_hash(packet)
|
|
runtime.emit_progress(progress, "complete", total, total)
|
|
return packet
|
|
|
|
|
|
def validate_packet(packet: dict[str, Any]) -> dict[str, Any]:
|
|
allowed = {"format", "verifier", "origin", "site", "prepared_at", "items", "affected_pages", "ui_changed", "review_hash"}
|
|
require_keys(packet, allowed, "packet")
|
|
if packet.get("format") != PACKET_FORMAT:
|
|
raise BatchError("unsupported batch packet format")
|
|
if packet.get("verifier") != verifier_identity():
|
|
raise BatchError("publication verifier changed after review; prepare and confirm a new packet")
|
|
if packet.get("site") not in {"cn", "en", "jp"}:
|
|
raise BatchError("packet site is invalid")
|
|
expected_hash = packet.get("review_hash")
|
|
if not isinstance(expected_hash, str) or not cms.CONTENT_VERSION.fullmatch(expected_hash):
|
|
raise BatchError("packet review_hash is invalid")
|
|
if packet_hash(packet) != expected_hash:
|
|
raise BatchError("packet changed after review")
|
|
origin = validate_text(packet.get("origin"), "packet origin", 2048)
|
|
if cms.validate_origin(origin)[0] != origin:
|
|
raise BatchError("packet origin is not canonical")
|
|
if cms._settings()[0] != origin:
|
|
raise BatchError("current CMS origin does not match the reviewed packet")
|
|
prepared_at = validate_text(packet.get("prepared_at"), "packet prepared_at", 64)
|
|
try:
|
|
prepared = dt.datetime.fromisoformat(prepared_at)
|
|
except ValueError:
|
|
raise BatchError("packet prepared_at is invalid") from None
|
|
if prepared.tzinfo is None or prepared.utcoffset() is None:
|
|
raise BatchError("packet prepared_at must include a timezone")
|
|
now = dt.datetime.now(dt.timezone.utc)
|
|
prepared = prepared.astimezone(dt.timezone.utc)
|
|
if prepared > now + PACKET_FUTURE_SKEW:
|
|
raise BatchError("packet prepared_at is unexpectedly in the future")
|
|
if now - prepared > PACKET_MAX_AGE:
|
|
raise BatchError("review packet is older than 24 hours; prepare and review a new packet")
|
|
raw_items = packet.get("items")
|
|
if not isinstance(raw_items, list) or not raw_items or len(raw_items) > MAX_ITEMS:
|
|
raise BatchError("packet items are invalid")
|
|
items: list[dict[str, Any]] = []
|
|
seen: set[str] = set()
|
|
item_keys = {"path", "resource", "schema_type", "id", "expected_version", "title", "url", "catid", "media_counts"}
|
|
for index, item in enumerate(raw_items):
|
|
if not isinstance(item, dict):
|
|
raise BatchError(f"packet.items[{index}] must be an object")
|
|
require_keys(item, item_keys, f"packet.items[{index}]")
|
|
path, resource, item_id = parse_item_path(item.get("path"))
|
|
if path in seen or resource != item.get("resource") or RESOURCE_SCHEMA[resource] != item.get("schema_type") or item_id != item.get("id"):
|
|
raise BatchError(f"packet.items[{index}] identity is invalid")
|
|
seen.add(path)
|
|
version = item.get("expected_version")
|
|
if not isinstance(version, str) or not cms.CONTENT_VERSION.fullmatch(version):
|
|
raise BatchError(f"packet.items[{index}] version is invalid")
|
|
validate_text(item.get("title"), f"packet.items[{index}].title")
|
|
url = validate_text(item.get("url"), f"packet.items[{index}].url", 4096)
|
|
if not cms._url_matches_site(url, packet["site"]):
|
|
raise BatchError(f"packet.items[{index}].url does not match site={packet['site']}")
|
|
counts = item.get("media_counts")
|
|
if (
|
|
not isinstance(counts, dict)
|
|
or set(counts) != {"thumb", "image", "bq", "down", "file"}
|
|
or any(type(value) is not int or value < 0 for value in counts.values())
|
|
):
|
|
raise BatchError(f"packet.items[{index}].media_counts is invalid")
|
|
items.append(dict(item))
|
|
ui_changed = packet.get("ui_changed")
|
|
if not isinstance(ui_changed, bool):
|
|
raise BatchError("packet.ui_changed must be true or false")
|
|
return {
|
|
**packet,
|
|
"items": items,
|
|
"affected_pages": validate_affected_pages(packet.get("affected_pages"), packet["site"]),
|
|
}
|
|
|
|
|
|
idempotency_key = runtime.idempotency_key
|
|
|
|
|
|
def checks_passed(verify: dict[str, Any]) -> bool:
|
|
checks = verify.get("checks")
|
|
return (
|
|
verify.get("visible_by_database_rules") is True
|
|
and isinstance(checks, dict)
|
|
and bool(checks)
|
|
and all(isinstance(value, list) and value and value[0] is True for value in checks.values())
|
|
)
|
|
|
|
|
|
def public_expectations_passed(result: dict[str, Any], page: dict[str, Any]) -> bool:
|
|
outcomes: list[bool] = []
|
|
if page.get("expected_text") is not None:
|
|
outcomes.append(result.get("expected_text_present") is True)
|
|
if page.get("expected_url") is not None:
|
|
outcomes.append(result.get("expected_href_present") is True)
|
|
return bool(outcomes) and all(outcomes)
|
|
|
|
|
|
def public_diagnostics(result: dict[str, Any]) -> dict[str, Any]:
|
|
keys = (
|
|
"expected_text_present",
|
|
"expected_text_match_mode",
|
|
"expected_href_present",
|
|
"visible_text_characters",
|
|
"visible_text_sha256",
|
|
"same_site_href_count",
|
|
"same_site_hrefs_sample",
|
|
)
|
|
return {key: result[key] for key in keys if key in result}
|
|
|
|
|
|
def is_frontend_stale_404(error: Exception) -> bool:
|
|
"""Recognize a cache-recoverable miss after database verification passed."""
|
|
return str(error) == "Public URL returned HTTP 404"
|
|
|
|
|
|
def fetch_affected_page(page: dict[str, Any], site: str) -> dict[str, Any]:
|
|
result = cms.fetch_public_url(
|
|
page["url"],
|
|
site,
|
|
page.get("expected_text"),
|
|
expected_href=page.get("expected_url"),
|
|
)
|
|
return {
|
|
"label": page["label"],
|
|
"url": page["url"],
|
|
"current": public_expectations_passed(result, page),
|
|
"diagnostics": public_diagnostics(result),
|
|
}
|
|
|
|
|
|
def validate_cache_result(response: dict[str, Any], dry_run: bool) -> dict[str, Any]:
|
|
data = data_of(response, "frontend cache clear")
|
|
targets = data.get("targets")
|
|
if data.get("scope") != "frontend" or data.get("dry_run") is not dry_run or not isinstance(targets, list):
|
|
raise BatchError("frontend cache response is inconsistent")
|
|
observed: set[tuple[str, str]] = set()
|
|
for target in targets:
|
|
if not isinstance(target, dict) or target.get("errors") != []:
|
|
raise BatchError("frontend cache target reported errors")
|
|
observed.add((target.get("name"), target.get("path")))
|
|
if observed != EXPECTED_CACHE_TARGETS or data.get("targets_count") != 2:
|
|
raise BatchError("frontend cache dry-run returned unexpected targets")
|
|
return data
|
|
|
|
|
|
def unpublish(item: dict[str, Any], version: str, review_hash: str, reason: str) -> dict[str, Any]:
|
|
body = {"site": item["site"], "expected_version": version}
|
|
key = f"batch-{review_hash[:12]}-{item['id']}-unpublish"
|
|
with idempotency_key(key):
|
|
response = api_call(["POST", f"{item['path']}/unpublish", canonical_json(body)])
|
|
data = data_of(response, "compensating unpublish")
|
|
next_version = data.get("version")
|
|
if (
|
|
data.get("id") != item["id"]
|
|
or data.get("published") is not False
|
|
or not isinstance(next_version, str)
|
|
or not cms.CONTENT_VERSION.fullmatch(next_version)
|
|
):
|
|
raise BatchError("compensating unpublish did not prove unpublished state", id=item["id"], reason=reason)
|
|
return {"id": item["id"], "attempted": True, "succeeded": True, "reason": reason, "version": next_version}
|
|
|
|
|
|
def compensate_published(
|
|
states: list[dict[str, Any]],
|
|
review_hash: str,
|
|
reason: str,
|
|
) -> list[dict[str, Any]]:
|
|
"""Best-effort reverse-order compensation for states proven published by this run."""
|
|
results: list[dict[str, Any]] = []
|
|
for state in reversed(states):
|
|
try:
|
|
results.append(unpublish(state["item"], state["version"], review_hash, reason))
|
|
except (BatchError, cms.ClientError) as exc:
|
|
failure: dict[str, Any] = {
|
|
"id": state["item"]["id"],
|
|
"attempted": True,
|
|
"succeeded": False,
|
|
"error": str(exc),
|
|
}
|
|
if isinstance(exc, cms.CmsRequestError):
|
|
failure["outcome_unknown"] = exc.request_sent and not exc.response_received
|
|
results.append(failure)
|
|
return results
|
|
|
|
|
|
def abort_published_batch(
|
|
message: str,
|
|
states: list[dict[str, Any]],
|
|
review_hash: str,
|
|
reason: str,
|
|
**details: Any,
|
|
) -> NoReturn:
|
|
compensation = compensate_published(states, review_hash, reason)
|
|
raise BatchError(message, compensation=compensation, **details)
|
|
|
|
|
|
def publish_batch(
|
|
packet: dict[str, Any],
|
|
confirmed: bool,
|
|
progress: runtime.ProgressCallback | None = None,
|
|
) -> dict[str, Any]:
|
|
if confirmed is not True:
|
|
raise BatchError("publish requires --confirmed under explicit human authorization for this scope")
|
|
normalized = validate_packet(packet)
|
|
cms.reset_http_trace()
|
|
site = normalized["site"]
|
|
total = len(normalized["items"])
|
|
runtime.emit_progress(progress, "preflight", 0, total)
|
|
for schema_type in sorted({item["schema_type"] for item in normalized["items"]}):
|
|
fetch_schema(schema_type, site)
|
|
for index, item in enumerate(normalized["items"], start=1):
|
|
detail = read_detail(item, site)
|
|
if (
|
|
detail.get("published") is not False
|
|
or detail["version"] != item["expected_version"]
|
|
or detail["title"] != item["title"]
|
|
or detail["url"] != item["url"]
|
|
or detail.get("catid") != item.get("catid")
|
|
or media_counts(detail) != item["media_counts"]
|
|
):
|
|
raise BatchError("content changed after review; prepare a new packet", path=item["path"])
|
|
runtime.emit_progress(progress, "validate", index, total, item["id"])
|
|
|
|
published: list[dict[str, Any]] = []
|
|
published_states: list[dict[str, Any]] = []
|
|
stale_products: list[dict[str, Any]] = []
|
|
review_hash = normalized["review_hash"]
|
|
for index, reviewed in enumerate(normalized["items"], start=1):
|
|
item = {**reviewed, "site": site}
|
|
body = {"site": site, "expected_version": reviewed["expected_version"], "confirmed_publish": True}
|
|
publish_key = f"batch-{review_hash[:12]}-{reviewed['id']}-publish"
|
|
try:
|
|
with idempotency_key(publish_key):
|
|
result = data_of(api_call(["POST", f"{reviewed['path']}/publish", canonical_json(body)]), "publish")
|
|
except cms.CmsRequestError as exc:
|
|
compensation = compensate_published(
|
|
published_states,
|
|
review_hash,
|
|
"a later publish request did not return a safe success",
|
|
)
|
|
raise BatchError(
|
|
"publish request did not return a safe success; stop without retry",
|
|
id=reviewed["id"],
|
|
outcome_unknown=exc.request_sent and not exc.response_received,
|
|
idempotency_key=publish_key,
|
|
compensation=compensation,
|
|
) from None
|
|
except (BatchError, cms.ClientError) as exc:
|
|
compensation = compensate_published(
|
|
published_states,
|
|
review_hash,
|
|
"a later publish attempt did not produce a valid response",
|
|
)
|
|
raise BatchError(
|
|
"publish attempt did not produce a valid response; inspect detail and audit before any replay",
|
|
id=reviewed["id"],
|
|
outcome_unknown=True,
|
|
idempotency_key=publish_key,
|
|
compensation=compensation,
|
|
) from None
|
|
latest = result.get("version")
|
|
if result.get("id") != reviewed["id"] or result.get("published") is not True or not isinstance(latest, str) or not cms.CONTENT_VERSION.fullmatch(latest):
|
|
compensation = compensate_published(
|
|
published_states,
|
|
review_hash,
|
|
"a later publish response did not prove public state",
|
|
)
|
|
raise BatchError(
|
|
"publish response did not prove public state; inspect detail and audit before any replay",
|
|
id=reviewed["id"],
|
|
outcome_unknown=True,
|
|
idempotency_key=publish_key,
|
|
compensation=compensation,
|
|
)
|
|
published_states.append({"item": item, "version": latest})
|
|
runtime.emit_progress(progress, "write", index, total, reviewed["id"])
|
|
try:
|
|
readback = read_detail(reviewed, site)
|
|
if readback.get("published") is not True or readback["version"] != latest:
|
|
raise BatchError("published readback is inconsistent")
|
|
runtime.emit_progress(progress, "readback", index, total, reviewed["id"])
|
|
verified = data_of(api_call(["POST", f"{reviewed['path']}/verify", canonical_json({"site": site})]), "verify")
|
|
if verified.get("id") != reviewed["id"] or not checks_passed(verified):
|
|
raise BatchError("post-publish verification failed")
|
|
except (BatchError, cms.ClientError) as exc:
|
|
abort_published_batch(
|
|
"post-publish checks failed; proven batch publications were unpublished",
|
|
published_states,
|
|
review_hash,
|
|
str(exc),
|
|
id=reviewed["id"],
|
|
)
|
|
try:
|
|
public = cms.fetch_public_url(reviewed["url"], site, reviewed["title"])
|
|
except cms.ClientError as exc:
|
|
if not is_frontend_stale_404(exc):
|
|
abort_published_batch(
|
|
"post-publish public fetch failed; proven batch publications were unpublished",
|
|
published_states,
|
|
review_hash,
|
|
str(exc),
|
|
id=reviewed["id"],
|
|
)
|
|
public = {"status": 404, "expected_text_present": False}
|
|
runtime.emit_progress(progress, "verify", index, total, reviewed["id"])
|
|
entry = {
|
|
"id": reviewed["id"], "title": reviewed["title"], "url": reviewed["url"],
|
|
"version": latest, "verified": True,
|
|
"public_fetch": public.get("status") == 200,
|
|
"expected_text_present": public.get("expected_text_present") is True,
|
|
"diagnostics": public_diagnostics(public),
|
|
}
|
|
published.append(entry)
|
|
if entry["expected_text_present"] is not True:
|
|
stale_products.append({"item": item, "version": latest})
|
|
|
|
affected_results: list[dict[str, Any]] = []
|
|
stale_pages: list[dict[str, Any]] = []
|
|
try:
|
|
for page in normalized["affected_pages"]:
|
|
observed = fetch_affected_page(page, site)
|
|
affected_results.append(observed)
|
|
if not observed["current"]:
|
|
stale_pages.append(page)
|
|
except cms.ClientError as exc:
|
|
abort_published_batch(
|
|
"affected public page fetch failed; proven batch publications were unpublished",
|
|
published_states,
|
|
review_hash,
|
|
str(exc),
|
|
)
|
|
|
|
cache = {"cleared": False, "reason": "not-needed"}
|
|
if stale_products or stale_pages:
|
|
runtime.emit_progress(progress, "cache", 0, 1)
|
|
try:
|
|
dry = validate_cache_result(
|
|
api_call(["POST", "/api/ai/cache/clear", canonical_json({"scope": "frontend", "dry_run": True})]),
|
|
True,
|
|
)
|
|
cache_key = f"batch-{review_hash[:12]}-frontend-cache"
|
|
with idempotency_key(cache_key):
|
|
applied = validate_cache_result(
|
|
api_call(["POST", "/api/ai/cache/clear", canonical_json({"scope": "frontend"})]),
|
|
False,
|
|
)
|
|
except (BatchError, cms.ClientError) as exc:
|
|
abort_published_batch(
|
|
"safe frontend cache refresh failed; proven batch publications were unpublished",
|
|
published_states,
|
|
review_hash,
|
|
str(exc),
|
|
)
|
|
cache = {
|
|
"cleared": True, "reason": "stale-public-page", "dry_run_targets": dry["targets_count"],
|
|
"cleared_targets": applied["targets_count"],
|
|
}
|
|
runtime.emit_progress(progress, "cache", 1, 1)
|
|
for stale in stale_products:
|
|
try:
|
|
public = cms.fetch_public_url(stale["item"]["url"], site, stale["item"]["title"])
|
|
except cms.ClientError as exc:
|
|
abort_published_batch(
|
|
"public page refetch failed after cache clear; proven batch publications were unpublished",
|
|
published_states,
|
|
review_hash,
|
|
str(exc),
|
|
id=stale["item"]["id"],
|
|
)
|
|
if public.get("expected_text_present") is not True:
|
|
abort_published_batch(
|
|
"public page remained stale after cache clear; proven batch publications were unpublished",
|
|
published_states,
|
|
review_hash,
|
|
"public page remained stale after cache clear",
|
|
id=stale["item"]["id"],
|
|
diagnostics=public_diagnostics(public),
|
|
)
|
|
for entry in published:
|
|
if entry["id"] == stale["item"]["id"]:
|
|
entry["expected_text_present"] = True
|
|
entry["diagnostics"] = public_diagnostics(public)
|
|
affected_results = []
|
|
try:
|
|
for page in normalized["affected_pages"]:
|
|
affected_results.append(fetch_affected_page(page, site))
|
|
except cms.ClientError as exc:
|
|
abort_published_batch(
|
|
"affected public page refetch failed; proven batch publications were unpublished",
|
|
published_states,
|
|
review_hash,
|
|
str(exc),
|
|
)
|
|
if not all(item["current"] for item in affected_results):
|
|
abort_published_batch(
|
|
"an affected public page remained stale; proven batch publications were unpublished",
|
|
published_states,
|
|
review_hash,
|
|
"affected public page remained stale after the safe cache clear",
|
|
affected_pages=affected_results,
|
|
)
|
|
|
|
result = {
|
|
"ok": True,
|
|
"site": site,
|
|
"verifier": normalized["verifier"],
|
|
"review_hash": review_hash,
|
|
"published": published,
|
|
"affected_pages": affected_results,
|
|
"cache": cache,
|
|
"verification_mode": "quick",
|
|
"extended_ui_check_required": normalized["ui_changed"],
|
|
"http_requests": len(cms.get_http_trace()),
|
|
}
|
|
runtime.emit_progress(progress, "complete", total, total)
|
|
return result
|
|
|
|
|
|
def _report_option(args: list[str]) -> tuple[list[str], str | None]:
|
|
cleaned: list[str] = []
|
|
report_path: str | None = None
|
|
index = 0
|
|
while index < len(args):
|
|
if args[index] == "--report-out":
|
|
if report_path is not None or index + 1 >= len(args):
|
|
raise BatchError("--report-out requires one unique path")
|
|
report_path = args[index + 1]
|
|
index += 2
|
|
continue
|
|
cleaned.append(args[index])
|
|
index += 1
|
|
return cleaned, report_path
|
|
|
|
|
|
def prepare_command(
|
|
args: list[str],
|
|
progress: runtime.ProgressCallback | None = None,
|
|
context: dict[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
if len(args) != 3 or args[1] != "--packet-out":
|
|
raise BatchError("Usage: cms_publish_batch.py prepare PLAN.json --packet-out PACKET.json [--report-out REPORT.json]")
|
|
plan = load_json_file(args[0], "plan")
|
|
if context is not None:
|
|
context["site"] = plan.get("site")
|
|
packet = prepare_batch(plan, progress=progress)
|
|
if context is not None:
|
|
context.update({"site": packet["site"], "review_hash": packet["review_hash"]})
|
|
write_packet(args[2], packet)
|
|
output = {
|
|
"ok": True,
|
|
"action": "prepare",
|
|
"site": packet["site"],
|
|
"origin": packet["origin"],
|
|
"review_hash": packet["review_hash"],
|
|
"verifier": packet["verifier"],
|
|
"packet_path": str(Path(args[2]).expanduser()),
|
|
"items": [
|
|
{
|
|
"id": item["id"], "title": item["title"], "url": item["url"],
|
|
"catid": item["catid"], "media_counts": item["media_counts"],
|
|
}
|
|
for item in packet["items"]
|
|
],
|
|
"affected_pages": packet["affected_pages"],
|
|
"ui_changed": packet["ui_changed"],
|
|
"human_confirmation_required": True,
|
|
"http_requests": len(cms.get_http_trace()),
|
|
}
|
|
return output
|
|
|
|
|
|
def publish_command(
|
|
args: list[str],
|
|
progress: runtime.ProgressCallback | None = None,
|
|
context: dict[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
if len(args) != 2 or args[1] != "--confirmed":
|
|
raise BatchError("Usage: cms_publish_batch.py publish PACKET.json --confirmed [--report-out REPORT.json]")
|
|
packet = load_json_file(args[0], "packet")
|
|
if context is not None:
|
|
context.update({"site": packet.get("site"), "review_hash": packet.get("review_hash")})
|
|
return publish_batch(packet, confirmed=True, progress=progress)
|
|
|
|
|
|
def usage(code: int = 0) -> NoReturn:
|
|
stream = sys.stdout if code == 0 else sys.stderr
|
|
print(
|
|
"Usage:\n"
|
|
" python scripts/cms_publish_batch.py prepare PLAN.json --packet-out PACKET.json [--report-out REPORT.json]\n"
|
|
" python scripts/cms_publish_batch.py publish PACKET.json --confirmed [--report-out REPORT.json]\n",
|
|
file=stream,
|
|
)
|
|
raise SystemExit(code)
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = sys.argv[1:] if argv is None else argv
|
|
if not args or args[0] in {"-h", "--help"}:
|
|
usage(0 if args else 2)
|
|
action = args[0]
|
|
tracker = runtime.ProgressTracker(action, stream=True)
|
|
context: dict[str, Any] = {}
|
|
report: runtime.SecureReport | None = None
|
|
result: dict[str, Any] | None = None
|
|
caught: BaseException | None = None
|
|
try:
|
|
command_args, report_path = _report_option(args[1:])
|
|
report = runtime.SecureReport(report_path)
|
|
if action == "prepare":
|
|
result = prepare_command(command_args, tracker, context)
|
|
elif action == "publish":
|
|
result = publish_command(command_args, tracker, context)
|
|
else:
|
|
raise BatchError(f"Unknown action: {action}")
|
|
output = result
|
|
code = 0
|
|
except cms.CmsRequestError as exc:
|
|
caught = exc
|
|
output = {
|
|
"ok": False,
|
|
"error": str(exc),
|
|
"outcome_unknown": exc.request_sent and not exc.response_received,
|
|
"details": exc.safe_details,
|
|
}
|
|
except (BatchError, cms.ClientError) as exc:
|
|
caught = exc
|
|
output = {"ok": False, "error": str(exc)}
|
|
if isinstance(exc, BatchError) and exc.details:
|
|
output["details"] = exc.details
|
|
except ValueError as exc:
|
|
caught = exc
|
|
output = {"ok": False, "error": str(exc)}
|
|
except KeyboardInterrupt as exc:
|
|
caught = exc
|
|
output = {"ok": False, "error": "Interrupted"}
|
|
code = 130
|
|
except Exception as exc:
|
|
caught = exc
|
|
output = {"ok": False, "error": "unexpected internal batch failure"}
|
|
if caught is not None and "code" not in locals():
|
|
code = 1
|
|
if report is not None:
|
|
report_result = result or {"http_requests": len(cms.get_http_trace())}
|
|
report.write(runtime.build_report(
|
|
action=action,
|
|
tracker=tracker,
|
|
ok=caught is None,
|
|
site=context.get("site"),
|
|
packet_review_hash=context.get("review_hash"),
|
|
result=report_result,
|
|
error=caught,
|
|
cms_module=cms,
|
|
))
|
|
print(canonical_json(output))
|
|
return code
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|