Files
jianli-boss/client/outbox.py
T
2026-08-26 13:33:21 +08:00

149 lines
5.0 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
本地投递 outbox(存储转发兜底)
====================================
上报先落盘再发送,防止网络抖动/云端不可用导致记录丢失:
- enqueue() 上报前先写盘,返回 entry_id
- ack() 发送成功后立即删除(常态下队列≈空,不积压)
- flush() 补发一轮:由心跳线程每 30s 驱动一次;按入队顺序发送,
网络未恢复时本轮立即停止,等下一轮,不做集中无效重试
防扩张约束:
- MAX_ITEMS 硬上限 1000 条,超出丢弃最旧并告警
- RETENTION_DAYS 保留期 7 天(与日志保留一致),过期丢弃
- 服务端按 user+job_id 幂等更新,重复补发不会产生重复记录
"""
import json
import threading
import time
import uuid
from pathlib import Path
import requests
from loguru import logger
OUTBOX_FILE = Path("client/data/outbox.json")
MAX_ITEMS = 1000
RETENTION_DAYS = 7
_lock = threading.Lock()
def _load() -> list[dict]:
if not OUTBOX_FILE.exists():
return []
try:
return json.loads(OUTBOX_FILE.read_text(encoding="utf-8")) or []
except Exception:
logger.warning("outbox 文件读取失败,按空队列处理")
return []
def _save(items: list[dict]) -> None:
OUTBOX_FILE.parent.mkdir(parents=True, exist_ok=True)
tmp = OUTBOX_FILE.with_suffix(".tmp")
tmp.write_text(json.dumps(items, ensure_ascii=False), encoding="utf-8")
tmp.replace(OUTBOX_FILE)
def enqueue(session_id: int | None, body: dict,
applied_delta: int = 0, skipped_delta: int = 0) -> str:
"""上报前写盘兜底,返回 entry_id;发送成功后调用 ack 删除"""
entry_id = uuid.uuid4().hex
with _lock:
items = _load()
items.append({
"id": entry_id,
"created_at": time.time(),
"session_id": session_id,
"body": body,
"applied_delta": applied_delta,
"skipped_delta": skipped_delta,
})
if len(items) > MAX_ITEMS:
dropped = len(items) - MAX_ITEMS
logger.warning(f"outbox 超 {MAX_ITEMS} 条上限,丢弃最旧 {dropped} 条")
items = items[-MAX_ITEMS:]
_save(items)
return entry_id
def ack(entry_id: str) -> None:
"""发送成功后从 outbox 删除"""
with _lock:
_save([i for i in _load() if i.get("id") != entry_id])
def pending_count() -> int:
with _lock:
return len(_load())
def _send_one(server_api, item: dict) -> str:
"""补发单条。返回:sent=成功 / keep=网络类失败保留 / drop=业务拒绝丢弃"""
body, sid = item["body"], item.get("session_id")
try:
server_api.report_application(sid, body)
except requests.HTTPError as e:
status = getattr(getattr(e, "response", None), "status_code", 0)
if 400 <= status < 500 and status not in (409, 429):
# 业务拒绝(422/401 等):去掉 session_id 再试一次,
# 仍被拒则为毒丸消息,丢弃避免卡住补发循环
try:
server_api.report_application(None, body)
except requests.HTTPError:
logger.warning(f"outbox 条目被服务端拒绝,丢弃:{body.get('job_id')}")
return "drop"
except Exception:
return "keep"
else:
return "keep"
except Exception:
return "keep"
# 补发成功:会话统计也补上(best-effort)
if item.get("session_id") and (item.get("applied_delta") or item.get("skipped_delta")):
try:
server_api.update_session_stats(
item["session_id"],
item.get("applied_delta", 0), item.get("skipped_delta", 0))
except Exception as e:
logger.debug(f"outbox 补发会话统计失败:{e}")
return "sent"
def flush(server_api) -> int:
"""补发一轮,返回成功条数。网络未恢复时本轮提前结束等下一轮"""
sent = 0
with _lock:
items = _load()
if not items:
return 0
# 保留期清理
cutoff = time.time() - RETENTION_DAYS * 86400
expired = [i for i in items if i.get("created_at", 0) < cutoff]
if expired:
logger.warning(f"outbox 超保留期({RETENTION_DAYS} 天),丢弃 {len(expired)} 条")
items = [i for i in items if i.get("created_at", 0) >= cutoff]
remaining: list[dict] = []
stopped = False
for it in items:
if stopped:
remaining.append(it)
continue
result = _send_one(server_api, it)
if result == "sent":
sent += 1
elif result == "drop":
continue
else: # keep:网络未恢复,本条及后续全部保留,等下一轮
remaining.append(it)
stopped = True
_save(remaining)
if sent:
logger.info(f"outbox 补发成功 {sent} 条,剩余 {pending_count()} 条")
return sent