149 lines
5.0 KiB
Python
149 lines
5.0 KiB
Python
"""
|
||
本地投递 outbox(存储转发兜底)
|
||
====================================
|
||
上报先落盘再发送,防止网络抖动/云端不可用导致记录丢失:
|
||
|
||
- enqueue() 上报前先写盘,返回 entry_id
|
||
- ack() 发送成功后立即删除(常态下队列≈空,不积压)
|
||
- flush() 补发一轮:由心跳线程每 30s 驱动一次;按入队顺序发送,
|
||
网络未恢复时本轮立即停止,等下一轮,不做集中无效重试
|
||
|
||
防扩张约束:
|
||
- MAX_ITEMS 硬上限 1000 条,超出丢弃最旧并告警
|
||
- RETENTION_DAYS 保留期 7 天(与日志保留一致),过期丢弃
|
||
- 服务端按 user+job_id 幂等更新,重复补发不会产生重复记录
|
||
"""
|
||
import json
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from pathlib import Path
|
||
|
||
import requests
|
||
from loguru import logger
|
||
|
||
OUTBOX_FILE = Path("client/data/outbox.json")
|
||
MAX_ITEMS = 1000
|
||
RETENTION_DAYS = 7
|
||
|
||
_lock = threading.Lock()
|
||
|
||
|
||
def _load() -> list[dict]:
|
||
if not OUTBOX_FILE.exists():
|
||
return []
|
||
try:
|
||
return json.loads(OUTBOX_FILE.read_text(encoding="utf-8")) or []
|
||
except Exception:
|
||
logger.warning("outbox 文件读取失败,按空队列处理")
|
||
return []
|
||
|
||
|
||
def _save(items: list[dict]) -> None:
|
||
OUTBOX_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||
tmp = OUTBOX_FILE.with_suffix(".tmp")
|
||
tmp.write_text(json.dumps(items, ensure_ascii=False), encoding="utf-8")
|
||
tmp.replace(OUTBOX_FILE)
|
||
|
||
|
||
def enqueue(session_id: int | None, body: dict,
|
||
applied_delta: int = 0, skipped_delta: int = 0) -> str:
|
||
"""上报前写盘兜底,返回 entry_id;发送成功后调用 ack 删除"""
|
||
entry_id = uuid.uuid4().hex
|
||
with _lock:
|
||
items = _load()
|
||
items.append({
|
||
"id": entry_id,
|
||
"created_at": time.time(),
|
||
"session_id": session_id,
|
||
"body": body,
|
||
"applied_delta": applied_delta,
|
||
"skipped_delta": skipped_delta,
|
||
})
|
||
if len(items) > MAX_ITEMS:
|
||
dropped = len(items) - MAX_ITEMS
|
||
logger.warning(f"outbox 超 {MAX_ITEMS} 条上限,丢弃最旧 {dropped} 条")
|
||
items = items[-MAX_ITEMS:]
|
||
_save(items)
|
||
return entry_id
|
||
|
||
|
||
def ack(entry_id: str) -> None:
|
||
"""发送成功后从 outbox 删除"""
|
||
with _lock:
|
||
_save([i for i in _load() if i.get("id") != entry_id])
|
||
|
||
|
||
def pending_count() -> int:
|
||
with _lock:
|
||
return len(_load())
|
||
|
||
|
||
def _send_one(server_api, item: dict) -> str:
|
||
"""补发单条。返回:sent=成功 / keep=网络类失败保留 / drop=业务拒绝丢弃"""
|
||
body, sid = item["body"], item.get("session_id")
|
||
try:
|
||
server_api.report_application(sid, body)
|
||
except requests.HTTPError as e:
|
||
status = getattr(getattr(e, "response", None), "status_code", 0)
|
||
if 400 <= status < 500 and status not in (409, 429):
|
||
# 业务拒绝(422/401 等):去掉 session_id 再试一次,
|
||
# 仍被拒则为毒丸消息,丢弃避免卡住补发循环
|
||
try:
|
||
server_api.report_application(None, body)
|
||
except requests.HTTPError:
|
||
logger.warning(f"outbox 条目被服务端拒绝,丢弃:{body.get('job_id')}")
|
||
return "drop"
|
||
except Exception:
|
||
return "keep"
|
||
else:
|
||
return "keep"
|
||
except Exception:
|
||
return "keep"
|
||
|
||
# 补发成功:会话统计也补上(best-effort)
|
||
if item.get("session_id") and (item.get("applied_delta") or item.get("skipped_delta")):
|
||
try:
|
||
server_api.update_session_stats(
|
||
item["session_id"],
|
||
item.get("applied_delta", 0), item.get("skipped_delta", 0))
|
||
except Exception as e:
|
||
logger.debug(f"outbox 补发会话统计失败:{e}")
|
||
return "sent"
|
||
|
||
|
||
def flush(server_api) -> int:
|
||
"""补发一轮,返回成功条数。网络未恢复时本轮提前结束等下一轮"""
|
||
sent = 0
|
||
with _lock:
|
||
items = _load()
|
||
if not items:
|
||
return 0
|
||
|
||
# 保留期清理
|
||
cutoff = time.time() - RETENTION_DAYS * 86400
|
||
expired = [i for i in items if i.get("created_at", 0) < cutoff]
|
||
if expired:
|
||
logger.warning(f"outbox 超保留期({RETENTION_DAYS} 天),丢弃 {len(expired)} 条")
|
||
items = [i for i in items if i.get("created_at", 0) >= cutoff]
|
||
|
||
remaining: list[dict] = []
|
||
stopped = False
|
||
for it in items:
|
||
if stopped:
|
||
remaining.append(it)
|
||
continue
|
||
result = _send_one(server_api, it)
|
||
if result == "sent":
|
||
sent += 1
|
||
elif result == "drop":
|
||
continue
|
||
else: # keep:网络未恢复,本条及后续全部保留,等下一轮
|
||
remaining.append(it)
|
||
stopped = True
|
||
_save(remaining)
|
||
|
||
if sent:
|
||
logger.info(f"outbox 补发成功 {sent} 条,剩余 {pending_count()} 条")
|
||
return sent
|