小鱼开发
144 lines
4.6 KiB
Python
144 lines
4.6 KiB
Python
"""
|
|
作业注册表 - Redis 运行时状态读写
|
|
==================================
|
|
|
|
所有 running 作业的状态统一存储在 Redis 中，供 Scheduler Tick 读取、更新。
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
from datetime import UTC
|
|
from typing import Any
|
|
|
|
from redis.asyncio import Redis
|
|
|
|
from app.scheduler.models import JobRecord
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
KEY_RUNNING_SET = "scheduler:running_tasks"
|
|
|
|
|
|
def _job_key(job_id: str) -> str:
|
|
return f"job:{job_id}"
|
|
|
|
|
|
class JobRegistry:
|
|
"""基于 Redis 的作业注册表"""
|
|
|
|
def __init__(self, redis: Redis):
|
|
self.redis = redis
|
|
|
|
async def create(
|
|
self,
|
|
job_id: str,
|
|
job_type: str,
|
|
user_id: str,
|
|
status: str = "pending",
|
|
params: dict[str, Any] | None = None,
|
|
ttl: int = 86400,
|
|
) -> None:
|
|
"""创建新的作业记录"""
|
|
from datetime import datetime
|
|
|
|
data = {
|
|
"type": job_type,
|
|
"user_id": user_id,
|
|
"status": status,
|
|
"progress": "0",
|
|
"message": "等待执行...",
|
|
"completed": "0",
|
|
"total": "0",
|
|
"created_at": datetime.now(UTC).isoformat(),
|
|
}
|
|
if params:
|
|
data["params"] = json.dumps(params, ensure_ascii=False)
|
|
|
|
await self.redis.hset(_job_key(job_id), mapping=data)
|
|
await self.redis.expire(_job_key(job_id), ttl)
|
|
logger.debug(f"Registry created: {job_id}, type={job_type}")
|
|
|
|
async def update(self, job_id: str, **fields: Any) -> None:
|
|
"""更新作业字段"""
|
|
mapping: dict[str, str] = {}
|
|
for key, value in fields.items():
|
|
if isinstance(value, dict | list):
|
|
mapping[key] = json.dumps(value, ensure_ascii=False)
|
|
elif value is None:
|
|
mapping[key] = ""
|
|
else:
|
|
mapping[key] = str(value)
|
|
await self.redis.hset(_job_key(job_id), mapping=mapping)
|
|
logger.debug(f"Registry updated: {job_id}, fields={list(fields.keys())}")
|
|
|
|
async def get(self, job_id: str) -> JobRecord | None:
|
|
"""读取完整作业记录"""
|
|
data = await self.redis.hgetall(_job_key(job_id))
|
|
if not data:
|
|
return None
|
|
|
|
def _parse(key: str, raw: str) -> Any:
|
|
if key in ("result", "params") and raw:
|
|
try:
|
|
return json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
return raw
|
|
if key in ("progress", "completed", "total"):
|
|
try:
|
|
return int(raw)
|
|
except ValueError:
|
|
return 0
|
|
return raw
|
|
|
|
parsed = {k: _parse(k, v) for k, v in data.items()}
|
|
job_type = parsed.get("type", "")
|
|
params_raw = parsed.get("params", {})
|
|
params = params_raw if isinstance(params_raw, dict) else {}
|
|
|
|
return JobRecord(
|
|
job_id=job_id,
|
|
job_type=job_type,
|
|
user_id=parsed.get("user_id", ""),
|
|
project_id=str(params.get("project_id", "")),
|
|
status=parsed.get("status", "unknown"),
|
|
progress=parsed.get("progress", 0),
|
|
message=parsed.get("message", ""),
|
|
completed=parsed.get("completed", 0),
|
|
total=parsed.get("total", 0),
|
|
result=parsed.get("result", {}),
|
|
error=parsed.get("error"),
|
|
params=params,
|
|
created_at=parsed.get("created_at", ""),
|
|
)
|
|
|
|
async def add_running(self, job_id: str) -> None:
|
|
"""将作业标记为 running（加入全局 running 集合）"""
|
|
await self.redis.sadd(KEY_RUNNING_SET, job_id)
|
|
|
|
async def remove_running(self, job_id: str) -> None:
|
|
"""将作业从全局 running 集合移除"""
|
|
await self.redis.srem(KEY_RUNNING_SET, job_id)
|
|
|
|
async def get_running_job_ids(self) -> list[str]:
|
|
"""获取所有 running 的作业 ID 列表"""
|
|
members = await self.redis.smembers(KEY_RUNNING_SET)
|
|
return list(members)
|
|
|
|
async def list_running_by_user(self, user_id: str) -> list[JobRecord]:
|
|
"""获取指定用户的所有 running 作业"""
|
|
job_ids = await self.get_running_job_ids()
|
|
if not job_ids:
|
|
return []
|
|
|
|
results: list[JobRecord] = []
|
|
for job_id in job_ids:
|
|
job = await self.get(job_id)
|
|
if job and job.user_id == user_id:
|
|
results.append(job)
|
|
return results
|
|
|
|
async def delete(self, job_id: str) -> None:
|
|
"""删除作业记录"""
|
|
await self.redis.delete(_job_key(job_id))
|
|
await self.redis.srem(KEY_RUNNING_SET, job_id)
|