背景:A6-BE-01..05 与 A6-INF-01,打通「提交 → 队列 → SSE 进度 → 完成/失败 → 取消 / 仅重试失败项」闭环,为 B3 / M1 的真实任务提供统一协议。 改动: - API media 模块:POST/GET /media-tasks、详情、SSE(Last-Event-ID 续推 + 15s 心跳 + 800ms 落库轮询)、取消(只影响未开始项)、仅重试失败项;Idempotency-Key 24h(异步提交 202;同 key 同 body 返回同一任务且不重复审计,不同 body 40902);范围外统一 404; - 队列骨架:BullMQ 5 九队列(5 组独立启停)、Outbox dispatcher(Redis 锁 + FOR UPDATE SKIP LOCKED + jobId 幂等 + 指数退避/死信)、worker 按 QUEUE 启停、demo 演练处理器(事务 + 行锁 + 取消/重试/幂等)、优雅退出与心跳; - 监控:GET /api/v1/health/queues、GET /api/v1/metrics(Prometheus 0.0.4)、infra/monitoring 告警规则与接入说明; - 通用:AuditInterceptor 支持幂等重放 skipAudit(默认行为不变)、AppModule 接线、.env.example 的 QUEUE 语义、bullmq 5.81.5 / worker @prisma/client / worker 覆盖率依赖。 影响面:OpenAPI 新增 6 个端点(openapi:diff 无破坏性变更);API 进程新增 outbox dispatcher 与 metrics/health 路由;worker 新增 BullMQ 消费;未改旧审计导出链路与既有接口。 回滚:回滚提交并停止 worker 新队列消费;task/outbox 新表数据不影响旧功能,可按需保留。 验证:api 单测 257、集成 92(连跑 2 次)、worker 27、shared 197;typecheck / lint / format:check / build / openapi:diff 全绿;本地真实 worker 启用 9 队列心跳,/health/queues、/metrics 实测,/media-tasks 未登录 401。
This commit is contained in:
1 parent
4a29cf815e
commit
633765f10f
70 files changed
+8674
-45
No files matched your search
@@ -0,0 +1,100 @@
|
||||
import {
|
||||
TASK_QUEUE_GROUPS,
|
||||
TASK_QUEUES,
|
||||
type TaskQueueGroup,
|
||||
type TaskQueueName,
|
||||
} from '@lingxi/shared';
|
||||
|
||||
/** 单个 worker 被分配到的队列定义(并发 / 重试参数全部来自 shared 的 TASK_QUEUES) */
|
||||
export interface WorkerQueueDefinition {
|
||||
name: TaskQueueName;
|
||||
group: TaskQueueGroup;
|
||||
concurrency: number;
|
||||
attempts: number;
|
||||
backoffDelayMs: number;
|
||||
}
|
||||
|
||||
/** `QUEUE=all`:启动全部队列(本地开发默认) */
|
||||
export const ALL_QUEUE_SELECTOR = 'all';
|
||||
|
||||
/**
|
||||
* 逻辑队列名 → BullMQ 物理队列名(BullMQ 禁止冒号),与 apps/api 的 toBullQueueName 保持一致。
|
||||
* 心跳 / 日志 / QUEUE 环境变量使用逻辑名,只有 BullMQ 实例使用物理名。
|
||||
*/
|
||||
export function toBullQueueName(name: TaskQueueName): string {
|
||||
return name.replaceAll(':', '-');
|
||||
}
|
||||
|
||||
export function isQueueName(value: string): value is TaskQueueName {
|
||||
return TASK_QUEUES.some((definition) => definition.name === value);
|
||||
}
|
||||
|
||||
export function isQueueGroup(value: string): value is TaskQueueGroup {
|
||||
return TASK_QUEUE_GROUPS.some((group) => group === value);
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析 QUEUE 环境变量:`all` / 分组名 / 队列名,逗号分隔(如 `QUEUE=verify`、`QUEUE=image,media:verify`)。
|
||||
* 非法值给出中文原因(含可用取值),避免 worker 静默空转。
|
||||
*/
|
||||
export function parseQueueSpec(spec: string): TaskQueueName[] {
|
||||
const tokens = spec
|
||||
.split(',')
|
||||
.map((token) => token.trim())
|
||||
.filter((token) => token.length > 0);
|
||||
|
||||
if (tokens.length === 0) {
|
||||
throw new Error('QUEUE 不能为空:请填 all、队列组名或队列名(如 media:verify)');
|
||||
}
|
||||
if (tokens.includes(ALL_QUEUE_SELECTOR)) {
|
||||
if (tokens.length > 1) {
|
||||
throw new Error('QUEUE=all 不能与其他分组 / 队列混用');
|
||||
}
|
||||
return TASK_QUEUES.map((definition) => definition.name);
|
||||
}
|
||||
|
||||
const selected: TaskQueueName[] = [];
|
||||
for (const token of tokens) {
|
||||
if (isQueueName(token)) {
|
||||
selected.push(token);
|
||||
continue;
|
||||
}
|
||||
if (isQueueGroup(token)) {
|
||||
const group = token;
|
||||
selected.push(
|
||||
...TASK_QUEUES.filter((definition) => definition.group === group).map(
|
||||
(definition) => definition.name,
|
||||
),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
throw new Error(
|
||||
`含未知队列 / 分组:${token}(可用:all、${TASK_QUEUE_GROUPS.join(' / ')} 或具体队列名)`,
|
||||
);
|
||||
}
|
||||
|
||||
return [...new Set(selected)];
|
||||
}
|
||||
|
||||
/** 把队列名展开为 worker 启动定义;WORKER_CONCURRENCY 覆盖全部队列并发 */
|
||||
export function buildQueueDefinitions(
|
||||
names: readonly string[],
|
||||
concurrencyOverride?: number,
|
||||
): WorkerQueueDefinition[] {
|
||||
return names.map((name) => {
|
||||
if (!isQueueName(name)) {
|
||||
throw new Error(`未登记的队列:${name}`);
|
||||
}
|
||||
const definition = TASK_QUEUES.find((item) => item.name === name);
|
||||
if (definition === undefined) {
|
||||
throw new Error(`未登记的队列:${name}`);
|
||||
}
|
||||
return {
|
||||
name,
|
||||
group: definition.group,
|
||||
concurrency: concurrencyOverride ?? definition.concurrency,
|
||||
attempts: definition.attempts,
|
||||
backoffDelayMs: definition.backoffDelayMs,
|
||||
};
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user