feat: 添加异步数据入库任务系统,支持全量/增量更新与实时进度展示

新增完整的异步任务管理体系:
1. 新增Redis缓存层实现任务状态实时存储与降级机制
2. 实现并发拉取+批量写入的高性能入库流程
3. 添加任务管理HTTP接口与路由
4. 重构行情控制器使用共享SQLite存储
5. 配套添加Docker Compose Redis启动配置
6. 前端新增任务管理面板支持操作与进度查看
This commit is contained in:
2026-08-18 10:09:52 +08:00
parent e28bfec957
commit 273fc90a95
18 changed files with 1682 additions and 80 deletions

View File

@@ -308,6 +308,64 @@
font-size: 13px;
}
/* ============ Task Status Badges ============ */
.task-badge {
display: inline-block;
padding: 3px 10px;
border-radius: 12px;
font-size: 11px;
font-weight: 600;
letter-spacing: 0.3px;
white-space: nowrap;
}
.task-badge.pending { background: #FFF3CD; color: #856404; }
.task-badge.running { background: #D6E4FF; color: #1E5FBB; }
.task-badge.success { background: #D4EDDA; color: #155724; }
.task-badge.failed { background: #F8D7DA; color: #721C24; }
.task-badge.cancelled { background: #E2E3E5; color: #383D41; }
.task-badge.running::before {
content: '';
display: inline-block;
width: 6px;
height: 6px;
border-radius: 50%;
background: #1E5FBB;
margin-right: 5px;
animation: pulse 1s ease-in-out infinite;
}
@keyframes pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.3; }
}
.progress-bar {
display: inline-block;
width: 80px;
height: 6px;
background: var(--gray-200);
border-radius: 3px;
vertical-align: middle;
margin: 0 6px;
overflow: hidden;
}
.progress-bar-fill {
height: 100%;
background: var(--blue);
border-radius: 3px;
transition: width 0.3s ease;
}
.btn-cancel {
padding: 4px 10px;
border: 1px solid var(--red);
background: transparent;
color: var(--red);
border-radius: var(--radius-sm);
font-size: 12px;
cursor: pointer;
transition: all var(--transition);
}
.btn-cancel:hover { background: var(--red); color: var(--white); }
.btn-cancel:disabled { opacity: 0.4; cursor: not-allowed; }
/* ============ Responsive ============ */
@media (max-width: 768px) {
.header { padding: 16px 20px; }
@@ -484,6 +542,59 @@
</div>
<div id="data-error" style="display:none"></div>
<div class="status-grid" id="data-cards"></div>
<!-- 任务控制 -->
<div class="section-card" style="margin-top:16px" data-section="task-control">
<div class="section-header">
<span class="section-title">数据更新任务</span>
<button type="button" class="section-toggle" aria-label="折叠">−</button>
</div>
<div class="section-body">
<div class="toolbar" style="margin-top:0">
<button class="btn-primary" id="task-full">全量更新</button>
<button class="btn-primary" id="task-incremental">增量更新</button>
<input type="date" id="task-since" title="增量起始日期(留空自动取最后K线日期)">
<input type="number" id="task-limit" placeholder="限制数量" min="0" style="width:120px" title="限制处理数量(0=全部)">
<span id="task-create-status"></span>
</div>
<div style="font-size:12px;color:var(--gray-500);margin-top:4px;line-height:1.6">
<b>全量更新</b>:拉取所有转债的历史K线(已入库的自动跳过)<br>
<b>增量更新</b>:仅同步已入库转债的最新数据(自上次入库日期起)
</div>
</div>
</div>
<!-- 任务列表 -->
<div class="section-card" data-section="task-list">
<div class="section-header">
<span class="section-title">任务列表</span>
<button type="button" class="section-toggle" aria-label="折叠">−</button>
</div>
<div class="section-body">
<div class="toolbar" style="margin-top:0">
<button class="btn-default" id="task-refresh">刷新任务</button>
<span id="task-list-status" style="font-size:13px;color:var(--gray-500)"></span>
</div>
<div class="table-wrap" style="max-height:480px;overflow-y:auto;margin-bottom:0">
<table id="task-table">
<thead>
<tr>
<th>任务ID</th>
<th>类型</th>
<th>状态</th>
<th>进度</th>
<th>成功</th>
<th>失败</th>
<th>开始时间</th>
<th>完成时间</th>
<th>操作</th>
</tr>
</thead>
<tbody></tbody>
</table>
</div>
</div>
</div>
</div>
</div>
@@ -559,7 +670,7 @@
document.querySelectorAll('.tab-content').forEach(c => c.classList.remove('active'));
tab.classList.add('active');
document.getElementById('tab-' + tab.dataset.tab).classList.add('active');
if (tab.dataset.tab === 'data') loadDataStatus();
if (tab.dataset.tab === 'data') { loadDataStatus(); loadTasks(); }
});
});
@@ -956,6 +1067,170 @@
document.getElementById('data-refresh').addEventListener('click', loadDataStatus);
// ============ 数据更新任务 ============
let taskPollTimer = null;
// 任务类型中文映射
const taskTypeLabel = { full: '全量更新', incremental: '增量更新' };
// 任务状态中文映射
const taskStatusLabel = {
pending: '等待中', running: '运行中', success: '成功', failed: '失败', cancelled: '已取消'
};
// 创建任务
async function createTask(type) {
const statusEl = document.getElementById('task-create-status');
const btnId = type === 'full' ? 'task-full' : 'task-incremental';
const btn = document.getElementById(btnId);
const since = document.getElementById('task-since').value;
const limit = +document.getElementById('task-limit').value || 0;
btn.disabled = true;
statusEl.textContent = '提交中...';
statusEl.style.color = 'var(--blue)';
try {
const body = { type, since, limit };
const resp = await apiFetch('/api/tasks', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body),
});
if (!resp.ok) {
const err = await resp.json().catch(() => ({}));
throw new Error(err.error || 'HTTP ' + resp.status);
}
const task = await resp.json();
showToast(`${taskTypeLabel[type]}任务已创建`, 'success');
statusEl.textContent = '';
// 立即刷新任务列表并启动轮询
await loadTasks();
startTaskPolling();
// 切换到任务列表区域
document.querySelector('[data-section="task-list"] .section-header').scrollIntoView({ behavior: 'smooth', block: 'start' });
} catch (e) {
showToast('创建任务失败: ' + e.message, 'error');
statusEl.textContent = '× ' + e.message;
statusEl.style.color = 'var(--red)';
} finally {
btn.disabled = false;
}
}
// 加载任务列表
async function loadTasks() {
const statusEl = document.getElementById('task-list-status');
const tbody = document.querySelector('#task-table tbody');
statusEl.textContent = '加载中...';
try {
const resp = await apiFetch('/api/tasks?limit=50');
if (!resp.ok) throw new Error('HTTP ' + resp.status);
const tasks = await resp.json();
renderTasks(tasks || []);
const running = (tasks || []).filter(t => t.status === 'running' || t.status === 'pending').length;
statusEl.textContent = `共 ${tasks.length} 条${running > 0 ? `(${running} 个运行中)` : ''}`;
// 有运行中任务时继续轮询,否则停止
if (running > 0) {
startTaskPolling();
} else {
stopTaskPolling();
}
} catch (e) {
tbody.innerHTML = `<tr><td colspan="9" style="text-align:center;padding:20px;color:var(--red)">加载失败: ${e.message}</td></tr>`;
statusEl.textContent = '加载失败';
}
}
// 渲染任务列表
function renderTasks(tasks) {
const tbody = document.querySelector('#task-table tbody');
if (tasks.length === 0) {
tbody.innerHTML = '<tr><td colspan="9" style="text-align:center;padding:24px;color:#888">暂无任务记录</td></tr>';
return;
}
tbody.innerHTML = tasks.map(t => {
const pct = t.total > 0 ? Math.round(t.done / t.total * 100) : 0;
const isActive = t.status === 'running' || t.status === 'pending';
const statusText = taskStatusLabel[t.status] || t.status;
const shortId = t.id.substring(0, 8);
const started = formatTime(t.startedAt);
const finished = t.finishedAt ? formatTime(t.finishedAt) : '-';
const failInfo = t.failed > 0 && t.failedCodes && t.failedCodes.length > 0
? ` title="失败代码: ${t.failedCodes.slice(0, 10).join(', ')}${t.failedCodes.length > 10 ? '...' : ''}"`
: '';
const errorInfo = t.error ? ` title="${t.error.replace(/"/g, '&quot;')}"` : '';
return `<tr>
<td style="font-family:monospace;font-size:12px" title="${t.id}">${shortId}</td>
<td>${taskTypeLabel[t.type] || t.type}</td>
<td><span class="task-badge ${t.status}"${errorInfo}>${statusText}</span></td>
<td>
<span style="font-size:12px">${t.done}/${t.total}</span>
<span class="progress-bar"><span class="progress-bar-fill" style="width:${pct}%"></span></span>
<span style="font-size:11px;color:var(--gray-500)">${pct}%</span>
</td>
<td style="color:var(--green);font-weight:600">${t.success}</td>
<td style="color:var(--red);font-weight:600"${failInfo}>${t.failed}</td>
<td style="font-size:12px">${started}</td>
<td style="font-size:12px">${finished}</td>
<td>${isActive
? `<button class="btn-cancel" onclick="cancelTask('${t.id}')">取消</button>`
: '<span style="color:var(--gray-500);font-size:12px">-</span>'}</td>
</tr>`;
}).join('');
}
// 取消任务
async function cancelTask(id) {
if (!confirm('确定取消该任务?')) return;
try {
const resp = await apiFetch(`/api/tasks/${id}/cancel`, { method: 'POST' });
if (!resp.ok) {
const err = await resp.json().catch(() => ({}));
throw new Error(err.error || 'HTTP ' + resp.status);
}
showToast('任务取消请求已发送', 'info');
await loadTasks();
} catch (e) {
showToast('取消失败: ' + e.message, 'error');
}
}
// 暴露到全局供 inline onclick 调用
window.cancelTask = cancelTask;
// 格式化时间:RFC3339 → 可读格式
function formatTime(s) {
if (!s) return '-';
const d = new Date(s);
if (isNaN(d)) return s;
const pad = n => String(n).padStart(2, '0');
return `${d.getFullYear()}-${pad(d.getMonth()+1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}`;
}
// 轮询:运行中任务时每 3 秒刷新一次
function startTaskPolling() {
if (taskPollTimer) return; // 已在轮询
taskPollTimer = setInterval(loadTasks, 3000);
}
function stopTaskPolling() {
if (taskPollTimer) {
clearInterval(taskPollTimer);
taskPollTimer = null;
}
}
document.getElementById('task-full').addEventListener('click', () => createTask('full'));
document.getElementById('task-incremental').addEventListener('click', () => createTask('incremental'));
document.getElementById('task-refresh').addEventListener('click', loadTasks);
// 数据管理Tab的折叠/展开
document.querySelectorAll('#tab-data .section-header').forEach(header => {
header.addEventListener('click', e => {
if (e.target.closest('.section-toggle')) e.stopPropagation();
const card = header.closest('.section-card');
const collapsed = card.classList.toggle('collapsed');
header.querySelector('.section-toggle').textContent = collapsed ? '+' : '−';
});
});
// 初始化
loadBonds();
</script>