feat: 添加异步数据入库任务系统,支持全量/增量更新与实时进度展示

新增完整的异步任务管理体系:
1. 新增Redis缓存层实现任务状态实时存储与降级机制
2. 实现并发拉取+批量写入的高性能入库流程
3. 添加任务管理HTTP接口与路由
4. 重构行情控制器使用共享SQLite存储
5. 配套添加Docker Compose Redis启动配置
6. 前端新增任务管理面板支持操作与进度查看
This commit is contained in:
2026-08-18 10:09:52 +08:00
parent e28bfec957
commit 273fc90a95
18 changed files with 1682 additions and 80 deletions

View File

@@ -0,0 +1,126 @@
package sqlite
import (
"context"
"encoding/json"
"fmt"
"stock-analysis/internal/cache"
)
// 任务相关 SQL。
const (
saveTaskSQL = `
INSERT INTO ingest_task (id, type, status, total, done, success, failed, failed_codes, params, error, started_at, finished_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (id) DO UPDATE SET
type = excluded.type,
status = excluded.status,
total = excluded.total,
done = excluded.done,
success = excluded.success,
failed = excluded.failed,
failed_codes = excluded.failed_codes,
params = excluded.params,
error = excluded.error,
started_at = excluded.started_at,
finished_at = excluded.finished_at;`
updateTaskProgressSQL = `UPDATE ingest_task SET total = ?, done = ?, success = ?, failed = ?, failed_codes = ? WHERE id = ?;`
finishTaskSQL = `UPDATE ingest_task SET status = ?, error = ?, finished_at = ? WHERE id = ?;`
getTaskSQL = `SELECT id, type, status, total, done, success, failed, failed_codes, params, error, started_at, finished_at FROM ingest_task WHERE id = ?;`
listTasksSQL = `SELECT id, type, status, total, done, success, failed, failed_codes, params, error, started_at, finished_at FROM ingest_task ORDER BY started_at DESC LIMIT ?;`
)
// SaveTask 保存或更新任务(INSERT OR REPLACE 语义)。
func (s *Store) SaveTask(ctx context.Context, t *cache.Task) error {
failedCodes, err := json.Marshal(t.FailedCodes)
if err != nil {
return fmt.Errorf("marshal failed codes: %w", err)
}
_, err = s.db.ExecContext(ctx, saveTaskSQL,
t.ID, t.Type, t.Status, t.Total, t.Done, t.Success, t.Failed,
string(failedCodes), t.Params, t.Error, t.StartedAt, t.FinishedAt)
if err != nil {
return fmt.Errorf("save task %s: %w", t.ID, err)
}
return nil
}
// UpdateTaskProgress 更新任务进度(total/done/success/failed/failed_codes)。
func (s *Store) UpdateTaskProgress(ctx context.Context, id string, total, done, success, failed int, failedCodes []string) error {
data, err := json.Marshal(failedCodes)
if err != nil {
return fmt.Errorf("marshal failed codes: %w", err)
}
_, err = s.db.ExecContext(ctx, updateTaskProgressSQL, total, done, success, failed, string(data), id)
if err != nil {
return fmt.Errorf("update task progress %s: %w", id, err)
}
return nil
}
// FinishTask 标记任务终态(status/error/finished_at)。
func (s *Store) FinishTask(ctx context.Context, id string, status cache.TaskStatus, errMsg, finishedAt string) error {
_, err := s.db.ExecContext(ctx, finishTaskSQL, status, errMsg, finishedAt, id)
if err != nil {
return fmt.Errorf("finish task %s: %w", id, err)
}
return nil
}
// GetTask 按 ID 查询任务。
func (s *Store) GetTask(ctx context.Context, id string) (*cache.Task, error) {
row := s.db.QueryRowContext(ctx, getTaskSQL, id)
t, err := scanTask(row)
if err != nil {
return nil, fmt.Errorf("get task %s: %w", id, err)
}
return t, nil
}
// ListTasks 返回最近的任务列表(按开始时间降序)。
func (s *Store) ListTasks(ctx context.Context, limit int) ([]*cache.Task, error) {
if limit <= 0 {
limit = 50
}
rows, err := s.db.QueryContext(ctx, listTasksSQL, limit)
if err != nil {
return nil, fmt.Errorf("list tasks: %w", err)
}
defer rows.Close()
var tasks []*cache.Task
for rows.Next() {
t, err := scanTask(rows)
if err != nil {
return nil, err
}
tasks = append(tasks, t)
}
return tasks, rows.Err()
}
// rowScanner 抽象 *sql.Row 和 *sql.Rows 的 Scan 方法。
type rowScanner interface {
Scan(dest ...any) error
}
// scanTask 从查询行扫描一条任务记录。
func scanTask(row rowScanner) (*cache.Task, error) {
var (
t cache.Task
failedCodes string
)
if err := row.Scan(&t.ID, &t.Type, &t.Status, &t.Total, &t.Done, &t.Success, &t.Failed,
&failedCodes, &t.Params, &t.Error, &t.StartedAt, &t.FinishedAt); err != nil {
return nil, err
}
if failedCodes != "" && failedCodes != "null" {
_ = json.Unmarshal([]byte(failedCodes), &t.FailedCodes)
}
return &t, nil
}