feat: 添加异步数据入库任务系统,支持全量/增量更新与实时进度展示
新增完整的异步任务管理体系: 1. 新增Redis缓存层实现任务状态实时存储与降级机制 2. 实现并发拉取+批量写入的高性能入库流程 3. 添加任务管理HTTP接口与路由 4. 重构行情控制器使用共享SQLite存储 5. 配套添加Docker Compose Redis启动配置 6. 前端新增任务管理面板支持操作与进度查看
This commit is contained in:
126
internal/storage/sqlite/task.go
Normal file
126
internal/storage/sqlite/task.go
Normal file
@@ -0,0 +1,126 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"stock-analysis/internal/cache"
|
||||
)
|
||||
|
||||
// 任务相关 SQL。
|
||||
const (
|
||||
saveTaskSQL = `
|
||||
INSERT INTO ingest_task (id, type, status, total, done, success, failed, failed_codes, params, error, started_at, finished_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
type = excluded.type,
|
||||
status = excluded.status,
|
||||
total = excluded.total,
|
||||
done = excluded.done,
|
||||
success = excluded.success,
|
||||
failed = excluded.failed,
|
||||
failed_codes = excluded.failed_codes,
|
||||
params = excluded.params,
|
||||
error = excluded.error,
|
||||
started_at = excluded.started_at,
|
||||
finished_at = excluded.finished_at;`
|
||||
|
||||
updateTaskProgressSQL = `UPDATE ingest_task SET total = ?, done = ?, success = ?, failed = ?, failed_codes = ? WHERE id = ?;`
|
||||
|
||||
finishTaskSQL = `UPDATE ingest_task SET status = ?, error = ?, finished_at = ? WHERE id = ?;`
|
||||
|
||||
getTaskSQL = `SELECT id, type, status, total, done, success, failed, failed_codes, params, error, started_at, finished_at FROM ingest_task WHERE id = ?;`
|
||||
|
||||
listTasksSQL = `SELECT id, type, status, total, done, success, failed, failed_codes, params, error, started_at, finished_at FROM ingest_task ORDER BY started_at DESC LIMIT ?;`
|
||||
)
|
||||
|
||||
// SaveTask 保存或更新任务(INSERT OR REPLACE 语义)。
|
||||
func (s *Store) SaveTask(ctx context.Context, t *cache.Task) error {
|
||||
failedCodes, err := json.Marshal(t.FailedCodes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal failed codes: %w", err)
|
||||
}
|
||||
_, err = s.db.ExecContext(ctx, saveTaskSQL,
|
||||
t.ID, t.Type, t.Status, t.Total, t.Done, t.Success, t.Failed,
|
||||
string(failedCodes), t.Params, t.Error, t.StartedAt, t.FinishedAt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("save task %s: %w", t.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateTaskProgress 更新任务进度(total/done/success/failed/failed_codes)。
|
||||
func (s *Store) UpdateTaskProgress(ctx context.Context, id string, total, done, success, failed int, failedCodes []string) error {
|
||||
data, err := json.Marshal(failedCodes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal failed codes: %w", err)
|
||||
}
|
||||
_, err = s.db.ExecContext(ctx, updateTaskProgressSQL, total, done, success, failed, string(data), id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update task progress %s: %w", id, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FinishTask 标记任务终态(status/error/finished_at)。
|
||||
func (s *Store) FinishTask(ctx context.Context, id string, status cache.TaskStatus, errMsg, finishedAt string) error {
|
||||
_, err := s.db.ExecContext(ctx, finishTaskSQL, status, errMsg, finishedAt, id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("finish task %s: %w", id, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetTask 按 ID 查询任务。
|
||||
func (s *Store) GetTask(ctx context.Context, id string) (*cache.Task, error) {
|
||||
row := s.db.QueryRowContext(ctx, getTaskSQL, id)
|
||||
t, err := scanTask(row)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get task %s: %w", id, err)
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// ListTasks 返回最近的任务列表(按开始时间降序)。
|
||||
func (s *Store) ListTasks(ctx context.Context, limit int) ([]*cache.Task, error) {
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, listTasksSQL, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list tasks: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var tasks []*cache.Task
|
||||
for rows.Next() {
|
||||
t, err := scanTask(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tasks = append(tasks, t)
|
||||
}
|
||||
return tasks, rows.Err()
|
||||
}
|
||||
|
||||
// rowScanner 抽象 *sql.Row 和 *sql.Rows 的 Scan 方法。
|
||||
type rowScanner interface {
|
||||
Scan(dest ...any) error
|
||||
}
|
||||
|
||||
// scanTask 从查询行扫描一条任务记录。
|
||||
func scanTask(row rowScanner) (*cache.Task, error) {
|
||||
var (
|
||||
t cache.Task
|
||||
failedCodes string
|
||||
)
|
||||
if err := row.Scan(&t.ID, &t.Type, &t.Status, &t.Total, &t.Done, &t.Success, &t.Failed,
|
||||
&failedCodes, &t.Params, &t.Error, &t.StartedAt, &t.FinishedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if failedCodes != "" && failedCodes != "null" {
|
||||
_ = json.Unmarshal([]byte(failedCodes), &t.FailedCodes)
|
||||
}
|
||||
return &t, nil
|
||||
}
|
||||
Reference in New Issue
Block a user