feat: 添加异步数据入库任务系统，支持全量/增量更新与实时进度展示

新增完整的异步任务管理体系： 1. 新增Redis缓存层实现任务状态实时存储与降级机制 2. 实现并发拉取+批量写入的高性能入库流程 3. 添加任务管理HTTP接口与路由 4. 重构行情控制器使用共享SQLite存储 5. 配套添加Docker Compose Redis启动配置 6. 前端新增任务管理面板支持操作与进度查看
2026-08-18 10:09:52 +08:00
parent e28bfec957
commit 273fc90a95
18 changed files with 1682 additions and 80 deletions
--- a/internal/storage/sqlite/store.go
+++ b/internal/storage/sqlite/store.go
@@ -80,6 +80,23 @@ CREATE TABLE IF NOT EXISTS convert_price_history (
 		    created_at TEXT    NOT NULL DEFAULT CURRENT_TIMESTAMP
 		);
 		CREATE INDEX IF NOT EXISTS idx_verify_code_email ON verify_code (email, purpose);
+
+		CREATE TABLE IF NOT EXISTS ingest_task (
+		    id           TEXT PRIMARY KEY,
+		    type         TEXT NOT NULL,
+		    status       TEXT NOT NULL,
+		    total        INTEGER NOT NULL DEFAULT 0,
+		    done         INTEGER NOT NULL DEFAULT 0,
+		    success      INTEGER NOT NULL DEFAULT 0,
+		    failed       INTEGER NOT NULL DEFAULT 0,
+		    failed_codes TEXT NOT NULL DEFAULT '',
+		    params       TEXT NOT NULL DEFAULT '',
+		    error        TEXT NOT NULL DEFAULT '',
+		    started_at   TEXT NOT NULL,
+		    finished_at  TEXT NOT NULL DEFAULT ''
+		);
+		CREATE INDEX IF NOT EXISTS idx_ingest_task_status ON ingest_task (status);
+		CREATE INDEX IF NOT EXISTS idx_ingest_task_started ON ingest_task (started_at);
 		`

 // 通用 SQL：幂等写入（存在则跳过）。
@@ -90,8 +107,8 @@ ON CONFLICT (code, date) DO NOTHING;
 `

 const (
-	querySQL   = `SELECT code, date, open, close, high, low, volume, amount, convert_price, convert_value, premium_rate FROM convertible_kline WHERE code = ? ORDER BY date;`
-	countSQL   = `SELECT COUNT(*) FROM convertible_kline WHERE code = ?;`
+	querySQL    = `SELECT code, date, open, close, high, low, volume, amount, convert_price, convert_value, premium_rate FROM convertible_kline WHERE code = ? ORDER BY date;`
+	countSQL    = `SELECT COUNT(*) FROM convertible_kline WHERE code = ?;`
 	lastDateSQL = `SELECT MAX(date) FROM convertible_kline WHERE code = ?;`
 	allKlineSQL = `SELECT code, date, open, close, high, low, volume, amount, convert_price, convert_value, premium_rate FROM convertible_kline ORDER BY code, date;`

@@ -148,9 +165,13 @@ type PremiumRow struct {

 // Store 是 SQLite 存储实现，实现 storage.KlineStore。
 type Store struct {
-	db *sql.DB
+	db   *sql.DB
+	path string
 }

+// Path 返回数据库文件路径。
+func (s *Store) Path() string { return s.path }
+
 // Open 打开（或创建）SQLite 数据库文件并初始化表结构。
 func Open(path string) (*Store, error) {
 	db, err := sql.Open("sqlite", path)
@@ -187,7 +208,7 @@ func Open(path string) (*Store, error) {
 		db.Close()
 		return nil, fmt.Errorf("set busy timeout: %w", err)
 	}
-	return &Store{db: db}, nil
+	return &Store{db: db, path: path}, nil
 }

 // initSchema 执行建表语句。
@@ -354,6 +375,74 @@ func (s *Store) ListCodes(ctx context.Context) ([]string, error) {
 	return codes, rows.Err()
 }

+// LastKlineDates 一次性返回所有已入库 code 的最新K线日期（YYYY-MM-DD）。
+// 用于增量同步预加载，替代逐只 LastKlineDate 查询，避免占用唯一写连接。
+func (s *Store) LastKlineDates(ctx context.Context) (map[string]string, error) {
+	rows, err := s.db.QueryContext(ctx, "SELECT code, MAX(date) FROM convertible_kline GROUP BY code")
+	if err != nil {
+		return nil, fmt.Errorf("last kline dates: %w", err)
+	}
+	defer rows.Close()
+
+	out := make(map[string]string)
+	for rows.Next() {
+		var code, date string
+		if err := rows.Scan(&code, &date); err != nil {
+			return nil, fmt.Errorf("scan last date: %w", err)
+		}
+		out[code] = date
+	}
+	return out, rows.Err()
+}
+
+// SaveKlinesBatch 单事务批量写入多只转债K线（collector 调用）。
+// 接收 map[转债代码]K线切片，在一个事务内遍历写入，带 3 次重试。
+func (s *Store) SaveKlinesBatch(ctx context.Context, batch map[string][]model.Kline) error {
+	if len(batch) == 0 {
+		return nil
+	}
+	const maxAttempt = 3
+	var lastErr error
+	for attempt := 0; attempt < maxAttempt; attempt++ {
+		if attempt > 0 {
+			time.Sleep(2 * time.Second)
+		}
+		if err := s.saveBatchOnce(ctx, batch); err != nil {
+			lastErr = err
+			continue
+		}
+		return nil
+	}
+	return lastErr
+}
+
+// saveBatchOnce 单次执行批量写入事务。
+func (s *Store) saveBatchOnce(ctx context.Context, batch map[string][]model.Kline) error {
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("begin tx: %w", err)
+	}
+	defer tx.Rollback() //nolint:errcheck
+
+	stmt, err := tx.PrepareContext(ctx, upsertSQL)
+	if err != nil {
+		return fmt.Errorf("prepare upsert: %w", err)
+	}
+	defer stmt.Close()
+
+	for code, klines := range batch {
+		for _, k := range klines {
+			if _, err := stmt.ExecContext(ctx, code, k.Date, k.Open, k.Close, k.High, k.Low, k.Volume, k.Amount, k.ConvertPrice, k.ConvertValue, k.PremiumRate); err != nil {
+				return fmt.Errorf("insert kline %s %s: %w", code, k.Date, err)
+			}
+		}
+	}
+	if err := tx.Commit(); err != nil {
+		return fmt.Errorf("commit tx: %w", err)
+	}
+	return nil
+}
+
 // LoadAllKlines 一次性加载全部标的的K线（按代码、日期升序）。
 // 用于回测等需要全量数据在内存的场景。
 func (s *Store) LoadAllKlines(ctx context.Context) (map[string][]model.Kline, error) {
--- a/internal/storage/sqlite/task.go
+++ b/internal/storage/sqlite/task.go
@@ -0,0 +1,126 @@
+package sqlite
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"stock-analysis/internal/cache"
+)
+
+// 任务相关 SQL。
+const (
+	saveTaskSQL = `
+INSERT INTO ingest_task (id, type, status, total, done, success, failed, failed_codes, params, error, started_at, finished_at)
+VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ON CONFLICT (id) DO UPDATE SET
+    type = excluded.type,
+    status = excluded.status,
+    total = excluded.total,
+    done = excluded.done,
+    success = excluded.success,
+    failed = excluded.failed,
+    failed_codes = excluded.failed_codes,
+    params = excluded.params,
+    error = excluded.error,
+    started_at = excluded.started_at,
+    finished_at = excluded.finished_at;`
+
+	updateTaskProgressSQL = `UPDATE ingest_task SET total = ?, done = ?, success = ?, failed = ?, failed_codes = ? WHERE id = ?;`
+
+	finishTaskSQL = `UPDATE ingest_task SET status = ?, error = ?, finished_at = ? WHERE id = ?;`
+
+	getTaskSQL = `SELECT id, type, status, total, done, success, failed, failed_codes, params, error, started_at, finished_at FROM ingest_task WHERE id = ?;`
+
+	listTasksSQL = `SELECT id, type, status, total, done, success, failed, failed_codes, params, error, started_at, finished_at FROM ingest_task ORDER BY started_at DESC LIMIT ?;`
+)
+
+// SaveTask 保存或更新任务（INSERT OR REPLACE 语义）。
+func (s *Store) SaveTask(ctx context.Context, t *cache.Task) error {
+	failedCodes, err := json.Marshal(t.FailedCodes)
+	if err != nil {
+		return fmt.Errorf("marshal failed codes: %w", err)
+	}
+	_, err = s.db.ExecContext(ctx, saveTaskSQL,
+		t.ID, t.Type, t.Status, t.Total, t.Done, t.Success, t.Failed,
+		string(failedCodes), t.Params, t.Error, t.StartedAt, t.FinishedAt)
+	if err != nil {
+		return fmt.Errorf("save task %s: %w", t.ID, err)
+	}
+	return nil
+}
+
+// UpdateTaskProgress 更新任务进度（total/done/success/failed/failed_codes）。
+func (s *Store) UpdateTaskProgress(ctx context.Context, id string, total, done, success, failed int, failedCodes []string) error {
+	data, err := json.Marshal(failedCodes)
+	if err != nil {
+		return fmt.Errorf("marshal failed codes: %w", err)
+	}
+	_, err = s.db.ExecContext(ctx, updateTaskProgressSQL, total, done, success, failed, string(data), id)
+	if err != nil {
+		return fmt.Errorf("update task progress %s: %w", id, err)
+	}
+	return nil
+}
+
+// FinishTask 标记任务终态（status/error/finished_at）。
+func (s *Store) FinishTask(ctx context.Context, id string, status cache.TaskStatus, errMsg, finishedAt string) error {
+	_, err := s.db.ExecContext(ctx, finishTaskSQL, status, errMsg, finishedAt, id)
+	if err != nil {
+		return fmt.Errorf("finish task %s: %w", id, err)
+	}
+	return nil
+}
+
+// GetTask 按 ID 查询任务。
+func (s *Store) GetTask(ctx context.Context, id string) (*cache.Task, error) {
+	row := s.db.QueryRowContext(ctx, getTaskSQL, id)
+	t, err := scanTask(row)
+	if err != nil {
+		return nil, fmt.Errorf("get task %s: %w", id, err)
+	}
+	return t, nil
+}
+
+// ListTasks 返回最近的任务列表（按开始时间降序）。
+func (s *Store) ListTasks(ctx context.Context, limit int) ([]*cache.Task, error) {
+	if limit <= 0 {
+		limit = 50
+	}
+	rows, err := s.db.QueryContext(ctx, listTasksSQL, limit)
+	if err != nil {
+		return nil, fmt.Errorf("list tasks: %w", err)
+	}
+	defer rows.Close()
+
+	var tasks []*cache.Task
+	for rows.Next() {
+		t, err := scanTask(rows)
+		if err != nil {
+			return nil, err
+		}
+		tasks = append(tasks, t)
+	}
+	return tasks, rows.Err()
+}
+
+// rowScanner 抽象 *sql.Row 和 *sql.Rows 的 Scan 方法。
+type rowScanner interface {
+	Scan(dest ...any) error
+}
+
+// scanTask 从查询行扫描一条任务记录。
+func scanTask(row rowScanner) (*cache.Task, error) {
+	var (
+		t           cache.Task
+		failedCodes string
+	)
+	if err := row.Scan(&t.ID, &t.Type, &t.Status, &t.Total, &t.Done, &t.Success, &t.Failed,
+		&failedCodes, &t.Params, &t.Error, &t.StartedAt, &t.FinishedAt); err != nil {
+		return nil, err
+	}
+	if failedCodes != "" && failedCodes != "null" {
+		_ = json.Unmarshal([]byte(failedCodes), &t.FailedCodes)
+	}
+	return &t, nil
+}