feat: 添加异步数据入库任务系统,支持全量/增量更新与实时进度展示
新增完整的异步任务管理体系: 1. 新增Redis缓存层实现任务状态实时存储与降级机制 2. 实现并发拉取+批量写入的高性能入库流程 3. 添加任务管理HTTP接口与路由 4. 重构行情控制器使用共享SQLite存储 5. 配套添加Docker Compose Redis启动配置 6. 前端新增任务管理面板支持操作与进度查看
This commit is contained in:
@@ -80,6 +80,23 @@ CREATE TABLE IF NOT EXISTS convert_price_history (
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_verify_code_email ON verify_code (email, purpose);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ingest_task (
|
||||
id TEXT PRIMARY KEY,
|
||||
type TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
total INTEGER NOT NULL DEFAULT 0,
|
||||
done INTEGER NOT NULL DEFAULT 0,
|
||||
success INTEGER NOT NULL DEFAULT 0,
|
||||
failed INTEGER NOT NULL DEFAULT 0,
|
||||
failed_codes TEXT NOT NULL DEFAULT '',
|
||||
params TEXT NOT NULL DEFAULT '',
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
started_at TEXT NOT NULL,
|
||||
finished_at TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_ingest_task_status ON ingest_task (status);
|
||||
CREATE INDEX IF NOT EXISTS idx_ingest_task_started ON ingest_task (started_at);
|
||||
`
|
||||
|
||||
// 通用 SQL:幂等写入(存在则跳过)。
|
||||
@@ -90,8 +107,8 @@ ON CONFLICT (code, date) DO NOTHING;
|
||||
`
|
||||
|
||||
const (
|
||||
querySQL = `SELECT code, date, open, close, high, low, volume, amount, convert_price, convert_value, premium_rate FROM convertible_kline WHERE code = ? ORDER BY date;`
|
||||
countSQL = `SELECT COUNT(*) FROM convertible_kline WHERE code = ?;`
|
||||
querySQL = `SELECT code, date, open, close, high, low, volume, amount, convert_price, convert_value, premium_rate FROM convertible_kline WHERE code = ? ORDER BY date;`
|
||||
countSQL = `SELECT COUNT(*) FROM convertible_kline WHERE code = ?;`
|
||||
lastDateSQL = `SELECT MAX(date) FROM convertible_kline WHERE code = ?;`
|
||||
allKlineSQL = `SELECT code, date, open, close, high, low, volume, amount, convert_price, convert_value, premium_rate FROM convertible_kline ORDER BY code, date;`
|
||||
|
||||
@@ -148,9 +165,13 @@ type PremiumRow struct {
|
||||
|
||||
// Store 是 SQLite 存储实现,实现 storage.KlineStore。
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
db *sql.DB
|
||||
path string
|
||||
}
|
||||
|
||||
// Path 返回数据库文件路径。
|
||||
func (s *Store) Path() string { return s.path }
|
||||
|
||||
// Open 打开(或创建)SQLite 数据库文件并初始化表结构。
|
||||
func Open(path string) (*Store, error) {
|
||||
db, err := sql.Open("sqlite", path)
|
||||
@@ -187,7 +208,7 @@ func Open(path string) (*Store, error) {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("set busy timeout: %w", err)
|
||||
}
|
||||
return &Store{db: db}, nil
|
||||
return &Store{db: db, path: path}, nil
|
||||
}
|
||||
|
||||
// initSchema 执行建表语句。
|
||||
@@ -354,6 +375,74 @@ func (s *Store) ListCodes(ctx context.Context) ([]string, error) {
|
||||
return codes, rows.Err()
|
||||
}
|
||||
|
||||
// LastKlineDates 一次性返回所有已入库 code 的最新K线日期(YYYY-MM-DD)。
|
||||
// 用于增量同步预加载,替代逐只 LastKlineDate 查询,避免占用唯一写连接。
|
||||
func (s *Store) LastKlineDates(ctx context.Context) (map[string]string, error) {
|
||||
rows, err := s.db.QueryContext(ctx, "SELECT code, MAX(date) FROM convertible_kline GROUP BY code")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("last kline dates: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make(map[string]string)
|
||||
for rows.Next() {
|
||||
var code, date string
|
||||
if err := rows.Scan(&code, &date); err != nil {
|
||||
return nil, fmt.Errorf("scan last date: %w", err)
|
||||
}
|
||||
out[code] = date
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SaveKlinesBatch 单事务批量写入多只转债K线(collector 调用)。
|
||||
// 接收 map[转债代码]K线切片,在一个事务内遍历写入,带 3 次重试。
|
||||
func (s *Store) SaveKlinesBatch(ctx context.Context, batch map[string][]model.Kline) error {
|
||||
if len(batch) == 0 {
|
||||
return nil
|
||||
}
|
||||
const maxAttempt = 3
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxAttempt; attempt++ {
|
||||
if attempt > 0 {
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
if err := s.saveBatchOnce(ctx, batch); err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// saveBatchOnce 单次执行批量写入事务。
|
||||
func (s *Store) saveBatchOnce(ctx context.Context, batch map[string][]model.Kline) error {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin tx: %w", err)
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck
|
||||
|
||||
stmt, err := tx.PrepareContext(ctx, upsertSQL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare upsert: %w", err)
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
for code, klines := range batch {
|
||||
for _, k := range klines {
|
||||
if _, err := stmt.ExecContext(ctx, code, k.Date, k.Open, k.Close, k.High, k.Low, k.Volume, k.Amount, k.ConvertPrice, k.ConvertValue, k.PremiumRate); err != nil {
|
||||
return fmt.Errorf("insert kline %s %s: %w", code, k.Date, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("commit tx: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadAllKlines 一次性加载全部标的的K线(按代码、日期升序)。
|
||||
// 用于回测等需要全量数据在内存的场景。
|
||||
func (s *Store) LoadAllKlines(ctx context.Context) (map[string][]model.Kline, error) {
|
||||
|
||||
Reference in New Issue
Block a user