feat: 添加异步数据入库任务系统,支持全量/增量更新与实时进度展示
新增完整的异步任务管理体系: 1. 新增Redis缓存层实现任务状态实时存储与降级机制 2. 实现并发拉取+批量写入的高性能入库流程 3. 添加任务管理HTTP接口与路由 4. 重构行情控制器使用共享SQLite存储 5. 配套添加Docker Compose Redis启动配置 6. 前端新增任务管理面板支持操作与进度查看
This commit is contained in:
74
internal/cache/cache.go
vendored
Normal file
74
internal/cache/cache.go
vendored
Normal file
@@ -0,0 +1,74 @@
|
||||
// Package cache 定义任务状态缓存接口,支持 Redis 实时状态 + 内存降级。
|
||||
// Redis 不可用时降级到 NoopCache,任务系统仍可运行(仅失去实时状态能力,回退到 SQLite)。
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TaskType 任务类型。
|
||||
type TaskType string
|
||||
|
||||
const (
|
||||
TaskTypeFull TaskType = "full" // 全量入库
|
||||
TaskTypeIncremental TaskType = "incremental" // 增量同步
|
||||
)
|
||||
|
||||
// TaskStatus 任务状态。
|
||||
type TaskStatus string
|
||||
|
||||
const (
|
||||
TaskStatusPending TaskStatus = "pending" // 已创建未启动
|
||||
TaskStatusRunning TaskStatus = "running" // 执行中
|
||||
TaskStatusSuccess TaskStatus = "success" // 成功
|
||||
TaskStatusFailed TaskStatus = "failed" // 失败
|
||||
TaskStatusCancelled TaskStatus = "cancelled" // 已取消
|
||||
)
|
||||
|
||||
// Task 任务运行态快照,Redis 实时状态与 SQLite 持久化共用同一结构。
|
||||
type Task struct {
|
||||
ID string `json:"id"`
|
||||
Type TaskType `json:"type"`
|
||||
Status TaskStatus `json:"status"`
|
||||
Total int `json:"total"`
|
||||
Done int `json:"done"`
|
||||
Success int `json:"success"`
|
||||
Failed int `json:"failed"`
|
||||
FailedCodes []string `json:"failedCodes,omitempty"`
|
||||
Params string `json:"params"`
|
||||
Error string `json:"error,omitempty"`
|
||||
StartedAt string `json:"startedAt"`
|
||||
FinishedAt string `json:"finishedAt,omitempty"`
|
||||
}
|
||||
|
||||
// Cache 任务状态缓存接口。
|
||||
// Redis 实现用于实时状态查询;NoopCache 用于 Redis 不可用时降级。
|
||||
type Cache interface {
|
||||
// GetTask 读取任务状态,不存在返回 nil,nil。
|
||||
GetTask(ctx context.Context, id string) (*Task, error)
|
||||
// SetTask 写入任务状态,ttl 为过期时间。
|
||||
SetTask(ctx context.Context, t *Task, ttl time.Duration) error
|
||||
// DeleteTask 删除任务状态。
|
||||
DeleteTask(ctx context.Context, id string) error
|
||||
// ListActiveTaskIDs 返回运行中任务的 ID 列表。
|
||||
ListActiveTaskIDs(ctx context.Context) ([]string, error)
|
||||
// AddActiveTask 将任务 ID 加入活跃集合。
|
||||
AddActiveTask(ctx context.Context, id string) error
|
||||
// RemoveActiveTask 从活跃集合移除任务 ID。
|
||||
RemoveActiveTask(ctx context.Context, id string) error
|
||||
// Ping 探测缓存可用性。
|
||||
Ping(ctx context.Context) error
|
||||
}
|
||||
|
||||
// NoopCache Cache 的空实现,Redis 不可用时降级使用。
|
||||
// 所有操作均为空操作,任务状态仅依赖 SQLite 持久化。
|
||||
type NoopCache struct{}
|
||||
|
||||
func (NoopCache) GetTask(ctx context.Context, id string) (*Task, error) { return nil, nil }
|
||||
func (NoopCache) SetTask(ctx context.Context, t *Task, ttl time.Duration) error { return nil }
|
||||
func (NoopCache) DeleteTask(ctx context.Context, id string) error { return nil }
|
||||
func (NoopCache) ListActiveTaskIDs(ctx context.Context) ([]string, error) { return nil, nil }
|
||||
func (NoopCache) AddActiveTask(ctx context.Context, id string) error { return nil }
|
||||
func (NoopCache) RemoveActiveTask(ctx context.Context, id string) error { return nil }
|
||||
func (NoopCache) Ping(ctx context.Context) error { return nil }
|
||||
107
internal/cache/redis.go
vendored
Normal file
107
internal/cache/redis.go
vendored
Normal file
@@ -0,0 +1,107 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// keyPrefix Redis 键前缀,避免与其他应用冲突。
|
||||
const keyPrefix = "stock-analysis:"
|
||||
|
||||
// taskKey 构造任务状态键。
|
||||
func taskKey(id string) string { return keyPrefix + "task:" + id }
|
||||
|
||||
// activeSetKey 构造活跃任务集合键。
|
||||
const activeSetKey = keyPrefix + "tasks:active"
|
||||
|
||||
// RedisCache 基于 go-redis/v9 的 Cache 实现。
|
||||
type RedisCache struct {
|
||||
client *redis.Client
|
||||
}
|
||||
|
||||
// NewRedisCache 创建 Redis 缓存实例。
|
||||
// addr 格式 "host:port",password 为空表示无密码,db 为 Redis 数据库编号。
|
||||
// 配置较短的超时和重试次数,Redis 不可用时快速降级到 NoopCache。
|
||||
func NewRedisCache(addr, password string, db int) (*RedisCache, error) {
|
||||
client := redis.NewClient(&redis.Options{
|
||||
Addr: addr,
|
||||
Password: password,
|
||||
DB: db,
|
||||
DialTimeout: 2 * time.Second,
|
||||
ReadTimeout: 2 * time.Second,
|
||||
WriteTimeout: 2 * time.Second,
|
||||
MaxRetries: 1,
|
||||
PoolSize: 10,
|
||||
})
|
||||
return &RedisCache{client: client}, nil
|
||||
}
|
||||
|
||||
// GetTask 从 Redis 读取任务状态(JSON 反序列化)。
|
||||
func (c *RedisCache) GetTask(ctx context.Context, id string) (*Task, error) {
|
||||
data, err := c.client.Get(ctx, taskKey(id)).Bytes()
|
||||
if err != nil {
|
||||
if err == redis.Nil {
|
||||
return nil, nil // 不存在
|
||||
}
|
||||
return nil, fmt.Errorf("redis get task %s: %w", id, err)
|
||||
}
|
||||
var t Task
|
||||
if err := json.Unmarshal(data, &t); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal task %s: %w", id, err)
|
||||
}
|
||||
return &t, nil
|
||||
}
|
||||
|
||||
// SetTask 写入任务状态(JSON 序列化),设置 TTL 防止过期数据残留。
|
||||
func (c *RedisCache) SetTask(ctx context.Context, t *Task, ttl time.Duration) error {
|
||||
data, err := json.Marshal(t)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal task %s: %w", t.ID, err)
|
||||
}
|
||||
if err := c.client.Set(ctx, taskKey(t.ID), data, ttl).Err(); err != nil {
|
||||
return fmt.Errorf("redis set task %s: %w", t.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteTask 删除任务状态。
|
||||
func (c *RedisCache) DeleteTask(ctx context.Context, id string) error {
|
||||
if err := c.client.Del(ctx, taskKey(id)).Err(); err != nil {
|
||||
return fmt.Errorf("redis del task %s: %w", id, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListActiveTaskIDs 返回活跃任务集合中的所有 ID。
|
||||
func (c *RedisCache) ListActiveTaskIDs(ctx context.Context) ([]string, error) {
|
||||
ids, err := c.client.SMembers(ctx, activeSetKey).Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("redis smembers: %w", err)
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// AddActiveTask 将任务 ID 加入活跃集合。
|
||||
func (c *RedisCache) AddActiveTask(ctx context.Context, id string) error {
|
||||
if err := c.client.SAdd(ctx, activeSetKey, id).Err(); err != nil {
|
||||
return fmt.Errorf("redis sadd %s: %w", id, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveActiveTask 从活跃集合移除任务 ID。
|
||||
func (c *RedisCache) RemoveActiveTask(ctx context.Context, id string) error {
|
||||
if err := c.client.SRem(ctx, activeSetKey, id).Err(); err != nil {
|
||||
return fmt.Errorf("redis srem %s: %w", id, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ping 探测 Redis 连通性。
|
||||
func (c *RedisCache) Ping(ctx context.Context) error {
|
||||
return c.client.Ping(ctx).Err()
|
||||
}
|
||||
@@ -15,14 +15,15 @@ import (
|
||||
)
|
||||
|
||||
// MarketController 提供行情与回测相关的 HTTP 接口。
|
||||
// store 为共享 SQLite 存储,与 job.Manager 共用同一连接池,避免每次请求重复开关连接。
|
||||
type MarketController struct {
|
||||
svc *service.MarketService
|
||||
dbPath string
|
||||
svc *service.MarketService
|
||||
store *sqlite.Store
|
||||
}
|
||||
|
||||
// NewMarketController 创建行情控制器。
|
||||
func NewMarketController(svc *service.MarketService, dbPath string) *MarketController {
|
||||
return &MarketController{svc: svc, dbPath: dbPath}
|
||||
func NewMarketController(svc *service.MarketService, store *sqlite.Store) *MarketController {
|
||||
return &MarketController{svc: svc, store: store}
|
||||
}
|
||||
|
||||
// ListBonds 处理 GET /api/bonds,返回全量可转债实时行情。
|
||||
@@ -93,13 +94,7 @@ func (c *MarketController) RunBacktest(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
ctx := r.Context()
|
||||
|
||||
store, err := sqlite.Open(c.dbPath)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
defer store.Close()
|
||||
store := c.store
|
||||
|
||||
feed, err := backtest.NewDataFeed(ctx, store)
|
||||
if err != nil {
|
||||
@@ -127,53 +122,53 @@ func (c *MarketController) RunBacktest(w http.ResponseWriter, r *http.Request) {
|
||||
MaxPrice: req.MaxPrice,
|
||||
})
|
||||
case "doublelow":
|
||||
strat = strategy.NewDoubleLow(strategy.DoubleLowParams{
|
||||
TopN: req.TopN,
|
||||
Rebalance: req.Rebalance,
|
||||
MinPrice: req.MinPrice,
|
||||
MaxPrice: req.MaxPrice,
|
||||
MaxPremium: req.MaxPremium,
|
||||
})
|
||||
case "premium":
|
||||
strat = strategy.NewPremium(strategy.PremiumParams{
|
||||
TopN: req.TopN,
|
||||
Rebalance: req.Rebalance,
|
||||
MinPrice: req.MinPrice,
|
||||
MaxPrice: req.MaxPrice,
|
||||
})
|
||||
default:
|
||||
strat = strategy.NewDoubleLow(strategy.DoubleLowParams{
|
||||
TopN: req.TopN,
|
||||
Rebalance: req.Rebalance,
|
||||
MinPrice: req.MinPrice,
|
||||
MaxPrice: req.MaxPrice,
|
||||
MaxPremium: req.MaxPremium,
|
||||
})
|
||||
case "premium":
|
||||
strat = strategy.NewPremium(strategy.PremiumParams{
|
||||
TopN: req.TopN,
|
||||
Rebalance: req.Rebalance,
|
||||
MinPrice: req.MinPrice,
|
||||
MaxPrice: req.MaxPrice,
|
||||
})
|
||||
default:
|
||||
writeError(w, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
|
||||
engine := backtest.NewEngine(params, strat)
|
||||
|
||||
if req.Fundamental {
|
||||
fundamentals, err := store.LoadAllFundamentals(ctx)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, fmt.Errorf("加载基本面数据: %w", err))
|
||||
return
|
||||
}
|
||||
stock2bond, err := store.GetStockBondMap(ctx)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, fmt.Errorf("获取正股映射: %w", err))
|
||||
return
|
||||
}
|
||||
filter := backtest.NewFundamentalFilter(stock2bond)
|
||||
engine.SetFundamentalFilter(filter, fundamentals)
|
||||
log.Printf("基本面过滤器已启用:%d 只正股数据", len(fundamentals))
|
||||
if req.Fundamental {
|
||||
fundamentals, err := store.LoadAllFundamentals(ctx)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, fmt.Errorf("加载基本面数据: %w", err))
|
||||
return
|
||||
}
|
||||
stock2bond, err := store.GetStockBondMap(ctx)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, fmt.Errorf("获取正股映射: %w", err))
|
||||
return
|
||||
}
|
||||
filter := backtest.NewFundamentalFilter(stock2bond)
|
||||
engine.SetFundamentalFilter(filter, fundamentals)
|
||||
log.Printf("基本面过滤器已启用:%d 只正股数据", len(fundamentals))
|
||||
}
|
||||
|
||||
res, err := engine.Run(ctx, feed)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
res, err := engine.Run(ctx, feed)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Printf("backtest done: %s, start=%s, end=%s, totalReturn=%.2f%%, curvePoints=%d, tradeDays=%d",
|
||||
res.Strategy, res.Start, res.End, res.Metrics.TotalReturn*100, len(res.Curve), res.Metrics.TradeDays)
|
||||
log.Printf("backtest done: %s, start=%s, end=%s, totalReturn=%.2f%%, curvePoints=%d, tradeDays=%d",
|
||||
res.Strategy, res.Start, res.End, res.Metrics.TotalReturn*100, len(res.Curve), res.Metrics.TradeDays)
|
||||
|
||||
bench := computeBenchmarkJSON(feed, req.StartDate, req.EndDate)
|
||||
bench := computeBenchmarkJSON(feed, req.StartDate, req.EndDate)
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"strategy": res.Strategy,
|
||||
@@ -251,12 +246,7 @@ type DBBond struct {
|
||||
// ListDBBonds 处理 GET /api/db/bonds,返回数据库中已有的转债列表。
|
||||
func (c *MarketController) ListDBBonds(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
store, err := sqlite.Open(c.dbPath)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
defer store.Close()
|
||||
store := c.store
|
||||
|
||||
codes, err := store.ListCodes(ctx)
|
||||
if err != nil {
|
||||
@@ -282,14 +272,8 @@ func (c *MarketController) ListDBBonds(w http.ResponseWriter, r *http.Request) {
|
||||
// GetDataStatus 处理 GET /api/status,返回数据库状态信息。
|
||||
func (c *MarketController) GetDataStatus(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
status := DataStatus{DBPath: c.dbPath}
|
||||
|
||||
store, err := sqlite.Open(c.dbPath)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusOK, status) // 返回部分信息
|
||||
return
|
||||
}
|
||||
defer store.Close()
|
||||
store := c.store
|
||||
status := DataStatus{DBPath: c.store.Path()}
|
||||
|
||||
codes, err := store.ListCodes(ctx)
|
||||
if err == nil {
|
||||
|
||||
122
internal/controller/task.go
Normal file
122
internal/controller/task.go
Normal file
@@ -0,0 +1,122 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"stock-analysis/internal/cache"
|
||||
"stock-analysis/internal/job"
|
||||
)
|
||||
|
||||
// 任务接口错误(包级哨兵错误,便于测试断言)。
|
||||
var (
|
||||
errBadTaskType = errString("无效的任务类型,支持 full / incremental")
|
||||
errMissingTaskID = errString("缺少任务 ID")
|
||||
errTaskNotFound = errString("任务不存在")
|
||||
errTaskNotRunning = errString("任务不存在或已结束,无法取消")
|
||||
)
|
||||
|
||||
// errString 将字符串转为 error,避免重复 fmt.Errorf 调用。
|
||||
type errString string
|
||||
|
||||
func (e errString) Error() string { return string(e) }
|
||||
|
||||
// TaskController 提供数据入库任务管理的 HTTP 接口。
|
||||
// 任务由 job.Manager 异步执行,状态实时写入 Redis,持久化到 SQLite。
|
||||
type TaskController struct {
|
||||
mgr *job.Manager
|
||||
}
|
||||
|
||||
// NewTaskController 创建任务控制器。
|
||||
func NewTaskController(mgr *job.Manager) *TaskController {
|
||||
return &TaskController{mgr: mgr}
|
||||
}
|
||||
|
||||
// taskRequest 创建任务请求参数。
|
||||
type taskRequest struct {
|
||||
Type string `json:"type"` // full | incremental
|
||||
Since string `json:"since"` // 增量起点(YYYY-MM-DD),可选,留空自动取最后K线日期
|
||||
Limit int `json:"limit"` // 限制处理数量,0 表示全部
|
||||
}
|
||||
|
||||
// CreateTask 处理 POST /api/tasks,创建数据入库任务(全量/增量)。
|
||||
// 任务异步执行,立即返回 pending 状态的任务快照,前端通过轮询 GetTask 获取进度。
|
||||
func (c *TaskController) CreateTask(w http.ResponseWriter, r *http.Request) {
|
||||
var req taskRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
|
||||
var typ cache.TaskType
|
||||
switch req.Type {
|
||||
case "full", "":
|
||||
typ = cache.TaskTypeFull
|
||||
case "incremental":
|
||||
typ = cache.TaskTypeIncremental
|
||||
default:
|
||||
writeError(w, http.StatusBadRequest, errBadTaskType)
|
||||
return
|
||||
}
|
||||
|
||||
task, err := c.mgr.Start(r.Context(), typ, req.Since, req.Limit)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, task)
|
||||
}
|
||||
|
||||
// ListTasks 处理 GET /api/tasks,返回任务列表(按开始时间降序)。
|
||||
// 默认返回最近 50 条,可通过 ?limit=N 调整。
|
||||
// 运行中任务会合并 Redis 实时进度。
|
||||
func (c *TaskController) ListTasks(w http.ResponseWriter, r *http.Request) {
|
||||
limit := 50
|
||||
if q := r.URL.Query().Get("limit"); q != "" {
|
||||
if n, err := strconv.Atoi(q); err == nil && n > 0 {
|
||||
limit = n
|
||||
}
|
||||
}
|
||||
tasks, err := c.mgr.List(r.Context(), limit)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, tasks)
|
||||
}
|
||||
|
||||
// GetTask 处理 GET /api/tasks/{id},返回单个任务状态。
|
||||
// 优先从 Redis 读取实时状态,回退 SQLite 持久化记录。
|
||||
func (c *TaskController) GetTask(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
if id == "" {
|
||||
writeError(w, http.StatusBadRequest, errMissingTaskID)
|
||||
return
|
||||
}
|
||||
task, err := c.mgr.Get(r.Context(), id)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusNotFound, err)
|
||||
return
|
||||
}
|
||||
if task == nil {
|
||||
writeError(w, http.StatusNotFound, errTaskNotFound)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, task)
|
||||
}
|
||||
|
||||
// CancelTask 处理 POST /api/tasks/{id}/cancel,取消运行中任务。
|
||||
// 仅运行中任务可取消,已完成任务忽略。
|
||||
func (c *TaskController) CancelTask(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
if id == "" {
|
||||
writeError(w, http.StatusBadRequest, errMissingTaskID)
|
||||
return
|
||||
}
|
||||
if ok := c.mgr.Cancel(id); !ok {
|
||||
writeError(w, http.StatusNotFound, errTaskNotRunning)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"message": "任务已取消"})
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -46,14 +47,26 @@ var historyHosts = []string{
|
||||
|
||||
// Client 是东方财富行情接口客户端,实现 datasource.MarketDataSource。
|
||||
type Client struct {
|
||||
http *http.Client
|
||||
lastReq time.Time
|
||||
http *http.Client
|
||||
lastReq time.Time
|
||||
mu sync.Mutex // 保护 lastReq,支持多 goroutine 并发调用
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
// NewClient 创建东财行情客户端。
|
||||
// NewClient 创建东财行情客户端(默认 2s 节流)。
|
||||
func NewClient() *Client {
|
||||
return &Client{
|
||||
http: &http.Client{Timeout: 15 * time.Second},
|
||||
http: &http.Client{Timeout: 15 * time.Second},
|
||||
interval: requestInterval,
|
||||
}
|
||||
}
|
||||
|
||||
// NewClientWithThrottle 创建可配置节流间隔的客户端。
|
||||
// interval=0 关闭内部节流,由外部全局限速器统一控制(job runner 场景)。
|
||||
func NewClientWithThrottle(interval time.Duration) *Client {
|
||||
return &Client{
|
||||
http: &http.Client{Timeout: 15 * time.Second},
|
||||
interval: interval,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -104,10 +117,16 @@ func (c *Client) fetch(ctx context.Context, reqURL string, out any) error {
|
||||
return json.Unmarshal(body, out)
|
||||
}
|
||||
|
||||
// throttle 保证请求间隔不小于 requestInterval。
|
||||
// throttle 保证请求间隔不小于 interval,线程安全。
|
||||
// interval=0 时为空操作,由外部全局限速器统一控制速率。
|
||||
func (c *Client) throttle() {
|
||||
if c.interval == 0 {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
elapsed := time.Since(c.lastReq)
|
||||
if wait := requestInterval - elapsed; wait > 0 {
|
||||
if wait := c.interval - elapsed; wait > 0 {
|
||||
time.Sleep(wait)
|
||||
}
|
||||
c.lastReq = time.Now()
|
||||
|
||||
82
internal/job/ingest.go
Normal file
82
internal/job/ingest.go
Normal file
@@ -0,0 +1,82 @@
|
||||
package job
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"stock-analysis/internal/cache"
|
||||
"stock-analysis/internal/model"
|
||||
)
|
||||
|
||||
// resolveCandidates 根据任务类型解析要处理的转债候选列表。
|
||||
//
|
||||
// full: 从 eastmoney datacenter 获取全量转债列表,过滤 ListDate 为空的
|
||||
// incremental: 从 SQLite 获取已入库的转债代码列表
|
||||
func resolveCandidates(ctx context.Context, store Store, lister BondLister, typ cache.TaskType, limit int) ([]model.ConvertibleBond, error) {
|
||||
switch typ {
|
||||
case cache.TaskTypeFull:
|
||||
bonds, err := lister.FetchAllBondCodes(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取转债列表: %w", err)
|
||||
}
|
||||
var candidates []model.ConvertibleBond
|
||||
for _, b := range bonds {
|
||||
if b.ListDate == "" {
|
||||
continue // 过滤未上市
|
||||
}
|
||||
candidates = append(candidates, b)
|
||||
}
|
||||
if limit > 0 && limit < len(candidates) {
|
||||
candidates = candidates[:limit]
|
||||
}
|
||||
return candidates, nil
|
||||
|
||||
case cache.TaskTypeIncremental:
|
||||
codes, err := store.ListCodes(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("加载已入库转债: %w", err)
|
||||
}
|
||||
candidates := make([]model.ConvertibleBond, 0, len(codes))
|
||||
for i, c := range codes {
|
||||
if limit > 0 && i >= limit {
|
||||
break
|
||||
}
|
||||
candidates = append(candidates, model.ConvertibleBond{Code: c})
|
||||
}
|
||||
return candidates, nil
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("未知任务类型: %s", typ)
|
||||
}
|
||||
}
|
||||
|
||||
// yearOf 从上市日期(2006-01-02)解析年份。
|
||||
func yearOf(listDate string) int {
|
||||
if len(listDate) >= 4 {
|
||||
var y int
|
||||
if _, err := fmt.Sscanf(listDate[:4], "%d", &y); err == nil && y > 2000 {
|
||||
return y
|
||||
}
|
||||
}
|
||||
return time.Now().Year()
|
||||
}
|
||||
|
||||
// resolveIncrementalBeg 确定增量拉取的起始日期(YYYYMMDD)。
|
||||
//
|
||||
// since 非空 → 直接使用(覆盖库中最后日期)
|
||||
// 否则 → lastDateMap[code] 次日;无记录返回空串(表示跳过)
|
||||
func resolveIncrementalBeg(code, since string, lastDateMap map[string]string) (string, bool) {
|
||||
if since != "" {
|
||||
return since, true
|
||||
}
|
||||
lastDate, ok := lastDateMap[code]
|
||||
if !ok || lastDate == "" {
|
||||
return "", false
|
||||
}
|
||||
t, err := time.Parse("2006-01-02", lastDate)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
return t.AddDate(0, 0, 1).Format("20060102"), true
|
||||
}
|
||||
187
internal/job/job.go
Normal file
187
internal/job/job.go
Normal file
@@ -0,0 +1,187 @@
|
||||
package job
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"stock-analysis/internal/cache"
|
||||
"stock-analysis/internal/datasource"
|
||||
)
|
||||
|
||||
// Config Manager 配置。
|
||||
type Config struct {
|
||||
Workers int // 并发 worker 数,默认 8
|
||||
BatchSize int // 批量写入阈值,默认 20
|
||||
RateInterval time.Duration // 全局限速间隔,默认 400ms(≈2.5 req/s)
|
||||
RateBurst int // 限速突发额度,默认 4
|
||||
}
|
||||
|
||||
// runningTask 记录运行中任务的取消句柄和状态快照。
|
||||
type runningTask struct {
|
||||
cancel context.CancelFunc
|
||||
task *cache.Task
|
||||
}
|
||||
|
||||
// Manager 管理任务注册表和全局限速器,允许多任务并发运行。
|
||||
type Manager struct {
|
||||
store Store
|
||||
cache cache.Cache
|
||||
ds datasource.MarketDataSource
|
||||
lister BondLister
|
||||
gate <-chan struct{} // 全局限速令牌(跨任务共享)
|
||||
workers int
|
||||
batchSize int
|
||||
|
||||
mu sync.RWMutex
|
||||
running map[string]*runningTask
|
||||
}
|
||||
|
||||
// NewManager 创建任务管理器,内部启动全局限速器。
|
||||
func NewManager(store Store, c cache.Cache, ds datasource.MarketDataSource, lister BondLister, cfg Config) *Manager {
|
||||
if cfg.Workers <= 0 {
|
||||
cfg.Workers = 8
|
||||
}
|
||||
if cfg.BatchSize <= 0 {
|
||||
cfg.BatchSize = 20
|
||||
}
|
||||
if cfg.RateInterval <= 0 {
|
||||
cfg.RateInterval = 400 * time.Millisecond
|
||||
}
|
||||
if cfg.RateBurst <= 0 {
|
||||
cfg.RateBurst = 4
|
||||
}
|
||||
|
||||
return &Manager{
|
||||
store: store,
|
||||
cache: c,
|
||||
ds: ds,
|
||||
lister: lister,
|
||||
gate: newRateLimiter(cfg.RateInterval, cfg.RateBurst),
|
||||
workers: cfg.Workers,
|
||||
batchSize: cfg.BatchSize,
|
||||
running: make(map[string]*runningTask),
|
||||
}
|
||||
}
|
||||
|
||||
// Start 创建并异步启动一个任务,立即返回任务快照(pending 状态)。
|
||||
func (m *Manager) Start(ctx context.Context, typ cache.TaskType, since string, limit int) (*cache.Task, error) {
|
||||
params, _ := json.Marshal(map[string]any{"since": since, "limit": limit})
|
||||
|
||||
task := &cache.Task{
|
||||
ID: uuid.New().String(),
|
||||
Type: typ,
|
||||
Status: cache.TaskStatusPending,
|
||||
StartedAt: time.Now().Format(time.RFC3339),
|
||||
Params: string(params),
|
||||
}
|
||||
|
||||
// 持久化到 SQLite + Redis
|
||||
if err := m.store.SaveTask(ctx, task); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = m.cache.SetTask(ctx, task, 24*time.Hour)
|
||||
_ = m.cache.AddActiveTask(ctx, task.ID)
|
||||
|
||||
// 创建可取消 context
|
||||
taskCtx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
m.mu.Lock()
|
||||
m.running[task.ID] = &runningTask{cancel: cancel, task: task}
|
||||
m.mu.Unlock()
|
||||
|
||||
// 异步执行
|
||||
runner := NewRunner(m.store, m.cache, m.ds, m.gate, m.workers, m.batchSize)
|
||||
go func() {
|
||||
defer cancel()
|
||||
runner.run(taskCtx, task, m.lister, since, limit)
|
||||
// 任务结束后从 running map 移除
|
||||
m.mu.Lock()
|
||||
delete(m.running, task.ID)
|
||||
m.mu.Unlock()
|
||||
}()
|
||||
|
||||
log.Printf("[job] task %s created: type=%s, since=%s, limit=%d", task.ID, typ, since, limit)
|
||||
return task, nil
|
||||
}
|
||||
|
||||
// Cancel 取消运行中任务,返回是否成功取消。
|
||||
func (m *Manager) Cancel(id string) bool {
|
||||
m.mu.RLock()
|
||||
rt, ok := m.running[id]
|
||||
m.mu.RUnlock()
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
rt.cancel()
|
||||
log.Printf("[job] task %s cancelled", id)
|
||||
return true
|
||||
}
|
||||
|
||||
// Get 获取任务状态:Redis 优先(实时),回退 SQLite。
|
||||
func (m *Manager) Get(ctx context.Context, id string) (*cache.Task, error) {
|
||||
// 1. 先查 Redis(实时状态)
|
||||
t, err := m.cache.GetTask(ctx, id)
|
||||
if err == nil && t != nil {
|
||||
return t, nil
|
||||
}
|
||||
// 2. 回退 SQLite
|
||||
return m.store.GetTask(ctx, id)
|
||||
}
|
||||
|
||||
// List 返回任务列表:SQLite 历史记录 + Redis 实时覆盖运行中任务。
|
||||
func (m *Manager) List(ctx context.Context, limit int) ([]*cache.Task, error) {
|
||||
tasks, err := m.store.ListTasks(ctx, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// 对运行中任务用 Redis 实时数据覆盖
|
||||
for _, t := range tasks {
|
||||
if t.Status == cache.TaskStatusRunning || t.Status == cache.TaskStatusPending {
|
||||
if rt, _ := m.cache.GetTask(ctx, t.ID); rt != nil {
|
||||
t.Done = rt.Done
|
||||
t.Success = rt.Success
|
||||
t.Failed = rt.Failed
|
||||
t.Status = rt.Status
|
||||
t.FailedCodes = rt.FailedCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
return tasks, nil
|
||||
}
|
||||
|
||||
// StopAll 优雅停止所有运行中任务(服务关闭时调用)。
|
||||
func (m *Manager) StopAll() {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
for id, rt := range m.running {
|
||||
rt.cancel()
|
||||
log.Printf("[job] stopping task %s on shutdown", id)
|
||||
}
|
||||
}
|
||||
|
||||
// newRateLimiter 创建全局限速器(令牌桶模式)。
|
||||
// 返回一个只读 channel,每次接收表示获得一个请求令牌。
|
||||
// 稳态速率 = 1 token / interval,初始突发 = burst。
|
||||
func newRateLimiter(interval time.Duration, burst int) <-chan struct{} {
|
||||
ch := make(chan struct{}, burst)
|
||||
// 预填突发额度
|
||||
for i := 0; i < burst; i++ {
|
||||
ch <- struct{}{}
|
||||
}
|
||||
go func() {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
select {
|
||||
case ch <- struct{}{}:
|
||||
default: // 满了丢弃,保持稳态速率
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ch
|
||||
}
|
||||
229
internal/job/runner.go
Normal file
229
internal/job/runner.go
Normal file
@@ -0,0 +1,229 @@
|
||||
package job
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"stock-analysis/internal/cache"
|
||||
"stock-analysis/internal/datasource"
|
||||
"stock-analysis/internal/model"
|
||||
)
|
||||
|
||||
// fetchResult 单只转债拉取结果,通过 channel 发往 collector。
|
||||
type fetchResult struct {
|
||||
code string
|
||||
klines []model.Kline
|
||||
err error
|
||||
}
|
||||
|
||||
// Runner 单任务执行器:N worker 并发拉取 + 单 collector 串行批量写入。
|
||||
//
|
||||
// 并发模型:
|
||||
// - 主 goroutine 遍历候选列表,为每只转债启动一个 worker goroutine
|
||||
// - worker 从 gate 获取全局限速令牌后拉取数据,结果发往 resultCh
|
||||
// - collector 单 goroutine 从 resultCh 收集,每 batchSize 只批量写 SQLite
|
||||
// - SQLite SetMaxOpenConns(1) 下,只有 collector 写入,避免锁竞争
|
||||
type Runner struct {
|
||||
store Store
|
||||
cache cache.Cache
|
||||
ds datasource.MarketDataSource
|
||||
gate <-chan struct{} // 全局限速令牌
|
||||
workers int
|
||||
batchSize int
|
||||
}
|
||||
|
||||
// NewRunner 创建执行器。
|
||||
func NewRunner(store Store, c cache.Cache, ds datasource.MarketDataSource, gate <-chan struct{}, workers, batchSize int) *Runner {
|
||||
return &Runner{
|
||||
store: store,
|
||||
cache: c,
|
||||
ds: ds,
|
||||
gate: gate,
|
||||
workers: workers,
|
||||
batchSize: batchSize,
|
||||
}
|
||||
}
|
||||
|
||||
// run 执行任务(异步调用,不阻塞调用方)。
|
||||
func (r *Runner) run(ctx context.Context, task *cache.Task, lister BondLister, since string, limit int) {
|
||||
// 1. 解析候选列表
|
||||
candidates, err := resolveCandidates(ctx, r.store, lister, task.Type, limit)
|
||||
if err != nil {
|
||||
r.failTask(ctx, task, err)
|
||||
return
|
||||
}
|
||||
task.Total = len(candidates)
|
||||
|
||||
// 2. 预加载:全量模式加载已入库集合(跳过断点续跑);增量模式加载最后日期映射
|
||||
var skipSet map[string]bool
|
||||
var lastDateMap map[string]string
|
||||
if task.Type == cache.TaskTypeFull {
|
||||
codes, err := r.store.ListCodes(ctx)
|
||||
if err == nil {
|
||||
skipSet = make(map[string]bool, len(codes))
|
||||
for _, c := range codes {
|
||||
skipSet[c] = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
lastDateMap, _ = r.store.LastKlineDates(ctx)
|
||||
}
|
||||
|
||||
// 3. 更新状态为 running
|
||||
task.Status = cache.TaskStatusRunning
|
||||
r.reportProgress(ctx, task, true)
|
||||
|
||||
log.Printf("[job] task %s started: type=%s, total=%d", task.ID, task.Type, task.Total)
|
||||
|
||||
// 4. 启动 collector(唯一 SQLite 写者)
|
||||
resultCh := make(chan fetchResult, r.workers*2)
|
||||
collectorDone := make(chan struct{})
|
||||
go r.collect(ctx, task, resultCh, collectorDone)
|
||||
|
||||
// 5. 并发拉取:主 goroutine 遍历候选,每只启动一个 worker
|
||||
sem := make(chan struct{}, r.workers)
|
||||
var wg sync.WaitGroup
|
||||
for _, b := range candidates {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
// 全量模式跳过已入库转债
|
||||
if task.Type == cache.TaskTypeFull && skipSet[b.Code] {
|
||||
task.Done++
|
||||
task.Success++
|
||||
r.reportProgress(ctx, task, false)
|
||||
continue
|
||||
}
|
||||
sem <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func(bond model.ConvertibleBond) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
|
||||
// 全局限速:获取令牌后才允许发请求
|
||||
select {
|
||||
case <-r.gate:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
|
||||
klines, err := r.fetchOne(ctx, task.Type, bond, since, lastDateMap)
|
||||
select {
|
||||
case resultCh <- fetchResult{code: bond.Code, klines: klines, err: err}:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}(b)
|
||||
}
|
||||
|
||||
// 6. 等待所有 worker 完成
|
||||
wg.Wait()
|
||||
close(resultCh)
|
||||
<-collectorDone
|
||||
|
||||
// 7. 确定终态
|
||||
r.finishTask(ctx, task)
|
||||
}
|
||||
|
||||
// fetchOne 拉取单只转债K线。
|
||||
func (r *Runner) fetchOne(ctx context.Context, typ cache.TaskType, bond model.ConvertibleBond, since string, lastDateMap map[string]string) ([]model.Kline, error) {
|
||||
switch typ {
|
||||
case cache.TaskTypeFull:
|
||||
beg := fmt.Sprintf("%04d0101", yearOf(bond.ListDate))
|
||||
return r.ds.FetchKline(ctx, bond.Code, "101", beg, "20500101")
|
||||
case cache.TaskTypeIncremental:
|
||||
beg, ok := resolveIncrementalBeg(bond.Code, since, lastDateMap)
|
||||
if !ok {
|
||||
return nil, nil // 无存档,跳过
|
||||
}
|
||||
return r.ds.FetchKline(ctx, bond.Code, "101", beg, "20500101")
|
||||
default:
|
||||
return nil, fmt.Errorf("未知任务类型: %s", typ)
|
||||
}
|
||||
}
|
||||
|
||||
// collect 单 goroutine 收集结果并批量写入 SQLite(唯一写者)。
|
||||
func (r *Runner) collect(ctx context.Context, task *cache.Task, resultCh <-chan fetchResult, done chan<- struct{}) {
|
||||
defer close(done)
|
||||
batch := make(map[string][]model.Kline)
|
||||
|
||||
for res := range resultCh {
|
||||
task.Done++
|
||||
if res.err != nil || len(res.klines) == 0 {
|
||||
task.Failed++
|
||||
if res.err != nil {
|
||||
task.FailedCodes = append(task.FailedCodes, res.code)
|
||||
}
|
||||
} else {
|
||||
task.Success++
|
||||
batch[res.code] = res.klines
|
||||
if len(batch) >= r.batchSize {
|
||||
if err := r.store.SaveKlinesBatch(ctx, batch); err != nil {
|
||||
log.Printf("[job] batch save error: %v", err)
|
||||
}
|
||||
batch = make(map[string][]model.Kline)
|
||||
}
|
||||
}
|
||||
r.reportProgress(ctx, task, false)
|
||||
}
|
||||
|
||||
// flush 残余 batch
|
||||
if len(batch) > 0 {
|
||||
if err := r.store.SaveKlinesBatch(ctx, batch); err != nil {
|
||||
log.Printf("[job] final batch save error: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reportProgress 上报进度:Redis 每次(廉价),SQLite 按策略写(减少写压力)。
|
||||
// force=true(状态变更)时用 SaveTask 写入完整状态(含 status/total),
|
||||
// 确保 Redis 不可用时 SQLite 也能反映运行状态。
|
||||
func (r *Runner) reportProgress(ctx context.Context, task *cache.Task, force bool) {
|
||||
_ = r.cache.SetTask(ctx, task, 24*time.Hour)
|
||||
if force {
|
||||
_ = r.store.SaveTask(ctx, task)
|
||||
} else if task.Done%5 == 0 {
|
||||
_ = r.store.UpdateTaskProgress(ctx, task.ID, task.Total, task.Done, task.Success, task.Failed, task.FailedCodes)
|
||||
}
|
||||
}
|
||||
|
||||
// finishTask 确定任务终态并持久化。
|
||||
// 使用 SaveTask 写入完整任务状态(含 total/done/success/failed),
|
||||
// 确保 SQLite 中最终记录与内存一致(Redis 不可用时尤为重要)。
|
||||
func (r *Runner) finishTask(ctx context.Context, task *cache.Task) {
|
||||
var errMsg string
|
||||
switch {
|
||||
case ctx.Err() != nil:
|
||||
task.Status = cache.TaskStatusCancelled
|
||||
case task.Success == 0 && task.Failed > 0:
|
||||
task.Status = cache.TaskStatusFailed
|
||||
errMsg = fmt.Sprintf("全部 %d 只失败", task.Failed)
|
||||
default:
|
||||
task.Status = cache.TaskStatusSuccess
|
||||
if task.Failed > 0 {
|
||||
errMsg = fmt.Sprintf("成功 %d,失败 %d", task.Success, task.Failed)
|
||||
}
|
||||
}
|
||||
task.Error = errMsg
|
||||
task.FinishedAt = time.Now().Format(time.RFC3339)
|
||||
|
||||
_ = r.store.SaveTask(ctx, task)
|
||||
_ = r.cache.SetTask(ctx, task, 24*time.Hour)
|
||||
_ = r.cache.RemoveActiveTask(ctx, task.ID)
|
||||
|
||||
log.Printf("[job] task %s finished: status=%s, total=%d, success=%d, failed=%d",
|
||||
task.ID, task.Status, task.Total, task.Success, task.Failed)
|
||||
}
|
||||
|
||||
// failTask 标记任务失败(启动阶段错误)。
|
||||
func (r *Runner) failTask(ctx context.Context, task *cache.Task, err error) {
|
||||
task.Status = cache.TaskStatusFailed
|
||||
task.Error = err.Error()
|
||||
task.FinishedAt = time.Now().Format(time.RFC3339)
|
||||
_ = r.store.SaveTask(ctx, task)
|
||||
_ = r.cache.SetTask(ctx, task, 24*time.Hour)
|
||||
_ = r.cache.RemoveActiveTask(ctx, task.ID)
|
||||
log.Printf("[job] task %s failed: %v", task.ID, err)
|
||||
}
|
||||
37
internal/job/store.go
Normal file
37
internal/job/store.go
Normal file
@@ -0,0 +1,37 @@
|
||||
// Package job 实现异步任务执行系统:并发拉取 + 批量写入 + 进度上报。
|
||||
//
|
||||
// 核心设计:
|
||||
// - Manager 管理任务注册表和全局限速器,允许多任务并发
|
||||
// - Runner 是单任务执行器:N worker 并发拉取,单 collector 串行写 SQLite
|
||||
// - Redis 存实时进度,SQLite 存持久化任务记录
|
||||
package job
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"stock-analysis/internal/cache"
|
||||
"stock-analysis/internal/model"
|
||||
)
|
||||
|
||||
// Store 定义 job 运行期需要的存储能力(K线 + 任务 CRUD)。
|
||||
// *sqlite.Store 实现此接口(编译期断言)。
|
||||
type Store interface {
|
||||
// K线批量写入(单事务,collector 调用)
|
||||
SaveKlinesBatch(ctx context.Context, batch map[string][]model.Kline) error
|
||||
// 返回所有已入库 code(全量断点续跑跳过判断)
|
||||
ListCodes(ctx context.Context) ([]string, error)
|
||||
// 一次性返回 code -> 最后K线日期 映射(增量起点预加载)
|
||||
LastKlineDates(ctx context.Context) (map[string]string, error)
|
||||
|
||||
// 任务持久化
|
||||
SaveTask(ctx context.Context, t *cache.Task) error
|
||||
UpdateTaskProgress(ctx context.Context, id string, total, done, success, failed int, failedCodes []string) error
|
||||
FinishTask(ctx context.Context, id string, status cache.TaskStatus, errMsg, finishedAt string) error
|
||||
GetTask(ctx context.Context, id string) (*cache.Task, error)
|
||||
ListTasks(ctx context.Context, limit int) ([]*cache.Task, error)
|
||||
}
|
||||
|
||||
// BondLister 全量转债列表获取接口(eastmoney.Client 实现)。
|
||||
type BondLister interface {
|
||||
FetchAllBondCodes(ctx context.Context) ([]model.ConvertibleBond, error)
|
||||
}
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
|
||||
// New 构建应用路由处理器。
|
||||
// webDir 为前端静态资源目录(View 层)。
|
||||
func New(mc *controller.MarketController, ac *controller.AuthController, webDir string) http.Handler {
|
||||
func New(mc *controller.MarketController, ac *controller.AuthController, tc *controller.TaskController, webDir string) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
// 公开接口:无需登录。
|
||||
@@ -29,6 +29,12 @@ func New(mc *controller.MarketController, ac *controller.AuthController, webDir
|
||||
mux.HandleFunc("GET /api/status", requireAuth(ac, mc.GetDataStatus))
|
||||
mux.HandleFunc("GET /api/db/bonds", requireAuth(ac, mc.ListDBBonds))
|
||||
|
||||
// 数据入库任务接口:创建/查询/取消(需要登录)。
|
||||
mux.HandleFunc("POST /api/tasks", requireAuth(ac, tc.CreateTask))
|
||||
mux.HandleFunc("GET /api/tasks", requireAuth(ac, tc.ListTasks))
|
||||
mux.HandleFunc("GET /api/tasks/{id}", requireAuth(ac, tc.GetTask))
|
||||
mux.HandleFunc("POST /api/tasks/{id}/cancel", requireAuth(ac, tc.CancelTask))
|
||||
|
||||
// 静态资源:登录页及样式脚本公开,首页受保护。
|
||||
fs := noCache(http.FileServer(http.Dir(webDir)))
|
||||
mux.Handle("/auth.html", fs)
|
||||
|
||||
@@ -80,6 +80,23 @@ CREATE TABLE IF NOT EXISTS convert_price_history (
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_verify_code_email ON verify_code (email, purpose);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ingest_task (
|
||||
id TEXT PRIMARY KEY,
|
||||
type TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
total INTEGER NOT NULL DEFAULT 0,
|
||||
done INTEGER NOT NULL DEFAULT 0,
|
||||
success INTEGER NOT NULL DEFAULT 0,
|
||||
failed INTEGER NOT NULL DEFAULT 0,
|
||||
failed_codes TEXT NOT NULL DEFAULT '',
|
||||
params TEXT NOT NULL DEFAULT '',
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
started_at TEXT NOT NULL,
|
||||
finished_at TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_ingest_task_status ON ingest_task (status);
|
||||
CREATE INDEX IF NOT EXISTS idx_ingest_task_started ON ingest_task (started_at);
|
||||
`
|
||||
|
||||
// 通用 SQL:幂等写入(存在则跳过)。
|
||||
@@ -90,8 +107,8 @@ ON CONFLICT (code, date) DO NOTHING;
|
||||
`
|
||||
|
||||
const (
|
||||
querySQL = `SELECT code, date, open, close, high, low, volume, amount, convert_price, convert_value, premium_rate FROM convertible_kline WHERE code = ? ORDER BY date;`
|
||||
countSQL = `SELECT COUNT(*) FROM convertible_kline WHERE code = ?;`
|
||||
querySQL = `SELECT code, date, open, close, high, low, volume, amount, convert_price, convert_value, premium_rate FROM convertible_kline WHERE code = ? ORDER BY date;`
|
||||
countSQL = `SELECT COUNT(*) FROM convertible_kline WHERE code = ?;`
|
||||
lastDateSQL = `SELECT MAX(date) FROM convertible_kline WHERE code = ?;`
|
||||
allKlineSQL = `SELECT code, date, open, close, high, low, volume, amount, convert_price, convert_value, premium_rate FROM convertible_kline ORDER BY code, date;`
|
||||
|
||||
@@ -148,9 +165,13 @@ type PremiumRow struct {
|
||||
|
||||
// Store 是 SQLite 存储实现,实现 storage.KlineStore。
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
db *sql.DB
|
||||
path string
|
||||
}
|
||||
|
||||
// Path 返回数据库文件路径。
|
||||
func (s *Store) Path() string { return s.path }
|
||||
|
||||
// Open 打开(或创建)SQLite 数据库文件并初始化表结构。
|
||||
func Open(path string) (*Store, error) {
|
||||
db, err := sql.Open("sqlite", path)
|
||||
@@ -187,7 +208,7 @@ func Open(path string) (*Store, error) {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("set busy timeout: %w", err)
|
||||
}
|
||||
return &Store{db: db}, nil
|
||||
return &Store{db: db, path: path}, nil
|
||||
}
|
||||
|
||||
// initSchema 执行建表语句。
|
||||
@@ -354,6 +375,74 @@ func (s *Store) ListCodes(ctx context.Context) ([]string, error) {
|
||||
return codes, rows.Err()
|
||||
}
|
||||
|
||||
// LastKlineDates 一次性返回所有已入库 code 的最新K线日期(YYYY-MM-DD)。
|
||||
// 用于增量同步预加载,替代逐只 LastKlineDate 查询,避免占用唯一写连接。
|
||||
func (s *Store) LastKlineDates(ctx context.Context) (map[string]string, error) {
|
||||
rows, err := s.db.QueryContext(ctx, "SELECT code, MAX(date) FROM convertible_kline GROUP BY code")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("last kline dates: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make(map[string]string)
|
||||
for rows.Next() {
|
||||
var code, date string
|
||||
if err := rows.Scan(&code, &date); err != nil {
|
||||
return nil, fmt.Errorf("scan last date: %w", err)
|
||||
}
|
||||
out[code] = date
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SaveKlinesBatch 单事务批量写入多只转债K线(collector 调用)。
|
||||
// 接收 map[转债代码]K线切片,在一个事务内遍历写入,带 3 次重试。
|
||||
func (s *Store) SaveKlinesBatch(ctx context.Context, batch map[string][]model.Kline) error {
|
||||
if len(batch) == 0 {
|
||||
return nil
|
||||
}
|
||||
const maxAttempt = 3
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxAttempt; attempt++ {
|
||||
if attempt > 0 {
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
if err := s.saveBatchOnce(ctx, batch); err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// saveBatchOnce 单次执行批量写入事务。
|
||||
func (s *Store) saveBatchOnce(ctx context.Context, batch map[string][]model.Kline) error {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin tx: %w", err)
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck
|
||||
|
||||
stmt, err := tx.PrepareContext(ctx, upsertSQL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare upsert: %w", err)
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
for code, klines := range batch {
|
||||
for _, k := range klines {
|
||||
if _, err := stmt.ExecContext(ctx, code, k.Date, k.Open, k.Close, k.High, k.Low, k.Volume, k.Amount, k.ConvertPrice, k.ConvertValue, k.PremiumRate); err != nil {
|
||||
return fmt.Errorf("insert kline %s %s: %w", code, k.Date, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("commit tx: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadAllKlines 一次性加载全部标的的K线(按代码、日期升序)。
|
||||
// 用于回测等需要全量数据在内存的场景。
|
||||
func (s *Store) LoadAllKlines(ctx context.Context) (map[string][]model.Kline, error) {
|
||||
|
||||
126
internal/storage/sqlite/task.go
Normal file
126
internal/storage/sqlite/task.go
Normal file
@@ -0,0 +1,126 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"stock-analysis/internal/cache"
|
||||
)
|
||||
|
||||
// 任务相关 SQL。
|
||||
const (
|
||||
saveTaskSQL = `
|
||||
INSERT INTO ingest_task (id, type, status, total, done, success, failed, failed_codes, params, error, started_at, finished_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
type = excluded.type,
|
||||
status = excluded.status,
|
||||
total = excluded.total,
|
||||
done = excluded.done,
|
||||
success = excluded.success,
|
||||
failed = excluded.failed,
|
||||
failed_codes = excluded.failed_codes,
|
||||
params = excluded.params,
|
||||
error = excluded.error,
|
||||
started_at = excluded.started_at,
|
||||
finished_at = excluded.finished_at;`
|
||||
|
||||
updateTaskProgressSQL = `UPDATE ingest_task SET total = ?, done = ?, success = ?, failed = ?, failed_codes = ? WHERE id = ?;`
|
||||
|
||||
finishTaskSQL = `UPDATE ingest_task SET status = ?, error = ?, finished_at = ? WHERE id = ?;`
|
||||
|
||||
getTaskSQL = `SELECT id, type, status, total, done, success, failed, failed_codes, params, error, started_at, finished_at FROM ingest_task WHERE id = ?;`
|
||||
|
||||
listTasksSQL = `SELECT id, type, status, total, done, success, failed, failed_codes, params, error, started_at, finished_at FROM ingest_task ORDER BY started_at DESC LIMIT ?;`
|
||||
)
|
||||
|
||||
// SaveTask 保存或更新任务(INSERT OR REPLACE 语义)。
|
||||
func (s *Store) SaveTask(ctx context.Context, t *cache.Task) error {
|
||||
failedCodes, err := json.Marshal(t.FailedCodes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal failed codes: %w", err)
|
||||
}
|
||||
_, err = s.db.ExecContext(ctx, saveTaskSQL,
|
||||
t.ID, t.Type, t.Status, t.Total, t.Done, t.Success, t.Failed,
|
||||
string(failedCodes), t.Params, t.Error, t.StartedAt, t.FinishedAt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("save task %s: %w", t.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateTaskProgress 更新任务进度(total/done/success/failed/failed_codes)。
|
||||
func (s *Store) UpdateTaskProgress(ctx context.Context, id string, total, done, success, failed int, failedCodes []string) error {
|
||||
data, err := json.Marshal(failedCodes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal failed codes: %w", err)
|
||||
}
|
||||
_, err = s.db.ExecContext(ctx, updateTaskProgressSQL, total, done, success, failed, string(data), id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update task progress %s: %w", id, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FinishTask 标记任务终态(status/error/finished_at)。
|
||||
func (s *Store) FinishTask(ctx context.Context, id string, status cache.TaskStatus, errMsg, finishedAt string) error {
|
||||
_, err := s.db.ExecContext(ctx, finishTaskSQL, status, errMsg, finishedAt, id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("finish task %s: %w", id, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetTask 按 ID 查询任务。
|
||||
func (s *Store) GetTask(ctx context.Context, id string) (*cache.Task, error) {
|
||||
row := s.db.QueryRowContext(ctx, getTaskSQL, id)
|
||||
t, err := scanTask(row)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get task %s: %w", id, err)
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// ListTasks 返回最近的任务列表(按开始时间降序)。
|
||||
func (s *Store) ListTasks(ctx context.Context, limit int) ([]*cache.Task, error) {
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, listTasksSQL, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list tasks: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var tasks []*cache.Task
|
||||
for rows.Next() {
|
||||
t, err := scanTask(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tasks = append(tasks, t)
|
||||
}
|
||||
return tasks, rows.Err()
|
||||
}
|
||||
|
||||
// rowScanner 抽象 *sql.Row 和 *sql.Rows 的 Scan 方法。
|
||||
type rowScanner interface {
|
||||
Scan(dest ...any) error
|
||||
}
|
||||
|
||||
// scanTask 从查询行扫描一条任务记录。
|
||||
func scanTask(row rowScanner) (*cache.Task, error) {
|
||||
var (
|
||||
t cache.Task
|
||||
failedCodes string
|
||||
)
|
||||
if err := row.Scan(&t.ID, &t.Type, &t.Status, &t.Total, &t.Done, &t.Success, &t.Failed,
|
||||
&failedCodes, &t.Params, &t.Error, &t.StartedAt, &t.FinishedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if failedCodes != "" && failedCodes != "null" {
|
||||
_ = json.Unmarshal([]byte(failedCodes), &t.FailedCodes)
|
||||
}
|
||||
return &t, nil
|
||||
}
|
||||
Reference in New Issue
Block a user