feat: 添加异步数据入库任务系统，支持全量/增量更新与实时进度展示
新增完整的异步任务管理体系： 1. 新增Redis缓存层实现任务状态实时存储与降级机制 2. 实现并发拉取+批量写入的高性能入库流程 3. 添加任务管理HTTP接口与路由 4. 重构行情控制器使用共享SQLite存储 5. 配套添加Docker Compose Redis启动配置 6. 前端新增任务管理面板支持操作与进度查看
This commit is contained in:
@@ -15,14 +15,15 @@ import (
|
||||
)
|
||||
|
||||
// MarketController 提供行情与回测相关的 HTTP 接口。
|
||||
// store 为共享 SQLite 存储，与 job.Manager 共用同一连接池，避免每次请求重复开关连接。
|
||||
type MarketController struct {
|
||||
svc *service.MarketService
|
||||
dbPath string
|
||||
svc *service.MarketService
|
||||
store *sqlite.Store
|
||||
}
|
||||
|
||||
// NewMarketController 创建行情控制器。
|
||||
func NewMarketController(svc *service.MarketService, dbPath string) *MarketController {
|
||||
return &MarketController{svc: svc, dbPath: dbPath}
|
||||
func NewMarketController(svc *service.MarketService, store *sqlite.Store) *MarketController {
|
||||
return &MarketController{svc: svc, store: store}
|
||||
}
|
||||
|
||||
// ListBonds 处理 GET /api/bonds，返回全量可转债实时行情。
|
||||
@@ -93,13 +94,7 @@ func (c *MarketController) RunBacktest(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
ctx := r.Context()
|
||||
|
||||
store, err := sqlite.Open(c.dbPath)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
defer store.Close()
|
||||
store := c.store
|
||||
|
||||
feed, err := backtest.NewDataFeed(ctx, store)
|
||||
if err != nil {
|
||||
@@ -127,53 +122,53 @@ func (c *MarketController) RunBacktest(w http.ResponseWriter, r *http.Request) {
|
||||
MaxPrice: req.MaxPrice,
|
||||
})
|
||||
case "doublelow":
|
||||
strat = strategy.NewDoubleLow(strategy.DoubleLowParams{
|
||||
TopN: req.TopN,
|
||||
Rebalance: req.Rebalance,
|
||||
MinPrice: req.MinPrice,
|
||||
MaxPrice: req.MaxPrice,
|
||||
MaxPremium: req.MaxPremium,
|
||||
})
|
||||
case "premium":
|
||||
strat = strategy.NewPremium(strategy.PremiumParams{
|
||||
TopN: req.TopN,
|
||||
Rebalance: req.Rebalance,
|
||||
MinPrice: req.MinPrice,
|
||||
MaxPrice: req.MaxPrice,
|
||||
})
|
||||
default:
|
||||
strat = strategy.NewDoubleLow(strategy.DoubleLowParams{
|
||||
TopN: req.TopN,
|
||||
Rebalance: req.Rebalance,
|
||||
MinPrice: req.MinPrice,
|
||||
MaxPrice: req.MaxPrice,
|
||||
MaxPremium: req.MaxPremium,
|
||||
})
|
||||
case "premium":
|
||||
strat = strategy.NewPremium(strategy.PremiumParams{
|
||||
TopN: req.TopN,
|
||||
Rebalance: req.Rebalance,
|
||||
MinPrice: req.MinPrice,
|
||||
MaxPrice: req.MaxPrice,
|
||||
})
|
||||
default:
|
||||
writeError(w, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
|
||||
engine := backtest.NewEngine(params, strat)
|
||||
|
||||
if req.Fundamental {
|
||||
fundamentals, err := store.LoadAllFundamentals(ctx)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, fmt.Errorf("加载基本面数据: %w", err))
|
||||
return
|
||||
}
|
||||
stock2bond, err := store.GetStockBondMap(ctx)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, fmt.Errorf("获取正股映射: %w", err))
|
||||
return
|
||||
}
|
||||
filter := backtest.NewFundamentalFilter(stock2bond)
|
||||
engine.SetFundamentalFilter(filter, fundamentals)
|
||||
log.Printf("基本面过滤器已启用：%d 只正股数据", len(fundamentals))
|
||||
if req.Fundamental {
|
||||
fundamentals, err := store.LoadAllFundamentals(ctx)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, fmt.Errorf("加载基本面数据: %w", err))
|
||||
return
|
||||
}
|
||||
stock2bond, err := store.GetStockBondMap(ctx)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, fmt.Errorf("获取正股映射: %w", err))
|
||||
return
|
||||
}
|
||||
filter := backtest.NewFundamentalFilter(stock2bond)
|
||||
engine.SetFundamentalFilter(filter, fundamentals)
|
||||
log.Printf("基本面过滤器已启用：%d 只正股数据", len(fundamentals))
|
||||
}
|
||||
|
||||
res, err := engine.Run(ctx, feed)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
res, err := engine.Run(ctx, feed)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Printf("backtest done: %s, start=%s, end=%s, totalReturn=%.2f%%, curvePoints=%d, tradeDays=%d",
|
||||
res.Strategy, res.Start, res.End, res.Metrics.TotalReturn*100, len(res.Curve), res.Metrics.TradeDays)
|
||||
log.Printf("backtest done: %s, start=%s, end=%s, totalReturn=%.2f%%, curvePoints=%d, tradeDays=%d",
|
||||
res.Strategy, res.Start, res.End, res.Metrics.TotalReturn*100, len(res.Curve), res.Metrics.TradeDays)
|
||||
|
||||
bench := computeBenchmarkJSON(feed, req.StartDate, req.EndDate)
|
||||
bench := computeBenchmarkJSON(feed, req.StartDate, req.EndDate)
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"strategy": res.Strategy,
|
||||
@@ -251,12 +246,7 @@ type DBBond struct {
|
||||
// ListDBBonds 处理 GET /api/db/bonds，返回数据库中已有的转债列表。
|
||||
func (c *MarketController) ListDBBonds(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
store, err := sqlite.Open(c.dbPath)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
defer store.Close()
|
||||
store := c.store
|
||||
|
||||
codes, err := store.ListCodes(ctx)
|
||||
if err != nil {
|
||||
@@ -282,14 +272,8 @@ func (c *MarketController) ListDBBonds(w http.ResponseWriter, r *http.Request) {
|
||||
// GetDataStatus 处理 GET /api/status，返回数据库状态信息。
|
||||
func (c *MarketController) GetDataStatus(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
status := DataStatus{DBPath: c.dbPath}
|
||||
|
||||
store, err := sqlite.Open(c.dbPath)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusOK, status) // 返回部分信息
|
||||
return
|
||||
}
|
||||
defer store.Close()
|
||||
store := c.store
|
||||
status := DataStatus{DBPath: c.store.Path()}
|
||||
|
||||
codes, err := store.ListCodes(ctx)
|
||||
if err == nil {
|
||||
|
||||
122
internal/controller/task.go
Normal file
122
internal/controller/task.go
Normal file
@@ -0,0 +1,122 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"stock-analysis/internal/cache"
|
||||
"stock-analysis/internal/job"
|
||||
)
|
||||
|
||||
// 任务接口错误（包级哨兵错误，便于测试断言）。
|
||||
var (
|
||||
errBadTaskType = errString("无效的任务类型，支持 full / incremental")
|
||||
errMissingTaskID = errString("缺少任务 ID")
|
||||
errTaskNotFound = errString("任务不存在")
|
||||
errTaskNotRunning = errString("任务不存在或已结束，无法取消")
|
||||
)
|
||||
|
||||
// errString 将字符串转为 error，避免重复 fmt.Errorf 调用。
|
||||
type errString string
|
||||
|
||||
func (e errString) Error() string { return string(e) }
|
||||
|
||||
// TaskController 提供数据入库任务管理的 HTTP 接口。
|
||||
// 任务由 job.Manager 异步执行，状态实时写入 Redis，持久化到 SQLite。
|
||||
type TaskController struct {
|
||||
mgr *job.Manager
|
||||
}
|
||||
|
||||
// NewTaskController 创建任务控制器。
|
||||
func NewTaskController(mgr *job.Manager) *TaskController {
|
||||
return &TaskController{mgr: mgr}
|
||||
}
|
||||
|
||||
// taskRequest 创建任务请求参数。
|
||||
type taskRequest struct {
|
||||
Type string `json:"type"` // full | incremental
|
||||
Since string `json:"since"` // 增量起点（YYYY-MM-DD），可选，留空自动取最后K线日期
|
||||
Limit int `json:"limit"` // 限制处理数量，0 表示全部
|
||||
}
|
||||
|
||||
// CreateTask 处理 POST /api/tasks，创建数据入库任务（全量/增量）。
|
||||
// 任务异步执行，立即返回 pending 状态的任务快照，前端通过轮询 GetTask 获取进度。
|
||||
func (c *TaskController) CreateTask(w http.ResponseWriter, r *http.Request) {
|
||||
var req taskRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
|
||||
var typ cache.TaskType
|
||||
switch req.Type {
|
||||
case "full", "":
|
||||
typ = cache.TaskTypeFull
|
||||
case "incremental":
|
||||
typ = cache.TaskTypeIncremental
|
||||
default:
|
||||
writeError(w, http.StatusBadRequest, errBadTaskType)
|
||||
return
|
||||
}
|
||||
|
||||
task, err := c.mgr.Start(r.Context(), typ, req.Since, req.Limit)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, task)
|
||||
}
|
||||
|
||||
// ListTasks 处理 GET /api/tasks，返回任务列表（按开始时间降序）。
|
||||
// 默认返回最近 50 条，可通过 ?limit=N 调整。
|
||||
// 运行中任务会合并 Redis 实时进度。
|
||||
func (c *TaskController) ListTasks(w http.ResponseWriter, r *http.Request) {
|
||||
limit := 50
|
||||
if q := r.URL.Query().Get("limit"); q != "" {
|
||||
if n, err := strconv.Atoi(q); err == nil && n > 0 {
|
||||
limit = n
|
||||
}
|
||||
}
|
||||
tasks, err := c.mgr.List(r.Context(), limit)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, tasks)
|
||||
}
|
||||
|
||||
// GetTask 处理 GET /api/tasks/{id}，返回单个任务状态。
|
||||
// 优先从 Redis 读取实时状态，回退 SQLite 持久化记录。
|
||||
func (c *TaskController) GetTask(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
if id == "" {
|
||||
writeError(w, http.StatusBadRequest, errMissingTaskID)
|
||||
return
|
||||
}
|
||||
task, err := c.mgr.Get(r.Context(), id)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusNotFound, err)
|
||||
return
|
||||
}
|
||||
if task == nil {
|
||||
writeError(w, http.StatusNotFound, errTaskNotFound)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, task)
|
||||
}
|
||||
|
||||
// CancelTask 处理 POST /api/tasks/{id}/cancel，取消运行中任务。
|
||||
// 仅运行中任务可取消，已完成任务忽略。
|
||||
func (c *TaskController) CancelTask(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
if id == "" {
|
||||
writeError(w, http.StatusBadRequest, errMissingTaskID)
|
||||
return
|
||||
}
|
||||
if ok := c.mgr.Cancel(id); !ok {
|
||||
writeError(w, http.StatusNotFound, errTaskNotRunning)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"message": "任务已取消"})
|
||||
}
|
||||
Reference in New Issue
Block a user