feat: 添加异步数据入库任务系统,支持全量/增量更新与实时进度展示

新增完整的异步任务管理体系:
1. 新增Redis缓存层实现任务状态实时存储与降级机制
2. 实现并发拉取+批量写入的高性能入库流程
3. 添加任务管理HTTP接口与路由
4. 重构行情控制器使用共享SQLite存储
5. 配套添加Docker Compose Redis启动配置
6. 前端新增任务管理面板支持操作与进度查看
This commit is contained in:
2026-08-18 10:09:52 +08:00
parent e28bfec957
commit 273fc90a95
18 changed files with 1682 additions and 80 deletions

View File

@@ -15,14 +15,15 @@ import (
)
// MarketController 提供行情与回测相关的 HTTP 接口。
// store 为共享 SQLite 存储,与 job.Manager 共用同一连接池,避免每次请求重复开关连接。
type MarketController struct {
svc *service.MarketService
dbPath string
svc *service.MarketService
store *sqlite.Store
}
// NewMarketController 创建行情控制器。
func NewMarketController(svc *service.MarketService, dbPath string) *MarketController {
return &MarketController{svc: svc, dbPath: dbPath}
func NewMarketController(svc *service.MarketService, store *sqlite.Store) *MarketController {
return &MarketController{svc: svc, store: store}
}
// ListBonds 处理 GET /api/bonds,返回全量可转债实时行情。
@@ -93,13 +94,7 @@ func (c *MarketController) RunBacktest(w http.ResponseWriter, r *http.Request) {
}
ctx := r.Context()
store, err := sqlite.Open(c.dbPath)
if err != nil {
writeError(w, http.StatusInternalServerError, err)
return
}
defer store.Close()
store := c.store
feed, err := backtest.NewDataFeed(ctx, store)
if err != nil {
@@ -127,53 +122,53 @@ func (c *MarketController) RunBacktest(w http.ResponseWriter, r *http.Request) {
MaxPrice: req.MaxPrice,
})
case "doublelow":
strat = strategy.NewDoubleLow(strategy.DoubleLowParams{
TopN: req.TopN,
Rebalance: req.Rebalance,
MinPrice: req.MinPrice,
MaxPrice: req.MaxPrice,
MaxPremium: req.MaxPremium,
})
case "premium":
strat = strategy.NewPremium(strategy.PremiumParams{
TopN: req.TopN,
Rebalance: req.Rebalance,
MinPrice: req.MinPrice,
MaxPrice: req.MaxPrice,
})
default:
strat = strategy.NewDoubleLow(strategy.DoubleLowParams{
TopN: req.TopN,
Rebalance: req.Rebalance,
MinPrice: req.MinPrice,
MaxPrice: req.MaxPrice,
MaxPremium: req.MaxPremium,
})
case "premium":
strat = strategy.NewPremium(strategy.PremiumParams{
TopN: req.TopN,
Rebalance: req.Rebalance,
MinPrice: req.MinPrice,
MaxPrice: req.MaxPrice,
})
default:
writeError(w, http.StatusBadRequest, err)
return
}
engine := backtest.NewEngine(params, strat)
if req.Fundamental {
fundamentals, err := store.LoadAllFundamentals(ctx)
if err != nil {
writeError(w, http.StatusInternalServerError, fmt.Errorf("加载基本面数据: %w", err))
return
}
stock2bond, err := store.GetStockBondMap(ctx)
if err != nil {
writeError(w, http.StatusInternalServerError, fmt.Errorf("获取正股映射: %w", err))
return
}
filter := backtest.NewFundamentalFilter(stock2bond)
engine.SetFundamentalFilter(filter, fundamentals)
log.Printf("基本面过滤器已启用:%d 只正股数据", len(fundamentals))
if req.Fundamental {
fundamentals, err := store.LoadAllFundamentals(ctx)
if err != nil {
writeError(w, http.StatusInternalServerError, fmt.Errorf("加载基本面数据: %w", err))
return
}
stock2bond, err := store.GetStockBondMap(ctx)
if err != nil {
writeError(w, http.StatusInternalServerError, fmt.Errorf("获取正股映射: %w", err))
return
}
filter := backtest.NewFundamentalFilter(stock2bond)
engine.SetFundamentalFilter(filter, fundamentals)
log.Printf("基本面过滤器已启用:%d 只正股数据", len(fundamentals))
}
res, err := engine.Run(ctx, feed)
if err != nil {
writeError(w, http.StatusInternalServerError, err)
return
}
res, err := engine.Run(ctx, feed)
if err != nil {
writeError(w, http.StatusInternalServerError, err)
return
}
log.Printf("backtest done: %s, start=%s, end=%s, totalReturn=%.2f%%, curvePoints=%d, tradeDays=%d",
res.Strategy, res.Start, res.End, res.Metrics.TotalReturn*100, len(res.Curve), res.Metrics.TradeDays)
log.Printf("backtest done: %s, start=%s, end=%s, totalReturn=%.2f%%, curvePoints=%d, tradeDays=%d",
res.Strategy, res.Start, res.End, res.Metrics.TotalReturn*100, len(res.Curve), res.Metrics.TradeDays)
bench := computeBenchmarkJSON(feed, req.StartDate, req.EndDate)
bench := computeBenchmarkJSON(feed, req.StartDate, req.EndDate)
writeJSON(w, http.StatusOK, map[string]any{
"strategy": res.Strategy,
@@ -251,12 +246,7 @@ type DBBond struct {
// ListDBBonds 处理 GET /api/db/bonds,返回数据库中已有的转债列表。
func (c *MarketController) ListDBBonds(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
store, err := sqlite.Open(c.dbPath)
if err != nil {
writeError(w, http.StatusInternalServerError, err)
return
}
defer store.Close()
store := c.store
codes, err := store.ListCodes(ctx)
if err != nil {
@@ -282,14 +272,8 @@ func (c *MarketController) ListDBBonds(w http.ResponseWriter, r *http.Request) {
// GetDataStatus 处理 GET /api/status,返回数据库状态信息。
func (c *MarketController) GetDataStatus(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
status := DataStatus{DBPath: c.dbPath}
store, err := sqlite.Open(c.dbPath)
if err != nil {
writeJSON(w, http.StatusOK, status) // 返回部分信息
return
}
defer store.Close()
store := c.store
status := DataStatus{DBPath: c.store.Path()}
codes, err := store.ListCodes(ctx)
if err == nil {