feat: 添加异步数据入库任务系统,支持全量/增量更新与实时进度展示
新增完整的异步任务管理体系: 1. 新增Redis缓存层实现任务状态实时存储与降级机制 2. 实现并发拉取+批量写入的高性能入库流程 3. 添加任务管理HTTP接口与路由 4. 重构行情控制器使用共享SQLite存储 5. 配套添加Docker Compose Redis启动配置 6. 前端新增任务管理面板支持操作与进度查看
This commit is contained in:
@@ -10,6 +10,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -46,14 +47,26 @@ var historyHosts = []string{
|
||||
|
||||
// Client 是东方财富行情接口客户端,实现 datasource.MarketDataSource。
|
||||
type Client struct {
|
||||
http *http.Client
|
||||
lastReq time.Time
|
||||
http *http.Client
|
||||
lastReq time.Time
|
||||
mu sync.Mutex // 保护 lastReq,支持多 goroutine 并发调用
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
// NewClient 创建东财行情客户端。
|
||||
// NewClient 创建东财行情客户端(默认 2s 节流)。
|
||||
func NewClient() *Client {
|
||||
return &Client{
|
||||
http: &http.Client{Timeout: 15 * time.Second},
|
||||
http: &http.Client{Timeout: 15 * time.Second},
|
||||
interval: requestInterval,
|
||||
}
|
||||
}
|
||||
|
||||
// NewClientWithThrottle 创建可配置节流间隔的客户端。
|
||||
// interval=0 关闭内部节流,由外部全局限速器统一控制(job runner 场景)。
|
||||
func NewClientWithThrottle(interval time.Duration) *Client {
|
||||
return &Client{
|
||||
http: &http.Client{Timeout: 15 * time.Second},
|
||||
interval: interval,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -104,10 +117,16 @@ func (c *Client) fetch(ctx context.Context, reqURL string, out any) error {
|
||||
return json.Unmarshal(body, out)
|
||||
}
|
||||
|
||||
// throttle 保证请求间隔不小于 requestInterval。
|
||||
// throttle 保证请求间隔不小于 interval,线程安全。
|
||||
// interval=0 时为空操作,由外部全局限速器统一控制速率。
|
||||
func (c *Client) throttle() {
|
||||
if c.interval == 0 {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
elapsed := time.Since(c.lastReq)
|
||||
if wait := requestInterval - elapsed; wait > 0 {
|
||||
if wait := c.interval - elapsed; wait > 0 {
|
||||
time.Sleep(wait)
|
||||
}
|
||||
c.lastReq = time.Now()
|
||||
|
||||
Reference in New Issue
Block a user