// crawl 抓取指定日期的市场报价 func (mr marketRecorder) crawl(companies []market.Company, date time.Time) error { ch := make(chan bool, mr.source.ParallelMax()) defer close(ch) var wg sync.WaitGroup wg.Add(len(companies)) _, offset := date.Zone() dailyQuote := market.DailyQuote{ Market: mr.Market, Date: date, UTCOffset: offset, } for _, company := range companies { go func(_market market.Market, _company market.Company, _date time.Time) { quote, err := mr.source.Crawl(_market, _company, _date) if err == nil { dailyQuote.Quotes = append(dailyQuote.Quotes, *quote) } <-ch wg.Done() }(mr.Market, company, date) // 限流 ch <- false } // 阻塞,直到抓取所有 wg.Wait() // 保存 err := mr.store.Save(dailyQuote) if err != nil { return fmt.Errorf("[%s] 保存上市公司在%s的分时数据时发生错误: %v", mr.Market.Name(), date.Format(datePattern), err) } log.Printf("[%s] 上市公司在%s的分时数据已经抓取结束", mr.Market.Name(), date.Format(datePattern)) return nil }