// Save 保存
func (s AmazonS3) Save(quote market.DailyQuote) error {
// gzip 最高压缩
buffer := new(bytes.Buffer)
w, err := gzip.NewWriterLevel(buffer, gzip.BestCompression)
if err != nil {
return err
}
_, err = w.Write(quote.Marshal())
if err != nil {
return err
}
w.Flush()
w.Close()
unzipped, err := ioutil.ReadAll(buffer)
if err != nil {
return err
}
// 上传
_, err = s.svc.PutObject(&s3.PutObjectInput{
Bucket: aws.String(s.config.Bucket),
Key: aws.String(s.savePath(quote.Market, quote.Date)),
Body: bytes.NewReader(unzipped),
StorageClass: aws.String(s3.ObjectStorageClassReducedRedundancy),
})
return err
}
// crawl 抓取指定日期的市场报价
func (mr marketRecorder) crawl(companies []market.Company, date time.Time) error {
ch := make(chan bool, mr.source.ParallelMax())
defer close(ch)
var wg sync.WaitGroup
wg.Add(len(companies))
_, offset := date.Zone()
dailyQuote := market.DailyQuote{
Market: mr.Market,
Date: date,
UTCOffset: offset,
}
for _, company := range companies {
go func(_market market.Market, _company market.Company, _date time.Time) {
quote, err := mr.source.Crawl(_market, _company, _date)
if err == nil {
dailyQuote.Quotes = append(dailyQuote.Quotes, *quote)
}
<-ch
wg.Done()
}(mr.Market, company, date)
// 限流
ch <- false
}
// 阻塞，直到抓取所有
wg.Wait()
// 保存
err := mr.store.Save(dailyQuote)
if err != nil {
return fmt.Errorf("[%s] 保存上市公司在%s的分时数据时发生错误: %v", mr.Market.Name(), date.Format(datePattern), err)
}
log.Printf("[%s] 上市公司在%s的分时数据已经抓取结束", mr.Market.Name(), date.Format(datePattern))
return nil
}
// Save 保存
func (s AliyunOSS) Save(quote market.DailyQuote) error {
// gzip 最高压缩
buffer := new(bytes.Buffer)
w, err := gzip.NewWriterLevel(buffer, gzip.BestCompression)
if err != nil {
return err
}
_, err = w.Write(quote.Marshal())
if err != nil {
return err
}
w.Flush()
w.Close()
unzipped, err := ioutil.ReadAll(buffer)
if err != nil {
return err
}
// 上传
return s.bucket.PutObject(s.objectKey(quote.Market, quote.Date), bytes.NewReader(unzipped))
}