Beispiel #1
0
// Save 保存
func (s AmazonS3) Save(quote market.DailyQuote) error {

	// gzip 最高压缩
	buffer := new(bytes.Buffer)
	w, err := gzip.NewWriterLevel(buffer, gzip.BestCompression)
	if err != nil {
		return err
	}
	_, err = w.Write(quote.Marshal())
	if err != nil {
		return err
	}
	w.Flush()
	w.Close()

	unzipped, err := ioutil.ReadAll(buffer)
	if err != nil {
		return err
	}

	// 上传
	_, err = s.svc.PutObject(&s3.PutObjectInput{
		Bucket:       aws.String(s.config.Bucket),
		Key:          aws.String(s.savePath(quote.Market, quote.Date)),
		Body:         bytes.NewReader(unzipped),
		StorageClass: aws.String(s3.ObjectStorageClassReducedRedundancy),
	})

	return err
}
Beispiel #2
0
// crawl 抓取指定日期的市场报价
func (mr marketRecorder) crawl(companies []market.Company, date time.Time) error {

	ch := make(chan bool, mr.source.ParallelMax())
	defer close(ch)

	var wg sync.WaitGroup
	wg.Add(len(companies))

	_, offset := date.Zone()

	dailyQuote := market.DailyQuote{
		Market:    mr.Market,
		Date:      date,
		UTCOffset: offset,
	}

	for _, company := range companies {

		go func(_market market.Market, _company market.Company, _date time.Time) {
			quote, err := mr.source.Crawl(_market, _company, _date)
			if err == nil {
				dailyQuote.Quotes = append(dailyQuote.Quotes, *quote)
			}

			<-ch
			wg.Done()
		}(mr.Market, company, date)

		// 限流
		ch <- false
	}
	//	阻塞,直到抓取所有
	wg.Wait()

	// 保存
	err := mr.store.Save(dailyQuote)
	if err != nil {
		return fmt.Errorf("[%s] 保存上市公司在%s的分时数据时发生错误: %v", mr.Market.Name(), date.Format(datePattern), err)
	}

	log.Printf("[%s] 上市公司在%s的分时数据已经抓取结束", mr.Market.Name(), date.Format(datePattern))

	return nil
}
Beispiel #3
0
// Save 保存
func (s AliyunOSS) Save(quote market.DailyQuote) error {

	// gzip 最高压缩
	buffer := new(bytes.Buffer)
	w, err := gzip.NewWriterLevel(buffer, gzip.BestCompression)
	if err != nil {
		return err
	}
	_, err = w.Write(quote.Marshal())
	if err != nil {
		return err
	}
	w.Flush()
	w.Close()

	unzipped, err := ioutil.ReadAll(buffer)
	if err != nil {
		return err
	}

	// 上传
	return s.bucket.PutObject(s.objectKey(quote.Market, quote.Date), bytes.NewReader(unzipped))
}