Exemplo n.º 1
0
Arquivo: zj.go Projeto: qgweb/new
// 店铺信息获取
func (this *ZjPut) ShopData(out chan interface{}, in chan int8) {
	var datacount = 0
	defer func() {
		// 统计数据 zhejiang_put , other_1461016800, 11111
		lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_shop",
			convert.ToString(datacount), "")
	}()

	es, err := lib.GetESObj()
	if err != nil {
		log.Error(err)
		in <- 1
		return
	}

	this.shopAdverts = this.GetPutShopInfo()
	for shopid, shopinfo := range this.shopAdverts {
		for _, adids := range shopinfo.ShopAdverts {
			date := timestamp.GetDayTimestamp(adids.Date * -1)
			var scrollid = ""
			var query = elastic.NewBoolQuery()
			query.Must(elastic.NewRangeQuery("timestamp").Gte(date))
			query.Must(elastic.NewTermQuery("shop", shopid))

			sr, err := es.Scroll().Index("zhejiang_tb_shop_trace").Type("shop").
				Query(query).Do()
			if err != nil {
				log.Error(err)
				continue
			}
			scrollid = sr.ScrollId
			for {
				sres, err := es.Scroll().Index("zhejiang_tb_shop_trace").Type("shop").
					Query(query).ScrollId(scrollid).Size(1000).Do()

				if err == elastic.EOS {
					break
				}
				if err != nil {
					log.Error(err)
					out <- 1
					return
				}
				for _, hit := range sres.Hits.Hits {
					item := make(map[string]interface{})
					err := json.Unmarshal(*hit.Source, &item)
					if err != nil {
						continue
					}
					ad := convert.ToString(item["ad"])
					ua := encrypt.DefaultMd5.Encode(convert.ToString(item["ua"]))
					datacount++
					out <- fmt.Sprintf("%s\t%s\t%s", ad, ua, adids.AdvertId)
				}

				scrollid = sres.ScrollId
			}
		}
	}
	log.Info("店铺ok")
	in <- 1
}
Exemplo n.º 2
0
Arquivo: zj.go Projeto: qgweb/new
// 电商数据获取
func (this *ZjPut) BusinessData(out chan interface{}, in chan int8) {
	var datacount = 0
	defer func() {
		// 统计数据 zhejiang_put , other_1461016800, 11111
		lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_business",
			convert.ToString(datacount), "")
	}()

	es, err := lib.GetESObj()
	if err != nil {
		out <- 1
		return
	}

	var bt = timestamp.GetHourTimestamp(-1)
	var et = timestamp.GetHourTimestamp(-73)
	var query = elastic.NewRangeQuery("timestamp").Gte(et).Lte(bt)
	var sid = ""
	res, err := es.Scroll().Index("zhejiang_tb_ad_trace").Type("ad").Query(query).Size(1000).Do()
	if err != nil {
		log.Error(err)
		out <- 1
		return
	}
	sid = res.ScrollId
	for {
		sres, err := es.Scroll().Index("zhejiang_tb_ad_trace").Type("ad").
			Query(query).ScrollId(sid).Size(1000).Do()
		if err == elastic.EOS {
			break
		}
		if err != nil {
			log.Error(err)
			out <- 1
			return
		}

		for _, hit := range sres.Hits.Hits {
			item := make(map[string]interface{})
			err := json.Unmarshal(*hit.Source, &item)
			if err != nil {
				continue
			}
			ad := convert.ToString(item["ad"])
			ua := encrypt.DefaultMd5.Encode(convert.ToString(item["ua"]))
			cids := item["cids"].([]interface{})
			ncids := make(map[string]int)
			ncidsary := make([]string, 0, len(cids))
			for _, v := range cids {
				if vv, ok := lib.TcatBig[convert.ToString(v)]; ok {
					ncids[vv] = 1
				}
			}
			for k, _ := range ncids {
				ncidsary = append(ncidsary, "tb_"+k)
			}
			if len(ncidsary) == 0 {
				continue
			}
			datacount++
			out <- fmt.Sprintf("%s\t%s\t%s", ad, ua, strings.Join(ncidsary, ","))
		}

		sid = sres.ScrollId
	}
	log.Info("电商ok")
	in <- 1
}