// 店铺信息获取
func (this *ZjPut) ShopData(out chan interface{}, in chan int8) {
var datacount = 0
defer func() {
// 统计数据 zhejiang_put , other_1461016800, 11111
lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_shop",
convert.ToString(datacount), "")
}()
es, err := lib.GetESObj()
if err != nil {
log.Error(err)
in <- 1
return
}
this.shopAdverts = this.GetPutShopInfo()
for shopid, shopinfo := range this.shopAdverts {
for _, adids := range shopinfo.ShopAdverts {
date := timestamp.GetDayTimestamp(adids.Date * -1)
var scrollid = ""
var query = elastic.NewBoolQuery()
query.Must(elastic.NewRangeQuery("timestamp").Gte(date))
query.Must(elastic.NewTermQuery("shop", shopid))
sr, err := es.Scroll().Index("zhejiang_tb_shop_trace").Type("shop").
Query(query).Do()
if err != nil {
log.Error(err)
continue
}
scrollid = sr.ScrollId
for {
sres, err := es.Scroll().Index("zhejiang_tb_shop_trace").Type("shop").
Query(query).ScrollId(scrollid).Size(1000).Do()
if err == elastic.EOS {
break
}
if err != nil {
log.Error(err)
out <- 1
return
}
for _, hit := range sres.Hits.Hits {
item := make(map[string]interface{})
err := json.Unmarshal(*hit.Source, &item)
if err != nil {
continue
}
ad := convert.ToString(item["ad"])
ua := encrypt.DefaultMd5.Encode(convert.ToString(item["ua"]))
datacount++
out <- fmt.Sprintf("%s\t%s\t%s", ad, ua, adids.AdvertId)
}
scrollid = sres.ScrollId
}
}
}
log.Info("店铺ok")
in <- 1
}
// 电商数据获取
func (this *ZjPut) BusinessData(out chan interface{}, in chan int8) {
var datacount = 0
defer func() {
// 统计数据 zhejiang_put , other_1461016800, 11111
lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_business",
convert.ToString(datacount), "")
}()
es, err := lib.GetESObj()
if err != nil {
out <- 1
return
}
var bt = timestamp.GetHourTimestamp(-1)
var et = timestamp.GetHourTimestamp(-73)
var query = elastic.NewRangeQuery("timestamp").Gte(et).Lte(bt)
var sid = ""
res, err := es.Scroll().Index("zhejiang_tb_ad_trace").Type("ad").Query(query).Size(1000).Do()
if err != nil {
log.Error(err)
out <- 1
return
}
sid = res.ScrollId
for {
sres, err := es.Scroll().Index("zhejiang_tb_ad_trace").Type("ad").
Query(query).ScrollId(sid).Size(1000).Do()
if err == elastic.EOS {
break
}
if err != nil {
log.Error(err)
out <- 1
return
}
for _, hit := range sres.Hits.Hits {
item := make(map[string]interface{})
err := json.Unmarshal(*hit.Source, &item)
if err != nil {
continue
}
ad := convert.ToString(item["ad"])
ua := encrypt.DefaultMd5.Encode(convert.ToString(item["ua"]))
cids := item["cids"].([]interface{})
ncids := make(map[string]int)
ncidsary := make([]string, 0, len(cids))
for _, v := range cids {
if vv, ok := lib.TcatBig[convert.ToString(v)]; ok {
ncids[vv] = 1
}
}
for k, _ := range ncids {
ncidsary = append(ncidsary, "tb_"+k)
}
if len(ncidsary) == 0 {
continue
}
datacount++
out <- fmt.Sprintf("%s\t%s\t%s", ad, ua, strings.Join(ncidsary, ","))
}
sid = sres.ScrollId
}
log.Info("电商ok")
in <- 1
}