func (th *TailHandler) Click(info []string) { var exstr = th.Script([]byte(strings.Join(info, "\t"))) var exmap map[string]string if err := json.Unmarshal(exstr, &exmap); err != nil { return } if len(exmap) == 0 { return } log.Info(exmap) var ml MLog p, c := th.getIp(info) ml.Province = p ml.City = c ml.AdwId = convert.ToInt(th.getmap(exmap, "pd")) ml.AdId = convert.ToInt(th.getmap(exmap, "hd")) ml.UA = th.getmap(exmap, "ua") ml.AD = th.getmap(exmap, "ad") ml.Click = 1 ml.Clock = convert.ToInt(time.Now().Format("15")) ml.Date = convert.ToInt(timestamp.GetDayTimestamp(0)) ml.Url = strings.TrimSpace(th.get_url(th.getmap(exmap, "ltu"), th.getmap(exmap, "lftu"))) ml.Domain = th.get_domain(ml.Url) ml.Cookie = th.get_cookie(info[9]) ml.CusId = encrypt.DefaultMd5.Encode(ml.AD + ml.UA + ml.Cookie) ml.CusId2 = ml.CusId ml.Money, ml.Fmoney = th.click_money(ml.AdwId, ml.AdId) if _, ok := th.SouceMap[th.getmap(exmap, "pd")]; ok { ml.Source = 1 } th.SaveData(ml) }
// 域名访客找回 func (this CproData) DomainVisitor(pkgId string, cookie string, domain string) error { var ( conn = db.GetHbaseConn() date = timestamp.GetDayTimestamp(0) tableName = "domain-cookie" ) if err := this.createTable(conn, tableName); err != nil { return err } put := hbase.NewPut([]byte(pkgId + "_" + date + "_" + cookie)) put.AddStringValue("base", "date", timestamp.GetDayTimestamp(0)) put.AddStringValue("base", "cookie", cookie) put.AddStringValue("base", "domain", domain) conn.Put(tableName, put) return nil }
func NewShPut() *ShPut { var sh = &ShPut{} sh.kf = dbfactory.NewKVFile(fmt.Sprintf("./%s.txt", convert.ToString(time.Now().Unix()))) sh.putTags = make(map[string]map[string]int) sh.Timestamp = timestamp.GetDayTimestamp(-1) sh.initPutAdverts() sh.initPutTags("TAGS_3*", "tb_", "mg_") sh.initPutTags("TAGS_5*", "url_", "") log.Warn(sh.putAdverts) return sh }
func (th *TailHandler) Pv(info []string) { var ml MLog ud, err := th.pv_param(info[3]) //log.Info(ud["lftu"]) if err != nil { return } if !th.PvParamCheck(ud) { return } p, c := th.getIp(info) ml.Province = p ml.City = c ml.AdwId = convert.ToInt(ud["pd"][0]) ml.AdId = convert.ToInt(ud["hd"][0]) ml.UA = info[7] ml.AD = th.get_ad(ud["ltu"][0]) ml.Click = 0 ml.Clock = convert.ToInt(time.Now().Format("15")) ml.Date = convert.ToInt(timestamp.GetDayTimestamp(0)) ml.Url = strings.TrimSpace(th.get_url(ud["ltu"][0], ud["lftu"][0])) if strings.Contains(ml.Url, "cpro.9xu.com") { log.Error(ud) } ml.Domain = th.get_domain(ml.Url) ml.Cookie = th.get_cookie(info[9]) ml.CusId = encrypt.DefaultMd5.Encode(ml.AD + ml.UA + ml.Cookie) ml.CusId2 = ml.CusId ml.Fmoney = 0 ml.Money = 0 ml.PV = 1 if _, ok := th.SouceMap[ud["pd"][0]]; ok { ml.Source = 1 } //fmt.Println(ml) th.SaveData(ml) }
// 店铺信息获取 func (this *ZjPut) ShopData(out chan interface{}, in chan int8) { var datacount = 0 defer func() { // 统计数据 zhejiang_put , other_1461016800, 11111 lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_shop", convert.ToString(datacount), "") }() es, err := lib.GetESObj() if err != nil { log.Error(err) in <- 1 return } this.shopAdverts = this.GetPutShopInfo() for shopid, shopinfo := range this.shopAdverts { for _, adids := range shopinfo.ShopAdverts { date := timestamp.GetDayTimestamp(adids.Date * -1) var scrollid = "" var query = elastic.NewBoolQuery() query.Must(elastic.NewRangeQuery("timestamp").Gte(date)) query.Must(elastic.NewTermQuery("shop", shopid)) sr, err := es.Scroll().Index("zhejiang_tb_shop_trace").Type("shop"). Query(query).Do() if err != nil { log.Error(err) continue } scrollid = sr.ScrollId for { sres, err := es.Scroll().Index("zhejiang_tb_shop_trace").Type("shop"). Query(query).ScrollId(scrollid).Size(1000).Do() if err == elastic.EOS { break } if err != nil { log.Error(err) out <- 1 return } for _, hit := range sres.Hits.Hits { item := make(map[string]interface{}) err := json.Unmarshal(*hit.Source, &item) if err != nil { continue } ad := convert.ToString(item["ad"]) ua := encrypt.DefaultMd5.Encode(convert.ToString(item["ua"])) datacount++ out <- fmt.Sprintf("%s\t%s\t%s", ad, ua, adids.AdvertId) } scrollid = sres.ScrollId } } } log.Info("店铺ok") in <- 1 }