// 域名找回信息获取 func (this *ZjPut) VisitorData(out chan interface{}, in chan int8) { var datacount = 0 defer func() { // 统计数据 zhejiang_put , other_1461016800, 11111 lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_visitor", convert.ToString(datacount), "") }() m, err := lib.GetMongoObj() if err != nil { log.Error(err) in <- 1 return } defer m.Close() qconf := mongodb.MongodbQueryConf{} qconf.Db = "data_source" qconf.Table = "zhejiang_visitor" qconf.Query = mongodb.MM{} m.Query(qconf, func(info map[string]interface{}) { ad := convert.ToString(info["ad"]) ua := encrypt.DefaultMd5.Encode(encrypt.DefaultBase64.Decode(convert.ToString(info["ua"]))) aids := convert.ToString(info["aids"]) datacount++ out <- fmt.Sprintf("%s\t%s\t%s", ad, ua, aids) }) log.Info("访客ok") in <- 1 }
// 获取当前时间戳 // 2006-01-02 15:04:05 func GetTimestamp(v ...interface{}) string { if len(v) > 0 { a, _ := time.ParseInLocation("2006-01-02 15:04:05", v[0].(string), time.Local) return convert.ToString(a.Unix()) } return convert.ToString(time.Now().Unix()) }
// 标签数据统计 func (this *ZjPut) tagDataStats() { fname := convert.ToString(time.Now().UnixNano()) + "_" this.kf.IDAdUaSet(fname, func(info map[string]int) { for k, v := range info { tagid := strings.TrimPrefix(k, fname) tagids := strings.Split(tagid, "_") // 标签统计数据 tags_stats , url_1461016800, 11111 lib.StatisticsData("tags_stats", fmt.Sprintf("zj_%s_%s_%s", this.Timestamp, tagids[0], tagids[1]), convert.ToString(v), "incr") } }, true) }
// 保存投放轨迹到投放系统 func (this *ZjPut) saveTraceToPutSys() { rdb, err := lib.GetPutRedisObj("put_redis_proxy_url") if err != nil { log.Error("redis连接失败", err) return } go func() { for { rdb.Receive() } }() //rdb.SelectDb("1") adcount := 0 this.kf.AdUaIdsSet(func(ad string, ua string, aids map[string]int8) { key := ad if ua != "ua" { key = encrypt.DefaultMd5.Encode(ad + "_" + ua) } for aid, _ := range aids { rdb.HSet(key, "advert:"+aid, aid) } rdb.Expire(key, 5400) adcount++ }) rdb.Flush() rdb.Close() // 广告数量统计数据 put_stats , Zj_1461016800, 11111 lib.StatisticsData("put_stats", fmt.Sprintf("zj_%s", this.Timestamp), convert.ToString(adcount), "") }
func (this *KVFile) WriteFile() error { wf, err := this.createFile() if err != nil { return err } defer wf.Close() this.overChan = make(chan int8, len(this.funAry)) tmpOverChan := make(chan int8) for _, f := range this.funAry { go func(fn ReadFun) { fn(this.rchan, this.overChan) }(f) } go func() { for { v, ok := <-this.rchan if !ok { tmpOverChan <- 1 break } wf.WriteString(convert.ToString(v) + "\n") } }() for i := 0; i < cap(this.overChan); i++ { <-this.overChan } close(this.rchan) <-tmpOverChan this.sortuniqm() return nil }
// 保存投放轨迹到电信redis func (this *ZjPut) saveTraceToDianxin() { var ( db = lib.GetConfVal("zhejiang::dx_redis_db") pwd = lib.GetConfVal("zhejiang::dx_redis_pwd") adcount = 0 ) rdb, err := lib.GetZJDxRedisObj() if err != nil { log.Error("redis连接失败", err) return } rdb.Auth(pwd) rdb.SelectDb(db) rdb.FlushDb() // ua默认md5加密 go func() { for { rdb.Receive() } }() this.kf.AdUaIdsSet(func(ad string, ua string, ids map[string]int8) { var key = ad + "|" + strings.ToUpper(ua) rdb.Set(key, "1") adcount++ }) rdb.Flush() rdb.Close() // 广告数量统计数据 dx_stats , Zj_1461016800, 11111 lib.StatisticsData("dx_stats", fmt.Sprintf("zj_%s", this.Timestamp), convert.ToString(adcount), "") }
func NewZjPut() *ZjPut { var zj = &ZjPut{} zj.kf = dbfactory.NewKVFile(fmt.Sprintf("./%s.txt", convert.ToString(time.Now().Unix()))) zj.putTags = make(map[string]map[string]int) zj.Timestamp = timestamp.GetHourTimestamp(-1) zj.initPutAdverts() zj.initPutTags("TAGS_3*", "tb_phone_", "mg_phone_") zj.initPutTags("TAGS_5*", "url_phone_", "") return zj }
func NewShPut() *ShPut { var sh = &ShPut{} sh.kf = dbfactory.NewKVFile(fmt.Sprintf("./%s.txt", convert.ToString(time.Now().Unix()))) sh.putTags = make(map[string]map[string]int) sh.Timestamp = timestamp.GetDayTimestamp(-1) sh.initPutAdverts() sh.initPutTags("TAGS_3*", "tb_", "mg_") sh.initPutTags("TAGS_5*", "url_", "") log.Warn(sh.putAdverts) return sh }
func NewCookiePut() *CookiePut { var zj = &CookiePut{} zj.kf = dbfactory.NewKVFile(fmt.Sprintf("./%s.txt", convert.ToString(time.Now().Unix()))) zj.putTags = make(map[string]map[string]int) zj.Timestamp = timestamp.GetHourTimestamp(-1) zj.initPutAdverts() zj.initPutTags("TAGS_5*", "cookie_", "") log.Info(zj.putAdverts) log.Info(zj.putTags) return zj }
func getGeop(w http.ResponseWriter, r *http.Request) { ad := strings.TrimSpace(r.URL.Query().Get("ad")) mlink, err := db.Get() lon := "" lat := "" if err != nil { log.Error(err) w.Write([]byte("")) return } defer mlink.Close() if ad == "" { w.WriteHeader(404) w.Write([]byte("")) return } qconf := mongodb.MongodbQueryConf{} qconf.Db = "lonlat_data" qconf.Table = "tbl_map" qconf.Select = mongodb.MM{"lon": 1, "lat": 1} qconf.Query = mongodb.MM{"ad": ad} info, err := mlink.One(qconf) if err != nil { log.Error(err) w.Write([]byte("")) return } if v, ok := info["lon"]; ok { lon = convert.ToString(v) } if v, ok := info["lat"]; ok { lat = convert.ToString(v) } w.Write([]byte(fmt.Sprintf("%s,%s", lon, lat))) }
// 保存广告对应的ad,ua func (this *ZjPut) saveAdvertSet() { tname := "advert_tj_zj_" + this.Timestamp + "_" fname := lib.GetConfVal("zhejiang::data_path") + tname this.kf.IDAdUaSet(fname, func(info map[string]int) { tm := this.Timestamp for k, v := range info { aid := strings.TrimPrefix(k, tname) // 广告数量统计数据 advert_stats , zj_1461016800_1111, 11111 lib.StatisticsData("advert_stats", fmt.Sprintf("zj_%s_%s", tm, aid), convert.ToString(v), "") } }, false) }
// 域名数据获取 func (this *ShPut) domainData(out chan interface{}, in chan int8) { var datacount = 0 defer func() { // 统计数据 jiangsu_put , url_1461016800, 11111 lib.StatisticsData("dsource_stats", "sh_"+this.Timestamp+"_url", convert.ToString(datacount), "") }() fname := "shanghai_url_" + this.Timestamp if err := lib.GetFdbData(fname, func(val string) { if v := lib.AddPrefix(val, "url_"); v != "" { datacount++ out <- v } }); err != nil { in <- 1 return } log.Info("域名ok") in <- 1 }
// 其他杂项数据获取 func (this *ZjPut) otherData(out chan interface{}, in chan int8) { var datacount = 0 defer func() { // 统计数据 zhejiang_put , other_1461016800, 11111 lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_other", convert.ToString(datacount), "") }() fname := "zhejiang_other_" + this.Timestamp if err := lib.GetFdbData(fname, func(val string) { if v := lib.AddPrefix(val, "mg_"); v != "" { datacount++ out <- v } }); err != nil { in <- 1 return } log.Info("其他ok") in <- 1 }
// 无限数据获取 func (this *CookiePut) coikieData(out chan interface{}, in chan int8) { var datacount = 0 defer func() { // 统计数据 jiangsu_put , url_1461016800, 11111 lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_cookie", convert.ToString(datacount), "") }() fname := "zhejiang_cookie_" + this.Timestamp if err := lib.GetFdbData(fname, func(val string) { if v := lib.AddPrefix2(val, "cookie_"); v != "" { datacount++ out <- v } }); err != nil { in <- 1 return } log.Info("cookieok") in <- 1 }
// 获取月的时间戳 func GetMonthTimestamp(month int) string { d := time.Now().AddDate(0, month, 0).Format("200601") a, _ := time.ParseInLocation("200601", d, time.Local) return convert.ToString(a.Unix()) }
// 获取某个小时的时间戳 func GetHourTimestamp(hour int) string { d := time.Now().Add(time.Hour * time.Duration(hour)).Format("2006010215") a, _ := time.ParseInLocation("2006010215", d, time.Local) return convert.ToString(a.Unix()) }
// 店铺信息获取 func (this *ZjPut) ShopData(out chan interface{}, in chan int8) { var datacount = 0 defer func() { // 统计数据 zhejiang_put , other_1461016800, 11111 lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_shop", convert.ToString(datacount), "") }() es, err := lib.GetESObj() if err != nil { log.Error(err) in <- 1 return } this.shopAdverts = this.GetPutShopInfo() for shopid, shopinfo := range this.shopAdverts { for _, adids := range shopinfo.ShopAdverts { date := timestamp.GetDayTimestamp(adids.Date * -1) var scrollid = "" var query = elastic.NewBoolQuery() query.Must(elastic.NewRangeQuery("timestamp").Gte(date)) query.Must(elastic.NewTermQuery("shop", shopid)) sr, err := es.Scroll().Index("zhejiang_tb_shop_trace").Type("shop"). Query(query).Do() if err != nil { log.Error(err) continue } scrollid = sr.ScrollId for { sres, err := es.Scroll().Index("zhejiang_tb_shop_trace").Type("shop"). Query(query).ScrollId(scrollid).Size(1000).Do() if err == elastic.EOS { break } if err != nil { log.Error(err) out <- 1 return } for _, hit := range sres.Hits.Hits { item := make(map[string]interface{}) err := json.Unmarshal(*hit.Source, &item) if err != nil { continue } ad := convert.ToString(item["ad"]) ua := encrypt.DefaultMd5.Encode(convert.ToString(item["ua"])) datacount++ out <- fmt.Sprintf("%s\t%s\t%s", ad, ua, adids.AdvertId) } scrollid = sres.ScrollId } } } log.Info("店铺ok") in <- 1 }
// 电商数据获取 func (this *ZjPut) BusinessData(out chan interface{}, in chan int8) { var datacount = 0 defer func() { // 统计数据 zhejiang_put , other_1461016800, 11111 lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_business", convert.ToString(datacount), "") }() es, err := lib.GetESObj() if err != nil { out <- 1 return } var bt = timestamp.GetHourTimestamp(-1) var et = timestamp.GetHourTimestamp(-73) var query = elastic.NewRangeQuery("timestamp").Gte(et).Lte(bt) var sid = "" res, err := es.Scroll().Index("zhejiang_tb_ad_trace").Type("ad").Query(query).Size(1000).Do() if err != nil { log.Error(err) out <- 1 return } sid = res.ScrollId for { sres, err := es.Scroll().Index("zhejiang_tb_ad_trace").Type("ad"). Query(query).ScrollId(sid).Size(1000).Do() if err == elastic.EOS { break } if err != nil { log.Error(err) out <- 1 return } for _, hit := range sres.Hits.Hits { item := make(map[string]interface{}) err := json.Unmarshal(*hit.Source, &item) if err != nil { continue } ad := convert.ToString(item["ad"]) ua := encrypt.DefaultMd5.Encode(convert.ToString(item["ua"])) cids := item["cids"].([]interface{}) ncids := make(map[string]int) ncidsary := make([]string, 0, len(cids)) for _, v := range cids { if vv, ok := lib.TcatBig[convert.ToString(v)]; ok { ncids[vv] = 1 } } for k, _ := range ncids { ncidsary = append(ncidsary, "tb_"+k) } if len(ncidsary) == 0 { continue } datacount++ out <- fmt.Sprintf("%s\t%s\t%s", ad, ua, strings.Join(ncidsary, ",")) } sid = sres.ScrollId } log.Info("电商ok") in <- 1 }
// 获取任意一天整点时间戳 func GetDayTimestamp(day int) string { d := time.Now().AddDate(0, 0, day).Format("20060102") a, _ := time.ParseInLocation("20060102", d, time.Local) return convert.ToString(a.Unix()) }