Example #1
0
File: url.go Project: qgweb/new
func (this *Domain) domainData(out chan interface{}, in chan int8) {
	var (
		tname = "zhejiang_urltrack_" + time.Now().Format("200601")
		btime = timestamp.GetHourTimestamp(-1)
		etime = timestamp.GetHourTimestamp(0)
	)

	sc := hbase.NewScan([]byte(tname), 10000, this.hb)
	sc.StartRow = []byte(btime)
	sc.StopRow = []byte(etime)

	for {
		row := sc.Next()
		if row == nil {
			break
		}

		ad := strings.TrimSpace(string(row.Columns["base:ad"].Value))
		ua := strings.TrimSpace(string(row.Columns["base:ua"].Value))
		cids := make([]string, 0, len(row.Columns)-2)

		for _, v := range row.Columns {
			if string(v.Family) == "cids" {
				cids = append(cids, string(v.Qual))
			}
		}

		out <- fmt.Sprintf("%s\t%s\t%s", ad, ua, strings.Join(cids, ","))
	}
	in <- 1
	sc.Close()
}
Example #2
0
File: other.go Project: qgweb/new
func (this *UserTrack) otherData(out chan interface{}, in chan int8) {
	var (
		eghour = timestamp.GetHourTimestamp(-1)
		bghour = timestamp.GetHourTimestamp(-25)
	)
	conf := mongodb.MongodbQueryConf{}
	conf.Db = "data_source"
	conf.Table = "useraction"
	conf.Query = mongodb.MM{"timestamp": mongodb.MM{"$gte": bghour, "$lte": eghour}, "domainId": "0"}
	this.mg_big.Query(conf, func(info map[string]interface{}) {
		ua := "ua"
		ad := convert.ToString(info["AD"])
		if u, ok := info["UA"]; ok {
			ua = convert.ToString(u)
		}
		cids := make([]string, 0, len(info["tag"].([]interface{})))
		for _, v := range info["tag"].([]interface{}) {
			if tags, ok := v.(map[string]interface{}); ok {
				if strings.TrimSpace(convert.ToString(tags["tagId"])) != "" {
					cids = append(cids, convert.ToString(tags["tagId"]))
				}
			}
		}
		out <- fmt.Sprintf("%s\t%s\t%s", ad, ua, strings.Join(cids, ","))
	})
	in <- 1
}
Example #3
0
File: js.go Project: qgweb/new
// 保存广告对应的ad,ua
func (this *JsPut) saveAdvertSet() {
	tname := "advert_tj_js_" + timestamp.GetHourTimestamp(-1) + "_"
	fname := lib.GetConfVal("jiangsu::data_path") + tname
	this.kf.IDAdUaSet(fname, func(info map[string]int) {
		tm := timestamp.GetHourTimestamp(-1)
		for k, v := range info {
			aid := strings.TrimPrefix(k, tname)
			// 广告数量统计数据 advert_stats , js_1461016800_1111, 11111
			lib.StatisticsData("advert_stats", fmt.Sprintf("js_%s_%s", tm, aid),
				convert.ToString(v), "")
		}
	}, false)
}
Example #4
0
File: js.go Project: qgweb/new
// 保存投放轨迹到电信ftp
func (this *JsPut) saveTraceToDianxin() {
	var (
		ftp     = lib.GetConfVal("jiangsu::ftp_path")
		ppath   = lib.GetConfVal("jiangsu::put_path")
		rk      = "account.10046.sha1." + time.Now().Add(-time.Hour).Format("200601021504")
		fname   = ppath + "/" + rk
		adcount = 0
	)

	f, err := os.Create(fname)
	if err != nil {
		log.Error("创建文件失败", err)
		return
	}
	defer f.Close()

	this.kf.AdSet(func(ad string) {
		if v, ok := this.areamap[ad]; ok {
			f.WriteString(ad + "," + v + "\n")
			adcount++
		}
	})
	cmd := exec.Command(ftp, rk)
	str, err := cmd.Output()
	log.Info(string(str), err)

	// 广告数量统计数据 dx_stats , js_1461016800, 11111
	lib.StatisticsData("dx_stats", fmt.Sprintf("js_%s", timestamp.GetHourTimestamp(-1)),
		convert.ToString(adcount), "")
}
Example #5
0
File: js.go Project: qgweb/new
// 保存投放轨迹到投放系统
func (this *JsPut) saveTraceToPutSys() {
	rdb, err := lib.GetPutRedisObj("put_redis_proxy_url")
	if err != nil {
		log.Error("redis连接失败", err)
		return
	}
	go func() {
		for {
			rdb.Receive()
		}
	}()
	//rdb.SelectDb("1")
	adcount := 0
	this.kf.AdUaIdsSet(func(ad string, ua string, aids map[string]int8) {
		key := ad
		if ua != "ua" {
			ua = encrypt.DefaultMd5.Encode(encrypt.DefaultBase64.Decode(ua))
			key = encrypt.DefaultMd5.Encode(ad + "_" + ua)
		}
		for aid, _ := range aids {
			rdb.HSet(key, "advert:"+aid, aid)
		}
		rdb.Expire(key, 5400)
		adcount++
	})
	rdb.Flush()
	rdb.Close()
	// 广告数量统计数据 put_stats , js_1461016800, 11111
	lib.StatisticsData("put_stats", fmt.Sprintf("js_%s", timestamp.GetHourTimestamp(-1)),
		convert.ToString(adcount), "")
}
Example #6
0
File: zj.go Project: qgweb/new
func NewZjPut() *ZjPut {
	var zj = &ZjPut{}
	zj.kf = dbfactory.NewKVFile(fmt.Sprintf("./%s.txt", convert.ToString(time.Now().Unix())))
	zj.putTags = make(map[string]map[string]int)
	zj.Timestamp = timestamp.GetHourTimestamp(-1)
	zj.initPutAdverts()
	zj.initPutTags("TAGS_3*", "tb_phone_", "mg_phone_")
	zj.initPutTags("TAGS_5*", "url_phone_", "")
	return zj
}
Example #7
0
File: js.go Project: qgweb/new
// 其他杂项数据获取
func (this *JsPut) otherData(out chan interface{}, in chan int8) {
	var datacount = 0
	defer func() {
		// 统计数据 jiangsu_put , other_1461016800, 11111
		lib.StatisticsData("dsource_stats", "js_"+timestamp.GetHourTimestamp(-1)+"_other",
			convert.ToString(datacount), "")
	}()

	fname := "jiangsu_other_" + timestamp.GetHourTimestamp(-1)
	if err := lib.GetFdbData(fname, func(val string) {
		if v := lib.AddPrefix(val, "mg_"); v != "" {
			datacount++
			out <- v
		}
	}); err != nil {
		in <- 1
		return
	}
	in <- 1
}
Example #8
0
File: ck.go Project: qgweb/new
func NewCookiePut() *CookiePut {
	var zj = &CookiePut{}
	zj.kf = dbfactory.NewKVFile(fmt.Sprintf("./%s.txt", convert.ToString(time.Now().Unix())))
	zj.putTags = make(map[string]map[string]int)
	zj.Timestamp = timestamp.GetHourTimestamp(-1)
	zj.initPutAdverts()
	zj.initPutTags("TAGS_5*", "cookie_", "")
	log.Info(zj.putAdverts)
	log.Info(zj.putTags)
	return zj
}
Example #9
0
File: js.go Project: qgweb/new
// 标签数据统计
func (this *JsPut) tagDataStats() {
	fname := convert.ToString(time.Now().UnixNano()) + "_"
	this.kf.IDAdUaSet(fname, func(info map[string]int) {
		for k, v := range info {
			tagid := strings.TrimPrefix(k, fname)
			tagids := strings.Split(tagid, "_")
			// 标签统计数据 tags_stats , url_1461016800, 11111
			lib.StatisticsData("tags_stats", fmt.Sprintf("js_%s_%s_%s", timestamp.GetHourTimestamp(-1), tagids[0], tagids[1]),
				convert.ToString(v), "incr")
		}
	}, true)
}
Example #10
0
File: zj.go Project: qgweb/new
// 电商数据获取
func (this *ZjPut) BusinessData(out chan interface{}, in chan int8) {
	var datacount = 0
	defer func() {
		// 统计数据 zhejiang_put , other_1461016800, 11111
		lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_business",
			convert.ToString(datacount), "")
	}()

	es, err := lib.GetESObj()
	if err != nil {
		out <- 1
		return
	}

	var bt = timestamp.GetHourTimestamp(-1)
	var et = timestamp.GetHourTimestamp(-73)
	var query = elastic.NewRangeQuery("timestamp").Gte(et).Lte(bt)
	var sid = ""
	res, err := es.Scroll().Index("zhejiang_tb_ad_trace").Type("ad").Query(query).Size(1000).Do()
	if err != nil {
		log.Error(err)
		out <- 1
		return
	}
	sid = res.ScrollId
	for {
		sres, err := es.Scroll().Index("zhejiang_tb_ad_trace").Type("ad").
			Query(query).ScrollId(sid).Size(1000).Do()
		if err == elastic.EOS {
			break
		}
		if err != nil {
			log.Error(err)
			out <- 1
			return
		}

		for _, hit := range sres.Hits.Hits {
			item := make(map[string]interface{})
			err := json.Unmarshal(*hit.Source, &item)
			if err != nil {
				continue
			}
			ad := convert.ToString(item["ad"])
			ua := encrypt.DefaultMd5.Encode(convert.ToString(item["ua"]))
			cids := item["cids"].([]interface{})
			ncids := make(map[string]int)
			ncidsary := make([]string, 0, len(cids))
			for _, v := range cids {
				if vv, ok := lib.TcatBig[convert.ToString(v)]; ok {
					ncids[vv] = 1
				}
			}
			for k, _ := range ncids {
				ncidsary = append(ncidsary, "tb_"+k)
			}
			if len(ncidsary) == 0 {
				continue
			}
			datacount++
			out <- fmt.Sprintf("%s\t%s\t%s", ad, ua, strings.Join(ncidsary, ","))
		}

		sid = sres.ScrollId
	}
	log.Info("电商ok")
	in <- 1
}