func (this *Domain) domainData(out chan interface{}, in chan int8) {
var (
tname = "zhejiang_urltrack_" + time.Now().Format("200601")
btime = timestamp.GetHourTimestamp(-1)
etime = timestamp.GetHourTimestamp(0)
)
sc := hbase.NewScan([]byte(tname), 10000, this.hb)
sc.StartRow = []byte(btime)
sc.StopRow = []byte(etime)
for {
row := sc.Next()
if row == nil {
break
}
ad := strings.TrimSpace(string(row.Columns["base:ad"].Value))
ua := strings.TrimSpace(string(row.Columns["base:ua"].Value))
cids := make([]string, 0, len(row.Columns)-2)
for _, v := range row.Columns {
if string(v.Family) == "cids" {
cids = append(cids, string(v.Qual))
}
}
out <- fmt.Sprintf("%s\t%s\t%s", ad, ua, strings.Join(cids, ","))
}
in <- 1
sc.Close()
}
func (this *UserTrack) otherData(out chan interface{}, in chan int8) {
var (
eghour = timestamp.GetHourTimestamp(-1)
bghour = timestamp.GetHourTimestamp(-25)
)
conf := mongodb.MongodbQueryConf{}
conf.Db = "data_source"
conf.Table = "useraction"
conf.Query = mongodb.MM{"timestamp": mongodb.MM{"$gte": bghour, "$lte": eghour}, "domainId": "0"}
this.mg_big.Query(conf, func(info map[string]interface{}) {
ua := "ua"
ad := convert.ToString(info["AD"])
if u, ok := info["UA"]; ok {
ua = convert.ToString(u)
}
cids := make([]string, 0, len(info["tag"].([]interface{})))
for _, v := range info["tag"].([]interface{}) {
if tags, ok := v.(map[string]interface{}); ok {
if strings.TrimSpace(convert.ToString(tags["tagId"])) != "" {
cids = append(cids, convert.ToString(tags["tagId"]))
}
}
}
out <- fmt.Sprintf("%s\t%s\t%s", ad, ua, strings.Join(cids, ","))
})
in <- 1
}
// 保存广告对应的ad，ua
func (this *JsPut) saveAdvertSet() {
tname := "advert_tj_js_" + timestamp.GetHourTimestamp(-1) + "_"
fname := lib.GetConfVal("jiangsu::data_path") + tname
this.kf.IDAdUaSet(fname, func(info map[string]int) {
tm := timestamp.GetHourTimestamp(-1)
for k, v := range info {
aid := strings.TrimPrefix(k, tname)
// 广告数量统计数据 advert_stats , js_1461016800_1111, 11111
lib.StatisticsData("advert_stats", fmt.Sprintf("js_%s_%s", tm, aid),
convert.ToString(v), "")
}
}, false)
}
// 保存投放轨迹到电信ftp
func (this *JsPut) saveTraceToDianxin() {
var (
ftp = lib.GetConfVal("jiangsu::ftp_path")
ppath = lib.GetConfVal("jiangsu::put_path")
rk = "account.10046.sha1." + time.Now().Add(-time.Hour).Format("200601021504")
fname = ppath + "/" + rk
adcount = 0
)
f, err := os.Create(fname)
if err != nil {
log.Error("创建文件失败", err)
return
}
defer f.Close()
this.kf.AdSet(func(ad string) {
if v, ok := this.areamap[ad]; ok {
f.WriteString(ad + "," + v + "\n")
adcount++
}
})
cmd := exec.Command(ftp, rk)
str, err := cmd.Output()
log.Info(string(str), err)
// 广告数量统计数据 dx_stats , js_1461016800, 11111
lib.StatisticsData("dx_stats", fmt.Sprintf("js_%s", timestamp.GetHourTimestamp(-1)),
convert.ToString(adcount), "")
}
// 保存投放轨迹到投放系统
func (this *JsPut) saveTraceToPutSys() {
rdb, err := lib.GetPutRedisObj("put_redis_proxy_url")
if err != nil {
log.Error("redis连接失败", err)
return
}
go func() {
for {
rdb.Receive()
}
}()
//rdb.SelectDb("1")
adcount := 0
this.kf.AdUaIdsSet(func(ad string, ua string, aids map[string]int8) {
key := ad
if ua != "ua" {
ua = encrypt.DefaultMd5.Encode(encrypt.DefaultBase64.Decode(ua))
key = encrypt.DefaultMd5.Encode(ad + "_" + ua)
}
for aid, _ := range aids {
rdb.HSet(key, "advert:"+aid, aid)
}
rdb.Expire(key, 5400)
adcount++
})
rdb.Flush()
rdb.Close()
// 广告数量统计数据 put_stats , js_1461016800, 11111
lib.StatisticsData("put_stats", fmt.Sprintf("js_%s", timestamp.GetHourTimestamp(-1)),
convert.ToString(adcount), "")
}
func NewZjPut() *ZjPut {
var zj = &ZjPut{}
zj.kf = dbfactory.NewKVFile(fmt.Sprintf("./%s.txt", convert.ToString(time.Now().Unix())))
zj.putTags = make(map[string]map[string]int)
zj.Timestamp = timestamp.GetHourTimestamp(-1)
zj.initPutAdverts()
zj.initPutTags("TAGS_3*", "tb_phone_", "mg_phone_")
zj.initPutTags("TAGS_5*", "url_phone_", "")
return zj
}
// 其他杂项数据获取
func (this *JsPut) otherData(out chan interface{}, in chan int8) {
var datacount = 0
defer func() {
// 统计数据 jiangsu_put , other_1461016800, 11111
lib.StatisticsData("dsource_stats", "js_"+timestamp.GetHourTimestamp(-1)+"_other",
convert.ToString(datacount), "")
}()
fname := "jiangsu_other_" + timestamp.GetHourTimestamp(-1)
if err := lib.GetFdbData(fname, func(val string) {
if v := lib.AddPrefix(val, "mg_"); v != "" {
datacount++
out <- v
}
}); err != nil {
in <- 1
return
}
in <- 1
}
func NewCookiePut() *CookiePut {
var zj = &CookiePut{}
zj.kf = dbfactory.NewKVFile(fmt.Sprintf("./%s.txt", convert.ToString(time.Now().Unix())))
zj.putTags = make(map[string]map[string]int)
zj.Timestamp = timestamp.GetHourTimestamp(-1)
zj.initPutAdverts()
zj.initPutTags("TAGS_5*", "cookie_", "")
log.Info(zj.putAdverts)
log.Info(zj.putTags)
return zj
}
// 标签数据统计
func (this *JsPut) tagDataStats() {
fname := convert.ToString(time.Now().UnixNano()) + "_"
this.kf.IDAdUaSet(fname, func(info map[string]int) {
for k, v := range info {
tagid := strings.TrimPrefix(k, fname)
tagids := strings.Split(tagid, "_")
// 标签统计数据 tags_stats , url_1461016800, 11111
lib.StatisticsData("tags_stats", fmt.Sprintf("js_%s_%s_%s", timestamp.GetHourTimestamp(-1), tagids[0], tagids[1]),
convert.ToString(v), "incr")
}
}, true)
}
// 电商数据获取
func (this *ZjPut) BusinessData(out chan interface{}, in chan int8) {
var datacount = 0
defer func() {
// 统计数据 zhejiang_put , other_1461016800, 11111
lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_business",
convert.ToString(datacount), "")
}()
es, err := lib.GetESObj()
if err != nil {
out <- 1
return
}
var bt = timestamp.GetHourTimestamp(-1)
var et = timestamp.GetHourTimestamp(-73)
var query = elastic.NewRangeQuery("timestamp").Gte(et).Lte(bt)
var sid = ""
res, err := es.Scroll().Index("zhejiang_tb_ad_trace").Type("ad").Query(query).Size(1000).Do()
if err != nil {
log.Error(err)
out <- 1
return
}
sid = res.ScrollId
for {
sres, err := es.Scroll().Index("zhejiang_tb_ad_trace").Type("ad").
Query(query).ScrollId(sid).Size(1000).Do()
if err == elastic.EOS {
break
}
if err != nil {
log.Error(err)
out <- 1
return
}
for _, hit := range sres.Hits.Hits {
item := make(map[string]interface{})
err := json.Unmarshal(*hit.Source, &item)
if err != nil {
continue
}
ad := convert.ToString(item["ad"])
ua := encrypt.DefaultMd5.Encode(convert.ToString(item["ua"]))
cids := item["cids"].([]interface{})
ncids := make(map[string]int)
ncidsary := make([]string, 0, len(cids))
for _, v := range cids {
if vv, ok := lib.TcatBig[convert.ToString(v)]; ok {
ncids[vv] = 1
}
}
for k, _ := range ncids {
ncidsary = append(ncidsary, "tb_"+k)
}
if len(ncidsary) == 0 {
continue
}
datacount++
out <- fmt.Sprintf("%s\t%s\t%s", ad, ua, strings.Join(ncidsary, ","))
}
sid = sres.ScrollId
}
log.Info("电商ok")
in <- 1
}