// 域名找回信息获取
func (this *ZjPut) VisitorData(out chan interface{}, in chan int8) {
var datacount = 0
defer func() {
// 统计数据 zhejiang_put , other_1461016800, 11111
lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_visitor",
convert.ToString(datacount), "")
}()
m, err := lib.GetMongoObj()
if err != nil {
log.Error(err)
in <- 1
return
}
defer m.Close()
qconf := mongodb.MongodbQueryConf{}
qconf.Db = "data_source"
qconf.Table = "zhejiang_visitor"
qconf.Query = mongodb.MM{}
m.Query(qconf, func(info map[string]interface{}) {
ad := convert.ToString(info["ad"])
ua := encrypt.DefaultMd5.Encode(encrypt.DefaultBase64.Decode(convert.ToString(info["ua"])))
aids := convert.ToString(info["aids"])
datacount++
out <- fmt.Sprintf("%s\t%s\t%s", ad, ua, aids)
})
log.Info("访客ok")
in <- 1
}
// 获取当前时间戳
// 2006-01-02 15:04:05
func GetTimestamp(v ...interface{}) string {
if len(v) > 0 {
a, _ := time.ParseInLocation("2006-01-02 15:04:05", v[0].(string), time.Local)
return convert.ToString(a.Unix())
}
return convert.ToString(time.Now().Unix())
}
// 标签数据统计
func (this *ZjPut) tagDataStats() {
fname := convert.ToString(time.Now().UnixNano()) + "_"
this.kf.IDAdUaSet(fname, func(info map[string]int) {
for k, v := range info {
tagid := strings.TrimPrefix(k, fname)
tagids := strings.Split(tagid, "_")
// 标签统计数据 tags_stats , url_1461016800, 11111
lib.StatisticsData("tags_stats", fmt.Sprintf("zj_%s_%s_%s", this.Timestamp, tagids[0], tagids[1]),
convert.ToString(v), "incr")
}
}, true)
}
// 保存投放轨迹到投放系统
func (this *ZjPut) saveTraceToPutSys() {
rdb, err := lib.GetPutRedisObj("put_redis_proxy_url")
if err != nil {
log.Error("redis连接失败", err)
return
}
go func() {
for {
rdb.Receive()
}
}()
//rdb.SelectDb("1")
adcount := 0
this.kf.AdUaIdsSet(func(ad string, ua string, aids map[string]int8) {
key := ad
if ua != "ua" {
key = encrypt.DefaultMd5.Encode(ad + "_" + ua)
}
for aid, _ := range aids {
rdb.HSet(key, "advert:"+aid, aid)
}
rdb.Expire(key, 5400)
adcount++
})
rdb.Flush()
rdb.Close()
// 广告数量统计数据 put_stats , Zj_1461016800, 11111
lib.StatisticsData("put_stats", fmt.Sprintf("zj_%s", this.Timestamp),
convert.ToString(adcount), "")
}
func (this *KVFile) WriteFile() error {
wf, err := this.createFile()
if err != nil {
return err
}
defer wf.Close()
this.overChan = make(chan int8, len(this.funAry))
tmpOverChan := make(chan int8)
for _, f := range this.funAry {
go func(fn ReadFun) {
fn(this.rchan, this.overChan)
}(f)
}
go func() {
for {
v, ok := <-this.rchan
if !ok {
tmpOverChan <- 1
break
}
wf.WriteString(convert.ToString(v) + "\n")
}
}()
for i := 0; i < cap(this.overChan); i++ {
<-this.overChan
}
close(this.rchan)
<-tmpOverChan
this.sortuniqm()
return nil
}
// 保存投放轨迹到电信redis
func (this *ZjPut) saveTraceToDianxin() {
var (
db = lib.GetConfVal("zhejiang::dx_redis_db")
pwd = lib.GetConfVal("zhejiang::dx_redis_pwd")
adcount = 0
)
rdb, err := lib.GetZJDxRedisObj()
if err != nil {
log.Error("redis连接失败", err)
return
}
rdb.Auth(pwd)
rdb.SelectDb(db)
rdb.FlushDb()
// ua默认md5加密
go func() {
for {
rdb.Receive()
}
}()
this.kf.AdUaIdsSet(func(ad string, ua string, ids map[string]int8) {
var key = ad + "|" + strings.ToUpper(ua)
rdb.Set(key, "1")
adcount++
})
rdb.Flush()
rdb.Close()
// 广告数量统计数据 dx_stats , Zj_1461016800, 11111
lib.StatisticsData("dx_stats", fmt.Sprintf("zj_%s", this.Timestamp),
convert.ToString(adcount), "")
}
func NewZjPut() *ZjPut {
var zj = &ZjPut{}
zj.kf = dbfactory.NewKVFile(fmt.Sprintf("./%s.txt", convert.ToString(time.Now().Unix())))
zj.putTags = make(map[string]map[string]int)
zj.Timestamp = timestamp.GetHourTimestamp(-1)
zj.initPutAdverts()
zj.initPutTags("TAGS_3*", "tb_phone_", "mg_phone_")
zj.initPutTags("TAGS_5*", "url_phone_", "")
return zj
}
func NewShPut() *ShPut {
var sh = &ShPut{}
sh.kf = dbfactory.NewKVFile(fmt.Sprintf("./%s.txt", convert.ToString(time.Now().Unix())))
sh.putTags = make(map[string]map[string]int)
sh.Timestamp = timestamp.GetDayTimestamp(-1)
sh.initPutAdverts()
sh.initPutTags("TAGS_3*", "tb_", "mg_")
sh.initPutTags("TAGS_5*", "url_", "")
log.Warn(sh.putAdverts)
return sh
}
func NewCookiePut() *CookiePut {
var zj = &CookiePut{}
zj.kf = dbfactory.NewKVFile(fmt.Sprintf("./%s.txt", convert.ToString(time.Now().Unix())))
zj.putTags = make(map[string]map[string]int)
zj.Timestamp = timestamp.GetHourTimestamp(-1)
zj.initPutAdverts()
zj.initPutTags("TAGS_5*", "cookie_", "")
log.Info(zj.putAdverts)
log.Info(zj.putTags)
return zj
}
func getGeop(w http.ResponseWriter, r *http.Request) {
ad := strings.TrimSpace(r.URL.Query().Get("ad"))
mlink, err := db.Get()
lon := ""
lat := ""
if err != nil {
log.Error(err)
w.Write([]byte(""))
return
}
defer mlink.Close()
if ad == "" {
w.WriteHeader(404)
w.Write([]byte(""))
return
}
qconf := mongodb.MongodbQueryConf{}
qconf.Db = "lonlat_data"
qconf.Table = "tbl_map"
qconf.Select = mongodb.MM{"lon": 1, "lat": 1}
qconf.Query = mongodb.MM{"ad": ad}
info, err := mlink.One(qconf)
if err != nil {
log.Error(err)
w.Write([]byte(""))
return
}
if v, ok := info["lon"]; ok {
lon = convert.ToString(v)
}
if v, ok := info["lat"]; ok {
lat = convert.ToString(v)
}
w.Write([]byte(fmt.Sprintf("%s,%s", lon, lat)))
}
// 保存广告对应的ad，ua
func (this *ZjPut) saveAdvertSet() {
tname := "advert_tj_zj_" + this.Timestamp + "_"
fname := lib.GetConfVal("zhejiang::data_path") + tname
this.kf.IDAdUaSet(fname, func(info map[string]int) {
tm := this.Timestamp
for k, v := range info {
aid := strings.TrimPrefix(k, tname)
// 广告数量统计数据 advert_stats , zj_1461016800_1111, 11111
lib.StatisticsData("advert_stats", fmt.Sprintf("zj_%s_%s", tm, aid),
convert.ToString(v), "")
}
}, false)
}
// 域名数据获取
func (this *ShPut) domainData(out chan interface{}, in chan int8) {
var datacount = 0
defer func() {
// 统计数据 jiangsu_put , url_1461016800, 11111
lib.StatisticsData("dsource_stats", "sh_"+this.Timestamp+"_url",
convert.ToString(datacount), "")
}()
fname := "shanghai_url_" + this.Timestamp
if err := lib.GetFdbData(fname, func(val string) {
if v := lib.AddPrefix(val, "url_"); v != "" {
datacount++
out <- v
}
}); err != nil {
in <- 1
return
}
log.Info("域名ok")
in <- 1
}
// 其他杂项数据获取
func (this *ZjPut) otherData(out chan interface{}, in chan int8) {
var datacount = 0
defer func() {
// 统计数据 zhejiang_put , other_1461016800, 11111
lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_other",
convert.ToString(datacount), "")
}()
fname := "zhejiang_other_" + this.Timestamp
if err := lib.GetFdbData(fname, func(val string) {
if v := lib.AddPrefix(val, "mg_"); v != "" {
datacount++
out <- v
}
}); err != nil {
in <- 1
return
}
log.Info("其他ok")
in <- 1
}
// 无限数据获取
func (this *CookiePut) coikieData(out chan interface{}, in chan int8) {
var datacount = 0
defer func() {
// 统计数据 jiangsu_put , url_1461016800, 11111
lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_cookie",
convert.ToString(datacount), "")
}()
fname := "zhejiang_cookie_" + this.Timestamp
if err := lib.GetFdbData(fname, func(val string) {
if v := lib.AddPrefix2(val, "cookie_"); v != "" {
datacount++
out <- v
}
}); err != nil {
in <- 1
return
}
log.Info("cookieok")
in <- 1
}
// 获取月的时间戳
func GetMonthTimestamp(month int) string {
d := time.Now().AddDate(0, month, 0).Format("200601")
a, _ := time.ParseInLocation("200601", d, time.Local)
return convert.ToString(a.Unix())
}
// 获取某个小时的时间戳
func GetHourTimestamp(hour int) string {
d := time.Now().Add(time.Hour * time.Duration(hour)).Format("2006010215")
a, _ := time.ParseInLocation("2006010215", d, time.Local)
return convert.ToString(a.Unix())
}
// 店铺信息获取
func (this *ZjPut) ShopData(out chan interface{}, in chan int8) {
var datacount = 0
defer func() {
// 统计数据 zhejiang_put , other_1461016800, 11111
lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_shop",
convert.ToString(datacount), "")
}()
es, err := lib.GetESObj()
if err != nil {
log.Error(err)
in <- 1
return
}
this.shopAdverts = this.GetPutShopInfo()
for shopid, shopinfo := range this.shopAdverts {
for _, adids := range shopinfo.ShopAdverts {
date := timestamp.GetDayTimestamp(adids.Date * -1)
var scrollid = ""
var query = elastic.NewBoolQuery()
query.Must(elastic.NewRangeQuery("timestamp").Gte(date))
query.Must(elastic.NewTermQuery("shop", shopid))
sr, err := es.Scroll().Index("zhejiang_tb_shop_trace").Type("shop").
Query(query).Do()
if err != nil {
log.Error(err)
continue
}
scrollid = sr.ScrollId
for {
sres, err := es.Scroll().Index("zhejiang_tb_shop_trace").Type("shop").
Query(query).ScrollId(scrollid).Size(1000).Do()
if err == elastic.EOS {
break
}
if err != nil {
log.Error(err)
out <- 1
return
}
for _, hit := range sres.Hits.Hits {
item := make(map[string]interface{})
err := json.Unmarshal(*hit.Source, &item)
if err != nil {
continue
}
ad := convert.ToString(item["ad"])
ua := encrypt.DefaultMd5.Encode(convert.ToString(item["ua"]))
datacount++
out <- fmt.Sprintf("%s\t%s\t%s", ad, ua, adids.AdvertId)
}
scrollid = sres.ScrollId
}
}
}
log.Info("店铺ok")
in <- 1
}
// 电商数据获取
func (this *ZjPut) BusinessData(out chan interface{}, in chan int8) {
var datacount = 0
defer func() {
// 统计数据 zhejiang_put , other_1461016800, 11111
lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_business",
convert.ToString(datacount), "")
}()
es, err := lib.GetESObj()
if err != nil {
out <- 1
return
}
var bt = timestamp.GetHourTimestamp(-1)
var et = timestamp.GetHourTimestamp(-73)
var query = elastic.NewRangeQuery("timestamp").Gte(et).Lte(bt)
var sid = ""
res, err := es.Scroll().Index("zhejiang_tb_ad_trace").Type("ad").Query(query).Size(1000).Do()
if err != nil {
log.Error(err)
out <- 1
return
}
sid = res.ScrollId
for {
sres, err := es.Scroll().Index("zhejiang_tb_ad_trace").Type("ad").
Query(query).ScrollId(sid).Size(1000).Do()
if err == elastic.EOS {
break
}
if err != nil {
log.Error(err)
out <- 1
return
}
for _, hit := range sres.Hits.Hits {
item := make(map[string]interface{})
err := json.Unmarshal(*hit.Source, &item)
if err != nil {
continue
}
ad := convert.ToString(item["ad"])
ua := encrypt.DefaultMd5.Encode(convert.ToString(item["ua"]))
cids := item["cids"].([]interface{})
ncids := make(map[string]int)
ncidsary := make([]string, 0, len(cids))
for _, v := range cids {
if vv, ok := lib.TcatBig[convert.ToString(v)]; ok {
ncids[vv] = 1
}
}
for k, _ := range ncids {
ncidsary = append(ncidsary, "tb_"+k)
}
if len(ncidsary) == 0 {
continue
}
datacount++
out <- fmt.Sprintf("%s\t%s\t%s", ad, ua, strings.Join(ncidsary, ","))
}
sid = sres.ScrollId
}
log.Info("电商ok")
in <- 1
}
// 获取任意一天整点时间戳
func GetDayTimestamp(day int) string {
d := time.Now().AddDate(0, 0, day).Format("20060102")
a, _ := time.ParseInLocation("20060102", d, time.Local)
return convert.ToString(a.Unix())
}