// 域名数据获取
func (this *ShPut) domainData(out chan interface{}, in chan int8) {
var datacount = 0
defer func() {
// 统计数据 jiangsu_put , url_1461016800, 11111
lib.StatisticsData("dsource_stats", "sh_"+this.Timestamp+"_url",
convert.ToString(datacount), "")
}()
fname := "shanghai_url_" + this.Timestamp
if err := lib.GetFdbData(fname, func(val string) {
if v := lib.AddPrefix(val, "url_"); v != "" {
datacount++
out <- v
}
}); err != nil {
in <- 1
return
}
log.Info("域名ok")
in <- 1
}
// 其他杂项数据获取
func (this *JsPut) otherData(out chan interface{}, in chan int8) {
var datacount = 0
defer func() {
// 统计数据 jiangsu_put , other_1461016800, 11111
lib.StatisticsData("dsource_stats", "js_"+timestamp.GetHourTimestamp(-1)+"_other",
convert.ToString(datacount), "")
}()
fname := "jiangsu_other_" + timestamp.GetHourTimestamp(-1)
if err := lib.GetFdbData(fname, func(val string) {
if v := lib.AddPrefix(val, "mg_"); v != "" {
datacount++
out <- v
}
}); err != nil {
in <- 1
return
}
in <- 1
}
// 其他杂项数据获取
func (this *ZjPut) otherData(out chan interface{}, in chan int8) {
var datacount = 0
defer func() {
// 统计数据 zhejiang_put , other_1461016800, 11111
lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_other",
convert.ToString(datacount), "")
}()
fname := "zhejiang_other_" + this.Timestamp
if err := lib.GetFdbData(fname, func(val string) {
if v := lib.AddPrefix(val, "mg_"); v != "" {
datacount++
out <- v
}
}); err != nil {
in <- 1
return
}
log.Info("其他ok")
in <- 1
}
// 无限数据获取
func (this *CookiePut) coikieData(out chan interface{}, in chan int8) {
var datacount = 0
defer func() {
// 统计数据 jiangsu_put , url_1461016800, 11111
lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_cookie",
convert.ToString(datacount), "")
}()
fname := "zhejiang_cookie_" + this.Timestamp
if err := lib.GetFdbData(fname, func(val string) {
if v := lib.AddPrefix2(val, "cookie_"); v != "" {
datacount++
out <- v
}
}); err != nil {
in <- 1
return
}
log.Info("cookieok")
in <- 1
}