//记录信息
func recordAdUa(param map[string]string) {
sess := GetSession()
defer sess.Close()
db := IniFile.Section("mongo").Key("db").String()
res, err := getKeyWord(param["ad"])
if err != nil {
res = ""
}
res = encrypt.GetEnDecoder(encrypt.TYPE_BASE64).Decode(res)
sess.DB(db).C("srtb").Insert(bson.M{"ad": param["ad"], "ua": encrypt.GetEnDecoder(encrypt.TYPE_BASE64).Decode(param["ua"]),
"keys": res, "date": time.Now().Format("2006-01-02")})
}
// 队列操作
func httpsqsQueue(px string) url.Values {
hurl := fmt.Sprintf("http://%s:%s/?name=%s&opt=%s&auth=%s", qshost, qsport,
rediskey+px, "get", qsauth)
r := httplib.Get(hurl)
transport := http.Transport{
DisableKeepAlives: true,
}
r.SetTransport(&transport)
res, err := r.String()
if err != nil {
log.Println("读取http队列出错,错误信息为:", err)
return nil
}
if string(res) == "HTTPSQS_GET_END" || string(res) == "HTTPSQS_ERROR" {
return nil
}
res = encrypt.GetEnDecoder(encrypt.TYPE_BASE64).Decode(res)
data, err := url.ParseQuery(res)
if err != nil {
log.Println("解析数据失败")
return nil
}
return data
}
func dispath(data url.Values, px string) {
if px == "_ad" && data.Get("date") != time.Now().Format("2006-01-02") {
return
}
data.Set("ua", encrypt.GetEnDecoder(encrypt.TYPE_BASE64).Decode(data.Get("ua")))
uidsAry := strings.Split(data.Get("uids"), ",")
wg := WaitGroup{}
ginfoAry := make([]map[string]interface{}, 0, len(uidsAry))
info := make(map[string]interface{})
var lock sync.Mutex
for i := 0; i < len(uidsAry); i++ {
wg.Wrap(func(param ...interface{}) {
gid := param[0]
//判断是否存在
ginfo, err := checkGoodsExist(gid.(string))
if err == mgo.ErrNotFound {
//抓取
ginfo = GrabGoodsInfo(gid.(string))
if ginfo != nil {
ginfo["exists"] = 0
}
} else {
ginfo["exists"] = 1
}
if ginfo != nil {
lock.Lock()
ginfoAry = append(ginfoAry, ginfo)
lock.Unlock()
}
}, uidsAry[i])
}
wg.Wait()
for k, _ := range data {
info[k] = data.Get(k)
}
info["ginfos"] = ginfoAry
j, err := json.Marshal(&info)
if err != nil {
log.Println("err")
return
}
pushMsgToNsq(j)
}
func createTjData(cate string) string {
sess := GetSession()
defer sess.Close()
var (
modb = IniFile.Section("mongo").Key("db").String()
mkey = IniFile.Section("default").Key("mkey").String()
clock, _ = IniFile.Section("default").Key("clock").Int()
ndate = time.Now().Format("2006-01-02")
pdate = time.Now().Add(-time.Hour * 24).Format("2006-01-02")
aulist = make(map[string]bool)
cateList []string
)
var fun = func(d string, ct string, cl string) {
var list []map[string]interface{}
sess.DB(modb).C(mkey).Find(bson.M{"date": d, "cids." + cl: bson.RegEx{Pattern: ",?" + ct + ",?"}}).
Select(bson.M{"ad": 1, "ua": 1, "_id": 0}).All(&list)
for _, v := range list {
ad := v["ad"].(string)
ua := encrypt.GetEnDecoder(encrypt.TYPE_BASE64).Encode(v["ua"].(string))
aulist[ad+"_"+ua] = true
}
}
cateList = strings.Split(cate, ",")
for i := clock; i <= 23; i++ {
for _, v := range cateList {
fun(pdate, v, fmt.Sprintf("%02d", i))
}
}
for i := 0; i <= clock; i++ {
for _, v := range cateList {
fun(ndate, v, fmt.Sprintf("%02d", i))
}
}
var str string
for k, _ := range aulist {
str += k + ","
}
if str != "" {
return str[0 : len(str)-1]
}
return ""
}
//请求出价
func reponsePrice(param map[string]string) {
// mux.Lock()
// defer mux.Unlock()
// //http://ip:port/uri?mid=$mid&prod=$prod&showType=$showType
// conn := pool.Get()
// defer conn.Close()
var (
host = IniFile.Section("dxhttp").Key("host").String()
port = IniFile.Section("dxhttp").Key("port").String()
adurl = IniFile.Section("adurl").Key("url").String()
)
if host == "" || port == "" {
log.Fatalln("读取电信配置文件出错")
}
if _, ok := param["ad"]; ok {
adurl = adurl + "?cox=" + param["ad"]
}
url := fmt.Sprintf("http://%s:%s/receive?mid=%s&prod=%s&showType=%s&token=%s&price=%s",
host, port, param["mid"], encrypt.GetEnDecoder(encrypt.TYPE_BASE64).Encode(adurl),
"03", "reBkYQmESMs=", "10")
// res, err := redis.Bytes(conn.Do("GET", "name"))
// if err != nil {
// log.Println(err)
// return
// }
// if string(res) == "" {
// fmt.Println(res)
// }
//resp, err := httpclient.Get(url)
resp, err := http.Get(url)
if err != nil {
log.Println("发送数据出错,错误信息为:", err)
}
if resp != nil && resp.StatusCode != 200 {
log.Println("返回数据出错,错误code为:", resp.StatusCode)
}
stotal++
}
//获取mysql链接
func GetMysqlConn() {
var (
mouser = IniFile.Section("mysql").Key("user").String()
mopwd = IniFile.Section("mysql").Key("pwd").String()
mohost = IniFile.Section("mysql").Key("host").String()
moport = IniFile.Section("mysql").Key("port").String()
modb = IniFile.Section("mysql").Key("db").String()
)
mopwd = encrypt.GetEnDecoder(encrypt.TYPE_BASE64).Decode(mopwd)
msqldb = orm.NewORM()
err := msqldb.Open(fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8", mouser, mopwd, mohost, moport, modb))
if err != nil {
log.Fatalln("初始化mysql链接失败,错误信息:", err)
}
}
func (this NSQHandler) HandleMessage(message *nsq.Message) error {
data := string(message.Body)
if data == "" {
log.Println("数据丢失")
return nil
}
data = encrypt.GetEnDecoder(encrypt.TYPE_BASE64).Decode(data)
urlData, err := url.ParseQuery(data)
if err != nil {
log.Println("解析数据失败")
return nil
}
dispath(urlData, this.Px)
return nil
}
//分配数据
func Dispath(data url.Values) {
defer func() {
if msg := recover(); msg != nil {
log.Println(msg)
}
}()
ad := data.Get("ad")
cookieId := data.Get("cookie")
ua := data.Get("ua")
gids := strings.Split(strings.TrimSpace(data.Get("uids")), ",")
cids := make([]string, 0, len(gids))
shopids := make([]string, 0, len(gids))
//多线程抓取
goodsList := make(chan map[string]string, 20)
overTag := make(chan bool)
readCount := 0
go func() {
for _, gid := range gids {
go func(gid string) {
//判断是否存在
ginfo, err := checkGoodsExist(gid)
if err != mgo.ErrNotFound && err != nil {
log.Println(err)
goodsList <- nil
return
}
if err == mgo.ErrNotFound {
//抓取商品
ginfo = AddGoodsInfo(gid)
if ginfo == nil {
goodsList <- nil
return
}
}
goodsList <- ginfo
//添加店铺表
if !checkShopExist(ginfo["shop_id"]) {
go AddShopInfo(ginfo)
}
}(gid)
}
}()
go func() {
for {
info := <-goodsList
if info != nil && len(info) != 0 {
cids = append(cids, info["cid"])
shopids = append(shopids, info["shop_id"])
}
readCount++
if readCount == len(gids) {
overTag <- true
break
}
}
}()
<-overTag
//添加对应关系
if len(cids) != 0 {
ua = encrypt.GetEnDecoder(encrypt.TYPE_BASE64).Decode(ua)
//添加商品对应各种关系表
go AddUidCids(map[string]string{
"ad": ad, "cookie": cookieId,
"ua": ua, "cids": strings.Join(cids, ","),
"shops": strings.Join(shopids, ","),
"clock": data.Get("clock"), "date": data.Get("date")})
}
}