Esempio n. 1
0
File: main.go Progetto: qgweb/gopro
//记录信息
func recordAdUa(param map[string]string) {
	sess := GetSession()
	defer sess.Close()

	db := IniFile.Section("mongo").Key("db").String()

	res, err := getKeyWord(param["ad"])
	if err != nil {
		res = ""
	}

	res = encrypt.GetEnDecoder(encrypt.TYPE_BASE64).Decode(res)

	sess.DB(db).C("srtb").Insert(bson.M{"ad": param["ad"], "ua": encrypt.GetEnDecoder(encrypt.TYPE_BASE64).Decode(param["ua"]),
		"keys": res, "date": time.Now().Format("2006-01-02")})
}
Esempio n. 2
0
File: ptag.go Progetto: qgweb/gopro
// 队列操作
func httpsqsQueue(px string) url.Values {
	hurl := fmt.Sprintf("http://%s:%s/?name=%s&opt=%s&auth=%s", qshost, qsport,
		rediskey+px, "get", qsauth)

	r := httplib.Get(hurl)
	transport := http.Transport{
		DisableKeepAlives: true,
	}
	r.SetTransport(&transport)
	res, err := r.String()

	if err != nil {
		log.Println("读取http队列出错,错误信息为:", err)
		return nil
	}

	if string(res) == "HTTPSQS_GET_END" || string(res) == "HTTPSQS_ERROR" {
		return nil
	}

	res = encrypt.GetEnDecoder(encrypt.TYPE_BASE64).Decode(res)

	data, err := url.ParseQuery(res)
	if err != nil {
		log.Println("解析数据失败")
		return nil
	}

	return data
}
Esempio n. 3
0
File: main.go Progetto: qgweb/gopro
func dispath(data url.Values, px string) {
	if px == "_ad" && data.Get("date") != time.Now().Format("2006-01-02") {
		return
	}

	data.Set("ua", encrypt.GetEnDecoder(encrypt.TYPE_BASE64).Decode(data.Get("ua")))
	uidsAry := strings.Split(data.Get("uids"), ",")

	wg := WaitGroup{}
	ginfoAry := make([]map[string]interface{}, 0, len(uidsAry))
	info := make(map[string]interface{})
	var lock sync.Mutex

	for i := 0; i < len(uidsAry); i++ {
		wg.Wrap(func(param ...interface{}) {
			gid := param[0]
			//判断是否存在
			ginfo, err := checkGoodsExist(gid.(string))
			if err == mgo.ErrNotFound {
				//抓取
				ginfo = GrabGoodsInfo(gid.(string))
				if ginfo != nil {
					ginfo["exists"] = 0
				}
			} else {
				ginfo["exists"] = 1
			}
			if ginfo != nil {
				lock.Lock()
				ginfoAry = append(ginfoAry, ginfo)
				lock.Unlock()
			}
		}, uidsAry[i])
	}

	wg.Wait()

	for k, _ := range data {
		info[k] = data.Get(k)
	}

	info["ginfos"] = ginfoAry

	j, err := json.Marshal(&info)
	if err != nil {
		log.Println("err")
		return
	}

	pushMsgToNsq(j)
}
Esempio n. 4
0
File: main.go Progetto: qgweb/gopro
func createTjData(cate string) string {
	sess := GetSession()
	defer sess.Close()
	var (
		modb     = IniFile.Section("mongo").Key("db").String()
		mkey     = IniFile.Section("default").Key("mkey").String()
		clock, _ = IniFile.Section("default").Key("clock").Int()
		ndate    = time.Now().Format("2006-01-02")
		pdate    = time.Now().Add(-time.Hour * 24).Format("2006-01-02")
		aulist   = make(map[string]bool)
		cateList []string
	)

	var fun = func(d string, ct string, cl string) {
		var list []map[string]interface{}
		sess.DB(modb).C(mkey).Find(bson.M{"date": d, "cids." + cl: bson.RegEx{Pattern: ",?" + ct + ",?"}}).
			Select(bson.M{"ad": 1, "ua": 1, "_id": 0}).All(&list)

		for _, v := range list {
			ad := v["ad"].(string)
			ua := encrypt.GetEnDecoder(encrypt.TYPE_BASE64).Encode(v["ua"].(string))
			aulist[ad+"_"+ua] = true
		}
	}

	cateList = strings.Split(cate, ",")

	for i := clock; i <= 23; i++ {
		for _, v := range cateList {
			fun(pdate, v, fmt.Sprintf("%02d", i))
		}

	}
	for i := 0; i <= clock; i++ {
		for _, v := range cateList {
			fun(ndate, v, fmt.Sprintf("%02d", i))
		}
	}

	var str string
	for k, _ := range aulist {
		str += k + ","
	}

	if str != "" {
		return str[0 : len(str)-1]
	}

	return ""
}
Esempio n. 5
0
File: main.go Progetto: qgweb/gopro
//请求出价
func reponsePrice(param map[string]string) {
	//	mux.Lock()
	//	defer mux.Unlock()

	//	//http://ip:port/uri?mid=$mid&prod=$prod&showType=$showType
	//	conn := pool.Get()
	//	defer conn.Close()

	var (
		host  = IniFile.Section("dxhttp").Key("host").String()
		port  = IniFile.Section("dxhttp").Key("port").String()
		adurl = IniFile.Section("adurl").Key("url").String()
	)

	if host == "" || port == "" {
		log.Fatalln("读取电信配置文件出错")
	}

	if _, ok := param["ad"]; ok {
		adurl = adurl + "?cox=" + param["ad"]
	}

	url := fmt.Sprintf("http://%s:%s/receive?mid=%s&prod=%s&showType=%s&token=%s&price=%s",
		host, port, param["mid"], encrypt.GetEnDecoder(encrypt.TYPE_BASE64).Encode(adurl),
		"03", "reBkYQmESMs=", "10")

	//	res, err := redis.Bytes(conn.Do("GET", "name"))
	//	if err != nil {
	//		log.Println(err)
	//		return
	//	}

	//	if string(res) == "" {
	//		fmt.Println(res)
	//	}

	//resp, err := httpclient.Get(url)
	resp, err := http.Get(url)

	if err != nil {
		log.Println("发送数据出错,错误信息为:", err)
	}
	if resp != nil && resp.StatusCode != 200 {
		log.Println("返回数据出错,错误code为:", resp.StatusCode)
	}
	stotal++
}
Esempio n. 6
0
File: main.go Progetto: qgweb/gopro
//获取mysql链接
func GetMysqlConn() {
	var (
		mouser = IniFile.Section("mysql").Key("user").String()
		mopwd  = IniFile.Section("mysql").Key("pwd").String()
		mohost = IniFile.Section("mysql").Key("host").String()
		moport = IniFile.Section("mysql").Key("port").String()
		modb   = IniFile.Section("mysql").Key("db").String()
	)

	mopwd = encrypt.GetEnDecoder(encrypt.TYPE_BASE64).Decode(mopwd)

	msqldb = orm.NewORM()
	err := msqldb.Open(fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8", mouser, mopwd, mohost, moport, modb))
	if err != nil {
		log.Fatalln("初始化mysql链接失败,错误信息:", err)
	}
}
Esempio n. 7
0
File: model.go Progetto: qgweb/gopro
func (this NSQHandler) HandleMessage(message *nsq.Message) error {
	data := string(message.Body)
	if data == "" {
		log.Println("数据丢失")
		return nil
	}

	data = encrypt.GetEnDecoder(encrypt.TYPE_BASE64).Decode(data)

	urlData, err := url.ParseQuery(data)
	if err != nil {
		log.Println("解析数据失败")
		return nil
	}

	dispath(urlData, this.Px)

	return nil
}
Esempio n. 8
0
File: ptag.go Progetto: qgweb/gopro
//分配数据
func Dispath(data url.Values) {
	defer func() {
		if msg := recover(); msg != nil {
			log.Println(msg)
		}
	}()

	ad := data.Get("ad")
	cookieId := data.Get("cookie")
	ua := data.Get("ua")
	gids := strings.Split(strings.TrimSpace(data.Get("uids")), ",")
	cids := make([]string, 0, len(gids))
	shopids := make([]string, 0, len(gids))

	//多线程抓取
	goodsList := make(chan map[string]string, 20)
	overTag := make(chan bool)
	readCount := 0

	go func() {
		for _, gid := range gids {
			go func(gid string) {
				//判断是否存在
				ginfo, err := checkGoodsExist(gid)

				if err != mgo.ErrNotFound && err != nil {
					log.Println(err)
					goodsList <- nil
					return
				}
				if err == mgo.ErrNotFound {
					//抓取商品
					ginfo = AddGoodsInfo(gid)

					if ginfo == nil {
						goodsList <- nil
						return
					}
				}

				goodsList <- ginfo

				//添加店铺表
				if !checkShopExist(ginfo["shop_id"]) {
					go AddShopInfo(ginfo)
				}
			}(gid)
		}
	}()

	go func() {
		for {
			info := <-goodsList

			if info != nil && len(info) != 0 {
				cids = append(cids, info["cid"])
				shopids = append(shopids, info["shop_id"])
			}

			readCount++

			if readCount == len(gids) {
				overTag <- true
				break
			}
		}
	}()

	<-overTag

	//添加对应关系
	if len(cids) != 0 {
		ua = encrypt.GetEnDecoder(encrypt.TYPE_BASE64).Decode(ua)
		//添加商品对应各种关系表
		go AddUidCids(map[string]string{
			"ad": ad, "cookie": cookieId,
			"ua": ua, "cids": strings.Join(cids, ","),
			"shops": strings.Join(shopids, ","),
			"clock": data.Get("clock"), "date": data.Get("date")})
	}
}