//添加商品 func AddGoodsInfo(gid string) (info map[string]string) { url := "http://item.taobao.com/item.htm?id=" + gid grab.SetGrabCookie(cookie) h := grab.GrabTaoHTML(url) p, _ := grab.ParseNode(h) //标签名称 title := grab.GetTitle(p) if title == "淘宝网 - 淘!我喜欢" || strings.Contains(title, "出错啦!") { //log.Println("商品不存在,id为:", gid) return nil } if strings.Contains(title, "访问受限") { log.Println("访问受限,id为", gid) time.Sleep(time.Minute * 2) return nil } //标签id cateId := grab.GetCategoryId(h) cateInfo, err := GetTaoCat(cateId) if err == mgo.ErrNotFound { //log.Println("分类ID:", gid, "-", cateId, "-", err) return nil } //特性 features := make(map[string]int) if v, ok := cateInfo["features"]; ok { for a, b := range v.(map[string]interface{}) { features[a] = b.(int) } } //属性 attrbuites := grab.GetAttrbuites(p) //性别 sex := 0 for k, v := range sexmap { if strings.Contains(title, k) { sex = v break } } //人群 people := 0 for k, v := range peoplemap { if strings.Contains(title, k) { people = v break } } // 店铺信息 shopId := grab.GetShopId(p) shopName := grab.GetShopName(p) shopUrl := grab.GetShopUrl(p) shopBoss := grab.GetShopBoss(p) //浏览数 count := 0 go func() { sess := GetSession() defer func() { sess.Close() }() sess.DB(modb).C("goods").Upsert(bson.M{"gid": gid}, bson.M{"$set": bson.M{ "tagname": cateInfo["name"], "tagid": cateId, "features": features, "attrbuites": attrbuites, "sex": sex, "people": people, "shop_id": shopId, "shop_name": shopName, "shop_url": shopUrl, "shop_box": shopBoss, "count": count}}) }() return map[string]string{ "cid": cateId, "shop_id": shopId, "shop_name": shopName, "shop_url": shopUrl, "shop_boss": shopBoss, } }
//添加商品 func GrabGoodsInfo(gid string) (info map[string]interface{}) { LABEL: url := "https://item.taobao.com/item.htm?id=" + gid grab.SetUserAgent(getUserAgent()) //grab.SetTransport(getHttpProxy()) h := grab.GrabTaoHTML(url) if h == "" { return nil } p, _ := grab.ParseNode(h) //标签名称 title := grab.GetTitle(p) if title == "淘宝网 - 淘!我喜欢" || strings.Contains(title, "出错啦!") { //log.Println("商品不存在,id为:", gid) return nil } if strings.Contains(title, "访问受限") { log.Println("访问受限,id为", gid) time.Sleep(time.Minute * 2) goto LABEL return nil } //标签id cateId := grab.GetCategoryId(h) //标签信息 cateInfo := make(map[string]interface{}) if v, ok := cateList[cateId]; !ok { return nil } else { cateInfo = v } //特性 features := make(map[string]int) if v, ok := cateInfo["features"]; ok { for a, b := range v.(map[string]interface{}) { features[a] = b.(int) } } //属性 attrbuites := grab.GetAttrbuites(p) //性别 sex := 0 for k, v := range sexmap { if strings.Contains(title, k) { sex = v break } } //人群 people := 0 for k, v := range peoplemap { if strings.Contains(title, k) { people = v break } } // 店铺信息 shopId := grab.GetShopId(p) shopName := grab.GetShopName(p) shopUrl := grab.GetShopUrl(p) shopBoss := grab.GetShopBoss(p) return map[string]interface{}{ "shop_id": shopId, "shop_name": shopName, "shop_url": shopUrl, "shop_boss": shopBoss, "gid": gid, "tagname": cateInfo["name"], "tagid": cateId, "features": features, "attrbuites": attrbuites, "sex": sex, "people": people, } }
//添加商品 func AddGoodsInfo(gid string) (cid string) { modb := IniFile.Section("mongo-xu_precise").Key("db").String() url := "http://item.taobao.com/item.htm?id=" + gid h := grab.GrabTaoHTML(url) p, _ := grab.ParseNode(h) sess := GetSession() //标签名称 title := grab.GetTitle(p) if title == "淘宝网 - 淘!我喜欢" { return "" } //标签id cateId := grab.GetCategoryId(h) cateInfo, err := GetTaoCat(cateId) if err == mgo.ErrNotFound { log.Println("分类ID:", cateId, err) return "" } //特性 features := make(map[string]int) if v, ok := cateInfo["features"]; ok { for a, b := range v.(map[string]interface{}) { features[a] = b.(int) } } //属性 attrbuites := grab.GetAttrbuites(p) //性别 sex := 0 for k, v := range sexmap { if strings.Contains(title, k) { sex = v break } } //人群 people := 0 for k, v := range peoplemap { if strings.Contains(title, k) { people = v break } } //浏览数 count := 0 // 店铺信息 shopId := grab.GetShopId(p) shopName := grab.GetShopName(p) shopUrl := grab.GetShopUrl(p) shopBoss := grab.GetShopBoss(p) go func() { sess.DB(modb).C("goods").Upsert(bson.M{"gid": gid}, bson.M{"gid": gid, "tagname": cateInfo["name"], "tagid": cateId, "features": features, "attrbuites": attrbuites, "sex": sex, "people": people, "shop_id": shopId, "shop_name": shopName, "shop_url": shopUrl, "shop_boss": shopBoss, "count": count, }) sess.Close() }() return cateId }