Exemplo n.º 1
0
func main() {

	pool := mongo.NewDialPool("localhost:27018", 1000)

	var i int = 1
	chs := make([]chan bool, 50)
	hasMore := true
	for ; i <= 50; i++ {
		ch := make(chan bool)

		chs[i-1] = ch

		go func(i int, ch chan bool) {
			conn, _ := pool.Get()
			db := &mongo.Database{conn, "meishi", mongo.DefaultLastErrorCmd}
			coll := db.C("foods")
			data, err := goquery.ParseUrl(targetUrl + strconv.Itoa(i))
			if nil != err {
				log.Fatalln("response fail ,", err)
				ch <- false
				return
			}

			nodes := data.Find("div.lp_result_list")
			size := nodes.Find("li").Length()
			if size <= 0 {
				hasMore = false
			}
			for idx := 0; idx < size; idx++ {
				item := nodes.Find("li").Eq(idx)

				link := item.Find("a")
				href := link.Attr("href")
				name := link.Attr("title")
				img := item.Find("img").Attr("src")
				// log.Println(name, "|", href, "|", img)
				if len(name) > 0 {

					err := coll.Upsert(mongo.M{"name": name}, mongo.M{"name": name, "img_url": img, "link": href})

					// log.Println("insert mongo|", err, "|", href)
					log.Println("err", err, "name:", name, "link:", link, "href:", href, "img", img)
				}
			}
			ch <- true
			log.Println(i)

		}(i, ch)
	}

	for i, val := range chs {
		<-val
		log.Println("end:", i)
	}
}
Exemplo n.º 2
0
func main() {

	pool := mongo.NewDialPool("localhost:27018", 1000)

	var i int = 1
	ch := make([]chan bool, 20)
	for ; i <= 20; i++ {
		ch[i-1] = make(chan bool)
		go func(i int, ch chan bool) {
			conn, _ := pool.Get()
			db := &mongo.Database{conn, "meishi", mongo.DefaultLastErrorCmd}
			coll := db.C("foods")
			data, err := goquery.ParseUrl(targetUrl + strconv.Itoa(i))
			if nil != err {
				log.Fatalln("response fail ,", err)
				ch <- false
				return
			}

			nodes := data.Find("#page-" + strconv.Itoa(i))
			size := nodes.Find("div.media").Length()
			for idx := 0; idx < size; idx++ {
				item := nodes.Find("div.media").Eq(idx)
				h2 := item.Find("h2")
				link := h2.Find("a")
				href := link.Attr("href")
				name := link.Attr("title")

				img := item.Find("img").Attr("src")
				// log.Println(name, "|", href, "|", img)
				if len(name) > 0 {
					err := coll.Insert(mongo.M{"name": name, "img_url": img, "link": href})
					log.Println("insert mongo|", err, "|", href)
				}
			}
			ch <- true
			log.Println(i)

		}(i, ch[i-1])
	}

	for i, val := range ch {
		<-val
		log.Println("end:", i)
	}
}
Exemplo n.º 3
0
func init() {
	pool = mongo.NewDialPool("localhost:27018", 1000)
}