Example #1
0
func main() {

	// POST data
	post_arg := url.Values{
		"name": {"admin"},
		"pwd":  {"admin"},
	}

	// http header
	header := make(http.Header)
	header.Set("Content-Type", "application/x-www-form-urlencoded")

	// Spider input:
	//  PageProcesser ;
	//  Task name used in Pipeline for record;
	// AddRequest Params:
	//  1. Url.
	//  2. Responce type is "html" or "json" or "jsonp" or "text".
	//  3. The urltag is name for marking url and distinguish different urls in PageProcesser and Pipeline.
	//  4. The method is POST or GET.
	//  5. The postdata is body string sent to sever.
	//  6. The header is header for http request.
	//  7. Cookies
	//  8. Http redirect function
	req := request.NewRequest("http://backadmin.hucong.net/main/user/login", "html", "site_login", "POST", post_arg.Encode(), header, nil, myRedirect, nil)

	spider.NewSpider(NewMyPageProcesser(), "TaskName").
		AddRequest(req).
		AddPipeline(pipeline.NewPipelineConsole()). // Print result on screen
		SetThreadnum(3).                            // Crawl request by three Coroutines
		Run()
}
Example #2
0
func (w *SocialWorker) SpiderMain() {
	spider.NewSpider(NewMyPageProcesser(w), "TaskName").
		AddUrlWithHeaderFile(crawlUrl.Url, "html", "./header_1.json"). // Start url, html is the responce type ("html" or "json" or "jsonp" or "text")
		AddPipeline(pipeline.NewPipelineConsole()).                    // Print result on screen
		SetThreadnum(1).                                               // Crawl request by three Coroutines
		Run()
}
Example #3
0
func main() {
	start_url := "http://www.jiexieyin.org"
	thread_num := uint(16)

	redisAddr := "127.0.0.1:6379"
	redisMaxConn := 10
	redisMaxIdle := 10

	proc := &MyProcessor{}

	sp := spider.NewSpider(proc, "redis_scheduler_example").
		//SetSleepTime("fixed", 6000, 6000).
		//SetScheduler(scheduler.NewQueueScheduler(true)).
		SetScheduler(scheduler.NewRedisScheduler(redisAddr, redisMaxConn, redisMaxIdle, true)).
		AddPipeline(pipeline.NewPipelineConsole()).
		SetThreadnum(thread_num)

	init := false
	for _, arg := range os.Args {
		if arg == "--init" {
			init = true
			break
		}
	}
	if init {
		sp.AddUrl(start_url, "html")
		mlog.LogInst().LogInfo("重新开始爬")
	} else {
		mlog.LogInst().LogInfo("继续爬")
	}
	sp.Run()
}
Example #4
0
func main() {
	// Spider input:
	//  PageProcesser ;
	//  Task name used in Pipeline for record;
	spider.NewSpider(NewMyPageProcesser(), "TaskName").
		AddUrl("https://github.com/hu17889?tab=repositories", "html"). // Start url, html is the responce type ("html" or "json" or "jsonp" or "text")
		AddPipeline(pipeline.NewPipelineConsole()).                    // Print result on screen
		SetThreadnum(3).                                               // Crawl request by three Coroutines
		Run()
}
Example #5
0
func main() {
	flag.Parse()
	// chan init
	exitChan = make(chan struct{})
	getURL = make(chan string, *threadNum)
	releaseSlot = make(chan int, *threadNum)

	// repetition and urlstor initialization
	c := redis.NewClient(&redis.Options{
		Addr:     *redisURL,
		Password: *redisPWD,
		DB:       *redisDB,
	})

	rep = repetition.RepetitionJudgement{}
	err := rep.InitializeVisited(c, "repetition")
	if err != nil {
		log.Fatal("redis error, ", err)
	}

	urlstr = urlstore.URLCrawlerStore{}
	_, err = urlstr.InitialURLsStore(c, "colNeedCrawl", "colNeedCommit", "colNeedCrawl", "colNeedCommit")
	visited, _ := rep.CheckIfVisited("http://www.dianping.com/")
	if !visited {
		rep.VisitedNewNode("http://www.dianping.com/")
		urlstr.UploadURL("http://www.dianping.com/")
	}
	if err != nil {
		log.Fatal("urlstore initialzation error, ", err)
	}

	// db initilization
	dbSession, err := storage.Link2Db(*databaseURL)
	defer dbSession.Close()
	if err != nil {
		log.Fatal("mongodb error, ", err)
	}
	collection = storage.Link2Collection(dbSession, *databaseName, *databaseUser, *databasePwd, *collectionName, *databaseAuth)
	go distributeURL(*threadNum, urlstr)
	// url initilziation
	for i := 0; i < *threadNum; i++ {
		releaseSlot <- 1
	}
	rootURL := GetOneURL()

	// Spider input:
	//  PageProcesser ;
	//  Task name used in Pipeline for record;
	spider.NewSpider(NewMyPageProcesser(), "TaskName").
		AddUrl(rootURL, "html").                    // Start url, html is the responce type ("html" or "json" or "jsonp" or "text")
		AddPipeline(pipeline.NewPipelineConsole()). // Print result on screen
		SetThreadnum((uint)(*threadNum)).           // Crawl request by three Coroutines
		Run()
}
func main() {
	// spider input:
	//  PageProcesser ;
	//  config path(default: WD/etc/main.conf);
	//  task name used in Pipeline for record;
	spider.NewSpider(NewMyPageProcesser(), "", "TaskName").
		AddUrl("https://github.com/hu17889?tab=repositories", "html"). // start url, html is the responce type ("html" or "json")
		AddPipeline(pipeline.NewPipelineConsole()).                    // print result on screen
		SetThreadnum(3).                                               // crawl request by three Coroutines
		Run()
}
Example #7
0
func main() {
	// Spider input:
	//  PageProcesser ;
	//  Task name used in Pipeline for record;
	req_url := "http://weixin.sogou.com/weixin?query=%E4%BA%91%E6%B5%AE&type=1&page=1&ie=utf8"
	spider.NewSpider(NewMyPageProcesser(), "TaskName").
		AddUrlWithHeaderFile(req_url, "html", "weixin.sogou.com.json"). // Start url, html is the responce type ("html" or "json" or "jsonp" or "text")
		AddPipeline(pipeline.NewPipelineConsole()).                     // Print result on screen
		SetThreadnum(3).                                                // Crawl request by three Coroutines
		Run()
}
Example #8
0
File: main.go Project: tuyuwei/test
func main() {
	// spider input:
	//  PageProcesser ;
	//  task name used in Pipeline for record;
	spider.NewSpider(NewMyPageProcesser(), "TaskName").
		AddUrls(getUrls(), "html").                 // start url, html is the responce type ("html" or "json")
		AddPipeline(pipeline.NewPipelineConsole()). // print result on screen
		SetThreadnum(1000).                         // crawl request by three Coroutines
		CloseStrace().
		OpenFileLog("/Users/tywei/workspace/devLanguage/go/crawl/logs").
		Run()
}
Example #9
0
func main() {
	// spider input:
	//  PageProcesser ;
	//  task name used in Pipeline for record;
	spider.NewSpider(NewMyPageProcesser(), "sina_stock_news").
		AddUrl("http://live.sina.com.cn/zt/api/l/get/finance/globalnews1/index.htm?format=json&id=63621&pagesize=10&dire=f", "json"). // start url, html is the responce type ("html" or "json" or "jsonp" or "text")
		AddPipeline(pipeline.NewPipelineConsole()).                                                                                   // Print result to std output
		AddPipeline(pipeline.NewPipelineFile("/tmp/sinafile")).                                                                       // Print result in file
		OpenFileLog("/tmp").                                                                                                          // Error info or other useful info in spider will be logged in file of defalt path like "WD/log/log.2014-9-1".
		SetSleepTime("rand", 1000, 3000).                                                                                             // Sleep time between 1s and 3s.
		Run()
	//AddPipeline(pipeline.NewPipelineFile("/tmp/tmpfile")). // print result in file
}
func main() {

	pageProcess := NewMyPageProcesser()
	pageProcess.Init()
	diandianSpider := spider.NewSpider(pageProcess, "TaskName").
		AddUrl("http://shinichr.diandian.com/", "html"). // Start url, html is the responce type ("html" or "json" or "jsonp" or "text")
		AddPipeline(pipeline.NewPipelineConsole()).      // Print result on screen
		AddPipeline(pipeline.NewPipelineFile("/home/shinichr/spider.tmp")).
		SetThreadnum(10000) // Crawl request by three Coroutines

	for i := 2; i < 10; i++ {
		url := fmt.Sprintf("http://shinichr.diandian.com/page/%d", i)
		diandianSpider.AddUrl(url, "html")
	}

	diandianSpider.Run()

	for url, _ := range pageProcess.visit_url {
		fmt.Println("spider:", url)
	}

}