示例#1
0
func TestNotRanWhenStopped(t *testing.T) {
	ran := false
	testSpider := spider.Get("http://google.com", func(ctx *spider.Context) error {
		ran = true
		return nil
	})
	dur := 1*time.Second + 100*time.Millisecond
	stopCh := make(chan struct{})

	sched := spider.NewScheduler()
	sched.Add(schedule.Every(1*time.Second), testSpider)
	sched.Start()

	go func() {
		sched.Stop()
		stopCh <- struct{}{}
	}()

	select {
	case <-time.After(dur):
		t.Error("Should not wait to much")
	case <-stopCh:
		if ran {
			t.Error("Spider ran but should not run when stopped")
		}
	}
}
示例#2
0
文件: wiki.go 项目: lamproae/spider
func main() {
	wikiHTMLSpider := &WikipediaHTMLSpider{"Albert Einstein"}
	wikiJSONSpider := &WikipediaJSONSpider{"Lionel Messi"}

	spider.Add(schedule.Every(7*time.Second), wikiHTMLSpider)
	spider.Add(schedule.Every(9*time.Second), wikiJSONSpider)
	spider.Add(schedule.Every(5*time.Second), spider.Get("https://google.com", func(ctx *spider.Context) error {
		_, err := ctx.DoRequest()
		if err != nil {
			fmt.Println("Made request to google")
		}
		return nil
	}))
	spider.Start()

	<-time.After(26 * time.Second)
}
示例#3
0
func TestInMemory(t *testing.T) {
	ran := false
	testSpider := spider.Get("http://google.com", func(ctx *spider.Context) error {
		ran = true
		return nil
	})
	sched := spider.NewScheduler()
	sched.Add(schedule.Every(1*time.Second), testSpider)
	sched.Start()
	dur := 1*time.Second + 500*time.Millisecond

	select {
	case <-time.After(dur):
		if !ran {
			t.Error("spider not ran")
		}
	}
	sched.Stop()
}
示例#4
0
文件: main.go 项目: schollz/gocapsule
func main() {
	if len(os.Args) == 1 {
		log.Fatal("You need to specify the external IP address")
	}
	ExternalIP = os.Args[1]
	loadSelfSpider()
	scheduler := spider.NewScheduler()
	// scheduler.Add(schedule.Every(600*time.Second), ScienceNewsSpider)
	scheduler.Add(schedule.Every(600*time.Second), InTheatersSpider)
	scheduler.Add(schedule.Every(600*time.Second), GroceriesSpider)
	scheduler.Add(schedule.Every(600*time.Second), CoolListSpider)
	scheduler.Add(schedule.Every(600*time.Second), TodoListSpider)
	scheduler.Add(schedule.Every(600*time.Second), SelfSpider)
	scheduler.Add(schedule.Every(600*time.Second), WeatherSpider)
	scheduler.Start()

	Open()
	defer Close()

	// Testing
	// Open()
	// var p WeatherData
	// p.Get(Today())
	// fmt.Println(p)
	// Close()

	router := gin.Default()
	router.LoadHTMLGlob("templates/*")
	router.GET("/", func(c *gin.Context) {
		day := Today()
		c.Redirect(307, "http://"+ExternalIP+"/"+day)
	})
	router.GET("/:year/:month/:day", func(c *gin.Context) {
		year := c.Param("year")
		month := c.Param("month")
		daynum := c.Param("day")
		day := year + "/" + month + "/" + daynum
		if day == Today() {
			// Open()
			var scienceNews ScienceNewsGroup
			scienceNews.Get(day)
			var inTheaters InTheatersGroup
			inTheaters.Get(day)
			var groceryList CowyoGroup
			groceryList.Get(day, "GroceryList")
			var coolList CowyoGroup
			coolList.Get(day, "CoolList")
			var todoList CowyoGroup
			todoList.Get(day, "TodoList")
			var weather WeatherData
			weather.Get(day)
			// Close()

			c.HTML(http.StatusOK, "index.tmpl", gin.H{
				"Date":        day,
				"InTheaters":  inTheaters.Data,
				"GroceryList": groceryList.Data,
				"CoolList":    coolList.Data,
				"TodoList":    todoList.Data,
				"Weather":     weather,
			})
		} else {
			var p SelfData
			p.Get(day)
			c.Data(http.StatusOK, "text/html", []byte(p.Html))
		}

	})

	router.Run(":8001")

	// Exit 5 seconds later to let time for the request to be done.
	<-time.After(3 * time.Second)
	fmt.Println("HI")
}