Beispiel #1
0
func (s *saveUrlSuite) TestSavingAList(c *C) {
	key := config.Env().Get("cacheKey").(string)
	saveUrls([]string{"one", "two", "three"}, key)

	redisC, _ := connectToRedis(config.Env().Get("redishost").(string))
	defer redisC.Close()

	urls, _ := redis.Strings(redisC.Do("LRANGE", key, 0, -1))
	c.Check(urls[0], Equals, "one")
	c.Check(urls[1], Equals, "two")
	c.Check(urls[2], Equals, "three")

	// check that they values are overwritten
	saveUrls([]string{"more", "values"}, key)
	urls, _ = redis.Strings(redisC.Do("LRANGE", key, 0, -1))
	c.Check(urls[0], Equals, "more")
	c.Check(urls[1], Equals, "values")
}
Beispiel #2
0
func ScrapeAction() error {
	// build up the storyUrls list
	urls, err := getNextUrl(startingUrl, []string{startingUrl})
	if err != nil {
		return err
	}

	return saveUrls(urls, config.Env().Get("cacheKey").(string))
}
Beispiel #3
0
func saveUrls(storyUrls []string, redisKey string) error {
	// save the found urls in the database
	redisC, err := connectToRedis(config.Env().Get("redishost").(string))
	if err != nil {
		return err
	}
	defer redisC.Close()

	// clear out the list if it already exists
	if _, err := redisC.Do("DEL", redisKey); err != nil {
		return err
	}

	// save the urls
	for _, url := range storyUrls {
		if _, err := redisC.Do("RPUSH", redisKey, url); err != nil {
			return err
		}
	}

	return nil
}