Esempio n. 1
0
func TestGinger(t *testing.T) {
	requests := queue.NewChannelQueue(nil)
	ginger.NewMemoryGinger(false)

	if f, err := ginger.NewFetch("http://www.eikeon.com/"); err == nil {
		f.URL = "http://www.eikeon.com/"
		f.Fetch()
		f.Put()
	} else {
		t.Error("unable to add fetch for http://eikeon.com/")
	}

	ginger.Qer(requests)

	go ginger.Worker(requests)

	time.Sleep(1 * time.Second)
	if response, err := ginger.DB.Scan("fetch", nil); err == nil {
		for _, i := range response.Items {
			f := ginger.DB.FromItem("fetch", i).(*ginger.Fetch)
			if f.URL == "http://www.eikeon.com/" {
				if f.StatusCode != 0 {
					goto found
				}
			}
		}
	} else {
		t.Log(err)
	}
	t.Error("Didn't find expected result")
found:
}
Esempio n. 2
0
func fetcher(q *sqs.Queue) {
	count := 0
	count_throttled := 0
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	for {
	receiveMessage:
		delay := 300 + r.Intn(100)
		resp, err := q.ReceiveMessage([]string{"All"}, 1, delay)
		if err != nil {
			log.Println("Error receiving message:", err)
			time.Sleep(time.Second)
			goto receiveMessage
		}
		if len(resp.Messages) == 0 {
			log.Println("No messages in queue. We're done.")
			break
		}
		for _, message := range resp.Messages {
			var msg struct {
				Message string
			}
			if err = json.NewDecoder(strings.NewReader(message.Body)).Decode(&msg); err != nil {
				log.Println(err)
			} else {
				url := msg.Message
				log.Println("url:", url)

				if f, err := ginger.NewFetch(url); err == nil {
					if f.NumFetchesLast(time.Second) < 1 {
						f.Fetch()
						log.Println("fetched:", url)
						count += 1
						const N = 100
						if count%N == 0 {
							gingerfetch <- N
						}
						if err := f.Put(); err != nil {
							log.Println("Error putting fetch:", err)
						}
					deleteMessage:
						_, err = q.DeleteMessage(message.ReceiptHandle)
						if err != nil {
							log.Println("error deleting message:", err)
							time.Sleep(100 * time.Millisecond)
							goto deleteMessage
						}
					} else {
						count_throttled += 1
						log.Println("throttling:", url)
						const N = 100
						if count_throttled%N == 0 {
							gingerfetchthrottled <- N
						}

					}
				} else {
					log.Println("could not create new fetch:", err)
				}
			}
		}
	}
}