func TestGinger(t *testing.T) {
requests := queue.NewChannelQueue(nil)
ginger.NewMemoryGinger(false)
if f, err := ginger.NewFetch("http://www.eikeon.com/"); err == nil {
f.URL = "http://www.eikeon.com/"
f.Fetch()
f.Put()
} else {
t.Error("unable to add fetch for http://eikeon.com/")
}
ginger.Qer(requests)
go ginger.Worker(requests)
time.Sleep(1 * time.Second)
if response, err := ginger.DB.Scan("fetch", nil); err == nil {
for _, i := range response.Items {
f := ginger.DB.FromItem("fetch", i).(*ginger.Fetch)
if f.URL == "http://www.eikeon.com/" {
if f.StatusCode != 0 {
goto found
}
}
}
} else {
t.Log(err)
}
t.Error("Didn't find expected result")
found:
}
func fetcher(q *sqs.Queue) {
count := 0
count_throttled := 0
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for {
receiveMessage:
delay := 300 + r.Intn(100)
resp, err := q.ReceiveMessage([]string{"All"}, 1, delay)
if err != nil {
log.Println("Error receiving message:", err)
time.Sleep(time.Second)
goto receiveMessage
}
if len(resp.Messages) == 0 {
log.Println("No messages in queue. We're done.")
break
}
for _, message := range resp.Messages {
var msg struct {
Message string
}
if err = json.NewDecoder(strings.NewReader(message.Body)).Decode(&msg); err != nil {
log.Println(err)
} else {
url := msg.Message
log.Println("url:", url)
if f, err := ginger.NewFetch(url); err == nil {
if f.NumFetchesLast(time.Second) < 1 {
f.Fetch()
log.Println("fetched:", url)
count += 1
const N = 100
if count%N == 0 {
gingerfetch <- N
}
if err := f.Put(); err != nil {
log.Println("Error putting fetch:", err)
}
deleteMessage:
_, err = q.DeleteMessage(message.ReceiptHandle)
if err != nil {
log.Println("error deleting message:", err)
time.Sleep(100 * time.Millisecond)
goto deleteMessage
}
} else {
count_throttled += 1
log.Println("throttling:", url)
const N = 100
if count_throttled%N == 0 {
gingerfetchthrottled <- N
}
}
} else {
log.Println("could not create new fetch:", err)
}
}
}
}
}
