func main() {
ginger.NewMemoryGinger(true)
ec := json.NewEncoder(os.Stdout)
count := 0
var last dynamodb.Key
for {
if qr, err := ginger.DB.Scan("fetch", &dynamodb.ScanOptions{ReturnConsumedCapacity: "TOTAL", ExclusiveStartKey: last}); err == nil {
count += qr.Count
for _, i := range qr.Items {
f := ginger.DB.FromItem("fetch", i)
if err = ec.Encode(&f); err != nil {
panic(err)
}
}
last = qr.LastEvaluatedKey
if last == nil {
break
}
} else {
log.Println("query error:", err)
}
}
log.Printf("Count: %d\n", count)
}
func TestGinger(t *testing.T) {
requests := queue.NewChannelQueue(nil)
ginger.NewMemoryGinger(false)
if f, err := ginger.NewFetch("http://www.eikeon.com/"); err == nil {
f.URL = "http://www.eikeon.com/"
f.Fetch()
f.Put()
} else {
t.Error("unable to add fetch for http://eikeon.com/")
}
ginger.Qer(requests)
go ginger.Worker(requests)
time.Sleep(1 * time.Second)
if response, err := ginger.DB.Scan("fetch", nil); err == nil {
for _, i := range response.Items {
f := ginger.DB.FromItem("fetch", i).(*ginger.Fetch)
if f.URL == "http://www.eikeon.com/" {
if f.StatusCode != 0 {
goto found
}
}
}
} else {
t.Log(err)
}
t.Error("Didn't find expected result")
found:
}
func main() {
ginger.NewMemoryGinger(true)
for _, code := range []string{"200", "301", "302", "404"} {
count := 0
f := dynamodb.KeyConditions{"StatusCode": {[]dynamodb.AttributeValue{{"N": code}}, "EQ"}}
var last dynamodb.Key
for {
if qr, err := ginger.DB.Scan("fetch", &dynamodb.ScanOptions{ScanFilter: f, ReturnConsumedCapacity: "TOTAL", Select: "COUNT", ExclusiveStartKey: last}); err == nil {
count += qr.Count
last = qr.LastEvaluatedKey
if last == nil {
break
}
} else {
log.Println("query error:", err)
}
}
log.Printf("Status Code: %s Total Count: %d\n", code, count)
}
}
func sendUrls(urls chan string, name string) {
g := ginger.NewMemoryGinger(true)
count := 0
for url := range urls {
count += 1
retryCount := 0
retryAdd:
if err := g.Add(url, name); err != nil {
time.Sleep(time.Second)
retryCount += 1
if retryCount < 120 {
log.Println("error:", err, "retrying")
goto retryAdd
} else {
log.Println("error:", err, "hit max retry... giving up.")
}
}
if count%1000 == 0 {
stathat.PostEZCount("gingerload", "*****@*****.**", 1000)
}
}
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
_ = ginger.NewMemoryGinger(true)
auth:
auth, err := aws.EnvAuth()
if err != nil {
log.Println("EnvAuth Failed:", err)
time.Sleep(time.Second)
goto auth
}
ssqs := sqs.New(auth, aws.USEast)
getQueue:
q, err := ssqs.GetQueue("ginger-fetchrequests-test")
if err != nil {
log.Println("error getting queue:", err)
time.Sleep(10 * time.Second)
goto getQueue
}
const C = 64
var wg sync.WaitGroup
wg.Add(C)
for i := 0; i < C; i++ {
go func() {
fetcher(q)
wg.Done()
}()
}
wg.Wait()
}
func main() {
address := flag.String("address", ":9999", "http service address")
dynamo := flag.Bool("dynamo", false, "use dynamodb")
flag.Parse()
g := ginger.NewMemoryGinger(*dynamo)
web.AddHandlers(g)
go func() {
log.Println("server listening on:", *address)
err := http.ListenAndServe(*address, nil)
if err != nil {
log.Print("ListenAndServe:", err)
}
}()
// We must use a buffered channel or risk missing the signal
notifyChannel := make(chan os.Signal, 1)
signal.Notify(notifyChannel, os.Interrupt, syscall.SIGHUP, syscall.SIGTERM)
sig := <-notifyChannel
log.Println("stopped:", sig)
}