Beispiel #1
0
func main() {
	ginger.NewMemoryGinger(true)

	ec := json.NewEncoder(os.Stdout)

	count := 0
	var last dynamodb.Key
	for {
		if qr, err := ginger.DB.Scan("fetch", &dynamodb.ScanOptions{ReturnConsumedCapacity: "TOTAL", ExclusiveStartKey: last}); err == nil {
			count += qr.Count
			for _, i := range qr.Items {
				f := ginger.DB.FromItem("fetch", i)
				if err = ec.Encode(&f); err != nil {
					panic(err)
				}
			}
			last = qr.LastEvaluatedKey
			if last == nil {
				break
			}
		} else {
			log.Println("query error:", err)
		}
	}
	log.Printf("Count: %d\n", count)

}
Beispiel #2
0
func TestGinger(t *testing.T) {
	requests := queue.NewChannelQueue(nil)
	ginger.NewMemoryGinger(false)

	if f, err := ginger.NewFetch("http://www.eikeon.com/"); err == nil {
		f.URL = "http://www.eikeon.com/"
		f.Fetch()
		f.Put()
	} else {
		t.Error("unable to add fetch for http://eikeon.com/")
	}

	ginger.Qer(requests)

	go ginger.Worker(requests)

	time.Sleep(1 * time.Second)
	if response, err := ginger.DB.Scan("fetch", nil); err == nil {
		for _, i := range response.Items {
			f := ginger.DB.FromItem("fetch", i).(*ginger.Fetch)
			if f.URL == "http://www.eikeon.com/" {
				if f.StatusCode != 0 {
					goto found
				}
			}
		}
	} else {
		t.Log(err)
	}
	t.Error("Didn't find expected result")
found:
}
Beispiel #3
0
func main() {
	ginger.NewMemoryGinger(true)

	for _, code := range []string{"200", "301", "302", "404"} {
		count := 0
		f := dynamodb.KeyConditions{"StatusCode": {[]dynamodb.AttributeValue{{"N": code}}, "EQ"}}
		var last dynamodb.Key
		for {
			if qr, err := ginger.DB.Scan("fetch", &dynamodb.ScanOptions{ScanFilter: f, ReturnConsumedCapacity: "TOTAL", Select: "COUNT", ExclusiveStartKey: last}); err == nil {
				count += qr.Count
				last = qr.LastEvaluatedKey
				if last == nil {
					break
				}
			} else {
				log.Println("query error:", err)
			}
		}
		log.Printf("Status Code: %s Total Count: %d\n", code, count)
	}
}
Beispiel #4
0
func sendUrls(urls chan string, name string) {
	g := ginger.NewMemoryGinger(true)
	count := 0
	for url := range urls {
		count += 1
		retryCount := 0
	retryAdd:
		if err := g.Add(url, name); err != nil {
			time.Sleep(time.Second)
			retryCount += 1
			if retryCount < 120 {
				log.Println("error:", err, "retrying")
				goto retryAdd
			} else {
				log.Println("error:", err, "hit max retry... giving up.")
			}
		}
		if count%1000 == 0 {
			stathat.PostEZCount("gingerload", "*****@*****.**", 1000)
		}
	}
}
Beispiel #5
0
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	_ = ginger.NewMemoryGinger(true)

auth:
	auth, err := aws.EnvAuth()
	if err != nil {
		log.Println("EnvAuth Failed:", err)
		time.Sleep(time.Second)
		goto auth
	}

	ssqs := sqs.New(auth, aws.USEast)

getQueue:
	q, err := ssqs.GetQueue("ginger-fetchrequests-test")
	if err != nil {
		log.Println("error getting queue:", err)
		time.Sleep(10 * time.Second)
		goto getQueue
	}

	const C = 64

	var wg sync.WaitGroup
	wg.Add(C)
	for i := 0; i < C; i++ {
		go func() {
			fetcher(q)
			wg.Done()

		}()
	}
	wg.Wait()
}
Beispiel #6
0
Datei: gw.go Projekt: edsu/ginger
func main() {
	address := flag.String("address", ":9999", "http service address")
	dynamo := flag.Bool("dynamo", false, "use dynamodb")
	flag.Parse()

	g := ginger.NewMemoryGinger(*dynamo)
	web.AddHandlers(g)

	go func() {
		log.Println("server listening on:", *address)
		err := http.ListenAndServe(*address, nil)
		if err != nil {
			log.Print("ListenAndServe:", err)
		}
	}()

	// We must use a buffered channel or risk missing the signal
	notifyChannel := make(chan os.Signal, 1)

	signal.Notify(notifyChannel, os.Interrupt, syscall.SIGHUP, syscall.SIGTERM)
	sig := <-notifyChannel
	log.Println("stopped:", sig)

}