func tailLog(db gorm.DB, eventSample *stripe.Event) error {
// Start paging from after the event sample that we capture before we
// started loading from snapshot.
params := &stripe.EventListParams{}
params.Filters.AddFilter("limit", "", strconv.Itoa(PageSize))
params.Filters.AddFilter("starting_after", "", eventSample.ID)
var apiDuration, dbDuration time.Duration
numProcessed := 0
totalStart := time.Now()
iterator := event.List(params)
timeIterator := func() bool {
start := time.Now()
ret := iterator.Next()
apiDuration = apiDuration + time.Now().Sub(start)
return ret
}
for timeIterator() {
start := time.Now()
event := iterator.Event()
switch event.Type {
case "charge.created":
charge := Charge{
// TODO: deserialize to proper charge object
ID: event.Data.Obj["id"].(string),
Amount: event.Data.Obj["amount"].(uint64),
Created: time.Unix(event.Data.Obj["created"].(int64), 0),
}
db.FirstOrCreate(&charge)
dbDuration = dbDuration + time.Now().Sub(start)
if db.Error != nil {
return db.Error
}
numProcessed = numProcessed + 1
if numProcessed%ReportingIncrement == 0 {
log.Printf("Working. Processed %v record(s).", ReportingIncrement)
}
}
}
if err := iterator.Err(); err != nil {
return err
}
log.Printf("")
log.Printf("== Log Tailing Results")
log.Printf("API duration: %v", apiDuration)
log.Printf("DB duration: %v", dbDuration)
log.Printf("Total duration: %v", time.Now().Sub(totalStart))
log.Printf("Total records processed: %v", numProcessed)
return nil
}
func getEventSample() (*stripe.Event, error) {
params := &stripe.EventListParams{}
params.Filters.AddFilter("limit", "", "1")
iterator := event.List(params)
for iterator.Next() {
return iterator.Event(), nil
}
err := iterator.Err()
return nil, err
}
func tailLog(producer sarama.SyncProducer, topic string) error {
numProcessed := 0
params := &stripe.EventListParams{}
params.Filters.AddFilter("limit", "", strconv.Itoa(PageSize))
iterator := event.List(params)
timeIterator := func() bool {
//start := time.Now()
ret := iterator.Next()
//log.Printf("API request took %v", time.Now().Sub(start))
return ret
}
var batch []*stripe.Event
for timeIterator() {
event := iterator.Event()
batch = append(batch, event)
if len(batch) == KafkaBatchSize {
err := processBatch(producer, topic, batch)
if err != nil {
return err
}
batch = nil
numProcessed = numProcessed + 1
if numProcessed%ReportingIncrement == 0 {
log.Printf("Working. Processed %v record(s).", ReportingIncrement)
}
}
}
if err := iterator.Err(); err != nil {
return err
}
return nil
}