Exemple #1
0
func tailLog(db gorm.DB, eventSample *stripe.Event) error {
	// Start paging from after the event sample that we capture before we
	// started loading from snapshot.
	params := &stripe.EventListParams{}
	params.Filters.AddFilter("limit", "", strconv.Itoa(PageSize))
	params.Filters.AddFilter("starting_after", "", eventSample.ID)

	var apiDuration, dbDuration time.Duration
	numProcessed := 0
	totalStart := time.Now()

	iterator := event.List(params)
	timeIterator := func() bool {
		start := time.Now()
		ret := iterator.Next()
		apiDuration = apiDuration + time.Now().Sub(start)
		return ret
	}

	for timeIterator() {
		start := time.Now()
		event := iterator.Event()

		switch event.Type {
		case "charge.created":
			charge := Charge{
				// TODO: deserialize to proper charge object
				ID:      event.Data.Obj["id"].(string),
				Amount:  event.Data.Obj["amount"].(uint64),
				Created: time.Unix(event.Data.Obj["created"].(int64), 0),
			}
			db.FirstOrCreate(&charge)
			dbDuration = dbDuration + time.Now().Sub(start)
			if db.Error != nil {
				return db.Error
			}

			numProcessed = numProcessed + 1
			if numProcessed%ReportingIncrement == 0 {
				log.Printf("Working. Processed %v record(s).", ReportingIncrement)
			}
		}
	}
	if err := iterator.Err(); err != nil {
		return err
	}

	log.Printf("")
	log.Printf("== Log Tailing Results")
	log.Printf("API duration:            %v", apiDuration)
	log.Printf("DB duration:             %v", dbDuration)
	log.Printf("Total duration:          %v", time.Now().Sub(totalStart))
	log.Printf("Total records processed: %v", numProcessed)
	return nil
}
Exemple #2
0
func getEventSample() (*stripe.Event, error) {
	params := &stripe.EventListParams{}
	params.Filters.AddFilter("limit", "", "1")
	iterator := event.List(params)

	for iterator.Next() {
		return iterator.Event(), nil
	}
	err := iterator.Err()
	return nil, err
}
Exemple #3
0
func tailLog(producer sarama.SyncProducer, topic string) error {
	numProcessed := 0

	params := &stripe.EventListParams{}
	params.Filters.AddFilter("limit", "", strconv.Itoa(PageSize))

	iterator := event.List(params)
	timeIterator := func() bool {
		//start := time.Now()
		ret := iterator.Next()
		//log.Printf("API request took %v", time.Now().Sub(start))
		return ret
	}

	var batch []*stripe.Event
	for timeIterator() {
		event := iterator.Event()

		batch = append(batch, event)

		if len(batch) == KafkaBatchSize {
			err := processBatch(producer, topic, batch)
			if err != nil {
				return err
			}
			batch = nil

			numProcessed = numProcessed + 1
			if numProcessed%ReportingIncrement == 0 {
				log.Printf("Working. Processed %v record(s).", ReportingIncrement)
			}
		}
	}
	if err := iterator.Err(); err != nil {
		return err
	}
	return nil
}