func main() { flag.Parse() if verbose { log.SetLevel(log.DebugLevel) } else { log.SetLevel(log.InfoLevel) } if flushFreq == nil || !flushFreq.IsPositive() { flushFreq, _ = goprimitives.NewDuration("1s") } cfg := &producer.KinesisProducerConfig{ FlushFrequency: flushFreq, FlushBytes: flushBytes, FlushMessages: flushMessages, FlushMaxMessages: flushMaxMessages, MaxOpenRequests: maxOpenRequests, MaxMessageBytes: maxMessageBytes, AWSDebugMode: false, MaxRetries: maxRetries, BufferSize: bufferSize, AckSuccess: true, } streamProducer := producer.NewStreamProducer(cfg) if err := streamProducer.ValidateStream(streamName); err != nil { log.WithField("error", err).Error("unable to validate stream") return } log.WithFields(log.Fields{ "stream": streamName, "sendTotal": sendTotal, }).Info("sending messages to kinesis") time.Sleep(3000 * time.Millisecond) for i := 0; i < sendTotal; i++ { streamProducer.Input() <- &producer.KinesisMessage{ Data: []byte(fmt.Sprintf("data record %d", i)), Stream: streamName, PartitionKey: fmt.Sprintf("%d", i), } } for { select { case <-streamProducer.Successes(): log.Debug("got a success message") case kinesisError := <-streamProducer.Errors(): kinesisError.Log("received an error") } } time.Sleep(10000 * time.Millisecond) //streamProducer.Close() }
func main() { flag.Parse() if verbose { log.SetLevel(log.DebugLevel) } else { log.SetLevel(log.InfoLevel) } if queryFreq == nil || !queryFreq.IsPositive() { queryFreq, _ = goprimitives.NewDuration("1s") } cfg := &kcl.Config{ ConsumerGroup: consumerGroup, StreamName: streamName, AWSDebugMode: false, NumRecords: numRecords, BufferSize: bufferSize, QueryFrequency: queryFreq, ReadCapacity: 10, WriteCapacity: 10, ConsumerExpirationSeconds: consumerExpirationSeconds, WorkerID: workerID, } consumer, err := kcl.NewStreamConsumer(cfg) if err != nil { log.WithField("error", err).Error("unable to create consumer") return } if err := consumer.ValidateStream(); err != nil { log.WithField("error", err).Error("unable to validate stream") return } consumer.Start() go printStats() for record := range consumer.Consume() { //log.WithFields(log.Fields{ // "data": string(data), //}).Debug("got consumption data") totalRecords++ if totalRecords == totalToConsume { log.Error("************** SHUT DOWN SHUTDOWN SHUTDOWN (currently doesn't work) *****************************") consumer.Shutdown() } record.Checkpoint() } log.WithFields(log.Fields{ "totalRecordsConsumed": totalRecords, "totalToConsumeBeforeClosing": totalToConsume, }).Error("info consuming records") }
func validateConfig(opts *Config) (*Config, error) { if opts.NumRecords <= 0 { opts.NumRecords = 1 } else if opts.NumRecords > 10000 { opts.NumRecords = 10000 } if opts.BufferSize <= 0 { opts.BufferSize = 1000 } if opts.ConsumerGroup == "" { return nil, fmt.Errorf("you must specify a consumer group") } if opts.ReadCapacity <= 0 { opts.ReadCapacity = 10 } if opts.WriteCapacity <= 0 { opts.WriteCapacity = 10 } if opts.WorkerID == "" { id, err := uuid.NewV4() if err != nil { return nil, err } opts.WorkerID = id.String() } if !opts.QueryFrequency.IsPositive() { opts.QueryFrequency, _ = goprimitives.NewDuration("1s") } if opts.ConsumerExpirationSeconds <= 0 { opts.ConsumerExpirationSeconds = 30 } if opts.IteratorType != "LATEST" || opts.IteratorType != "TRIM_HORIZON" { opts.IteratorType = "LATEST" } return opts, nil }