func main() { flag.Parse() log.SetHandler(text.New(os.Stderr)) log.SetLevel(log.DebugLevel) // set up producer svc := kinesis.New(session.New()) p := producer.New(producer.Config{ StreamName: *stream, BacklogSize: 500, Client: svc, }) p.Start() // open data file f, err := os.Open("/tmp/users.txt") if err != nil { log.Fatal("Cannot open users.txt file") } defer f.Close() // loop over file data b := bufio.NewScanner(f) for b.Scan() { err := p.Put(b.Bytes(), "site") if err != nil { log.WithError(err).Fatal("error producing") } } p.Stop() }
// NewConfig handler sending logs to Kinesis. The `config` given is passed to the batch // Kinesis producer, and a random value is used as the partition key for even distribution. func NewConfig(config k.Config) *Handler { producer := k.New(config) producer.Start() return &Handler{ producer: producer, gen: fastuuid.MustNewGenerator(), } }