func putRecordBatch(svc *firehose.Firehose, channel string, data []byte, total, batch int) {
ts := time.Now()
for i := 0; i < total/batch; i++ {
var records []*firehose.Record
for j := 0; j < batch; j++ {
records = append(records, &firehose.Record{Data: data})
}
params := &firehose.PutRecordBatchInput{
DeliveryStreamName: aws.String(channel),
Records: records,
}
svc.PutRecordBatch(params)
}
duration := float64(time.Since(ts)) / float64(time.Millisecond)
//log.Printf("Pushed a total of %d (msg size %d) in batch of %d in %fms", total, len(data), batch, duration)
fmt.Printf("%d,%d,%d,%f\n", total, len(data), batch, duration)
}
func putRecordBuffered(svc *firehose.Firehose, channel string, data []byte, total, batch int) {
ts := time.Now()
for i := 0; i < total/batch; i++ {
var buf []byte
for j := 0; j < batch; j++ {
buf = append(buf, data...)
buf = append(buf, byte('\n'))
}
params := &firehose.PutRecordInput{
DeliveryStreamName: aws.String(channel),
Record: &firehose.Record{Data: buf},
}
svc.PutRecord(params)
}
duration := float64(time.Since(ts)) / float64(time.Millisecond)
//log.Printf("Pushed a total of %d (msg size %d) buffered into %d in %fms", total, len(data), batch, duration)
fmt.Printf("%d,%d,%d,%f\n", total, len(data), batch, duration)
}
func firehoseStreamStateRefreshFunc(conn *firehose.Firehose, sn string) resource.StateRefreshFunc {
return func() (interface{}, string, error) {
describeOpts := &firehose.DescribeDeliveryStreamInput{
DeliveryStreamName: aws.String(sn),
}
resp, err := conn.DescribeDeliveryStream(describeOpts)
if err != nil {
if awsErr, ok := err.(awserr.Error); ok {
if awsErr.Code() == "ResourceNotFoundException" {
return 42, "DESTROYED", nil
}
return nil, awsErr.Code(), err
}
return nil, "failed", err
}
return resp.DeliveryStreamDescription, *resp.DeliveryStreamDescription.DeliveryStreamStatus, nil
}
}