func getRecords(ksis *kinesis.Kinesis, streamName, ShardId string) {
args := kinesis.NewArgs()
args.Add("StreamName", streamName)
args.Add("ShardId", ShardId)
args.Add("ShardIteratorType", "TRIM_HORIZON")
resp10, _ := ksis.GetShardIterator(args)
shardIterator := resp10.ShardIterator
for {
args = kinesis.NewArgs()
args.Add("ShardIterator", shardIterator)
resp11, err := ksis.GetRecords(args)
if len(resp11.Records) > 0 {
fmt.Printf("GetRecords Data BEGIN\n")
for _, d := range resp11.Records {
res, err := d.GetData()
fmt.Printf("GetRecords Data: %v, err: %v\n", string(res), err)
}
fmt.Printf("GetRecords Data END\n")
} else if resp11.NextShardIterator == "" || shardIterator == resp11.NextShardIterator || err != nil {
fmt.Printf("GetRecords ERROR: %v\n", err)
break
}
shardIterator = resp11.NextShardIterator
}
}
func getRecords(ksis kinesis.KinesisClient, streamName, ShardId string) {
args := kinesis.NewArgs()
args.Add("StreamName", streamName)
args.Add("ShardId", ShardId)
args.Add("ShardIteratorType", "TRIM_HORIZON")
resp10, _ := ksis.GetShardIterator(args)
shardIterator := resp10.ShardIterator
for {
args = kinesis.NewArgs()
args.Add("ShardIterator", shardIterator)
resp11, err := ksis.GetRecords(args)
if err != nil {
time.Sleep(1000 * time.Millisecond)
continue
}
if len(resp11.Records) > 0 {
fmt.Printf("GetRecords Data BEGIN\n")
for _, d := range resp11.Records {
fmt.Printf("GetRecords Data: %v\n", string(d.GetData()))
}
fmt.Printf("GetRecords Data END\n")
} else if resp11.NextShardIterator == "" || shardIterator == resp11.NextShardIterator || err != nil {
fmt.Printf("GetRecords ERROR: %v\n", err)
break
}
shardIterator = resp11.NextShardIterator
time.Sleep(1000 * time.Millisecond)
}
}
func getShardRecords(client kinesis.KinesisClient, streamName string, shardId string, messages chan []kinesis.GetRecordsRecords) {
args := kinesis.NewArgs()
args.Add("StreamName", streamName)
args.Add("ShardId", shardId)
args.Add("ShardIteratorType", "LATEST") // AT_SEQUENCE_NUMBER | AFTER_SEQUENCE_NUMBER | TRIM_HORIZON | LATEST
resp, _ := client.GetShardIterator(args)
shardIterator := resp.ShardIterator
for {
args = kinesis.NewArgs()
args.Add("ShardIterator", shardIterator)
records, err := client.GetRecords(args)
if err != nil {
time.Sleep(3000 * time.Millisecond)
continue
}
messages <- records.Records
shardIterator = records.NextShardIterator
time.Sleep(2000 * time.Millisecond)
}
}
// CreateStream creates a new stream
func CreateStream(k *kinesis.Kinesis, tempStreamName string, shards int) bool {
if err := k.CreateStream(tempStreamName, shards); err != nil {
return false
}
// wait for Stream ready state
timeout := make(chan bool, 30)
resp := &kinesis.DescribeStreamResp{}
log.Printf("Waiting for stream to be created")
for {
args := kinesis.NewArgs()
args.Add("StreamName", tempStreamName)
resp, _ = k.DescribeStream(args)
log.Printf(".")
if resp.StreamDescription.StreamStatus != "ACTIVE" {
time.Sleep(4 * time.Second)
timeout <- true
} else {
break
}
}
return true
}
func GetMessages(k *kinesis.Kinesis, streamName string, shardID string) (out [][]byte, ok bool) {
shardIterator := GetShardIterator(k, streamName, shardID)
for {
args := kinesis.NewArgs()
args.Add("ShardIterator", shardIterator)
recordResp, err := k.GetRecords(args)
if err == nil {
if len(recordResp.Records) > 0 {
for _, d := range recordResp.Records {
res, err := d.GetData()
if err != nil {
LogFile(err.Error())
}
//fmt.Printf("GetRecords Data: %v, err: %v\n", string(res), err)
out = append(out, res)
}
} else {
break
}
}
shardIterator = recordResp.NextShardIterator
}
return out, true
}
func GetShardIterator(k *kinesis.Kinesis, streamName string, shardId string) string {
args := kinesis.NewArgs()
args.Add("StreamName", streamName)
args.Add("ShardId", shardId)
args.Add("ShardIteratorType", "TRIM_HORIZON")
a, _ := k.GetShardIterator(args)
return a.ShardIterator
}
// send sends a data blob to Kinesis.
func (writer *BufferedKinesisWriter) send(message []byte) (err error) {
if message != nil {
args := kinesis.NewArgs()
args.Add("PartitionKey", "test")
args.Add("StreamName", writer.streamName)
args.Add("Data", message)
_, err = writer.client.PutRecord(args)
}
return
}
// SendMessage generates a new JSON blob and sends to a stream
func SendMessage(k *kinesis.Kinesis, streamName string, msg *Message, comms chan bool) {
args := kinesis.NewArgs()
args.Add("StreamName", streamName)
args.Add("PartitionKey", fmt.Sprintf("partitionKey-%d", 1))
if data, ok := msg.ToJSON(); ok {
args.AddData(data)
if _, err := k.PutRecord(args); err == nil {
comms <- true
return
}
}
log.Println("Failed to send message")
comms <- false
return
}
// CheckStream verifies whether a stream already exists
func CheckStream(k *kinesis.Kinesis, streamname string) bool {
args := kinesis.NewArgs()
args.Add("StreamName", streamname)
temp, err := k.ListStreams(args)
if err != nil {
log.Println("Could not obtain list of streams.")
} else {
for _, b := range temp.StreamNames {
if b == streamname {
return true
}
}
}
return false
}
func main() {
auth, err := kinesis.NewAuthFromEnv()
if err != nil {
fmt.Printf("Unable to retrieve authentication credentials from the environment: %v", err)
os.Exit(1)
}
awsRegion := *region
stream := *streamName
fmt.Println("authenticating")
ksis := kinesis.New(auth, awsRegion)
fmt.Println("finding shards")
args := kinesis.NewArgs()
args.Add("StreamName", stream)
description, err := ksis.DescribeStream(args)
if err != nil {
fmt.Printf("Unable to retrieve description from kinesis API: %v", err)
os.Exit(1)
}
messages := make(chan []kinesis.GetRecordsRecords)
for _, shard := range description.StreamDescription.Shards {
go getShardRecords(ksis, stream, shard.ShardId, messages)
}
for {
for _, d := range <-messages {
fmt.Println(string((d.GetData())))
}
}
}
func main() {
fmt.Println("Begin")
streamName := "test"
// set env variables AWS_ACCESS_KEY and AWS_SECRET_KEY AWS_REGION_NAME
ksis := kinesis.New("", "", kinesis.Region{})
err := ksis.CreateStream(streamName, 2)
if err != nil {
fmt.Printf("CreateStream ERROR: %v\n", err)
}
args := kinesis.NewArgs()
resp2, _ := ksis.ListStreams(args)
fmt.Printf("ListStreams: %v\n", resp2)
resp3 := &kinesis.DescribeStreamResp{}
timeout := make(chan bool, 30)
for {
args = kinesis.NewArgs()
args.Add("StreamName", streamName)
resp3, _ = ksis.DescribeStream(args)
fmt.Printf("DescribeStream: %v\n", resp3)
if resp3.StreamDescription.StreamStatus != "ACTIVE" {
time.Sleep(4 * time.Second)
timeout <- true
} else {
break
}
}
for i := 0; i < 10; i++ {
args = kinesis.NewArgs()
args.Add("StreamName", streamName)
args.AddData([]byte(fmt.Sprintf("Hello AWS Kinesis %d", i)))
args.Add("PartitionKey", fmt.Sprintf("partitionKey-%d", i))
resp4, err := ksis.PutRecord(args)
if err != nil {
fmt.Printf("PutRecord err: %v\n", err)
} else {
fmt.Printf("PutRecord: %v\n", resp4)
}
}
for _, shard := range resp3.StreamDescription.Shards {
go getRecords(ksis, streamName, shard.ShardId)
}
var inputGuess string
fmt.Scanf("%s\n", &inputGuess)
err1 := ksis.DeleteStream("test")
if err1 != nil {
fmt.Printf("DeleteStream ERROR: %v\n", err1)
}
fmt.Println("End")
}
func main() {
fmt.Println("Begin")
var (
err error
auth kinesis.Auth
)
streamName := "test"
// set env variables AWS_ACCESS_KEY and AWS_SECRET_KEY AWS_REGION_NAME
auth, err = kinesis.NewAuthFromEnv()
if err != nil {
fmt.Printf("Unable to retrieve authentication credentials from the environment: %v", err)
os.Exit(1)
}
ksis := kinesis.New(auth, "")
err = ksis.CreateStream(streamName, 2)
if err != nil {
fmt.Printf("CreateStream ERROR: %v\n", err)
}
args := kinesis.NewArgs()
resp2, _ := ksis.ListStreams(args)
fmt.Printf("ListStreams: %v\n", resp2)
resp3 := &kinesis.DescribeStreamResp{}
timeout := make(chan bool, 30)
for {
args = kinesis.NewArgs()
args.Add("StreamName", streamName)
resp3, _ = ksis.DescribeStream(args)
fmt.Printf("DescribeStream: %v\n", resp3)
if resp3.StreamDescription.StreamStatus != "ACTIVE" {
time.Sleep(4 * time.Second)
timeout <- true
} else {
break
}
}
// Put records individually
for i := 0; i < 10; i++ {
args = kinesis.NewArgs()
args.Add("StreamName", streamName)
data := []byte(fmt.Sprintf("Hello AWS Kinesis %d", i))
partitionKey := fmt.Sprintf("partitionKey-%d", i)
args.AddRecord(data, partitionKey)
resp4, err := ksis.PutRecord(args)
if err != nil {
fmt.Printf("PutRecord err: %v\n", err)
} else {
fmt.Printf("PutRecord: %v\n", resp4)
}
}
for _, shard := range resp3.StreamDescription.Shards {
go getRecords(ksis, streamName, shard.ShardId)
}
// Put records in batch
args = kinesis.NewArgs()
args.Add("StreamName", streamName)
for i := 0; i < 10; i++ {
args.AddRecord(
[]byte(fmt.Sprintf("Hello AWS Kinesis %d", i)),
fmt.Sprintf("partitionKey-%d", i),
)
}
resp4, err := ksis.PutRecords(args)
if err != nil {
fmt.Printf("PutRecords err: %v\n", err)
} else {
fmt.Printf("PutRecords: %v\n", resp4)
}
// Wait for user input
var inputGuess string
fmt.Scanf("%s\n", &inputGuess)
// Delete the stream
err1 := ksis.DeleteStream("test")
if err1 != nil {
fmt.Printf("DeleteStream ERROR: %v\n", err1)
}
fmt.Println("End")
}
