Beispiel #1
0
func getRecords(ksis *kinesis.Kinesis, streamName, ShardId string) {
	args := kinesis.NewArgs()
	args.Add("StreamName", streamName)
	args.Add("ShardId", ShardId)
	args.Add("ShardIteratorType", "TRIM_HORIZON")
	resp10, _ := ksis.GetShardIterator(args)

	shardIterator := resp10.ShardIterator

	for {
		args = kinesis.NewArgs()
		args.Add("ShardIterator", shardIterator)
		resp11, err := ksis.GetRecords(args)

		if len(resp11.Records) > 0 {
			fmt.Printf("GetRecords Data BEGIN\n")
			for _, d := range resp11.Records {
				res, err := d.GetData()
				fmt.Printf("GetRecords Data: %v, err: %v\n", string(res), err)
			}
			fmt.Printf("GetRecords Data END\n")
		} else if resp11.NextShardIterator == "" || shardIterator == resp11.NextShardIterator || err != nil {
			fmt.Printf("GetRecords ERROR: %v\n", err)
			break
		}

		shardIterator = resp11.NextShardIterator
	}
}
Beispiel #2
0
func getRecords(ksis kinesis.KinesisClient, streamName, ShardId string) {
	args := kinesis.NewArgs()
	args.Add("StreamName", streamName)
	args.Add("ShardId", ShardId)
	args.Add("ShardIteratorType", "TRIM_HORIZON")
	resp10, _ := ksis.GetShardIterator(args)

	shardIterator := resp10.ShardIterator

	for {
		args = kinesis.NewArgs()
		args.Add("ShardIterator", shardIterator)
		resp11, err := ksis.GetRecords(args)
		if err != nil {
			time.Sleep(1000 * time.Millisecond)
			continue
		}

		if len(resp11.Records) > 0 {
			fmt.Printf("GetRecords Data BEGIN\n")
			for _, d := range resp11.Records {
				fmt.Printf("GetRecords Data: %v\n", string(d.GetData()))
			}
			fmt.Printf("GetRecords Data END\n")
		} else if resp11.NextShardIterator == "" || shardIterator == resp11.NextShardIterator || err != nil {
			fmt.Printf("GetRecords ERROR: %v\n", err)
			break
		}

		shardIterator = resp11.NextShardIterator
		time.Sleep(1000 * time.Millisecond)
	}
}
Beispiel #3
0
func getShardRecords(client kinesis.KinesisClient, streamName string, shardId string, messages chan []kinesis.GetRecordsRecords) {
	args := kinesis.NewArgs()
	args.Add("StreamName", streamName)
	args.Add("ShardId", shardId)
	args.Add("ShardIteratorType", "LATEST") // AT_SEQUENCE_NUMBER | AFTER_SEQUENCE_NUMBER | TRIM_HORIZON | LATEST
	resp, _ := client.GetShardIterator(args)

	shardIterator := resp.ShardIterator

	for {
		args = kinesis.NewArgs()
		args.Add("ShardIterator", shardIterator)
		records, err := client.GetRecords(args)

		if err != nil {
			time.Sleep(3000 * time.Millisecond)
			continue
		}

		messages <- records.Records

		shardIterator = records.NextShardIterator
		time.Sleep(2000 * time.Millisecond)
	}
}
Beispiel #4
0
// CreateStream creates a new stream
func CreateStream(k *kinesis.Kinesis, tempStreamName string, shards int) bool {
	if err := k.CreateStream(tempStreamName, shards); err != nil {
		return false
	}

	// wait for Stream ready state
	timeout := make(chan bool, 30)
	resp := &kinesis.DescribeStreamResp{}
	log.Printf("Waiting for stream to be created")
	for {
		args := kinesis.NewArgs()
		args.Add("StreamName", tempStreamName)
		resp, _ = k.DescribeStream(args)
		log.Printf(".")

		if resp.StreamDescription.StreamStatus != "ACTIVE" {
			time.Sleep(4 * time.Second)
			timeout <- true
		} else {
			break
		}
	}

	return true
}
Beispiel #5
0
func GetMessages(k *kinesis.Kinesis, streamName string, shardID string) (out [][]byte, ok bool) {
	shardIterator := GetShardIterator(k, streamName, shardID)

	for {
		args := kinesis.NewArgs()
		args.Add("ShardIterator", shardIterator)
		recordResp, err := k.GetRecords(args)
		if err == nil {

			if len(recordResp.Records) > 0 {
				for _, d := range recordResp.Records {
					res, err := d.GetData()
					if err != nil {
						LogFile(err.Error())
					}
					//fmt.Printf("GetRecords Data: %v, err: %v\n", string(res), err)
					out = append(out, res)
				}
			} else {

				break
			}
		}

		shardIterator = recordResp.NextShardIterator

	}
	return out, true
}
Beispiel #6
0
func GetShardIterator(k *kinesis.Kinesis, streamName string, shardId string) string {
	args := kinesis.NewArgs()
	args.Add("StreamName", streamName)
	args.Add("ShardId", shardId)
	args.Add("ShardIteratorType", "TRIM_HORIZON")
	a, _ := k.GetShardIterator(args)
	return a.ShardIterator
}
Beispiel #7
0
// send sends a data blob to Kinesis.
func (writer *BufferedKinesisWriter) send(message []byte) (err error) {
	if message != nil {
		args := kinesis.NewArgs()
		args.Add("PartitionKey", "test")
		args.Add("StreamName", writer.streamName)
		args.Add("Data", message)
		_, err = writer.client.PutRecord(args)
	}
	return
}
Beispiel #8
0
// SendMessage generates a new JSON blob and sends to a stream
func SendMessage(k *kinesis.Kinesis, streamName string, msg *Message, comms chan bool) {
	args := kinesis.NewArgs()
	args.Add("StreamName", streamName)
	args.Add("PartitionKey", fmt.Sprintf("partitionKey-%d", 1))
	if data, ok := msg.ToJSON(); ok {
		args.AddData(data)
		if _, err := k.PutRecord(args); err == nil {
			comms <- true
			return
		}
	}
	log.Println("Failed to send message")
	comms <- false
	return
}
Beispiel #9
0
// CheckStream verifies whether a stream already exists
func CheckStream(k *kinesis.Kinesis, streamname string) bool {
	args := kinesis.NewArgs()
	args.Add("StreamName", streamname)
	temp, err := k.ListStreams(args)
	if err != nil {
		log.Println("Could not obtain list of streams.")
	} else {

		for _, b := range temp.StreamNames {
			if b == streamname {
				return true
			}
		}
	}
	return false
}
Beispiel #10
0
func main() {
	auth, err := kinesis.NewAuthFromEnv()

	if err != nil {
		fmt.Printf("Unable to retrieve authentication credentials from the environment: %v", err)
		os.Exit(1)
	}

	awsRegion := *region
	stream := *streamName

	fmt.Println("authenticating")
	ksis := kinesis.New(auth, awsRegion)

	fmt.Println("finding shards")
	args := kinesis.NewArgs()
	args.Add("StreamName", stream)
	description, err := ksis.DescribeStream(args)

	if err != nil {
		fmt.Printf("Unable to retrieve description from kinesis API: %v", err)
		os.Exit(1)
	}

	messages := make(chan []kinesis.GetRecordsRecords)

	for _, shard := range description.StreamDescription.Shards {
		go getShardRecords(ksis, stream, shard.ShardId, messages)
	}

	for {
		for _, d := range <-messages {
			fmt.Println(string((d.GetData())))
		}
	}
}
Beispiel #11
0
func main() {
	fmt.Println("Begin")

	streamName := "test"
	// set env variables AWS_ACCESS_KEY and AWS_SECRET_KEY AWS_REGION_NAME
	ksis := kinesis.New("", "", kinesis.Region{})

	err := ksis.CreateStream(streamName, 2)
	if err != nil {
		fmt.Printf("CreateStream ERROR: %v\n", err)
	}

	args := kinesis.NewArgs()
	resp2, _ := ksis.ListStreams(args)
	fmt.Printf("ListStreams: %v\n", resp2)

	resp3 := &kinesis.DescribeStreamResp{}

	timeout := make(chan bool, 30)
	for {

		args = kinesis.NewArgs()
		args.Add("StreamName", streamName)
		resp3, _ = ksis.DescribeStream(args)
		fmt.Printf("DescribeStream: %v\n", resp3)

		if resp3.StreamDescription.StreamStatus != "ACTIVE" {
			time.Sleep(4 * time.Second)
			timeout <- true
		} else {
			break
		}

	}

	for i := 0; i < 10; i++ {
		args = kinesis.NewArgs()
		args.Add("StreamName", streamName)
		args.AddData([]byte(fmt.Sprintf("Hello AWS Kinesis %d", i)))
		args.Add("PartitionKey", fmt.Sprintf("partitionKey-%d", i))
		resp4, err := ksis.PutRecord(args)
		if err != nil {
			fmt.Printf("PutRecord err: %v\n", err)
		} else {
			fmt.Printf("PutRecord: %v\n", resp4)
		}
	}

	for _, shard := range resp3.StreamDescription.Shards {
		go getRecords(ksis, streamName, shard.ShardId)
	}

	var inputGuess string
	fmt.Scanf("%s\n", &inputGuess)

	err1 := ksis.DeleteStream("test")
	if err1 != nil {
		fmt.Printf("DeleteStream ERROR: %v\n", err1)
	}

	fmt.Println("End")
}
Beispiel #12
0
func main() {
	fmt.Println("Begin")
	var (
		err  error
		auth kinesis.Auth
	)

	streamName := "test"
	// set env variables AWS_ACCESS_KEY and AWS_SECRET_KEY AWS_REGION_NAME
	auth, err = kinesis.NewAuthFromEnv()
	if err != nil {
		fmt.Printf("Unable to retrieve authentication credentials from the environment: %v", err)
		os.Exit(1)
	}
	ksis := kinesis.New(auth, "")

	err = ksis.CreateStream(streamName, 2)
	if err != nil {
		fmt.Printf("CreateStream ERROR: %v\n", err)
	}

	args := kinesis.NewArgs()
	resp2, _ := ksis.ListStreams(args)
	fmt.Printf("ListStreams: %v\n", resp2)

	resp3 := &kinesis.DescribeStreamResp{}

	timeout := make(chan bool, 30)
	for {

		args = kinesis.NewArgs()
		args.Add("StreamName", streamName)
		resp3, _ = ksis.DescribeStream(args)
		fmt.Printf("DescribeStream: %v\n", resp3)

		if resp3.StreamDescription.StreamStatus != "ACTIVE" {
			time.Sleep(4 * time.Second)
			timeout <- true
		} else {
			break
		}

	}

	// Put records individually
	for i := 0; i < 10; i++ {
		args = kinesis.NewArgs()
		args.Add("StreamName", streamName)
		data := []byte(fmt.Sprintf("Hello AWS Kinesis %d", i))
		partitionKey := fmt.Sprintf("partitionKey-%d", i)
		args.AddRecord(data, partitionKey)
		resp4, err := ksis.PutRecord(args)
		if err != nil {
			fmt.Printf("PutRecord err: %v\n", err)
		} else {
			fmt.Printf("PutRecord: %v\n", resp4)
		}
	}

	for _, shard := range resp3.StreamDescription.Shards {
		go getRecords(ksis, streamName, shard.ShardId)
	}

	// Put records in batch
	args = kinesis.NewArgs()
	args.Add("StreamName", streamName)

	for i := 0; i < 10; i++ {
		args.AddRecord(
			[]byte(fmt.Sprintf("Hello AWS Kinesis %d", i)),
			fmt.Sprintf("partitionKey-%d", i),
		)
	}

	resp4, err := ksis.PutRecords(args)
	if err != nil {
		fmt.Printf("PutRecords err: %v\n", err)
	} else {
		fmt.Printf("PutRecords: %v\n", resp4)
	}

	// Wait for user input
	var inputGuess string
	fmt.Scanf("%s\n", &inputGuess)

	// Delete the stream
	err1 := ksis.DeleteStream("test")
	if err1 != nil {
		fmt.Printf("DeleteStream ERROR: %v\n", err1)
	}

	fmt.Println("End")
}