예제 #1
0
// CreateStream creates a new stream
func CreateStream(k *kinesis.Kinesis, tempStreamName string, shards int) bool {
	if err := k.CreateStream(tempStreamName, shards); err != nil {
		return false
	}

	// wait for Stream ready state
	timeout := make(chan bool, 30)
	resp := &kinesis.DescribeStreamResp{}
	log.Printf("Waiting for stream to be created")
	for {
		args := kinesis.NewArgs()
		args.Add("StreamName", tempStreamName)
		resp, _ = k.DescribeStream(args)
		log.Printf(".")

		if resp.StreamDescription.StreamStatus != "ACTIVE" {
			time.Sleep(4 * time.Second)
			timeout <- true
		} else {
			break
		}
	}

	return true
}
예제 #2
0
func getRecords(ksis *kinesis.Kinesis, streamName, ShardId string) {
	args := kinesis.NewArgs()
	args.Add("StreamName", streamName)
	args.Add("ShardId", ShardId)
	args.Add("ShardIteratorType", "TRIM_HORIZON")
	resp10, _ := ksis.GetShardIterator(args)

	shardIterator := resp10.ShardIterator

	for {
		args = kinesis.NewArgs()
		args.Add("ShardIterator", shardIterator)
		resp11, err := ksis.GetRecords(args)

		if len(resp11.Records) > 0 {
			fmt.Printf("GetRecords Data BEGIN\n")
			for _, d := range resp11.Records {
				res, err := d.GetData()
				fmt.Printf("GetRecords Data: %v, err: %v\n", string(res), err)
			}
			fmt.Printf("GetRecords Data END\n")
		} else if resp11.NextShardIterator == "" || shardIterator == resp11.NextShardIterator || err != nil {
			fmt.Printf("GetRecords ERROR: %v\n", err)
			break
		}

		shardIterator = resp11.NextShardIterator
	}
}
예제 #3
0
파일: consumer.go 프로젝트: vly/haystack
func GetMessages(k *kinesis.Kinesis, streamName string, shardID string) (out [][]byte, ok bool) {
	shardIterator := GetShardIterator(k, streamName, shardID)

	for {
		args := kinesis.NewArgs()
		args.Add("ShardIterator", shardIterator)
		recordResp, err := k.GetRecords(args)
		if err == nil {

			if len(recordResp.Records) > 0 {
				for _, d := range recordResp.Records {
					res, err := d.GetData()
					if err != nil {
						LogFile(err.Error())
					}
					//fmt.Printf("GetRecords Data: %v, err: %v\n", string(res), err)
					out = append(out, res)
				}
			} else {

				break
			}
		}

		shardIterator = recordResp.NextShardIterator

	}
	return out, true
}
예제 #4
0
파일: consumer.go 프로젝트: vly/haystack
func GetShardIterator(k *kinesis.Kinesis, streamName string, shardId string) string {
	args := kinesis.NewArgs()
	args.Add("StreamName", streamName)
	args.Add("ShardId", shardId)
	args.Add("ShardIteratorType", "TRIM_HORIZON")
	a, _ := k.GetShardIterator(args)
	return a.ShardIterator
}
예제 #5
0
// SendMessage generates a new JSON blob and sends to a stream
func SendMessage(k *kinesis.Kinesis, streamName string, msg *Message, comms chan bool) {
	args := kinesis.NewArgs()
	args.Add("StreamName", streamName)
	args.Add("PartitionKey", fmt.Sprintf("partitionKey-%d", 1))
	if data, ok := msg.ToJSON(); ok {
		args.AddData(data)
		if _, err := k.PutRecord(args); err == nil {
			comms <- true
			return
		}
	}
	log.Println("Failed to send message")
	comms <- false
	return
}
예제 #6
0
// CheckStream verifies whether a stream already exists
func CheckStream(k *kinesis.Kinesis, streamname string) bool {
	args := kinesis.NewArgs()
	args.Add("StreamName", streamname)
	temp, err := k.ListStreams(args)
	if err != nil {
		log.Println("Could not obtain list of streams.")
	} else {

		for _, b := range temp.StreamNames {
			if b == streamname {
				return true
			}
		}
	}
	return false
}
예제 #7
0
// DeleteStream deletes an existing stream
func DeleteStream(k *kinesis.Kinesis, streamName string) bool {
	if err := k.DeleteStream(streamName); err == nil {
		return true
	}
	return false
}