func getKinesis(route *router.Route) *kinesis.Kinesis {
var auth *kinesis.AuthCredentials
var err error
if getEnvVar("AWS_ACCESS_KEY", "") != "" {
// Get auth from env variables AWS_ACCESS_KEY and AWS_SECRET_KEY
auth, err = kinesis.NewAuthFromEnv()
if err != nil {
fmt.Printf("Unable to retrieve authentication credentials from the environment: %v", err)
os.Exit(1)
}
} else {
// Get auth from meta-data-server
auth, err = kinesis.NewAuthFromMetadata()
if err != nil {
fmt.Printf("Unable to retrieve authentication credentials from the the meta-data-server: %v", err)
os.Exit(1)
}
}
// AWS region
aws_region := kinesis.NewRegionFromEnv()
fmt.Printf("# KINESIS Adapter - Using region: %s\n", aws_region)
return kinesis.New(auth, aws_region)
}
// InitKinesis initialises a connection to the Kinesis service
func InitKinesis() (*kinesis.Kinesis, bool) {
k := new(Keys)
if ok := loadTokens(k); ok {
ksis := kinesis.New(k.AccessKey, k.SecretKey)
return ksis, true
}
return nil, false
}
func NewBufferedKinesisWriter(accessKey, secretKey, streamName string, buffer int) *BufferedKinesisWriter {
writer := new(BufferedKinesisWriter)
writer.client = kinesis.New(accessKey, secretKey)
writer.messages = make(chan []byte, buffer)
writer.streamName = streamName
writer.flushLock = new(sync.Mutex)
return writer
}
func getKinesis(route *router.Route) *kinesis.Kinesis {
// set env variables AWS_ACCESS_KEY and AWS_SECRET_KEY AWS_REGION_NAME
auth, err := kinesis.NewAuthFromEnv()
if err != nil {
fmt.Printf("Unable to retrieve authentication credentials from the environment: %v", err)
os.Exit(1)
}
// AWS region
aws_region := kinesis.NewRegionFromEnv()
fmt.Printf("# KINESIS Adapter - Using region: %s\n", aws_region)
return kinesis.New(auth, aws_region)
}
func main() {
auth, err := kinesis.NewAuthFromEnv()
if err != nil {
fmt.Printf("Unable to retrieve authentication credentials from the environment: %v", err)
os.Exit(1)
}
awsRegion := *region
stream := *streamName
fmt.Println("authenticating")
ksis := kinesis.New(auth, awsRegion)
fmt.Println("finding shards")
args := kinesis.NewArgs()
args.Add("StreamName", stream)
description, err := ksis.DescribeStream(args)
if err != nil {
fmt.Printf("Unable to retrieve description from kinesis API: %v", err)
os.Exit(1)
}
messages := make(chan []kinesis.GetRecordsRecords)
for _, shard := range description.StreamDescription.Shards {
go getShardRecords(ksis, stream, shard.ShardId, messages)
}
for {
for _, d := range <-messages {
fmt.Println(string((d.GetData())))
}
}
}
func main() {
fmt.Println("Begin")
streamName := "test"
// set env variables AWS_ACCESS_KEY and AWS_SECRET_KEY AWS_REGION_NAME
ksis := kinesis.New("", "", kinesis.Region{})
err := ksis.CreateStream(streamName, 2)
if err != nil {
fmt.Printf("CreateStream ERROR: %v\n", err)
}
args := kinesis.NewArgs()
resp2, _ := ksis.ListStreams(args)
fmt.Printf("ListStreams: %v\n", resp2)
resp3 := &kinesis.DescribeStreamResp{}
timeout := make(chan bool, 30)
for {
args = kinesis.NewArgs()
args.Add("StreamName", streamName)
resp3, _ = ksis.DescribeStream(args)
fmt.Printf("DescribeStream: %v\n", resp3)
if resp3.StreamDescription.StreamStatus != "ACTIVE" {
time.Sleep(4 * time.Second)
timeout <- true
} else {
break
}
}
for i := 0; i < 10; i++ {
args = kinesis.NewArgs()
args.Add("StreamName", streamName)
args.AddData([]byte(fmt.Sprintf("Hello AWS Kinesis %d", i)))
args.Add("PartitionKey", fmt.Sprintf("partitionKey-%d", i))
resp4, err := ksis.PutRecord(args)
if err != nil {
fmt.Printf("PutRecord err: %v\n", err)
} else {
fmt.Printf("PutRecord: %v\n", resp4)
}
}
for _, shard := range resp3.StreamDescription.Shards {
go getRecords(ksis, streamName, shard.ShardId)
}
var inputGuess string
fmt.Scanf("%s\n", &inputGuess)
err1 := ksis.DeleteStream("test")
if err1 != nil {
fmt.Printf("DeleteStream ERROR: %v\n", err1)
}
fmt.Println("End")
}
func main() {
fmt.Println("Begin")
var (
err error
auth kinesis.Auth
)
streamName := "test"
// set env variables AWS_ACCESS_KEY and AWS_SECRET_KEY AWS_REGION_NAME
auth, err = kinesis.NewAuthFromEnv()
if err != nil {
fmt.Printf("Unable to retrieve authentication credentials from the environment: %v", err)
os.Exit(1)
}
ksis := kinesis.New(auth, "")
err = ksis.CreateStream(streamName, 2)
if err != nil {
fmt.Printf("CreateStream ERROR: %v\n", err)
}
args := kinesis.NewArgs()
resp2, _ := ksis.ListStreams(args)
fmt.Printf("ListStreams: %v\n", resp2)
resp3 := &kinesis.DescribeStreamResp{}
timeout := make(chan bool, 30)
for {
args = kinesis.NewArgs()
args.Add("StreamName", streamName)
resp3, _ = ksis.DescribeStream(args)
fmt.Printf("DescribeStream: %v\n", resp3)
if resp3.StreamDescription.StreamStatus != "ACTIVE" {
time.Sleep(4 * time.Second)
timeout <- true
} else {
break
}
}
// Put records individually
for i := 0; i < 10; i++ {
args = kinesis.NewArgs()
args.Add("StreamName", streamName)
data := []byte(fmt.Sprintf("Hello AWS Kinesis %d", i))
partitionKey := fmt.Sprintf("partitionKey-%d", i)
args.AddRecord(data, partitionKey)
resp4, err := ksis.PutRecord(args)
if err != nil {
fmt.Printf("PutRecord err: %v\n", err)
} else {
fmt.Printf("PutRecord: %v\n", resp4)
}
}
for _, shard := range resp3.StreamDescription.Shards {
go getRecords(ksis, streamName, shard.ShardId)
}
// Put records in batch
args = kinesis.NewArgs()
args.Add("StreamName", streamName)
for i := 0; i < 10; i++ {
args.AddRecord(
[]byte(fmt.Sprintf("Hello AWS Kinesis %d", i)),
fmt.Sprintf("partitionKey-%d", i),
)
}
resp4, err := ksis.PutRecords(args)
if err != nil {
fmt.Printf("PutRecords err: %v\n", err)
} else {
fmt.Printf("PutRecords: %v\n", resp4)
}
// Wait for user input
var inputGuess string
fmt.Scanf("%s\n", &inputGuess)
// Delete the stream
err1 := ksis.DeleteStream("test")
if err1 != nil {
fmt.Printf("DeleteStream ERROR: %v\n", err1)
}
fmt.Println("End")
}