func getKinesis(route *router.Route) *kinesis.Kinesis {
	var auth *kinesis.AuthCredentials
	var err error

	if getEnvVar("AWS_ACCESS_KEY", "") != "" {
		// Get auth from env variables AWS_ACCESS_KEY and AWS_SECRET_KEY
		auth, err = kinesis.NewAuthFromEnv()
		if err != nil {
			fmt.Printf("Unable to retrieve authentication credentials from the environment: %v", err)
			os.Exit(1)
		}
	} else {
		// Get auth from meta-data-server
		auth, err = kinesis.NewAuthFromMetadata()
		if err != nil {
			fmt.Printf("Unable to retrieve authentication credentials from the the meta-data-server: %v", err)
			os.Exit(1)
		}
	}

	// AWS region
	aws_region := kinesis.NewRegionFromEnv()
	fmt.Printf("# KINESIS Adapter - Using region: %s\n", aws_region)

	return kinesis.New(auth, aws_region)
}
Exemple #2
0
// InitKinesis initialises a connection to the Kinesis service
func InitKinesis() (*kinesis.Kinesis, bool) {
	k := new(Keys)
	if ok := loadTokens(k); ok {
		ksis := kinesis.New(k.AccessKey, k.SecretKey)
		return ksis, true
	}
	return nil, false
}
Exemple #3
0
func NewBufferedKinesisWriter(accessKey, secretKey, streamName string, buffer int) *BufferedKinesisWriter {
	writer := new(BufferedKinesisWriter)
	writer.client = kinesis.New(accessKey, secretKey)
	writer.messages = make(chan []byte, buffer)
	writer.streamName = streamName
	writer.flushLock = new(sync.Mutex)
	return writer
}
func getKinesis(route *router.Route) *kinesis.Kinesis {
	// set env variables AWS_ACCESS_KEY and AWS_SECRET_KEY AWS_REGION_NAME
	auth, err := kinesis.NewAuthFromEnv()
	if err != nil {
		fmt.Printf("Unable to retrieve authentication credentials from the environment: %v", err)
		os.Exit(1)
	}

	// AWS region
	aws_region := kinesis.NewRegionFromEnv()
	fmt.Printf("# KINESIS Adapter - Using region: %s\n", aws_region)

	return kinesis.New(auth, aws_region)
}
Exemple #5
0
func main() {
	auth, err := kinesis.NewAuthFromEnv()

	if err != nil {
		fmt.Printf("Unable to retrieve authentication credentials from the environment: %v", err)
		os.Exit(1)
	}

	awsRegion := *region
	stream := *streamName

	fmt.Println("authenticating")
	ksis := kinesis.New(auth, awsRegion)

	fmt.Println("finding shards")
	args := kinesis.NewArgs()
	args.Add("StreamName", stream)
	description, err := ksis.DescribeStream(args)

	if err != nil {
		fmt.Printf("Unable to retrieve description from kinesis API: %v", err)
		os.Exit(1)
	}

	messages := make(chan []kinesis.GetRecordsRecords)

	for _, shard := range description.StreamDescription.Shards {
		go getShardRecords(ksis, stream, shard.ShardId, messages)
	}

	for {
		for _, d := range <-messages {
			fmt.Println(string((d.GetData())))
		}
	}
}
Exemple #6
0
func main() {
	fmt.Println("Begin")

	streamName := "test"
	// set env variables AWS_ACCESS_KEY and AWS_SECRET_KEY AWS_REGION_NAME
	ksis := kinesis.New("", "", kinesis.Region{})

	err := ksis.CreateStream(streamName, 2)
	if err != nil {
		fmt.Printf("CreateStream ERROR: %v\n", err)
	}

	args := kinesis.NewArgs()
	resp2, _ := ksis.ListStreams(args)
	fmt.Printf("ListStreams: %v\n", resp2)

	resp3 := &kinesis.DescribeStreamResp{}

	timeout := make(chan bool, 30)
	for {

		args = kinesis.NewArgs()
		args.Add("StreamName", streamName)
		resp3, _ = ksis.DescribeStream(args)
		fmt.Printf("DescribeStream: %v\n", resp3)

		if resp3.StreamDescription.StreamStatus != "ACTIVE" {
			time.Sleep(4 * time.Second)
			timeout <- true
		} else {
			break
		}

	}

	for i := 0; i < 10; i++ {
		args = kinesis.NewArgs()
		args.Add("StreamName", streamName)
		args.AddData([]byte(fmt.Sprintf("Hello AWS Kinesis %d", i)))
		args.Add("PartitionKey", fmt.Sprintf("partitionKey-%d", i))
		resp4, err := ksis.PutRecord(args)
		if err != nil {
			fmt.Printf("PutRecord err: %v\n", err)
		} else {
			fmt.Printf("PutRecord: %v\n", resp4)
		}
	}

	for _, shard := range resp3.StreamDescription.Shards {
		go getRecords(ksis, streamName, shard.ShardId)
	}

	var inputGuess string
	fmt.Scanf("%s\n", &inputGuess)

	err1 := ksis.DeleteStream("test")
	if err1 != nil {
		fmt.Printf("DeleteStream ERROR: %v\n", err1)
	}

	fmt.Println("End")
}
Exemple #7
0
func main() {
	fmt.Println("Begin")
	var (
		err  error
		auth kinesis.Auth
	)

	streamName := "test"
	// set env variables AWS_ACCESS_KEY and AWS_SECRET_KEY AWS_REGION_NAME
	auth, err = kinesis.NewAuthFromEnv()
	if err != nil {
		fmt.Printf("Unable to retrieve authentication credentials from the environment: %v", err)
		os.Exit(1)
	}
	ksis := kinesis.New(auth, "")

	err = ksis.CreateStream(streamName, 2)
	if err != nil {
		fmt.Printf("CreateStream ERROR: %v\n", err)
	}

	args := kinesis.NewArgs()
	resp2, _ := ksis.ListStreams(args)
	fmt.Printf("ListStreams: %v\n", resp2)

	resp3 := &kinesis.DescribeStreamResp{}

	timeout := make(chan bool, 30)
	for {

		args = kinesis.NewArgs()
		args.Add("StreamName", streamName)
		resp3, _ = ksis.DescribeStream(args)
		fmt.Printf("DescribeStream: %v\n", resp3)

		if resp3.StreamDescription.StreamStatus != "ACTIVE" {
			time.Sleep(4 * time.Second)
			timeout <- true
		} else {
			break
		}

	}

	// Put records individually
	for i := 0; i < 10; i++ {
		args = kinesis.NewArgs()
		args.Add("StreamName", streamName)
		data := []byte(fmt.Sprintf("Hello AWS Kinesis %d", i))
		partitionKey := fmt.Sprintf("partitionKey-%d", i)
		args.AddRecord(data, partitionKey)
		resp4, err := ksis.PutRecord(args)
		if err != nil {
			fmt.Printf("PutRecord err: %v\n", err)
		} else {
			fmt.Printf("PutRecord: %v\n", resp4)
		}
	}

	for _, shard := range resp3.StreamDescription.Shards {
		go getRecords(ksis, streamName, shard.ShardId)
	}

	// Put records in batch
	args = kinesis.NewArgs()
	args.Add("StreamName", streamName)

	for i := 0; i < 10; i++ {
		args.AddRecord(
			[]byte(fmt.Sprintf("Hello AWS Kinesis %d", i)),
			fmt.Sprintf("partitionKey-%d", i),
		)
	}

	resp4, err := ksis.PutRecords(args)
	if err != nil {
		fmt.Printf("PutRecords err: %v\n", err)
	} else {
		fmt.Printf("PutRecords: %v\n", resp4)
	}

	// Wait for user input
	var inputGuess string
	fmt.Scanf("%s\n", &inputGuess)

	// Delete the stream
	err1 := ksis.DeleteStream("test")
	if err1 != nil {
		fmt.Printf("DeleteStream ERROR: %v\n", err1)
	}

	fmt.Println("End")
}