Beispiel #1
0
func (self *Reader) run() {
	defer finish(self.done, self.src) // To signal that job is done

	server := dynamodb.New(*self.auth, self.region)

	//just test the connection to dyno table
	tableDesc, err := server.DescribeTable(self.tableName)
	if err != nil {
		log.Fatal("Could not DescribeTable", err)
	}
	pk, err := tableDesc.BuildPrimaryKey()
	if err != nil {
		log.Fatal("Could not BuildPrimaryKey", err)
	}

	table := server.NewTable(self.tableName, pk)

	items, startKey, err := table.ParallelScanPartialLimit(
		self.attributeComparisons, nil,
		self.seg_id, self.total_seg, int64(self.batch_size))

	if err == nil {
		for _, item := range items {
			self.work <- item
			self.count += 1
		}

		for startKey != nil {
			items, startKey, err = table.ParallelScanPartialLimit(
				self.attributeComparisons, startKey,
				self.seg_id, self.total_seg, int64(self.batch_size))

			if err == nil {
				for _, item := range items {
					self.work <- item
					self.count += 1
				}

			} else {
				e := Event{
					self.src,
					"error",
					err.Error(),
				}
				self.events <- e
			}
		}

	} else {
		e := Event{
			self.src,
			"error",
			err.Error(),
		}
		self.events <- e
	}

}
Beispiel #2
0
func getDynamoServer(ctx *Context, region string) (*dynamodb.Server, error) {

	Log(INFO, ctx, "getDynamoServer", "region", region)

	// If we don't have real access keys, just try local.  See
	// ../local-dynamodb/.  Otherwise go for real DynamoDB.  ToDo:
	// insist on using the real DynamoDB.

	// ToDo: Combine with router/aws.go FindAWS()

	if region == "local" {
		r := aws.Region{DynamoDBEndpoint: "http://127.0.0.1:8000"}
		auth := aws.Auth{AccessKey: "DUMMY_KEY", SecretKey: "DUMMY_SECRET"}
		return dynamodb.New(auth, r), nil
	} else if strings.HasPrefix(region, "http:") {
		r := aws.Region{DynamoDBEndpoint: region}
		auth, err := aws.GetAuth("", "", "", time.Now().Add(100000*time.Hour))
		if err != nil {
			Log(INFO, ctx, "router.FindAWS", "warning", err)
			return nil, err
		}
		return dynamodb.New(auth, r), nil
	} else {
		auth, err := aws.EnvAuth()
		if err != nil {
			Log(INFO, ctx, "getDynamoServer", "warning", err, "when", "aws.EnvAuth")
			// return nil, nil, err
			// ToDo: Fix 100000 ...
			auth, err = aws.GetAuth("", "", "", time.Now().Add(100000*time.Hour))
			if err != nil {
				Log(INFO, ctx, "router.FindAWS", "warning", err)
				return nil, err
			}
		}
		r, found := aws.Regions[region]
		if !found {
			err = fmt.Errorf("Bad region name '%s'", region)
			Log(INFO, ctx, "getDynamoServer", "error", err)
			return nil, err
		}

		return dynamodb.New(auth, r), nil
	}
}
Beispiel #3
0
func New(service string, region string, tableName string) *Discovery {
	pk, err := tableDescription.BuildPrimaryKey()
	if err != nil {
		log.Fatal(err)
	}

	auth, err := aws.GetAuth("", "", "", time.Now())
	dbServer := dynamodb.New(auth, aws.GetRegion(region))
	table := dbServer.NewTable(tableName, pk)
	return &Discovery{table: table, Service: service}
}
Beispiel #4
0
func (f *Factory) New(uri *url.URL) bridge.RegistryAdapter {
	pk, err := tableDescription.BuildPrimaryKey()
	if err != nil {
		log.Fatal(err)
	}

	auth, err := aws.GetAuth("", "", "", time.Now())

	dbServer := dynamodb.New(auth, aws.GetRegion(uri.Host))
	table := dbServer.NewTable(strings.TrimPrefix(uri.Path, "/"), pk)
	return &MikroAdapter{table: table}
}
Beispiel #5
0
func (self *Writer) run() {
	defer finish(self.done, self.src) // To signal that job is done
	server := dynamodb.New(*self.auth, self.region)

	//just test the connection to dyno table
	tableDesc, err := server.DescribeTable(self.tableName)
	if err != nil {
		log.Fatal("Could not DescribeTable", err)
	}
	pk, err := tableDesc.BuildPrimaryKey()
	if err != nil {
		log.Fatal("Could not BuildPrimaryKey", err)
	}

	table := server.NewTable(self.tableName, pk)
	batch := [][]dynamodb.Attribute{}

	for w := range self.work {
		item := map_to_item(w)
		batch = append(batch, *item)

		if len(batch) == self.batch_size {
			err = self.batch_shoot(table, batch)
			if err != nil {
				self.events <- Event{
					self.src,
					"error",
					err.Error(),
				}
			}
			batch = nil
		}
	}

	if len(batch) > 0 {
		// will do that again if batch isn't empty
		err = self.batch_shoot(table, batch)
		if err != nil {
			self.events <- Event{
				self.src,
				"error",
				err.Error(),
			}
		}
		batch = nil
	}
}
Beispiel #6
0
func sniff(auth *aws.Auth,
	region aws.Region,
	src string,
	dst string) (*dynamodb.TableDescriptionT, *dynamodb.TableDescriptionT, error) {
	server := dynamodb.New(*auth, region)

	//just test the connection to dyno table
	srcDesc, err := server.DescribeTable(src)

	if err != nil {
		return nil, nil, err
	}

	dstDesc, err := server.DescribeTable(dst)

	if err != nil {
		return nil, nil, err
	}

	return srcDesc, dstDesc, nil
}