Ejemplo n.º 1
0
// query using LSI or GSI
func (t *DynamoTable) GetByIndex(idx string, values ...Any) ([]map[string]interface{}, error) {
	index, ok := t.indexes[idx]
	if !ok {
		log.Error("[DynamoDB] Cannot find the index name, table="+t.name, idx)
		log.Error("[DynamoDB] indexes on table="+t.name, t.indexes)
	}

	hashKey := index.GetHashKeyName()
	rangeKey := index.GetRangeKeyName()

	keys := make(map[string]*SDK.Condition)
	keys[hashKey] = &SDK.Condition{
		AttributeValueList: []*SDK.AttributeValue{createAttributeValue(values[0])},
		ComparisonOperator: String(ComparisonOperatorEQ),
	}
	if len(values) > 1 && rangeKey != "" {
		keys[rangeKey] = &SDK.Condition{
			AttributeValueList: []*SDK.AttributeValue{createAttributeValue(values[1])},
			ComparisonOperator: String(ComparisonOperatorEQ),
		}
	}

	in := &SDK.QueryInput{
		TableName:     String(t.name),
		KeyConditions: keys,
		IndexName:     &idx,
	}
	return t.Query(in)
}
Ejemplo n.º 2
0
// Get message from the queue with limit
func (q *Queue) Fetch(num int) (*SDK.ReceiveMessageOutput, error) {
	// use long-polling for 1sec when to get multiple messages
	var wait int
	if num > 1 {
		wait = 1
	} else {
		wait = 0
	}

	// receive message from server
	resp, err := q.client.ReceiveMessage(&SDK.ReceiveMessageInput{
		QueueURL:            q.url,
		WaitTimeSeconds:     Long(wait),
		MaxNumberOfMessages: Long(num),
		VisibilityTimeout:   Long(defaultExpireSecond),
	})
	if err != nil {
		log.Error("[SQS] error on `ReceiveMessage` operation, queue="+q.name, err.Error())
	}

	// delete messages automatically
	if q.autoDel && len(resp.Messages) > 0 {
		q.AddDeleteList(resp.Messages)
		defer q.DeleteListItems()
	}

	return resp, err
}
Ejemplo n.º 3
0
// Send messages in the send spool
func (q *Queue) Send() error {
	// pack the messages ten each to meet the SQS restriction.
	messages := make(map[int][]*SDK.SendMessageBatchRequestEntry)
	if len(q.messages) > 10 {
		for i, msg := range q.messages {
			v := (i + 1) / 10
			messages[v] = append(messages[v], msg)
		}
	} else {
		messages[0] = append(messages[0], q.messages...)
	}

	var err error = nil
	errStr := ""
	// send message
	for i := 0; i < len(messages); i++ {
		e := q.send(messages[i])
		if e != nil {
			log.Error("[SQS] error on `SendMessageBatch` operation, queue="+q.name, e.Error())
			errStr = errStr + "," + e.Error()
		}
	}
	if errStr != "" {
		err = errors.New(errStr)
	}
	return err
}
Ejemplo n.º 4
0
// get mapped-items with Query operation
func (t *DynamoTable) Query(in *SDK.QueryInput) ([]map[string]interface{}, error) {
	req, err := t.db.client.Query(in)
	if err != nil {
		log.Error("[DynamoDB] Error in `Query` operation, table="+t.name, err)
		return nil, err
	}
	return t.convertItemsToMapArray(req.Items), nil
}
Ejemplo n.º 5
0
// Create new SQS Queue
func (svc *AmazonSQS) CreateQueue(in *SDK.CreateQueueInput) error {
	data, err := svc.client.CreateQueue(in)
	if err != nil {
		log.Error("[SQS] Error on `CreateQueue` operation, queue="+*in.QueueName, err)
		return err
	}
	log.Info("[SQS] Complete CreateQueue, queue="+*in.QueueName, *(data.QueueURL))
	return nil
}
Ejemplo n.º 6
0
// Create new DynamoDB table
func (d *AmazonDynamoDB) CreateTable(in *SDK.CreateTableInput) error {
	data, err := d.client.CreateTable(in)
	if err != nil {
		log.Error("[DynamoDB] Error on `CreateTable` operation, table="+*in.TableName, err)
		return err
	}
	log.Info("[DynamoDB] Complete CreateTable, table="+*in.TableName, data.TableDescription.TableStatus)
	return nil
}
Ejemplo n.º 7
0
// Add message spool from map data
func (q *Queue) AddMessageMap(message map[string]interface{}) error {
	msg, err := json.Marshal(message)
	if err != nil {
		log.Error("[SQS] error on `json.Marshal`, msg="+fmt.Sprint(msg), err.Error())
		return err
	}
	q.AddMessage(string(msg))
	return nil
}
Ejemplo n.º 8
0
func (t *DynamoTable) Desc() (*SDK.TableDescription, error) {
	req, err := t.db.client.DescribeTable(&SDK.DescribeTableInput{
		TableName: String(t.name),
	})
	if err != nil {
		log.Error("[DynamoDB] Error in `DescribeTable` operation, table="+t.name, err)
		return nil, err
	}
	return req.Table, nil
}
Ejemplo n.º 9
0
// Delete all messages on the Queue
func (q *Queue) Purge() error {
	_, err := q.client.PurgeQueue(&SDK.PurgeQueueInput{
		QueueURL: q.url,
	})
	if err != nil {
		log.Error("[SQS] error on `PurgeQueue`, queue="+q.name, err.Error())
		return err
	}
	return nil
}
Ejemplo n.º 10
0
// GetEndpoint
func (svc *AmazonSNS) GetEndpoint(arn string) (*SNSEndpoint, error) {
	in := &SDK.GetEndpointAttributesInput{
		EndpointARN: String(arn),
	}
	resp, err := svc.Client.GetEndpointAttributes(in)
	if err != nil {
		log.Error("[SNS] error on `GetEndpointAttributes` operation, arn="+arn, err.Error())
		return nil, err
	}
	attr := resp.Attributes
	ep := svc.NewApplicationEndpoint(arn)
	ep.token = *attr["Token"]
	ep.enable, err = strconv.ParseBool(*attr["Enabled"])
	if err != nil {
		log.Error("[SNS] error on `Endpoint Attributes` Enabled="+*attr["Enabled"], err.Error())
		ep.enable = false
	}
	return ep, err
}
Ejemplo n.º 11
0
// Delete a message from server
func (q *Queue) DeleteMessage(msg *SDK.Message) error {
	_, err := q.client.DeleteMessage(&SDK.DeleteMessageInput{
		QueueURL:      q.url,
		ReceiptHandle: msg.ReceiptHandle,
	})
	if err != nil {
		log.Error("[SQS] error on `DeleteMessage`, queue="+q.name, err.Error())
	}
	return err
}
Ejemplo n.º 12
0
// delete object of target path
func (b *Bucket) DeleteObject(path string) error {
	_, err := b.client.DeleteObject(&SDK.DeleteObjectInput{
		Bucket: String(b.name),
		Key:    String(path),
	})
	if err != nil {
		log.Error("[S3] error on `DeleteObject` operation, bucket="+b.name, err.Error())
	}
	return err
}
Ejemplo n.º 13
0
// Create Endpoint(add device) and return `EndpointARN`
func (a *SNSApp) createEndpoint(token string) (string, error) {
	in := &SDK.CreatePlatformEndpointInput{
		PlatformApplicationARN: String(a.arn),
		Token: String(token),
	}
	resp, err := a.svc.Client.CreatePlatformEndpoint(in)
	if err != nil {
		log.Error("[SNS] error on `CreatePlatformEndpoint` operation, token="+token, err.Error())
		return "", err
	}
	return *resp.EndpointARN, nil
}
Ejemplo n.º 14
0
// Delete DynamoDB table
func (d *AmazonDynamoDB) DeleteTable(name string) error {
	in := &SDK.DeleteTableInput{
		TableName: String(name),
	}
	data, err := d.client.DeleteTable(in)
	if err != nil {
		log.Error("[DynamoDB] Error on `DeleteTable` operation, table="+*in.TableName, err)
		return err
	}
	log.Info("[DynamoDB] Complete DeleteTable, table="+*in.TableName, data.TableDescription.TableStatus)
	return nil
}
Ejemplo n.º 15
0
// fetch object from target S3 path
func (b *Bucket) getObject(path string) (io.Reader, error) {
	req := SDK.GetObjectInput{
		Bucket: &b.name,
		Key:    &path,
	}
	out, err := b.client.GetObject(&req)
	if err != nil {
		log.Error("[S3] error on `GetObject` operation, bucket="+b.name, err.Error())
		return nil, err
	}
	return out.Body, err
}
Ejemplo n.º 16
0
// Subscribe
func (t *SNSTopic) Subscribe(endpoint *SNSEndpoint) (string, error) {
	resp, err := t.svc.Client.Subscribe(&SDK.SubscribeInput{
		Endpoint: String(endpoint.arn),
		Protocol: String(endpoint.protocol),
		TopicARN: String(t.arn),
	})
	if err != nil {
		log.Error("[SNS] error on `Subscribe` operation, topic="+t.arn, err.Error())
		return "", err
	}
	return *resp.SubscriptionARN, nil
}
Ejemplo n.º 17
0
// get mapped-items with Scan operation
func (t *DynamoTable) Scan() ([]map[string]interface{}, error) {
	in := &SDK.ScanInput{
		TableName: String(t.name),
		Limit:     Long(1000),
	}
	req, err := t.db.client.Scan(in)
	if err != nil {
		log.Error("[DynamoDB] Error in `Scan` operation, table="+t.name, err)
		return nil, err
	}
	return t.convertItemsToMapArray(req.Items), nil
}
Ejemplo n.º 18
0
// Create SNS Topic and return `TopicARN`
func (svc *AmazonSNS) createTopic(name string) (string, error) {
	prefix := config.GetConfigValue(snsConfigSectionName, "prefix", defaultPrefix)
	in := &SDK.CreateTopicInput{
		Name: String(prefix + name),
	}
	resp, err := svc.Client.CreateTopic(in)
	if err != nil {
		log.Error("[SNS] error on `CreateTopic` operation, name="+name, err.Error())
		return "", err
	}
	return *resp.TopicARN, nil
}
Ejemplo n.º 19
0
// Delete a packed message
func (q *Queue) delete(msg []*SDK.DeleteMessageBatchRequestEntry) error {
	if len(msg) < 1 {
		return nil
	}
	res, err := q.client.DeleteMessageBatch(&SDK.DeleteMessageBatchInput{
		Entries:  msg,
		QueueURL: q.url,
	})
	if err != nil {
		log.Error("[SQS] error on `DeleteMessageBatch`, queue="+q.name, err.Error())
		q.failedDelete = append(q.failedDelete, res.Failed...)
	}

	return err
}
Ejemplo n.º 20
0
// execute put operation for all tables in write spool
func (d *AmazonDynamoDB) PutAll() error {
	var errs []string
	for name, _ := range d.writeTables {
		err := d.tables[name].Put()
		if err != nil {
			errs = append(errs, err.Error())
			log.Error("[DynamoDB] Error on `Put` operation, table="+name, err.Error())
		}
		d.removeWriteTable(name)
	}
	if len(errs) != 0 {
		return errors.New(strings.Join(errs, "\n"))
	}
	return nil
}
Ejemplo n.º 21
0
// put object to server
func (b *Bucket) Put() error {
	var err error = nil
	errStr := ""
	// save file
	for _, obj := range b.objects {
		_, e := b.client.PutObject(obj)
		if e != nil {
			log.Error("[S3] error on `PutObject` operation, bucket="+b.name, e.Error())
			errStr = errStr + "," + e.Error()
		}
	}
	if errStr != "" {
		err = errors.New(errStr)
	}
	return err
}
Ejemplo n.º 22
0
// Get SNSApp struct
func (svc *AmazonSNS) GetApp(typ string) (*SNSApp, error) {
	// get the app from cache
	app, ok := svc.apps[typ]
	if ok {
		return app, nil
	}
	arn := config.GetConfigValue(snsConfigSectionName, "app."+typ, "")
	if arn == "" {
		errMsg := "[SNS] error, cannot find ARN setting"
		log.Error(errMsg, typ)
		return nil, errors.New(errMsg)
	}
	app = svc.NewApp(arn, typ)
	svc.apps[typ] = app
	return app, nil
}
Ejemplo n.º 23
0
// updateThroughput updates dynamodb table provisioned throughput
func (t *DynamoTable) updateThroughput(in *SDK.UpdateTableInput) error {
	if t.isSameThroughput(in) {
		return nil
	}
	_, err := t.db.client.UpdateTable(in)
	if err != nil {
		log.Error("[DynamoDB] Error in `UpdateTable` operation, table="+t.name, err)
		return err
	}
	desc, err := t.Desc()
	if err != nil {
		return err
	}
	t.table = desc
	return nil
}
Ejemplo n.º 24
0
// Count left messages on the Queue
func (q *Queue) CountMessage() (int, int, error) {
	out, err := q.client.GetQueueAttributes(&SDK.GetQueueAttributesInput{
		QueueURL: q.url,
		AttributeNames: []*string{
			String("ApproximateNumberOfMessages"),
			String("ApproximateNumberOfMessagesNotVisible"),
		},
	})
	if err != nil {
		log.Error("[SQS] error on `GetQueueAttributes`, queue="+q.name, err.Error())
		return 0, 0, err
	}
	m := out.Attributes
	visible, _ := strconv.Atoi(*m["ApproximateNumberOfMessages"])
	invisible, _ := strconv.Atoi(*m["ApproximateNumberOfMessagesNotVisible"])
	return visible, invisible, nil
}
Ejemplo n.º 25
0
// GetOne retrieves a single item by GetOne(HashKey [, RangeKey])
func (t *DynamoTable) GetOne(values ...Any) (map[string]interface{}, error) {
	key := NewItem()
	key.AddAttribute(t.GetHashKeyName(), values[0])
	if len(values) > 1 && t.GetRangeKeyName() != "" {
		key.AddAttribute(t.GetRangeKeyName(), values[1])
	}

	in := &SDK.GetItemInput{
		TableName: String(t.name),
		Key:       key.data,
	}
	req, err := t.db.client.GetItem(in)
	if err != nil {
		log.Error("[DynamoDB] Error in `GetItem` operation, table="+t.name, err)
		return nil, err
	}
	return Unmarshal(req.Item), nil
}
Ejemplo n.º 26
0
// Register endpoint(device) to application
func (svc *AmazonSNS) RegisterEndpoint(device, token string) (*SNSEndpoint, error) {
	var app *SNSApp
	var err error
	switch device {
	case "ios", "apns":
		app, err = svc.GetAppAPNS()
	case "android", "gcm":
		app, err = svc.GetAppGCM()
	default:
		errMsg := "[SNS] Unsupported device, device=" + device
		log.Error(errMsg, token)
		return nil, errors.New(errMsg)
	}
	if err != nil {
		return nil, err
	}
	return app.CreateEndpoint(token)
}
Ejemplo n.º 27
0
// delete item
func (t *DynamoTable) Delete(values ...Any) error {
	key := NewItem()
	key.AddAttribute(t.GetHashKeyName(), values[0])
	if len(values) > 1 && t.GetRangeKeyName() != "" {
		key.AddAttribute(t.GetRangeKeyName(), values[1])
	}

	in := &SDK.DeleteItemInput{
		TableName: String(t.name),
		Key:       key.data,
	}
	_, err := t.db.client.DeleteItem(in)
	if err != nil {
		log.Error("[DynamoDB] Error in `DeleteItem` operation, table="+t.name, err)
		return err
	}
	return nil
}
Ejemplo n.º 28
0
// Get only the body of messages
func (q *Queue) FetchBody(num int) []string {
	var messages []string
	resp, err := q.Fetch(num)
	if err != nil {
		log.Error("[SQS] error on `FetchBody`, queue="+q.name, err.Error())
		return messages
	}
	if len(resp.Messages) == 0 {
		return messages
	}
	for _, msg := range resp.Messages {
		messages = append(messages, *msg.Body)
	}
	q.AddDeleteList(resp.Messages)
	if q.autoDel {
		defer q.DeleteListItems()
	}
	return messages
}
Ejemplo n.º 29
0
// Publish notification for arn(topic or endpoint)
func (svc *AmazonSNS) Publish(arn string, msg string, opt map[string]interface{}) error {
	msg = truncateMessage(msg)
	m := make(map[string]string)
	m["default"] = msg
	m["GCM"] = composeMessageGCM(msg)
	m["APNS"] = composeMessageAPNS(msg, opt)
	m["APNS_SANDBOX"] = m["APNS"]
	jsonString, _ := json.Marshal(m)
	resp, err := svc.Client.Publish(&SDK.PublishInput{
		TargetARN:        String(arn),
		Message:          String(string(jsonString)),
		MessageStructure: String("json"),
	})
	if err != nil {
		log.Error("[SNS] error on `Publish` operation, arn="+arn, err.Error())
		return err
	}
	log.Info("[SNS] publish message", *resp.MessageID)
	return nil
}
Ejemplo n.º 30
0
// Get a queue
func (svc *AmazonSQS) GetQueue(queue string) (*Queue, error) {
	queueName := GetQueuePrefix() + queue

	// get the queue from cache
	q, ok := svc.queues[queueName]
	if ok {
		return q, nil
	}

	// get the queue from server
	url, err := svc.client.GetQueueURL(&SDK.GetQueueURLInput{
		QueueName:              String(queueName),
		QueueOwnerAWSAccountID: nil,
	})
	if err != nil {
		log.Error("[SQS] error on `GetQueueURL` operation, queue="+queueName, err.Error())
		return nil, err
	}
	q = NewQueue(queueName, url.QueueURL, svc.client)
	svc.queues[queueName] = q
	return q, nil
}