// query using LSI or GSI
func (t *DynamoTable) GetByIndex(idx string, values ...Any) ([]map[string]interface{}, error) {
index, ok := t.indexes[idx]
if !ok {
log.Error("[DynamoDB] Cannot find the index name, table="+t.name, idx)
log.Error("[DynamoDB] indexes on table="+t.name, t.indexes)
}
hashKey := index.GetHashKeyName()
rangeKey := index.GetRangeKeyName()
keys := make(map[string]*SDK.Condition)
keys[hashKey] = &SDK.Condition{
AttributeValueList: []*SDK.AttributeValue{createAttributeValue(values[0])},
ComparisonOperator: String(ComparisonOperatorEQ),
}
if len(values) > 1 && rangeKey != "" {
keys[rangeKey] = &SDK.Condition{
AttributeValueList: []*SDK.AttributeValue{createAttributeValue(values[1])},
ComparisonOperator: String(ComparisonOperatorEQ),
}
}
in := &SDK.QueryInput{
TableName: String(t.name),
KeyConditions: keys,
IndexName: &idx,
}
return t.Query(in)
}
// Send messages in the send spool
func (q *Queue) Send() error {
// pack the messages ten each to meet the SQS restriction.
messages := make(map[int][]*SDK.SendMessageBatchRequestEntry)
if len(q.messages) > 10 {
for i, msg := range q.messages {
v := (i + 1) / 10
messages[v] = append(messages[v], msg)
}
} else {
messages[0] = append(messages[0], q.messages...)
}
err := &SQSError{}
// send message
for i := 0; i < len(messages); i++ {
e := q.send(messages[i])
if e != nil {
log.Error("[SQS] error on `SendMessageBatch` operation, queue="+q.name, e.Error())
err.AddMessage(e.Error())
}
}
if err.HasError() {
return err
}
return nil
}
// Get message from the queue with limit
func (q *Queue) Fetch(num int) ([]*Message, error) {
// use long-polling for 1sec when to get multiple messages
var wait int = 0
if num > 1 {
wait = 1
}
// receive message from server
resp, err := q.client.ReceiveMessage(&SDK.ReceiveMessageInput{
QueueUrl: q.url,
WaitTimeSeconds: Long(wait),
MaxNumberOfMessages: Long(num),
VisibilityTimeout: Long(defaultExpireSecond),
})
if err != nil {
log.Error("[SQS] error on `ReceiveMessage` operation, queue="+q.name, err.Error())
}
var list []*Message
if resp == nil || len(resp.Messages) == 0 {
return list, err
}
// delete messages automatically
if q.autoDel {
q.AddDeleteList(resp.Messages)
defer q.DeleteListItems()
}
for _, msg := range resp.Messages {
list = append(list, NewMessage(msg))
}
return list, err
}
// Send messages in the send spool
func (q *Queue) Send() error {
// pack the messages ten each to meet the SQS restriction.
messages := make(map[int][]*SDK.SendMessageBatchRequestEntry)
if len(q.messages) > 10 {
for i, msg := range q.messages {
v := (i + 1) / 10
messages[v] = append(messages[v], msg)
}
} else {
messages[0] = append(messages[0], q.messages...)
}
var err error = nil
errStr := ""
// send message
for i := 0; i < len(messages); i++ {
e := q.send(messages[i])
if e != nil {
log.Error("[SQS] error on `SendMessageBatch` operation, queue="+q.name, e.Error())
errStr = errStr + "," + e.Error()
}
}
if errStr != "" {
err = errors.New(errStr)
}
return err
}
// Get message from the queue with limit
func (q *Queue) Fetch(num int) (*SDK.ReceiveMessageOutput, error) {
// use long-polling for 1sec when to get multiple messages
var wait int
if num > 1 {
wait = 1
} else {
wait = 0
}
// receive message from server
resp, err := q.client.ReceiveMessage(&SDK.ReceiveMessageInput{
QueueURL: q.url,
WaitTimeSeconds: Long(wait),
MaxNumberOfMessages: Long(num),
VisibilityTimeout: Long(defaultExpireSecond),
})
if err != nil {
log.Error("[SQS] error on `ReceiveMessage` operation, queue="+q.name, err.Error())
}
// delete messages automatically
if q.autoDel && len(resp.Messages) > 0 {
q.AddDeleteList(resp.Messages)
defer q.DeleteListItems()
}
return resp, err
}
// make sns message for Apple Push Notification Service
func composeMessageAPNS(msg string, opt map[string]interface{}) string {
aps := make(map[string]interface{})
aps[apnsKeyMessage] = msg
aps[apnsKeySound] = "default"
if v, ok := opt[apnsKeySound]; ok {
aps[apnsKeySound] = v
}
if v, ok := opt[apnsKeyBadge]; ok {
aps[apnsKeyBadge] = v
}
message := make(map[string]interface{})
message["aps"] = aps
for k, v := range opt {
switch k {
case apnsKeySound:
continue
case apnsKeyBadge:
continue
default:
message[k] = v
}
}
payload, err := json.Marshal(message)
if err != nil {
log.Error("[SNS] error on json.Marshal", err.Error())
}
return string(payload)
}
func (q *Query) AndLT(key string, val interface{}) {
if _, ok := q.conditions[key]; ok {
log.Error("[DynamoDB] only contain one condition per key, key="+key, val)
}
cond := newQueryCondition(key, val)
cond.Condition = conditionLT
q.conditions[key] = cond
}
// get mapped-items with Query operation
func (t *DynamoTable) Query(in *SDK.QueryInput) ([]map[string]interface{}, error) {
req, err := t.db.client.Query(in)
if err != nil {
log.Error("[DynamoDB] Error in `Query` operation, table="+t.name, err)
return nil, err
}
return t.convertItemsToMapArray(req.Items), nil
}
func (q *Query) AndBETWEEN(key string, from, to interface{}) {
if _, ok := q.conditions[key]; ok {
log.Error("[DynamoDB] only contain one condition per key, key="+key, from)
}
cond := newQueryCondition(key, from)
cond.SubValue = to
cond.Condition = conditionBETWEEN
q.conditions[key] = cond
}
// Create new DynamoDB table
func (d *AmazonDynamoDB) CreateTable(in *SDK.CreateTableInput) error {
data, err := d.client.CreateTable(in)
if err != nil {
log.Error("[DynamoDB] Error on `CreateTable` operation, table="+*in.TableName, err)
return err
}
log.Info("[DynamoDB] Complete CreateTable, table="+*in.TableName, data.TableDescription.TableStatus)
return nil
}
// RegisterEndpointWithUserData creates endpoint(device) and CustomUserData to application
func (svc *AmazonSNS) RegisterEndpointWithUserData(device, token, userData string) (*SNSEndpoint, error) {
app, err := svc.getApp(device)
if err != nil {
log.Error(err.Error(), token)
return nil, err
}
app.SetUserData(userData)
return app.CreateEndpoint(token)
}
// RegisterEndpoint creates endpoint(device) to application
func (svc *AmazonSNS) RegisterEndpoint(device, token string) (*SNSEndpoint, error) {
app, err := svc.getApp(device)
if err != nil {
log.Error(err.Error(), token)
return nil, err
}
return app.CreateEndpoint(token)
}
// Create new SQS Queue
func (svc *AmazonSQS) CreateQueue(in *SDK.CreateQueueInput) error {
data, err := svc.client.CreateQueue(in)
if err != nil {
log.Error("[SQS] Error on `CreateQueue` operation, queue="+*in.QueueName, err)
return err
}
log.Info("[SQS] Complete CreateQueue, queue="+*in.QueueName, *(data.QueueUrl))
return nil
}
// Add message spool from map data
func (q *Queue) AddMessageMap(message map[string]interface{}) error {
msg, err := json.Marshal(message)
if err != nil {
log.Error("[SQS] error on `json.Marshal`, msg="+fmt.Sprint(msg), err.Error())
return err
}
q.AddMessage(string(msg))
return nil
}
// GetEndpoint
func (svc *AmazonSNS) GetEndpoint(arn string) (*SNSEndpoint, error) {
in := &SDK.GetEndpointAttributesInput{
EndpointArn: String(arn),
}
resp, err := svc.Client.GetEndpointAttributes(in)
if err != nil {
log.Error("[SNS] error on `GetEndpointAttributes` operation, arn="+arn, err.Error())
return nil, err
}
attr := resp.Attributes
ep := svc.NewApplicationEndpoint(arn)
ep.token = *attr["Token"]
ep.enable, err = strconv.ParseBool(*attr["Enabled"])
if err != nil {
log.Error("[SNS] error on `Endpoint Attributes` Enabled="+*attr["Enabled"], err.Error())
ep.enable = false
}
return ep, err
}
func (t *DynamoTable) Desc() (*SDK.TableDescription, error) {
req, err := t.db.client.DescribeTable(&SDK.DescribeTableInput{
TableName: String(t.name),
})
if err != nil {
log.Error("[DynamoDB] Error in `DescribeTable` operation, table="+t.name, err)
return nil, err
}
return req.Table, nil
}
// delete object of target path
func (b *Bucket) DeleteObject(path string) error {
_, err := b.client.DeleteObject(&SDK.DeleteObjectInput{
Bucket: String(b.name),
Key: String(path),
})
if err != nil {
log.Error("[S3] error on `DeleteObject` operation, bucket="+b.name, err.Error())
}
return err
}
// Delete all messages on the Queue
func (q *Queue) Purge() error {
_, err := q.client.PurgeQueue(&SDK.PurgeQueueInput{
QueueURL: q.url,
})
if err != nil {
log.Error("[SQS] error on `PurgeQueue`, queue="+q.name, err.Error())
return err
}
return nil
}
// Delete a message from server
func (q *Queue) DeleteMessage(msg *SDK.Message) error {
_, err := q.client.DeleteMessage(&SDK.DeleteMessageInput{
QueueURL: q.url,
ReceiptHandle: msg.ReceiptHandle,
})
if err != nil {
log.Error("[SQS] error on `DeleteMessage`, queue="+q.name, err.Error())
}
return err
}
// Create SNS Topic and return `TopicARN`
func (svc *AmazonSNS) createTopic(name string) (string, error) {
prefix := config.GetConfigValue(snsConfigSectionName, "prefix", defaultPrefix)
in := &SDK.CreateTopicInput{
Name: String(prefix + name),
}
resp, err := svc.Client.CreateTopic(in)
if err != nil {
log.Error("[SNS] error on `CreateTopic` operation, name="+name, err.Error())
return "", err
}
return *resp.TopicArn, nil
}
// Delete DynamoDB table
func (d *AmazonDynamoDB) DeleteTable(name string) error {
in := &SDK.DeleteTableInput{
TableName: String(name),
}
data, err := d.client.DeleteTable(in)
if err != nil {
log.Error("[DynamoDB] Error on `DeleteTable` operation, table="+*in.TableName, err)
return err
}
log.Info("[DynamoDB] Complete DeleteTable, table="+*in.TableName, data.TableDescription.TableStatus)
return nil
}
// fetch object from target S3 path
func (b *Bucket) getObject(path string) (io.Reader, error) {
req := SDK.GetObjectInput{
Bucket: &b.name,
Key: &path,
}
out, err := b.client.GetObject(&req)
if err != nil {
log.Error("[S3] error on `GetObject` operation, bucket="+b.name, err.Error())
return nil, err
}
return out.Body, err
}
// Subscribe
func (t *SNSTopic) Subscribe(endpoint *SNSEndpoint) (string, error) {
resp, err := t.svc.Client.Subscribe(&SDK.SubscribeInput{
Endpoint: String(endpoint.arn),
Protocol: String(endpoint.protocol),
TopicARN: String(t.arn),
})
if err != nil {
log.Error("[SNS] error on `Subscribe` operation, topic="+t.arn, err.Error())
return "", err
}
return *resp.SubscriptionARN, nil
}
// get mapped-items with Scan operation
func (t *DynamoTable) Scan() ([]map[string]interface{}, error) {
in := &SDK.ScanInput{
TableName: String(t.name),
Limit: Long(1000),
}
req, err := t.db.client.Scan(in)
if err != nil {
log.Error("[DynamoDB] Error in `Scan` operation, table="+t.name, err)
return nil, err
}
return t.convertItemsToMapArray(req.Items), nil
}
// Create Endpoint(add device) and return `EndpointARN`
func (a *SNSApp) createEndpoint(token string) (string, error) {
in := &SDK.CreatePlatformEndpointInput{
PlatformApplicationARN: String(a.arn),
Token: String(token),
}
resp, err := a.svc.Client.CreatePlatformEndpoint(in)
if err != nil {
log.Error("[SNS] error on `CreatePlatformEndpoint` operation, token="+token, err.Error())
return "", err
}
return *resp.EndpointARN, nil
}
func (ct *CreateTableInput) addGSI(name string, key ...string) {
var schema []*SDK.KeySchemaElement
switch len(key) {
case 1:
schema = NewKeySchema(NewHashKeyElement(key[0]))
case 2:
schema = NewKeySchema(NewHashKeyElement(key[0]), NewRangeKeyElement(key[1]))
default:
log.Error("[DynamoDB] keys must have 1 or 2, name="+name, len(key))
return
}
tp := NewProvisionedThroughput(ct.ReadCapacity, ct.WriteCapacity)
gsi := NewGSI(name, schema, tp)
ct.GSI = append(ct.GSI, gsi)
}
// execute put operation for all tables in write spool
func (d *AmazonDynamoDB) PutAll() error {
var errs []string
for name, _ := range d.writeTables {
err := d.tables[name].Put()
if err != nil {
errs = append(errs, err.Error())
log.Error("[DynamoDB] Error on `Put` operation, table="+name, err.Error())
}
d.removeWriteTable(name)
}
if len(errs) != 0 {
return errors.New(strings.Join(errs, "\n"))
}
return nil
}
// Delete a packed message
func (q *Queue) delete(msg []*SDK.DeleteMessageBatchRequestEntry) error {
if len(msg) < 1 {
return nil
}
res, err := q.client.DeleteMessageBatch(&SDK.DeleteMessageBatchInput{
Entries: msg,
QueueURL: q.url,
})
if err != nil {
log.Error("[SQS] error on `DeleteMessageBatch`, queue="+q.name, err.Error())
q.failedDelete = append(q.failedDelete, res.Failed...)
}
return err
}
// updateThroughput updates dynamodb table provisioned throughput
func (t *DynamoTable) updateThroughput(in *SDK.UpdateTableInput) error {
if t.isSameThroughput(in) {
return nil
}
_, err := t.db.client.UpdateTable(in)
if err != nil {
log.Error("[DynamoDB] Error in `UpdateTable` operation, table="+t.name, err)
return err
}
desc, err := t.Desc()
if err != nil {
return err
}
t.table = desc
return nil
}
func (q *Query) query(in *SDK.QueryInput) (*QueryResult, error) {
if !q.HasCondition() {
errData := &DynamoError{}
errData.AddMessage("condition is missing, you must specify at least one condition")
return nil, errData
}
in.KeyConditionExpression = q.formatConditions()
in.ExpressionAttributeValues = q.formatConditionValues()
in.ExpressionAttributeNames = q.formatConditionNames()
if q.HasIndex() {
in.IndexName = String(q.index)
}
if q.HasLimit() {
in.Limit = Long(q.limit)
}
req, err := q.table.db.client.Query(in)
if err != nil {
log.Error("[DynamoDB] Error in `Query` operation, table="+q.table.name, err)
return nil, err
}
res := &QueryResult{
Items: req.Items,
LastEvaluatedKey: req.LastEvaluatedKey,
Count: *req.Count,
ScannedCount: *req.ScannedCount,
}
return res, nil
}