func (ds *LevelDBStorage) DeleteData(id ...string) error {
wb := new(leveldb.Batch)
for _, i := range id {
wb.Delete(enc.UnsafeStringToBytes(i))
}
return ds.db.Write(wb, nil)
}
// FlushCache flushes all cache into database.
func (ds *LevelDBStorage) FlushCache() {
ds.saveLock.Lock()
ds.cacheLock.Lock()
ds.tmpItemCache = ds.itemCache
ds.itemCache = make(ItemCache)
ds.cacheLock.Unlock()
wb := &leveldb.Batch{}
count := 0
for k, v := range ds.tmpItemCache {
if count >= 100 {
ds.db.Write(wb, nil)
wb.Reset()
count = 0
}
key := enc.UnsafeStringToBytes(k)
if v == nil {
wb.Delete(key)
} else {
wb.Put(key, v)
}
count++
}
ds.db.Write(wb, nil)
ds.saveLock.Unlock()
}
// EncodeAttrTo encodes messages attributes to the appropriate SQS AWS format.
func EncodeAttrTo(name string, data *sqsmsg.UserAttribute, b []byte) []byte {
nLen := len(name)
b = append(b, byte(nLen>>24), byte(nLen>>16), byte(nLen>>8), byte(nLen))
b = append(b, enc.UnsafeStringToBytes(name)...)
tLen := len(data.Type)
b = append(b, byte(tLen>>24), byte(tLen>>16), byte(tLen>>8), byte(tLen))
b = append(b, enc.UnsafeStringToBytes(data.Type)...)
if strings.HasPrefix(data.Type, "String") || strings.HasPrefix(data.Type, "Number") {
b = append(b, 1)
} else if strings.HasPrefix(data.Type, "Binary") {
b = append(b, 2)
}
valLen := len(data.Value)
b = append(b, byte(valLen>>24), byte(valLen>>16), byte(valLen>>8), byte(valLen))
b = append(b, enc.UnsafeStringToBytes(data.Value)...)
return b
}
func MakeMessageAttr(name string, sqsAttr *sqsmsg.UserAttribute) *MessageAttribute {
if strings.HasPrefix(sqsAttr.Type, "Binary") {
encodedBin := make([]byte, base64.StdEncoding.EncodedLen(len(sqsAttr.Value)))
base64.StdEncoding.Encode(encodedBin, enc.UnsafeStringToBytes(sqsAttr.Value))
return &MessageAttribute{
Name: name,
Type: sqsAttr.Type,
BinaryValue: enc.UnsafeBytesToString(encodedBin),
}
} else {
return &MessageAttribute{
Name: name,
Type: sqsAttr.Type,
StringValue: sqsAttr.Value,
}
}
}
// GetData looks data looks for and item going through each layer of cache finally looking into database.
func (ds *LevelDBStorage) GetData(id string) []byte {
ds.cacheLock.Lock()
data, ok := ds.itemCache[id]
if ok {
ds.cacheLock.Unlock()
return data
}
data, ok = ds.tmpItemCache[id]
if ok {
ds.cacheLock.Unlock()
return data
}
ds.cacheLock.Unlock()
value, e := ds.db.Get(enc.UnsafeStringToBytes(id), nil)
if e != nil {
return nil
}
return value
}
func MakeMessageResponse(iMsg apis.IResponseItem, opts *ReceiveMessageOptions,
sqsQuery *urlutils.SQSQuery) *MessageResponse {
msg, ok := iMsg.(*pqueue.MsgResponseItem)
if !ok {
return nil
}
sqsMsg := &sqsmsg.SQSMessagePayload{}
payload := msg.Payload()
if err := sqsMsg.Unmarshal([]byte(payload)); err != nil {
// Recovering from error. Non SQS messages will be filled with bulk info.
sqsMsg.Payload = string(msg.Payload())
sqsMsg.SenderId = "unknown"
sqsMsg.SentTimestamp = strconv.FormatInt(utils.Uts(), 10)
sqsMsg.MD5OfMessageAttributes = fmt.Sprintf("%x", md5.Sum(nil))
sqsMsg.MD5OfMessageBody = fmt.Sprintf("%x", md5.Sum(enc.UnsafeStringToBytes(sqsMsg.Payload)))
}
msgMeta := msg.GetMeta()
output := &MessageResponse{
MD5OfMessageAttributes: sqsMsg.MD5OfMessageAttributes,
MD5OfMessageBody: sqsMsg.MD5OfMessageBody,
ReceiptHandle: msg.Receipt(),
MessageId: msg.ID(),
Body: sqsMsg.Payload,
}
if opts.AllSysAttributes {
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrSenderId, Value: sqsMsg.SenderId,
})
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrSentTimestamp, Value: sqsMsg.SentTimestamp,
})
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrApproximateReceiveCount, Value: strconv.FormatInt(msgMeta.PopCount, 10),
})
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrApproximateFirstReceiveTimestamp, Value: strconv.FormatInt(utils.Uts(), 10),
})
} else {
for _, k := range opts.Attributes {
switch k {
case AttrSenderId:
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrSenderId, Value: sqsMsg.SenderId,
})
case AttrSentTimestamp:
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrSentTimestamp, Value: sqsMsg.SentTimestamp,
})
case AttrApproximateReceiveCount:
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrApproximateReceiveCount, Value: strconv.FormatInt(msgMeta.PopCount, 10),
})
case AttrApproximateFirstReceiveTimestamp:
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrApproximateFirstReceiveTimestamp, Value: strconv.FormatInt(utils.Uts(), 10),
})
}
}
}
if opts.AllMessageAttributes {
for attrName, attrData := range sqsMsg.UserAttributes {
output.MessageAttributes = append(
output.MessageAttributes, MakeMessageAttr(attrName, attrData))
}
} else {
for _, attrName := range opts.MessageAttributes {
if v, ok := sqsMsg.UserAttributes[attrName]; ok {
output.MessageAttributes = append(
output.MessageAttributes, MakeMessageAttr(attrName, v))
}
}
}
if !ok {
log.Error("Failed to cast response item")
return nil
}
return output
}
// StoreData data directly into the database stores service metadata into database.
func (ds *LevelDBStorage) StoreData(key string, data []byte) error {
return ds.db.Put(enc.UnsafeStringToBytes(key), data, nil)
}
// IterData returns an iterator over all data with prefix.
func (ds *LevelDBStorage) IterData(prefix string) apis.ItemIterator {
iter := ds.db.NewIterator(new(util.Range), nil)
return makeItemIterator(iter, enc.UnsafeStringToBytes(prefix))
}
func PushAMessage(pq *pqueue.PQueue, senderId string, paramList []string) sqs_response.SQSResponse {
out := &MessageParams{
DelaySeconds: -1,
MessageBody: "",
}
attrs, err := urlutils.ParseNNotationAttr("MessageAttribute.", paramList, out.Parse, NewReqQueueAttr)
if err != nil {
return err
}
attrsLen := len(attrs)
outAttrs := make(map[string]*sqsmsg.UserAttribute)
for i := 1; i <= attrsLen; i++ {
a, ok := attrs[i]
if !ok {
return sqserr.InvalidParameterValueError("The request must contain non-empty message attribute name.")
}
reqMsgAttr, _ := a.(*ReqMsgAttr)
sqs_err := validation.ValidateMessageAttrName(reqMsgAttr.Name)
if sqs_err != nil {
return sqs_err
}
sqs_err = validation.ValidateMessageAttrName(reqMsgAttr.DataType)
if sqs_err != nil {
return sqs_err
}
if reqMsgAttr.BinaryValue != "" && reqMsgAttr.StringValue != "" {
return sqserr.InvalidParameterValueError(
"Message attribute name '%s' has multiple values.", reqMsgAttr.Name)
}
if _, ok := outAttrs[reqMsgAttr.Name]; ok {
return sqserr.InvalidParameterValueError(
"Message attribute name '%s' already exists.", reqMsgAttr.Name)
}
if strings.HasPrefix(reqMsgAttr.DataType, "Number") {
if _, err := strconv.Atoi(reqMsgAttr.StringValue); err != nil {
return sqserr.InvalidParameterValueError(
"Could not cast message attribute '%s' value to number.", reqMsgAttr.Name)
}
}
if reqMsgAttr.BinaryValue != "" {
if reqMsgAttr.DataType != "Binary" {
return sqserr.InvalidParameterValueError(
"The message attribute '%s' with type 'Binary' must use field 'Binary'", reqMsgAttr.Name)
}
outAttrs[reqMsgAttr.Name] = &sqsmsg.UserAttribute{
Type: reqMsgAttr.DataType,
Value: reqMsgAttr.BinaryValue,
}
continue
}
if reqMsgAttr.StringValue != "" {
if reqMsgAttr.DataType != "String" && reqMsgAttr.DataType != "Number" {
return sqserr.InvalidParameterValueError(
"The message attribute '%s' with type 'String' must use field 'String'", reqMsgAttr.Name)
}
outAttrs[reqMsgAttr.Name] = &sqsmsg.UserAttribute{
Type: reqMsgAttr.DataType,
Value: reqMsgAttr.StringValue,
}
}
}
msgId := IdGen.RandId()
if out.DelaySeconds < 0 {
out.DelaySeconds = pq.Config().DeliveryDelay
} else if out.DelaySeconds > conf.CFG_PQ.MaxDeliveryDelay {
return sqserr.InvalidParameterValueError(
"Delay secods must be between 0 and %d", conf.CFG_PQ.MaxDeliveryDelay/1000)
}
bodyMd5str := fmt.Sprintf("%x", md5.Sum(enc.UnsafeStringToBytes(out.MessageBody)))
attrMd5 := CalcAttrMd5(outAttrs)
msgPayload := sqsmsg.SQSMessagePayload{
UserAttributes: outAttrs,
MD5OfMessageBody: bodyMd5str,
MD5OfMessageAttributes: attrMd5,
SenderId: senderId,
SentTimestamp: strconv.FormatInt(utils.Uts(), 10),
Payload: out.MessageBody,
}
d, marshalErr := msgPayload.Marshal()
if marshalErr != nil {
log.Error("Failed to serialize message payload: %v", err)
}
payload := enc.UnsafeBytesToString(d)
resp := pq.Push(msgId, payload, pq.Config().MsgTtl, out.DelaySeconds, 1)
if resp.IsError() {
e, _ := resp.(error)
return sqserr.InvalidParameterValueError(e.Error())
}
return &SendMessageResponse{
MessageId: msgId,
MD5OfMessageBody: bodyMd5str,
MD5OfMessageAttributes: attrMd5,
RequestId: "req",
}
}