func (ds *LevelDBStorage) DeleteData(id ...string) error { wb := new(leveldb.Batch) for _, i := range id { wb.Delete(enc.UnsafeStringToBytes(i)) } return ds.db.Write(wb, nil) }
// FlushCache flushes all cache into database. func (ds *LevelDBStorage) FlushCache() { ds.saveLock.Lock() ds.cacheLock.Lock() ds.tmpItemCache = ds.itemCache ds.itemCache = make(ItemCache) ds.cacheLock.Unlock() wb := &leveldb.Batch{} count := 0 for k, v := range ds.tmpItemCache { if count >= 100 { ds.db.Write(wb, nil) wb.Reset() count = 0 } key := enc.UnsafeStringToBytes(k) if v == nil { wb.Delete(key) } else { wb.Put(key, v) } count++ } ds.db.Write(wb, nil) ds.saveLock.Unlock() }
// EncodeAttrTo encodes messages attributes to the appropriate SQS AWS format. func EncodeAttrTo(name string, data *sqsmsg.UserAttribute, b []byte) []byte { nLen := len(name) b = append(b, byte(nLen>>24), byte(nLen>>16), byte(nLen>>8), byte(nLen)) b = append(b, enc.UnsafeStringToBytes(name)...) tLen := len(data.Type) b = append(b, byte(tLen>>24), byte(tLen>>16), byte(tLen>>8), byte(tLen)) b = append(b, enc.UnsafeStringToBytes(data.Type)...) if strings.HasPrefix(data.Type, "String") || strings.HasPrefix(data.Type, "Number") { b = append(b, 1) } else if strings.HasPrefix(data.Type, "Binary") { b = append(b, 2) } valLen := len(data.Value) b = append(b, byte(valLen>>24), byte(valLen>>16), byte(valLen>>8), byte(valLen)) b = append(b, enc.UnsafeStringToBytes(data.Value)...) return b }
func MakeMessageAttr(name string, sqsAttr *sqsmsg.UserAttribute) *MessageAttribute { if strings.HasPrefix(sqsAttr.Type, "Binary") { encodedBin := make([]byte, base64.StdEncoding.EncodedLen(len(sqsAttr.Value))) base64.StdEncoding.Encode(encodedBin, enc.UnsafeStringToBytes(sqsAttr.Value)) return &MessageAttribute{ Name: name, Type: sqsAttr.Type, BinaryValue: enc.UnsafeBytesToString(encodedBin), } } else { return &MessageAttribute{ Name: name, Type: sqsAttr.Type, StringValue: sqsAttr.Value, } } }
// GetData looks data looks for and item going through each layer of cache finally looking into database. func (ds *LevelDBStorage) GetData(id string) []byte { ds.cacheLock.Lock() data, ok := ds.itemCache[id] if ok { ds.cacheLock.Unlock() return data } data, ok = ds.tmpItemCache[id] if ok { ds.cacheLock.Unlock() return data } ds.cacheLock.Unlock() value, e := ds.db.Get(enc.UnsafeStringToBytes(id), nil) if e != nil { return nil } return value }
func MakeMessageResponse(iMsg apis.IResponseItem, opts *ReceiveMessageOptions, sqsQuery *urlutils.SQSQuery) *MessageResponse { msg, ok := iMsg.(*pqueue.MsgResponseItem) if !ok { return nil } sqsMsg := &sqsmsg.SQSMessagePayload{} payload := msg.Payload() if err := sqsMsg.Unmarshal([]byte(payload)); err != nil { // Recovering from error. Non SQS messages will be filled with bulk info. sqsMsg.Payload = string(msg.Payload()) sqsMsg.SenderId = "unknown" sqsMsg.SentTimestamp = strconv.FormatInt(utils.Uts(), 10) sqsMsg.MD5OfMessageAttributes = fmt.Sprintf("%x", md5.Sum(nil)) sqsMsg.MD5OfMessageBody = fmt.Sprintf("%x", md5.Sum(enc.UnsafeStringToBytes(sqsMsg.Payload))) } msgMeta := msg.GetMeta() output := &MessageResponse{ MD5OfMessageAttributes: sqsMsg.MD5OfMessageAttributes, MD5OfMessageBody: sqsMsg.MD5OfMessageBody, ReceiptHandle: msg.Receipt(), MessageId: msg.ID(), Body: sqsMsg.Payload, } if opts.AllSysAttributes { output.Attributes = append(output.Attributes, &SysAttribute{ Name: AttrSenderId, Value: sqsMsg.SenderId, }) output.Attributes = append(output.Attributes, &SysAttribute{ Name: AttrSentTimestamp, Value: sqsMsg.SentTimestamp, }) output.Attributes = append(output.Attributes, &SysAttribute{ Name: AttrApproximateReceiveCount, Value: strconv.FormatInt(msgMeta.PopCount, 10), }) output.Attributes = append(output.Attributes, &SysAttribute{ Name: AttrApproximateFirstReceiveTimestamp, Value: strconv.FormatInt(utils.Uts(), 10), }) } else { for _, k := range opts.Attributes { switch k { case AttrSenderId: output.Attributes = append(output.Attributes, &SysAttribute{ Name: AttrSenderId, Value: sqsMsg.SenderId, }) case AttrSentTimestamp: output.Attributes = append(output.Attributes, &SysAttribute{ Name: AttrSentTimestamp, Value: sqsMsg.SentTimestamp, }) case AttrApproximateReceiveCount: output.Attributes = append(output.Attributes, &SysAttribute{ Name: AttrApproximateReceiveCount, Value: strconv.FormatInt(msgMeta.PopCount, 10), }) case AttrApproximateFirstReceiveTimestamp: output.Attributes = append(output.Attributes, &SysAttribute{ Name: AttrApproximateFirstReceiveTimestamp, Value: strconv.FormatInt(utils.Uts(), 10), }) } } } if opts.AllMessageAttributes { for attrName, attrData := range sqsMsg.UserAttributes { output.MessageAttributes = append( output.MessageAttributes, MakeMessageAttr(attrName, attrData)) } } else { for _, attrName := range opts.MessageAttributes { if v, ok := sqsMsg.UserAttributes[attrName]; ok { output.MessageAttributes = append( output.MessageAttributes, MakeMessageAttr(attrName, v)) } } } if !ok { log.Error("Failed to cast response item") return nil } return output }
// StoreData data directly into the database stores service metadata into database. func (ds *LevelDBStorage) StoreData(key string, data []byte) error { return ds.db.Put(enc.UnsafeStringToBytes(key), data, nil) }
// IterData returns an iterator over all data with prefix. func (ds *LevelDBStorage) IterData(prefix string) apis.ItemIterator { iter := ds.db.NewIterator(new(util.Range), nil) return makeItemIterator(iter, enc.UnsafeStringToBytes(prefix)) }
func PushAMessage(pq *pqueue.PQueue, senderId string, paramList []string) sqs_response.SQSResponse { out := &MessageParams{ DelaySeconds: -1, MessageBody: "", } attrs, err := urlutils.ParseNNotationAttr("MessageAttribute.", paramList, out.Parse, NewReqQueueAttr) if err != nil { return err } attrsLen := len(attrs) outAttrs := make(map[string]*sqsmsg.UserAttribute) for i := 1; i <= attrsLen; i++ { a, ok := attrs[i] if !ok { return sqserr.InvalidParameterValueError("The request must contain non-empty message attribute name.") } reqMsgAttr, _ := a.(*ReqMsgAttr) sqs_err := validation.ValidateMessageAttrName(reqMsgAttr.Name) if sqs_err != nil { return sqs_err } sqs_err = validation.ValidateMessageAttrName(reqMsgAttr.DataType) if sqs_err != nil { return sqs_err } if reqMsgAttr.BinaryValue != "" && reqMsgAttr.StringValue != "" { return sqserr.InvalidParameterValueError( "Message attribute name '%s' has multiple values.", reqMsgAttr.Name) } if _, ok := outAttrs[reqMsgAttr.Name]; ok { return sqserr.InvalidParameterValueError( "Message attribute name '%s' already exists.", reqMsgAttr.Name) } if strings.HasPrefix(reqMsgAttr.DataType, "Number") { if _, err := strconv.Atoi(reqMsgAttr.StringValue); err != nil { return sqserr.InvalidParameterValueError( "Could not cast message attribute '%s' value to number.", reqMsgAttr.Name) } } if reqMsgAttr.BinaryValue != "" { if reqMsgAttr.DataType != "Binary" { return sqserr.InvalidParameterValueError( "The message attribute '%s' with type 'Binary' must use field 'Binary'", reqMsgAttr.Name) } outAttrs[reqMsgAttr.Name] = &sqsmsg.UserAttribute{ Type: reqMsgAttr.DataType, Value: reqMsgAttr.BinaryValue, } continue } if reqMsgAttr.StringValue != "" { if reqMsgAttr.DataType != "String" && reqMsgAttr.DataType != "Number" { return sqserr.InvalidParameterValueError( "The message attribute '%s' with type 'String' must use field 'String'", reqMsgAttr.Name) } outAttrs[reqMsgAttr.Name] = &sqsmsg.UserAttribute{ Type: reqMsgAttr.DataType, Value: reqMsgAttr.StringValue, } } } msgId := IdGen.RandId() if out.DelaySeconds < 0 { out.DelaySeconds = pq.Config().DeliveryDelay } else if out.DelaySeconds > conf.CFG_PQ.MaxDeliveryDelay { return sqserr.InvalidParameterValueError( "Delay secods must be between 0 and %d", conf.CFG_PQ.MaxDeliveryDelay/1000) } bodyMd5str := fmt.Sprintf("%x", md5.Sum(enc.UnsafeStringToBytes(out.MessageBody))) attrMd5 := CalcAttrMd5(outAttrs) msgPayload := sqsmsg.SQSMessagePayload{ UserAttributes: outAttrs, MD5OfMessageBody: bodyMd5str, MD5OfMessageAttributes: attrMd5, SenderId: senderId, SentTimestamp: strconv.FormatInt(utils.Uts(), 10), Payload: out.MessageBody, } d, marshalErr := msgPayload.Marshal() if marshalErr != nil { log.Error("Failed to serialize message payload: %v", err) } payload := enc.UnsafeBytesToString(d) resp := pq.Push(msgId, payload, pq.Config().MsgTtl, out.DelaySeconds, 1) if resp.IsError() { e, _ := resp.(error) return sqserr.InvalidParameterValueError(e.Error()) } return &SendMessageResponse{ MessageId: msgId, MD5OfMessageBody: bodyMd5str, MD5OfMessageAttributes: attrMd5, RequestId: "req", } }