func TestGetMessageInfo(t *testing.T) {
Convey("Pushed message should return parameters", t, func() {
q := CreateNewTestQueue()
defer q.Close()
q.Push("d1", "p", 10000, 1000, 9)
q.Push("d2", "p", 10000, 0, 11)
So(q.GetMessageInfo("d3"), ShouldResemble, mpqerr.ERR_MSG_NOT_FOUND)
m1, _ := q.GetMessageInfo("d1").(*resp.DictResponse)
m2, _ := q.GetMessageInfo("d2").(*resp.DictResponse)
msgInfo1 := m1.GetDict()
msgInfo2 := m2.GetDict()
So(msgInfo1[MSG_INFO_ID], ShouldEqual, "d1")
So(msgInfo1[MSG_INFO_LOCKED], ShouldEqual, true)
So(msgInfo1[MSG_INFO_UNLOCK_TS], ShouldBeGreaterThan, utils.Uts())
So(msgInfo1[MSG_INFO_POP_COUNT], ShouldEqual, 0)
So(msgInfo1[MSG_INFO_PRIORITY], ShouldEqual, 9)
So(msgInfo1[MSG_INFO_EXPIRE_TS], ShouldBeGreaterThan, utils.Uts())
So(msgInfo2[MSG_INFO_ID], ShouldEqual, "d2")
So(msgInfo2[MSG_INFO_LOCKED], ShouldEqual, false)
So(msgInfo2[MSG_INFO_UNLOCK_TS], ShouldBeLessThanOrEqualTo, utils.Uts())
So(msgInfo2[MSG_INFO_POP_COUNT], ShouldEqual, 0)
So(msgInfo2[MSG_INFO_PRIORITY], ShouldEqual, 11)
So(msgInfo2[MSG_INFO_EXPIRE_TS], ShouldBeGreaterThan, utils.Uts())
})
}
func TestMessagesMovedToAnotherQueue(t *testing.T) {
Convey("Elements should move from one queue to the other when number of pop attempts exceeded", t, func() {
log.InitLogging()
log.SetLevel(1)
db.SetDatabase(NewInMemDBService())
fsl := NewFakeSvcLoader()
q1 := CreateTestQueueWithName(fsl, "q1")
failQueue := CreateTestQueueWithName(fsl, "fq")
p := &PQueueParams{
MsgTTL: int64Ptr(10000),
MaxMsgSize: int64Ptr(256000),
MaxMsgsInQueue: int64Ptr(100000),
DeliveryDelay: int64Ptr(0),
PopCountLimit: int64Ptr(2),
PopLockTimeout: int64Ptr(1000),
FailQueue: "fq",
}
q1.SetParams(p)
q1.StartUpdate()
failQueue.StartUpdate()
defer q1.Close()
defer failQueue.Close()
Convey("Two elements should be moved to another queue", func() {
VerifyOkResponse(q1.Push("d1", "p", 10000, 0, 11))
VerifyOkResponse(q1.Push("d2", "p", 10000, 0, 11))
VerifyServiceSize(q1, 2)
VerifyItemsRespSize(q1.Pop(100, 0, 10, true), 2)
q1.checkTimeouts(utils.Uts() + 10000)
VerifyServiceSize(q1, 2)
VerifyItemsRespSize(q1.Pop(100, 0, 10, true), 2)
q1.checkTimeouts(utils.Uts() + 10000)
VerifyServiceSize(q1, 0)
// Need to wait while message transferring is happening.
for i := 0; i < 10000; i++ {
time.Sleep(time.Microsecond * 1)
if failQueue.Info().Size == 2 {
break
}
}
VerifyServiceSize(failQueue, 2)
})
})
}
func TestStatus(t *testing.T) {
Convey("Queue status should be correct", t, func() {
q := CreateNewTestQueue()
defer q.Close()
Convey("Empty status should be default", func() {
s, _ := q.GetCurrentStatus().(*resp.DictResponse)
status := s.GetDict()
So(status[PQ_STATUS_MAX_QUEUE_SIZE], ShouldEqual, 100001)
So(status[PQ_STATUS_MSG_TTL], ShouldEqual, 100000)
So(status[PQ_STATUS_DELIVERY_DELAY], ShouldEqual, 1)
So(status[PQ_STATUS_POP_LOCK_TIMEOUT], ShouldEqual, 10000)
So(status[PQ_STATUS_POP_COUNT_LIMIT], ShouldEqual, 4)
So(status[PQ_STATUS_CREATE_TS], ShouldEqual, 123)
So(status[PQ_STATUS_LAST_PUSH_TS], ShouldEqual, 12)
So(status[PQ_STATUS_LAST_POP_TS], ShouldEqual, 13)
So(status[PQ_STATUS_TOTAL_MSGS], ShouldEqual, 0)
So(status[PQ_STATUS_IN_FLIGHT_MSG], ShouldEqual, 0)
So(status[PQ_STATUS_AVAILABLE_MSGS], ShouldEqual, 0)
So(q.Info().ID, ShouldEqual, "1")
So(q.Info().Type, ShouldEqual, apis.ServiceTypePriorityQueue)
})
Convey("Status for several messages in flight", func() {
q.Push("d1", "p", 10000, 0, 9)
q.Push("d2", "p", 10000, 0, 9)
q.Push("d3", "p", 10000, 0, 9)
VerifySingleItem(q.Pop(100000, 0, 1, true), "d1", "p")
VerifyServiceSize(q, 3)
s, _ := q.GetCurrentStatus().(*resp.DictResponse)
status := s.GetDict()
So(status[PQ_STATUS_MAX_QUEUE_SIZE], ShouldEqual, 100001)
So(status[PQ_STATUS_MSG_TTL], ShouldEqual, 100000)
So(status[PQ_STATUS_DELIVERY_DELAY], ShouldEqual, 1)
So(status[PQ_STATUS_POP_LOCK_TIMEOUT], ShouldEqual, 10000)
So(status[PQ_STATUS_POP_COUNT_LIMIT], ShouldEqual, 4)
So(status[PQ_STATUS_CREATE_TS], ShouldBeLessThanOrEqualTo, utils.Uts())
So(status[PQ_STATUS_LAST_PUSH_TS], ShouldBeLessThanOrEqualTo, utils.Uts())
So(status[PQ_STATUS_LAST_POP_TS], ShouldBeLessThanOrEqualTo, utils.Uts())
So(status[PQ_STATUS_TOTAL_MSGS], ShouldEqual, 3)
So(status[PQ_STATUS_IN_FLIGHT_MSG], ShouldEqual, 1)
So(status[PQ_STATUS_AVAILABLE_MSGS], ShouldEqual, 2)
})
})
}
func TestReleaseInFlight(t *testing.T) {
Convey("One item should expire", t, func() {
q := CreateNewTestQueue()
defer q.Close()
q.Push("d1", "p", 10000, 100, 11)
r, _ := q.ReleaseInFlight(utils.Uts() + 1000).(*resp.IntResponse)
So(r.Value, ShouldEqual, 1)
VerifySingleItem(q.Pop(0, 0, 1, false), "d1", "p")
})
}
func TestExpiration(t *testing.T) {
Convey("One item should expire", t, func() {
q := CreateNewTestQueue()
defer q.Close()
q.Push("d1", "p", 10000, 0, 11)
r, _ := q.TimeoutItems(utils.Uts() + 100000).(*resp.IntResponse)
So(r.Value, ShouldEqual, 1)
VerifyServiceSize(q, 0)
})
}
func NewServiceDescription(name, sType string, exportId uint64) *ServiceDescription {
return &ServiceDescription{
ExportId: exportId,
SType: sType,
Name: name,
CreateTs: utils.Uts(),
Disabled: false,
ToDelete: false,
ServiceId: enc.EncodeTo36Base(exportId),
}
}
func DefaultPQConfig() *conf.PQConfig {
cfg := &conf.Config{}
flags.ParseArgs(cfg, []string{"firempq"})
conf.CFG = cfg
conf.CFG_PQ = &cfg.PQueueConfig
return &conf.PQConfig{
MaxMsgsInQueue: conf.CFG_PQ.DefaultMaxQueueSize,
MsgTtl: conf.CFG_PQ.DefaultMessageTTL,
DeliveryDelay: conf.CFG_PQ.DefaultDeliveryDelay,
PopLockTimeout: conf.CFG_PQ.DefaultLockTimeout,
LastPushTs: utils.Uts(),
LastPopTs: utils.Uts(),
PopLimitQueueName: "",
MaxMsgSize: conf.CFG_PQ.MaxMessageSize,
PopWaitTimeout: conf.CFG_PQ.DefaultPopWaitTimeout,
LastUpdateTs: utils.Uts(),
}
}
func TestAutoExpiration(t *testing.T) {
q := CreateNewTestQueue()
defer q.Close()
Convey("Two messages should expire, one message should still be in the queue", t, func() {
q.Push("data1", "p1", 1000, 0, 12)
q.Push("data2", "p2", 1000, 0, 12)
q.Push("data3", "p3", 10000, 0, 12)
VerifyServiceSize(q, 3)
q.checkTimeouts(utils.Uts() + 1300)
VerifyServiceSize(q, 1)
})
}
func TestPushPopAndTimeUnlockItems(t *testing.T) {
q := CreateNewTestQueue()
defer q.Close()
Convey("Test push and pop messages", t, func() {
q.Push("data1", "p1", 10000, 0, 12)
q.Push("data2", "p2", 10000, 0, 12)
VerifyServiceSize(q, 2)
VerifyItems(q.Pop(10000, 0, 10, true), 2, "data1", "p1", "data2", "p2")
VerifyServiceSize(q, 2)
// Unlock item data1 it should become available again.
q.UpdateLockById("data1", 0)
q.checkTimeouts(utils.Uts() + 110)
VerifySingleItem(q.Pop(10000, 0, 1, true), "data1", "p1")
VerifyServiceSize(q, 2)
VerifyOkResponse(q.DeleteLockedById("data1"))
VerifyServiceSize(q, 1)
q.DeleteLockedById("data2")
VerifyServiceSize(q, 0)
})
}
func MakeMessageResponse(iMsg apis.IResponseItem, opts *ReceiveMessageOptions,
sqsQuery *urlutils.SQSQuery) *MessageResponse {
msg, ok := iMsg.(*pqueue.MsgResponseItem)
if !ok {
return nil
}
sqsMsg := &sqsmsg.SQSMessagePayload{}
payload := msg.Payload()
if err := sqsMsg.Unmarshal([]byte(payload)); err != nil {
// Recovering from error. Non SQS messages will be filled with bulk info.
sqsMsg.Payload = string(msg.Payload())
sqsMsg.SenderId = "unknown"
sqsMsg.SentTimestamp = strconv.FormatInt(utils.Uts(), 10)
sqsMsg.MD5OfMessageAttributes = fmt.Sprintf("%x", md5.Sum(nil))
sqsMsg.MD5OfMessageBody = fmt.Sprintf("%x", md5.Sum(enc.UnsafeStringToBytes(sqsMsg.Payload)))
}
msgMeta := msg.GetMeta()
output := &MessageResponse{
MD5OfMessageAttributes: sqsMsg.MD5OfMessageAttributes,
MD5OfMessageBody: sqsMsg.MD5OfMessageBody,
ReceiptHandle: msg.Receipt(),
MessageId: msg.ID(),
Body: sqsMsg.Payload,
}
if opts.AllSysAttributes {
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrSenderId, Value: sqsMsg.SenderId,
})
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrSentTimestamp, Value: sqsMsg.SentTimestamp,
})
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrApproximateReceiveCount, Value: strconv.FormatInt(msgMeta.PopCount, 10),
})
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrApproximateFirstReceiveTimestamp, Value: strconv.FormatInt(utils.Uts(), 10),
})
} else {
for _, k := range opts.Attributes {
switch k {
case AttrSenderId:
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrSenderId, Value: sqsMsg.SenderId,
})
case AttrSentTimestamp:
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrSentTimestamp, Value: sqsMsg.SentTimestamp,
})
case AttrApproximateReceiveCount:
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrApproximateReceiveCount, Value: strconv.FormatInt(msgMeta.PopCount, 10),
})
case AttrApproximateFirstReceiveTimestamp:
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrApproximateFirstReceiveTimestamp, Value: strconv.FormatInt(utils.Uts(), 10),
})
}
}
}
if opts.AllMessageAttributes {
for attrName, attrData := range sqsMsg.UserAttributes {
output.MessageAttributes = append(
output.MessageAttributes, MakeMessageAttr(attrName, attrData))
}
} else {
for _, attrName := range opts.MessageAttributes {
if v, ok := sqsMsg.UserAttributes[attrName]; ok {
output.MessageAttributes = append(
output.MessageAttributes, MakeMessageAttr(attrName, v))
}
}
}
if !ok {
log.Error("Failed to cast response item")
return nil
}
return output
}
func PushAMessage(pq *pqueue.PQueue, senderId string, paramList []string) sqs_response.SQSResponse {
out := &MessageParams{
DelaySeconds: -1,
MessageBody: "",
}
attrs, err := urlutils.ParseNNotationAttr("MessageAttribute.", paramList, out.Parse, NewReqQueueAttr)
if err != nil {
return err
}
attrsLen := len(attrs)
outAttrs := make(map[string]*sqsmsg.UserAttribute)
for i := 1; i <= attrsLen; i++ {
a, ok := attrs[i]
if !ok {
return sqserr.InvalidParameterValueError("The request must contain non-empty message attribute name.")
}
reqMsgAttr, _ := a.(*ReqMsgAttr)
sqs_err := validation.ValidateMessageAttrName(reqMsgAttr.Name)
if sqs_err != nil {
return sqs_err
}
sqs_err = validation.ValidateMessageAttrName(reqMsgAttr.DataType)
if sqs_err != nil {
return sqs_err
}
if reqMsgAttr.BinaryValue != "" && reqMsgAttr.StringValue != "" {
return sqserr.InvalidParameterValueError(
"Message attribute name '%s' has multiple values.", reqMsgAttr.Name)
}
if _, ok := outAttrs[reqMsgAttr.Name]; ok {
return sqserr.InvalidParameterValueError(
"Message attribute name '%s' already exists.", reqMsgAttr.Name)
}
if strings.HasPrefix(reqMsgAttr.DataType, "Number") {
if _, err := strconv.Atoi(reqMsgAttr.StringValue); err != nil {
return sqserr.InvalidParameterValueError(
"Could not cast message attribute '%s' value to number.", reqMsgAttr.Name)
}
}
if reqMsgAttr.BinaryValue != "" {
if reqMsgAttr.DataType != "Binary" {
return sqserr.InvalidParameterValueError(
"The message attribute '%s' with type 'Binary' must use field 'Binary'", reqMsgAttr.Name)
}
outAttrs[reqMsgAttr.Name] = &sqsmsg.UserAttribute{
Type: reqMsgAttr.DataType,
Value: reqMsgAttr.BinaryValue,
}
continue
}
if reqMsgAttr.StringValue != "" {
if reqMsgAttr.DataType != "String" && reqMsgAttr.DataType != "Number" {
return sqserr.InvalidParameterValueError(
"The message attribute '%s' with type 'String' must use field 'String'", reqMsgAttr.Name)
}
outAttrs[reqMsgAttr.Name] = &sqsmsg.UserAttribute{
Type: reqMsgAttr.DataType,
Value: reqMsgAttr.StringValue,
}
}
}
msgId := IdGen.RandId()
if out.DelaySeconds < 0 {
out.DelaySeconds = pq.Config().DeliveryDelay
} else if out.DelaySeconds > conf.CFG_PQ.MaxDeliveryDelay {
return sqserr.InvalidParameterValueError(
"Delay secods must be between 0 and %d", conf.CFG_PQ.MaxDeliveryDelay/1000)
}
bodyMd5str := fmt.Sprintf("%x", md5.Sum(enc.UnsafeStringToBytes(out.MessageBody)))
attrMd5 := CalcAttrMd5(outAttrs)
msgPayload := sqsmsg.SQSMessagePayload{
UserAttributes: outAttrs,
MD5OfMessageBody: bodyMd5str,
MD5OfMessageAttributes: attrMd5,
SenderId: senderId,
SentTimestamp: strconv.FormatInt(utils.Uts(), 10),
Payload: out.MessageBody,
}
d, marshalErr := msgPayload.Marshal()
if marshalErr != nil {
log.Error("Failed to serialize message payload: %v", err)
}
payload := enc.UnsafeBytesToString(d)
resp := pq.Push(msgId, payload, pq.Config().MsgTtl, out.DelaySeconds, 1)
if resp.IsError() {
e, _ := resp.(error)
return sqserr.InvalidParameterValueError(e.Error())
}
return &SendMessageResponse{
MessageId: msgId,
MD5OfMessageBody: bodyMd5str,
MD5OfMessageAttributes: attrMd5,
RequestId: "req",
}
}