Example #1
0
func TestGetMessageInfo(t *testing.T) {
	Convey("Pushed message should return parameters", t, func() {
		q := CreateNewTestQueue()
		defer q.Close()
		q.Push("d1", "p", 10000, 1000, 9)
		q.Push("d2", "p", 10000, 0, 11)

		So(q.GetMessageInfo("d3"), ShouldResemble, mpqerr.ERR_MSG_NOT_FOUND)

		m1, _ := q.GetMessageInfo("d1").(*resp.DictResponse)
		m2, _ := q.GetMessageInfo("d2").(*resp.DictResponse)

		msgInfo1 := m1.GetDict()
		msgInfo2 := m2.GetDict()

		So(msgInfo1[MSG_INFO_ID], ShouldEqual, "d1")
		So(msgInfo1[MSG_INFO_LOCKED], ShouldEqual, true)
		So(msgInfo1[MSG_INFO_UNLOCK_TS], ShouldBeGreaterThan, utils.Uts())
		So(msgInfo1[MSG_INFO_POP_COUNT], ShouldEqual, 0)
		So(msgInfo1[MSG_INFO_PRIORITY], ShouldEqual, 9)
		So(msgInfo1[MSG_INFO_EXPIRE_TS], ShouldBeGreaterThan, utils.Uts())

		So(msgInfo2[MSG_INFO_ID], ShouldEqual, "d2")
		So(msgInfo2[MSG_INFO_LOCKED], ShouldEqual, false)
		So(msgInfo2[MSG_INFO_UNLOCK_TS], ShouldBeLessThanOrEqualTo, utils.Uts())
		So(msgInfo2[MSG_INFO_POP_COUNT], ShouldEqual, 0)
		So(msgInfo2[MSG_INFO_PRIORITY], ShouldEqual, 11)
		So(msgInfo2[MSG_INFO_EXPIRE_TS], ShouldBeGreaterThan, utils.Uts())

	})
}
Example #2
0
func TestMessagesMovedToAnotherQueue(t *testing.T) {
	Convey("Elements should move from one queue to the other when number of pop attempts exceeded", t, func() {

		log.InitLogging()
		log.SetLevel(1)
		db.SetDatabase(NewInMemDBService())

		fsl := NewFakeSvcLoader()
		q1 := CreateTestQueueWithName(fsl, "q1")
		failQueue := CreateTestQueueWithName(fsl, "fq")
		p := &PQueueParams{
			MsgTTL:         int64Ptr(10000),
			MaxMsgSize:     int64Ptr(256000),
			MaxMsgsInQueue: int64Ptr(100000),
			DeliveryDelay:  int64Ptr(0),
			PopCountLimit:  int64Ptr(2),
			PopLockTimeout: int64Ptr(1000),
			FailQueue:      "fq",
		}
		q1.SetParams(p)

		q1.StartUpdate()
		failQueue.StartUpdate()

		defer q1.Close()
		defer failQueue.Close()

		Convey("Two elements should be moved to another queue", func() {
			VerifyOkResponse(q1.Push("d1", "p", 10000, 0, 11))
			VerifyOkResponse(q1.Push("d2", "p", 10000, 0, 11))
			VerifyServiceSize(q1, 2)

			VerifyItemsRespSize(q1.Pop(100, 0, 10, true), 2)
			q1.checkTimeouts(utils.Uts() + 10000)
			VerifyServiceSize(q1, 2)

			VerifyItemsRespSize(q1.Pop(100, 0, 10, true), 2)
			q1.checkTimeouts(utils.Uts() + 10000)
			VerifyServiceSize(q1, 0)

			// Need to wait while message transferring is happening.
			for i := 0; i < 10000; i++ {
				time.Sleep(time.Microsecond * 1)
				if failQueue.Info().Size == 2 {
					break
				}
			}
			VerifyServiceSize(failQueue, 2)
		})

	})
}
Example #3
0
func TestStatus(t *testing.T) {
	Convey("Queue status should be correct", t, func() {
		q := CreateNewTestQueue()
		defer q.Close()
		Convey("Empty status should be default", func() {
			s, _ := q.GetCurrentStatus().(*resp.DictResponse)
			status := s.GetDict()
			So(status[PQ_STATUS_MAX_QUEUE_SIZE], ShouldEqual, 100001)
			So(status[PQ_STATUS_MSG_TTL], ShouldEqual, 100000)
			So(status[PQ_STATUS_DELIVERY_DELAY], ShouldEqual, 1)
			So(status[PQ_STATUS_POP_LOCK_TIMEOUT], ShouldEqual, 10000)
			So(status[PQ_STATUS_POP_COUNT_LIMIT], ShouldEqual, 4)
			So(status[PQ_STATUS_CREATE_TS], ShouldEqual, 123)
			So(status[PQ_STATUS_LAST_PUSH_TS], ShouldEqual, 12)
			So(status[PQ_STATUS_LAST_POP_TS], ShouldEqual, 13)
			So(status[PQ_STATUS_TOTAL_MSGS], ShouldEqual, 0)
			So(status[PQ_STATUS_IN_FLIGHT_MSG], ShouldEqual, 0)
			So(status[PQ_STATUS_AVAILABLE_MSGS], ShouldEqual, 0)

			So(q.Info().ID, ShouldEqual, "1")
			So(q.Info().Type, ShouldEqual, apis.ServiceTypePriorityQueue)
		})
		Convey("Status for several messages in flight", func() {
			q.Push("d1", "p", 10000, 0, 9)
			q.Push("d2", "p", 10000, 0, 9)
			q.Push("d3", "p", 10000, 0, 9)

			VerifySingleItem(q.Pop(100000, 0, 1, true), "d1", "p")
			VerifyServiceSize(q, 3)

			s, _ := q.GetCurrentStatus().(*resp.DictResponse)
			status := s.GetDict()
			So(status[PQ_STATUS_MAX_QUEUE_SIZE], ShouldEqual, 100001)
			So(status[PQ_STATUS_MSG_TTL], ShouldEqual, 100000)
			So(status[PQ_STATUS_DELIVERY_DELAY], ShouldEqual, 1)
			So(status[PQ_STATUS_POP_LOCK_TIMEOUT], ShouldEqual, 10000)
			So(status[PQ_STATUS_POP_COUNT_LIMIT], ShouldEqual, 4)
			So(status[PQ_STATUS_CREATE_TS], ShouldBeLessThanOrEqualTo, utils.Uts())
			So(status[PQ_STATUS_LAST_PUSH_TS], ShouldBeLessThanOrEqualTo, utils.Uts())
			So(status[PQ_STATUS_LAST_POP_TS], ShouldBeLessThanOrEqualTo, utils.Uts())
			So(status[PQ_STATUS_TOTAL_MSGS], ShouldEqual, 3)
			So(status[PQ_STATUS_IN_FLIGHT_MSG], ShouldEqual, 1)
			So(status[PQ_STATUS_AVAILABLE_MSGS], ShouldEqual, 2)
		})

	})
}
Example #4
0
func TestReleaseInFlight(t *testing.T) {
	Convey("One item should expire", t, func() {
		q := CreateNewTestQueue()
		defer q.Close()
		q.Push("d1", "p", 10000, 100, 11)
		r, _ := q.ReleaseInFlight(utils.Uts() + 1000).(*resp.IntResponse)
		So(r.Value, ShouldEqual, 1)
		VerifySingleItem(q.Pop(0, 0, 1, false), "d1", "p")
	})
}
Example #5
0
func TestExpiration(t *testing.T) {
	Convey("One item should expire", t, func() {
		q := CreateNewTestQueue()
		defer q.Close()
		q.Push("d1", "p", 10000, 0, 11)
		r, _ := q.TimeoutItems(utils.Uts() + 100000).(*resp.IntResponse)
		So(r.Value, ShouldEqual, 1)
		VerifyServiceSize(q, 0)
	})
}
Example #6
0
func NewServiceDescription(name, sType string, exportId uint64) *ServiceDescription {
	return &ServiceDescription{
		ExportId:  exportId,
		SType:     sType,
		Name:      name,
		CreateTs:  utils.Uts(),
		Disabled:  false,
		ToDelete:  false,
		ServiceId: enc.EncodeTo36Base(exportId),
	}
}
Example #7
0
func DefaultPQConfig() *conf.PQConfig {
	cfg := &conf.Config{}
	flags.ParseArgs(cfg, []string{"firempq"})

	conf.CFG = cfg
	conf.CFG_PQ = &cfg.PQueueConfig

	return &conf.PQConfig{
		MaxMsgsInQueue:    conf.CFG_PQ.DefaultMaxQueueSize,
		MsgTtl:            conf.CFG_PQ.DefaultMessageTTL,
		DeliveryDelay:     conf.CFG_PQ.DefaultDeliveryDelay,
		PopLockTimeout:    conf.CFG_PQ.DefaultLockTimeout,
		LastPushTs:        utils.Uts(),
		LastPopTs:         utils.Uts(),
		PopLimitQueueName: "",
		MaxMsgSize:        conf.CFG_PQ.MaxMessageSize,
		PopWaitTimeout:    conf.CFG_PQ.DefaultPopWaitTimeout,
		LastUpdateTs:      utils.Uts(),
	}
}
Example #8
0
func TestAutoExpiration(t *testing.T) {
	q := CreateNewTestQueue()
	defer q.Close()
	Convey("Two messages should expire, one message should still be in the queue", t, func() {
		q.Push("data1", "p1", 1000, 0, 12)
		q.Push("data2", "p2", 1000, 0, 12)
		q.Push("data3", "p3", 10000, 0, 12)
		VerifyServiceSize(q, 3)
		q.checkTimeouts(utils.Uts() + 1300)
		VerifyServiceSize(q, 1)
	})
}
Example #9
0
func TestPushPopAndTimeUnlockItems(t *testing.T) {
	q := CreateNewTestQueue()
	defer q.Close()
	Convey("Test push and pop messages", t, func() {
		q.Push("data1", "p1", 10000, 0, 12)
		q.Push("data2", "p2", 10000, 0, 12)
		VerifyServiceSize(q, 2)
		VerifyItems(q.Pop(10000, 0, 10, true), 2, "data1", "p1", "data2", "p2")
		VerifyServiceSize(q, 2)

		// Unlock item data1 it should become available again.
		q.UpdateLockById("data1", 0)
		q.checkTimeouts(utils.Uts() + 110)
		VerifySingleItem(q.Pop(10000, 0, 1, true), "data1", "p1")
		VerifyServiceSize(q, 2)

		VerifyOkResponse(q.DeleteLockedById("data1"))

		VerifyServiceSize(q, 1)
		q.DeleteLockedById("data2")
		VerifyServiceSize(q, 0)
	})
}
Example #10
0
func MakeMessageResponse(iMsg apis.IResponseItem, opts *ReceiveMessageOptions,
	sqsQuery *urlutils.SQSQuery) *MessageResponse {
	msg, ok := iMsg.(*pqueue.MsgResponseItem)
	if !ok {
		return nil
	}
	sqsMsg := &sqsmsg.SQSMessagePayload{}
	payload := msg.Payload()
	if err := sqsMsg.Unmarshal([]byte(payload)); err != nil {
		// Recovering from error. Non SQS messages will be filled with bulk info.
		sqsMsg.Payload = string(msg.Payload())
		sqsMsg.SenderId = "unknown"
		sqsMsg.SentTimestamp = strconv.FormatInt(utils.Uts(), 10)
		sqsMsg.MD5OfMessageAttributes = fmt.Sprintf("%x", md5.Sum(nil))
		sqsMsg.MD5OfMessageBody = fmt.Sprintf("%x", md5.Sum(enc.UnsafeStringToBytes(sqsMsg.Payload)))
	}

	msgMeta := msg.GetMeta()

	output := &MessageResponse{
		MD5OfMessageAttributes: sqsMsg.MD5OfMessageAttributes,
		MD5OfMessageBody:       sqsMsg.MD5OfMessageBody,
		ReceiptHandle:          msg.Receipt(),
		MessageId:              msg.ID(),
		Body:                   sqsMsg.Payload,
	}

	if opts.AllSysAttributes {
		output.Attributes = append(output.Attributes, &SysAttribute{
			Name: AttrSenderId, Value: sqsMsg.SenderId,
		})
		output.Attributes = append(output.Attributes, &SysAttribute{
			Name: AttrSentTimestamp, Value: sqsMsg.SentTimestamp,
		})
		output.Attributes = append(output.Attributes, &SysAttribute{
			Name: AttrApproximateReceiveCount, Value: strconv.FormatInt(msgMeta.PopCount, 10),
		})
		output.Attributes = append(output.Attributes, &SysAttribute{
			Name: AttrApproximateFirstReceiveTimestamp, Value: strconv.FormatInt(utils.Uts(), 10),
		})
	} else {
		for _, k := range opts.Attributes {
			switch k {
			case AttrSenderId:
				output.Attributes = append(output.Attributes, &SysAttribute{
					Name: AttrSenderId, Value: sqsMsg.SenderId,
				})
			case AttrSentTimestamp:
				output.Attributes = append(output.Attributes, &SysAttribute{
					Name: AttrSentTimestamp, Value: sqsMsg.SentTimestamp,
				})
			case AttrApproximateReceiveCount:
				output.Attributes = append(output.Attributes, &SysAttribute{
					Name: AttrApproximateReceiveCount, Value: strconv.FormatInt(msgMeta.PopCount, 10),
				})
			case AttrApproximateFirstReceiveTimestamp:
				output.Attributes = append(output.Attributes, &SysAttribute{
					Name: AttrApproximateFirstReceiveTimestamp, Value: strconv.FormatInt(utils.Uts(), 10),
				})
			}
		}
	}

	if opts.AllMessageAttributes {
		for attrName, attrData := range sqsMsg.UserAttributes {
			output.MessageAttributes = append(
				output.MessageAttributes, MakeMessageAttr(attrName, attrData))
		}
	} else {
		for _, attrName := range opts.MessageAttributes {
			if v, ok := sqsMsg.UserAttributes[attrName]; ok {
				output.MessageAttributes = append(
					output.MessageAttributes, MakeMessageAttr(attrName, v))
			}
		}
	}

	if !ok {
		log.Error("Failed to cast response item")
		return nil
	}
	return output
}
Example #11
0
func PushAMessage(pq *pqueue.PQueue, senderId string, paramList []string) sqs_response.SQSResponse {
	out := &MessageParams{
		DelaySeconds: -1,
		MessageBody:  "",
	}
	attrs, err := urlutils.ParseNNotationAttr("MessageAttribute.", paramList, out.Parse, NewReqQueueAttr)
	if err != nil {
		return err
	}
	attrsLen := len(attrs)
	outAttrs := make(map[string]*sqsmsg.UserAttribute)

	for i := 1; i <= attrsLen; i++ {
		a, ok := attrs[i]
		if !ok {
			return sqserr.InvalidParameterValueError("The request must contain non-empty message attribute name.")
		}
		reqMsgAttr, _ := a.(*ReqMsgAttr)

		sqs_err := validation.ValidateMessageAttrName(reqMsgAttr.Name)
		if sqs_err != nil {
			return sqs_err
		}

		sqs_err = validation.ValidateMessageAttrName(reqMsgAttr.DataType)
		if sqs_err != nil {
			return sqs_err
		}

		if reqMsgAttr.BinaryValue != "" && reqMsgAttr.StringValue != "" {
			return sqserr.InvalidParameterValueError(
				"Message attribute name '%s' has multiple values.", reqMsgAttr.Name)
		}

		if _, ok := outAttrs[reqMsgAttr.Name]; ok {
			return sqserr.InvalidParameterValueError(
				"Message attribute name '%s' already exists.", reqMsgAttr.Name)
		}

		if strings.HasPrefix(reqMsgAttr.DataType, "Number") {
			if _, err := strconv.Atoi(reqMsgAttr.StringValue); err != nil {
				return sqserr.InvalidParameterValueError(
					"Could not cast message attribute '%s' value to number.", reqMsgAttr.Name)
			}
		}

		if reqMsgAttr.BinaryValue != "" {
			if reqMsgAttr.DataType != "Binary" {
				return sqserr.InvalidParameterValueError(
					"The message attribute '%s' with type 'Binary' must use field 'Binary'", reqMsgAttr.Name)
			}
			outAttrs[reqMsgAttr.Name] = &sqsmsg.UserAttribute{
				Type:  reqMsgAttr.DataType,
				Value: reqMsgAttr.BinaryValue,
			}
			continue
		}

		if reqMsgAttr.StringValue != "" {
			if reqMsgAttr.DataType != "String" && reqMsgAttr.DataType != "Number" {
				return sqserr.InvalidParameterValueError(
					"The message attribute '%s' with type 'String' must use field 'String'", reqMsgAttr.Name)
			}
			outAttrs[reqMsgAttr.Name] = &sqsmsg.UserAttribute{
				Type:  reqMsgAttr.DataType,
				Value: reqMsgAttr.StringValue,
			}
		}
	}

	msgId := IdGen.RandId()
	if out.DelaySeconds < 0 {
		out.DelaySeconds = pq.Config().DeliveryDelay
	} else if out.DelaySeconds > conf.CFG_PQ.MaxDeliveryDelay {
		return sqserr.InvalidParameterValueError(
			"Delay secods must be between 0 and %d", conf.CFG_PQ.MaxDeliveryDelay/1000)
	}
	bodyMd5str := fmt.Sprintf("%x", md5.Sum(enc.UnsafeStringToBytes(out.MessageBody)))
	attrMd5 := CalcAttrMd5(outAttrs)

	msgPayload := sqsmsg.SQSMessagePayload{
		UserAttributes:         outAttrs,
		MD5OfMessageBody:       bodyMd5str,
		MD5OfMessageAttributes: attrMd5,
		SenderId:               senderId,
		SentTimestamp:          strconv.FormatInt(utils.Uts(), 10),
		Payload:                out.MessageBody,
	}

	d, marshalErr := msgPayload.Marshal()
	if marshalErr != nil {
		log.Error("Failed to serialize message payload: %v", err)
	}
	payload := enc.UnsafeBytesToString(d)

	resp := pq.Push(msgId, payload, pq.Config().MsgTtl, out.DelaySeconds, 1)
	if resp.IsError() {
		e, _ := resp.(error)
		return sqserr.InvalidParameterValueError(e.Error())
	}

	return &SendMessageResponse{
		MessageId:              msgId,
		MD5OfMessageBody:       bodyMd5str,
		MD5OfMessageAttributes: attrMd5,
		RequestId:              "req",
	}
}