Exemple #1
0
// Push message to the queue.
// Pushing message automatically enables auto expiration.
func (pq *PQueue) Push(params []string) iface.IResponse {
	var err *common.ErrorResponse
	var msgId string
	var priority int64 = pq.config.MaxPriority - 1
	var payload string = ""

	for len(params) > 0 {
		switch params[0] {
		case PRM_ID:
			params, msgId, err = common.ParseStringParam(params, 1, 128)
		case PRM_PRIORITY:
			params, priority, err = common.ParseInt64Params(params, 0, pq.config.MaxPriority-1)
		case PRM_PAYLOAD:
			params, payload, err = common.ParseStringParam(params, 1, 512*1024)
		default:
			return makeUnknownParamResponse(params[0])
		}
		if err != nil {
			return err
		}
	}
	if len(msgId) == 0 {
		msgId = common.GenRandMsgId()
	}

	pq.config.LastPushTs = common.Uts()

	pq.lock.Lock()
	defer pq.lock.Unlock()
	pq.msgSerialNumber += 1
	msg := NewPQMessage(msgId, priority, pq.msgSerialNumber)
	return pq.storeMessage(msg, payload)
}
Exemple #2
0
func NewPQueue(desc *common.ServiceDescription, priorities int64, size int64) *PQueue {
	defaults := conf.CFG.PQueueConfig
	config := &PQConfig{
		MaxPriority:    priorities,
		MaxSize:        size,
		MsgTtl:         defaults.DefaultMessageTtl,
		DeliveryDelay:  defaults.DefaultDeliveryDelay,
		PopLockTimeout: defaults.DefaultLockTimeout,
		PopCountLimit:  defaults.DefaultPopCountLimit,
		LastPushTs:     common.Uts(),
		LastPopTs:      common.Uts(),
		InactivityTtl:  0,
	}

	queue := initPQueue(desc, config)
	queue.database.SaveServiceConfig(queue.serviceId, config)
	return queue
}
Exemple #3
0
func (pq *PQueue) updateLoop() {
	for pq.closedState.IsFalse() {
		if pq.update(common.Uts()) {
			time.Sleep(time.Millisecond)
		} else {
			time.Sleep(conf.CFG.UpdateInterval * time.Millisecond)
		}
	}
}
Exemple #4
0
func (pq *PQueue) lockMessage(msg *PQMessage, lockTimeout int64) {
	nowTs := common.Uts()
	pq.config.LastPopTs = nowTs
	// Increase number of pop attempts.

	msg.UnlockTs = nowTs + lockTimeout
	pq.expireHeap.PopById(msg.Id)
	pq.inFlightHeap.PushItem(msg.Id, msg.UnlockTs)
}
Exemple #5
0
func (pq *PQueue) loadAllMessages() {
	nowTs := common.Uts()
	log.Debug("Initializing queue: %s", pq.desc.Name)
	iter := pq.database.IterServiceItems(pq.serviceId)
	defer iter.Close()

	msgs := MessageSlice{}
	delIds := []string{}

	cfg := pq.config
	for iter.Valid() {
		pqmsg := UnmarshalPQMessage(string(iter.Key), iter.Value)
		// Store list if message IDs that should be removed.
		if pqmsg.CreatedTs+cfg.MsgTtl < nowTs ||
			(pqmsg.PopCount >= cfg.PopCountLimit &&
				cfg.PopCountLimit > 0) {
			delIds = append(delIds, pqmsg.Id)
		} else {
			msgs = append(msgs, pqmsg)
		}
		iter.Next()
	}
	log.Debug("Loaded %d messages for %s queue", len(msgs), pq.desc.Name)
	if len(delIds) > 0 {
		log.Debug("Deleting %d expired messages", len(delIds))
		for _, msgId := range delIds {
			pq.database.DeleteItem(pq.serviceId, msgId)
		}

	}
	// Sorting data guarantees that messages will be available in the same order as they arrived.
	sort.Sort(msgs)

	// Update serial number to match the latest message.
	if len(msgs) > 0 {
		pq.msgSerialNumber = msgs[len(msgs)-1].SerialNumber
	}

	for _, msg := range msgs {
		pq.msgMap[msg.Id] = msg
		if msg.UnlockTs > nowTs {
			pq.inFlightHeap.PushItem(msg.Id, msg.UnlockTs)
		} else {
			pq.expireHeap.PushItem(msg.Id, msg.CreatedTs+cfg.MsgTtl)
			pq.availMsgs.Push(msg.Id, msg.Priority)
		}
	}

	log.Debug("Messages available: %d", pq.expireHeap.Len())
	log.Debug("Messages are in flight: %d", pq.inFlightHeap.Len())
}
Exemple #6
0
func main1() {
	//	f, _ := os.Create("pp.dat")
	//	pprof.StartCPUProfile(f)
	//	defer pprof.StopCPUProfile()

	fc := facade.CreateFacade()
	defer fc.Close()
	for i := 0; i < 1; i++ {
		qid := "tst_queue_" + strconv.Itoa(i)
		err := fc.CreateService(common.STYPE_PRIORITY_QUEUE, qid, nil)
		if err != nil {
			log.Notice("%s: %s", err, qid)
		}
	}
	start_ts := common.Uts()
	log.Notice("Started")
	for i := 0; i < 1; i++ {
		qid := "tst_queue_" + strconv.Itoa(i)
		q, _ := fc.GetService(qid)
		addMessages(q)
	}
	log.Notice("Finished. Elapsed: %d", common.Uts()-start_ts)
}
Exemple #7
0
func TestAutoExpiration(t *testing.T) {
	q := CreateTestQueue()
	q.Clear()
	defer q.Close()
	defer q.Clear()

	q.config.MsgTtl = 10
	q.Push([]string{PRM_ID, "dd1", PRM_PRIORITY, "12", PRM_PAYLOAD, "p1"})
	q.Push([]string{PRM_ID, "dd2", PRM_PRIORITY, "12", PRM_PAYLOAD, "p2"})

	// Wait for auto expiration.
	q.update(common.Uts() + 1300)
	msg := q.Pop([]string{}).GetResponse()
	cmp(t, msg, "+DATA *0")
	if len(q.msgMap) != 0 {
		t.Error("Messages map must be empty!")
	}
}
Exemple #8
0
// Set a user defined message lock timeout. Only locked message timeout can be set.
func (pq *PQueue) SetLockTimeout(params []string) iface.IResponse {
	var err *common.ErrorResponse
	var msgId string
	var lockTimeout int64 = -1

	for len(params) > 0 {
		switch params[0] {
		case PRM_ID:
			params, msgId, err = common.ParseStringParam(params, 1, 128)
		case PRM_LOCK_TIMEOUT:
			params, lockTimeout, err = common.ParseInt64Params(params, 0, 24*1000*3600)
		default:
			return makeUnknownParamResponse(params[0])
		}
		if err != nil {
			return err
		}
	}

	if len(msgId) == 0 {
		return common.ERR_MSG_ID_NOT_DEFINED
	}

	if lockTimeout < 0 {
		return common.ERR_MSG_TIMEOUT_NOT_DEFINED
	}

	pq.lock.Lock()
	defer pq.lock.Unlock()

	msg, err := pq.unflightMessage(msgId)
	if err != nil {
		return err
	}

	msg.UnlockTs = common.Uts() + int64(lockTimeout)

	pq.inFlightHeap.PushItem(msgId, msg.UnlockTs)
	pq.database.StoreItem(pq.serviceId, msg)

	return common.OK_RESPONSE
}
Exemple #9
0
func TestPushPopAndTimeUnlockItems(t *testing.T) {
	q := CreateTestQueue()
	q.Clear()
	defer q.Close()
	defer q.Clear()

	q.Push([]string{PRM_ID, "data1", PRM_PRIORITY, "12", PRM_PAYLOAD, "p1"})
	q.Push([]string{PRM_ID, "data2", PRM_PRIORITY, "12", PRM_PAYLOAD, "p2"})

	pop_msg1 := q.Pop([]string{}).GetResponse()
	pop_msg2 := q.Pop([]string{}).GetResponse()

	cmp(t, pop_msg1, "+DATA *1 %2 ID $5 data1 PL $2 p1")
	cmp(t, pop_msg2, "+DATA *1 %2 ID $5 data2 PL $2 p2")

	params := []string{PRM_ID, "data1", PRM_LOCK_TIMEOUT, "0"}

	q.Call(ACTION_SET_LOCK_TIMEOUT, params)

	q.update(common.Uts() + 110000)

	pop_msg3 := q.Pop(nil).GetResponse()
	cmp(t, pop_msg3, "+DATA *1 %2 ID $5 data1 PL $2 p1")
}
Exemple #10
0
// Returns current server unix time stamp in milliseconds.
func tsHandler(tokens []string) iface.IResponse {
	if len(tokens) > 0 {
		return common.ERR_CMD_WITH_NO_PARAMS
	}
	return common.NewIntResponse(common.Uts())
}
Exemple #11
0
func NewPQMessage(id string, priority int64, serialNumber uint64) *PQMessage {
	return &PQMessage{id, PQueueMsgData{priority, common.Uts(), 0, 0, serialNumber}}
}