// Push message to the queue.
// Pushing message automatically enables auto expiration.
func (pq *PQueue) Push(params []string) iface.IResponse {
var err *common.ErrorResponse
var msgId string
var priority int64 = pq.config.MaxPriority - 1
var payload string = ""
for len(params) > 0 {
switch params[0] {
case PRM_ID:
params, msgId, err = common.ParseStringParam(params, 1, 128)
case PRM_PRIORITY:
params, priority, err = common.ParseInt64Params(params, 0, pq.config.MaxPriority-1)
case PRM_PAYLOAD:
params, payload, err = common.ParseStringParam(params, 1, 512*1024)
default:
return makeUnknownParamResponse(params[0])
}
if err != nil {
return err
}
}
if len(msgId) == 0 {
msgId = common.GenRandMsgId()
}
pq.config.LastPushTs = common.Uts()
pq.lock.Lock()
defer pq.lock.Unlock()
pq.msgSerialNumber += 1
msg := NewPQMessage(msgId, priority, pq.msgSerialNumber)
return pq.storeMessage(msg, payload)
}
func NewPQueue(desc *common.ServiceDescription, priorities int64, size int64) *PQueue {
defaults := conf.CFG.PQueueConfig
config := &PQConfig{
MaxPriority: priorities,
MaxSize: size,
MsgTtl: defaults.DefaultMessageTtl,
DeliveryDelay: defaults.DefaultDeliveryDelay,
PopLockTimeout: defaults.DefaultLockTimeout,
PopCountLimit: defaults.DefaultPopCountLimit,
LastPushTs: common.Uts(),
LastPopTs: common.Uts(),
InactivityTtl: 0,
}
queue := initPQueue(desc, config)
queue.database.SaveServiceConfig(queue.serviceId, config)
return queue
}
func (pq *PQueue) updateLoop() {
for pq.closedState.IsFalse() {
if pq.update(common.Uts()) {
time.Sleep(time.Millisecond)
} else {
time.Sleep(conf.CFG.UpdateInterval * time.Millisecond)
}
}
}
func (pq *PQueue) lockMessage(msg *PQMessage, lockTimeout int64) {
nowTs := common.Uts()
pq.config.LastPopTs = nowTs
// Increase number of pop attempts.
msg.UnlockTs = nowTs + lockTimeout
pq.expireHeap.PopById(msg.Id)
pq.inFlightHeap.PushItem(msg.Id, msg.UnlockTs)
}
func (pq *PQueue) loadAllMessages() {
nowTs := common.Uts()
log.Debug("Initializing queue: %s", pq.desc.Name)
iter := pq.database.IterServiceItems(pq.serviceId)
defer iter.Close()
msgs := MessageSlice{}
delIds := []string{}
cfg := pq.config
for iter.Valid() {
pqmsg := UnmarshalPQMessage(string(iter.Key), iter.Value)
// Store list if message IDs that should be removed.
if pqmsg.CreatedTs+cfg.MsgTtl < nowTs ||
(pqmsg.PopCount >= cfg.PopCountLimit &&
cfg.PopCountLimit > 0) {
delIds = append(delIds, pqmsg.Id)
} else {
msgs = append(msgs, pqmsg)
}
iter.Next()
}
log.Debug("Loaded %d messages for %s queue", len(msgs), pq.desc.Name)
if len(delIds) > 0 {
log.Debug("Deleting %d expired messages", len(delIds))
for _, msgId := range delIds {
pq.database.DeleteItem(pq.serviceId, msgId)
}
}
// Sorting data guarantees that messages will be available in the same order as they arrived.
sort.Sort(msgs)
// Update serial number to match the latest message.
if len(msgs) > 0 {
pq.msgSerialNumber = msgs[len(msgs)-1].SerialNumber
}
for _, msg := range msgs {
pq.msgMap[msg.Id] = msg
if msg.UnlockTs > nowTs {
pq.inFlightHeap.PushItem(msg.Id, msg.UnlockTs)
} else {
pq.expireHeap.PushItem(msg.Id, msg.CreatedTs+cfg.MsgTtl)
pq.availMsgs.Push(msg.Id, msg.Priority)
}
}
log.Debug("Messages available: %d", pq.expireHeap.Len())
log.Debug("Messages are in flight: %d", pq.inFlightHeap.Len())
}
func main1() {
// f, _ := os.Create("pp.dat")
// pprof.StartCPUProfile(f)
// defer pprof.StopCPUProfile()
fc := facade.CreateFacade()
defer fc.Close()
for i := 0; i < 1; i++ {
qid := "tst_queue_" + strconv.Itoa(i)
err := fc.CreateService(common.STYPE_PRIORITY_QUEUE, qid, nil)
if err != nil {
log.Notice("%s: %s", err, qid)
}
}
start_ts := common.Uts()
log.Notice("Started")
for i := 0; i < 1; i++ {
qid := "tst_queue_" + strconv.Itoa(i)
q, _ := fc.GetService(qid)
addMessages(q)
}
log.Notice("Finished. Elapsed: %d", common.Uts()-start_ts)
}
func TestAutoExpiration(t *testing.T) {
q := CreateTestQueue()
q.Clear()
defer q.Close()
defer q.Clear()
q.config.MsgTtl = 10
q.Push([]string{PRM_ID, "dd1", PRM_PRIORITY, "12", PRM_PAYLOAD, "p1"})
q.Push([]string{PRM_ID, "dd2", PRM_PRIORITY, "12", PRM_PAYLOAD, "p2"})
// Wait for auto expiration.
q.update(common.Uts() + 1300)
msg := q.Pop([]string{}).GetResponse()
cmp(t, msg, "+DATA *0")
if len(q.msgMap) != 0 {
t.Error("Messages map must be empty!")
}
}
// Set a user defined message lock timeout. Only locked message timeout can be set.
func (pq *PQueue) SetLockTimeout(params []string) iface.IResponse {
var err *common.ErrorResponse
var msgId string
var lockTimeout int64 = -1
for len(params) > 0 {
switch params[0] {
case PRM_ID:
params, msgId, err = common.ParseStringParam(params, 1, 128)
case PRM_LOCK_TIMEOUT:
params, lockTimeout, err = common.ParseInt64Params(params, 0, 24*1000*3600)
default:
return makeUnknownParamResponse(params[0])
}
if err != nil {
return err
}
}
if len(msgId) == 0 {
return common.ERR_MSG_ID_NOT_DEFINED
}
if lockTimeout < 0 {
return common.ERR_MSG_TIMEOUT_NOT_DEFINED
}
pq.lock.Lock()
defer pq.lock.Unlock()
msg, err := pq.unflightMessage(msgId)
if err != nil {
return err
}
msg.UnlockTs = common.Uts() + int64(lockTimeout)
pq.inFlightHeap.PushItem(msgId, msg.UnlockTs)
pq.database.StoreItem(pq.serviceId, msg)
return common.OK_RESPONSE
}
func TestPushPopAndTimeUnlockItems(t *testing.T) {
q := CreateTestQueue()
q.Clear()
defer q.Close()
defer q.Clear()
q.Push([]string{PRM_ID, "data1", PRM_PRIORITY, "12", PRM_PAYLOAD, "p1"})
q.Push([]string{PRM_ID, "data2", PRM_PRIORITY, "12", PRM_PAYLOAD, "p2"})
pop_msg1 := q.Pop([]string{}).GetResponse()
pop_msg2 := q.Pop([]string{}).GetResponse()
cmp(t, pop_msg1, "+DATA *1 %2 ID $5 data1 PL $2 p1")
cmp(t, pop_msg2, "+DATA *1 %2 ID $5 data2 PL $2 p2")
params := []string{PRM_ID, "data1", PRM_LOCK_TIMEOUT, "0"}
q.Call(ACTION_SET_LOCK_TIMEOUT, params)
q.update(common.Uts() + 110000)
pop_msg3 := q.Pop(nil).GetResponse()
cmp(t, pop_msg3, "+DATA *1 %2 ID $5 data1 PL $2 p1")
}
// Returns current server unix time stamp in milliseconds.
func tsHandler(tokens []string) iface.IResponse {
if len(tokens) > 0 {
return common.ERR_CMD_WITH_NO_PARAMS
}
return common.NewIntResponse(common.Uts())
}
func NewPQMessage(id string, priority int64, serialNumber uint64) *PQMessage {
return &PQMessage{id, PQueueMsgData{priority, common.Uts(), 0, 0, serialNumber}}
}