// MultiPush implements MultiPush interface
func (u *UnitedQueue) MultiPush(key string, datas [][]byte) error {
key = strings.TrimPrefix(key, "/")
key = strings.TrimSuffix(key, "/")
for i, data := range datas {
if len(data) <= 0 {
cause := "message " + strconv.Itoa(i) + " has no content"
return utils.NewError(
utils.ErrBadRequest,
cause,
)
}
}
u.topicsLock.RLock()
t, ok := u.topics[key]
u.topicsLock.RUnlock()
if !ok {
return utils.NewError(
utils.ErrTopicNotExisted,
`queue multiPush`,
)
}
return t.mPush(datas)
}
func (u *UnitedQueue) create(key, rec string, fromEtcd bool) error {
key = strings.TrimPrefix(key, "/")
key = strings.TrimSuffix(key, "/")
var topicName, lineName string
var err error
parts := strings.Split(key, "/")
if len(parts) < 1 || len(parts) > 2 {
return utils.NewError(
utils.ErrBadKey,
`create key parts error: `+utils.ItoaQuick(len(parts)),
)
}
topicName = parts[0]
if topicName == "" {
return utils.NewError(
utils.ErrBadKey,
`create topic is nil`,
)
}
if len(parts) == 2 {
lineName = parts[1]
var recycle time.Duration
if rec != "" {
recycle, err = time.ParseDuration(rec)
if err != nil {
return utils.NewError(
utils.ErrBadRequest,
err.Error(),
)
}
}
u.topicsLock.RLock()
t, ok := u.topics[topicName]
u.topicsLock.RUnlock()
if !ok {
return utils.NewError(
utils.ErrTopicNotExisted,
`queue create`,
)
}
err = t.createLine(lineName, recycle, fromEtcd)
if err != nil {
// log.Printf("create line[%s] error: %s", lineName, err)
return err
}
} else {
err = u.createTopic(topicName, fromEtcd)
if err != nil {
// log.Printf("create topic[%s] error: %s", topicName, err)
return err
}
}
return err
}
// Confirm implements Confirm interface
func (u *UnitedQueue) Confirm(key string) error {
key = strings.TrimPrefix(key, "/")
key = strings.TrimSuffix(key, "/")
parts := strings.Split(key, "/")
if len(parts) != 3 {
return utils.NewError(
utils.ErrBadKey,
`confirm key parts error: `+utils.ItoaQuick(len(parts)),
)
}
topicName := parts[0]
lineName := parts[1]
id, err := strconv.ParseUint(parts[2], 10, 0)
if err != nil {
return utils.NewError(
utils.ErrBadKey,
`confirm key parse id error: `+err.Error(),
)
}
u.topicsLock.RLock()
t, ok := u.topics[topicName]
u.topicsLock.RUnlock()
if !ok {
// log.Printf("topic[%s] not existed.", topicName)
return utils.NewError(
utils.ErrTopicNotExisted,
`queue confirm`,
)
}
return t.confirm(lineName, id)
}
// Pop implements Pop interface
func (u *UnitedQueue) Pop(key string) (string, []byte, error) {
key = strings.TrimPrefix(key, "/")
key = strings.TrimSuffix(key, "/")
parts := strings.Split(key, "/")
if len(parts) != 2 {
return "", nil, utils.NewError(
utils.ErrBadKey,
`pop key parts error: `+utils.ItoaQuick(len(parts)),
)
}
tName := parts[0]
lName := parts[1]
u.topicsLock.RLock()
t, ok := u.topics[tName]
u.topicsLock.RUnlock()
if !ok {
// log.Printf("topic[%s] not existed.", tName)
return "", nil, utils.NewError(
utils.ErrTopicNotExisted,
`queue pop`,
)
}
id, data, err := t.pop(lName)
if err != nil {
return "", nil, err
}
return utils.Acatui(key, "/", id), data, nil
}
func (u *UnitedQueue) createTopic(name string, fromEtcd bool) error {
u.topicsLock.RLock()
_, ok := u.topics[name]
u.topicsLock.RUnlock()
if ok {
return utils.NewError(
utils.ErrTopicExisted,
`queue createTopic`,
)
}
t, err := u.newTopic(name)
if err != nil {
return err
}
u.topicsLock.Lock()
defer u.topicsLock.Unlock()
u.topics[name] = t
err = u.exportQueue()
if err != nil {
t.remove()
delete(u.topics, name)
return err
}
if !fromEtcd {
u.registerTopic(t.name)
}
log.Printf("topic[%s] created.", name)
return nil
}
func (r *RedisEntry) onQmpop(cmd *command) *reply {
key := cmd.stringAtIndex(1)
n, err := cmd.intAtIndex(2)
if err != nil {
return errorReply(utils.NewError(
utils.ErrBadRequest,
err.Error(),
))
}
ids, values, err := r.messageQueue.MultiPop(key, n)
if err != nil {
return errorReply(err)
}
np := len(ids)
vals := make([]interface{}, np*2)
for i, index := 0, 0; i < np; i++ {
vals[index] = values[i]
index++
vals[index] = ids[i]
index++
}
return multiBulksReply(vals)
}
func (s *UnitedAdmin) statHandler(w http.ResponseWriter, req *http.Request, key string) {
if req.Method != "GET" {
http.Error(w, "405 Method Not Allowed!", http.StatusMethodNotAllowed)
return
}
qs, err := s.messageQueue.Stat(key)
if err != nil {
writeErrorHTTP(w, err)
return
}
// log.Printf("qs: %v", qs)
data, err := qs.ToJSON()
if err != nil {
writeErrorHTTP(w, utils.NewError(
utils.ErrInternalError,
err.Error(),
))
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write(data)
}
func (u *UnitedQueue) removeTopic(name string, fromEtcd bool) error {
u.topicsLock.Lock()
defer u.topicsLock.Unlock()
t, ok := u.topics[name]
if !ok {
return utils.NewError(
utils.ErrTopicNotExisted,
`queue remove`,
)
}
delete(u.topics, name)
err := u.exportQueue()
if err != nil {
u.topics[name] = t
return err
}
if !fromEtcd {
u.unRegisterTopic(name)
}
return t.remove()
}
func (t *topic) createLine(name string, recycle time.Duration, fromEtcd bool) error {
t.linesLock.Lock()
defer t.linesLock.Unlock()
_, ok := t.lines[name]
if ok {
return utils.NewError(
utils.ErrLineExisted,
`topic createLine`,
)
}
l, err := t.newLine(name, recycle)
if err != nil {
return err
}
t.lines[name] = l
err = t.exportTopic()
if err != nil {
t.linesLock.Lock()
delete(t.lines, name)
t.linesLock.Unlock()
return err
}
if !fromEtcd {
t.q.registerLine(t.name, l.name, l.recycle.String())
}
log.Printf("topic[%s] line[%s:%v] created.", t.name, name, recycle)
return nil
}
func (t *topic) removeLine(name string, fromEtcd bool) error {
t.linesLock.RLock()
defer t.linesLock.RUnlock()
l, ok := t.lines[name]
if !ok {
// log.Printf("topic[%s] line[%s] not existed.", t.name, name)
return utils.NewError(
utils.ErrLineNotExisted,
`topic statLine`,
)
}
delete(t.lines, name)
err := t.exportTopic()
if err != nil {
t.linesLock.Lock()
t.lines[name] = l
t.linesLock.Unlock()
return err
}
if !fromEtcd {
t.q.unRegisterLine(t.name, name)
}
return l.remove()
}
func (l *line) pop() (uint64, []byte, error) {
l.inflightLock.Lock()
defer l.inflightLock.Unlock()
now := time.Now()
if l.recycle > 0 {
m := l.inflight.Front()
if m != nil {
msg := m.Value.(*InflightMessage)
exp := time.Unix(0, msg.Exptime)
if now.After(exp) {
// log.Printf("key[%s/%d] is expired.", l.name, msg.Tid)
msg.Exptime = now.Add(l.recycle).UnixNano()
data, err := l.t.getData(msg.Tid)
if err != nil {
return 0, nil, err
}
l.inflight.Remove(m)
l.inflight.PushBack(msg)
// log.Printf("key[%s/%s/%d] poped.", l.t.name, l.name, msg.Tid)
return msg.Tid, data, nil
}
}
}
l.headLock.Lock()
defer l.headLock.Unlock()
tid := l.head
topicTail := l.t.getTail()
if l.head >= topicTail {
// log.Printf("line[%s] is blank. head:%d - tail:%d", l.name, l.head, l.t.tail)
return 0, nil, utils.NewError(
utils.ErrNone,
`line pop`,
)
}
data, err := l.t.getData(tid)
if err != nil {
return 0, nil, err
}
l.head++
if l.recycle > 0 {
msg := new(InflightMessage)
msg.Tid = tid
msg.Exptime = now.Add(l.recycle).UnixNano()
l.inflight.PushBack(msg)
// log.Printf("key[%s/%s/%d] flighted.", l.t.name, l.name, l.head)
l.imap[tid] = true
}
return tid, data, nil
}
// Empty implements Empty interface
func (u *UnitedQueue) Empty(key string) error {
key = strings.TrimPrefix(key, "/")
key = strings.TrimSuffix(key, "/")
var topicName, lineName string
parts := strings.Split(key, "/")
if len(parts) < 1 || len(parts) > 2 {
return utils.NewError(
utils.ErrBadKey,
`empty key parts error: `+utils.ItoaQuick(len(parts)),
)
}
topicName = parts[0]
if topicName == "" {
return utils.NewError(
utils.ErrBadKey,
`empty topic is nil`,
)
}
u.topicsLock.RLock()
t, ok := u.topics[topicName]
u.topicsLock.RUnlock()
if !ok {
return utils.NewError(
utils.ErrTopicNotExisted,
`queue empty`,
)
}
if len(parts) == 2 {
lineName = parts[1]
return t.emptyLine(lineName)
// err = t.emptyLine(lineName)
// if err != nil {
// log.Printf("empty line[%s] error: %s", lineName, err)
// }
// return err
}
return t.empty()
}
func (u *UnitedQueue) delData(key string) error {
err := u.storage.Del(key)
if err != nil {
// log.Printf("key[%s] del data error: %s", key, err)
return utils.NewError(
utils.ErrInternalError,
err.Error(),
)
}
return nil
}
func (u *UnitedQueue) setData(key string, data []byte) error {
err := u.storage.Set(key, data)
if err != nil {
// log.Printf("key[%s] set data error: %s", key, err)
return utils.NewError(
utils.ErrInternalError,
err.Error(),
)
}
return nil
}
func (u *UnitedQueue) getData(key string) ([]byte, error) {
data, err := u.storage.Get(key)
if err != nil {
// log.Printf("key[%s] get data error: %s", key, err)
return nil, utils.NewError(
utils.ErrInternalError,
err.Error(),
)
}
return data, nil
}
// Stat implements Stat interface
func (u *UnitedQueue) Stat(key string) (*Stat, error) {
key = strings.TrimPrefix(key, "/")
key = strings.TrimSuffix(key, "/")
var topicName, lineName string
parts := strings.Split(key, "/")
if len(parts) < 1 || len(parts) > 2 {
return nil, utils.NewError(
utils.ErrBadKey,
`empty key parts error: `+utils.ItoaQuick(len(parts)),
)
}
topicName = parts[0]
if topicName == "" {
return nil, utils.NewError(
utils.ErrBadKey,
`stat topic is nil`,
)
}
u.topicsLock.RLock()
t, ok := u.topics[topicName]
u.topicsLock.RUnlock()
if !ok {
return nil, utils.NewError(
utils.ErrTopicNotExisted,
`queue stat`,
)
}
if len(parts) == 2 {
lineName = parts[1]
return t.statLine(lineName)
}
qs := t.stat()
return qs, nil
}
func (u *UnitedQueue) remove(key string, fromEtcd bool) error {
key = strings.TrimPrefix(key, "/")
key = strings.TrimSuffix(key, "/")
var topicName, lineName string
parts := strings.Split(key, "/")
if len(parts) < 1 || len(parts) > 2 {
return utils.NewError(
utils.ErrBadKey,
`remove key parts error: `+utils.ItoaQuick(len(parts)),
)
}
topicName = parts[0]
if topicName == "" {
return utils.NewError(
utils.ErrBadKey,
`rmove topic is nil`,
)
}
if len(parts) == 1 {
return u.removeTopic(topicName, fromEtcd)
}
u.topicsLock.RLock()
t, ok := u.topics[topicName]
u.topicsLock.RUnlock()
if !ok {
return utils.NewError(
utils.ErrTopicNotExisted,
`queue remove`,
)
}
lineName = parts[1]
return t.removeLine(lineName, fromEtcd)
}
// Push implements Push interface
func (u *UnitedQueue) Push(key string, data []byte) error {
key = strings.TrimPrefix(key, "/")
key = strings.TrimSuffix(key, "/")
if len(data) <= 0 {
return utils.NewError(
utils.ErrBadRequest,
`message has no content`,
)
}
u.topicsLock.RLock()
t, ok := u.topics[key]
u.topicsLock.RUnlock()
if !ok {
return utils.NewError(
utils.ErrTopicNotExisted,
`queue push`,
)
}
return t.push(data)
}
func (t *topic) emptyLine(name string) error {
t.linesLock.RLock()
l, ok := t.lines[name]
t.linesLock.RUnlock()
if !ok {
// log.Printf("topic[%s] line[%s] not existed.", t.name, name)
return utils.NewError(
utils.ErrLineNotExisted,
`topic emptyLine`,
)
}
return l.empty()
}
func (t *topic) confirm(name string, id uint64) error {
t.linesLock.RLock()
l, ok := t.lines[name]
t.linesLock.RUnlock()
if !ok {
// log.Printf("topic[%s] line[%s] not existed.", t.name, name)
return utils.NewError(
utils.ErrLineNotExisted,
`topic confirm`,
)
}
return l.confirm(id)
}
func (t *topic) mPop(name string, n int) ([]uint64, [][]byte, error) {
t.linesLock.RLock()
l, ok := t.lines[name]
t.linesLock.RUnlock()
if !ok {
// log.Printf("topic[%s] line[%s] not existed.", t.name, name)
return nil, nil, utils.NewError(
utils.ErrLineNotExisted,
`topic mPop`,
)
}
return l.mPop(n)
}
func (l *line) confirm(id uint64) error {
if l.recycle == 0 {
return utils.NewError(
utils.ErrNotDelivered,
`line confirm`,
)
}
l.headLock.RLock()
defer l.headLock.RUnlock()
head := l.head
if id >= head {
return utils.NewError(
utils.ErrNotDelivered,
`line confirm`,
)
}
l.inflightLock.Lock()
defer l.inflightLock.Unlock()
for m := l.inflight.Front(); m != nil; m = m.Next() {
msg := m.Value.(*inflightMessage)
if msg.Tid == id {
l.inflight.Remove(m)
// log.Printf("key[%s/%s/%d] comfirmed.", l.t.name, l.name, id)
l.imap[id] = false
l.updateiHead()
return nil
}
}
return utils.NewError(
utils.ErrNotDelivered,
`line confirm`,
)
}
// MultiPop implements MultiPop interface
func (u *UnitedQueue) MultiPop(key string, n int) ([]string, [][]byte, error) {
key = strings.TrimPrefix(key, "/")
key = strings.TrimSuffix(key, "/")
parts := strings.Split(key, "/")
if len(parts) != 2 {
return nil, nil, utils.NewError(
utils.ErrBadKey,
`mPop key parts error: `+utils.ItoaQuick(len(parts)),
)
}
tName := parts[0]
lName := parts[1]
u.topicsLock.RLock()
t, ok := u.topics[tName]
u.topicsLock.RUnlock()
if !ok {
// log.Printf("topic[%s] not existed.", tName)
return nil, nil, utils.NewError(
utils.ErrTopicNotExisted,
`queue multiPop`,
)
}
ids, datas, err := t.mPop(lName, n)
if err != nil {
return nil, nil, err
}
keys := make([]string, len(ids))
for i, id := range ids {
keys[i] = utils.Acatui(key, "/", id)
}
return keys, datas, nil
}
func (t *topic) statLine(name string) (*Stat, error) {
t.linesLock.RLock()
l, ok := t.lines[name]
t.linesLock.RUnlock()
if !ok {
// log.Printf("topic[%s] line[%s] not existed.", t.name, name)
return nil, utils.NewError(
utils.ErrLineNotExisted,
`topic statLine`,
)
}
qs := l.stat()
return qs, nil
}
func (r *RedisEntry) onQpush(cmd *command) *reply {
key := cmd.stringAtIndex(1)
val, err := cmd.argAtIndex(2)
if err != nil {
return errorReply(utils.NewError(
utils.ErrBadRequest,
err.Error(),
))
}
err = r.messageQueue.Push(key, val)
if err != nil {
return errorReply(err)
}
return statusReply("OK")
}
func (t *topic) exportTopic() error {
ts := t.genTopicStore()
buf, err := ts.Marshal()
if err != nil {
return utils.NewError(
utils.ErrInternalError,
err.Error(),
)
}
err = t.q.setData(t.name, buf)
if err != nil {
return err
}
// log.Printf("topic[%s] export finisded.", t.name)
return nil
}
func (h *HTTPEntry) pushHandler(w http.ResponseWriter, req *http.Request, key string) {
err := req.ParseForm()
if err != nil {
writeErrorHTTP(w, utils.NewError(
utils.ErrInternalError,
err.Error(),
))
return
}
data := []byte(req.FormValue("value"))
err = h.messageQueue.Push(key, data)
if err != nil {
writeErrorHTTP(w, err)
return
}
w.WriteHeader(http.StatusNoContent)
}
func (u *UnitedQueue) exportQueue() error {
// log.Printf("start export queue...")
qs := u.genQueueStore()
buf, err := qs.Marshal()
if err != nil {
return utils.NewError(
utils.ErrInternalError,
err.Error(),
)
}
err = u.setData(storageKeyWord, buf)
if err != nil {
return err
}
// log.Printf("united queue export finisded.")
return nil
}
func (l *line) exportLine() error {
// log.Printf("start export line[%s]...", l.name)
ls := l.genLineStore()
buf, err := ls.Marshal()
if err != nil {
return utils.NewError(
utils.ErrInternalError,
err.Error(),
)
}
lineStoreKey := l.t.name + "/" + l.name
err = l.t.q.setData(lineStoreKey, buf)
if err != nil {
return err
}
// log.Printf("line[%s] export finisded.", l.name)
return nil
}
func (t *topic) exportTopic() error {
topicStoreValue := t.genTopicStore()
buffer := bytes.NewBuffer(nil)
enc := gob.NewEncoder(buffer)
err := enc.Encode(topicStoreValue)
if err != nil {
return utils.NewError(
utils.ErrInternalError,
err.Error(),
)
}
err = t.q.setData(t.name, buffer.Bytes())
if err != nil {
return err
}
// log.Printf("topic[%s] export finisded.", t.name)
return nil
}