Пример #1
0
// MultiPush implements MultiPush interface
func (u *UnitedQueue) MultiPush(key string, datas [][]byte) error {
	key = strings.TrimPrefix(key, "/")
	key = strings.TrimSuffix(key, "/")

	for i, data := range datas {
		if len(data) <= 0 {
			cause := "message " + strconv.Itoa(i) + " has no content"
			return utils.NewError(
				utils.ErrBadRequest,
				cause,
			)
		}
	}

	u.topicsLock.RLock()
	t, ok := u.topics[key]
	u.topicsLock.RUnlock()
	if !ok {
		return utils.NewError(
			utils.ErrTopicNotExisted,
			`queue multiPush`,
		)
	}

	return t.mPush(datas)
}
Пример #2
0
func (u *UnitedQueue) create(key, rec string, fromEtcd bool) error {
	key = strings.TrimPrefix(key, "/")
	key = strings.TrimSuffix(key, "/")

	var topicName, lineName string
	var err error
	parts := strings.Split(key, "/")
	if len(parts) < 1 || len(parts) > 2 {
		return utils.NewError(
			utils.ErrBadKey,
			`create key parts error: `+utils.ItoaQuick(len(parts)),
		)
	}

	topicName = parts[0]
	if topicName == "" {
		return utils.NewError(
			utils.ErrBadKey,
			`create topic is nil`,
		)
	}

	if len(parts) == 2 {
		lineName = parts[1]
		var recycle time.Duration
		if rec != "" {
			recycle, err = time.ParseDuration(rec)
			if err != nil {
				return utils.NewError(
					utils.ErrBadRequest,
					err.Error(),
				)
			}
		}

		u.topicsLock.RLock()
		t, ok := u.topics[topicName]
		u.topicsLock.RUnlock()
		if !ok {
			return utils.NewError(
				utils.ErrTopicNotExisted,
				`queue create`,
			)
		}

		err = t.createLine(lineName, recycle, fromEtcd)
		if err != nil {
			// log.Printf("create line[%s] error: %s", lineName, err)
			return err
		}
	} else {
		err = u.createTopic(topicName, fromEtcd)
		if err != nil {
			// log.Printf("create topic[%s] error: %s", topicName, err)
			return err
		}
	}

	return err
}
Пример #3
0
// Confirm implements Confirm interface
func (u *UnitedQueue) Confirm(key string) error {
	key = strings.TrimPrefix(key, "/")
	key = strings.TrimSuffix(key, "/")

	parts := strings.Split(key, "/")
	if len(parts) != 3 {
		return utils.NewError(
			utils.ErrBadKey,
			`confirm key parts error: `+utils.ItoaQuick(len(parts)),
		)
	}
	topicName := parts[0]
	lineName := parts[1]
	id, err := strconv.ParseUint(parts[2], 10, 0)
	if err != nil {
		return utils.NewError(
			utils.ErrBadKey,
			`confirm key parse id error: `+err.Error(),
		)
	}

	u.topicsLock.RLock()
	t, ok := u.topics[topicName]
	u.topicsLock.RUnlock()
	if !ok {
		// log.Printf("topic[%s] not existed.", topicName)
		return utils.NewError(
			utils.ErrTopicNotExisted,
			`queue confirm`,
		)
	}

	return t.confirm(lineName, id)
}
Пример #4
0
// Pop implements Pop interface
func (u *UnitedQueue) Pop(key string) (string, []byte, error) {
	key = strings.TrimPrefix(key, "/")
	key = strings.TrimSuffix(key, "/")

	parts := strings.Split(key, "/")
	if len(parts) != 2 {
		return "", nil, utils.NewError(
			utils.ErrBadKey,
			`pop key parts error: `+utils.ItoaQuick(len(parts)),
		)
	}

	tName := parts[0]
	lName := parts[1]

	u.topicsLock.RLock()
	t, ok := u.topics[tName]
	u.topicsLock.RUnlock()
	if !ok {
		// log.Printf("topic[%s] not existed.", tName)
		return "", nil, utils.NewError(
			utils.ErrTopicNotExisted,
			`queue pop`,
		)
	}

	id, data, err := t.pop(lName)
	if err != nil {
		return "", nil, err
	}

	return utils.Acatui(key, "/", id), data, nil
}
Пример #5
0
func (u *UnitedQueue) createTopic(name string, fromEtcd bool) error {
	u.topicsLock.RLock()
	_, ok := u.topics[name]
	u.topicsLock.RUnlock()
	if ok {
		return utils.NewError(
			utils.ErrTopicExisted,
			`queue createTopic`,
		)
	}

	t, err := u.newTopic(name)
	if err != nil {
		return err
	}

	u.topicsLock.Lock()
	defer u.topicsLock.Unlock()
	u.topics[name] = t

	err = u.exportQueue()
	if err != nil {
		t.remove()
		delete(u.topics, name)
		return err
	}

	if !fromEtcd {
		u.registerTopic(t.name)
	}
	log.Printf("topic[%s] created.", name)
	return nil
}
Пример #6
0
func (r *RedisEntry) onQmpop(cmd *command) *reply {
	key := cmd.stringAtIndex(1)
	n, err := cmd.intAtIndex(2)
	if err != nil {
		return errorReply(utils.NewError(
			utils.ErrBadRequest,
			err.Error(),
		))
	}

	ids, values, err := r.messageQueue.MultiPop(key, n)
	if err != nil {
		return errorReply(err)
	}

	np := len(ids)

	vals := make([]interface{}, np*2)
	for i, index := 0, 0; i < np; i++ {
		vals[index] = values[i]
		index++

		vals[index] = ids[i]
		index++
	}

	return multiBulksReply(vals)
}
Пример #7
0
func (s *UnitedAdmin) statHandler(w http.ResponseWriter, req *http.Request, key string) {
	if req.Method != "GET" {
		http.Error(w, "405 Method Not Allowed!", http.StatusMethodNotAllowed)
		return
	}

	qs, err := s.messageQueue.Stat(key)
	if err != nil {
		writeErrorHTTP(w, err)
		return
	}

	// log.Printf("qs: %v", qs)
	data, err := qs.ToJSON()
	if err != nil {
		writeErrorHTTP(w, utils.NewError(
			utils.ErrInternalError,
			err.Error(),
		))
		return
	}

	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusOK)
	w.Write(data)
}
Пример #8
0
func (u *UnitedQueue) removeTopic(name string, fromEtcd bool) error {
	u.topicsLock.Lock()
	defer u.topicsLock.Unlock()

	t, ok := u.topics[name]
	if !ok {
		return utils.NewError(
			utils.ErrTopicNotExisted,
			`queue remove`,
		)
	}

	delete(u.topics, name)
	err := u.exportQueue()
	if err != nil {
		u.topics[name] = t
		return err
	}

	if !fromEtcd {
		u.unRegisterTopic(name)
	}

	return t.remove()
}
Пример #9
0
func (t *topic) createLine(name string, recycle time.Duration, fromEtcd bool) error {
	t.linesLock.Lock()
	defer t.linesLock.Unlock()
	_, ok := t.lines[name]
	if ok {
		return utils.NewError(
			utils.ErrLineExisted,
			`topic createLine`,
		)
	}

	l, err := t.newLine(name, recycle)
	if err != nil {
		return err
	}

	t.lines[name] = l

	err = t.exportTopic()
	if err != nil {
		t.linesLock.Lock()
		delete(t.lines, name)
		t.linesLock.Unlock()
		return err
	}

	if !fromEtcd {
		t.q.registerLine(t.name, l.name, l.recycle.String())
	}

	log.Printf("topic[%s] line[%s:%v] created.", t.name, name, recycle)
	return nil
}
Пример #10
0
func (t *topic) removeLine(name string, fromEtcd bool) error {
	t.linesLock.RLock()
	defer t.linesLock.RUnlock()
	l, ok := t.lines[name]
	if !ok {
		// log.Printf("topic[%s] line[%s] not existed.", t.name, name)
		return utils.NewError(
			utils.ErrLineNotExisted,
			`topic statLine`,
		)
	}

	delete(t.lines, name)
	err := t.exportTopic()
	if err != nil {
		t.linesLock.Lock()
		t.lines[name] = l
		t.linesLock.Unlock()
		return err
	}

	if !fromEtcd {
		t.q.unRegisterLine(t.name, name)
	}

	return l.remove()
}
Пример #11
0
func (l *line) pop() (uint64, []byte, error) {
	l.inflightLock.Lock()
	defer l.inflightLock.Unlock()

	now := time.Now()
	if l.recycle > 0 {

		m := l.inflight.Front()
		if m != nil {
			msg := m.Value.(*InflightMessage)
			exp := time.Unix(0, msg.Exptime)
			if now.After(exp) {
				// log.Printf("key[%s/%d] is expired.", l.name, msg.Tid)
				msg.Exptime = now.Add(l.recycle).UnixNano()
				data, err := l.t.getData(msg.Tid)
				if err != nil {
					return 0, nil, err
				}
				l.inflight.Remove(m)
				l.inflight.PushBack(msg)
				// log.Printf("key[%s/%s/%d] poped.", l.t.name, l.name, msg.Tid)
				return msg.Tid, data, nil
			}
		}
	}

	l.headLock.Lock()
	defer l.headLock.Unlock()
	tid := l.head

	topicTail := l.t.getTail()
	if l.head >= topicTail {
		// log.Printf("line[%s] is blank. head:%d - tail:%d", l.name, l.head, l.t.tail)
		return 0, nil, utils.NewError(
			utils.ErrNone,
			`line pop`,
		)
	}

	data, err := l.t.getData(tid)
	if err != nil {
		return 0, nil, err
	}

	l.head++

	if l.recycle > 0 {
		msg := new(InflightMessage)
		msg.Tid = tid
		msg.Exptime = now.Add(l.recycle).UnixNano()

		l.inflight.PushBack(msg)
		// log.Printf("key[%s/%s/%d] flighted.", l.t.name, l.name, l.head)
		l.imap[tid] = true
	}

	return tid, data, nil
}
Пример #12
0
// Empty implements Empty interface
func (u *UnitedQueue) Empty(key string) error {
	key = strings.TrimPrefix(key, "/")
	key = strings.TrimSuffix(key, "/")

	var topicName, lineName string
	parts := strings.Split(key, "/")
	if len(parts) < 1 || len(parts) > 2 {
		return utils.NewError(
			utils.ErrBadKey,
			`empty key parts error: `+utils.ItoaQuick(len(parts)),
		)
	}

	topicName = parts[0]

	if topicName == "" {
		return utils.NewError(
			utils.ErrBadKey,
			`empty topic is nil`,
		)
	}

	u.topicsLock.RLock()
	t, ok := u.topics[topicName]
	u.topicsLock.RUnlock()
	if !ok {
		return utils.NewError(
			utils.ErrTopicNotExisted,
			`queue empty`,
		)
	}

	if len(parts) == 2 {
		lineName = parts[1]
		return t.emptyLine(lineName)
		// err = t.emptyLine(lineName)
		// if err != nil {
		// 	log.Printf("empty line[%s] error: %s", lineName, err)
		// }
		// return err
	}

	return t.empty()
}
Пример #13
0
func (u *UnitedQueue) delData(key string) error {
	err := u.storage.Del(key)
	if err != nil {
		// log.Printf("key[%s] del data error: %s", key, err)
		return utils.NewError(
			utils.ErrInternalError,
			err.Error(),
		)
	}
	return nil
}
Пример #14
0
func (u *UnitedQueue) setData(key string, data []byte) error {
	err := u.storage.Set(key, data)
	if err != nil {
		// log.Printf("key[%s] set data error: %s", key, err)
		return utils.NewError(
			utils.ErrInternalError,
			err.Error(),
		)
	}
	return nil
}
Пример #15
0
func (u *UnitedQueue) getData(key string) ([]byte, error) {
	data, err := u.storage.Get(key)
	if err != nil {
		// log.Printf("key[%s] get data error: %s", key, err)
		return nil, utils.NewError(
			utils.ErrInternalError,
			err.Error(),
		)
	}
	return data, nil
}
Пример #16
0
// Stat implements Stat interface
func (u *UnitedQueue) Stat(key string) (*Stat, error) {
	key = strings.TrimPrefix(key, "/")
	key = strings.TrimSuffix(key, "/")

	var topicName, lineName string
	parts := strings.Split(key, "/")
	if len(parts) < 1 || len(parts) > 2 {
		return nil, utils.NewError(
			utils.ErrBadKey,
			`empty key parts error: `+utils.ItoaQuick(len(parts)),
		)
	}

	topicName = parts[0]
	if topicName == "" {
		return nil, utils.NewError(
			utils.ErrBadKey,
			`stat topic is nil`,
		)
	}

	u.topicsLock.RLock()
	t, ok := u.topics[topicName]
	u.topicsLock.RUnlock()
	if !ok {
		return nil, utils.NewError(
			utils.ErrTopicNotExisted,
			`queue stat`,
		)
	}

	if len(parts) == 2 {
		lineName = parts[1]
		return t.statLine(lineName)
	}

	qs := t.stat()
	return qs, nil
}
Пример #17
0
func (u *UnitedQueue) remove(key string, fromEtcd bool) error {
	key = strings.TrimPrefix(key, "/")
	key = strings.TrimSuffix(key, "/")

	var topicName, lineName string
	parts := strings.Split(key, "/")
	if len(parts) < 1 || len(parts) > 2 {
		return utils.NewError(
			utils.ErrBadKey,
			`remove key parts error: `+utils.ItoaQuick(len(parts)),
		)
	}

	topicName = parts[0]
	if topicName == "" {
		return utils.NewError(
			utils.ErrBadKey,
			`rmove topic is nil`,
		)
	}

	if len(parts) == 1 {
		return u.removeTopic(topicName, fromEtcd)
	}

	u.topicsLock.RLock()
	t, ok := u.topics[topicName]
	u.topicsLock.RUnlock()
	if !ok {
		return utils.NewError(
			utils.ErrTopicNotExisted,
			`queue remove`,
		)
	}

	lineName = parts[1]
	return t.removeLine(lineName, fromEtcd)
}
Пример #18
0
// Push implements Push interface
func (u *UnitedQueue) Push(key string, data []byte) error {
	key = strings.TrimPrefix(key, "/")
	key = strings.TrimSuffix(key, "/")

	if len(data) <= 0 {
		return utils.NewError(
			utils.ErrBadRequest,
			`message has no content`,
		)
	}

	u.topicsLock.RLock()
	t, ok := u.topics[key]
	u.topicsLock.RUnlock()
	if !ok {
		return utils.NewError(
			utils.ErrTopicNotExisted,
			`queue push`,
		)
	}

	return t.push(data)
}
Пример #19
0
func (t *topic) emptyLine(name string) error {
	t.linesLock.RLock()
	l, ok := t.lines[name]
	t.linesLock.RUnlock()
	if !ok {
		// log.Printf("topic[%s] line[%s] not existed.", t.name, name)
		return utils.NewError(
			utils.ErrLineNotExisted,
			`topic emptyLine`,
		)
	}

	return l.empty()
}
Пример #20
0
func (t *topic) confirm(name string, id uint64) error {
	t.linesLock.RLock()
	l, ok := t.lines[name]
	t.linesLock.RUnlock()
	if !ok {
		// log.Printf("topic[%s] line[%s] not existed.", t.name, name)
		return utils.NewError(
			utils.ErrLineNotExisted,
			`topic confirm`,
		)
	}

	return l.confirm(id)
}
Пример #21
0
func (t *topic) mPop(name string, n int) ([]uint64, [][]byte, error) {
	t.linesLock.RLock()
	l, ok := t.lines[name]
	t.linesLock.RUnlock()
	if !ok {
		// log.Printf("topic[%s] line[%s] not existed.", t.name, name)
		return nil, nil, utils.NewError(
			utils.ErrLineNotExisted,
			`topic mPop`,
		)
	}

	return l.mPop(n)
}
Пример #22
0
Файл: uLine.go Проект: jnan77/uq
func (l *line) confirm(id uint64) error {
	if l.recycle == 0 {
		return utils.NewError(
			utils.ErrNotDelivered,
			`line confirm`,
		)
	}

	l.headLock.RLock()
	defer l.headLock.RUnlock()
	head := l.head
	if id >= head {
		return utils.NewError(
			utils.ErrNotDelivered,
			`line confirm`,
		)
	}

	l.inflightLock.Lock()
	defer l.inflightLock.Unlock()

	for m := l.inflight.Front(); m != nil; m = m.Next() {
		msg := m.Value.(*inflightMessage)
		if msg.Tid == id {
			l.inflight.Remove(m)
			// log.Printf("key[%s/%s/%d] comfirmed.", l.t.name, l.name, id)
			l.imap[id] = false
			l.updateiHead()
			return nil
		}
	}

	return utils.NewError(
		utils.ErrNotDelivered,
		`line confirm`,
	)
}
Пример #23
0
// MultiPop implements MultiPop interface
func (u *UnitedQueue) MultiPop(key string, n int) ([]string, [][]byte, error) {
	key = strings.TrimPrefix(key, "/")
	key = strings.TrimSuffix(key, "/")

	parts := strings.Split(key, "/")
	if len(parts) != 2 {
		return nil, nil, utils.NewError(
			utils.ErrBadKey,
			`mPop key parts error: `+utils.ItoaQuick(len(parts)),
		)
	}

	tName := parts[0]
	lName := parts[1]

	u.topicsLock.RLock()
	t, ok := u.topics[tName]
	u.topicsLock.RUnlock()
	if !ok {
		// log.Printf("topic[%s] not existed.", tName)
		return nil, nil, utils.NewError(
			utils.ErrTopicNotExisted,
			`queue multiPop`,
		)
	}

	ids, datas, err := t.mPop(lName, n)
	if err != nil {
		return nil, nil, err
	}

	keys := make([]string, len(ids))
	for i, id := range ids {
		keys[i] = utils.Acatui(key, "/", id)
	}
	return keys, datas, nil
}
Пример #24
0
func (t *topic) statLine(name string) (*Stat, error) {
	t.linesLock.RLock()
	l, ok := t.lines[name]
	t.linesLock.RUnlock()
	if !ok {
		// log.Printf("topic[%s] line[%s] not existed.", t.name, name)
		return nil, utils.NewError(
			utils.ErrLineNotExisted,
			`topic statLine`,
		)
	}

	qs := l.stat()
	return qs, nil
}
Пример #25
0
func (r *RedisEntry) onQpush(cmd *command) *reply {
	key := cmd.stringAtIndex(1)
	val, err := cmd.argAtIndex(2)
	if err != nil {
		return errorReply(utils.NewError(
			utils.ErrBadRequest,
			err.Error(),
		))
	}

	err = r.messageQueue.Push(key, val)
	if err != nil {
		return errorReply(err)
	}
	return statusReply("OK")
}
Пример #26
0
func (t *topic) exportTopic() error {
	ts := t.genTopicStore()
	buf, err := ts.Marshal()
	if err != nil {
		return utils.NewError(
			utils.ErrInternalError,
			err.Error(),
		)
	}

	err = t.q.setData(t.name, buf)
	if err != nil {
		return err
	}

	// log.Printf("topic[%s] export finisded.", t.name)
	return nil
}
Пример #27
0
func (h *HTTPEntry) pushHandler(w http.ResponseWriter, req *http.Request, key string) {
	err := req.ParseForm()
	if err != nil {
		writeErrorHTTP(w, utils.NewError(
			utils.ErrInternalError,
			err.Error(),
		))
		return
	}

	data := []byte(req.FormValue("value"))
	err = h.messageQueue.Push(key, data)
	if err != nil {
		writeErrorHTTP(w, err)
		return
	}
	w.WriteHeader(http.StatusNoContent)
}
Пример #28
0
func (u *UnitedQueue) exportQueue() error {
	// log.Printf("start export queue...")
	qs := u.genQueueStore()
	buf, err := qs.Marshal()
	if err != nil {
		return utils.NewError(
			utils.ErrInternalError,
			err.Error(),
		)
	}

	err = u.setData(storageKeyWord, buf)
	if err != nil {
		return err
	}

	// log.Printf("united queue export finisded.")
	return nil
}
Пример #29
0
func (l *line) exportLine() error {
	// log.Printf("start export line[%s]...", l.name)
	ls := l.genLineStore()
	buf, err := ls.Marshal()
	if err != nil {
		return utils.NewError(
			utils.ErrInternalError,
			err.Error(),
		)
	}

	lineStoreKey := l.t.name + "/" + l.name
	err = l.t.q.setData(lineStoreKey, buf)
	if err != nil {
		return err
	}

	// log.Printf("line[%s] export finisded.", l.name)
	return nil
}
Пример #30
0
func (t *topic) exportTopic() error {
	topicStoreValue := t.genTopicStore()

	buffer := bytes.NewBuffer(nil)
	enc := gob.NewEncoder(buffer)
	err := enc.Encode(topicStoreValue)
	if err != nil {
		return utils.NewError(
			utils.ErrInternalError,
			err.Error(),
		)
	}

	err = t.q.setData(t.name, buffer.Bytes())
	if err != nil {
		return err
	}

	// log.Printf("topic[%s] export finisded.", t.name)
	return nil
}