Example #1
0
File: store.go Project: husio/maze
func NewStore(dir string) (*store, error) {
	_ = os.MkdirAll(dir, 0777)

	lfd, err := os.OpenFile(
		path.Join(dir, "log"),
		os.O_RDWR|os.O_CREATE|os.O_APPEND,
		0666)
	if err != nil {
		return nil, errwrap.Err(err, "cannot open log file")
	}

	xfd, err := os.OpenFile(
		path.Join(dir, "index"),
		os.O_RDWR|os.O_CREATE|os.O_APPEND,
		0666)
	if err != nil {
		_ = lfd.Close()
		return nil, errwrap.Err(err, "cannot open index file")
	}

	idx, err := newIndex(xfd)
	if err != nil {
		_ = lfd.Close()
		_ = xfd.Close()
		return nil, errwrap.Err(err, "cannot create index")
	}

	s := &store{
		log:   lfd,
		index: idx,
	}
	return s, nil
}
Example #2
0
File: store.go Project: husio/maze
func (s *store) Append(msgs [][]byte) (uint64, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	offset, err := s.log.Seek(0, os.SEEK_END)
	if err != nil {
		return 0, errwrap.Err(err, "cannot seek")
	}

	tick := uint64(time.Now().UnixNano())
	msx := make([]int64, len(msgs))
	mx := uint64(s.index.Count())
	for i, b := range msgs {
		if err := proto.SetMsgAttrs(b, 0, 0, tick, mx); err != nil {
			return 0, errwrap.Err(err, "cannot set message attributes")
		}
		if n, err := s.log.Write(b); err != nil {
			return 0, errwrap.Err(err, "cannot write to backend")
		} else if n != len(b) {
			return 0, errors.New("cannot write full message to backend")
		}

		mx += 1
		msx[i] = offset
		offset += int64(len(b))
	}

	if err := s.index.Append(msx...); err != nil {
		// XXX seek messages back
		return 0, errwrap.Err(err, "cannot write to index")
	}

	return mx, nil
}
Example #3
0
func handleReqTopics(r io.Reader, w io.Writer, pool *topicPool) error {
	dec := proto.NewDecoder(r)
	cid := dec.DecodeUint32()

	topics := pool.Topics()

	var b bytes.Buffer
	enc := proto.NewEncoder(&b)
	enc.EncodeUint8(proto.TopicsResp)
	enc.EncodeUint32(cid)
	enc.EncodeArrayLen(len(topics))
	for _, t := range topics {
		enc.EncodeShortBytes([]byte(t.Name))
		enc.EncodeUint64(t.Offset)
	}
	if err := enc.Err(); err != nil {
		_ = writeErrResp(w, cid, "cannot encode response")
		return errwrap.Err(err, "cannot encode response")
	}
	if _, err := b.WriteTo(w); err != nil {
		_ = writeErrResp(w, cid, "cannot write response")
		return errwrap.Err(err, "cannot write response")
	}
	return nil
}
Example #4
0
func writeErrResp(w io.Writer, corelationID uint32, text string) error {
	var b bytes.Buffer
	enc := proto.NewEncoder(&b)
	enc.EncodeUint8(proto.ErrResp)
	enc.EncodeUint32(corelationID)
	enc.EncodeBytes([]byte(text))

	if err := enc.Err(); err != nil {
		return errwrap.Err(err, "cannot encode response")
	}
	if _, err := b.WriteTo(w); err != nil {
		return errwrap.Err(err, "cannot write response")
	}
	return nil
}
Example #5
0
File: server.go Project: husio/maze
func (s *Server) ServeWith(ln net.Listener) error {
	for {
		c, err := ln.Accept()
		if err != nil {
			return errwrap.Err(err, "cannot accept connection")
		}
		go s.handleClient(c)
	}
}
Example #6
0
File: index.go Project: husio/maze
// Offset return store offset for message with given index
func (x *storeindex) Offset(msgindex uint64) (int64, error) {
	if msgindex > uint64(x.count) {
		return 0, ErrOutOfScope
	}
	if _, err := x.rw.Seek(int64(msgindex)*8, os.SEEK_SET); err != nil {
		return 0, errwrap.Err(err, "cannot seek index")
	}
	var b [8]byte
	n, err := x.rw.Read(b[:])
	if err != nil {
		if err == io.EOF {
			return 0, ErrOutOfScope
		}
		return 0, errwrap.Err(err, "cannot read offset")
	}
	if n != len(b) {
		return 0, fmt.Errorf("not enough data: %d", n)
	}
	return int64(order.Uint64(b[:])), nil
}
Example #7
0
File: index.go Project: husio/maze
func newIndex(rw io.ReadWriteSeeker) (*storeindex, error) {
	off, err := rw.Seek(0, os.SEEK_END)
	if err != nil {
		return nil, errwrap.Err(err, "cannot seek")
	}
	if off%8 != 0 {
		return nil, fmt.Errorf("corrupted index store: %d", off)
	}
	x := &storeindex{
		count: uint64(off / 8),
		rw:    rw,
	}
	return x, nil
}
Example #8
0
File: server.go Project: husio/maze
func NewServer(dir string) (*Server, error) {
	s := &Server{
		topics: newTopicPool(dir),
	}
	// scan dir, searching for existing topics
	dirs, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, errwrap.Err(err, "cannot read %s", dir)
	}
	for _, d := range dirs {
		if !d.IsDir() {
			continue
		}
		ddir := path.Join(dir, d.Name())
		files, err := ioutil.ReadDir(ddir)
		if err != nil {
			return nil, errwrap.Err(err, "cannot read: %s", ddir)
		}
		var found int
		for _, f := range files {
			if n := f.Name(); n == "log" || n == "index" {
				found++
			}
		}
		if found != 2 {
			continue
		}
		// both 'log' and 'index' files exist - it's topic
		if store, err := NewStore(ddir); err != nil {
			return nil, errwrap.Err(err, "cannot create store: %s", ddir)
		} else {
			s.topics.topics[d.Name()] = store
		}
	}

	return s, nil
}
Example #9
0
File: server.go Project: husio/maze
func (pool *topicPool) Topic(name string) (Store, error) {
	pool.mu.Lock()
	defer pool.mu.Unlock()

	s, ok := pool.topics[name]
	if !ok {
		if st, err := NewStore(path.Join(pool.dir, name)); err != nil {
			return nil, errwrap.Err(err, "cannot create store")
		} else {
			s = st
		}
		pool.topics[name] = s
	}
	return s, nil
}
Example #10
0
func handleReqSlice(r io.Reader, w io.Writer, pool *topicPool) error {
	dec := proto.NewDecoder(r)
	cid := dec.DecodeUint32()
	topic := dec.DecodeShortBytes()
	offset := dec.DecodeUint64()
	limit := dec.DecodeUint16()
	timeout := time.Duration(int64(dec.DecodeUint16())) * time.Millisecond

	if err := dec.Err(); err != nil {
		_ = writeErrResp(w, cid, "cannot decode request")
		return errwrap.Err(err, "cannot decode request")
	}
	if len(topic) == 0 {
		_ = writeErrResp(w, cid, "topic name not given")
		return nil
	}

	s, err := pool.Topic(string(topic))
	if err != nil {
		_ = writeErrResp(w, cid, "topic not available")
		return errwrap.Err(err, "cannot decode")
	}

	msgs, err := s.Slice(offset, uint64(limit), timeout)
	if err != nil {
		_ = writeErrResp(w, cid, "cannot slice")
		return errwrap.Err(err, "cannot slice")
	}

	var b bytes.Buffer
	enc := proto.NewEncoder(&b)
	enc.EncodeUint8(proto.SliceResp)
	enc.EncodeUint32(cid)
	enc.EncodeArrayLen(len(msgs))
	if err := enc.Err(); err != nil {
		_ = writeErrResp(w, cid, "cannot encode response")
		return errwrap.Err(err, "cannot encode response")
	}
	if _, err := b.WriteTo(w); err != nil {
		_ = writeErrResp(w, cid, "cannot write response")
		return errwrap.Err(err, "cannot write response")
	}

	for _, b := range msgs {
		if n, err := w.Write(b); err != nil {
			_ = writeErrResp(w, cid, "cannot write response")
			return errwrap.Err(err, "cannot write response")
		} else if n != len(b) {
			_ = writeErrResp(w, cid, "cannot write response")
			return errors.New("cannot write full response")
		}
	}

	return nil
}
Example #11
0
func handleReqAppend(r io.Reader, w io.Writer, pool *topicPool) error {
	dec := proto.NewDecoder(r)
	cid := dec.DecodeUint32()
	topic := dec.DecodeShortBytes()
	msgs := make([][]byte, dec.DecodeArrayLen())
	if err := dec.Err(); err != nil {
		_ = writeErrResp(w, cid, "cannot decode request")
		return errwrap.Err(err, "cannot decode")
	}
	if len(topic) == 0 {
		_ = writeErrResp(w, cid, "topic name not given")
		return nil
	}
	if len(msgs) == 0 {
		return writeErrResp(w, cid, "no messages")
	}

	var mrd proto.MsgReader
	for i := range msgs {
		b, err := mrd.ReadBytes(r)
		if err != nil {
			_ = writeErrResp(w, cid, "cannot read message")
			return errwrap.Err(err, "cannot read message")
		}
		msgs[i] = b
	}

	s, err := pool.Topic(string(topic))
	if err != nil {
		_ = writeErrResp(w, cid, "topic not available")
		return errwrap.Err(err, "cannot decode")
	}
	index, err := s.Append(msgs)
	if err != nil {
		_ = writeErrResp(w, cid, "cannot store messages")
		return errwrap.Err(err, "storage error: cannot append")
	}

	var b bytes.Buffer
	enc := proto.NewEncoder(&b)
	enc.EncodeUint8(proto.AppendResp)
	enc.EncodeUint32(cid)
	enc.EncodeUint64(index)

	if err := enc.Err(); err != nil {
		_ = writeErrResp(w, cid, "cannot encode response")
		return errwrap.Err(err, "cannot encode response")
	}
	if _, err := b.WriteTo(w); err != nil {
		_ = writeErrResp(w, cid, "cannot write response")
		return errwrap.Err(err, "cannot write response")
	}
	return nil
}
Example #12
0
File: index.go Project: husio/maze
// Append appends to index message storage offsets.
func (x *storeindex) Append(msgoffsets ...int64) error {
	b := make([]byte, 8*len(msgoffsets))
	for i, off := range msgoffsets {
		if off < 0 {
			return fmt.Errorf("offset must be positive value")
		}
		order.PutUint64(b[i*8:(i+1)*8], uint64(off))
	}

	n, err := x.rw.Write(b)
	if err != nil {
		return errwrap.Err(err, "cannot write to index")
	}
	if n != len(b) {
		return fmt.Errorf("expected to write %d, wrote %d", len(b), n)
	}
	x.count += uint64(len(msgoffsets))
	return nil
}
Example #13
0
File: store.go Project: husio/maze
// Slicing with offset == number of messages will block for max timeout before
// returning response. If during wait time any new message was written to
// topic, it will be returned. Otherwise if timeout was reached and no new
// messages was published, timeout error is returned.
// Slicing with offset > number of messages instantly returns out of bound
// error.
func (s *store) Slice(offset, limit uint64, timeout time.Duration) ([][]byte, error) {
	if limit == 0 {
		return nil, nil
	}

	s.mu.RLock()
	amount := s.index.Count()
	s.mu.RUnlock()

	// TODO use notification instead of busy looping
	deadline := time.After(timeout)

waitMessages:
	for {
		switch {
		case offset == amount:
			select {
			case <-deadline:
				return nil, errors.New("no new messages")
			case <-time.After(10 * time.Millisecond):
				s.mu.RLock()
				amount = s.index.Count()
				s.mu.RUnlock()
			}
		case offset > amount:
			return nil, errors.New("out of bound")
		default:
			break waitMessages
		}
	}

	s.mu.RLock()
	defer s.mu.RUnlock()

	if offset+limit > amount {
		limit = amount - offset
	}

	if pos, err := s.index.Offset(offset); err != nil {
		return nil, errwrap.Err(err, "cannot read index")
	} else {
		if _, err := s.log.Seek(pos, os.SEEK_SET); err != nil {
			return nil, errwrap.Err(err, "cannot seek")
		}
	}

	res := make([][]byte, 0, limit)
	var rd proto.MsgReader
	for i := 0; i < int(limit); i++ {
		b, err := rd.ReadBytes(s.log)
		switch err {
		case nil:
		case proto.ErrNotEnoughData, io.EOF:
			return res, nil
		default:
			return res, err
		}
		res = append(res, b)
	}
	return res, nil
}