func NewStore(dir string) (*store, error) {
_ = os.MkdirAll(dir, 0777)
lfd, err := os.OpenFile(
path.Join(dir, "log"),
os.O_RDWR|os.O_CREATE|os.O_APPEND,
0666)
if err != nil {
return nil, errwrap.Err(err, "cannot open log file")
}
xfd, err := os.OpenFile(
path.Join(dir, "index"),
os.O_RDWR|os.O_CREATE|os.O_APPEND,
0666)
if err != nil {
_ = lfd.Close()
return nil, errwrap.Err(err, "cannot open index file")
}
idx, err := newIndex(xfd)
if err != nil {
_ = lfd.Close()
_ = xfd.Close()
return nil, errwrap.Err(err, "cannot create index")
}
s := &store{
log: lfd,
index: idx,
}
return s, nil
}
func (s *store) Append(msgs [][]byte) (uint64, error) {
s.mu.Lock()
defer s.mu.Unlock()
offset, err := s.log.Seek(0, os.SEEK_END)
if err != nil {
return 0, errwrap.Err(err, "cannot seek")
}
tick := uint64(time.Now().UnixNano())
msx := make([]int64, len(msgs))
mx := uint64(s.index.Count())
for i, b := range msgs {
if err := proto.SetMsgAttrs(b, 0, 0, tick, mx); err != nil {
return 0, errwrap.Err(err, "cannot set message attributes")
}
if n, err := s.log.Write(b); err != nil {
return 0, errwrap.Err(err, "cannot write to backend")
} else if n != len(b) {
return 0, errors.New("cannot write full message to backend")
}
mx += 1
msx[i] = offset
offset += int64(len(b))
}
if err := s.index.Append(msx...); err != nil {
// XXX seek messages back
return 0, errwrap.Err(err, "cannot write to index")
}
return mx, nil
}
func handleReqTopics(r io.Reader, w io.Writer, pool *topicPool) error {
dec := proto.NewDecoder(r)
cid := dec.DecodeUint32()
topics := pool.Topics()
var b bytes.Buffer
enc := proto.NewEncoder(&b)
enc.EncodeUint8(proto.TopicsResp)
enc.EncodeUint32(cid)
enc.EncodeArrayLen(len(topics))
for _, t := range topics {
enc.EncodeShortBytes([]byte(t.Name))
enc.EncodeUint64(t.Offset)
}
if err := enc.Err(); err != nil {
_ = writeErrResp(w, cid, "cannot encode response")
return errwrap.Err(err, "cannot encode response")
}
if _, err := b.WriteTo(w); err != nil {
_ = writeErrResp(w, cid, "cannot write response")
return errwrap.Err(err, "cannot write response")
}
return nil
}
func writeErrResp(w io.Writer, corelationID uint32, text string) error {
var b bytes.Buffer
enc := proto.NewEncoder(&b)
enc.EncodeUint8(proto.ErrResp)
enc.EncodeUint32(corelationID)
enc.EncodeBytes([]byte(text))
if err := enc.Err(); err != nil {
return errwrap.Err(err, "cannot encode response")
}
if _, err := b.WriteTo(w); err != nil {
return errwrap.Err(err, "cannot write response")
}
return nil
}
func (s *Server) ServeWith(ln net.Listener) error {
for {
c, err := ln.Accept()
if err != nil {
return errwrap.Err(err, "cannot accept connection")
}
go s.handleClient(c)
}
}
// Offset return store offset for message with given index
func (x *storeindex) Offset(msgindex uint64) (int64, error) {
if msgindex > uint64(x.count) {
return 0, ErrOutOfScope
}
if _, err := x.rw.Seek(int64(msgindex)*8, os.SEEK_SET); err != nil {
return 0, errwrap.Err(err, "cannot seek index")
}
var b [8]byte
n, err := x.rw.Read(b[:])
if err != nil {
if err == io.EOF {
return 0, ErrOutOfScope
}
return 0, errwrap.Err(err, "cannot read offset")
}
if n != len(b) {
return 0, fmt.Errorf("not enough data: %d", n)
}
return int64(order.Uint64(b[:])), nil
}
func newIndex(rw io.ReadWriteSeeker) (*storeindex, error) {
off, err := rw.Seek(0, os.SEEK_END)
if err != nil {
return nil, errwrap.Err(err, "cannot seek")
}
if off%8 != 0 {
return nil, fmt.Errorf("corrupted index store: %d", off)
}
x := &storeindex{
count: uint64(off / 8),
rw: rw,
}
return x, nil
}
func NewServer(dir string) (*Server, error) {
s := &Server{
topics: newTopicPool(dir),
}
// scan dir, searching for existing topics
dirs, err := ioutil.ReadDir(dir)
if err != nil {
return nil, errwrap.Err(err, "cannot read %s", dir)
}
for _, d := range dirs {
if !d.IsDir() {
continue
}
ddir := path.Join(dir, d.Name())
files, err := ioutil.ReadDir(ddir)
if err != nil {
return nil, errwrap.Err(err, "cannot read: %s", ddir)
}
var found int
for _, f := range files {
if n := f.Name(); n == "log" || n == "index" {
found++
}
}
if found != 2 {
continue
}
// both 'log' and 'index' files exist - it's topic
if store, err := NewStore(ddir); err != nil {
return nil, errwrap.Err(err, "cannot create store: %s", ddir)
} else {
s.topics.topics[d.Name()] = store
}
}
return s, nil
}
func (pool *topicPool) Topic(name string) (Store, error) {
pool.mu.Lock()
defer pool.mu.Unlock()
s, ok := pool.topics[name]
if !ok {
if st, err := NewStore(path.Join(pool.dir, name)); err != nil {
return nil, errwrap.Err(err, "cannot create store")
} else {
s = st
}
pool.topics[name] = s
}
return s, nil
}
func handleReqSlice(r io.Reader, w io.Writer, pool *topicPool) error {
dec := proto.NewDecoder(r)
cid := dec.DecodeUint32()
topic := dec.DecodeShortBytes()
offset := dec.DecodeUint64()
limit := dec.DecodeUint16()
timeout := time.Duration(int64(dec.DecodeUint16())) * time.Millisecond
if err := dec.Err(); err != nil {
_ = writeErrResp(w, cid, "cannot decode request")
return errwrap.Err(err, "cannot decode request")
}
if len(topic) == 0 {
_ = writeErrResp(w, cid, "topic name not given")
return nil
}
s, err := pool.Topic(string(topic))
if err != nil {
_ = writeErrResp(w, cid, "topic not available")
return errwrap.Err(err, "cannot decode")
}
msgs, err := s.Slice(offset, uint64(limit), timeout)
if err != nil {
_ = writeErrResp(w, cid, "cannot slice")
return errwrap.Err(err, "cannot slice")
}
var b bytes.Buffer
enc := proto.NewEncoder(&b)
enc.EncodeUint8(proto.SliceResp)
enc.EncodeUint32(cid)
enc.EncodeArrayLen(len(msgs))
if err := enc.Err(); err != nil {
_ = writeErrResp(w, cid, "cannot encode response")
return errwrap.Err(err, "cannot encode response")
}
if _, err := b.WriteTo(w); err != nil {
_ = writeErrResp(w, cid, "cannot write response")
return errwrap.Err(err, "cannot write response")
}
for _, b := range msgs {
if n, err := w.Write(b); err != nil {
_ = writeErrResp(w, cid, "cannot write response")
return errwrap.Err(err, "cannot write response")
} else if n != len(b) {
_ = writeErrResp(w, cid, "cannot write response")
return errors.New("cannot write full response")
}
}
return nil
}
func handleReqAppend(r io.Reader, w io.Writer, pool *topicPool) error {
dec := proto.NewDecoder(r)
cid := dec.DecodeUint32()
topic := dec.DecodeShortBytes()
msgs := make([][]byte, dec.DecodeArrayLen())
if err := dec.Err(); err != nil {
_ = writeErrResp(w, cid, "cannot decode request")
return errwrap.Err(err, "cannot decode")
}
if len(topic) == 0 {
_ = writeErrResp(w, cid, "topic name not given")
return nil
}
if len(msgs) == 0 {
return writeErrResp(w, cid, "no messages")
}
var mrd proto.MsgReader
for i := range msgs {
b, err := mrd.ReadBytes(r)
if err != nil {
_ = writeErrResp(w, cid, "cannot read message")
return errwrap.Err(err, "cannot read message")
}
msgs[i] = b
}
s, err := pool.Topic(string(topic))
if err != nil {
_ = writeErrResp(w, cid, "topic not available")
return errwrap.Err(err, "cannot decode")
}
index, err := s.Append(msgs)
if err != nil {
_ = writeErrResp(w, cid, "cannot store messages")
return errwrap.Err(err, "storage error: cannot append")
}
var b bytes.Buffer
enc := proto.NewEncoder(&b)
enc.EncodeUint8(proto.AppendResp)
enc.EncodeUint32(cid)
enc.EncodeUint64(index)
if err := enc.Err(); err != nil {
_ = writeErrResp(w, cid, "cannot encode response")
return errwrap.Err(err, "cannot encode response")
}
if _, err := b.WriteTo(w); err != nil {
_ = writeErrResp(w, cid, "cannot write response")
return errwrap.Err(err, "cannot write response")
}
return nil
}
// Append appends to index message storage offsets.
func (x *storeindex) Append(msgoffsets ...int64) error {
b := make([]byte, 8*len(msgoffsets))
for i, off := range msgoffsets {
if off < 0 {
return fmt.Errorf("offset must be positive value")
}
order.PutUint64(b[i*8:(i+1)*8], uint64(off))
}
n, err := x.rw.Write(b)
if err != nil {
return errwrap.Err(err, "cannot write to index")
}
if n != len(b) {
return fmt.Errorf("expected to write %d, wrote %d", len(b), n)
}
x.count += uint64(len(msgoffsets))
return nil
}
// Slicing with offset == number of messages will block for max timeout before
// returning response. If during wait time any new message was written to
// topic, it will be returned. Otherwise if timeout was reached and no new
// messages was published, timeout error is returned.
// Slicing with offset > number of messages instantly returns out of bound
// error.
func (s *store) Slice(offset, limit uint64, timeout time.Duration) ([][]byte, error) {
if limit == 0 {
return nil, nil
}
s.mu.RLock()
amount := s.index.Count()
s.mu.RUnlock()
// TODO use notification instead of busy looping
deadline := time.After(timeout)
waitMessages:
for {
switch {
case offset == amount:
select {
case <-deadline:
return nil, errors.New("no new messages")
case <-time.After(10 * time.Millisecond):
s.mu.RLock()
amount = s.index.Count()
s.mu.RUnlock()
}
case offset > amount:
return nil, errors.New("out of bound")
default:
break waitMessages
}
}
s.mu.RLock()
defer s.mu.RUnlock()
if offset+limit > amount {
limit = amount - offset
}
if pos, err := s.index.Offset(offset); err != nil {
return nil, errwrap.Err(err, "cannot read index")
} else {
if _, err := s.log.Seek(pos, os.SEEK_SET); err != nil {
return nil, errwrap.Err(err, "cannot seek")
}
}
res := make([][]byte, 0, limit)
var rd proto.MsgReader
for i := 0; i < int(limit); i++ {
b, err := rd.ReadBytes(s.log)
switch err {
case nil:
case proto.ErrNotEnoughData, io.EOF:
return res, nil
default:
return res, err
}
res = append(res, b)
}
return res, nil
}
