Exemplo n.º 1
0
func newTableReader(base string, index int64, useMmap bool) (*tableReader, error) {
	if index <= 0 {
		return nil, fmt.Errorf("invalid index %d", index)
	}
	t := new(tableReader)
	t.base = base
	t.index = index

	t.useMmap = useMmap

	var err error

	if err = t.check(); err != nil {
		log.Errorf("check %d error: %s, try to repair", t.index, err.Error())

		if err = t.repair(); err != nil {
			log.Errorf("repair %d error: %s", t.index, err.Error())
			return nil, err
		}
	}

	t.close()

	return t, nil
}
Exemplo n.º 2
0
func (s *FileStore) purgeTableReaders(purges []*tableReader) {
	for _, r := range purges {
		dataName := fmtTableDataName(r.base, r.index)
		metaName := fmtTableMetaName(r.base, r.index)
		r.Close()
		if err := os.Remove(dataName); err != nil {
			log.Errorf("purge table data %s err: %s", dataName, err.Error())
		}
		if err := os.Remove(metaName); err != nil {
			log.Errorf("purge table meta %s err: %s", metaName, err.Error())
		}

	}
}
Exemplo n.º 3
0
func (s *FileStore) load() error {
	fs, err := ioutil.ReadDir(s.base)
	if err != nil {
		return err
	}

	s.rs = make(tableReaders, 0, len(fs))

	var r *tableReader
	var index int64
	for _, f := range fs {
		if _, err := fmt.Sscanf(f.Name(), "%08d.data", &index); err == nil {
			if r, err = newTableReader(s.base, index, s.cfg.Replication.UseMmap); err != nil {
				log.Errorf("load table %s err: %s", f.Name(), err.Error())
			} else {
				s.rs = append(s.rs, r)
			}
		}
	}

	if err := s.rs.check(); err != nil {
		return err
	}

	return nil
}
Exemplo n.º 4
0
func (s *FileStore) Close() error {
	close(s.quit)

	s.wm.Lock()
	s.rm.Lock()

	if r, err := s.w.Flush(); err != nil {
		if err != errNilHandler {
			log.Errorf("close err: %s", err.Error())
		}
	} else {
		r.Close()
		s.w.Close()
	}

	for i := range s.rs {
		s.rs[i].Close()
	}

	s.rs = tableReaders{}

	s.rm.Unlock()
	s.wm.Unlock()

	return nil
}
Exemplo n.º 5
0
func (m *mmapReadFile) Close() error {
	if m.m != nil {
		if err := m.m.Unmap(); err != nil {
			log.Errorf("unmap %s error %s", m.name, err.Error())
		}
		m.m = nil
	}

	if m.f != nil {
		if err := m.f.Close(); err != nil {
			log.Errorf("close %s error %s", m.name, err.Error())
		}
		m.f = nil
	}

	return nil
}
Exemplo n.º 6
0
func (r *Replication) run() {
	defer r.wg.Done()

	syncTc := time.NewTicker(1 * time.Second)
	purgeTc := time.NewTicker(1 * time.Hour)

	for {
		select {
		case <-purgeTc.C:
			n := (r.cfg.Replication.ExpiredLogDays * 24 * 3600)
			r.m.Lock()
			err := r.s.PurgeExpired(int64(n))
			r.m.Unlock()
			if err != nil {
				log.Errorf("purge expired log error %s", err.Error())
			}
		case <-syncTc.C:
			if r.cfg.Replication.SyncLog == 1 {
				r.m.Lock()
				err := r.s.Sync()
				r.m.Unlock()
				if err != nil {
					log.Errorf("sync store error %s", err.Error())
				}
			}
			if r.cfg.Replication.SyncLog != 2 {
				//we will sync commit id every 1 second
				r.m.Lock()
				err := r.updateCommitID(r.commitID, true)
				r.m.Unlock()

				if err != nil {
					log.Errorf("sync commitid error %s", err.Error())
				}
			}
		case <-r.quit:
			syncTc.Stop()
			purgeTc.Stop()
			return
		}
	}
}
Exemplo n.º 7
0
func (t *tableWriter) storeLog(l *Log) error {
	if l.ID == 0 {
		return ErrStoreLogID
	}

	if t.closed {
		return fmt.Errorf("table writer is closed")
	}

	if t.last > 0 && l.ID != t.last+1 {
		return ErrStoreLogID
	}

	if t.data != nil && t.data.Offset() > t.maxLogSize {
		return errTableNeedFlush
	}

	var err error
	if err = t.openFile(); err != nil {
		return err
	}

	offsetPos := t.data.Offset()
	if err = l.Encode(t.data); err != nil {
		return err
	}

	binary.BigEndian.PutUint32(t.posBuf, uint32(offsetPos))
	if _, err = t.meta.Write(t.posBuf); err != nil {
		return err
	}

	if t.first == 0 {
		t.first = l.ID
	}

	t.last = l.ID
	t.lastTime = l.CreateTime

	if t.syncType == 2 {
		if err := t.data.Sync(); err != nil {
			log.Errorf("sync table error %s", err.Error())
		}
	}

	return nil
}
Exemplo n.º 8
0
func (l *Ledis) handleReplication() error {
	l.wLock.Lock()
	defer l.wLock.Unlock()

	defer AsyncNotify(l.rDoneCh)

	rl := &rpl.Log{}

	var err error
	for {
		if err = l.r.NextNeedCommitLog(rl); err != nil {
			if err != rpl.ErrNoBehindLog {
				log.Errorf("get next commit log err, %s", err.Error)
				return err
			} else {
				return nil
			}
		} else {
			l.rbatch.Rollback()

			if rl.Compression == 1 {
				//todo optimize
				if rl.Data, err = snappy.Decode(nil, rl.Data); err != nil {
					log.Errorf("decode log error %s", err.Error())
					return err
				}
			}

			if bd, err := store.NewBatchData(rl.Data); err != nil {
				log.Errorf("decode batch log error %s", err.Error())
				return err
			} else if err = bd.Replay(l.rbatch); err != nil {
				log.Errorf("replay batch log error %s", err.Error())
			}

			l.commitLock.Lock()
			if err = l.rbatch.Commit(); err != nil {
				log.Errorf("commit log error %s", err.Error())
			} else if err = l.r.UpdateCommitID(rl.ID); err != nil {
				log.Errorf("update commit id error %s", err.Error())
			}

			l.commitLock.Unlock()
			if err != nil {
				return err
			}
		}

	}
}
Exemplo n.º 9
0
func (r *Replication) Close() error {
	close(r.quit)

	r.wg.Wait()

	r.m.Lock()
	defer r.m.Unlock()

	if r.s != nil {
		r.s.Close()
		r.s = nil
	}

	if err := r.updateCommitID(r.commitID, true); err != nil {
		log.Errorf("update commit id err %s", err.Error())
	}

	if r.commitLog != nil {
		r.commitLog.Close()
		r.commitLog = nil
	}

	return nil
}
Exemplo n.º 10
0
func (t *tableReader) repair() error {
	t.close()

	var err error
	var data writeFile
	var meta writeFile

	//repair will use raw file mode
	data, err = newWriteFile(false, fmtTableDataName(t.base, t.index), 0)
	data.SetOffset(int64(data.Size()))

	meta, err = newWriteFile(false, fmtTableMetaName(t.base, t.index), int64(defaultLogNumInFile*4))

	var l Log
	var pos int64 = 0
	var nextPos int64 = 0
	b := make([]byte, 4)

	t.first = 0
	t.last = 0

	for {
		nextPos, err = t.decodeLogHead(&l, data, pos)
		if err != nil {
			//if error, we may lost all logs from pos
			log.Errorf("%s may lost logs from %d", data.Name(), pos)
			break
		}

		if l.ID == 0 {
			log.Errorf("%s may lost logs from %d, invalid log 0", data.Name(), pos)
			break
		}

		if t.first == 0 {
			t.first = l.ID
		}

		if t.last == 0 {
			t.last = l.ID
		} else if l.ID <= t.last {
			log.Errorf("%s may lost logs from %d, invalid logid %d", t.data.Name(), pos, l.ID)
			break
		}

		t.last = l.ID
		t.lastTime = l.CreateTime

		binary.BigEndian.PutUint32(b, uint32(pos))
		meta.Write(b)

		pos = nextPos

		t.lastTime = l.CreateTime
	}

	var e error
	if err := meta.Close(); err != nil {
		e = err
	}

	data.SetOffset(pos)

	if _, err = data.Write(magic); err != nil {
		log.Errorf("write magic error %s", err.Error())
	}

	if err = data.Close(); err != nil {
		return err
	}

	return e
}