func NewWAL(path string) *WAL {
return &WAL{
path: path,
// these options should be overriden by any options in the config
LogOutput: os.Stderr,
SegmentSize: DefaultSegmentSize,
logger: log.New(os.Stderr, "[tsm1wal] ", log.LstdFlags),
closing: make(chan struct{}),
stats: &WALStatistics{},
limiter: limiter.NewFixed(defaultWaitingWALWrites),
}
}
func NewWAL(path string) *WAL {
logger := zap.New(zap.NullEncoder())
return &WAL{
path: path,
// these options should be overriden by any options in the config
SegmentSize: DefaultSegmentSize,
closing: make(chan struct{}),
stats: &WALStatistics{},
limiter: limiter.NewFixed(defaultWaitingWALWrites),
logger: logger,
traceLogger: logger,
}
}
// walkShards apply a function to each shard in parallel. If any of the
// functions return an error, the first error is returned.
func (s *Store) walkShards(shards []*Shard, fn func(sh *Shard) error) error {
// struct to hold the result of opening each reader in a goroutine
type res struct {
err error
}
t := limiter.NewFixed(runtime.GOMAXPROCS(0))
resC := make(chan res)
var n int
for _, sh := range shards {
n++
go func(sh *Shard) {
t.Take()
defer t.Release()
if err := fn(sh); err != nil {
resC <- res{err: fmt.Errorf("shard %d: %s", sh.id, err)}
return
}
resC <- res{}
}(sh)
}
var err error
for i := 0; i < n; i++ {
res := <-resC
if res.err != nil {
err = res.err
}
}
close(resC)
return err
}
func (s *Store) loadShards() error {
// struct to hold the result of opening each reader in a goroutine
type res struct {
s *Shard
err error
}
t := limiter.NewFixed(runtime.GOMAXPROCS(0))
resC := make(chan *res)
var n int
// loop through the current database indexes
for db := range s.databaseIndexes {
rps, err := ioutil.ReadDir(filepath.Join(s.path, db))
if err != nil {
return err
}
for _, rp := range rps {
// retention policies should be directories. Skip anything that is not a dir.
if !rp.IsDir() {
s.Logger.Printf("Skipping retention policy dir: %s. Not a directory", rp.Name())
continue
}
shards, err := ioutil.ReadDir(filepath.Join(s.path, db, rp.Name()))
if err != nil {
return err
}
for _, sh := range shards {
n++
go func(index *DatabaseIndex, db, rp, sh string) {
t.Take()
defer t.Release()
start := time.Now()
path := filepath.Join(s.path, db, rp, sh)
walPath := filepath.Join(s.EngineOptions.Config.WALDir, db, rp, sh)
// Shard file names are numeric shardIDs
shardID, err := strconv.ParseUint(sh, 10, 64)
if err != nil {
resC <- &res{err: fmt.Errorf("%s is not a valid ID. Skipping shard.", sh)}
return
}
shard := NewShard(shardID, s.databaseIndexes[db], path, walPath, s.EngineOptions)
shard.SetLogOutput(s.logOutput)
err = shard.Open()
if err != nil {
resC <- &res{err: fmt.Errorf("Failed to open shard: %d: %s", shardID, err)}
return
}
resC <- &res{s: shard}
s.Logger.Printf("%s opened in %s", path, time.Now().Sub(start))
}(s.databaseIndexes[db], db, rp.Name(), sh.Name())
}
}
}
for i := 0; i < n; i++ {
res := <-resC
if res.err != nil {
s.Logger.Println(res.err)
continue
}
s.shards[res.s.id] = res.s
}
close(resC)
return nil
}