func NewWAL(path string) *WAL { return &WAL{ path: path, // these options should be overriden by any options in the config LogOutput: os.Stderr, SegmentSize: DefaultSegmentSize, logger: log.New(os.Stderr, "[tsm1wal] ", log.LstdFlags), closing: make(chan struct{}), stats: &WALStatistics{}, limiter: limiter.NewFixed(defaultWaitingWALWrites), } }
func NewWAL(path string) *WAL { logger := zap.New(zap.NullEncoder()) return &WAL{ path: path, // these options should be overriden by any options in the config SegmentSize: DefaultSegmentSize, closing: make(chan struct{}), stats: &WALStatistics{}, limiter: limiter.NewFixed(defaultWaitingWALWrites), logger: logger, traceLogger: logger, } }
// walkShards apply a function to each shard in parallel. If any of the // functions return an error, the first error is returned. func (s *Store) walkShards(shards []*Shard, fn func(sh *Shard) error) error { // struct to hold the result of opening each reader in a goroutine type res struct { err error } t := limiter.NewFixed(runtime.GOMAXPROCS(0)) resC := make(chan res) var n int for _, sh := range shards { n++ go func(sh *Shard) { t.Take() defer t.Release() if err := fn(sh); err != nil { resC <- res{err: fmt.Errorf("shard %d: %s", sh.id, err)} return } resC <- res{} }(sh) } var err error for i := 0; i < n; i++ { res := <-resC if res.err != nil { err = res.err } } close(resC) return err }
func (s *Store) loadShards() error { // struct to hold the result of opening each reader in a goroutine type res struct { s *Shard err error } t := limiter.NewFixed(runtime.GOMAXPROCS(0)) resC := make(chan *res) var n int // loop through the current database indexes for db := range s.databaseIndexes { rps, err := ioutil.ReadDir(filepath.Join(s.path, db)) if err != nil { return err } for _, rp := range rps { // retention policies should be directories. Skip anything that is not a dir. if !rp.IsDir() { s.Logger.Printf("Skipping retention policy dir: %s. Not a directory", rp.Name()) continue } shards, err := ioutil.ReadDir(filepath.Join(s.path, db, rp.Name())) if err != nil { return err } for _, sh := range shards { n++ go func(index *DatabaseIndex, db, rp, sh string) { t.Take() defer t.Release() start := time.Now() path := filepath.Join(s.path, db, rp, sh) walPath := filepath.Join(s.EngineOptions.Config.WALDir, db, rp, sh) // Shard file names are numeric shardIDs shardID, err := strconv.ParseUint(sh, 10, 64) if err != nil { resC <- &res{err: fmt.Errorf("%s is not a valid ID. Skipping shard.", sh)} return } shard := NewShard(shardID, s.databaseIndexes[db], path, walPath, s.EngineOptions) shard.SetLogOutput(s.logOutput) err = shard.Open() if err != nil { resC <- &res{err: fmt.Errorf("Failed to open shard: %d: %s", shardID, err)} return } resC <- &res{s: shard} s.Logger.Printf("%s opened in %s", path, time.Now().Sub(start)) }(s.databaseIndexes[db], db, rp.Name(), sh.Name()) } } } for i := 0; i < n; i++ { res := <-resC if res.err != nil { s.Logger.Println(res.err) continue } s.shards[res.s.id] = res.s } close(resC) return nil }