// NewKeyValue returns a KeyValue implementation on top of a // github.com/cznic/kv file. func NewKeyValue(cfg jsonconfig.Obj) (sorted.KeyValue, error) { file := cfg.RequiredString("file") if err := cfg.Validate(); err != nil { return nil, err } createOpen := kv.Open verb := "opening" if _, err := os.Stat(file); os.IsNotExist(err) { createOpen = kv.Create verb = "creating" } opts := &kv.Options{ Locker: func(dbname string) (io.Closer, error) { lkfile := dbname + ".lock" cl, err := lock.Lock(lkfile) if err != nil { return nil, fmt.Errorf("failed to acquire lock on %s: %v", lkfile, err) } return cl, nil }, } db, err := createOpen(file, opts) if err != nil { return nil, fmt.Errorf("error %s %s: %v", verb, file, err) } is := &kvis{ db: db, opts: opts, path: file, } return is, nil }
// Open opens the named kv DB file for reading/writing. It // creates the file if it does not exist yet. func Open(dbFile string, opts *kv.Options) (*kv.DB, error) { createOpen := kv.Open verb := "opening" if _, err := os.Stat(dbFile); os.IsNotExist(err) { createOpen = kv.Create verb = "creating" } if opts == nil { opts = &kv.Options{} } if opts.Locker == nil { opts.Locker = func(dbFile string) (io.Closer, error) { lkfile := dbFile + ".lock" cl, err := lock.Lock(lkfile) if err != nil { return nil, fmt.Errorf("failed to acquire lock on %s: %v", lkfile, err) } return cl, nil } } if v, _ := strconv.ParseBool(os.Getenv("CAMLI_KV_VERIFY")); v { opts.VerifyDbBeforeOpen = true opts.VerifyDbAfterOpen = true opts.VerifyDbBeforeClose = true opts.VerifyDbAfterClose = true } db, err := createOpen(dbFile, opts) if err != nil { return nil, fmt.Errorf("error %s %s: %v", verb, dbFile, err) } return db, nil }
// Open opens the named kv DB file for reading/writing. It // creates the file if it does not exist yet. func Open(filePath string, opts *kv.Options) (*kv.DB, error) { // TODO(mpl): use it in index pkg and such createOpen := kv.Open if _, err := os.Stat(filePath); os.IsNotExist(err) { createOpen = kv.Create } if opts == nil { opts = &kv.Options{} } if opts.Locker == nil { opts.Locker = func(fullPath string) (io.Closer, error) { return lock.Lock(filePath + ".lock") } } return createOpen(filePath, opts) }
func NewStorage(file string) (index.Storage, io.Closer, error) { createOpen := kv.Open if _, err := os.Stat(file); os.IsNotExist(err) { createOpen = kv.Create } db, err := createOpen(file, &kv.Options{ Locker: func(dbname string) (io.Closer, error) { lkfile := dbname + ".lock" return lock.Lock(lkfile) }, }) if err != nil { return nil, nil, err } is := &kvis{ db: db, } return is, struct{ io.Closer }{db}, nil }
// openForWrite will create or open pack file n for writes, create a lock // visible external to the process and seek to the end of the file ready for // appending new data. // This function is not thread safe, s.mu should be locked by the caller. func (s *storage) openForWrite(n int) error { fn := s.filename(n) l, err := lock.Lock(fn + ".lock") if err != nil { return err } f, err := os.OpenFile(fn, os.O_RDWR|os.O_CREATE, 0666) if err != nil { l.Close() return err } openFdsVar.Add(s.root, 1) debug.Printf("diskpacked: opened for write %q", fn) s.size, err = f.Seek(0, os.SEEK_END) if err != nil { return err } s.writer = f s.writeLock = l return nil }
// openCurrent makes sure the current data file is open as s.current. func (s *storage) openCurrent() error { if s.current == nil { // First run; find the latest file data file and open it // and seek to the end. // If no data files exist, leave s.current as nil. for { _, err := os.Stat(s.filename(s.currentN)) if os.IsNotExist(err) { break } if err != nil { return err } s.currentN++ } if s.currentN > 0 { s.currentN-- l, err := lock.Lock(s.filename(s.currentN) + ".lock") if err != nil { return err } f, err := os.OpenFile(s.filename(s.currentN), os.O_RDWR, 0666) if err != nil { l.Close() return err } o, err := f.Seek(0, os.SEEK_END) if err != nil { l.Close() return err } s.current, s.currentL, s.currentO = f, l, o } } // If s.current is open and it's too big,close it and advance currentN. if s.current != nil && s.currentO > s.maxFileSize { f, l := s.current, s.currentL s.current, s.currentL, s.currentO = nil, nil, 0 s.currentN++ if err := f.Close(); err != nil { l.Close() return err } if err := l.Close(); err != nil { return err } } // If we don't have the current file open, make one. if s.current == nil { l, err := lock.Lock(s.filename(s.currentN) + ".lock") if err != nil { return err } f, err := os.Create(s.filename(s.currentN)) if err != nil { l.Close() return err } s.current, s.currentL, s.currentO = f, l, 0 } return nil }
func reindexOne(index sorted.KeyValue, overwrite, verbose bool, r io.ReadSeeker, name string, packId int64) error { l, err := lock.Lock(name + ".lock") defer l.Close() var pos, size int64 errAt := func(prefix, suffix string) error { if prefix != "" { prefix = prefix + " " } if suffix != "" { suffix = " " + suffix } return fmt.Errorf(prefix+"at %d (0x%x) in %q:"+suffix, pos, pos, name) } var batch sorted.BatchMutation if overwrite { batch = index.BeginBatch() } allOk := true br := bufio.NewReaderSize(r, 512) for { if b, err := br.ReadByte(); err != nil { if err == io.EOF { break } return errAt("error while reading", err.Error()) } else if b != '[' { return errAt(fmt.Sprintf("found byte 0x%x", b), "but '[' should be here!") } chunk, err := br.ReadSlice(']') if err != nil { if err == io.EOF { break } return errAt("error reading blob header", err.Error()) } m := len(chunk) chunk = chunk[:m-1] i := bytes.IndexByte(chunk, byte(' ')) if i <= 0 { return errAt("", fmt.Sprintf("bad header format (no space in %q)", chunk)) } if size, err = strconv.ParseInt(string(chunk[i+1:]), 10, 64); err != nil { return errAt(fmt.Sprintf("cannot parse size %q as int", chunk[i+1:]), err.Error()) } ref, ok := blob.Parse(string(chunk[:i])) if !ok { return errAt("", fmt.Sprintf("cannot parse %q as blobref", chunk[:i])) } if verbose { log.Printf("found %s at %d", ref, pos) } meta := blobMeta{packId, pos + 1 + int64(m), size}.String() if overwrite && batch != nil { batch.Set(ref.String(), meta) } else { if old, err := index.Get(ref.String()); err != nil { allOk = false if err == sorted.ErrNotFound { log.Println(ref.String() + ": cannot find in index!") } else { log.Println(ref.String()+": error getting from index: ", err.Error()) } } else if old != meta { allOk = false log.Printf("%s: index mismatch - index=%s data=%s", ref.String(), old, meta) } } pos += 1 + int64(m) // TODO(tgulacsi78): not just seek, but check the hashes of the files // maybe with a different command-line flag, only. if pos, err = r.Seek(pos+size, 0); err != nil { return errAt("", "cannot seek +"+strconv.FormatInt(size, 10)+" bytes") } // drain the buffer after the underlying reader Seeks io.CopyN(ioutil.Discard, br, int64(br.Buffered())) } if overwrite && batch != nil { log.Printf("overwriting %s from %s", index, name) if err = index.CommitBatch(batch); err != nil { return err } } else if !allOk { return fmt.Errorf("index does not match data in %q", name) } return nil }