func (l *Layer) GetLocationContents(id [16]byte, after string, count int) ([]string, error) { var rang kvl.RangeQuery rang.Low = keys.LexNext(tuple.MustAppend(nil, "locationlist", id, after)) rang.High = keys.PrefixNext(tuple.MustAppend(nil, "locationlist", id)) if count > 0 { rang.Limit = count + 1 } ps, err := l.index.Range(rang) if err != nil { return nil, err } names := make([]string, 0, len(ps)) for _, p := range ps { var typ, name string var loc [16]byte err := tuple.UnpackInto(p.Key, &typ, &loc, &name) if err != nil { return nil, err } if name > after { names = append(names, name) if count > 0 && len(names) >= count { break } } } return names, nil }
func walDump(timestamps []int64) []byte { data := tuple.MustAppend(nil, len(timestamps)) for _, t := range timestamps { data = tuple.MustAppend(data, t) } return data }
func (l *Location) toPair() kvl.Pair { var p kvl.Pair p.Key = tuple.MustAppend(nil, "location", l.UUID) p.Value = tuple.MustAppend(nil, 0, l.URL, l.Name, l.Dead, l.LastSeen) for _, split := range l.AllocSplit { p.Value = tuple.MustAppend(p.Value, split) } return p }
func (f *File) toPair() kvl.Pair { var p kvl.Pair p.Key = fileKey(f.Path) p.Value = tuple.MustAppend(nil, 0, f.Size, f.SHA256, f.WriteTime, f.PrefixID, f.DataChunks, f.MappingValue) for _, loc := range f.Locations { p.Value = tuple.MustAppend(p.Value, loc) } return p }
func (l *Layer) GetConfig(key string) ([]byte, error) { p, err := l.inner.Get(tuple.MustAppend(nil, "config", key)) if err != nil && err != kvl.ErrNotFound { return nil, err } return p.Value, nil }
func (l *Layer) ListFiles(after string, limit int) ([]File, error) { if limit < 0 { return nil, ErrBadArgument } var query kvl.RangeQuery query.Low = fileKey(after) query.High = keys.PrefixNext(tuple.MustAppend(nil, "file")) if limit > 0 { query.Limit = limit + 1 } pairs, err := l.inner.Range(query) if err != nil { return nil, err } files := make([]File, 0, len(pairs)) for _, pair := range pairs { var f File err := f.fromPair(pair) if err != nil { return nil, err } if f.Path > after { files = append(files, f) if limit > 0 && len(files) == limit { break } } } return files, nil }
func (l *Layer) PathForPrefixID(id [16]byte) (string, error) { p, err := l.index.Get(tuple.MustAppend(nil, "file", "prefix", id)) if err != nil { return "", err } return string(p.Value), nil }
func (l *Layer) GetFilesByLocation(id [16]byte, count int) ([]File, error) { var rang kvl.RangeQuery rang.Low, rang.High = keys.PrefixRange(tuple.MustAppend(nil, "file", "location", id)) rang.Limit = count ps, err := l.index.Range(rang) if err != nil { return nil, err } fs := make([]File, 0, len(ps)) for _, p := range ps { var typ, detail, path string var loc [16]byte err := tuple.UnpackInto(p.Key, &typ, &detail, &loc, &path) if err != nil { return nil, err } f, err := l.GetFile(path) if err != nil { return nil, err } if f != nil { fs = append(fs, *f) } } return fs, nil }
func (l *Layer) WALClear(id [16]byte) error { err := l.LinearizeOn(uuid.Fmt(id)) if err != nil { return err } key := tuple.MustAppend(nil, "wal2", id) p, err := l.inner.Get(key) if err != nil && err != kvl.ErrNotFound { return err } timestamps, err := walParse(p.Value) if err != nil { return err } if len(timestamps) == 0 { return errors.New("WAL does not have locks on that id") } // Remove most recent timestamp. // // In the case of hung/stopped processes that marked an id but never // cleared it, this effectively assumes that the hung one is the *oldest* // one, making the WAL lock unneccessarily longer than it should be. This // is not ideal, but is safe. timestamps = timestamps[:len(timestamps)-1] if len(timestamps) == 0 { return l.inner.Delete(key) } return l.inner.Set(kvl.Pair{key, walDump(timestamps)}) }
func (l *Layer) LocationShouldHave(id [16]byte, name string) (bool, error) { _, err := l.index.Get(tuple.MustAppend(nil, "locationlist", id, name)) if err != nil { if err == kvl.ErrNotFound { return false, nil } return false, err } return true, nil }
func (f *File) indexPairs() []kvl.Pair { ret := make([]kvl.Pair, 0, len(f.Locations)*2+1) for idx, loc := range f.Locations { localKey := fmt.Sprintf("%v_%x_%v", uuid.Fmt(f.PrefixID), f.SHA256[:8], idx) ret = append(ret, kvl.Pair{ tuple.MustAppend(nil, "file", "location", loc, f.Path), nil, }, kvl.Pair{ tuple.MustAppend(nil, "locationlist", loc, localKey), nil, }) } ret = append(ret, kvl.Pair{ tuple.MustAppend(nil, "file", "prefix", f.PrefixID), []byte(f.Path), }) return ret }
func (l *Layer) WALCheck(id [16]byte) (bool, error) { err := l.LinearizeOn(uuid.Fmt(id)) if err != nil { return false, err } _, err = l.inner.Get(tuple.MustAppend(nil, "wal2", id)) if err != nil { if err == kvl.ErrNotFound { err = nil } return false, err } return true, nil }
func (l *Layer) WALMark(id [16]byte) error { err := l.LinearizeOn(uuid.Fmt(id)) if err != nil { return err } key := tuple.MustAppend(nil, "wal2", id) p, err := l.inner.Get(key) if err != nil && err != kvl.ErrNotFound { return err } timestamps, err := walParse(p.Value) if err != nil { return err } timestamps = append(timestamps, time.Now().Unix()) return l.inner.Set(kvl.Pair{key, walDump(timestamps)}) }
func (l *Layer) SetConfig(key string, data []byte) error { return l.inner.Set(kvl.Pair{tuple.MustAppend(nil, "config", key), data}) }
func (l *Layer) WALClearOld() error { var rang kvl.RangeQuery rang.Low, rang.High = keys.PrefixRange(tuple.MustAppend(nil, "wal2")) rang.Limit = 100 for { ps, err := l.inner.Range(rang) if err != nil { return err } now := time.Now().Unix() for _, p := range ps { var typ string var id [16]byte err = tuple.UnpackInto(p.Key, &typ, &id) if err != nil { return err } timestamps, err := walParse(p.Value) if err != nil { return err } newestAge := time.Duration(now-timestamps[len(timestamps)-1]) * time.Second oldestAge := time.Duration(now-timestamps[0]) * time.Second if newestAge < -walUnsafeFutureAge { log.Printf("WARNING: WAL entry %v is %v in the future! Skipping it.", uuid.Fmt(id), newestAge) } else if oldestAge > walUnsafeOldAge { log.Printf("WARNING: WAL entry %v is %v in the past! Skipping it.", uuid.Fmt(id), oldestAge) } else if oldestAge > walExpireAge { log.Printf("Removing expired WAL entry %v (%v old)", uuid.Fmt(id), oldestAge) // Remove the oldest timestamp (only); we'll get to the other timestamps // in future passes if needed. timestamps = timestamps[1:] if len(timestamps) == 0 { err = l.inner.Delete(p.Key) if err != nil { return err } } else { err = l.inner.Set(kvl.Pair{p.Key, walDump(timestamps)}) if err != nil { return err } } } } if len(ps) < rang.Limit { break } rang.Low = ps[len(ps)-1].Key } return nil }
func fileKey(path string) []byte { return tuple.MustAppend(nil, "file", path) }