func (l *Layer) GetLocationContents(id [16]byte, after string, count int) ([]string, error) {
var rang kvl.RangeQuery
rang.Low = keys.LexNext(tuple.MustAppend(nil, "locationlist", id, after))
rang.High = keys.PrefixNext(tuple.MustAppend(nil, "locationlist", id))
if count > 0 {
rang.Limit = count + 1
}
ps, err := l.index.Range(rang)
if err != nil {
return nil, err
}
names := make([]string, 0, len(ps))
for _, p := range ps {
var typ, name string
var loc [16]byte
err := tuple.UnpackInto(p.Key, &typ, &loc, &name)
if err != nil {
return nil, err
}
if name > after {
names = append(names, name)
if count > 0 && len(names) >= count {
break
}
}
}
return names, nil
}
func walDump(timestamps []int64) []byte {
data := tuple.MustAppend(nil, len(timestamps))
for _, t := range timestamps {
data = tuple.MustAppend(data, t)
}
return data
}
func (l *Location) toPair() kvl.Pair {
var p kvl.Pair
p.Key = tuple.MustAppend(nil, "location", l.UUID)
p.Value = tuple.MustAppend(nil, 0, l.URL, l.Name, l.Dead, l.LastSeen)
for _, split := range l.AllocSplit {
p.Value = tuple.MustAppend(p.Value, split)
}
return p
}
func (f *File) toPair() kvl.Pair {
var p kvl.Pair
p.Key = fileKey(f.Path)
p.Value = tuple.MustAppend(nil,
0, f.Size, f.SHA256, f.WriteTime, f.PrefixID, f.DataChunks,
f.MappingValue)
for _, loc := range f.Locations {
p.Value = tuple.MustAppend(p.Value, loc)
}
return p
}
func (l *Layer) GetConfig(key string) ([]byte, error) {
p, err := l.inner.Get(tuple.MustAppend(nil, "config", key))
if err != nil && err != kvl.ErrNotFound {
return nil, err
}
return p.Value, nil
}
func (l *Layer) ListFiles(after string, limit int) ([]File, error) {
if limit < 0 {
return nil, ErrBadArgument
}
var query kvl.RangeQuery
query.Low = fileKey(after)
query.High = keys.PrefixNext(tuple.MustAppend(nil, "file"))
if limit > 0 {
query.Limit = limit + 1
}
pairs, err := l.inner.Range(query)
if err != nil {
return nil, err
}
files := make([]File, 0, len(pairs))
for _, pair := range pairs {
var f File
err := f.fromPair(pair)
if err != nil {
return nil, err
}
if f.Path > after {
files = append(files, f)
if limit > 0 && len(files) == limit {
break
}
}
}
return files, nil
}
func (l *Layer) PathForPrefixID(id [16]byte) (string, error) {
p, err := l.index.Get(tuple.MustAppend(nil, "file", "prefix", id))
if err != nil {
return "", err
}
return string(p.Value), nil
}
func (l *Layer) GetFilesByLocation(id [16]byte, count int) ([]File, error) {
var rang kvl.RangeQuery
rang.Low, rang.High = keys.PrefixRange(tuple.MustAppend(nil,
"file", "location", id))
rang.Limit = count
ps, err := l.index.Range(rang)
if err != nil {
return nil, err
}
fs := make([]File, 0, len(ps))
for _, p := range ps {
var typ, detail, path string
var loc [16]byte
err := tuple.UnpackInto(p.Key, &typ, &detail, &loc, &path)
if err != nil {
return nil, err
}
f, err := l.GetFile(path)
if err != nil {
return nil, err
}
if f != nil {
fs = append(fs, *f)
}
}
return fs, nil
}
func (l *Layer) WALClear(id [16]byte) error {
err := l.LinearizeOn(uuid.Fmt(id))
if err != nil {
return err
}
key := tuple.MustAppend(nil, "wal2", id)
p, err := l.inner.Get(key)
if err != nil && err != kvl.ErrNotFound {
return err
}
timestamps, err := walParse(p.Value)
if err != nil {
return err
}
if len(timestamps) == 0 {
return errors.New("WAL does not have locks on that id")
}
// Remove most recent timestamp.
//
// In the case of hung/stopped processes that marked an id but never
// cleared it, this effectively assumes that the hung one is the *oldest*
// one, making the WAL lock unneccessarily longer than it should be. This
// is not ideal, but is safe.
timestamps = timestamps[:len(timestamps)-1]
if len(timestamps) == 0 {
return l.inner.Delete(key)
}
return l.inner.Set(kvl.Pair{key, walDump(timestamps)})
}
func (l *Layer) LocationShouldHave(id [16]byte, name string) (bool, error) {
_, err := l.index.Get(tuple.MustAppend(nil, "locationlist", id, name))
if err != nil {
if err == kvl.ErrNotFound {
return false, nil
}
return false, err
}
return true, nil
}
func (f *File) indexPairs() []kvl.Pair {
ret := make([]kvl.Pair, 0, len(f.Locations)*2+1)
for idx, loc := range f.Locations {
localKey := fmt.Sprintf("%v_%x_%v",
uuid.Fmt(f.PrefixID), f.SHA256[:8], idx)
ret = append(ret, kvl.Pair{
tuple.MustAppend(nil, "file", "location", loc, f.Path),
nil,
}, kvl.Pair{
tuple.MustAppend(nil, "locationlist", loc, localKey),
nil,
})
}
ret = append(ret, kvl.Pair{
tuple.MustAppend(nil, "file", "prefix", f.PrefixID),
[]byte(f.Path),
})
return ret
}
func (l *Layer) WALCheck(id [16]byte) (bool, error) {
err := l.LinearizeOn(uuid.Fmt(id))
if err != nil {
return false, err
}
_, err = l.inner.Get(tuple.MustAppend(nil, "wal2", id))
if err != nil {
if err == kvl.ErrNotFound {
err = nil
}
return false, err
}
return true, nil
}
func (l *Layer) WALMark(id [16]byte) error {
err := l.LinearizeOn(uuid.Fmt(id))
if err != nil {
return err
}
key := tuple.MustAppend(nil, "wal2", id)
p, err := l.inner.Get(key)
if err != nil && err != kvl.ErrNotFound {
return err
}
timestamps, err := walParse(p.Value)
if err != nil {
return err
}
timestamps = append(timestamps, time.Now().Unix())
return l.inner.Set(kvl.Pair{key, walDump(timestamps)})
}
func (l *Layer) SetConfig(key string, data []byte) error {
return l.inner.Set(kvl.Pair{tuple.MustAppend(nil, "config", key), data})
}
func (l *Layer) WALClearOld() error {
var rang kvl.RangeQuery
rang.Low, rang.High = keys.PrefixRange(tuple.MustAppend(nil, "wal2"))
rang.Limit = 100
for {
ps, err := l.inner.Range(rang)
if err != nil {
return err
}
now := time.Now().Unix()
for _, p := range ps {
var typ string
var id [16]byte
err = tuple.UnpackInto(p.Key, &typ, &id)
if err != nil {
return err
}
timestamps, err := walParse(p.Value)
if err != nil {
return err
}
newestAge := time.Duration(now-timestamps[len(timestamps)-1]) * time.Second
oldestAge := time.Duration(now-timestamps[0]) * time.Second
if newestAge < -walUnsafeFutureAge {
log.Printf("WARNING: WAL entry %v is %v in the future! Skipping it.",
uuid.Fmt(id), newestAge)
} else if oldestAge > walUnsafeOldAge {
log.Printf("WARNING: WAL entry %v is %v in the past! Skipping it.",
uuid.Fmt(id), oldestAge)
} else if oldestAge > walExpireAge {
log.Printf("Removing expired WAL entry %v (%v old)",
uuid.Fmt(id), oldestAge)
// Remove the oldest timestamp (only); we'll get to the other timestamps
// in future passes if needed.
timestamps = timestamps[1:]
if len(timestamps) == 0 {
err = l.inner.Delete(p.Key)
if err != nil {
return err
}
} else {
err = l.inner.Set(kvl.Pair{p.Key, walDump(timestamps)})
if err != nil {
return err
}
}
}
}
if len(ps) < rang.Limit {
break
}
rang.Low = ps[len(ps)-1].Key
}
return nil
}
func fileKey(path string) []byte {
return tuple.MustAppend(nil, "file", path)
}