Exemplo n.º 1
0
func (l *Layer) GetLocationContents(id [16]byte, after string, count int) ([]string, error) {
	var rang kvl.RangeQuery
	rang.Low = keys.LexNext(tuple.MustAppend(nil, "locationlist", id, after))
	rang.High = keys.PrefixNext(tuple.MustAppend(nil, "locationlist", id))
	if count > 0 {
		rang.Limit = count + 1
	}

	ps, err := l.index.Range(rang)
	if err != nil {
		return nil, err
	}

	names := make([]string, 0, len(ps))

	for _, p := range ps {
		var typ, name string
		var loc [16]byte
		err := tuple.UnpackInto(p.Key, &typ, &loc, &name)
		if err != nil {
			return nil, err
		}

		if name > after {
			names = append(names, name)
			if count > 0 && len(names) >= count {
				break
			}
		}
	}

	return names, nil
}
Exemplo n.º 2
0
func walDump(timestamps []int64) []byte {
	data := tuple.MustAppend(nil, len(timestamps))
	for _, t := range timestamps {
		data = tuple.MustAppend(data, t)
	}
	return data
}
Exemplo n.º 3
0
func (l *Location) toPair() kvl.Pair {
	var p kvl.Pair

	p.Key = tuple.MustAppend(nil, "location", l.UUID)

	p.Value = tuple.MustAppend(nil, 0, l.URL, l.Name, l.Dead, l.LastSeen)
	for _, split := range l.AllocSplit {
		p.Value = tuple.MustAppend(p.Value, split)
	}

	return p
}
Exemplo n.º 4
0
func (f *File) toPair() kvl.Pair {
	var p kvl.Pair

	p.Key = fileKey(f.Path)

	p.Value = tuple.MustAppend(nil,
		0, f.Size, f.SHA256, f.WriteTime, f.PrefixID, f.DataChunks,
		f.MappingValue)
	for _, loc := range f.Locations {
		p.Value = tuple.MustAppend(p.Value, loc)
	}

	return p
}
Exemplo n.º 5
0
func (l *Layer) GetConfig(key string) ([]byte, error) {
	p, err := l.inner.Get(tuple.MustAppend(nil, "config", key))
	if err != nil && err != kvl.ErrNotFound {
		return nil, err
	}
	return p.Value, nil
}
Exemplo n.º 6
0
func (l *Layer) ListFiles(after string, limit int) ([]File, error) {
	if limit < 0 {
		return nil, ErrBadArgument
	}

	var query kvl.RangeQuery
	query.Low = fileKey(after)
	query.High = keys.PrefixNext(tuple.MustAppend(nil, "file"))
	if limit > 0 {
		query.Limit = limit + 1
	}
	pairs, err := l.inner.Range(query)
	if err != nil {
		return nil, err
	}

	files := make([]File, 0, len(pairs))
	for _, pair := range pairs {
		var f File
		err := f.fromPair(pair)
		if err != nil {
			return nil, err
		}

		if f.Path > after {
			files = append(files, f)
			if limit > 0 && len(files) == limit {
				break
			}
		}
	}

	return files, nil
}
Exemplo n.º 7
0
func (l *Layer) PathForPrefixID(id [16]byte) (string, error) {
	p, err := l.index.Get(tuple.MustAppend(nil, "file", "prefix", id))
	if err != nil {
		return "", err
	}
	return string(p.Value), nil
}
Exemplo n.º 8
0
func (l *Layer) GetFilesByLocation(id [16]byte, count int) ([]File, error) {
	var rang kvl.RangeQuery
	rang.Low, rang.High = keys.PrefixRange(tuple.MustAppend(nil,
		"file", "location", id))
	rang.Limit = count

	ps, err := l.index.Range(rang)
	if err != nil {
		return nil, err
	}

	fs := make([]File, 0, len(ps))

	for _, p := range ps {
		var typ, detail, path string
		var loc [16]byte
		err := tuple.UnpackInto(p.Key, &typ, &detail, &loc, &path)
		if err != nil {
			return nil, err
		}

		f, err := l.GetFile(path)
		if err != nil {
			return nil, err
		}

		if f != nil {
			fs = append(fs, *f)
		}
	}

	return fs, nil
}
Exemplo n.º 9
0
func (l *Layer) WALClear(id [16]byte) error {
	err := l.LinearizeOn(uuid.Fmt(id))
	if err != nil {
		return err
	}

	key := tuple.MustAppend(nil, "wal2", id)
	p, err := l.inner.Get(key)
	if err != nil && err != kvl.ErrNotFound {
		return err
	}

	timestamps, err := walParse(p.Value)
	if err != nil {
		return err
	}

	if len(timestamps) == 0 {
		return errors.New("WAL does not have locks on that id")
	}

	// Remove most recent timestamp.
	//
	// In the case of hung/stopped processes that marked an id but never
	// cleared it, this effectively assumes that the hung one is the *oldest*
	// one, making the WAL lock unneccessarily longer than it should be. This
	// is not ideal, but is safe.
	timestamps = timestamps[:len(timestamps)-1]

	if len(timestamps) == 0 {
		return l.inner.Delete(key)
	}

	return l.inner.Set(kvl.Pair{key, walDump(timestamps)})
}
Exemplo n.º 10
0
func (l *Layer) LocationShouldHave(id [16]byte, name string) (bool, error) {
	_, err := l.index.Get(tuple.MustAppend(nil, "locationlist", id, name))
	if err != nil {
		if err == kvl.ErrNotFound {
			return false, nil
		}
		return false, err
	}
	return true, nil
}
Exemplo n.º 11
0
func (f *File) indexPairs() []kvl.Pair {
	ret := make([]kvl.Pair, 0, len(f.Locations)*2+1)

	for idx, loc := range f.Locations {
		localKey := fmt.Sprintf("%v_%x_%v",
			uuid.Fmt(f.PrefixID), f.SHA256[:8], idx)

		ret = append(ret, kvl.Pair{
			tuple.MustAppend(nil, "file", "location", loc, f.Path),
			nil,
		}, kvl.Pair{
			tuple.MustAppend(nil, "locationlist", loc, localKey),
			nil,
		})
	}

	ret = append(ret, kvl.Pair{
		tuple.MustAppend(nil, "file", "prefix", f.PrefixID),
		[]byte(f.Path),
	})

	return ret
}
Exemplo n.º 12
0
func (l *Layer) WALCheck(id [16]byte) (bool, error) {
	err := l.LinearizeOn(uuid.Fmt(id))
	if err != nil {
		return false, err
	}

	_, err = l.inner.Get(tuple.MustAppend(nil, "wal2", id))
	if err != nil {
		if err == kvl.ErrNotFound {
			err = nil
		}
		return false, err
	}
	return true, nil
}
Exemplo n.º 13
0
func (l *Layer) WALMark(id [16]byte) error {
	err := l.LinearizeOn(uuid.Fmt(id))
	if err != nil {
		return err
	}

	key := tuple.MustAppend(nil, "wal2", id)
	p, err := l.inner.Get(key)
	if err != nil && err != kvl.ErrNotFound {
		return err
	}

	timestamps, err := walParse(p.Value)
	if err != nil {
		return err
	}

	timestamps = append(timestamps, time.Now().Unix())

	return l.inner.Set(kvl.Pair{key, walDump(timestamps)})
}
Exemplo n.º 14
0
func (l *Layer) SetConfig(key string, data []byte) error {
	return l.inner.Set(kvl.Pair{tuple.MustAppend(nil, "config", key), data})
}
Exemplo n.º 15
0
func (l *Layer) WALClearOld() error {
	var rang kvl.RangeQuery
	rang.Low, rang.High = keys.PrefixRange(tuple.MustAppend(nil, "wal2"))
	rang.Limit = 100

	for {
		ps, err := l.inner.Range(rang)
		if err != nil {
			return err
		}

		now := time.Now().Unix()

		for _, p := range ps {
			var typ string
			var id [16]byte
			err = tuple.UnpackInto(p.Key, &typ, &id)
			if err != nil {
				return err
			}

			timestamps, err := walParse(p.Value)
			if err != nil {
				return err
			}

			newestAge := time.Duration(now-timestamps[len(timestamps)-1]) * time.Second
			oldestAge := time.Duration(now-timestamps[0]) * time.Second

			if newestAge < -walUnsafeFutureAge {
				log.Printf("WARNING: WAL entry %v is %v in the future! Skipping it.",
					uuid.Fmt(id), newestAge)
			} else if oldestAge > walUnsafeOldAge {
				log.Printf("WARNING: WAL entry %v is %v in the past! Skipping it.",
					uuid.Fmt(id), oldestAge)
			} else if oldestAge > walExpireAge {
				log.Printf("Removing expired WAL entry %v (%v old)",
					uuid.Fmt(id), oldestAge)

				// Remove the oldest timestamp (only); we'll get to the other timestamps
				// in future passes if needed.
				timestamps = timestamps[1:]

				if len(timestamps) == 0 {
					err = l.inner.Delete(p.Key)
					if err != nil {
						return err
					}
				} else {
					err = l.inner.Set(kvl.Pair{p.Key, walDump(timestamps)})
					if err != nil {
						return err
					}
				}
			}
		}

		if len(ps) < rang.Limit {
			break
		}

		rang.Low = ps[len(ps)-1].Key
	}

	return nil
}
Exemplo n.º 16
0
func fileKey(path string) []byte {
	return tuple.MustAppend(nil, "file", path)
}