Example #1
0
func (l *Layer) GetLocationContents(id [16]byte, after string, count int) ([]string, error) {
	var rang kvl.RangeQuery
	rang.Low = keys.LexNext(tuple.MustAppend(nil, "locationlist", id, after))
	rang.High = keys.PrefixNext(tuple.MustAppend(nil, "locationlist", id))
	if count > 0 {
		rang.Limit = count + 1
	}

	ps, err := l.index.Range(rang)
	if err != nil {
		return nil, err
	}

	names := make([]string, 0, len(ps))

	for _, p := range ps {
		var typ, name string
		var loc [16]byte
		err := tuple.UnpackInto(p.Key, &typ, &loc, &name)
		if err != nil {
			return nil, err
		}

		if name > after {
			names = append(names, name)
			if count > 0 && len(names) >= count {
				break
			}
		}
	}

	return names, nil
}
Example #2
0
func (l *Layer) GetFilesByLocation(id [16]byte, count int) ([]File, error) {
	var rang kvl.RangeQuery
	rang.Low, rang.High = keys.PrefixRange(tuple.MustAppend(nil,
		"file", "location", id))
	rang.Limit = count

	ps, err := l.index.Range(rang)
	if err != nil {
		return nil, err
	}

	fs := make([]File, 0, len(ps))

	for _, p := range ps {
		var typ, detail, path string
		var loc [16]byte
		err := tuple.UnpackInto(p.Key, &typ, &detail, &loc, &path)
		if err != nil {
			return nil, err
		}

		f, err := l.GetFile(path)
		if err != nil {
			return nil, err
		}

		if f != nil {
			fs = append(fs, *f)
		}
	}

	return fs, nil
}
Example #3
0
func (l *Location) fromPair(p kvl.Pair) error {
	var typ string
	err := tuple.UnpackInto(p.Key, &typ, &l.UUID)
	if err != nil {
		return err
	}
	if typ != "location" {
		return ErrBadKeyType
	}

	var version int
	left, err := tuple.UnpackIntoPartial(p.Value,
		&version, &l.URL, &l.Name, &l.Dead, &l.LastSeen)
	if err != nil {
		return err
	}

	if version != 0 {
		return ErrUnknownMetaVersion
	}

	l.AllocSplit = nil
	for len(left) > 0 {
		var next string
		left, err = tuple.UnpackIntoPartial(left, &next)
		if err != nil {
			return err
		}
		l.AllocSplit = append(l.AllocSplit, next)
	}

	return nil
}
Example #4
0
func (f *File) fromPair(p kvl.Pair) error {
	var typ string
	err := tuple.UnpackInto(p.Key, &typ, &f.Path)
	if err != nil {
		return err
	}
	if typ != "file" {
		return ErrBadKeyType
	}

	var version int
	left, err := tuple.UnpackIntoPartial(p.Value, &version, &f.Size, &f.SHA256,
		&f.WriteTime, &f.PrefixID, &f.DataChunks, &f.MappingValue)
	if version != 0 {
		return ErrUnknownMetaVersion
	}

	f.Locations = nil
	for len(left) > 0 {
		var next [16]byte
		left, err = tuple.UnpackIntoPartial(left, &next)
		if err != nil {
			return err
		}
		f.Locations = append(f.Locations, next)
	}

	return nil
}
Example #5
0
func (l *Layer) WALClearOld() error {
	var rang kvl.RangeQuery
	rang.Low, rang.High = keys.PrefixRange(tuple.MustAppend(nil, "wal2"))
	rang.Limit = 100

	for {
		ps, err := l.inner.Range(rang)
		if err != nil {
			return err
		}

		now := time.Now().Unix()

		for _, p := range ps {
			var typ string
			var id [16]byte
			err = tuple.UnpackInto(p.Key, &typ, &id)
			if err != nil {
				return err
			}

			timestamps, err := walParse(p.Value)
			if err != nil {
				return err
			}

			newestAge := time.Duration(now-timestamps[len(timestamps)-1]) * time.Second
			oldestAge := time.Duration(now-timestamps[0]) * time.Second

			if newestAge < -walUnsafeFutureAge {
				log.Printf("WARNING: WAL entry %v is %v in the future! Skipping it.",
					uuid.Fmt(id), newestAge)
			} else if oldestAge > walUnsafeOldAge {
				log.Printf("WARNING: WAL entry %v is %v in the past! Skipping it.",
					uuid.Fmt(id), oldestAge)
			} else if oldestAge > walExpireAge {
				log.Printf("Removing expired WAL entry %v (%v old)",
					uuid.Fmt(id), oldestAge)

				// Remove the oldest timestamp (only); we'll get to the other timestamps
				// in future passes if needed.
				timestamps = timestamps[1:]

				if len(timestamps) == 0 {
					err = l.inner.Delete(p.Key)
					if err != nil {
						return err
					}
				} else {
					err = l.inner.Set(kvl.Pair{p.Key, walDump(timestamps)})
					if err != nil {
						return err
					}
				}
			}
		}

		if len(ps) < rang.Limit {
			break
		}

		rang.Low = ps[len(ps)-1].Key
	}

	return nil
}