// NewIterator creates a new Iterator for the given source and metric.
func (db *DB) NewIterator(source, metric string) (*Iterator, error) {
var p partition.Partition = nil
i := db.partitionList.NewIterator()
for i.Next() {
val, _ := i.Value()
val.Hold()
if val.HasMetric(source, metric) {
if p != nil {
p.Release()
}
p = val
} else {
val.Release()
}
}
if p == nil {
return nil, errors.New("catena: couldn't find metric for iterator")
}
partitionIter, err := p.NewIterator(source, metric)
p.Release()
if err != nil {
return nil, err
}
return &Iterator{
source: source,
metric: metric,
db: db,
curPartition: p,
Iterator: partitionIter,
}, nil
}
// Reset moves i to the first available timestamp.
func (i *Iterator) Reset() error {
i.Iterator.Close()
var p partition.Partition
partitionListIter := i.db.partitionList.NewIterator()
for partitionListIter.Next() {
val, _ := partitionListIter.Value()
val.Hold()
if val.HasMetric(i.source, i.metric) {
if p != nil {
p.Release()
}
p = val
} else {
val.Release()
}
}
if p == nil {
return errors.New("catena: couldn't find metric for iterator")
}
defer p.Release()
i.curPartition = p
partitionIter, err := p.NewIterator(i.source, i.metric)
if err != nil {
return err
}
i.Iterator = partitionIter
return nil
}
func comparePartitions(a, b partition.Partition) int {
// Highest timestamp first
return int(b.MinTimestamp() - a.MinTimestamp())
}
// InsertRows inserts the given rows into the database.
func (db *DB) InsertRows(rows []Row) error {
keyToRows := map[int][]Row{}
for _, row := range rows {
key := int(row.Timestamp / db.partitionSize)
keyToRows[key] = append(keyToRows[key], row)
}
keys := []int{}
for key := range keyToRows {
keys = append(keys, key)
}
sort.Ints(keys)
for _, key := range keys {
rowsForKey := keyToRows[key]
minTimestampInRows := int64(0)
maxTimestampInRows := int64(0)
for i, row := range rowsForKey {
if i == 1 {
minTimestampInRows = row.Timestamp
maxTimestampInRows = row.Timestamp
}
if row.Timestamp > maxTimestampInRows {
maxTimestampInRows = row.Timestamp
}
if row.Timestamp < minTimestampInRows {
minTimestampInRows = row.Timestamp
}
}
var p partition.Partition
FIND_PARTITION:
i := db.partitionList.NewIterator()
for i.Next() {
val, _ := i.Value()
val.Hold()
if val.MinTimestamp()/db.partitionSize == int64(key) {
p = val
goto VALID_PARTITION
}
if val.MinTimestamp()/db.partitionSize < int64(key) && val.MaxTimestamp()/db.partitionSize >= int64(key) {
p = val
goto VALID_PARTITION
}
val.Release()
}
db.partitionCreateLock.Lock()
if p == nil {
if int64(key) < atomic.LoadInt64(&db.minTimestamp)/db.partitionSize {
db.partitionCreateLock.Unlock()
return errors.New("catena: row(s) being inserted are too old")
}
if db.partitionList.Size() == 0 ||
int64(key) > atomic.LoadInt64(&db.maxTimestamp)/db.partitionSize {
// Need to make a new partition
newPartitionID := atomic.LoadInt64(&db.lastPartitionID) + 1
w, err := wal.NewFileWAL(filepath.Join(db.baseDir,
fmt.Sprintf("%d.wal", newPartitionID)))
if err != nil {
// Couldn't create the WAL. Maybe another writer has created
// the WAL file. Retry.
db.partitionCreateLock.Unlock()
goto FIND_PARTITION
}
p = memory.NewMemoryPartition(w)
db.partitionList.Insert(p)
p.Hold()
if db.partitionList.Size() == 1 {
atomic.SwapInt64(&db.minTimestamp, minTimestampInRows)
atomic.SwapInt64(&db.maxTimestamp, maxTimestampInRows)
}
if !atomic.CompareAndSwapInt64(&db.lastPartitionID, newPartitionID-1, newPartitionID) {
p.Release()
p.Destroy()
db.partitionCreateLock.Unlock()
goto FIND_PARTITION
}
}
if p == nil {
db.partitionCreateLock.Unlock()
goto FIND_PARTITION
}
}
db.partitionCreateLock.Unlock()
VALID_PARTITION:
if p.ReadOnly() {
p.Release()
return errors.New("catena: insert into read-only partition")
}
err := p.InsertRows(*(*[]partition.Row)(unsafe.Pointer(&rows)))
if err != nil {
p.Release()
return err
}
p.Release()
for min := atomic.LoadInt64(&db.minTimestamp); min > minTimestampInRows; min = atomic.LoadInt64(&db.minTimestamp) {
if atomic.CompareAndSwapInt64(&db.minTimestamp, min, minTimestampInRows) {
break
}
}
for max := atomic.LoadInt64(&db.maxTimestamp); max < maxTimestampInRows; max = atomic.LoadInt64(&db.maxTimestamp) {
if atomic.CompareAndSwapInt64(&db.maxTimestamp, max, maxTimestampInRows) {
break
}
}
}
return nil
}
// loadPartitions reads a slice of partition file names
// and updates the internal partition state.
func (db *DB) loadPartitions(names []string) error {
// Slice of partition IDs
partitions := []int{}
isWAL := map[int]bool{}
for _, name := range names {
partitionNum := -1
wal := false
if strings.HasSuffix(name, ".wal") {
_, err := fmt.Sscanf(name, "%d.wal", &partitionNum)
if err != nil {
return err
}
wal = true
}
if strings.HasSuffix(name, ".part") {
_, err := fmt.Sscanf(name, "%d.part", &partitionNum)
if err != nil {
return err
}
}
if partitionNum < 0 {
return errors.New(fmt.Sprintf("catena: invalid partition %s", name))
}
if seenWAL, seen := isWAL[partitionNum]; seen {
if (seenWAL && !wal) || (!seenWAL && wal) {
// We have both a .wal and a .part, so
// we'll get rid of the .part and recompact.
wal = true
err := os.Remove(filepath.Join(db.baseDir, fmt.Sprintf("%d.part", partitionNum)))
if err != nil {
return err
}
}
}
isWAL[partitionNum] = wal
}
for partitionNum := range isWAL {
partitions = append(partitions, partitionNum)
}
// Sort the partitions in increasing order.
sort.Ints(partitions)
for _, part := range partitions {
if int64(part) > db.lastPartitionID {
db.lastPartitionID = int64(part)
}
var p partition.Partition
var err error
var filename string
if isWAL[part] {
filename = filepath.Join(db.baseDir,
fmt.Sprintf("%d.wal", part))
w, err := wal.OpenFileWAL(filename)
if err != nil {
return err
}
p, err = memory.RecoverMemoryPartition(w)
if err != nil {
return err
}
} else {
filename = filepath.Join(db.baseDir,
fmt.Sprintf("%d.part", part))
p, err = disk.OpenDiskPartition(filename)
if err != nil {
return err
}
}
// No need for locks here.
if db.partitionList.Size() == 1 {
db.minTimestamp = p.MinTimestamp()
db.maxTimestamp = p.MaxTimestamp()
}
if db.minTimestamp > p.MinTimestamp() {
db.minTimestamp = p.MinTimestamp()
}
if db.maxTimestamp < p.MaxTimestamp() {
db.maxTimestamp = p.MaxTimestamp()
}
db.partitionList.Insert(p)
}
return nil
}