Beispiel #1
0
// NewIterator creates a new Iterator for the given source and metric.
func (db *DB) NewIterator(source, metric string) (*Iterator, error) {
	var p partition.Partition = nil

	i := db.partitionList.NewIterator()
	for i.Next() {
		val, _ := i.Value()

		val.Hold()

		if val.HasMetric(source, metric) {
			if p != nil {
				p.Release()

			}

			p = val
		} else {
			val.Release()
		}
	}

	if p == nil {
		return nil, errors.New("catena: couldn't find metric for iterator")
	}

	partitionIter, err := p.NewIterator(source, metric)
	p.Release()

	if err != nil {
		return nil, err
	}

	return &Iterator{
		source:       source,
		metric:       metric,
		db:           db,
		curPartition: p,
		Iterator:     partitionIter,
	}, nil
}
Beispiel #2
0
// Reset moves i to the first available timestamp.
func (i *Iterator) Reset() error {
	i.Iterator.Close()

	var p partition.Partition

	partitionListIter := i.db.partitionList.NewIterator()
	for partitionListIter.Next() {
		val, _ := partitionListIter.Value()
		val.Hold()

		if val.HasMetric(i.source, i.metric) {
			if p != nil {
				p.Release()
			}

			p = val
		} else {
			val.Release()
		}
	}

	if p == nil {
		return errors.New("catena: couldn't find metric for iterator")
	}

	defer p.Release()

	i.curPartition = p

	partitionIter, err := p.NewIterator(i.source, i.metric)
	if err != nil {
		return err
	}

	i.Iterator = partitionIter
	return nil
}
Beispiel #3
0
func comparePartitions(a, b partition.Partition) int {
	// Highest timestamp first
	return int(b.MinTimestamp() - a.MinTimestamp())
}
Beispiel #4
0
// InsertRows inserts the given rows into the database.
func (db *DB) InsertRows(rows []Row) error {
	keyToRows := map[int][]Row{}

	for _, row := range rows {
		key := int(row.Timestamp / db.partitionSize)
		keyToRows[key] = append(keyToRows[key], row)
	}

	keys := []int{}
	for key := range keyToRows {
		keys = append(keys, key)
	}
	sort.Ints(keys)

	for _, key := range keys {
		rowsForKey := keyToRows[key]

		minTimestampInRows := int64(0)
		maxTimestampInRows := int64(0)

		for i, row := range rowsForKey {
			if i == 1 {
				minTimestampInRows = row.Timestamp
				maxTimestampInRows = row.Timestamp
			}

			if row.Timestamp > maxTimestampInRows {
				maxTimestampInRows = row.Timestamp
			}

			if row.Timestamp < minTimestampInRows {
				minTimestampInRows = row.Timestamp
			}
		}

		var p partition.Partition

	FIND_PARTITION:

		i := db.partitionList.NewIterator()
		for i.Next() {
			val, _ := i.Value()
			val.Hold()

			if val.MinTimestamp()/db.partitionSize == int64(key) {
				p = val
				goto VALID_PARTITION
			}

			if val.MinTimestamp()/db.partitionSize < int64(key) && val.MaxTimestamp()/db.partitionSize >= int64(key) {
				p = val
				goto VALID_PARTITION
			}

			val.Release()
		}

		db.partitionCreateLock.Lock()
		if p == nil {
			if int64(key) < atomic.LoadInt64(&db.minTimestamp)/db.partitionSize {
				db.partitionCreateLock.Unlock()
				return errors.New("catena: row(s) being inserted are too old")
			}

			if db.partitionList.Size() == 0 ||
				int64(key) > atomic.LoadInt64(&db.maxTimestamp)/db.partitionSize {

				// Need to make a new partition
				newPartitionID := atomic.LoadInt64(&db.lastPartitionID) + 1
				w, err := wal.NewFileWAL(filepath.Join(db.baseDir,
					fmt.Sprintf("%d.wal", newPartitionID)))
				if err != nil {
					// Couldn't create the WAL. Maybe another writer has created
					// the WAL file. Retry.
					db.partitionCreateLock.Unlock()
					goto FIND_PARTITION
				}

				p = memory.NewMemoryPartition(w)
				db.partitionList.Insert(p)
				p.Hold()

				if db.partitionList.Size() == 1 {
					atomic.SwapInt64(&db.minTimestamp, minTimestampInRows)
					atomic.SwapInt64(&db.maxTimestamp, maxTimestampInRows)
				}

				if !atomic.CompareAndSwapInt64(&db.lastPartitionID, newPartitionID-1, newPartitionID) {
					p.Release()
					p.Destroy()
					db.partitionCreateLock.Unlock()
					goto FIND_PARTITION
				}
			}

			if p == nil {
				db.partitionCreateLock.Unlock()
				goto FIND_PARTITION
			}
		}

		db.partitionCreateLock.Unlock()

	VALID_PARTITION:

		if p.ReadOnly() {
			p.Release()
			return errors.New("catena: insert into read-only partition")
		}

		err := p.InsertRows(*(*[]partition.Row)(unsafe.Pointer(&rows)))
		if err != nil {
			p.Release()
			return err
		}

		p.Release()

		for min := atomic.LoadInt64(&db.minTimestamp); min > minTimestampInRows; min = atomic.LoadInt64(&db.minTimestamp) {
			if atomic.CompareAndSwapInt64(&db.minTimestamp, min, minTimestampInRows) {
				break
			}
		}

		for max := atomic.LoadInt64(&db.maxTimestamp); max < maxTimestampInRows; max = atomic.LoadInt64(&db.maxTimestamp) {
			if atomic.CompareAndSwapInt64(&db.maxTimestamp, max, maxTimestampInRows) {
				break
			}
		}
	}

	return nil
}
Beispiel #5
0
// loadPartitions reads a slice of partition file names
// and updates the internal partition state.
func (db *DB) loadPartitions(names []string) error {

	// Slice of partition IDs
	partitions := []int{}

	isWAL := map[int]bool{}

	for _, name := range names {
		partitionNum := -1

		wal := false

		if strings.HasSuffix(name, ".wal") {
			_, err := fmt.Sscanf(name, "%d.wal", &partitionNum)
			if err != nil {
				return err
			}

			wal = true
		}

		if strings.HasSuffix(name, ".part") {
			_, err := fmt.Sscanf(name, "%d.part", &partitionNum)
			if err != nil {
				return err
			}
		}

		if partitionNum < 0 {
			return errors.New(fmt.Sprintf("catena: invalid partition %s", name))
		}

		if seenWAL, seen := isWAL[partitionNum]; seen {
			if (seenWAL && !wal) || (!seenWAL && wal) {
				// We have both a .wal and a .part, so
				// we'll get rid of the .part and recompact.
				wal = true
				err := os.Remove(filepath.Join(db.baseDir, fmt.Sprintf("%d.part", partitionNum)))
				if err != nil {
					return err
				}
			}
		}

		isWAL[partitionNum] = wal
	}

	for partitionNum := range isWAL {
		partitions = append(partitions, partitionNum)
	}

	// Sort the partitions in increasing order.
	sort.Ints(partitions)

	for _, part := range partitions {
		if int64(part) > db.lastPartitionID {
			db.lastPartitionID = int64(part)
		}

		var p partition.Partition
		var err error
		var filename string

		if isWAL[part] {
			filename = filepath.Join(db.baseDir,
				fmt.Sprintf("%d.wal", part))

			w, err := wal.OpenFileWAL(filename)
			if err != nil {
				return err
			}

			p, err = memory.RecoverMemoryPartition(w)
			if err != nil {
				return err
			}

		} else {
			filename = filepath.Join(db.baseDir,
				fmt.Sprintf("%d.part", part))

			p, err = disk.OpenDiskPartition(filename)
			if err != nil {
				return err
			}
		}

		// No need for locks here.

		if db.partitionList.Size() == 1 {
			db.minTimestamp = p.MinTimestamp()
			db.maxTimestamp = p.MaxTimestamp()
		}

		if db.minTimestamp > p.MinTimestamp() {
			db.minTimestamp = p.MinTimestamp()
		}

		if db.maxTimestamp < p.MaxTimestamp() {
			db.maxTimestamp = p.MaxTimestamp()
		}

		db.partitionList.Insert(p)
	}

	return nil
}