예제 #1
0
func TestMemoryPartition1(t *testing.T) {
	os.RemoveAll("/tmp/wal.wal")
	timestamps := 100
	sources := 100
	metrics := 100

	WAL, err := wal.NewFileWAL("/tmp/wal.wal")
	if err != nil {
		t.Fatal(err)
	}

	p := memory.NewMemoryPartition(WAL)

	workQueue := make(chan []partition.Row, timestamps*sources)

	parallelism := runtime.NumCPU()
	runtime.GOMAXPROCS(parallelism)

	for i := 0; i < timestamps; i++ {
		for j := 0; j < sources; j++ {

			rows := make([]partition.Row, metrics)

			for k := 0; k < metrics; k++ {
				rows[k] = partition.Row{
					Source: "source_" + fmt.Sprint(j),
					Metric: "metric_" + fmt.Sprint(k),
					Point: partition.Point{
						Timestamp: int64(i),
						Value:     0,
					},
				}
			}

			workQueue <- rows
		}
	}

	wg := sync.WaitGroup{}
	wg.Add(parallelism)

	start := time.Now()

	for i := 0; i < parallelism; i++ {
		go func() {
			for rows := range workQueue {
				err := p.InsertRows(rows)
				if err != nil {
					t.Fatal(err)
				}
			}
			wg.Done()
		}()
	}

	close(workQueue)

	wg.Wait()

	t.Logf("%0.2f rows / sec\n", float64(timestamps*sources*metrics)/time.Now().Sub(start).Seconds())

	i, err := p.NewIterator("source_0", "metric_0")
	if err != nil {
		t.Fatal(err)
	}

	expected := int64(0)
	for i.Next() == nil {
		if i.Point().Timestamp != expected {
			t.Fatalf("expected timestamp %d; got %d", expected, i.Point().Timestamp)
		}

		expected++
	}
	i.Close()
	if expected != int64(timestamps) {
		t.Fatal(expected)
	}

	p.Close()

	WAL, err = wal.OpenFileWAL("/tmp/wal.wal")
	if err != nil {
		t.Fatal(err)
	}

	start = time.Now()

	p, err = memory.RecoverMemoryPartition(WAL)
	if err != nil {
		t.Fatal(err)
	}

	t.Logf("%0.2f rows / sec\n", float64(timestamps*sources*metrics)/time.Now().Sub(start).Seconds())

	expected = int64(0)
	i, err = p.NewIterator("source_0", "metric_0")
	if err != nil {
		t.Fatal(err)
	}
	for i.Next() == nil {
		if i.Point().Timestamp != expected {
			t.Fatalf("expected timestamp %d; got %d", expected, i.Point().Timestamp)
		}

		expected++
	}
	i.Close()

	if expected != int64(timestamps) {
		t.Fatal(expected)
	}

	p.SetReadOnly()

	f, err := os.Create("/tmp/compact.part")
	if err != nil {
		p.Destroy()
		t.Fatal(err)
	}

	err = p.Compact(f)
	if err != nil {
		p.Destroy()
		t.Fatal(err)
	}

	err = p.Destroy()
	if err != nil {
		t.Fatal(err)
	}

	f.Close()

	d, err := disk.OpenDiskPartition("/tmp/compact.part")
	if err != nil {
		t.Fatal(err)
	}

	diskIter, err := d.NewIterator("source_0", "metric_0")
	if err != nil {
		t.Fatal(err)
	}

	expected = 0
	for diskIter.Next() == nil {
		if diskIter.Point().Timestamp != expected {
			t.Fatalf("expected timestamp %d; got %d", expected, diskIter.Point().Timestamp)
		}

		expected++
	}
	diskIter.Close()

	err = d.Destroy()
	if err != nil {
		t.Fatal(err)
	}
}
예제 #2
0
파일: insert.go 프로젝트: jmptrader/catena
// InsertRows inserts the given rows into the database.
func (db *DB) InsertRows(rows []Row) error {
	keyToRows := map[int][]Row{}

	for _, row := range rows {
		key := int(row.Timestamp / db.partitionSize)
		keyToRows[key] = append(keyToRows[key], row)
	}

	keys := []int{}
	for key := range keyToRows {
		keys = append(keys, key)
	}
	sort.Ints(keys)

	for _, key := range keys {
		rowsForKey := keyToRows[key]

		minTimestampInRows := int64(0)
		maxTimestampInRows := int64(0)

		for i, row := range rowsForKey {
			if i == 1 {
				minTimestampInRows = row.Timestamp
				maxTimestampInRows = row.Timestamp
			}

			if row.Timestamp > maxTimestampInRows {
				maxTimestampInRows = row.Timestamp
			}

			if row.Timestamp < minTimestampInRows {
				minTimestampInRows = row.Timestamp
			}
		}

		var p partition.Partition

	FIND_PARTITION:

		i := db.partitionList.NewIterator()
		for i.Next() {
			val, _ := i.Value()
			val.Hold()

			if val.MinTimestamp()/db.partitionSize == int64(key) {
				p = val
				goto VALID_PARTITION
			}

			if val.MinTimestamp()/db.partitionSize < int64(key) && val.MaxTimestamp()/db.partitionSize >= int64(key) {
				p = val
				goto VALID_PARTITION
			}

			val.Release()
		}

		db.partitionCreateLock.Lock()
		if p == nil {
			if int64(key) < atomic.LoadInt64(&db.minTimestamp)/db.partitionSize {
				db.partitionCreateLock.Unlock()
				return errors.New("catena: row(s) being inserted are too old")
			}

			if db.partitionList.Size() == 0 ||
				int64(key) > atomic.LoadInt64(&db.maxTimestamp)/db.partitionSize {

				// Need to make a new partition
				newPartitionID := atomic.LoadInt64(&db.lastPartitionID) + 1
				w, err := wal.NewFileWAL(filepath.Join(db.baseDir,
					fmt.Sprintf("%d.wal", newPartitionID)))
				if err != nil {
					// Couldn't create the WAL. Maybe another writer has created
					// the WAL file. Retry.
					db.partitionCreateLock.Unlock()
					goto FIND_PARTITION
				}

				p = memory.NewMemoryPartition(w)
				db.partitionList.Insert(p)
				p.Hold()

				if db.partitionList.Size() == 1 {
					atomic.SwapInt64(&db.minTimestamp, minTimestampInRows)
					atomic.SwapInt64(&db.maxTimestamp, maxTimestampInRows)
				}

				if !atomic.CompareAndSwapInt64(&db.lastPartitionID, newPartitionID-1, newPartitionID) {
					p.Release()
					p.Destroy()
					db.partitionCreateLock.Unlock()
					goto FIND_PARTITION
				}
			}

			if p == nil {
				db.partitionCreateLock.Unlock()
				goto FIND_PARTITION
			}
		}

		db.partitionCreateLock.Unlock()

	VALID_PARTITION:

		if p.ReadOnly() {
			p.Release()
			return errors.New("catena: insert into read-only partition")
		}

		err := p.InsertRows(*(*[]partition.Row)(unsafe.Pointer(&rows)))
		if err != nil {
			p.Release()
			return err
		}

		p.Release()

		for min := atomic.LoadInt64(&db.minTimestamp); min > minTimestampInRows; min = atomic.LoadInt64(&db.minTimestamp) {
			if atomic.CompareAndSwapInt64(&db.minTimestamp, min, minTimestampInRows) {
				break
			}
		}

		for max := atomic.LoadInt64(&db.maxTimestamp); max < maxTimestampInRows; max = atomic.LoadInt64(&db.maxTimestamp) {
			if atomic.CompareAndSwapInt64(&db.maxTimestamp, max, maxTimestampInRows) {
				break
			}
		}
	}

	return nil
}