Exemplo n.º 1
0
// compact drops old partitions and compacts older memory to
// read-only disk partitions.
func (db *DB) compact() {

	// Look for partitions to drop
	i := db.partitionList.NewIterator()
	seen := 0
	lastMin := int64(0)
	for i.Next() {
		p, err := i.Value()
		if err != nil {
			break
		}

		seen++
		if seen <= db.maxPartitions {
			lastMin = p.MinTimestamp()
			continue
		}

		atomic.SwapInt64(&db.minTimestamp, lastMin)

		// Remove it from the list
		db.partitionList.Remove(p)

		// Make sure we're the only ones accessing the partition
		p.ExclusiveHold()
		p.Destroy()
		p.ExclusiveRelease()
	}

	// Find partitions to compact
	toCompact := []partition.Partition{}

	seen = 0
	i = db.partitionList.NewIterator()
	for i.Next() {
		seen++
		if seen <= 2 {
			// Skip the latest two in-memory partitions
			continue
		}

		p, _ := i.Value()

		p.Hold()
		if !p.ReadOnly() {
			p.Release()
			p.ExclusiveHold()
			p.SetReadOnly()
			p.ExclusiveRelease()

			toCompact = append(toCompact, p)
		} else {
			p.Release()
		}
	}

	for _, p := range toCompact {
		// p is read-only, so no need to lock.
		memPart := p.(*memory.MemoryPartition)

		// Create the disk partition file
		filename := strings.TrimSuffix(memPart.Filename(), ".wal") + ".part"
		f, err := os.Create(filename)
		if err != nil {
			// ???
			return
		}

		// Compact
		err = memPart.Compact(f)
		if err != nil {
			// ???
			return
		}

		// Close and reopen.
		f.Sync()
		f.Close()

		diskPart, err := disk.OpenDiskPartition(filename)
		if err != nil {
			// ???
			return
		}

		// Swap the memory partition with the disk partition.
		db.partitionList.Swap(memPart, diskPart)

		memPart.ExclusiveHold()
		memPart.Destroy()
		memPart.ExclusiveRelease()
	}
}
Exemplo n.º 2
0
func TestMemoryPartition1(t *testing.T) {
	os.RemoveAll("/tmp/wal.wal")
	timestamps := 100
	sources := 100
	metrics := 100

	WAL, err := wal.NewFileWAL("/tmp/wal.wal")
	if err != nil {
		t.Fatal(err)
	}

	p := memory.NewMemoryPartition(WAL)

	workQueue := make(chan []partition.Row, timestamps*sources)

	parallelism := runtime.NumCPU()
	runtime.GOMAXPROCS(parallelism)

	for i := 0; i < timestamps; i++ {
		for j := 0; j < sources; j++ {

			rows := make([]partition.Row, metrics)

			for k := 0; k < metrics; k++ {
				rows[k] = partition.Row{
					Source: "source_" + fmt.Sprint(j),
					Metric: "metric_" + fmt.Sprint(k),
					Point: partition.Point{
						Timestamp: int64(i),
						Value:     0,
					},
				}
			}

			workQueue <- rows
		}
	}

	wg := sync.WaitGroup{}
	wg.Add(parallelism)

	start := time.Now()

	for i := 0; i < parallelism; i++ {
		go func() {
			for rows := range workQueue {
				err := p.InsertRows(rows)
				if err != nil {
					t.Fatal(err)
				}
			}
			wg.Done()
		}()
	}

	close(workQueue)

	wg.Wait()

	t.Logf("%0.2f rows / sec\n", float64(timestamps*sources*metrics)/time.Now().Sub(start).Seconds())

	i, err := p.NewIterator("source_0", "metric_0")
	if err != nil {
		t.Fatal(err)
	}

	expected := int64(0)
	for i.Next() == nil {
		if i.Point().Timestamp != expected {
			t.Fatalf("expected timestamp %d; got %d", expected, i.Point().Timestamp)
		}

		expected++
	}
	i.Close()
	if expected != int64(timestamps) {
		t.Fatal(expected)
	}

	p.Close()

	WAL, err = wal.OpenFileWAL("/tmp/wal.wal")
	if err != nil {
		t.Fatal(err)
	}

	start = time.Now()

	p, err = memory.RecoverMemoryPartition(WAL)
	if err != nil {
		t.Fatal(err)
	}

	t.Logf("%0.2f rows / sec\n", float64(timestamps*sources*metrics)/time.Now().Sub(start).Seconds())

	expected = int64(0)
	i, err = p.NewIterator("source_0", "metric_0")
	if err != nil {
		t.Fatal(err)
	}
	for i.Next() == nil {
		if i.Point().Timestamp != expected {
			t.Fatalf("expected timestamp %d; got %d", expected, i.Point().Timestamp)
		}

		expected++
	}
	i.Close()

	if expected != int64(timestamps) {
		t.Fatal(expected)
	}

	p.SetReadOnly()

	f, err := os.Create("/tmp/compact.part")
	if err != nil {
		p.Destroy()
		t.Fatal(err)
	}

	err = p.Compact(f)
	if err != nil {
		p.Destroy()
		t.Fatal(err)
	}

	err = p.Destroy()
	if err != nil {
		t.Fatal(err)
	}

	f.Close()

	d, err := disk.OpenDiskPartition("/tmp/compact.part")
	if err != nil {
		t.Fatal(err)
	}

	diskIter, err := d.NewIterator("source_0", "metric_0")
	if err != nil {
		t.Fatal(err)
	}

	expected = 0
	for diskIter.Next() == nil {
		if diskIter.Point().Timestamp != expected {
			t.Fatalf("expected timestamp %d; got %d", expected, diskIter.Point().Timestamp)
		}

		expected++
	}
	diskIter.Close()

	err = d.Destroy()
	if err != nil {
		t.Fatal(err)
	}
}
Exemplo n.º 3
0
// loadPartitions reads a slice of partition file names
// and updates the internal partition state.
func (db *DB) loadPartitions(names []string) error {

	// Slice of partition IDs
	partitions := []int{}

	isWAL := map[int]bool{}

	for _, name := range names {
		partitionNum := -1

		wal := false

		if strings.HasSuffix(name, ".wal") {
			_, err := fmt.Sscanf(name, "%d.wal", &partitionNum)
			if err != nil {
				return err
			}

			wal = true
		}

		if strings.HasSuffix(name, ".part") {
			_, err := fmt.Sscanf(name, "%d.part", &partitionNum)
			if err != nil {
				return err
			}
		}

		if partitionNum < 0 {
			return errors.New(fmt.Sprintf("catena: invalid partition %s", name))
		}

		if seenWAL, seen := isWAL[partitionNum]; seen {
			if (seenWAL && !wal) || (!seenWAL && wal) {
				// We have both a .wal and a .part, so
				// we'll get rid of the .part and recompact.
				wal = true
				err := os.Remove(filepath.Join(db.baseDir, fmt.Sprintf("%d.part", partitionNum)))
				if err != nil {
					return err
				}
			}
		}

		isWAL[partitionNum] = wal
	}

	for partitionNum := range isWAL {
		partitions = append(partitions, partitionNum)
	}

	// Sort the partitions in increasing order.
	sort.Ints(partitions)

	for _, part := range partitions {
		if int64(part) > db.lastPartitionID {
			db.lastPartitionID = int64(part)
		}

		var p partition.Partition
		var err error
		var filename string

		if isWAL[part] {
			filename = filepath.Join(db.baseDir,
				fmt.Sprintf("%d.wal", part))

			w, err := wal.OpenFileWAL(filename)
			if err != nil {
				return err
			}

			p, err = memory.RecoverMemoryPartition(w)
			if err != nil {
				return err
			}

		} else {
			filename = filepath.Join(db.baseDir,
				fmt.Sprintf("%d.part", part))

			p, err = disk.OpenDiskPartition(filename)
			if err != nil {
				return err
			}
		}

		// No need for locks here.

		if db.partitionList.Size() == 1 {
			db.minTimestamp = p.MinTimestamp()
			db.maxTimestamp = p.MaxTimestamp()
		}

		if db.minTimestamp > p.MinTimestamp() {
			db.minTimestamp = p.MinTimestamp()
		}

		if db.maxTimestamp < p.MaxTimestamp() {
			db.maxTimestamp = p.MaxTimestamp()
		}

		db.partitionList.Insert(p)
	}

	return nil
}