// compact drops old partitions and compacts older memory to
// read-only disk partitions.
func (db *DB) compact() {
// Look for partitions to drop
i := db.partitionList.NewIterator()
seen := 0
lastMin := int64(0)
for i.Next() {
p, err := i.Value()
if err != nil {
break
}
seen++
if seen <= db.maxPartitions {
lastMin = p.MinTimestamp()
continue
}
atomic.SwapInt64(&db.minTimestamp, lastMin)
// Remove it from the list
db.partitionList.Remove(p)
// Make sure we're the only ones accessing the partition
p.ExclusiveHold()
p.Destroy()
p.ExclusiveRelease()
}
// Find partitions to compact
toCompact := []partition.Partition{}
seen = 0
i = db.partitionList.NewIterator()
for i.Next() {
seen++
if seen <= 2 {
// Skip the latest two in-memory partitions
continue
}
p, _ := i.Value()
p.Hold()
if !p.ReadOnly() {
p.Release()
p.ExclusiveHold()
p.SetReadOnly()
p.ExclusiveRelease()
toCompact = append(toCompact, p)
} else {
p.Release()
}
}
for _, p := range toCompact {
// p is read-only, so no need to lock.
memPart := p.(*memory.MemoryPartition)
// Create the disk partition file
filename := strings.TrimSuffix(memPart.Filename(), ".wal") + ".part"
f, err := os.Create(filename)
if err != nil {
// ???
return
}
// Compact
err = memPart.Compact(f)
if err != nil {
// ???
return
}
// Close and reopen.
f.Sync()
f.Close()
diskPart, err := disk.OpenDiskPartition(filename)
if err != nil {
// ???
return
}
// Swap the memory partition with the disk partition.
db.partitionList.Swap(memPart, diskPart)
memPart.ExclusiveHold()
memPart.Destroy()
memPart.ExclusiveRelease()
}
}
func TestMemoryPartition1(t *testing.T) {
os.RemoveAll("/tmp/wal.wal")
timestamps := 100
sources := 100
metrics := 100
WAL, err := wal.NewFileWAL("/tmp/wal.wal")
if err != nil {
t.Fatal(err)
}
p := memory.NewMemoryPartition(WAL)
workQueue := make(chan []partition.Row, timestamps*sources)
parallelism := runtime.NumCPU()
runtime.GOMAXPROCS(parallelism)
for i := 0; i < timestamps; i++ {
for j := 0; j < sources; j++ {
rows := make([]partition.Row, metrics)
for k := 0; k < metrics; k++ {
rows[k] = partition.Row{
Source: "source_" + fmt.Sprint(j),
Metric: "metric_" + fmt.Sprint(k),
Point: partition.Point{
Timestamp: int64(i),
Value: 0,
},
}
}
workQueue <- rows
}
}
wg := sync.WaitGroup{}
wg.Add(parallelism)
start := time.Now()
for i := 0; i < parallelism; i++ {
go func() {
for rows := range workQueue {
err := p.InsertRows(rows)
if err != nil {
t.Fatal(err)
}
}
wg.Done()
}()
}
close(workQueue)
wg.Wait()
t.Logf("%0.2f rows / sec\n", float64(timestamps*sources*metrics)/time.Now().Sub(start).Seconds())
i, err := p.NewIterator("source_0", "metric_0")
if err != nil {
t.Fatal(err)
}
expected := int64(0)
for i.Next() == nil {
if i.Point().Timestamp != expected {
t.Fatalf("expected timestamp %d; got %d", expected, i.Point().Timestamp)
}
expected++
}
i.Close()
if expected != int64(timestamps) {
t.Fatal(expected)
}
p.Close()
WAL, err = wal.OpenFileWAL("/tmp/wal.wal")
if err != nil {
t.Fatal(err)
}
start = time.Now()
p, err = memory.RecoverMemoryPartition(WAL)
if err != nil {
t.Fatal(err)
}
t.Logf("%0.2f rows / sec\n", float64(timestamps*sources*metrics)/time.Now().Sub(start).Seconds())
expected = int64(0)
i, err = p.NewIterator("source_0", "metric_0")
if err != nil {
t.Fatal(err)
}
for i.Next() == nil {
if i.Point().Timestamp != expected {
t.Fatalf("expected timestamp %d; got %d", expected, i.Point().Timestamp)
}
expected++
}
i.Close()
if expected != int64(timestamps) {
t.Fatal(expected)
}
p.SetReadOnly()
f, err := os.Create("/tmp/compact.part")
if err != nil {
p.Destroy()
t.Fatal(err)
}
err = p.Compact(f)
if err != nil {
p.Destroy()
t.Fatal(err)
}
err = p.Destroy()
if err != nil {
t.Fatal(err)
}
f.Close()
d, err := disk.OpenDiskPartition("/tmp/compact.part")
if err != nil {
t.Fatal(err)
}
diskIter, err := d.NewIterator("source_0", "metric_0")
if err != nil {
t.Fatal(err)
}
expected = 0
for diskIter.Next() == nil {
if diskIter.Point().Timestamp != expected {
t.Fatalf("expected timestamp %d; got %d", expected, diskIter.Point().Timestamp)
}
expected++
}
diskIter.Close()
err = d.Destroy()
if err != nil {
t.Fatal(err)
}
}
// loadPartitions reads a slice of partition file names
// and updates the internal partition state.
func (db *DB) loadPartitions(names []string) error {
// Slice of partition IDs
partitions := []int{}
isWAL := map[int]bool{}
for _, name := range names {
partitionNum := -1
wal := false
if strings.HasSuffix(name, ".wal") {
_, err := fmt.Sscanf(name, "%d.wal", &partitionNum)
if err != nil {
return err
}
wal = true
}
if strings.HasSuffix(name, ".part") {
_, err := fmt.Sscanf(name, "%d.part", &partitionNum)
if err != nil {
return err
}
}
if partitionNum < 0 {
return errors.New(fmt.Sprintf("catena: invalid partition %s", name))
}
if seenWAL, seen := isWAL[partitionNum]; seen {
if (seenWAL && !wal) || (!seenWAL && wal) {
// We have both a .wal and a .part, so
// we'll get rid of the .part and recompact.
wal = true
err := os.Remove(filepath.Join(db.baseDir, fmt.Sprintf("%d.part", partitionNum)))
if err != nil {
return err
}
}
}
isWAL[partitionNum] = wal
}
for partitionNum := range isWAL {
partitions = append(partitions, partitionNum)
}
// Sort the partitions in increasing order.
sort.Ints(partitions)
for _, part := range partitions {
if int64(part) > db.lastPartitionID {
db.lastPartitionID = int64(part)
}
var p partition.Partition
var err error
var filename string
if isWAL[part] {
filename = filepath.Join(db.baseDir,
fmt.Sprintf("%d.wal", part))
w, err := wal.OpenFileWAL(filename)
if err != nil {
return err
}
p, err = memory.RecoverMemoryPartition(w)
if err != nil {
return err
}
} else {
filename = filepath.Join(db.baseDir,
fmt.Sprintf("%d.part", part))
p, err = disk.OpenDiskPartition(filename)
if err != nil {
return err
}
}
// No need for locks here.
if db.partitionList.Size() == 1 {
db.minTimestamp = p.MinTimestamp()
db.maxTimestamp = p.MaxTimestamp()
}
if db.minTimestamp > p.MinTimestamp() {
db.minTimestamp = p.MinTimestamp()
}
if db.maxTimestamp < p.MaxTimestamp() {
db.maxTimestamp = p.MaxTimestamp()
}
db.partitionList.Insert(p)
}
return nil
}