func TestMemoryPartition1(t *testing.T) { os.RemoveAll("/tmp/wal.wal") timestamps := 100 sources := 100 metrics := 100 WAL, err := wal.NewFileWAL("/tmp/wal.wal") if err != nil { t.Fatal(err) } p := memory.NewMemoryPartition(WAL) workQueue := make(chan []partition.Row, timestamps*sources) parallelism := runtime.NumCPU() runtime.GOMAXPROCS(parallelism) for i := 0; i < timestamps; i++ { for j := 0; j < sources; j++ { rows := make([]partition.Row, metrics) for k := 0; k < metrics; k++ { rows[k] = partition.Row{ Source: "source_" + fmt.Sprint(j), Metric: "metric_" + fmt.Sprint(k), Point: partition.Point{ Timestamp: int64(i), Value: 0, }, } } workQueue <- rows } } wg := sync.WaitGroup{} wg.Add(parallelism) start := time.Now() for i := 0; i < parallelism; i++ { go func() { for rows := range workQueue { err := p.InsertRows(rows) if err != nil { t.Fatal(err) } } wg.Done() }() } close(workQueue) wg.Wait() t.Logf("%0.2f rows / sec\n", float64(timestamps*sources*metrics)/time.Now().Sub(start).Seconds()) i, err := p.NewIterator("source_0", "metric_0") if err != nil { t.Fatal(err) } expected := int64(0) for i.Next() == nil { if i.Point().Timestamp != expected { t.Fatalf("expected timestamp %d; got %d", expected, i.Point().Timestamp) } expected++ } i.Close() if expected != int64(timestamps) { t.Fatal(expected) } p.Close() WAL, err = wal.OpenFileWAL("/tmp/wal.wal") if err != nil { t.Fatal(err) } start = time.Now() p, err = memory.RecoverMemoryPartition(WAL) if err != nil { t.Fatal(err) } t.Logf("%0.2f rows / sec\n", float64(timestamps*sources*metrics)/time.Now().Sub(start).Seconds()) expected = int64(0) i, err = p.NewIterator("source_0", "metric_0") if err != nil { t.Fatal(err) } for i.Next() == nil { if i.Point().Timestamp != expected { t.Fatalf("expected timestamp %d; got %d", expected, i.Point().Timestamp) } expected++ } i.Close() if expected != int64(timestamps) { t.Fatal(expected) } p.SetReadOnly() f, err := os.Create("/tmp/compact.part") if err != nil { p.Destroy() t.Fatal(err) } err = p.Compact(f) if err != nil { p.Destroy() t.Fatal(err) } err = p.Destroy() if err != nil { t.Fatal(err) } f.Close() d, err := disk.OpenDiskPartition("/tmp/compact.part") if err != nil { t.Fatal(err) } diskIter, err := d.NewIterator("source_0", "metric_0") if err != nil { t.Fatal(err) } expected = 0 for diskIter.Next() == nil { if diskIter.Point().Timestamp != expected { t.Fatalf("expected timestamp %d; got %d", expected, diskIter.Point().Timestamp) } expected++ } diskIter.Close() err = d.Destroy() if err != nil { t.Fatal(err) } }
// loadPartitions reads a slice of partition file names // and updates the internal partition state. func (db *DB) loadPartitions(names []string) error { // Slice of partition IDs partitions := []int{} isWAL := map[int]bool{} for _, name := range names { partitionNum := -1 wal := false if strings.HasSuffix(name, ".wal") { _, err := fmt.Sscanf(name, "%d.wal", &partitionNum) if err != nil { return err } wal = true } if strings.HasSuffix(name, ".part") { _, err := fmt.Sscanf(name, "%d.part", &partitionNum) if err != nil { return err } } if partitionNum < 0 { return errors.New(fmt.Sprintf("catena: invalid partition %s", name)) } if seenWAL, seen := isWAL[partitionNum]; seen { if (seenWAL && !wal) || (!seenWAL && wal) { // We have both a .wal and a .part, so // we'll get rid of the .part and recompact. wal = true err := os.Remove(filepath.Join(db.baseDir, fmt.Sprintf("%d.part", partitionNum))) if err != nil { return err } } } isWAL[partitionNum] = wal } for partitionNum := range isWAL { partitions = append(partitions, partitionNum) } // Sort the partitions in increasing order. sort.Ints(partitions) for _, part := range partitions { if int64(part) > db.lastPartitionID { db.lastPartitionID = int64(part) } var p partition.Partition var err error var filename string if isWAL[part] { filename = filepath.Join(db.baseDir, fmt.Sprintf("%d.wal", part)) w, err := wal.OpenFileWAL(filename) if err != nil { return err } p, err = memory.RecoverMemoryPartition(w) if err != nil { return err } } else { filename = filepath.Join(db.baseDir, fmt.Sprintf("%d.part", part)) p, err = disk.OpenDiskPartition(filename) if err != nil { return err } } // No need for locks here. if db.partitionList.Size() == 1 { db.minTimestamp = p.MinTimestamp() db.maxTimestamp = p.MaxTimestamp() } if db.minTimestamp > p.MinTimestamp() { db.minTimestamp = p.MinTimestamp() } if db.maxTimestamp < p.MaxTimestamp() { db.maxTimestamp = p.MaxTimestamp() } db.partitionList.Insert(p) } return nil }