// startTestWriter creates a writer which initiates a sequence of // transactions, each which writes up to 10 times to random keys with // random values. If not nil, txnChannel is written to non-blockingly // every time a new transaction starts. func startTestWriter( db *client.DB, i int64, valBytes int32, wg *sync.WaitGroup, retries *int32, txnChannel chan struct{}, done <-chan struct{}, t *testing.T, ) { src := rand.New(rand.NewSource(i)) defer func() { if wg != nil { wg.Done() } }() for j := 0; ; j++ { select { case <-done: return default: first := true err := db.Txn(context.TODO(), func(txn *client.Txn) error { if first && txnChannel != nil { select { case txnChannel <- struct{}{}: default: } } else if !first && retries != nil { atomic.AddInt32(retries, 1) } first = false for j := 0; j <= int(src.Int31n(10)); j++ { key := randutil.RandBytes(src, 10) val := randutil.RandBytes(src, int(src.Int31n(valBytes))) if err := txn.Put(key, val); err != nil { log.Infof(context.Background(), "experienced an error in routine %d: %s", i, err) return err } } return nil }) if err != nil { t.Error(err) } else { time.Sleep(1 * time.Millisecond) } } } }
func runMVCCConditionalPut(emk engineMaker, valueSize int, createFirst bool, b *testing.B) { rng, _ := randutil.NewPseudoRand() value := roachpb.MakeValueFromBytes(randutil.RandBytes(rng, valueSize)) keyBuf := append(make([]byte, 0, 64), []byte("key-")...) eng := emk(b, fmt.Sprintf("cput_%d", valueSize)) defer eng.Close() b.SetBytes(int64(valueSize)) var expected *roachpb.Value if createFirst { for i := 0; i < b.N; i++ { key := roachpb.Key(encoding.EncodeUvarintAscending(keyBuf[:4], uint64(i))) ts := makeTS(timeutil.Now().UnixNano(), 0) if err := MVCCPut(context.Background(), eng, nil, key, ts, value, nil); err != nil { b.Fatalf("failed put: %s", err) } } expected = &value } b.ResetTimer() for i := 0; i < b.N; i++ { key := roachpb.Key(encoding.EncodeUvarintAscending(keyBuf[:4], uint64(i))) ts := makeTS(timeutil.Now().UnixNano(), 0) if err := MVCCConditionalPut(context.Background(), eng, nil, key, ts, value, expected, nil); err != nil { b.Fatalf("failed put: %s", err) } } b.StopTimer() }
func TestRandBytes(t *testing.T) { rand, _ := randutil.NewPseudoRand() for i := 0; i < 100; i++ { x := randutil.RandBytes(rand, i) if len(x) != i { t.Errorf("got array with unexpected length: %d (expected %d)", len(x), i) } } }
func fillTestRange(t testing.TB, rep *Replica, size int64) { src := rand.New(rand.NewSource(0)) for i := int64(0); i < size/int64(keySize+valSize); i++ { key := keys.MakeRowSentinelKey(randutil.RandBytes(src, keySize)) val := randutil.RandBytes(src, valSize) pArgs := putArgs(key, val) if _, pErr := client.SendWrappedWith(context.Background(), rep, roachpb.Header{ RangeID: rangeID, }, &pArgs); pErr != nil { t.Fatal(pErr) } } rep.mu.Lock() after := rep.mu.state.Stats.Total() rep.mu.Unlock() if after < size { t.Fatalf("range not full after filling: wrote %d, but range at %d", size, after) } }
func testPutInner(ctx context.Context, t *testing.T, c cluster.Cluster, cfg cluster.TestConfig) { db, err := c.NewClient(ctx, 0) if err != nil { t.Fatal(err) } errs := make(chan error, c.NumNodes()) start := timeutil.Now() deadline := start.Add(cfg.Duration) var count int64 for i := 0; i < c.NumNodes(); i++ { go func() { r, _ := randutil.NewPseudoRand() value := randutil.RandBytes(r, 8192) for timeutil.Now().Before(deadline) { k := atomic.AddInt64(&count, 1) v := value[:r.Intn(len(value))] if err := db.Put(ctx, fmt.Sprintf("%08d", k), v); err != nil { errs <- err return } } errs <- nil }() } for i := 0; i < c.NumNodes(); { baseCount := atomic.LoadInt64(&count) select { case <-stopper.ShouldStop(): t.Fatalf("interrupted") case err := <-errs: if err != nil { t.Fatal(err) } i++ case <-time.After(1 * time.Second): // Periodically print out progress so that we know the test is still // running. loadedCount := atomic.LoadInt64(&count) log.Infof(ctx, "%d (%d/s)", loadedCount, loadedCount-baseCount) c.Assert(ctx, t) if err := cluster.Consistent(ctx, c, 0); err != nil { t.Fatal(err) } } } elapsed := timeutil.Since(start) log.Infof(ctx, "%d %.1f/sec", count, float64(count)/elapsed.Seconds()) }
func bankDataInsertStmts(count int) []string { rng, _ := randutil.NewPseudoRand() var statements []string var insert bytes.Buffer for i := 0; i < count; i += 1000 { insert.Reset() insert.WriteString(`INSERT INTO bench.bank VALUES `) for j := i; j < i+1000 && j < count; j++ { if j != i { insert.WriteRune(',') } payload := randutil.RandBytes(rng, backupRestoreRowPayloadSize) fmt.Fprintf(&insert, `(%d, %d, '%s')`, j, 0, payload) } statements = append(statements, insert.String()) } return statements }
func runBatchApplyBatchRepr( emk engineMaker, writeOnly bool, valueSize, batchSize int, b *testing.B, ) { rng, _ := randutil.NewPseudoRand() value := roachpb.MakeValueFromBytes(randutil.RandBytes(rng, valueSize)) keyBuf := append(make([]byte, 0, 64), []byte("key-")...) eng := emk(b, fmt.Sprintf("batch_apply_batch_repr_%d_%d", valueSize, batchSize)) defer eng.Close() var repr []byte { batch := eng.NewBatch() for i := 0; i < batchSize; i++ { key := roachpb.Key(encoding.EncodeUvarintAscending(keyBuf[:4], uint64(i))) ts := makeTS(timeutil.Now().UnixNano(), 0) if err := MVCCPut(context.Background(), batch, nil, key, ts, value, nil); err != nil { b.Fatal(err) } } repr = batch.Repr() batch.Close() } b.SetBytes(int64(len(repr))) b.ResetTimer() for i := 0; i < b.N; i++ { var batch Batch if writeOnly { batch = eng.NewWriteOnlyBatch() } else { batch = eng.NewBatch() } if err := batch.ApplyBatchRepr(repr); err != nil { b.Fatal(err) } batch.Close() } b.StopTimer() }
func BenchmarkMVCCPutDelete_RocksDB(b *testing.B) { rocksdb := setupMVCCInMemRocksDB(b, "put_delete") defer rocksdb.Close() r := rand.New(rand.NewSource(int64(timeutil.Now().UnixNano()))) value := roachpb.MakeValueFromBytes(randutil.RandBytes(r, 10)) zeroTS := hlc.ZeroTimestamp var blockNum int64 for i := 0; i < b.N; i++ { blockID := r.Int63() blockNum++ key := encoding.EncodeVarintAscending(nil, blockID) key = encoding.EncodeVarintAscending(key, blockNum) if err := MVCCPut(context.Background(), rocksdb, nil, key, zeroTS, value, nil /* txn */); err != nil { b.Fatal(err) } if err := MVCCDelete(context.Background(), rocksdb, nil, key, zeroTS, nil /* txn */); err != nil { b.Fatal(err) } } }
func runMVCCBatchPut(emk engineMaker, valueSize, batchSize int, b *testing.B) { rng, _ := randutil.NewPseudoRand() value := roachpb.MakeValueFromBytes(randutil.RandBytes(rng, valueSize)) keyBuf := append(make([]byte, 0, 64), []byte("key-")...) eng := emk(b, fmt.Sprintf("batch_put_%d_%d", valueSize, batchSize)) defer eng.Close() b.SetBytes(int64(valueSize)) b.ResetTimer() for i := 0; i < b.N; i += batchSize { end := i + batchSize if end > b.N { end = b.N } batch := eng.NewBatch() for j := i; j < end; j++ { key := roachpb.Key(encoding.EncodeUvarintAscending(keyBuf[:4], uint64(j))) ts := makeTS(timeutil.Now().UnixNano(), 0) if err := MVCCPut(context.Background(), batch, nil, key, ts, value, nil); err != nil { b.Fatalf("failed put: %s", err) } } if err := batch.Commit(); err != nil { b.Fatal(err) } batch.Close() } b.StopTimer() }
// setupMVCCData writes up to numVersions values at each of numKeys // keys. The number of versions written for each key is chosen // randomly according to a uniform distribution. Each successive // version is written starting at 5ns and then in 5ns increments. This // allows scans at various times, starting at t=5ns, and continuing to // t=5ns*(numVersions+1). A version for each key will be read on every // such scan, but the dynamics of the scan will change depending on // the historical timestamp. Earlier timestamps mean scans which must // skip more historical versions; later timestamps mean scans which // skip fewer. // // The creation of the database is time consuming, especially for larger // numbers of versions. The database is persisted between runs and stored in // the current directory as "mvcc_scan_<versions>_<keys>_<valueBytes>" (which // is also returned). func setupMVCCData( emk engineMaker, numVersions, numKeys, valueBytes int, b *testing.B, ) (Engine, string) { loc := fmt.Sprintf("mvcc_data_%d_%d_%d", numVersions, numKeys, valueBytes) exists := true if _, err := os.Stat(loc); os.IsNotExist(err) { exists = false } eng := emk(b, loc) if exists { readAllFiles(filepath.Join(loc, "*")) return eng, loc } log.Infof(context.Background(), "creating mvcc data: %s", loc) // Generate the same data every time. rng := rand.New(rand.NewSource(1449168817)) keys := make([]roachpb.Key, numKeys) var order []int for i := 0; i < numKeys; i++ { keys[i] = roachpb.Key(encoding.EncodeUvarintAscending([]byte("key-"), uint64(i))) keyVersions := rng.Intn(numVersions) + 1 for j := 0; j < keyVersions; j++ { order = append(order, i) } } // Randomize the order in which the keys are written. for i, n := 0, len(order); i < n-1; i++ { j := i + rng.Intn(n-i) order[i], order[j] = order[j], order[i] } counts := make([]int, numKeys) batch := eng.NewBatch() for i, idx := range order { // Output the keys in ~20 batches. If we used a single batch to output all // of the keys rocksdb would create a single sstable. We want multiple // sstables in order to exercise filtering of which sstables are examined // during iterator seeking. We fix the number of batches we output so that // optimizations which change the data size result in the same number of // sstables. if scaled := len(order) / 20; i > 0 && (i%scaled) == 0 { log.Infof(context.Background(), "committing (%d/~%d)", i/scaled, 20) if err := batch.Commit(); err != nil { b.Fatal(err) } batch.Close() batch = eng.NewBatch() if err := eng.Flush(); err != nil { b.Fatal(err) } } key := keys[idx] ts := makeTS(int64(counts[idx]+1)*5, 0) counts[idx]++ value := roachpb.MakeValueFromBytes(randutil.RandBytes(rng, valueBytes)) value.InitChecksum(key) if err := MVCCPut(context.Background(), batch, nil, key, ts, value, nil); err != nil { b.Fatal(err) } } if err := batch.Commit(); err != nil { b.Fatal(err) } batch.Close() if err := eng.Flush(); err != nil { b.Fatal(err) } return eng, loc }
func TestReadWriteBlocks(t *testing.T) { db, stop := initTestDB(t) defer stop() id := uint64(10) rng, _ := randutil.NewPseudoRand() length := BlockSize*3 + 500 part1 := randutil.RandBytes(rng, length) if err := write(db, id, 0, 0, part1); err != nil { t.Fatal(err) } readData, err := read(db, id, 0, uint64(length)) if err != nil { t.Fatal(err) } if !bytes.Equal(part1, readData) { t.Errorf("Bytes differ. lengths: %d, expected %d", len(readData), len(part1)) } verboseData, err := getAllBlocks(db, id) if err != nil { t.Fatal(err) } if !bytes.Equal(verboseData, part1) { t.Errorf("Bytes differ. lengths: %d, expected %d", len(verboseData), len(part1)) } // Write with hole in the middle. part2 := make([]byte, BlockSize*2+250, BlockSize*2+250) fullData := append(part1, part2...) part3 := randutil.RandBytes(rng, BlockSize+123) if err := write(db, id, uint64(len(part1)), uint64(len(fullData)), part3); err != nil { t.Fatal(err) } fullData = append(fullData, part3...) readData, err = read(db, id, 0, uint64(len(fullData))) if err != nil { t.Fatal(err) } if !bytes.Equal(fullData, readData) { t.Errorf("Bytes differ. lengths: %d, expected %d", len(readData), len(fullData)) } verboseData, err = getAllBlocks(db, id) if err != nil { t.Fatal(err) } if !bytes.Equal(verboseData, fullData) { t.Errorf("Bytes differ. lengths: %d, expected %d", len(verboseData), len(fullData)) } // Now write into the middle of the file. part2 = randutil.RandBytes(rng, len(part2)) if err := write(db, id, uint64(len(fullData)), uint64(len(part1)), part2); err != nil { t.Fatal(err) } fullData = append(part1, part2...) fullData = append(fullData, part3...) readData, err = read(db, id, 0, uint64(len(fullData))) if err != nil { t.Fatal(err) } if !bytes.Equal(fullData, readData) { t.Errorf("Bytes differ. lengths: %d, expected %d", len(readData), len(fullData)) } verboseData, err = getAllBlocks(db, id) if err != nil { t.Fatal(err) } if !bytes.Equal(verboseData, fullData) { t.Errorf("Bytes differ. lengths: %d, expected %d", len(verboseData), len(fullData)) } // New file. id2 := uint64(20) if err := write(db, id2, 0, 0, []byte("1")); err != nil { t.Fatal(err) } readData, err = read(db, id2, 0, 1) if err != nil { t.Fatal(err) } if string(readData) != "1" { t.Fatalf("mismatch: %s", readData) } if err := write(db, id2, 1, 0, []byte("22")); err != nil { t.Fatal(err) } readData, err = read(db, id2, 0, 2) if err != nil { t.Fatal(err) } if string(readData) != "22" { t.Fatalf("mismatch: %s", readData) } id3 := uint64(30) part1 = randutil.RandBytes(rng, BlockSize) // Write 5 blocks. var offset uint64 for i := 0; i < 5; i++ { if err := write(db, id3, offset, offset, part1); err != nil { t.Fatal(err) } offset += BlockSize } }