// Ingest loads some data in an sstable into an empty range. Only the keys
// between startKey and endKey are loaded. If newTableID is non-zero, every
// row's key is rewritten to be for that table.
func Ingest(
ctx context.Context,
txn *client.Txn,
path string,
checksum uint32,
startKey, endKey roachpb.Key,
newTableID sqlbase.ID,
) error {
// TODO(mjibson): An appropriate value for this should be determined. The
// current value was guessed at but appears to work well.
const batchSize = 10000
// TODO(dan): Check if the range being ingested into is empty. If newTableID
// is non-zero, it'll have to be derived from startKey and endKey.
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
crc := crc32.New(crc32.MakeTable(crc32.Castagnoli))
if _, err := io.Copy(crc, f); err != nil {
return nil
}
if c := crc.Sum32(); c != checksum {
return errors.Errorf("%s: checksum mismatch got %d expected %d", path, c, checksum)
}
sst, err := engine.MakeRocksDBSstFileReader()
if err != nil {
return err
}
defer sst.Close()
if err := sst.AddFile(path); err != nil {
return err
}
b := txn.NewBatch()
var v roachpb.Value
count := 0
ingestFunc := func(kv engine.MVCCKeyValue) (bool, error) {
v = roachpb.Value{RawBytes: kv.Value}
v.ClearChecksum()
if log.V(3) {
log.Infof(ctx, "Put %s %s\n", kv.Key.Key, v.PrettyPrint())
}
b.Put(kv.Key.Key, &v)
count++
if count > batchSize {
if err := txn.Run(b); err != nil {
return true, err
}
b = txn.NewBatch()
count = 0
}
return false, nil
}
if newTableID != 0 {
// MakeRekeyMVCCKeyValFunc modifies the keys, but this is safe because
// the one we get back from rocksDBIterator.Key is a copy (not a
// reference to the mmaped file.)
ingestFunc = MakeRekeyMVCCKeyValFunc(newTableID, ingestFunc)
}
startKeyMVCC, endKeyMVCC := engine.MVCCKey{Key: startKey}, engine.MVCCKey{Key: endKey}
if err := sst.Iterate(startKeyMVCC, endKeyMVCC, ingestFunc); err != nil {
return err
}
return txn.Run(b)
}