// process iterates through all keys in a replica's range, calling the garbage
// collector for each key and associated set of values. GC'd keys are batched
// into GC calls. Extant intents are resolved if intents are older than
// intentAgeThreshold.
func (gcq *gcQueue) process(now roachpb.Timestamp, repl *Replica,
sysCfg *config.SystemConfig) error {
snap := repl.rm.Engine().NewSnapshot()
desc := repl.Desc()
iter := newRangeDataIterator(desc, snap)
defer iter.Close()
defer snap.Close()
// Lookup the GC policy for the zone containing this key range.
zone, err := sysCfg.GetZoneConfigForKey(desc.StartKey)
if err != nil {
return fmt.Errorf("could not find GC policy for range %s: %s", repl, err)
}
policy := zone.GC
gcMeta := roachpb.NewGCMetadata(now.WallTime)
gc := engine.NewGarbageCollector(now, *policy)
// Compute intent expiration (intent age at which we attempt to resolve).
intentExp := now
intentExp.WallTime -= intentAgeThreshold.Nanoseconds()
// TODO(tschottdorf): execution will use a leader-assigned local
// timestamp to compute intent age. While this should be fine, could
// consider adding a Now timestamp to GCRequest which would be used
// instead.
gcArgs := &roachpb.GCRequest{
RequestHeader: roachpb.RequestHeader{
RangeID: desc.RangeID,
},
}
var mu sync.Mutex
var oldestIntentNanos int64 = math.MaxInt64
var expBaseKey roachpb.Key
var keys []roachpb.EncodedKey
var vals [][]byte
// Maps from txn ID to txn and intent key slice.
txnMap := map[string]*roachpb.Transaction{}
intentMap := map[string][]roachpb.Intent{}
// updateOldestIntent atomically updates the oldest intent.
updateOldestIntent := func(intentNanos int64) {
mu.Lock()
defer mu.Unlock()
if intentNanos < oldestIntentNanos {
oldestIntentNanos = intentNanos
}
}
// processKeysAndValues is invoked with each key and its set of
// values. Intents older than the intent age threshold are sent for
// resolution and values after the MVCC metadata, and possible
// intent, are sent for garbage collection.
processKeysAndValues := func() {
// If there's more than a single value for the key, possibly send for GC.
if len(keys) > 1 {
meta := &engine.MVCCMetadata{}
if err := proto.Unmarshal(vals[0], meta); err != nil {
log.Errorf("unable to unmarshal MVCC metadata for key %q: %s", keys[0], err)
} else {
// In the event that there's an active intent, send for
// intent resolution if older than the threshold.
startIdx := 1
if meta.Txn != nil {
// Keep track of intent to resolve if older than the intent
// expiration threshold.
if meta.Timestamp.Less(intentExp) {
id := string(meta.Txn.ID)
txnMap[id] = meta.Txn
intentMap[id] = append(intentMap[id], roachpb.Intent{Key: expBaseKey})
} else {
updateOldestIntent(meta.Txn.OrigTimestamp.WallTime)
}
// With an active intent, GC ignores MVCC metadata & intent value.
startIdx = 2
}
// See if any values may be GC'd.
if gcTS := gc.Filter(keys[startIdx:], vals[startIdx:]); !gcTS.Equal(roachpb.ZeroTimestamp) {
// TODO(spencer): need to split the requests up into
// multiple requests in the event that more than X keys
// are added to the request.
gcArgs.Keys = append(gcArgs.Keys, roachpb.GCRequest_GCKey{Key: expBaseKey, Timestamp: gcTS})
}
}
}
}
// Iterate through the keys and values of this replica's range.
for ; iter.Valid(); iter.Next() {
baseKey, ts, isValue, err := engine.MVCCDecodeKey(iter.Key())
if err != nil {
log.Errorf("unable to decode MVCC key: %q: %v", iter.Key(), err)
continue
}
if !isValue {
// Moving to the next key (& values).
processKeysAndValues()
expBaseKey = baseKey
keys = []roachpb.EncodedKey{iter.Key()}
vals = [][]byte{iter.Value()}
} else {
if !baseKey.Equal(expBaseKey) {
log.Errorf("unexpectedly found a value for %q with ts=%s; expected key %q", baseKey, ts, expBaseKey)
continue
}
keys = append(keys, iter.Key())
vals = append(vals, iter.Value())
}
}
if iter.Error() != nil {
return iter.Error()
}
// Handle last collected set of keys/vals.
processKeysAndValues()
// Process push transactions in parallel.
var wg sync.WaitGroup
for _, txn := range txnMap {
wg.Add(1)
go gcq.pushTxn(repl, now, txn, updateOldestIntent, &wg)
}
wg.Wait()
// Resolve all intents.
var intents []roachpb.Intent
for id, txn := range txnMap {
if txn.Status != roachpb.PENDING {
for _, intent := range intentMap[id] {
intent.Txn = *txn
intents = append(intents, intent)
}
}
}
done := true
if len(intents) > 0 {
done = false
repl.resolveIntents(repl.context(), intents)
}
// Set start and end keys.
if len(gcArgs.Keys) > 0 {
done = false
gcArgs.Key = gcArgs.Keys[0].Key
gcArgs.EndKey = gcArgs.Keys[len(gcArgs.Keys)-1].Key.Next()
}
if done {
return nil
}
// Send GC request through range.
gcMeta.OldestIntentNanos = proto.Int64(oldestIntentNanos)
gcArgs.GCMeta = *gcMeta
if _, err := client.SendWrapped(repl, repl.context(), gcArgs); err != nil {
return err
}
// Store current timestamp as last verification for this replica, as
// we've just successfully scanned.
if err := repl.SetLastVerificationTimestamp(now); err != nil {
log.Errorf("failed to set last verification timestamp for replica %s: %s", repl, err)
}
return nil
}
// process iterates through all keys in a replica's range, calling the garbage
// collector for each key and associated set of values. GC'd keys are batched
// into GC calls. Extant intents are resolved if intents are older than
// intentAgeThreshold.
func (gcq *gcQueue) process(now roachpb.Timestamp, repl *Replica,
sysCfg *config.SystemConfig) error {
snap := repl.store.Engine().NewSnapshot()
desc := repl.Desc()
iter := newReplicaDataIterator(desc, snap)
defer iter.Close()
defer snap.Close()
// Lookup the GC policy for the zone containing this key range.
zone, err := sysCfg.GetZoneConfigForKey(desc.StartKey)
if err != nil {
return fmt.Errorf("could not find GC policy for range %s: %s", repl, err)
}
policy := zone.GC
gcMeta := roachpb.NewGCMetadata(now.WallTime)
gc := engine.NewGarbageCollector(now, *policy)
// Compute intent expiration (intent age at which we attempt to resolve).
intentExp := now
intentExp.WallTime -= intentAgeThreshold.Nanoseconds()
txnExp := now
txnExp.WallTime -= txnCleanupThreshold.Nanoseconds()
gcArgs := &roachpb.GCRequest{}
// TODO(tschottdorf): This is one of these instances in which we want
// to be more careful that the request ends up on the correct Replica,
// and we might have to worry about mixing range-local and global keys
// in a batch which might end up spanning Ranges by the time it executes.
gcArgs.Key = desc.StartKey.AsRawKey()
gcArgs.EndKey = desc.EndKey.AsRawKey()
var expBaseKey roachpb.Key
var keys []engine.MVCCKey
var vals [][]byte
// Maps from txn ID to txn and intent key slice.
txnMap := map[string]*roachpb.Transaction{}
intentSpanMap := map[string][]roachpb.Span{}
// processKeysAndValues is invoked with each key and its set of
// values. Intents older than the intent age threshold are sent for
// resolution and values after the MVCC metadata, and possible
// intent, are sent for garbage collection.
processKeysAndValues := func() {
// If there's more than a single value for the key, possibly send for GC.
if len(keys) > 1 {
meta := &engine.MVCCMetadata{}
if err := proto.Unmarshal(vals[0], meta); err != nil {
log.Errorf("unable to unmarshal MVCC metadata for key %q: %s", keys[0], err)
} else {
// In the event that there's an active intent, send for
// intent resolution if older than the threshold.
startIdx := 1
if meta.Txn != nil {
// Keep track of intent to resolve if older than the intent
// expiration threshold.
if meta.Timestamp.Less(intentExp) {
id := string(meta.Txn.ID)
txnMap[id] = meta.Txn
intentSpanMap[id] = append(intentSpanMap[id], roachpb.Span{Key: expBaseKey})
}
// With an active intent, GC ignores MVCC metadata & intent value.
startIdx = 2
}
// See if any values may be GC'd.
if gcTS := gc.Filter(keys[startIdx:], vals[startIdx:]); !gcTS.Equal(roachpb.ZeroTimestamp) {
// TODO(spencer): need to split the requests up into
// multiple requests in the event that more than X keys
// are added to the request.
gcArgs.Keys = append(gcArgs.Keys, roachpb.GCRequest_GCKey{Key: expBaseKey, Timestamp: gcTS})
}
}
}
}
// Iterate through the keys and values of this replica's range.
for ; iter.Valid(); iter.Next() {
baseKey, ts, isValue, err := engine.MVCCDecodeKey(iter.Key())
if err != nil {
log.Errorf("unable to decode MVCC key: %q: %v", iter.Key(), err)
continue
}
if !isValue {
// Moving to the next key (& values).
processKeysAndValues()
expBaseKey = baseKey
keys = []engine.MVCCKey{iter.Key()}
vals = [][]byte{iter.Value()}
} else {
if !baseKey.Equal(expBaseKey) {
log.Errorf("unexpectedly found a value for %q with ts=%s; expected key %q", baseKey, ts, expBaseKey)
continue
}
keys = append(keys, iter.Key())
vals = append(vals, iter.Value())
}
}
if iter.Error() != nil {
return iter.Error()
}
// Handle last collected set of keys/vals.
processKeysAndValues()
txnKeys, err := processTransactionTable(repl, txnMap, txnExp)
if err != nil {
return err
}
// From now on, all newly added keys are range-local.
// TODO(tschottdorf): Might need to use two requests at some point since we
// hard-coded the full non-local key range in the header, but that does
// not take into account the range-local keys. It will be OK as long as
// we send directly to the Replica, though.
gcArgs.Keys = append(gcArgs.Keys, txnKeys...)
// Process push transactions in parallel.
var wg sync.WaitGroup
for _, txn := range txnMap {
if txn.Status != roachpb.PENDING {
continue
}
wg.Add(1)
go pushTxn(repl, now, txn, roachpb.ABORT_TXN, &wg)
}
wg.Wait()
// Resolve all intents.
var intents []roachpb.Intent
for id, txn := range txnMap {
if txn.Status != roachpb.PENDING {
for _, intent := range intentSpanMap[id] {
intents = append(intents, roachpb.Intent{Span: intent, Txn: *txn})
}
}
}
if err := repl.resolveIntents(repl.context(), intents, true /* wait */, false /* !poison */); err != nil {
return err
}
// Deal with any leftover sequence cache keys. There shouldn't be many of
// them.
gcArgs.Keys = append(gcArgs.Keys, processSequenceCache(repl, now, txnExp, txnMap)...)
// Send GC request through range.
gcArgs.GCMeta = *gcMeta
var ba roachpb.BatchRequest
// Technically not needed since we're talking directly to the Range.
ba.RangeID = desc.RangeID
ba.Timestamp = now
ba.Add(gcArgs)
if _, pErr := repl.Send(repl.context(), ba); pErr != nil {
return pErr.GoError()
}
// Store current timestamp as last verification for this replica, as
// we've just successfully scanned.
if err := repl.SetLastVerificationTimestamp(now); err != nil {
log.Errorf("failed to set last verification timestamp for replica %s: %s", repl, err)
}
return nil
}