Esempio n. 1
0
// Filter makes decisions about garbage collection based on the
// garbage collection policy for batches of values for the same key.
// The GC policy is determined via the policyFn specified when the
// GarbageCollector was created. Returns a slice of deletions, one
// per incoming keys. If an index in the returned array is set to
// true, then that value will be garbage collected.
func (gc *GarbageCollector) Filter(keys []Key, values [][]byte) []bool {
	if len(keys) == 1 {
		return nil
	}
	// Look up the policy which applies to this set of MVCC values.
	_, decKey := encoding.DecodeBinary(keys[0])
	policy := gc.policyFn(decKey)
	if policy == nil || policy.TTLSeconds <= 0 {
		return nil
	}
	toDelete := make([]bool, len(keys))
	expiration := gc.now
	expiration.WallTime -= int64(policy.TTLSeconds) * 1E9

	var survivors bool
	for i, key := range keys {
		_, ts, isValue := mvccDecodeKey(key)
		if i == 0 {
			if isValue {
				log.Errorf("unexpected MVCC value encountered: %q", key)
				return make([]bool, len(keys))
			}
			continue
		}
		if !isValue {
			log.Errorf("unexpected MVCC metadata encountered: %q", key)
			return make([]bool, len(keys))
		}
		mvccVal := proto.MVCCValue{}
		if err := gogoproto.Unmarshal(values[i], &mvccVal); err != nil {
			log.Errorf("unable to unmarshal MVCC value %q: %v", key, err)
			return make([]bool, len(keys))
		}
		if i == 1 {
			// If the first value isn't a deletion tombstone, set survivors to true.
			if !mvccVal.Deleted {
				survivors = true
			}
		} else {
			if ts.Less(expiration) {
				// If we encounter a version older than our GC timestamp, mark for deletion.
				toDelete[i] = true
			} else if !mvccVal.Deleted {
				// Otherwise, if not marked for GC and not a tombstone, set survivors true.
				survivors = true
			}
		}
	}
	// If there are no remaining non-deleted, versioned entries, mark
	// all keys for deletion, including the MVCC metadata entry.
	if !survivors {
		for i := range keys {
			toDelete[i] = true
		}
	}
	return toDelete
}
Esempio n. 2
0
// ResolveWriteIntentRange commits or aborts (rolls back) the range of
// write intents specified by start and end keys for a given txnID
// according to commit parameter.  ResolveWriteIntentRange will skip
// write intents of other txnIDs. Specify max=0 for unbounded
// resolves.
func (mvcc *MVCC) ResolveWriteIntentRange(key Key, endKey Key, max int64, txnID []byte, commit bool) (int64, error) {
	if len(txnID) == 0 {
		return 0, util.Error("missing txnID in request")
	}

	binKey := encoding.EncodeBinary(nil, key)
	binEndKey := encoding.EncodeBinary(nil, endKey)
	nextKey := binKey

	num := int64(0)
	for {
		kvs, err := mvcc.engine.Scan(nextKey, binEndKey, 1)
		if err != nil {
			return num, err
		}
		// No more keys exists in the given range.
		if len(kvs) == 0 {
			break
		}

		remainder, currentKey := encoding.DecodeBinary(kvs[0].Key)
		if len(remainder) != 0 {
			return 0, util.Errorf("expected an MVCC metadata key: %s", kvs[0].Key)
		}
		_, _, existingTxnID, err := mvcc.getInternal(kvs[0].Key, proto.MaxTimestamp, txnID)
		// Return the error unless its a writeIntentError, which
		// will occur in the event we scan a key with a write
		// intent belonging to a different transaction.
		if _, ok := err.(*writeIntentError); err != nil && !ok {
			return num, err
		}
		// ResolveWriteIntent only needs to deal with the write
		// intents for the given txnID.
		if err == nil && bytes.Equal(existingTxnID, txnID) {
			// commits or aborts (rolls back) the write intent of
			// the given txnID.
			err = mvcc.ResolveWriteIntent(currentKey, txnID, commit)
			if err != nil {
				return num, err
			}
			num++
		}

		if max != 0 && max == num {
			break
		}

		// In order to efficiently skip the possibly long list of
		// old versions for this key; refer to Scan for details.
		nextKey = encoding.EncodeBinary(nil, NextKey(currentKey))
	}

	return num, nil
}
Esempio n. 3
0
// Scan scans the key range specified by start key through end key up
// to some maximum number of results. Specify max=0 for unbounded scans.
func (mvcc *MVCC) Scan(key Key, endKey Key, max int64, timestamp proto.Timestamp, txn *proto.Transaction) ([]proto.KeyValue, error) {
	binKey := encoding.EncodeBinary(nil, key)
	binEndKey := encoding.EncodeBinary(nil, endKey)
	nextKey := binKey

	res := []proto.KeyValue{}
	for {
		kvs, err := mvcc.engine.Scan(nextKey, binEndKey, 1)
		if err != nil {
			return nil, err
		}
		// No more keys exists in the given range.
		if len(kvs) == 0 {
			break
		}

		remainder, currentKey := encoding.DecodeBinary(kvs[0].Key)
		if len(remainder) != 0 {
			return nil, util.Errorf("expected an MVCC metadata key: %s", kvs[0].Key)
		}
		value, err := mvcc.Get(currentKey, timestamp, txn)
		if err != nil {
			return res, err
		}

		if value != nil {
			res = append(res, proto.KeyValue{Key: currentKey, Value: *value})
		}

		if max != 0 && max == int64(len(res)) {
			break
		}

		// In order to efficiently skip the possibly long list of
		// old versions for this key, we move instead to the next
		// highest key and the for loop continues by scanning again
		// with nextKey.
		// Let's say you have:
		// a
		// a<T=2>
		// a<T=1>
		// aa
		// aa<T=3>
		// aa<T=2>
		// b
		// b<T=5>
		// In this case, if we scan from "a"-"b", we wish to skip
		// a<T=2> and a<T=1> and find "aa'.
		nextKey = encoding.EncodeBinary(nil, NextKey(currentKey))
	}

	return res, nil
}
Esempio n. 4
0
// mvccDecodeKey decodes encodedKey into key and Timestamp. The final
// returned bool is true if this is an MVCC value and false if this is
// MVCC metadata. Note that the returned key is exactly the value of
// key passed to mvccEncodeKey. A separate DecodeBinary step must be
// carried out to decode it if necessary.
// If a decode process fails, a panic ensues.
func mvccDecodeKey(encodedKey []byte) (Key, proto.Timestamp, bool) {
	tsBytes, _ := encoding.DecodeBinary(encodedKey)
	key := encodedKey[:len(encodedKey)-len(tsBytes)]
	if len(tsBytes) == 0 {
		return key, proto.Timestamp{}, false
	}
	tsBytes, walltime := encoding.DecodeUint64Decreasing(tsBytes)
	tsBytes, logical := encoding.DecodeUint32Decreasing(tsBytes)
	if len(tsBytes) > 0 {
		panic(fmt.Sprintf("leftover bytes on mvcc key decode: %v", tsBytes))
	}
	return key, proto.Timestamp{WallTime: int64(walltime), Logical: int32(logical)}, true
}
Esempio n. 5
0
// FindSplitKey suggests a split key from the given user-space key range that
// aims to roughly cut into half the total number of bytes used (in raw key and
// value byte strings) in both subranges. It will operate on a snapshot of the
// underlying engine if a snapshotID is given, and in that case may safely be
// invoked in a goroutine.
// TODO(Tobias): leverage the work done here anyways to gather stats.
func (mvcc *MVCC) FindSplitKey(key Key, endKey Key, snapshotID string) (Key, error) {
	rs := util.NewWeightedReservoirSample(splitReservoirSize, nil)
	h := rs.Heap.(*util.WeightedValueHeap)

	// We expect most keys to contain anywhere between 2^4 to 2^14 bytes, so we
	// normalize to obtain typical weights that are numerically unproblematic.
	// The relevant expression is rand(0,1)**(1/weight).
	normalize := float64(1 << 6)
	binStartKey := encoding.EncodeBinary(nil, key)
	binEndKey := encoding.EncodeBinary(nil, endKey)
	totalSize := 0
	err := iterateRangeSnapshot(mvcc.engine, binStartKey, binEndKey,
		splitScanRowCount, snapshotID, func(kvs []proto.RawKeyValue) error {
			for _, kv := range kvs {
				byteCount := len(kv.Key) + len(kv.Value)
				rs.ConsiderWeighted(splitSampleItem{kv.Key, totalSize}, float64(byteCount)/normalize)
				totalSize += byteCount
			}
			return nil
		})
	if err != nil {
		return nil, err
	}

	if totalSize == 0 {
		return nil, util.Errorf("the range is empty")
	}

	// Inspect the sample to get the closest candidate that has sizeBefore >= totalSize/2.
	candidate := (*h)[0].Value.(splitSampleItem)
	cb := candidate.sizeBefore
	halfSize := totalSize / 2
	for i := 1; i < len(*h); i++ {
		if sb := (*h)[i].Value.(splitSampleItem).sizeBefore; (cb < halfSize && cb < sb) ||
			(cb > halfSize && cb > sb && sb > halfSize) {
			// The current candidate hasn't yet cracked 50% and the this value
			// is closer to doing so or we're already above but now we can
			// decrese the gap.
			candidate = (*h)[i].Value.(splitSampleItem)
			cb = candidate.sizeBefore
		}
	}
	// The key is an MVCC key, so to avoid corrupting MVCC we get the
	// associated sentinel metadata key, which is fine to split in front of.
	decodedKey, _, _ := mvccDecodeKey(candidate.Key)
	rest, humanKey := encoding.DecodeBinary(decodedKey)
	if len(rest) > 0 {
		return nil, util.Errorf("corrupt key encountered")
	}
	return humanKey, nil
}
Esempio n. 6
0
// ResolveWriteIntentRange commits or aborts (rolls back) the range of
// write intents specified by start and end keys for a given txn
// according to commit parameter. ResolveWriteIntentRange will skip
// write intents of other txns. Specify max=0 for unbounded resolves.
func (mvcc *MVCC) ResolveWriteIntentRange(key Key, endKey Key, max int64, txn *proto.Transaction, commit bool) (int64, error) {
	if txn == nil {
		return 0, util.Error("no txn specified")
	}

	binKey := encoding.EncodeBinary(nil, key)
	binEndKey := encoding.EncodeBinary(nil, endKey)
	nextKey := binKey

	num := int64(0)
	for {
		kvs, err := mvcc.engine.Scan(nextKey, binEndKey, 1)
		if err != nil {
			return num, err
		}
		// No more keys exists in the given range.
		if len(kvs) == 0 {
			break
		}

		remainder, currentKey := encoding.DecodeBinary(kvs[0].Key)
		if len(remainder) != 0 {
			return 0, util.Errorf("expected an MVCC metadata key: %s", kvs[0].Key)
		}
		err = mvcc.ResolveWriteIntent(currentKey, txn, commit)
		if err != nil {
			log.Warningf("failed to resolve intent for key %q: %v", currentKey, err)
		} else {
			num++
			if max != 0 && max == num {
				break
			}
		}

		// In order to efficiently skip the possibly long list of
		// old versions for this key; refer to Scan for details.
		nextKey = encoding.EncodeBinary(nil, NextKey(currentKey))
	}

	return num, nil
}