Пример #1
0
func (kv *consulKV) Snapshot(prefix string) (kvdb.Kvdb, uint64, error) {
	// Create a new bootstrap key : lowest index
	r := rand.New(rand.NewSource(time.Now().UnixNano())).Int63()
	bootStrapKeyLow := bootstrap + strconv.FormatInt(r, 10) +
		strconv.FormatInt(time.Now().UnixNano(), 10)
	val, _ := common.ToBytes(time.Now().UnixNano())
	kvPair, err := kv.Put(bootStrapKeyLow, val, 0)
	if err != nil {
		return nil, 0, fmt.Errorf("Failed to create snap bootstrap key %v, "+
			"err: %v", bootStrapKeyLow, err)
	}
	lowestKvdbIndex := kvPair.ModifiedIndex

	options := &api.QueryOptions{
		AllowStale:        false,
		RequireConsistent: true,
	}

	listKey := kv.domain + prefix
	listKey = stripConsecutiveForwardslash(listKey)
	pairs, _, err := kv.client.KV().List(listKey, options)
	if err != nil {
		return nil, 0, err
	}

	kvPairs := kv.pairToKvs("enumerate", pairs, nil)
	snapDb, err := mem.New(
		kv.domain,
		nil,
		map[string]string{mem.KvSnap: "true"},
		kv.FatalCb,
	)
	if err != nil {
		return nil, 0, err
	}

	for _, kvPair := range kvPairs {
		_, err := snapDb.SnapPut(kvPair)
		if err != nil {
			return nil, 0, fmt.Errorf("Failed creating snap: %v", err)
		}
	}

	// Create bootrap key : highest index
	bootStrapKeyHigh := bootstrap + strconv.FormatInt(r, 10) +
		strconv.FormatInt(time.Now().UnixNano(), 10)
	val, _ = common.ToBytes(time.Now().UnixNano())

	kvPair, err = kv.Put(bootStrapKeyHigh, val, 0)
	if err != nil {
		return nil, 0, fmt.Errorf("Failed to create snap bootstrap key %v, "+
			"err: %v", bootStrapKeyHigh, err)
	}
	highestKvdbIndex := kvPair.ModifiedIndex

	// In consul Delete does not increment kvdb index.
	// Hence the put (bootstrap) key and delete both return the same index
	if lowestKvdbIndex+1 != highestKvdbIndex {
		// create a watch to get all changes
		// between lowestKvdbIndex and highestKvdbIndex
		done := make(chan error)
		mutex := &sync.Mutex{}
		cb := func(
			prefix string,
			opaque interface{},
			kvp *kvdb.KVPair,
			err error,
		) error {
			var watchErr error
			var sendErr error
			var m *sync.Mutex
			ok := false

			if err != nil {
				if err == kvdb.ErrWatchStopped {
					return nil
				}
				watchErr = err
				sendErr = err
				goto errordone
			}

			if kvp == nil {
				watchErr = fmt.Errorf("kvp is nil")
				sendErr = watchErr
				goto errordone

			}

			m, ok = opaque.(*sync.Mutex)
			if !ok {
				watchErr = fmt.Errorf("Failed to get mutex")
				sendErr = watchErr
				goto errordone
			}

			m.Lock()
			defer m.Unlock()

			if kvp.ModifiedIndex > highestKvdbIndex {
				// done applying changes, just return
				watchErr = fmt.Errorf("done")
				sendErr = nil
				goto errordone
			} else if kvp.ModifiedIndex == highestKvdbIndex {
				// last update that we needed. Put it inside snap db
				// and return
				_, err = snapDb.SnapPut(kvp)
				if err != nil {
					watchErr = fmt.Errorf("Failed to apply update to snap: %v", err)
					sendErr = watchErr
				} else {
					watchErr = fmt.Errorf("done")
					sendErr = nil
				}
				goto errordone
			} else {
				_, err = snapDb.SnapPut(kvp)
				if err != nil {
					watchErr = fmt.Errorf("Failed to apply update to snap: %v", err)
					sendErr = watchErr
					goto errordone
				}
			}

			return nil
		errordone:
			done <- sendErr
			return watchErr
		}

		if err := kv.WatchTree("", lowestKvdbIndex, mutex,
			cb); err != nil {
			return nil, 0, fmt.Errorf("Failed to start watch: %v", err)
		}
		err = <-done
		if err != nil {
			return nil, 0, err
		}
	}

	_, err = kv.Delete(bootStrapKeyLow)
	if err != nil {
		return nil, 0, fmt.Errorf("Failed to delete snap bootstrap key: %v, "+
			"err: %v", bootStrapKeyLow, err)
	}
	_, err = kv.Delete(bootStrapKeyHigh)
	if err != nil {
		return nil, 0, fmt.Errorf("Failed to delete snap bootstrap key: %v, "+
			"err: %v", bootStrapKeyHigh, err)
	}
	return snapDb, highestKvdbIndex, nil
}
Пример #2
0
func (kv *etcdKV) Snapshot(prefix string) (kvdb.Kvdb, uint64, error) {
	// Create a new bootstrap key
	r := rand.New(rand.NewSource(time.Now().UnixNano())).Int63()
	bootStrapKey := ec.Bootstrap + strconv.FormatInt(r, 10) +
		strconv.FormatInt(time.Now().UnixNano(), 10)
	kvPair, err := kv.Put(bootStrapKey, time.Now().UnixNano(), 0)
	if err != nil {
		return nil, 0, fmt.Errorf("Failed to create snap bootstrap key %v, "+
			"err: %v", bootStrapKey, err)
	}
	lowestKvdbIndex := kvPair.ModifiedIndex

	kvPairs, err := kv.Enumerate(prefix)
	if err != nil {
		return nil, 0, fmt.Errorf("Failed to enumerate %v: err: %v", prefix,
			err)
	}
	snapDb, err := mem.New(
		kv.domain,
		nil,
		map[string]string{mem.KvSnap: "true"},
		kv.FatalCb,
	)
	if err != nil {
		return nil, 0, fmt.Errorf("Failed to create in-mem kv store: %v", err)
	}

	for i := 0; i < len(kvPairs); i++ {
		kvPair := kvPairs[i]
		if len(kvPair.Value) > 0 {
			// Only create a leaf node
			_, err := snapDb.SnapPut(kvPair)
			if err != nil {
				return nil, 0, fmt.Errorf("Failed creating snap: %v", err)
			}
		} else {
			newKvPairs, err := kv.Enumerate(kvPair.Key)
			if err != nil {
				return nil, 0, fmt.Errorf("Failed to get child keys: %v", err)
			}
			if len(newKvPairs) > 0 {
				kvPairs = append(kvPairs, newKvPairs...)
			}
		}
	}

	kvPair, err = kv.Delete(bootStrapKey)
	if err != nil {
		return nil, 0, fmt.Errorf("Failed to delete snap bootstrap key: %v, "+
			"err: %v", bootStrapKey, err)
	}
	highestKvdbIndex := kvPair.ModifiedIndex
	if lowestKvdbIndex+1 != highestKvdbIndex {
		// create a watch to get all changes
		// between lowestKvdbIndex and highestKvdbIndex
		done := make(chan error)
		mutex := &sync.Mutex{}
		cb := func(
			prefix string,
			opaque interface{},
			kvp *kvdb.KVPair,
			err error,
		) error {
			var watchErr error
			var sendErr error
			var m *sync.Mutex
			ok := false

			if err != nil {
				if err == kvdb.ErrWatchStopped {
					return nil
				}
				watchErr = err
				sendErr = err
				goto errordone
			}

			if kvp == nil {

				watchErr = fmt.Errorf("kvp is nil")
				sendErr = watchErr
				goto errordone

			}

			m, ok = opaque.(*sync.Mutex)
			if !ok {
				watchErr = fmt.Errorf("Failed to get mutex")
				sendErr = watchErr
				goto errordone
			}

			m.Lock()
			defer m.Unlock()

			if kvp.ModifiedIndex >= highestKvdbIndex {
				// Done applying changes.
				watchErr = fmt.Errorf("done")
				sendErr = nil
				goto errordone
			}

			_, err = snapDb.SnapPut(kvp)
			if err != nil {
				watchErr = fmt.Errorf("Failed to apply update to snap: %v", err)
				sendErr = watchErr
				goto errordone
			}

			return nil
		errordone:
			done <- sendErr
			return watchErr
		}

		if err := kv.WatchTree("", lowestKvdbIndex, mutex,
			cb); err != nil {
			return nil, 0, fmt.Errorf("Failed to start watch: %v", err)
		}
		err = <-done
		if err != nil {
			return nil, 0, err
		}
	}

	return snapDb, highestKvdbIndex, nil
}