func (tracker *FleetTracker) scaleDown(delta int, rpcEvent *RPCEvent) error {
logger.WithFields(
"leverEnv", rpcEvent.Environment,
"leverService", rpcEvent.Service,
"codeVersion", rpcEvent.CodeVersion,
"servingID", rpcEvent.ServingID,
"deltaInstances", delta,
).Info("Scaling down")
consulHealth := config.GetConsulClient().Health()
entries, _, err := consulHealth.Service(
rpcEvent.ServingID, "", true, &consulapi.QueryOptions{
RequireConsistent: true,
})
if err != nil {
logger.WithFields(
"err", err,
"servingID", rpcEvent.ServingID,
).Error("Error trying to ask Consul for instances")
}
if len(entries) < delta {
delta = len(entries)
}
tmpRand := leverutil.GetRand()
permutation := tmpRand.Perm(len(entries))
leverutil.PutRand(tmpRand)
shuffled := make([]*consulapi.ServiceEntry, len(entries))
for from, to := range permutation {
shuffled[to] = entries[from]
}
toRemove := shuffled[:delta]
hadErrors := false
for _, entry := range toRemove {
err = hostman.StopInstance(
tracker.grpcPool,
&hostman.InstanceKey{
Environment: rpcEvent.Environment,
Service: rpcEvent.Service,
InstanceID: entry.Service.ID,
ServingID: rpcEvent.ServingID,
}, entry.Node.Node)
if err != nil {
logger.WithFields("err", err).Error(
"Error trying to stop instance remotely")
hadErrors = true
}
}
if hadErrors {
return fmt.Errorf("There were errors during scale down")
}
return nil
}
// RandomInstaceID returns a random instanceID that serves provided servingID.
func (manager *Manager) RandomInstaceID(
servingID string) (instanceID string, err error) {
manager.servingIDsLock.RLock()
defer manager.servingIDsLock.RUnlock()
instances, ok := manager.servingIDs[servingID]
if !ok {
return "", fmt.Errorf("Serving ID not found")
}
tmpRand := leverutil.GetRand()
instanceIndex := tmpRand.Intn(len(instances))
leverutil.PutRand(tmpRand)
index := 0
for instanceID := range instances {
if index == instanceIndex {
return instanceID, nil
}
index++
}
panic(fmt.Errorf("Should never happen"))
}
func (tracker *valueTracker) value(value uint64) {
tracker.lock.RLock()
if tracker.dataSize < valueTrackerCapacity {
// Did not yet fill data vector.
// Switch locks.
tracker.lock.RUnlock()
tracker.lock.Lock()
// Make sure nothing changed.
if tracker.dataSize == valueTrackerCapacity {
// Sample space filled up while switching locks.
// Try again.
tracker.lock.Unlock()
tracker.value(value)
}
tracker.data[tracker.dataSize] = value
tracker.dataSize++
atomic.AddUint64(&tracker.sumValues, value)
atomic.AddUint64(&tracker.sumSqValuesS32, (value>>16)*(value>>16))
tracker.lock.Unlock()
} else {
// Data vector full. Replace a random value in the sample.
tmpRand := leverutil.GetRand()
index := tmpRand.Intn(tracker.dataSize)
leverutil.PutRand(tmpRand)
tracker.dataLocks[index%valueTrackerDataBuckets].Lock()
oldValue := tracker.data[index]
tracker.data[index] = value
tracker.dataLocks[index%valueTrackerDataBuckets].Unlock()
// Update the sums as a result (subtract old value and add new value).
delta := value + ^uint64(oldValue-1)
atomic.AddUint64(&tracker.sumValues, delta)
deltaSq := (value>>16)*(value>>16) +
^uint64((oldValue>>16)*(oldValue>>16)-1)
atomic.AddUint64(&tracker.sumSqValuesS32, deltaSq)
tracker.lock.RUnlock()
}
}