func (a *autoScaleConfig) rebalanceIfNeeded(evt *event.Event, pool string, nodes []*cluster.Node, sResult *scalerResult) error {
if len(sResult.ToRemove) > 0 {
return nil
}
if sResult.ToAdd > 0 {
sResult.ToRebalance = true
}
rebalanceFilter := map[string]string{poolMetadataName: pool}
if !sResult.ToRebalance {
// No action yet, check if we need rebalance
_, gap, err := a.provisioner.containerGapInNodes(nodes)
if err != nil {
return errors.Wrapf(err, "unable to obtain container gap in nodes")
}
buf := safe.NewBuffer(nil)
dryProvisioner, err := a.provisioner.rebalanceContainersByFilter(buf, nil, rebalanceFilter, true)
if err != nil {
return errors.Wrapf(err, "unable to run dry rebalance to check if rebalance is needed. log: %s", buf.String())
}
if dryProvisioner == nil {
return nil
}
_, gapAfter, err := dryProvisioner.containerGapInNodes(nodes)
if err != nil {
return errors.Wrap(err, "couldn't find containers from rebalanced nodes")
}
if math.Abs((float64)(gap-gapAfter)) > 2.0 {
sResult.ToRebalance = true
if sResult.Reason == "" {
sResult.Reason = fmt.Sprintf("gap is %d, after rebalance gap will be %d", gap, gapAfter)
}
}
}
if sResult.ToRebalance {
evt.Logf("running rebalance, for %q: %#v", pool, sResult)
buf := safe.NewBuffer(nil)
writer := io.MultiWriter(buf, evt)
_, err := a.provisioner.rebalanceContainersByFilter(writer, nil, rebalanceFilter, false)
if err != nil {
return errors.Wrapf(err, "unable to rebalance containers. log: %s", buf.String())
}
}
return nil
}
func (a *autoScaleConfig) removeMultipleNodes(evt *event.Event, chosenNodes []cluster.Node) error {
nodeAddrs := make([]string, len(chosenNodes))
nodeHosts := make([]string, len(chosenNodes))
for i, node := range chosenNodes {
_, hasIaas := node.Metadata["iaas"]
if !hasIaas {
return errors.Errorf("no IaaS information in node (%s) metadata: %#v", node.Address, node.Metadata)
}
nodeAddrs[i] = node.Address
nodeHosts[i] = net.URLToHost(node.Address)
}
err := a.provisioner.Cluster().UnregisterNodes(nodeAddrs...)
if err != nil {
return errors.Wrapf(err, "unable to unregister nodes (%s) for removal", strings.Join(nodeAddrs, ", "))
}
buf := safe.NewBuffer(nil)
err = a.provisioner.moveContainersFromHosts(nodeHosts, "", buf)
if err != nil {
for _, node := range chosenNodes {
a.provisioner.Cluster().Register(node)
}
return errors.Wrapf(err, "unable to move containers from nodes (%s). log: %s", strings.Join(nodeAddrs, ", "), buf.String())
}
wg := sync.WaitGroup{}
for i := range chosenNodes {
wg.Add(1)
go func(i int) {
defer wg.Done()
node := chosenNodes[i]
m, err := iaas.FindMachineByIdOrAddress(node.Metadata["iaas-id"], net.URLToHost(node.Address))
if err != nil {
evt.Logf("unable to find machine for removal in iaas: %s", err)
return
}
err = m.Destroy()
if err != nil {
evt.Logf("unable to destroy machine in IaaS: %s", err)
}
}(i)
}
wg.Wait()
return nil
}
func (a *autoScaleConfig) addNode(evt *event.Event, modelNodes []*cluster.Node) (*cluster.Node, error) {
metadata, err := chooseMetadataFromNodes(modelNodes)
if err != nil {
return nil, err
}
_, hasIaas := metadata["iaas"]
if !hasIaas {
return nil, errors.Errorf("no IaaS information in nodes metadata: %#v", metadata)
}
machine, err := iaas.CreateMachineForIaaS(metadata["iaas"], metadata)
if err != nil {
return nil, errors.Wrap(err, "unable to create machine")
}
newAddr := machine.FormatNodeAddress()
evt.Logf("new machine created: %s - Waiting for docker to start...", newAddr)
createOpts := provision.AddNodeOptions{
Address: newAddr,
Metadata: metadata,
WaitTO: a.WaitTimeNewMachine,
CaCert: machine.CaCert,
ClientCert: machine.ClientCert,
ClientKey: machine.ClientKey,
}
err = a.provisioner.AddNode(createOpts)
if err != nil {
machine.Destroy()
a.provisioner.Cluster().Unregister(newAddr)
return nil, errors.Wrapf(err, "error adding new node %s", newAddr)
}
createdNode, err := a.provisioner.Cluster().GetNode(newAddr)
if err != nil {
machine.Destroy()
a.provisioner.Cluster().Unregister(newAddr)
return nil, errors.Wrapf(err, "error retrieving new node %s", newAddr)
}
evt.Logf("new machine created: %s - started!", newAddr)
return &createdNode, nil
}