// newPublisher returns a publishFunc that publishes a single UnitState
// by the given name to the provided Registry, with the given TTL
func newPublisher(reg registry.Registry, ttl time.Duration) publishFunc {
return func(name string, us *unit.UnitState) {
if us == nil {
log.V(1).Infof("Destroying UnitState(%s) in Registry", name)
err := reg.RemoveUnitState(name)
if err != nil {
log.Errorf("Failed to destroy UnitState(%s) in Registry: %v", name, err)
}
} else {
// Sanity check - don't want to publish incomplete UnitStates
// TODO(jonboulle): consider teasing apart a separate UnitState-like struct
// so we can rely on a UnitState always being fully hydrated?
// See https://github.com/coreos/fleet/issues/720
//if len(us.UnitHash) == 0 {
// log.Errorf("Refusing to push UnitState(%s), no UnitHash: %#v", name, us)
if len(us.MachineID) == 0 {
log.Errorf("Refusing to push UnitState(%s), no MachineID: %#v", name, us)
} else {
log.V(1).Infof("Pushing UnitState(%s) to Registry: %#v", name, us)
reg.SaveUnitState(name, us, ttl)
}
}
}
}
// ParseFilepath expands ~ and ~user constructions.
// If user or $HOME is unknown, do nothing.
func ParseFilepath(path string) string {
if !strings.HasPrefix(path, "~") {
return path
}
i := strings.Index(path, "/")
if i < 0 {
i = len(path)
}
var home string
if i == 1 {
if home = os.Getenv("HOME"); home == "" {
usr, err := user.Current()
if err != nil {
log.V(1).Infof("Failed to get current home directory: %v", err)
return path
}
home = usr.HomeDir
}
} else {
usr, err := user.Lookup(path[1:i])
if err != nil {
log.V(1).Infof("Failed to get %v's home directory: %v", path[1:i], err)
return path
}
home = usr.HomeDir
}
path = filepath.Join(home, path[i:])
return path
}
func (r *reconciler) Run(stop chan bool) {
trigger := make(chan struct{})
go func() {
abort := make(chan struct{})
for {
select {
case <-stop:
close(abort)
return
case <-r.eStream.Next(abort):
trigger <- struct{}{}
}
}
}()
ticker := time.After(r.ival)
for {
select {
case <-stop:
log.V(1).Info("Reconciler exiting due to stop signal")
return
case <-ticker:
ticker = time.After(r.ival)
log.V(1).Info("Reconciler tick")
r.rFunc()
case <-trigger:
ticker = time.After(r.ival)
log.V(1).Info("Reconciler triggered")
r.rFunc()
}
}
}
func runUnloadUnit(args []string) (exit int) {
units, err := findUnits(args)
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
return 1
}
wait := make([]string, 0)
for _, s := range units {
if job.JobState(s.CurrentState) == job.JobStateInactive {
log.V(1).Infof("Unit(%s) already %s, skipping.", s.Name, job.JobStateInactive)
continue
}
log.V(1).Infof("Setting target state of Unit(%s) to %s", s.Name, job.JobStateInactive)
cAPI.SetUnitTargetState(s.Name, string(job.JobStateInactive))
wait = append(wait, s.Name)
}
if !sharedFlags.NoBlock {
errchan := waitForUnitStates(wait, job.JobStateInactive, sharedFlags.BlockAttempts, os.Stdout)
for err := range errchan {
fmt.Fprintf(os.Stderr, "%v\n", err)
exit = 1
}
}
return
}
func watch(client etcd.Client, key string, stop chan struct{}) (res *etcd.Result) {
for res == nil {
select {
case <-stop:
log.V(1).Infof("Gracefully closing etcd watch loop: key=%s", key)
return
default:
req := &etcd.Watch{
Key: key,
WaitIndex: 0,
Recursive: true,
}
log.V(1).Infof("Creating etcd watcher: %v", req)
var err error
res, err = client.Wait(req, stop)
if err != nil {
log.Errorf("etcd watcher %v returned error: %v", req, err)
}
}
// Let's not slam the etcd server in the event that we know
// an unexpected error occurred.
time.Sleep(time.Second)
}
return
}
// addrToHostPort takes the given address and parses it into a string suitable
// for use in the 'hostnames' field in a known_hosts file. For more details,
// see the `SSH_KNOWN_HOSTS FILE FORMAT` section of `man 8 sshd`
func (kc *HostKeyChecker) addrToHostPort(a string) (string, error) {
if !strings.Contains(a, ":") {
// No port, so return unadulterated
return a, nil
}
host, p, err := net.SplitHostPort(a)
if err != nil {
log.V(1).Infof("Unable to parse addr %s: %v", a, err)
return "", err
}
port, err := strconv.Atoi(p)
if err != nil {
log.V(1).Infof("Error parsing port %s: %v", p, err)
return "", err
}
// Default port should be omitted from the entry.
// (see `put_host_port` in openssh/misc.c)
if port == 0 || port == sshDefaultPort {
// IPv6 addresses must be enclosed in square brackets
if strings.Contains(host, ":") {
host = fmt.Sprintf("[%s]", host)
}
return host, nil
}
return fmt.Sprintf("[%s]:%d", host, port), nil
}
// HasMetadata determine if the Metadata of a given MachineState
// matches the indicated values.
func HasMetadata(state *MachineState, metadata map[string][]string) bool {
for key, values := range metadata {
local, ok := state.Metadata[key]
if !ok {
log.V(1).Infof("No local values found for Metadata(%s)", key)
return false
}
log.V(1).Infof("Asserting local Metadata(%s) meets requirements", key)
var localMatch bool
for _, val := range values {
if local == val {
log.V(1).Infof("Local Metadata(%s) meets requirement", key)
localMatch = true
}
}
if !localMatch {
log.V(1).Infof("Local Metadata(%s) does not match requirement", key)
return false
}
}
return true
}
func (lt *LoggingHTTPTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
log.V(1).Infof("HTTP %s %s", req.Method, req.URL.String())
resp, err = lt.Transport.RoundTrip(req)
if err == nil {
log.V(1).Infof("HTTP %s %s %s", req.Method, req.URL.String(), resp.Status)
}
return
}
func (ar *actionResolver) one(req *http.Request, cancel <-chan struct{}) (resp *http.Response, body []byte, err error) {
log.V(1).Infof("etcd: sending HTTP request %s %s", req.Method, req.URL)
resp, body, err = ar.requestFunc(req, cancel)
if err != nil {
log.V(1).Infof("etcd: recv error response from %s %s: %v", req.Method, req.URL, err)
return
}
log.V(1).Infof("etcd: recv response from %s %s: %s", req.Method, req.URL, resp.Status)
return
}
func findAddressInMachineList(lookup string) (string, bool) {
states, err := cAPI.Machines()
if err != nil {
log.V(1).Infof("Unable to retrieve list of active machines from the Registry: %v", err)
return "", false
}
var match *machine.MachineState
for i := range states {
machState := states[i]
if !strings.HasPrefix(machState.ID, lookup) {
continue
} else if match != nil {
fmt.Fprintln(os.Stderr, "Found more than one Machine, be more specific.")
os.Exit(1)
}
match = &machState
}
if match == nil {
return "", false
}
return match.PublicIP, true
}
func globMatches(pattern, target string) bool {
matched, err := path.Match(pattern, target)
if err != nil {
log.V(1).Infof("Received error while matching pattern '%s': %v", pattern, err)
}
return matched
}
// check attempts to beat a Heart several times within a timeout, returning the
// log index at which the beat succeeded or an error
func (m *Monitor) check(hrt Heart) (idx uint64, err error) {
// time out after a third of the machine presence TTL, attempting
// the heartbeat up to four times
timeout := m.TTL / 3
interval := timeout / 4
tchan := time.After(timeout)
next := time.After(0)
for idx == 0 {
select {
case <-tchan:
err = errors.New("Monitor timed out before successful heartbeat")
return
case <-next:
idx, err = hrt.Beat(m.TTL)
if err != nil {
log.V(1).Infof("Monitor heartbeat function returned err, retrying in %v: %v", interval, err)
}
next = time.After(interval)
}
}
return
}
func createUnit(name string, uf *unit.UnitFile) (*schema.Unit, error) {
if uf == nil {
return nil, fmt.Errorf("nil unit provided")
}
u := schema.Unit{
Name: name,
Options: schema.MapUnitFileToSchemaUnitOptions(uf),
}
// TODO(jonboulle): this dependency on the API package is awkward, and
// redundant with the check in api.unitsResource.set, but it is a
// workaround to implementing the same check in the RegistryClient. It
// will disappear once RegistryClient is deprecated.
if err := api.ValidateName(name); err != nil {
return nil, err
}
if err := api.ValidateOptions(u.Options); err != nil {
return nil, err
}
err := cAPI.CreateUnit(&u)
if err != nil {
return nil, fmt.Errorf("failed creating unit %s: %v", name, err)
}
log.V(1).Infof("Created Unit(%s) in Registry", name)
return &u, nil
}
func NewCoreOSMachine(static MachineState, um unit.UnitManager) *CoreOSMachine {
log.V(1).Infof("Created CoreOSMachine with static state %s", static)
m := &CoreOSMachine{
staticState: static,
um: um,
}
return m
}
func acquireLeadership(lReg registry.LeaseRegistry, machID string, ver int, ttl time.Duration) registry.Lease {
existing, err := lReg.GetLease(engineLeaseName)
if err != nil {
log.Errorf("Unable to determine current lessee: %v", err)
return nil
}
var l registry.Lease
if existing == nil {
l, err = lReg.AcquireLease(engineLeaseName, machID, ver, ttl)
if err != nil {
log.Errorf("Engine leadership acquisition failed: %v", err)
return nil
} else if l == nil {
log.V(1).Infof("Unable to acquire engine leadership")
return nil
}
log.Infof("Engine leadership acquired")
return l
}
if existing.Version() >= ver {
log.V(1).Infof("Lease already held by Machine(%s) operating at acceptable version %d", existing.MachineID(), existing.Version())
return existing
}
rem := existing.TimeRemaining()
l, err = lReg.StealLease(engineLeaseName, machID, ver, ttl+rem, existing.Index())
if err != nil {
log.Errorf("Engine leadership steal failed: %v", err)
return nil
} else if l == nil {
log.V(1).Infof("Unable to steal engine leadership")
return nil
}
log.Infof("Stole engine leadership from Machine(%s)", existing.MachineID())
if rem > 0 {
log.Infof("Waiting %v for previous lease to expire before continuing reconciliation", rem)
<-time.After(rem)
}
return l
}
func renewLeadership(l registry.Lease, ttl time.Duration) registry.Lease {
err := l.Renew(ttl)
if err != nil {
log.Errorf("Engine leadership lost, renewal failed: %v", err)
return nil
}
log.V(1).Infof("Engine leadership renewed")
return l
}
func runStopUnit(args []string) (exit int) {
units, err := findUnits(args)
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
return 1
}
stopping := make([]string, 0)
for _, u := range units {
if !suToGlobal(u) {
if job.JobState(u.CurrentState) == job.JobStateInactive {
fmt.Fprintf(os.Stderr, "Unable to stop unit %s in state %s\n", u.Name, job.JobStateInactive)
return 1
} else if job.JobState(u.CurrentState) == job.JobStateLoaded {
log.V(1).Infof("Unit(%s) already %s, skipping.", u.Name, job.JobStateLoaded)
continue
}
}
log.V(1).Infof("Setting target state of Unit(%s) to %s", u.Name, job.JobStateLoaded)
cAPI.SetUnitTargetState(u.Name, string(job.JobStateLoaded))
if suToGlobal(u) {
fmt.Printf("Triggered global unit %s stop\n", u.Name)
} else {
stopping = append(stopping, u.Name)
}
}
if !sharedFlags.NoBlock {
errchan := waitForUnitStates(stopping, job.JobStateLoaded, sharedFlags.BlockAttempts, os.Stdout)
for err := range errchan {
fmt.Fprintf(os.Stderr, "Error waiting for units: %v\n", err)
exit = 1
}
} else {
for _, name := range stopping {
fmt.Printf("Triggered unit %s stop\n", name)
}
}
return
}
// setTargetStateOfUnits ensures that the target state for the given Units is set
// to the given state in the Registry.
// On success, a slice of the Units for which a state change was made is returned.
// Any error encountered is immediately returned (i.e. this is not a transaction).
func setTargetStateOfUnits(units []string, state job.JobState) ([]*schema.Unit, error) {
triggered := make([]*schema.Unit, 0)
for _, name := range units {
u, err := cAPI.Unit(name)
if err != nil {
return nil, fmt.Errorf("error retrieving unit %s from registry: %v", name, err)
} else if u == nil {
return nil, fmt.Errorf("unable to find unit %s", name)
} else if job.JobState(u.DesiredState) == state {
log.V(1).Infof("Unit(%s) already %s, skipping.", u.Name, u.DesiredState)
continue
}
log.V(1).Infof("Setting Unit(%s) target state to %s", u.Name, state)
cAPI.SetUnitTargetState(u.Name, string(state))
triggered = append(triggered, u)
}
return triggered, nil
}
// getUnitFromFile attempts to load a Unit from a given filename
// It returns the Unit or nil, and any error encountered
func getUnitFromFile(file string) (*unit.UnitFile, error) {
out, err := ioutil.ReadFile(file)
if err != nil {
return nil, err
}
unitName := path.Base(file)
log.V(1).Infof("Unit(%s) found in local filesystem", unitName)
return unit.NewUnitFile(string(out))
}
// HasMetadata determine if the Metadata of a given MachineState
// matches the indicated values.
func HasMetadata(state *MachineState, metadata map[string]pkg.Set) bool {
for key, values := range metadata {
local, ok := state.Metadata[key]
if !ok {
log.V(1).Infof("No local values found for Metadata(%s)", key)
return false
}
log.V(1).Infof("Asserting local Metadata(%s) meets requirements", key)
if values.Contains(local) {
log.V(1).Infof("Local Metadata(%s) meets requirement", key)
} else {
log.V(1).Infof("Local Metadata(%s) does not match requirement", key)
return false
}
}
return true
}
func getDefaultGatewayIface() *net.Interface {
log.V(1).Infof("Attempting to retrieve IP route info from netlink")
routes, err := netlink.NetworkGetRoutes()
if err != nil {
log.V(1).Infof("Unable to detect default interface: %v", err)
return nil
}
if len(routes) == 0 {
log.V(1).Infof("Netlink returned zero routes")
return nil
}
for _, route := range routes {
if route.Default {
if route.Iface == nil {
log.V(1).Infof("Found default route but could not determine interface")
}
log.V(1).Infof("Found default route with interface %v", route.Iface.Name)
return route.Iface
}
}
log.V(1).Infof("Unable to find default route")
return nil
}
func runUnloadUnit(args []string) (exit int) {
units, err := findUnits(args)
if err != nil {
stderr("%v", err)
return 1
}
wait := make([]string, 0)
for _, s := range units {
if !suToGlobal(s) {
if job.JobState(s.CurrentState) == job.JobStateInactive {
log.V(1).Infof("Target state of Unit(%s) already %s, skipping.", s.Name, job.JobStateInactive)
continue
}
}
log.V(1).Infof("Setting target state of Unit(%s) to %s", s.Name, job.JobStateInactive)
cAPI.SetUnitTargetState(s.Name, string(job.JobStateInactive))
if suToGlobal(s) {
stdout("Triggered global unit %s unload", s.Name)
} else {
wait = append(wait, s.Name)
}
}
if !sharedFlags.NoBlock {
errchan := waitForUnitStates(wait, job.JobStateInactive, sharedFlags.BlockAttempts, os.Stdout)
for err := range errchan {
stderr("Error waiting for units: %v", err)
exit = 1
}
} else {
for _, name := range wait {
stdout("Triggered unit %s unload", name)
}
}
return
}
func (a *Agent) heartbeatJobs(ttl time.Duration, stop chan bool) {
heartbeat := func() {
machID := a.Machine.State().ID
launched := a.cache.launchedJobs()
for _, j := range launched {
go a.registry.UnitHeartbeat(j, machID, ttl)
}
}
interval := ttl / 2
ticker := time.Tick(interval)
for {
select {
case <-stop:
log.V(1).Info("HeartbeatJobs exiting due to stop signal")
return
case <-ticker:
log.V(1).Info("HeartbeatJobs tick")
heartbeat()
}
}
}
// PeriodicRefresh updates the current state of the CoreOSMachine at the
// interval indicated. Operation ceases when the provided channel is closed.
func (m *CoreOSMachine) PeriodicRefresh(interval time.Duration, stop chan bool) {
ticker := time.NewTicker(interval)
for {
select {
case <-stop:
log.V(1).Info("Halting CoreOSMachine.PeriodicRefresh")
ticker.Stop()
return
case <-ticker.C:
m.Refresh()
}
}
}
func createUnit(name string, uf *unit.UnitFile) (*schema.Unit, error) {
u := schema.Unit{
Name: name,
Options: schema.MapUnitFileToSchemaUnitOptions(uf),
}
err := cAPI.CreateUnit(&u)
if err != nil {
return nil, fmt.Errorf("failed creating unit %s: %v", name, err)
}
log.V(1).Infof("Created Unit(%s) in Registry", name)
return &u, nil
}
func (r *reconciler) Run(stop chan bool) {
trigger := make(chan struct{})
go func() {
abort := make(chan struct{})
for {
select {
case <-stop:
close(abort)
return
case <-r.eStream.Next(abort):
trigger <- struct{}{}
}
}
}()
ticker := r.clock.After(r.ival)
// When starting up, reconcile once immediately
log.V(1).Info("Initial reconciliation commencing")
r.rFunc()
for {
select {
case <-stop:
log.V(1).Info("Reconciler exiting due to stop signal")
return
case <-ticker:
ticker = r.clock.After(r.ival)
log.V(1).Info("Reconciler tick")
r.rFunc()
case <-trigger:
ticker = r.clock.After(r.ival)
log.V(1).Info("Reconciler triggered")
r.rFunc()
}
}
}
// Monitor ensures a Heart is still beating until a channel is closed, returning
// an error if the heartbeats fail.
func (m *Monitor) Monitor(hrt Heart, stop chan bool) error {
ticker := time.Tick(m.ival)
for {
select {
case <-stop:
log.V(1).Info("Monitor exiting due to stop signal")
return nil
case <-ticker:
if _, err := m.check(hrt); err != nil {
return err
}
}
}
}
func findAddressInRunningUnits(name string) (string, bool) {
name = unitNameMangle(name)
u, err := cAPI.Unit(name)
if err != nil {
log.V(1).Infof("Unable to retrieve Unit(%s) from Repository: %v", name, err)
}
if u == nil {
return "", false
}
m := cachedMachineState(u.MachineID)
if m != nil && m.PublicIP != "" {
return m.PublicIP, true
}
return "", false
}
// ensureLeader will attempt to renew a non-nil Lease, falling back to
// acquiring a new Lease on the lead engine role.
func ensureLeader(prev registry.Lease, reg registry.Registry, machID string, ttl time.Duration) (cur registry.Lease) {
if prev != nil {
err := prev.Renew(ttl)
if err == nil {
log.V(1).Infof("Engine leadership renewed")
cur = prev
return
} else {
log.Errorf("Engine leadership lost, renewal failed: %v", err)
}
}
var err error
cur, err = reg.LeaseRole(engineRoleName, machID, ttl)
if err != nil {
log.Errorf("Engine leadership acquisition failed: %v", err)
} else if cur == nil {
log.V(1).Infof("Unable to acquire engine leadership")
} else {
log.Infof("Engine leadership acquired")
}
return
}
// Run periodically attempts to reconcile the provided Agent until the stop
// channel is closed. Run will also reconcile in reaction to calls to Trigger.
// While a reconciliation is being attempted, calls to Trigger are ignored.
func (ar *AgentReconciler) Run(a *Agent, stop chan bool) {
reconcile := func() {
start := time.Now()
ar.Reconcile(a)
elapsed := time.Now().Sub(start)
msg := fmt.Sprintf("AgentReconciler completed reconciliation in %s", elapsed)
if elapsed > reconcileInterval {
log.Warning(msg)
} else {
log.V(1).Info(msg)
}
}
reconciler := pkg.NewPeriodicReconciler(reconcileInterval, reconcile, ar.rStream)
reconciler.Run(stop)
}