func (r *RunCommand) runOnRemote(localPath string, cmdWithArgsStr string) (*ExecRes, error) { machine, err := mountcli.NewMountcli().FindMountNameByPath(localPath) if err != nil { return nil, err } fullCmdPath, err := r.getCmdRemotePath(machine, localPath) if err != nil { return nil, err } // track metrics metrics.TrackRun(machine, config.VersionNum()) return r.runOnMachine(machine, fullCmdPath, cmdWithArgsStr) }
func (m *MetricClient) StartMountStatusTicker(mount string) (err error) { var ( i int failures int ) // stop previous tickers if any m.StopMountStatusTicker(mount) // start new ticker and save for future use ticker := time.NewTicker(m.Interval) m.tickers[mount] = ticker path, err := mountcli.NewMountcli().FindMountedPathByName(mount) if err != nil { TrackMountCheckFailure(mount, err.Error()) return err } for range ticker.C { ms := NewDefaultMountStatus(path) // alterate between reading & writing; this is req. since kernel // catches file on write, if we read right away, it'll return contents // from kernel cache and not from mount like we want it to if i%2 == 0 { err = ms.Write() } else { err = ms.CheckContents() } // we only care about failures and not success if err != nil { TrackMountCheckFailure(mount, err.Error()) failures += 1 } // if it errors more than limit, return from ticker if failures > m.LimitFailures { return nil } i += 1 } return nil }
// getCmdRemotePath return the path on remote machine where the command should // be run. func (r *RunCommand) getCmdRemotePath(machine, localPath string) (string, error) { relativePath, err := mountcli.NewMountcli().FindRelativeMountPath(localPath) if err != nil { return "", err } mounts, err := r.getMounts() if err != nil { return "", err } for _, m := range mounts { if m.MountName == machine { // join path in remote machine return filepath.Join(m.RemotePath, relativePath), nil } } return "", mountcli.ErrNotInMount }
// initDefaultRepairers creates the repairers for this Command if the // Command.Repairers field is *nil*. This allows a caller can specify their own // repairers if desired. func (c *Command) initDefaultRepairers() error { if c.Repairers != nil { return nil } // TODO: Re-enable. Currently disabled because we're manually running it // before the checkMachineExist() call inside of Run(). // //// The kontrol repairer will check if we're connected to kontrol yet, and //// attempt to wait for it. Eventually restarting if needed. //kontrolRepair := &KontrolRepair{ // Log: c.Log.New("KontrolRepair"), // Stdout: c.Stdout, // Klient: c.Klient, // RetryOptions: RetryOptions{ // StatusRetries: 3, // StatusDelay: 10 * time.Second, // }, // Exec: &exec.CommandRun{ // Stdin: c.Stdin, // Stdout: c.Stdout, // }, //} // The kite unreachable repairer ensures that the remote machine is on, and // kite is reachable. No repair action is possible. kiteUnreachableRepair := &KiteUnreachableRepair{ Log: c.Log.New("KiteUnreachableRepair"), Stdout: c.Stdout, Klient: c.Klient, StatusRetries: 10, StatusDelay: 1 * time.Second, MachineName: c.Options.MountName, } // The token expired repair checks for token expired. This should be placed *before* // TokenNotYetValidRepair, so that after we restart, we can check if the token // is valid. tokenExpired := &TokenExpiredRepair{ Log: c.Log.New("TokenExpiredRepair"), Stdout: c.Stdout, Klient: c.Klient, RepairWaitForToken: 5 * time.Second, MachineName: c.Options.MountName, } // The token not yet valid repairer will check if we're failing from the token // not yet valid error, and wait for it to become valid. tokenNotValidYetRepair := &TokenNotYetValidRepair{ Log: c.Log.New("TokenNotYetValidRepair"), Stdout: c.Stdout, Klient: c.Klient, RepairRetries: 5, RepairDelay: 3 * time.Second, MachineName: c.Options.MountName, } mountExistsRepair := &MountExistsRepair{ Log: c.Log.New("MountExistsRepair"), Stdout: util.NewFprint(c.Stdout), MountName: c.Options.MountName, Klient: c.Klient, Mountcli: mountcli.NewMountcli(), } permDeniedRepair := &PermDeniedRepair{ Log: c.Log.New("PermDeniedRepair"), Stdout: util.NewFprint(c.Stdout), MountName: c.Options.MountName, Klient: c.Klient, } mountEmptyRepair := &MountEmptyRepair{ Log: c.Log.New("MountEmptyRepair"), Stdout: util.NewFprint(c.Stdout), MountName: c.Options.MountName, Klient: c.Klient, } deviceNotConfiguredRepair := &DeviceNotConfiguredRepair{ Log: c.Log.New("DeviceNotConfiguredRepair"), Stdout: util.NewFprint(c.Stdout), MountName: c.Options.MountName, Klient: c.Klient, } writeReadRepair := &WriteReadRepair{ Log: c.Log.New("WriteReadRepair"), Stdout: util.NewFprint(c.Stdout), MountName: c.Options.MountName, Klient: c.Klient, } // A collection of Repairers responsible for actually repairing a given mount. // Executed in the order they are defined, the effectiveness of the Repairers // may depend on the order they are run in. An example being TokenNotValidYetRepair // likely should be run *after* a restart, as Tokens not being valid yet usually // happens after a restart. c.Repairers = []Repairer{ //kontrolRepair, kiteUnreachableRepair, tokenExpired, tokenNotValidYetRepair, mountExistsRepair, permDeniedRepair, mountEmptyRepair, deviceNotConfiguredRepair, writeReadRepair, } return nil }
func (r *Remote) restoreMount(m *mount.Mount) (err error) { if r.mockedRestoreMount != nil { return r.mockedRestoreMount(m) } // The two New methods is to tweak how the log is displayed. log := logging.NewLogger("remote").New("restoreMount").New( "mountName", m.MountName, "syncMount", m.MountFolder.OneWaySyncMount, "prefetchAll", m.MountFolder.PrefetchAll, ) // Enable debug for the mount that was originally using debug. if m.MountFolder.Debug { log.SetLevel(logging.DEBUG) } // First get the plain machine, we don't care about it being dialed or valid as // we will potentially just be setting the status with it. remoteMachine, err := r.GetMachine(m.MountName) if err != nil { return err } // If the machine does not have an http tracker, create it so that we can // get accurate online/offline information. if !remoteMachine.HasHTTPTracker() { // No need to return here, this just means we won't get accurate information // about online/offline *before the machine is valid*. This is used mainly // in the defer, to mark the remounting machine as offline. // // Later, we'll get a valid and dialed machine, which is assured to have an // http tracker or fail trying. if err := remoteMachine.InitHTTPTracker(); err != nil { log.Error("Unable to init http tracker before remount. err:%s", err) } } // Update the status based on the return value. Note that it's possible to // return before this call, if we can't get the machine, but that's a non-issue // for updating the machine status, since we failed to get the machine, and // can't possible update the status. defer func() { if err != nil { // Update the user that we failed, and are retrying. switch { case !remoteMachine.IsOnline() && remoteMachine.HasHTTPTracker(): // The machine is offline / unreachable, so don't set the status to // remounting specifically. // // TODO: Check if we have internet here? remoteMachine.SetStatus(machine.MachineOffline, remountingButOffline) default: // Machine status is not offline/disconnected, therefor it may be // online and/or connected - but we failed to mount for another unknown // reason. Use a generic status. remoteMachine.SetStatus(machine.MachineRemounting, autoRemountingAgain) } } else { // If there's no errors, clear the status. remoteMachine.SetStatus(machine.MachineStatusUnknown, "") } }() // Now try to get a valid, dialed machine. We're doing this *after* the // machine's setstatus defer, so that we can set autoRemountingAgain as needed. // // Note that we're not getting the instance here, because if we cannot get a // dialed machine then remoteMachine will be set to nil, causing a panic // in the defer above. Regardless, it's the same instance, we don't need it. if _, err := r.GetDialedMachine(m.MountName); err != nil { return err } if remoteMachine.IsMountingLocked() { log.Warning("Restore mount was attempted but the machine is mount locked") return machine.ErrMachineActionIsLocked } // Lock and defer unlock the machine mount actions remoteMachine.LockMounting() defer remoteMachine.UnlockMounting() fsMountName, _ := mountcli.NewMountcli().FindMountNameByPath(m.LocalPath) if fsMountName != "" { failOnUnmount := true // Mount path exists, but the name doesn't match our mount name. // This occurs if the folder has been mounted by something else (ie, // the user), so to be safe we should not mount this folder. if fsMountName != m.MountName { log.Warning( "The path %q has a fs mountName of %q, but %q was expected.", m.LocalPath, fsMountName, m.MountName, ) failOnUnmount = false } log.Info("Automatically unmounting") m.Log = mount.MountLogger(m, log) // Mount path exists, and the names match. Unmount it, so that we // can remount it below. if err := m.Unmount(); err != nil { if failOnUnmount { log.Error("Failed to automatically unmount. err:%s", err) return err } else { log.Error( "Failed to automatically unmount, but ignoring unmount error. Continuing. err:%s", err, ) } } } // Mount path has been unmounted, or didn't exist locally. // Remount it, to improve UX. log.Info("Automatically mounting") // Construct our mounter mounter := &mount.Mounter{ Log: log, Options: m.MountFolder, Machine: remoteMachine, IP: remoteMachine.IP, KiteTracker: remoteMachine.KiteTracker, Transport: remoteMachine, PathUnmounter: fuseklient.Unmount, EventSub: r.eventSub, } if err := mounter.MountExisting(m); err != nil { return err } // remote.cache is normally responsible for creating the intervaler, but // because cache is not creating one here, we need to do it manually. if remoteMachine.Intervaler == nil { if !m.SyncIntervalOpts.IsZero() { rs := rsync.NewClient(log) // After the progress chan is done, start our SyncInterval startIntervalerIfNeeded(log, remoteMachine, rs, m.SyncIntervalOpts) // Assign the rsync intervaler to the mount. m.Intervaler = remoteMachine.Intervaler } else { log.Warning( "Unable to restore Interval for remote, SyncOpts is zero value. This likely means that SyncOpts were not saved or didn't exist in the previous binary. machineName:%s", remoteMachine.Name, ) } } return nil }