func (mon *SlaveMonitor) slaveDidBeginRegistration(fd int) {
// Having just started the process, we expect an IO, which we convert to a UNIX domain socket
fileName := strconv.Itoa(rand.Int())
slaveFile := os.NewFile(uintptr(fd), fileName)
slaveUsock, err := unixsocket.NewFromFile(slaveFile)
if err != nil {
slog.Error(err)
}
// We now expect the slave to use this fd they send us to send a Pid&Identifier Message
msg, err := slaveUsock.ReadMessage()
if err != nil {
slog.Error(err)
}
pid, identifier, err := messages.ParsePidMessage(msg)
// And the last step before executing its action, the slave sends us a pipe it will later use to
// send us all the features it's loaded.
featurePipeFd, err := slaveUsock.ReadFD()
if err != nil {
slog.Error(err)
}
slaveNode := mon.tree.FindSlaveByName(identifier)
if slaveNode == nil {
Error("slavemonitor.go:slaveDidBeginRegistration:Unknown identifier:" + identifier)
}
slaveNode.SlaveWasInitialized(pid, slaveUsock, featurePipeFd)
}
func (mon *SlaveMonitor) slaveDidBeginRegistration(fd int) {
// Having just started the process, we expect an IO, which we convert to a UNIX domain socket
fileName := strconv.Itoa(rand.Int())
slaveFile := unixsocket.FdToFile(fd, fileName)
slaveUsock, err := unixsocket.NewUsockFromFile(slaveFile)
if err != nil {
slog.Error(err)
}
if err = slaveUsock.Conn.SetReadBuffer(1024); err != nil {
slog.Error(err)
}
if err = slaveUsock.Conn.SetWriteBuffer(1024); err != nil {
slog.Error(err)
}
// We now expect the slave to use this fd they send us to send a Pid&Identifier Message
msg, err := slaveUsock.ReadMessage()
if err != nil {
slog.Error(err)
}
pid, identifier, err := ParsePidMessage(msg)
slaveNode := mon.tree.FindSlaveByName(identifier)
if slaveNode == nil {
Error("slavemonitor.go:slaveDidBeginRegistration:Unknown identifier:" + identifier)
}
slaveNode.SlaveWasInitialized(pid, slaveUsock)
}
// In "SBooting", we have a pid and socket to the process we will use,
// but it has not yet finished initializing (generally, running the code
// specific to this slave. When we receive a message about the success or
// failure of this operation, we transition to either crashed or ready.
func (s *SlaveNode) doBootingState() string { // -> {SCrashed, SReady}
// The slave will execute its action and respond with a status...
// Note we don't hold the mutex while waiting for the action to execute.
msg, err := s.socket.ReadMessage()
if err != nil {
slog.Error(err)
}
s.trace("in booting state")
s.L.Lock()
defer s.L.Unlock()
msg, err = messages.ParseActionResponseMessage(msg)
if err != nil {
slog.ErrorString("[" + s.Name + "] " + err.Error())
}
if msg == "OK" {
return SReady
}
// Drain the process's feature messages, if we have any, so
// that reloads happen when any load-time problems get fixed:
s.L.Unlock()
s.handleMessages()
s.L.Lock()
// Clean up:
if s.Pid > 0 {
syscall.Kill(s.Pid, syscall.SIGKILL)
}
s.wipe()
s.Error = msg
return SCrashed
}
// In "SBooting", we have a pid and socket to the process we will use,
// but it has not yet finished initializing (generally, running the code
// specific to this slave. When we receive a message about the success or
// failure of this operation, we transition to either crashed or ready.
func (s *SlaveNode) doBootingState() string { // -> {SCrashed, SReady}
// The slave will execute its action and respond with a status...
// Note we don't hold the mutex while waiting for the action to execute.
msg, err := s.socket.ReadMessage()
if err != nil {
slog.Error(err)
}
s.L.Lock()
defer s.L.Unlock()
msg, err = messages.ParseActionResponseMessage(msg)
if err != nil {
slog.ErrorString("[" + s.Name + "] " + err.Error())
}
if msg == "OK" {
return SReady
}
if s.Pid > 0 {
syscall.Kill(s.Pid, syscall.SIGKILL)
}
s.wipe()
s.Error = msg
return SCrashed
}
func (s *SlaveNode) handleFeatureMessage(msg string) {
if file, err := ParseFeatureMessage(msg); err != nil {
slog.Error(err)
} else {
s.Features[file] = true
AddFile(file)
}
}
// This should only be called while holding a lock on s.L.
// This unfortunately holds the mutex for a little while, and if the
// command dies super early, the entire slave pretty well deadlocks.
// TODO: review this.
func (s *SlaveNode) bootCommand(request *CommandRequest) {
identifier := request.Name
// TODO: If crashed, do something different...
msg := messages.CreateSpawnCommandMessage(identifier)
_, err := s.socket.WriteMessage(msg)
if err != nil {
slog.Error(err)
return
}
commandFD, err := s.socket.ReadFD()
if err != nil {
fmt.Println(s.socket)
slog.Error(err)
return
}
fileName := strconv.Itoa(rand.Int())
commandFile := unixsocket.FdToFile(commandFD, fileName)
request.Retchan <- commandFile
}
func (s *SlaveNode) bootSlave(slave *SlaveNode) {
s.L.Lock()
defer s.L.Unlock()
s.trace("now sending slave boot request for %s", slave.Name)
msg := messages.CreateSpawnSlaveMessage(slave.Name)
_, err := s.socket.WriteMessage(msg)
if err != nil {
slog.Error(err)
}
}
// This should only be called while holding a lock on s.L.
// This unfortunately holds the mutex for a little while, and if the
// command dies super early, the entire slave pretty well deadlocks.
// TODO: review this.
func (s *SlaveNode) bootCommand(request *CommandRequest) {
if s.State == SCrashed {
request.Retchan <- &CommandReply{SCrashed, nil}
return
}
identifier := request.Name
msg := messages.CreateSpawnCommandMessage(identifier)
_, err := s.socket.WriteMessage(msg)
if err != nil {
slog.Error(err)
return
}
commandFD, err := s.socket.ReadFD()
if err != nil {
fmt.Println(s.socket)
slog.Error(err)
return
}
fileName := strconv.Itoa(rand.Int())
commandFile := unixsocket.FdToFile(commandFD, fileName)
request.Retchan <- &CommandReply{s.State, commandFile}
}
// This should only be called while holding a lock on s.L.
func (s *SlaveNode) bootSlave(slave *SlaveNode) {
if s.Error != "" {
slave.L.Lock()
slave.Error = s.Error
slave.event <- true
slave.L.Unlock()
return
}
msg := messages.CreateSpawnSlaveMessage(slave.Name)
_, err := s.socket.WriteMessage(msg)
if err != nil {
slog.Error(err)
}
}
// This unfortunately holds the mutex for a little while, and if the
// command dies super early, the entire slave pretty well deadlocks.
// TODO: review this.
func (s *SlaveNode) bootCommand(request *CommandRequest) {
s.L.Lock()
defer s.L.Unlock()
s.trace("now sending command boot request %v", request)
identifier := request.Name
msg := messages.CreateSpawnCommandMessage(identifier)
_, err := s.socket.WriteMessage(msg)
if err != nil {
slog.Error(err)
return
}
commandFD, err := s.socket.ReadFD()
if err != nil {
fmt.Println(s.socket)
slog.Error(err)
return
}
fileName := strconv.Itoa(rand.Int())
commandFile := os.NewFile(uintptr(commandFD), fileName)
request.Retchan <- &CommandReply{s.state, commandFile}
}
func StartSlaveMonitor(tree *ProcessTree, fileChanges <-chan []string, done chan bool) chan bool {
quit := make(chan bool)
go func() {
localMasterFile, remoteMasterFile, err := unixsocket.Socketpair(syscall.SOCK_DGRAM)
if err != nil {
Error("Couldn't create socketpair")
}
monitor := &SlaveMonitor{tree, remoteMasterFile}
defer monitor.cleanupChildren()
localMasterSocket, err := unixsocket.NewFromFile(localMasterFile)
if err != nil {
Error("Couldn't Open UNIXSocket")
}
// We just want this unix socket to be a channel so we can select on it...
registeringFds := make(chan int, 3)
go func() {
for {
if fd, err := localMasterSocket.ReadFD(); err != nil {
slog.Error(err)
} else {
registeringFds <- fd
}
}
}()
for _, slave := range monitor.tree.SlavesByName {
go slave.Run(monitor)
}
for {
select {
case <-quit:
done <- true
return
case fd := <-registeringFds:
go monitor.slaveDidBeginRegistration(fd)
case files := <-fileChanges:
if len(files) > 0 {
tree.RestartNodesWithFeatures(files)
}
}
}
}()
return quit
}
// see docs/client_master_handshake.md
func handleClientConnection(tree *processtree.ProcessTree, usock *unixsocket.Usock) {
defer usock.Close()
// we have established first contact to the client.
command, clientPid, argCount, argFD, err := receiveCommandArgumentsAndPid(usock, nil)
commandNode, slaveNode, err := findCommandAndSlaveNodes(tree, command, err)
if err != nil {
// connection was established, no data was sent. Ignore.
return
}
command = commandNode.Name // resolve aliases
clientFile, err := receiveTTY(usock, err)
defer clientFile.Close()
if err == nil && slaveNode.Error != "" {
writeStacktrace(usock, slaveNode, clientFile)
return
}
commandUsock, err := bootNewCommand(slaveNode, command, err)
if err != nil {
// If a client connects while the command is just
// booting up, it actually makes it here - still
// expects a backtrace, of course.
writeStacktrace(usock, slaveNode, clientFile)
return
}
defer commandUsock.Close()
err = sendClientPidAndArgumentsToCommand(commandUsock, clientPid, argCount, argFD, err)
err = sendTTYToCommand(commandUsock, clientFile, err)
cmdPid, err := receivePidFromCommand(commandUsock, err)
err = sendCommandPidToClient(usock, cmdPid, err)
exitStatus, err := receiveExitStatus(commandUsock, err)
err = sendExitStatus(usock, exitStatus, err)
if err != nil {
slog.Error(err)
}
// Done! Hooray!
}
// see docs/client_master_handshake.md
func handleClientConnection(tree *processtree.ProcessTree, usock *unixsocket.Usock) {
defer usock.Close()
// we have established first contact to the client.
command, clientPid, arguments, err := receiveCommandArgumentsAndPid(usock, nil)
commandNode, slaveNode, err := findCommandAndSlaveNodes(tree, command, err)
if err != nil {
// connection was established, no data was sent. Ignore.
return
}
command = commandNode.Name // resolve aliases
clientFile, err := receiveTTY(usock, err)
defer clientFile.Close()
if err == nil && slaveNode.Error != "" {
// we can skip steps 3-5 as they deal with the command process we're not spawning.
// Write a fake pid (step 6)
usock.WriteMessage("0")
// Write the error message to the terminal
clientFile.Write([]byte(slaveNode.Error))
// Skip step 7, and write an exit code to the client (step 8)
usock.WriteMessage("1")
return
}
commandUsock, err := bootNewCommand(slaveNode, command, err)
defer commandUsock.Close()
err = sendClientPidAndArgumentsToCommand(commandUsock, clientPid, arguments, err)
err = sendTTYToCommand(commandUsock, clientFile, err)
cmdPid, err := receivePidFromCommand(commandUsock, err)
err = sendCommandPidToClient(usock, cmdPid, err)
exitStatus, err := receiveExitStatus(commandUsock, err)
err = sendExitStatus(usock, exitStatus, err)
if err != nil {
slog.Error(err)
}
// Done! Hooray!
}
func Start(tree *processtree.ProcessTree, done chan bool) chan bool {
quit := make(chan bool)
go func() {
theChart = &StatusChart{}
theChart.RootSlave = tree.Root
theChart.numberOfSlaves = len(tree.SlavesByName)
theChart.Commands = tree.Commands
theChart.update = make(chan bool, 10)
theChart.directLogger = slog.NewShinyLogger(os.Stdout, os.Stderr)
scw := &StringChannelWriter{make(chan string, 10)}
slog.DefaultLogger = slog.NewShinyLogger(scw, scw)
termios, err := ttyutils.NoEcho(uintptr(os.Stdout.Fd()))
if err != nil {
slog.Error(err)
}
ticker := time.Tick(1000 * time.Millisecond)
for {
select {
case <-quit:
ttyutils.RestoreTerminalState(uintptr(os.Stdout.Fd()), termios)
done <- true
return
case <-ticker:
theChart.draw()
case output := <-scw.Notif:
theChart.L.Lock()
if theChart.drawnInitial {
print(output)
}
theChart.extraOutput += output
theChart.L.Unlock()
theChart.draw()
case <-tree.StateChanged:
theChart.draw()
case <-theChart.update:
theChart.draw()
}
}
}()
return quit
}
func (s *StatusChart) lengthOfOutput() int {
ts, err := ttyutils.Winsize(os.Stdout)
if err != nil {
slog.Error(err)
}
width := int(ts.Columns)
lines := strings.Split(s.extraOutput, "\n")
numLines := 0
for _, line := range lines {
n := (len(line) + width - 1) / width
if n == 0 {
n = 1
}
numLines += n
}
return numLines - 1
}
func StartSlaveMonitor(tree *ProcessTree, quit chan bool) {
localMasterFile, remoteMasterFile, err := unixsocket.Socketpair(syscall.SOCK_DGRAM)
if err != nil {
Error("Couldn't create socketpair")
}
monitor := &SlaveMonitor{tree, remoteMasterFile}
localMasterSocket, err := unixsocket.NewUsockFromFile(localMasterFile)
if err != nil {
Error("Couldn't Open UNIXSocket")
}
// We just want this unix socket to be a channel so we can select on it...
registeringFds := make(chan int, 3)
go func() {
for {
fd, err := localMasterSocket.ReadFD()
if err != nil {
slog.Error(err)
}
registeringFds <- fd
}
}()
for _, slave := range monitor.tree.SlavesByName {
go slave.Run(monitor)
}
for {
select {
case <-quit:
monitor.cleanupChildren()
quit <- true
return
case fd := <-registeringFds:
go monitor.slaveDidBeginRegistration(fd)
}
}
}
func startWatchingFile(file string) {
_, err := watcherIn.Write([]byte(file + "\n"))
if err != nil {
slog.Error(err)
}
}