Example #1
0
File: cmd.go Project: pressly/qmd
func (cmd *Cmd) Kill() error {
	switch cmd.State {
	case Running:
		cmd.State = Terminated
		lg.Debugf("Cmd:\tKilling %v\n", cmd.JobID)
		pgid, err := syscall.Getpgid(cmd.Cmd.Process.Pid)
		if err != nil {
			// Fall-back on error. Kill the main process only.
			cmd.Cmd.Process.Kill()
			break
		}
		// Kill the whole process group.
		syscall.Kill(-pgid, 15)

	case Finished:
		lg.Debug("Cmd:\tKilling pgroup %v\n", cmd.JobID)
		pgid, err := syscall.Getpgid(cmd.Cmd.Process.Pid)
		if err != nil {
			break
		}
		// Make sure to kill the whole process group,
		// so there are no subprocesses left.
		syscall.Kill(-pgid, 15)

	case Initialized:
		// This one is tricky, as the cmd's Start() might have
		// been called and is already in progress, but the cmd's
		// state is not Running yet.
		usCallingStartOnce := false
		cmd.StartOnce.Do(func() {
			cmd.WaitOnce.Do(func() {
				cmd.State = Invalidated
				cmd.StatusCode = -2
				cmd.Err = errors.New("invalidated")
				lg.Debugf("Cmd: Invalidating %v\n", cmd.JobID)
				close(cmd.Finished)
			})
			close(cmd.Started)
			usCallingStartOnce = true
		})
		if !usCallingStartOnce {
			// It was cmd.Start() that called StartOnce.Do(), not us,
			// thus we need to wait for Started and try to Kill again:
			<-cmd.Started
			cmd.Kill()
		}
	}

	return cmd.Err
}
Example #2
0
File: queue.go Project: pressly/qmd
func (qmd *Qmd) ListenQueue() {
	qmd.WaitListenQueue.Add(1)
	defer qmd.WaitListenQueue.Done()

	lg.Debug("Queue:\tListening")

	for {
		select {
		// Wait for some worker to become available.
		case worker := <-qmd.Workers:
			// Dequeue job or try again.
			job, err := qmd.Dequeue()
			if err != nil {
				qmd.Workers <- worker
				break
			}
			lg.Debugf("Queue:\tDequeued job %v", job.ID)
			// Send the job to the worker.
			worker <- job

		case <-qmd.ClosingListenQueue:
			lg.Debug("Queue:\tStopped listening")
			return
		}
	}
}
Example #3
0
File: cmd.go Project: pressly/qmd
func (cmd *Cmd) waitOnce() {
	err := cmd.Cmd.Wait()
	cmd.Duration = time.Since(cmd.StartTime)
	cmd.EndTime = cmd.StartTime.Add(cmd.Duration)
	if cmd.State != Terminated {
		cmd.State = Finished
	}

	if err != nil {
		cmd.Err = err
		if e, ok := err.(*exec.ExitError); ok {
			if s, ok := e.Sys().(syscall.WaitStatus); ok {
				cmd.StatusCode = s.ExitStatus()
			}
		}
	}

	if f, err := os.Open(cmd.QmdOutFile); err == nil {
		_, err := cmd.QmdOut.ReadFrom(f)
		if err != nil {
			cmd.Err = err
		}
		f.Close()
	}

	close(cmd.Finished)
	lg.Debugf("Cmd:\tFinished %v", cmd.JobID)
}
Example #4
0
File: cmd.go Project: pressly/qmd
func (cmd *Cmd) Cleanup() error {
	lg.Debugf("Cmd:\tCleaning %v\n", cmd.JobID)

	// Make sure to kill the whole process group,
	// so there are no subprocesses left.
	cmd.Kill()

	// Remove working directory.
	err := os.RemoveAll(cmd.Cmd.Dir)
	if err != nil {
		return err
	}

	return nil
}
Example #5
0
// Update walks ScriptDir directory for shell scripts and updates the files cache.
func (s *Scripts) Update(dir string) error {
	info, err := os.Stat(dir)
	if err != nil {
		return errors.New("script_dir=\"" + dir + "\": no such directory")
	}
	if !info.IsDir() {
		return errors.New("script_dir=\"" + dir + "\": not a directory")
	}

	files := map[string]string{}
	if err := filepath.Walk(dir, func(file string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			if path.Ext(file) == ".sh" {
				rel, err := filepath.Rel(dir, file)
				if err != nil {
					return err
				}
				files[rel] = file
			}
		}
		return nil
	}); err != nil {
		return err
	}

	if len(files) == 0 {
		return errors.New("script_dir=\"" + dir + "\" is empty")
	}

	s.Lock()
	defer s.Unlock()

	if !reflect.DeepEqual(s.files, files) {
		lg.Debug("Scripts:	Loading new files from script_dir:")
		for rel, _ := range files {
			lg.Debugf("Scripts:	 - %v", rel)
		}
	}

	s.files = files
	return nil
}
Example #6
0
func CreateJob(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	// Low, high and urgent priorities only (high is default).
	priority := r.URL.Query().Get("priority")
	switch priority {
	case "low", "high", "urgent":
		// NOP.
	case "":
		priority = "high"
	default:
		http.Error(w, "unknown priority \""+priority+"\"", 422)
		return
	}

	// Decode request data.
	var req *api.ScriptsRequest
	err := json.NewDecoder(r.Body).Decode(&req)
	if err != nil {
		http.Error(w, "parse request body: "+err.Error(), 422)
		return
	}
	req.Script = chi.URLParams(ctx)["filename"]

	// Make sure ASYNC callback is valid URL.
	if req.CallbackURL != "" {
		req.CallbackURL, err = urlx.NormalizeString(req.CallbackURL)
		if err != nil {
			http.Error(w, "parse request body: "+err.Error(), 422)
			return
		}
	}

	// Enqueue the request.
	data, err := json.Marshal(req)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	lg.Debugf("Handler:\tEnqueue \"%v\" request", priority)
	job, err := Qmd.Enqueue(string(data), priority)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	// Async.
	if req.CallbackURL != "" {
		resp, _ := Qmd.GetAsyncResponse(req, job.ID)
		w.Write(resp)
		lg.Debugf("Handler:\tResponded with job %s ASYNC result", job.ID)

		go func() {
			//TODO: Retry callback if it failed?
			err := Qmd.PostResponseCallback(req, job.ID)
			if err != nil {
				lg.Errorf("can't post callback to %v", err)
			}
		}()
		return
	}

	// Sync.
	lg.Debugf("Handler:\tWaiting for job %s", job.ID)

	resp, _ := Qmd.GetResponse(job.ID)
	w.Write(resp)

	lg.Debugf("Handler:\tResponded with job %s result", job.ID)

	// 	// Kill the job, if client closes the connection before
	// 	// it receives the data.
	// 	done := make(chan struct{})
	// 	defer close(done)
	// 	connClosed := w.(http.CloseNotifier).CloseNotify()
	// 	go func() {
	// 		select {
	// 		case <-connClosed:
	// 			job.Kill()
	// 		case <-done:
	// 		}
	// 	}()
}
Example #7
0
File: cmd.go Project: pressly/qmd
func (cmd *Cmd) startOnce() {
	lg.Debugf("Cmd:\tStarting %v", cmd.JobID)

	cmd.Cmd.Dir += "/" + cmd.JobID
	cmd.QmdOutFile = cmd.Cmd.Dir + "/QMD_OUT"
	cmd.Cmd.Env = append(os.Environ(),
		"QMD_TMP="+cmd.Cmd.Dir,
		"QMD_STORE="+cmd.StoreDir,
		"QMD_OUT="+cmd.QmdOutFile,
	)

	cmd.Cmd.SysProcAttr = &syscall.SysProcAttr{
		Setpgid: true,
		//TODO: Chroot: cmd.Cmd.Dir,
	}

	cmd.Cmd.Stdout = &cmd.CmdOut
	cmd.Cmd.Stderr = &cmd.CmdOut

	// Create working directory.
	err := os.MkdirAll(cmd.Cmd.Dir, 0777)
	if err != nil {
		cmd.Err = err
	}

	// Create QMD_OUT file.
	qmdOut, err := os.Create(cmd.QmdOutFile)
	if err != nil {
		cmd.Err = err
	}
	qmdOut.Close()

	for file, data := range cmd.ExtraWorkDirFiles {
		// Must be a simple filename without slashes.
		if strings.Index(file, "/") != -1 {
			cmd.Err = errors.New("extra files must not contain any slashes in the path")
			goto failedToStart
		}
		err = ioutil.WriteFile(cmd.Cmd.Dir+"/"+file, []byte(data), 0644)
		if err != nil {
			cmd.Err = err
			goto failedToStart
		}
	}

	if err := cmd.Cmd.Start(); err != nil {
		cmd.Err = err
		goto failedToStart
	}

	cmd.StartTime = time.Now()
	cmd.State = Running
	close(cmd.Started)
	cmd.Err = nil
	return

failedToStart:
	cmd.StatusCode = -1
	cmd.State = Failed
	cmd.WaitOnce.Do(func() {
		close(cmd.Finished)
	})
	close(cmd.Started)
	lg.Debugf("Cmd:\tFailed to start %v: %v", cmd.JobID, cmd.Err)
}
Example #8
0
func (qmd *Qmd) startWorker(id int, workers chan Worker) {
	qmd.WaitWorkers.Add(1)
	defer qmd.WaitWorkers.Done()

	worker := make(Worker)
	for {
		// Mark this worker as available.
		workers <- worker

		select {
		// Wait for a job.
		case job := <-worker:
			msg := fmt.Errorf("Worker %v:\tGot \"%v\" job %v/jobs/%v", id, job.Queue, qmd.Config.URL, job.ID)
			lg.Error(msg)
			qmd.Slack.Notify(msg.Error())

			var req *api.ScriptsRequest
			err := json.Unmarshal([]byte(job.Data), &req)
			if err != nil {
				qmd.Queue.Ack(job)
				msg := fmt.Errorf("Worker %v:\tfailed: %v", err)
				lg.Error(msg)
				qmd.Slack.Notify(msg.Error())
				break
			}

			script, err := qmd.GetScript(req.Script)
			if err != nil {
				qmd.Queue.Ack(job)
				msg := fmt.Errorf("Worker %v:\tfailed: %v", err)
				lg.Error(msg)
				qmd.Slack.Notify(msg.Error())
				break
			}

			// Create QMD job to run the command.
			cmd, err := qmd.Cmd(exec.Command(script, req.Args...))
			if err != nil {
				qmd.Queue.Ack(job)
				msg := fmt.Errorf("Worker %v:\tfailed: %v", err)
				lg.Error(msg)
				qmd.Slack.Notify(msg.Error())
				break
			}
			cmd.JobID = job.ID
			cmd.CallbackURL = req.CallbackURL
			cmd.ExtraWorkDirFiles = req.Files

			// Run a job.
			go cmd.Run()
			<-cmd.Started

			select {
			// Wait for the job to finish.
			case <-cmd.Finished:

			// Or kill it, if it doesn't finish in a specified time.
			case <-time.After(time.Duration(qmd.Config.MaxExecTime) * time.Second):
				cmd.Kill()
				cmd.Wait()
				cmd.Cleanup()

			// Or kill it, if QMD is closing.
			case <-qmd.ClosingWorkers:
				lg.Debugf("Worker %d:\tStopping (busy)", id)
				cmd.Kill()
				cmd.Cleanup()
				qmd.Queue.Nack(job)
				msg := fmt.Errorf("Worker %d:\tNACKed job %v/jobs/%v", id, qmd.Config.URL, job.ID)
				lg.Error(msg)
				qmd.Slack.Notify(msg.Error())
				return
			}

			// Response.
			resp := api.ScriptsResponse{
				ID:     job.ID,
				Script: req.Script,
				Args:   req.Args,
				Files:  req.Files,
			}

			// "OK" and "ERR" for backward compatibility.
			if cmd.StatusCode == 0 {
				resp.Status = "OK"
			} else {
				resp.Status = "ERR"
			}

			resp.EndTime = cmd.EndTime
			resp.Duration = fmt.Sprintf("%f", cmd.Duration.Seconds())
			resp.QmdOut = cmd.QmdOut.String()
			resp.ExecLog = cmd.CmdOut.String()
			resp.StartTime = cmd.StartTime
			if cmd.Err != nil {
				resp.Err = cmd.Err.Error()
			}

			qmd.DB.SaveResponse(&resp)

			qmd.Queue.Ack(job)
			msg = fmt.Errorf("Worker %v:\tACKed job %v/jobs/%v", id, qmd.Config.URL, job.ID)
			lg.Error(msg)
			qmd.Slack.Notify(msg.Error())

		case <-qmd.ClosingWorkers:
			lg.Debugf("Worker %d:\tStopping (idle)", id)
			return
		}
	}
}
Example #9
0
func (qmd *Qmd) StartWorkers() {
	lg.Debugf("Starting %v QMD workers", qmd.Config.MaxJobs)
	for i := 0; i < qmd.Config.MaxJobs; i++ {
		go qmd.startWorker(i, qmd.Workers)
	}
}