// GET: /job
// To do:
// - Iterate job queries
func (cr *JobController) ReadMany(cx *goweb.Context) {
LogRequest(cx.Request)
// Try to authenticate user.
u, err := request.Authenticate(cx.Request)
if err != nil && err.Error() != e.NoAuth {
cx.RespondWithErrorMessage(err.Error(), http.StatusUnauthorized)
return
}
// If no auth was provided, and anonymous read is allowed, use the public user
if u == nil {
if conf.ANON_READ == true {
u = &user.User{Uuid: "public"}
} else {
cx.RespondWithErrorMessage(e.NoAuth, http.StatusUnauthorized)
return
}
}
// Gather query params
query := &Query{Li: cx.Request.URL.Query()}
// Setup query and jobs objects
q := bson.M{}
jobs := core.Jobs{}
// Add authorization checking to query if the user is not an admin
if u.Admin == false {
q["$or"] = []bson.M{bson.M{"acl.read": "public"}, bson.M{"acl.read": u.Uuid}, bson.M{"acl.owner": u.Uuid}, bson.M{"acl": bson.M{"$exists": "false"}}}
}
limit := conf.DEFAULT_PAGE_SIZE
offset := 0
order := "updatetime"
direction := "desc"
if query.Has("limit") {
limit, err = strconv.Atoi(query.Value("limit"))
if err != nil {
cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
return
}
}
if query.Has("offset") {
offset, err = strconv.Atoi(query.Value("offset"))
if err != nil {
cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
return
}
}
if query.Has("order") {
order = query.Value("order")
}
if query.Has("direction") {
direction = query.Value("direction")
}
// Gather params to make db query. Do not include the
// following list.
skip := map[string]int{"limit": 1,
"offset": 1,
"query": 1,
"recent": 1,
"order": 1,
"direction": 1,
"active": 1,
"suspend": 1,
"registered": 1,
}
if query.Has("query") {
for key, val := range query.All() {
_, s := skip[key]
if !s {
queryvalues := strings.Split(val[0], ",")
q[key] = bson.M{"$in": queryvalues}
}
}
} else if query.Has("active") {
q["state"] = bson.M{"$in": core.JOB_STATS_ACTIVE}
} else if query.Has("suspend") {
q["state"] = core.JOB_STAT_SUSPEND
} else if query.Has("registered") {
q["state"] = bson.M{"$in": core.JOB_STATS_REGISTERED}
}
//getting real active (in-progress) job (some jobs are in "submitted" states but not in the queue,
//because they may have failed and not recovered from the mongodb).
if query.Has("active") {
err := jobs.GetAll(q, order, direction)
if err != nil {
logger.Error("err " + err.Error())
cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
return
}
filtered_jobs := []core.Job{}
act_jobs := core.QMgr.GetActiveJobs()
length := jobs.Length()
skip := 0
count := 0
for i := 0; i < length; i++ {
job := jobs.GetJobAt(i)
if _, ok := act_jobs[job.Id]; ok {
if skip < offset {
skip += 1
continue
}
job.Registered = true
filtered_jobs = append(filtered_jobs, job)
count += 1
if count == limit {
break
}
}
}
cx.RespondWithPaginatedData(filtered_jobs, limit, offset, len(act_jobs))
return
}
//geting suspended job in the current queue (excluding jobs in db but not in qmgr)
if query.Has("suspend") {
err := jobs.GetAll(q, order, direction)
if err != nil {
logger.Error("err " + err.Error())
cx.RespondWithError(http.StatusBadRequest)
return
}
filtered_jobs := []core.Job{}
suspend_jobs := core.QMgr.GetSuspendJobs()
length := jobs.Length()
skip := 0
count := 0
for i := 0; i < length; i++ {
job := jobs.GetJobAt(i)
if _, ok := suspend_jobs[job.Id]; ok {
if skip < offset {
skip += 1
continue
}
job.Registered = true
filtered_jobs = append(filtered_jobs, job)
count += 1
if count == limit {
break
}
}
}
cx.RespondWithPaginatedData(filtered_jobs, limit, offset, len(suspend_jobs))
return
}
if query.Has("registered") {
err := jobs.GetAll(q, order, direction)
if err != nil {
logger.Error("err " + err.Error())
cx.RespondWithError(http.StatusBadRequest)
return
}
paged_jobs := []core.Job{}
registered_jobs := []core.Job{}
length := jobs.Length()
total := 0
for i := 0; i < length; i++ {
job := jobs.GetJobAt(i)
if core.QMgr.IsJobRegistered(job.Id) {
job.Registered = true
registered_jobs = append(registered_jobs, job)
total += 1
}
}
count := 0
for i := offset; i < len(registered_jobs); i++ {
paged_jobs = append(paged_jobs, registered_jobs[i])
count += 1
if count == limit {
break
}
}
cx.RespondWithPaginatedData(paged_jobs, limit, offset, total)
return
}
total, err := jobs.GetPaginated(q, limit, offset, order, direction)
if err != nil {
logger.Error("err " + err.Error())
cx.RespondWithError(http.StatusBadRequest)
return
}
filtered_jobs := []core.Job{}
length := jobs.Length()
for i := 0; i < length; i++ {
job := jobs.GetJobAt(i)
if core.QMgr.IsJobRegistered(job.Id) {
job.Registered = true
} else {
job.Registered = false
}
filtered_jobs = append(filtered_jobs, job)
}
cx.RespondWithPaginatedData(filtered_jobs, limit, offset, total)
return
}
// GET: /job
// To do:
// - Iterate job queries
func (cr *JobController) ReadMany(cx *goweb.Context) {
LogRequest(cx.Request)
// Try to authenticate user.
u, err := request.Authenticate(cx.Request)
if err != nil && err.Error() != e.NoAuth {
cx.RespondWithErrorMessage(err.Error(), http.StatusUnauthorized)
return
}
// Gather query params
query := &Query{Li: cx.Request.URL.Query()}
// Setup query and jobs objects
q := bson.M{}
jobs := core.Jobs{}
if u != nil {
// Add authorization checking to query if the user is not an admin
if u.Admin == false {
q["$or"] = []bson.M{bson.M{"acl.read": "public"}, bson.M{"acl.read": u.Uuid}, bson.M{"acl.owner": u.Uuid}, bson.M{"acl": bson.M{"$exists": "false"}}}
}
} else {
// User is anonymous
if conf.ANON_READ {
// select on only jobs that are publicly readable
q["acl.read"] = "public"
} else {
cx.RespondWithErrorMessage(e.NoAuth, http.StatusUnauthorized)
return
}
}
limit := conf.DEFAULT_PAGE_SIZE
offset := 0
order := "info.submittime"
direction := "desc"
if query.Has("limit") {
limit, err = strconv.Atoi(query.Value("limit"))
if err != nil {
cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
return
}
}
if query.Has("offset") {
offset, err = strconv.Atoi(query.Value("offset"))
if err != nil {
cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
return
}
}
if query.Has("order") {
order = query.Value("order")
}
if query.Has("direction") {
direction = query.Value("direction")
}
// Gather params to make db query. Do not include the
// following list.
skip := map[string]int{"limit": 1,
"offset": 1,
"query": 1,
"recent": 1,
"order": 1,
"direction": 1,
"active": 1,
"suspend": 1,
"registered": 1,
"verbosity": 1,
"userattr": 1,
}
if query.Has("query") {
const shortForm = "2006-01-02"
date_query := bson.M{}
for key, val := range query.All() {
_, s := skip[key]
if !s {
// special case for date range, either full date-time or just date
if (key == "date_start") || (key == "date_end") {
opr := "$gte"
if key == "date_end" {
opr = "$lt"
}
if t_long, err := time.Parse(time.RFC3339, val[0]); err != nil {
if t_short, err := time.Parse(shortForm, val[0]); err != nil {
cx.RespondWithErrorMessage("Invalid datetime format: "+val[0], http.StatusBadRequest)
return
} else {
date_query[opr] = t_short
}
} else {
date_query[opr] = t_long
}
} else {
// handle either multiple values for key, or single comma-spereated value
if len(val) == 1 {
queryvalues := strings.Split(val[0], ",")
q[key] = bson.M{"$in": queryvalues}
} else if len(val) > 1 {
q[key] = bson.M{"$in": val}
}
}
}
}
// add submittime and completedtime range query
if len(date_query) > 0 {
q["$or"] = []bson.M{bson.M{"info.submittime": date_query}, bson.M{"info.completedtime": date_query}}
}
} else if query.Has("active") {
q["state"] = bson.M{"$in": core.JOB_STATS_ACTIVE}
} else if query.Has("suspend") {
q["state"] = core.JOB_STAT_SUSPEND
} else if query.Has("registered") {
q["state"] = bson.M{"$in": core.JOB_STATS_REGISTERED}
}
//getting real active (in-progress) job (some jobs are in "submitted" states but not in the queue,
//because they may have failed and not recovered from the mongodb).
if query.Has("active") {
err := jobs.GetAll(q, order, direction)
if err != nil {
logger.Error("err " + err.Error())
cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
return
}
filtered_jobs := []core.Job{}
act_jobs := core.QMgr.GetActiveJobs()
length := jobs.Length()
skip := 0
count := 0
for i := 0; i < length; i++ {
job := jobs.GetJobAt(i)
if _, ok := act_jobs[job.Id]; ok {
if skip < offset {
skip += 1
continue
}
job.Registered = true
filtered_jobs = append(filtered_jobs, job)
count += 1
if count == limit {
break
}
}
}
cx.RespondWithPaginatedData(filtered_jobs, limit, offset, len(act_jobs))
return
}
//geting suspended job in the current queue (excluding jobs in db but not in qmgr)
if query.Has("suspend") {
err := jobs.GetAll(q, order, direction)
if err != nil {
logger.Error("err " + err.Error())
cx.RespondWithError(http.StatusBadRequest)
return
}
filtered_jobs := []core.Job{}
suspend_jobs := core.QMgr.GetSuspendJobs()
length := jobs.Length()
skip := 0
count := 0
for i := 0; i < length; i++ {
job := jobs.GetJobAt(i)
if _, ok := suspend_jobs[job.Id]; ok {
if skip < offset {
skip += 1
continue
}
job.Registered = true
filtered_jobs = append(filtered_jobs, job)
count += 1
if count == limit {
break
}
}
}
cx.RespondWithPaginatedData(filtered_jobs, limit, offset, len(suspend_jobs))
return
}
if query.Has("registered") {
err := jobs.GetAll(q, order, direction)
if err != nil {
logger.Error("err " + err.Error())
cx.RespondWithError(http.StatusBadRequest)
return
}
paged_jobs := []core.Job{}
registered_jobs := []core.Job{}
length := jobs.Length()
total := 0
for i := 0; i < length; i++ {
job := jobs.GetJobAt(i)
if core.QMgr.IsJobRegistered(job.Id) {
job.Registered = true
registered_jobs = append(registered_jobs, job)
total += 1
}
}
count := 0
for i := offset; i < len(registered_jobs); i++ {
paged_jobs = append(paged_jobs, registered_jobs[i])
count += 1
if count == limit {
break
}
}
cx.RespondWithPaginatedData(paged_jobs, limit, offset, total)
return
}
if query.Has("verbosity") && (query.Value("verbosity") == "minimal") {
// TODO - have mongo query only return fields needed to populate JobMin struct
total, err := jobs.GetPaginated(q, limit, offset, order, direction)
if err != nil {
logger.Error("err " + err.Error())
cx.RespondWithError(http.StatusBadRequest)
return
}
minimal_jobs := []core.JobMin{}
length := jobs.Length()
for i := 0; i < length; i++ {
job := jobs.GetJobAt(i)
// create and populate minimal job
mjob := core.JobMin{}
mjob.Id = job.Id
mjob.Name = job.Info.Name
mjob.SubmitTime = job.Info.SubmitTime
mjob.CompletedTime = job.Info.CompletedTime
// get size of input
var size_sum int64 = 0
for _, v := range job.Tasks[0].Inputs {
size_sum = size_sum + v.Size
}
mjob.Size = size_sum
// add userattr fields
if query.Has("userattr") {
mjob.UserAttr = map[string]string{}
for _, attr := range query.List("userattr") {
if val, ok := job.Info.UserAttr[attr]; ok {
mjob.UserAttr[attr] = val
}
}
}
if (job.State == "completed") || (job.State == "deleted") {
// if completed or deleted move on, empty task array
mjob.State = append(mjob.State, job.State)
} else if job.State == "suspend" {
// get failed task if info available, otherwise empty task array
mjob.State = append(mjob.State, "suspend")
parts := strings.Split(job.LastFailed, "_")
if (len(parts) == 2) || (len(parts) == 3) {
if tid, err := strconv.Atoi(parts[1]); err == nil {
mjob.Task = append(mjob.Task, tid)
}
}
} else {
// get multiple tasks in state queued or in-progress
for j := 0; j < len(job.Tasks); j++ {
task := job.Tasks[j]
if (task.State == "in-progress") || (task.State == "queued") {
mjob.State = append(mjob.State, task.State)
mjob.Task = append(mjob.Task, j)
}
}
// otherwise get oldest pending or init task
if len(mjob.State) == 0 {
for j := 0; j < len(job.Tasks); j++ {
task := job.Tasks[j]
if (task.State == "pending") || (task.State == "init") {
mjob.State = append(mjob.State, task.State)
mjob.Task = append(mjob.Task, j)
break
}
}
}
}
minimal_jobs = append(minimal_jobs, mjob)
}
cx.RespondWithPaginatedData(minimal_jobs, limit, offset, total)
return
}
total, err := jobs.GetPaginated(q, limit, offset, order, direction)
if err != nil {
logger.Error("err " + err.Error())
cx.RespondWithError(http.StatusBadRequest)
return
}
filtered_jobs := []core.Job{}
length := jobs.Length()
for i := 0; i < length; i++ {
job := jobs.GetJobAt(i)
if core.QMgr.IsJobRegistered(job.Id) {
job.Registered = true
} else {
job.Registered = false
}
filtered_jobs = append(filtered_jobs, job)
}
cx.RespondWithPaginatedData(filtered_jobs, limit, offset, total)
return
}
// GET: /job
// To do:
// - Iterate job queries
func (cr *JobController) ReadMany(cx *goweb.Context) {
LogRequest(cx.Request)
// Gather query params
query := &Query{Li: cx.Request.URL.Query()}
// Setup query and jobs objects
q := bson.M{}
jobs := core.Jobs{}
// Gather params to make db query. Do not include the
// following list.
skip := map[string]int{"limit": 1, "skip": 1, "query": 1, "recent": 1}
if query.Has("query") {
for key, val := range query.All() {
_, s := skip[key]
if !s {
queryvalues := strings.Split(val[0], ",")
q[key] = bson.M{"$in": queryvalues}
}
}
} else if query.Has("active") {
q["state"] = core.JOB_STAT_INPROGRESS
} else if query.Has("suspend") {
q["state"] = core.JOB_STAT_SUSPEND
}
// Limit and skip. Set default if both are not specified
if query.Has("limit") || query.Has("skip") || query.Has("recent") {
var lim, off, recent int
if query.Has("limit") {
lim, _ = strconv.Atoi(query.Value("limit"))
} else {
lim = 100
}
if query.Has("skip") {
off, _ = strconv.Atoi(query.Value("skip"))
} else {
off = 0
}
if query.Has("recent") {
recent, _ = strconv.Atoi(query.Value("recent"))
} else {
recent = 0
}
var err error
if recent > 0 {
_, err = jobs.GetAllRecent(q, recent)
} else {
err = jobs.GetAllLimitOffset(q, lim, off)
}
if err != nil {
logger.Error("err " + err.Error())
cx.RespondWithError(http.StatusBadRequest)
return
}
} else {
// Get jobs from db
err := jobs.GetAll(q)
if err != nil {
logger.Error("err " + err.Error())
cx.RespondWithError(http.StatusBadRequest)
return
}
}
//getting real active (in-progress) job (some jobs are in "submitted" states but not in the queue,
//because they may have failed and not recovered from the mongodb).
if query.Has("active") {
filtered_jobs := []core.Job{}
act_jobs := core.QMgr.GetActiveJobs()
length := jobs.Length()
for i := 0; i < length; i++ {
job := jobs.GetJobAt(i)
if _, ok := act_jobs[job.Id]; ok {
filtered_jobs = append(filtered_jobs, job)
}
}
cx.RespondWithData(filtered_jobs)
return
}
//geting suspended job in the current queue (excluding jobs in db but not in qmgr)
if query.Has("suspend") {
filtered_jobs := []core.Job{}
suspend_jobs := core.QMgr.GetSuspendJobs()
length := jobs.Length()
for i := 0; i < length; i++ {
job := jobs.GetJobAt(i)
if _, ok := suspend_jobs[job.Id]; ok {
filtered_jobs = append(filtered_jobs, job)
}
}
cx.RespondWithData(filtered_jobs)
return
}
cx.RespondWithData(jobs)
return
}