Example #1
0
//poll ready tasks and push into workQueue
func (qm *ServerMgr) updateQueue() (err error) {
	for _, task := range qm.taskMap {
		if qm.isTaskReady(task) {
			if err := qm.taskEnQueue(task); err != nil {
				jobid, _ := GetJobIdByTaskId(task.Id)
				qm.SuspendJob(jobid, fmt.Sprintf("failed enqueuing task %s, err=%s", task.Id, err.Error()), task.Id)
			}
		}
	}
	for id, work := range qm.workQueue.workMap {
		if work == nil || work.Info == nil {
			jid, err := GetJobIdByWorkId(id)
			if err != nil {
				qm.workQueue.Delete(id)
				logger.Error(fmt.Sprintf("error: in updateQueue() workunit %s is nil, cannot get job id", id))
				continue
			}

			if work == nil {
				qm.workQueue.Delete(id)
				qm.SuspendJob(jid, fmt.Sprintf("workunit %s is nil", id), id)
				logger.Error(fmt.Sprintf("error: workunit %s is nil, deleted from queue", id))
			} else if work.Info == nil {
				qm.workQueue.Delete(id)
				qm.SuspendJob(jid, fmt.Sprintf("workunit %s has Info=nil", id), id)
				logger.Error(fmt.Sprintf("error: workunit %s has Info=nil, deleted from queue", id))
			}
		}
	}
	return
}
Example #2
0
func launchAPI(control chan int, port int) {
	c := controller.NewServerController()
	goweb.ConfigureDefaultFormatters()
	r := &goweb.RouteManager{}
	r.Map("/job/{jid}/acl/{type}", c.JobAcl["typed"])
	r.Map("/job/{jid}/acl", c.JobAcl["base"])
	r.Map("/cgroup/{cgid}/acl/{type}", c.ClientGroupAcl["typed"])
	r.Map("/cgroup/{cgid}/acl", c.ClientGroupAcl["base"])
	r.Map("/cgroup/{cgid}/token", c.ClientGroupToken)
	r.MapRest("/job", c.Job)
	r.MapRest("/work", c.Work)
	r.MapRest("/cgroup", c.ClientGroup)
	r.MapRest("/client", c.Client)
	r.MapRest("/queue", c.Queue)
	r.MapRest("/logger", c.Logger)
	r.MapRest("/awf", c.Awf)
	r.MapFunc("*", controller.ResourceDescription, goweb.GetMethod)
	if conf.SSL_ENABLED {
		err := goweb.ListenAndServeRoutesTLS(fmt.Sprintf(":%d", conf.API_PORT), conf.SSL_CERT_FILE, conf.SSL_KEY_FILE, r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: api: %v\n", err)
			logger.Error("ERROR: api: " + err.Error())
		}
	} else {
		err := goweb.ListenAndServeRoutes(fmt.Sprintf(":%d", conf.API_PORT), r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: api: %v\n", err)
			logger.Error("ERROR: api: " + err.Error())
		}
	}
	control <- 1 //we are ending
}
Example #3
0
func StartClientWorkers() {
	control := make(chan int)
	go heartBeater(control)
	go workStealer(control)
	go dataMover(control)
	go processor(control)
	go deliverer(control)
	for {
		who := <-control //block till someone dies and then restart it
		switch who {
		case ID_HEARTBEATER:
			go heartBeater(control)
			logger.Error("heartBeater died and restarted")
		case ID_WORKSTEALER:
			go workStealer(control)
			logger.Error("workStealer died and restarted")
		case ID_DATAMOVER:
			go dataMover(control)
			logger.Error("dataMover died and restarted")
		case ID_WORKER:
			go processor(control)
			logger.Error("worker died and restarted")
		case ID_DELIVERER:
			go deliverer(control)
			logger.Error("deliverer died and restarted")
		}
	}
}
Example #4
0
// GET: /client/{id}
func (cr *ClientController) Read(id string, cx *goweb.Context) {
	// Gather query params
	query := &Query{Li: cx.Request.URL.Query()}

	if query.Has("heartbeat") { //handle heartbeat
		cg, err := request.AuthenticateClientGroup(cx.Request)
		if err != nil {
			if err.Error() == e.NoAuth || err.Error() == e.UnAuth || err.Error() == e.InvalidAuth {
				if conf.CLIENT_AUTH_REQ == true {
					cx.RespondWithError(http.StatusUnauthorized)
					return
				}
			} else {
				logger.Error("Err@AuthenticateClientGroup: " + err.Error())
				cx.RespondWithError(http.StatusInternalServerError)
				return
			}
		}

		hbmsg, err := core.QMgr.ClientHeartBeat(id, cg)
		if err != nil {
			cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		} else {
			cx.RespondWithData(hbmsg)
		}
		return
	}

	LogRequest(cx.Request) //skip heartbeat in access log

	// Try to authenticate user.
	u, err := request.Authenticate(cx.Request)
	if err != nil && err.Error() != e.NoAuth {
		cx.RespondWithErrorMessage(err.Error(), http.StatusUnauthorized)
		return
	}

	// If no auth was provided, and anonymous read is allowed, use the public user
	if u == nil {
		if conf.ANON_READ == true {
			u = &user.User{Uuid: "public"}
		} else {
			cx.RespondWithErrorMessage(e.NoAuth, http.StatusUnauthorized)
			return
		}
	}

	client, err := core.QMgr.GetClientByUser(id, u)
	if err != nil {
		if err.Error() == e.ClientNotFound {
			cx.RespondWithErrorMessage(e.ClientNotFound, http.StatusBadRequest)
		} else {
			logger.Error("Error in GET client:" + err.Error())
			cx.RespondWithError(http.StatusBadRequest)
		}
		return
	}
	cx.RespondWithData(client)
	return
}
Example #5
0
// POST: /job
func (cr *JobController) Create(cx *goweb.Context) {
	// Log Request and check for Auth
	LogRequest(cx.Request)

	// Parse uploaded form
	params, files, err := ParseMultipartForm(cx.Request)

	if err != nil {
		if err.Error() == "request Content-Type isn't multipart/form-data" {
			cx.RespondWithErrorMessage("No job file is submitted", http.StatusBadRequest)
		} else {
			// Some error other than request encoding. Theoretically
			// could be a lost db connection between user lookup and parsing.
			// Blame the user, Its probaby their fault anyway.
			logger.Error("Error parsing form: " + err.Error())
			cx.RespondWithError(http.StatusBadRequest)
		}
		return
	}

	_, has_upload := files["upload"]
	_, has_awf := files["awf"]

	if !has_upload && !has_awf {
		cx.RespondWithErrorMessage("No job script or awf is submitted", http.StatusBadRequest)
		return
	}

	//send job submission request and get back an assigned job number (jid)
	var jid string
	jid, err = core.QMgr.JobRegister()
	if err != nil {
		logger.Error("Err@job_Create:GetNextJobNum: " + err.Error())
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}

	var job *core.Job
	job, err = core.CreateJobUpload(params, files, jid)
	if err != nil {
		logger.Error("Err@job_Create:CreateJobUpload: " + err.Error())
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}

	if token, err := request.RetrieveToken(cx.Request); err == nil {
		job.SetDataToken(token)
	}

	core.QMgr.EnqueueTasksByJobId(job.Id, job.TaskList())

	//log event about job submission (JB)
	logger.Event(event.JOB_SUBMISSION, "jobid="+job.Id+";jid="+job.Jid+";name="+job.Info.Name+";project="+job.Info.Project)
	cx.RespondWithData(job)
	return
}
Example #6
0
func deliverer(control chan int) {
	fmt.Printf("deliverer lanched, client=%s\n", core.Self.Id)
	defer fmt.Printf("deliverer exiting...\n")
	for {
		processed := <-fromProcessor
		work := processed.workunit
		workmap[work.Id] = ID_DELIVERER
		perfstat := processed.perfstat

		//post-process for works computed successfully: push output data to Shock
		move_start := time.Now().Unix()
		if work.State == core.WORK_STAT_COMPUTED {
			if err := core.PushOutputData(work); err != nil {
				work.State = core.WORK_STAT_FAIL
				logger.Error("err@pushOutputData: workid=" + work.Id + ", err=" + err.Error())
			} else {
				work.State = core.WORK_STAT_DONE
			}
		}
		move_end := time.Now().Unix()
		perfstat.DataOut = move_end - move_start
		perfstat.Deliver = move_end
		perfstat.ClientResp = perfstat.Deliver - perfstat.Checkout
		perfstat.ClientId = core.Self.Id

		//notify server the final process results
		if err := core.NotifyWorkunitProcessed(work, perfstat); err != nil {
			time.Sleep(3 * time.Second) //wait 3 seconds and try another time
			if err := core.NotifyWorkunitProcessed(work, perfstat); err != nil {
				fmt.Printf("!!!NotifyWorkunitDone returned error: %s\n", err.Error())
				logger.Error("err@NotifyWorkunitProcessed: workid=" + work.Id + ", err=" + err.Error())
				//mark this work in Current_work map as false, something needs to be done in the future
				//to clean this kind of work that has been proccessed but its result can't be sent to server!
				core.Self.Current_work[work.Id] = false //server doesn't know this yet
			}
		}
		//now final status report sent to server, update some local info
		if work.State == core.WORK_STAT_DONE {
			logger.Event(event.WORK_DONE, "workid="+work.Id)
			core.Self.Total_completed += 1

			if conf.AUTO_CLEAN_DIR {
				if err := work.RemoveDir(); err != nil {
					logger.Error("[email protected](): workid=" + work.Id + ", err=" + err.Error())
				}
			}
		} else {
			logger.Event(event.WORK_RETURN, "workid="+work.Id)
			core.Self.Total_failed += 1
		}
		delete(core.Self.Current_work, work.Id)
		delete(workmap, work.Id)

	}
	control <- ID_DELIVERER //we are ending
}
Example #7
0
// POST: /user
// To create a new user make a empty POST to /user with user:password
// Basic Auth encoded in the header. Return new user object.
func (cr *UserController) Create(cx *goweb.Context) {
	// Log Request
	LogRequest(cx.Request)

	if _, ok := cx.Request.Header["Authorization"]; !ok {
		cx.RespondWithError(http.StatusUnauthorized)
		return
	}
	header := cx.Request.Header.Get("Authorization")
	tmpAuthArray := strings.Split(header, " ")

	authValues, err := base64.URLEncoding.DecodeString(tmpAuthArray[1])
	if err != nil {
		err = errors.New("Failed to decode encoded auth settings in http request.")
		cx.RespondWithError(http.StatusBadRequest)
		return
	}

	authValuesArray := strings.Split(string(authValues), ":")
	if conf.ANON_CREATEUSER == false && len(authValuesArray) != 4 {
		if len(authValuesArray) == 2 {
			cx.RespondWithErrorMessage(e.UnAuth, http.StatusUnauthorized)
			return
		} else {
			cx.RespondWithError(http.StatusBadRequest)
			return
		}
	}
	name := authValuesArray[0]
	passwd := authValuesArray[1]
	admin := false
	if len(authValuesArray) == 4 {
		if authValuesArray[2] != fmt.Sprint(conf.SECRET_KEY) {
			cx.RespondWithErrorMessage(e.UnAuth, http.StatusUnauthorized)
			return
		} else if authValuesArray[3] == "true" {
			admin = true
		}
	}

	u, err := user.New(name, passwd, admin)
	if err != nil {
		// Duplicate key check
		if e.MongoDupKeyRegex.MatchString(err.Error()) {
			logger.Error("Err@user_Create: duplicate key error")
			cx.RespondWithErrorMessage("Username not available", http.StatusBadRequest)
			return
		} else {
			logger.Error("Err@user_Create: " + err.Error())
			cx.RespondWithError(http.StatusInternalServerError)
			return
		}
	}
	cx.RespondWithData(u)
	return
}
Example #8
0
func getMetaDataField(field string) (result string, err error) {
	var url = fmt.Sprintf("%s/%s", conf.OPENSTACK_METADATA_URL, field) // TODO this is not OPENSTACK, this is EC2
	logger.Debug(1, fmt.Sprintf("url=%s", url))

	for i := 0; i < 3; i++ {
		var res *http.Response
		c := make(chan error)
		go func() {
			res, err = http.Get(url)
			if err != nil {
				c <- err //we are ending with error
				return
			}

			defer res.Body.Close()
			bodybytes, err := ioutil.ReadAll(res.Body)
			if err != nil {
				c <- err //we are ending with error
				return
			}
			result = string(bodybytes[:])

			c <- nil //we are ending without error
		}()
		select {
		case err = <-c:
			//go ahead
		case <-time.After(conf.INSTANCE_METADATA_TIMEOUT): //GET timeout
			err = errors.New("timeout: " + url)
		}

		if err != nil {
			logger.Error(fmt.Sprintf("warning: (iteration=%d) %s \"%s\"", i, url, err.Error()))
			continue
		} else if result == "" {
			logger.Error(fmt.Sprintf("warning: (iteration=%d) %s empty result", i, url))
			continue
		}

		break

	}

	if err != nil {
		return "", err
	}

	if result == "" {
		return "", errors.New(fmt.Sprintf("metadata result empty, %s", url))
	}

	logger.Debug(1, fmt.Sprintf("Intance Metadata %s => \"%s\"", url, result))
	return
}
Example #9
0
File: app.go Project: MG-RAST/AWE
func (apr AppRegistry) GetAppPackage(app_package string) (ap *AppPackage, err error) {

	ap, ok := apr[app_package]
	if ok {
		return ap, nil
	}

	package_url := conf.APP_REGISTRY_URL + "/" + app_package + ".json"

	var new_app_package AppPackage

	for i := 0; i < 3; i++ {

		if i > 0 {
			time.Sleep(1000 * time.Millisecond)
		}
		logger.Debug(1, fmt.Sprintf("downloading app package \"%s\"", package_url))

		res, err := httpclient.GetTimeout(package_url, nil, nil, nil, 5000*time.Millisecond)

		if err != nil {
			logger.Error("warning: " + conf.APP_REGISTRY_URL + " " + err.Error())
			continue
		}

		app_package_json, err := ioutil.ReadAll(res.Body)
		if err != nil {
			logger.Error(fmt.Sprintf("warning, could not read app registry json: %s", err.Error()))
			continue
		}

		// transform json into go struct interface
		//var f map[string]interface{}
		err = json.Unmarshal(app_package_json, &new_app_package)

		if err != nil {
			logger.Error("error unmarshaling app package " + app_package + ", error=" + err.Error())
			continue
		}

		apr[app_package] = &new_app_package
		ap = &new_app_package

		logger.Debug(1, fmt.Sprintf("app package unmarshalled"))
		return ap, nil
	}

	ap = nil
	err = errors.New("could not get app package from " + package_url)
	return

}
Example #10
0
func (qm *CQMgr) filterWorkByClient(clientid string) (ids []string) {
	client := qm.clientMap[clientid]
	for id, _ := range qm.workQueue.wait {
		if _, ok := qm.workQueue.workMap[id]; !ok {
			logger.Error(fmt.Sprintf("error: workunit %s is in wait queue but not in workMap", id))
			continue
		}
		work := qm.workQueue.workMap[id]

		// In case of edge case where pointer to workunit is in queue but workunit has been deleted
		// If work.Info is nil, this will cause errors in execution
		// These will be deleted by servermgr.updateQueue()
		if work == nil || work.Info == nil {
			continue
		}

		if client == nil {
			fmt.Fprintf(os.Stderr, "error: Skip_work for client %s is nil", clientid)
			logger.Error(fmt.Sprintf("error: client %s is nil", clientid))
			continue
		}

		if client.Skip_work == nil {
			fmt.Fprintf(os.Stderr, "error: Skip_work for client %s is nil", clientid)
			logger.Error(fmt.Sprintf("error: Skip_work for client %s is nil", clientid))
			continue
		}

		//skip works that are in the client's skip-list
		if contains(client.Skip_work, work.Id) {
			logger.Debug(2, fmt.Sprintf("2) contains(client.Skip_work, work.Id) %s", id))
			continue
		}
		//skip works that have dedicate client groups which this client doesn't belong to
		if len(work.Info.ClientGroups) > 0 {
			eligible_groups := strings.Split(work.Info.ClientGroups, ",")
			if !contains(eligible_groups, client.Group) {
				logger.Debug(2, fmt.Sprintf("3) !contains(eligible_groups, client.Group) %s", id))
				continue
			}
		}
		//append works whos apps are supported by the client
		if contains(client.Apps, work.Cmd.Name) || contains(client.Apps, conf.ALL_APP) {
			ids = append(ids, id)
		} else {
			logger.Debug(2, fmt.Sprintf("3) contains(client.Apps, work.Cmd.Name) || contains(client.Apps, conf.ALL_APP) %s", id))
		}
	}
	return ids
}
Example #11
0
func (qm *ServerMgr) Handle() {
	for {
		select {
		case <-qm.jsReq:
			jid := qm.getNextJid()
			qm.jsAck <- jid
			logger.Debug(2, fmt.Sprintf("qmgr:receive a job submission request, assigned jid=%s\n", jid))

		case task := <-qm.taskIn:
			logger.Debug(2, fmt.Sprintf("qmgr:task recived from chan taskIn, id=%s\n", task.Id))
			qm.addTask(task)

		case coReq := <-qm.coReq:
			logger.Debug(2, fmt.Sprintf("qmgr: workunit checkout request received, Req=%v\n", coReq))
			works, err := qm.popWorks(coReq)
			ack := CoAck{workunits: works, err: err}
			qm.coAck <- ack

		case notice := <-qm.feedback:
			logger.Debug(2, fmt.Sprintf("qmgr: workunit feedback received, workid=%s, status=%s, clientid=%s\n", notice.WorkId, notice.Status, notice.ClientId))
			if err := qm.handleWorkStatusChange(notice); err != nil {
				logger.Error("handleWorkStatusChange(): " + err.Error())
			}

		case <-qm.reminder:
			logger.Debug(3, "time to update workunit queue....\n")
			qm.updateQueue()
			if conf.DEV_MODE {
				fmt.Println(qm.ShowStatus())
			}
		}
	}
}
Example #12
0
func (qm *ServerMgr) ClientHandle() {
	for {
		select {
		case coReq := <-qm.coReq:
			logger.Debug(2, fmt.Sprintf("qmgr: workunit checkout request received, Req=%v", coReq))
			var ack CoAck
			if qm.suspendQueue {
				// queue is suspended, return suspend error
				ack = CoAck{workunits: nil, err: errors.New(e.QueueSuspend)}
			} else {
				qm.updateQueue()
				works, err := qm.popWorks(coReq)
				if err == nil {
					qm.UpdateJobTaskToInProgress(works)
				}
				ack = CoAck{workunits: works, err: err}
			}
			qm.coAck <- ack
		case notice := <-qm.feedback:
			logger.Debug(2, fmt.Sprintf("qmgr: workunit feedback received, workid=%s, status=%s, clientid=%s", notice.WorkId, notice.Status, notice.ClientId))
			if err := qm.handleWorkStatusChange(notice); err != nil {
				logger.Error("handleWorkStatusChange(): " + err.Error())
			}
			qm.updateQueue()
		}
	}
}
Example #13
0
func resetTask(task *Task, info *Info) {
	task.Info = info
	task.State = TASK_STAT_PENDING
	task.RemainWork = task.TotalWork
	task.ComputeTime = 0
	task.CompletedDate = time.Time{}
	// reset all inputs with an origin
	for _, input := range task.Inputs {
		if input.Origin != "" {
			input.Node = "-"
			input.Url = ""
			input.Size = 0
		}
	}
	// reset / delete all outputs
	for _, output := range task.Outputs {
		if dataUrl, _ := output.DataUrl(); dataUrl != "" {
			// delete dataUrl if is shock node
			if strings.HasSuffix(dataUrl, shock.DATA_SUFFIX) {
				if err := shock.ShockDelete(output.Host, output.Node, output.DataToken); err == nil {
					logger.Debug(2, fmt.Sprintf("Deleted node %s from shock", output.Node))
				} else {
					logger.Error(fmt.Sprintf("resetTask: unable to deleted node %s from shock: %s", output.Node, err.Error()))
				}
			}
		}
		output.Node = "-"
		output.Url = ""
		output.Size = 0
	}
	// delete all workunit logs
	for _, log := range conf.WORKUNIT_LOGS {
		deleteStdLogByTask(task.Id, log)
	}
}
Example #14
0
// GET: /work/{id}
// get a workunit by id, read-only
func (cr *WorkController) Read(id string, cx *goweb.Context) {
	LogRequest(cx.Request)

	// Gather query params
	query := &Query{Li: cx.Request.URL.Query()}

	if query.Has("datatoken") && query.Has("client") { //a client is requesting data token for this job
		//***insert code to authenticate and check ACL***
		clientid := query.Value("client")
		token, err := core.QMgr.FetchDataToken(id, clientid)
		if err != nil {
			cx.RespondWithErrorMessage("error in getting token for job "+id, http.StatusBadRequest)
		}
		cx.RespondWithData(token)
		return
	}

	// Load workunit by id
	workunit, err := core.QMgr.GetWorkById(id)

	if err != nil {
		if err.Error() != e.QueueEmpty {
			logger.Error("Err@work_Read:core.QMgr.GetWorkById(): " + err.Error())
		}
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}
	// Base case respond with workunit in json
	cx.RespondWithData(workunit)
	return
}
Example #15
0
// GET: /client/{id}
func (cr *ClientController) Read(id string, cx *goweb.Context) {
	// Gather query params
	query := &Query{Li: cx.Request.URL.Query()}

	if query.Has("heartbeat") { //handle heartbeat
		hbmsg, err := core.QMgr.ClientHeartBeat(id)
		if err != nil {
			cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		} else {
			cx.RespondWithData(hbmsg)
		}
		return
	}

	LogRequest(cx.Request) //skip heartbeat in access log

	client, err := core.QMgr.GetClient(id)
	if err != nil {
		if err.Error() == e.ClientNotFound {
			cx.RespondWithErrorMessage(e.ClientNotFound, http.StatusBadRequest)
		} else {
			logger.Error("Error in GET client:" + err.Error())
			cx.RespondWithError(http.StatusBadRequest)
		}
		return
	}
	cx.RespondWithData(client)
}
Example #16
0
func NewProfileClient(filepath string) (client *Client, err error) {
	client = new(Client)
	jsonstream, err := ioutil.ReadFile(filepath)
	if err != nil {
		return nil, err
	}
	if err := json.Unmarshal(jsonstream, client); err != nil {
		logger.Error("failed to unmashal json stream for client profile: " + string(jsonstream[:]))
		return nil, err
	}
	if client.Id == "" {
		client.Id = uuid.New()
	}
	if client.RegTime.IsZero() {
		client.RegTime = time.Now()
	}
	if client.Apps == nil {
		client.Apps = []string{}
	}
	client.Skip_work = []string{}
	client.Status = CLIENT_STAT_ACTIVE_IDLE
	if client.Current_work == nil {
		client.Current_work = map[string]bool{}
	}
	client.Tag = true
	return
}
Example #17
0
// GET: /user
func (cr *UserController) ReadMany(cx *goweb.Context) {
	// Log Request and check for Auth
	LogRequest(cx.Request)
	u, err := request.Authenticate(cx.Request)

	if err != nil {
		if err.Error() == e.NoAuth || err.Error() == e.UnAuth {
			cx.RespondWithError(http.StatusUnauthorized)
			return
		} else {
			logger.Error("Err@user_Read: " + err.Error())
			cx.RespondWithError(http.StatusInternalServerError)
			return
		}
	}
	if u.Admin {
		users := user.Users{}
		user.AdminGet(&users)
		if len(users) > 0 {
			cx.RespondWithData(users)
			return
		} else {
			cx.RespondWithNotFound()
			return
		}
	} else {
		cx.RespondWithError(http.StatusUnauthorized)
		return
	}
}
Example #18
0
// fetchProfile validiates token by using it to fetch user profile
func fetchProfile(t string) (u *user.User, err error) {
	client := &http.Client{
		Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}},
	}
	req, err := http.NewRequest("GET", conf.GLOBUS_PROFILE_URL+"/"+clientId(t), nil)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Authorization", "Globus-Goauthtoken "+t)
	if resp, err := client.Do(req); err == nil {
		defer resp.Body.Close()
		if resp.StatusCode == http.StatusOK {
			if body, err := ioutil.ReadAll(resp.Body); err == nil {
				u = &user.User{}
				if err = json.Unmarshal(body, &u); err != nil {
					return nil, err
				} else {
					if err = u.SetMongoInfo(); err != nil {
						return nil, err
					}
				}
			}
		} else if resp.StatusCode == http.StatusForbidden {
			return nil, errors.New(e.InvalidAuth)
		} else {
			err_str := "Authentication failed: Unexpected response status: " + resp.Status
			logger.Error(err_str)
			return nil, errors.New(err_str)
		}
	} else {
		return nil, err
	}
	return
}
Example #19
0
//check whether a pending task is ready to enqueue (dependent tasks are all done)
func (qm *ServerMgr) isTaskReady(task *Task) (ready bool) {
	ready = false

	//skip if the belonging job is suspended
	jobid, _ := GetJobIdByTaskId(task.Id)
	if qm.isSusJob(jobid) {
		return false
	}

	if task.State == TASK_STAT_PENDING {
		ready = true
		for _, predecessor := range task.DependsOn {
			if pretask, ok := qm.getTask(predecessor); ok {
				if pretask.State != TASK_STAT_COMPLETED &&
					pretask.State != TASK_STAT_PASSED &&
					pretask.State != TASK_STAT_SKIPPED &&
					pretask.State != TASK_STAT_FAIL_SKIP {
					ready = false
				}
			} else {
				logger.Error("warning: predecessor " + predecessor + " is unknown")
			}
		}
	}
	if task.Skip == 1 && task.Skippable() {
		qm.skipTask(task)
		ready = false
	}
	return
}
Example #20
0
func AuthError(err error, cx *goweb.Context) {
	if err.Error() == e.InvalidAuth {
		cx.RespondWithErrorMessage("Invalid authorization header or content", http.StatusBadRequest)
		return
	}
	logger.Error("Error at Auth: " + err.Error())
	cx.RespondWithError(http.StatusInternalServerError)
	return
}
Example #21
0
func (qm *ServerMgr) taskEnQueue(task *Task) (err error) {

	logger.Debug(2, "trying to enqueue task "+task.Id)

	if err := qm.locateInputs(task); err != nil {
		logger.Error("qmgr.taskEnQueue locateInputs:" + err.Error())
		return err
	}

	//create shock index on input nodes (if set in workflow document)
	if err := task.CreateIndex(); err != nil {
		logger.Error("qmgr.taskEnQueue CreateIndex:" + err.Error())
		return err
	}

	//init partition
	if err := task.InitPartIndex(); err != nil {
		logger.Error("qmgr.taskEnQueue InitPartitionIndex:" + err.Error())
		return err
	}

	if err := qm.createOutputNode(task); err != nil {
		logger.Error("qmgr.taskEnQueue createOutputNode:" + err.Error())
		return err
	}
	if err := qm.parseTask(task); err != nil {
		logger.Error("qmgr.taskEnQueue parseTask:" + err.Error())
		return err
	}
	task.State = TASK_STAT_QUEUED
	task.CreatedDate = time.Now()
	task.StartedDate = time.Now() //to-do: will be changed to the time when the first workunit is checked out
	qm.updateJobTask(task)        //task status PENDING->QUEUED

	//log event about task enqueue (TQ)
	logger.Event(event.TASK_ENQUEUE, fmt.Sprintf("taskid=%s;totalwork=%d", task.Id, task.TotalWork))
	qm.CreateTaskPerf(task.Id)

	if IsFirstTask(task.Id) {
		jobid, _ := GetJobIdByTaskId(task.Id)
		UpdateJobState(jobid, JOB_STAT_QUEUED, []string{JOB_STAT_INIT, JOB_STAT_SUSPEND})
	}
	return
}
Example #22
0
func (qm *ServerMgr) InitMaxJid() (err error) {
	jidfile := conf.DATA_PATH + "/maxjid"

	if _, err := os.Stat(jidfile); err != nil {

		f, err := os.Create(jidfile)
		if err != nil {
			fmt.Fprintf(os.Stderr, fmt.Sprintf("error creating jidfile ", err.Error())) // logger does not work
			logger.Error(fmt.Sprintf("error creating jidfile ", err.Error()))
			return err
		}
		f.WriteString("10000")
		qm.nextJid = "10001"
		f.Close()
	} else {

		buf, err := ioutil.ReadFile(jidfile)
		if err != nil {
			if conf.DEBUG_LEVEL > 0 {
				fmt.Println("error ioutil.ReadFile(jidfile)")
			}
			return err
		}
		bufstr := strings.TrimSpace(string(buf))

		maxjid, err := strconv.Atoi(bufstr)
		if err != nil {
			if conf.DEBUG_LEVEL > 0 {
				fmt.Println(fmt.Sprintf("error strconv.Atoi(bufstr), bufstr=\"%s\"", bufstr))
			}
			fmt.Fprintf(os.Stderr, fmt.Sprintf("Could not convert \"%s\" into int", bufstr)) // logger does not work
			logger.Error(fmt.Sprintf("Could not convert \"%s\" into int", bufstr))
			return err
		}

		qm.nextJid = strconv.Itoa(maxjid + 1)

	}
	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("in InitMaxJid C")
	}
	logger.Debug(2, fmt.Sprintf("qmgr:jid initialized, next jid=%s\n", qm.nextJid))
	return
}
Example #23
0
func launchSite(control chan int, port int) {
	goweb.ConfigureDefaultFormatters()
	r := &goweb.RouteManager{}
	r.MapFunc("*", controller.SiteDir)
	if conf.SSL_ENABLED {
		err := goweb.ListenAndServeRoutesTLS(fmt.Sprintf(":%d", conf.SITE_PORT), conf.SSL_CERT_FILE, conf.SSL_KEY_FILE, r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: site: %v\n", err)
			logger.Error("ERROR: site: " + err.Error())
		}
	} else {
		err := goweb.ListenAndServeRoutes(fmt.Sprintf(":%d", conf.SITE_PORT), r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: site: %v\n", err)
			logger.Error("ERROR: site: " + err.Error())
		}
	}
	control <- 1 //we are ending
}
Example #24
0
// POST: /client - register a new client
func (cr *ClientController) Create(cx *goweb.Context) {
	// Log Request and check for Auth
	LogRequest(cx.Request)

	cg, err := request.AuthenticateClientGroup(cx.Request)
	if err != nil {
		if err.Error() == e.NoAuth || err.Error() == e.UnAuth || err.Error() == e.InvalidAuth {
			if conf.CLIENT_AUTH_REQ == true {
				cx.RespondWithError(http.StatusUnauthorized)
				return
			}
		} else {
			logger.Error("Err@AuthenticateClientGroup: " + err.Error())
			cx.RespondWithError(http.StatusInternalServerError)
			return
		}
	}

	// Parse uploaded form
	_, files, err := ParseMultipartForm(cx.Request)
	if err != nil {
		if err.Error() != "request Content-Type isn't multipart/form-data" {
			logger.Error("Error parsing form: " + err.Error())
			cx.RespondWithError(http.StatusBadRequest)
			return
		}
	}

	client, err := core.QMgr.RegisterNewClient(files, cg)
	if err != nil {
		msg := "Error in registering new client:" + err.Error()
		logger.Error(msg)
		cx.RespondWithErrorMessage(msg, http.StatusBadRequest)
		return
	}

	//log event about client registration (CR)
	logger.Event(event.CLIENT_REGISTRATION, "clientid="+client.Id+";name="+client.Name+";host="+client.Host+";group="+client.Group+";instance_id="+client.InstanceId+";instance_type="+client.InstanceType+";domain="+client.Domain)

	cx.RespondWithData(client)
	return
}
Example #25
0
func StartProxyWorkers() {
	control := make(chan int)
	go heartBeater(control)
	go workStealer(control)
	go redistributor(control)
	for {
		who := <-control //block till someone dies and then restart it
		switch who {
		case ID_HEARTBEATER:
			go heartBeater(control)
			logger.Error("heartBeater died and restarted")
		case ID_WORKSTEALER:
			go workStealer(control)
			logger.Error("workStealer died and restarted")
		case ID_REDISTRIBUTOR:
			go redistributor(control)
			logger.Error("deliverer died and restarted")
		}
	}
}
Example #26
0
func launchSite(control chan int, port int) {
	goweb.ConfigureDefaultFormatters()
	r := &goweb.RouteManager{}

	site_directory := conf.SITE_PATH
	fileinfo, err := os.Stat(site_directory)
	if err != nil {
		message := fmt.Sprintf("ERROR: site, path %s does not exist: %s", site_directory, err.Error())
		if os.IsNotExist(err) {
			message += " IsNotExist"
		}

		fmt.Fprintf(os.Stderr, message, "\n")
		logger.Error(message)

		os.Exit(1)
	} else {
		if !fileinfo.IsDir() {
			message := fmt.Sprintf("ERROR: site, path %s exists, but is not a directory", site_directory)
			fmt.Fprintf(os.Stderr, message, "\n")
			logger.Error(message)
			os.Exit(1)
		}

	}
	r.MapFunc("*", controller.SiteDir)
	if conf.SSL_ENABLED {
		err := goweb.ListenAndServeRoutesTLS(fmt.Sprintf(":%d", conf.SITE_PORT), conf.SSL_CERT_FILE, conf.SSL_KEY_FILE, r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: site: %v\n", err)
			logger.Error("ERROR: site: " + err.Error())
		}
	} else {
		err := goweb.ListenAndServeRoutes(fmt.Sprintf(":%d", conf.SITE_PORT), r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: site: %v\n", err)
			logger.Error("ERROR: site: " + err.Error())
		}
	}
	control <- 1 //we are ending
}
Example #27
0
File: cqmgr.go Project: MG-RAST/AWE
func (qm *CQMgr) filterWorkByClient(clientid string) (ids []string, err error) {
	logger.Debug(3, fmt.Sprintf("starting filterWorkByClient() for client: %s", clientid))

	client, ok := qm.GetClient(clientid)
	if !ok {
		err_msg := fmt.Sprintf("error: unregistered client %s trying to checkout workunit, most likely cause is client disappeared after request to checkout workunit combined with slow response to workunit checkout request", clientid)
		fmt.Fprintln(os.Stderr, err_msg)
		logger.Error(err_msg)
		return nil, errors.New(e.ClientNotFound)
	}

	for _, id := range qm.workQueue.WaitList() {
		work, ok := qm.workQueue.Get(id)
		if !ok {
			logger.Error(fmt.Sprintf("error: workunit %s is in wait queue but not in workMap", id))
			continue
		}
		//skip works that are in the client's skip-list
		if client.Contains_Skip_work(work.Id) {
			logger.Debug(2, fmt.Sprintf("2) client.Contains_Skip_work(work.Id) %s", id))
			continue
		}
		//skip works that have dedicate client groups which this client doesn't belong to
		if len(work.Info.ClientGroups) > 0 {
			eligible_groups := strings.Split(work.Info.ClientGroups, ",")
			if !contains(eligible_groups, client.Group) {
				logger.Debug(2, fmt.Sprintf("3) !contains(eligible_groups, client.Group) %s", id))
				continue
			}
		}
		//append works whos apps are supported by the client
		if contains(client.Apps, work.Cmd.Name) || contains(client.Apps, conf.ALL_APP) {
			ids = append(ids, id)
		} else {
			logger.Debug(2, fmt.Sprintf("3) contains(client.Apps, work.Cmd.Name) || contains(client.Apps, conf.ALL_APP) %s", id))
		}
	}
	logger.Debug(3, fmt.Sprintf("done with filterWorkByClient() for client: %s", clientid))

	return ids, nil
}
Example #28
0
File: task.go Project: MG-RAST/AWE
func (task *Task) CreateIndex() (err error) {
	for _, io := range task.Inputs {
		if len(io.ShockIndex) > 0 {
			idxinfo, err := io.GetIndexInfo()
			if err != nil {
				errMsg := "could not retrieve index info from input shock node, taskid=" + task.Id + ", error=" + err.Error()
				logger.Error("error: " + errMsg)
				return errors.New(errMsg)
			}

			if _, ok := idxinfo[io.ShockIndex]; !ok {
				if err := ShockPutIndex(io.Host, io.Node, io.ShockIndex, task.Info.DataToken); err != nil {
					errMsg := "failed to create index on shock node for taskid=" + task.Id + ", error=" + err.Error()
					logger.Error("error: " + errMsg)
					return errors.New(errMsg)
				}
			}
		}
	}
	return
}
Example #29
0
File: io.go Project: wtangiit/AWE
func (io *IO) GetFileSize() int64 {
	if io.Size > 0 {
		return io.Size
	}
	shocknode, err := io.GetShockNode()
	if err != nil {
		logger.Error(fmt.Sprintf("GetFileSize error: %s, node: %s", err.Error(), io.Node))
		return -1
	}
	io.Size = shocknode.File.Size
	return io.Size
}
Example #30
0
func ReRegisterWithSelf(host string) (client *core.Client, err error) {
	fmt.Printf("lost contact with server, try to re-register\n")
	client, err = RegisterWithAuth(host, core.Self)
	if err != nil {
		logger.Error("Error: fail to re-register, clientid=" + core.Self.Id)
		fmt.Printf("failed to re-register\n")
	} else {
		logger.Event(event.CLIENT_AUTO_REREGI, "clientid="+core.Self.Id)
		fmt.Printf("re-register successfully\n")
	}
	return
}