Beispiel #1
0
func handleRequest(c net.Conn, message interface{}) {
	o.Debug("Request Recieved.  Decoding!")
	ptr, ok := message.(*o.ProtoTaskRequest)
	if !ok {
		o.Assert("CC stuffed up - handleRequest got something that wasn't a ProtoTaskRequest.")
	}
	job := o.JobFromProto(ptr)
	/* search the registry for the job */
	o.Debug("Request for Job.ID %d", job.Id)
	existing := o.JobGet(job.Id)
	if nil != existing {
		if existing.MyResponse.IsFinished() {
			o.Debug("job%d: Resending Response", job.Id)
			sendResponse(c, existing.MyResponse)
		}
	} else {
		// check to see if we have the score
		// add the Job to our Registry
		job.MyResponse = o.NewTaskResponse()
		job.MyResponse.Id = job.Id
		job.MyResponse.State = o.RESP_PENDING
		o.JobAdd(job)
		o.Info("Added New Job %d to our local registry", job.Id)
		// and then push it onto the pending job list so we know it needs actioning.
		appendPendingJob(job)
	}
}
Beispiel #2
0
func handleRequest(c net.Conn, message interface{}) {
	o.Debug("Request Recieved.  Decoding!")
	ptr, ok := message.(*o.TaskRequest)
	if !ok {
		o.Assert("CC stuffed up - handleRequest got something that wasn't a TaskRequest.")
	}
	task := TaskFromProto(ptr)
	/* search the registry for the task */
	o.Debug("Request for Job.ID %d", task.Id)
	existing := TaskGet(task.Id)
	if nil != existing {
		if existing.MyResponse.IsFinished() {
			o.Debug("job%d: Resending Response", task.Id)
			sendResponse(c, existing.MyResponse)
		}
	} else {
		// check to see if we have the score
		// add the Job to our Registry
		task.MyResponse = NewTaskResponse()
		task.MyResponse.id = task.Id
		task.MyResponse.State = RESP_PENDING
		TaskAdd(task)
		o.Info("Added New Task (Job ID %d) to our local registry", task.Id)
		// and then push it onto the pending job list so we know it needs actioning.
		appendPendingTask(task)
	}
}
Beispiel #3
0
// Ugh.
func (job *JobRequest) updateState() {
	if job.Results == nil {
		o.Assert("job.Results nil for jobid %d", job.Id)
		return
	}
	was_finished := job.State.Finished()
	switch job.Scope {
	case SCOPE_ONEOF:
		// look for a success (any success) in the responses
		var success bool
		for host, res := range job.Results {
			if res == nil {
				o.Debug("nil result for %s?", host)
				continue
			}
			if res.State == RESP_FINISHED {
				success = true
				break
			}
		}
		// update the job state based upon these findings
		if success {
			job.State = JOB_SUCCESSFUL
		} else {
			if len(job.Players) < 1 {
				job.State = JOB_FAILED
			} else {
				job.State = JOB_PENDING
			}
		}
	case SCOPE_ALLOF:
		var success, failed int

		for pidx := range job.Players {
			p := job.Players[pidx]
			resp, exists := job.Results[p]
			if exists {
				if resp.DidFail() {
					failed++
				} else if resp.State == RESP_FINISHED {
					success++
				}
			}
		}
		if (success + failed) < len(job.Players) {
			job.State = JOB_PENDING
		} else if success == len(job.Players) {
			job.State = JOB_SUCCESSFUL
		} else if failed == len(job.Players) {
			job.State = JOB_FAILED
		} else {
			job.State = JOB_FAILED_PARTIAL
		}
	}
	if !was_finished && job.State.Finished() {
		o.Debug("job%d: finished; setting expiry time", job.Id)
		regInternalMarkJobForExpiry(job)
	}
}
Beispiel #4
0
func handleAck(c net.Conn, message interface{}) {
	o.Debug("Ack Received")
	ack, ok := message.(*o.Acknowledgement)
	if !ok {
		o.Assert("CC stuffed up - handleAck got something that wasn't a ProtoAcknowledgement.")
	}
	acknowledgeResponse(ack.Id)
}
Beispiel #5
0
func sendResponse(c net.Conn, resp *TaskResponse) {
	//FIXME: update retry time on Response
	o.Debug("Sending Response!")
	ptr := resp.Encode()
	p, err := o.Encode(ptr)
	o.MightFail(err, "Failed to encode response")
	_, err = p.Send(c)
	if err != nil {
		o.Warn("Transmission error: %s", err)
		c.Close()
		prequeueResponse(resp)
		lostConnection <- 1
	} else {
		appendUnacknowledgedResponse(resp)
	}
}
Beispiel #6
0
func manageRegistry() {
	for {
		select {
		case req := <-chanRegistryRequest:
			resp := new(registryResponse)
			// by default, we failed.
			resp.success = false
			// find the operation
			handler, exists := registryHandlers[req.operation]
			if exists {
				handler(req, resp)
			}
			if req.responseChannel != nil {
				req.responseChannel <- resp
			}
		case <-expiryChan:
			o.Debug("job%d: expiring job record", expiryJobid)
			regInternalExpireJob(expiryJobid)
			expiryChan = nil
			regInternalFindNextExpiry()
		}
	}
}
Beispiel #7
0
func handleAudienceRequest(c net.Conn) {
	defer c.Close()

	r, _ := c.(io.Reader)
	w, _ := c.(io.Writer)
	dec := json.NewDecoder(r)
	enc := json.NewEncoder(w)

	outobj := new(GenericJsonRequest)
	err := dec.Decode(outobj)
	if err != nil {
		o.Warn("Error decoding JSON talking to audience: %s", err)
		return
	}

	if nil == outobj.Op {
		o.Warn("Malformed JSON message talking to audience.  Missing Op")
		return
	}
	switch *(outobj.Op) {
	case "status":
		if nil == outobj.Id {
			o.Warn("Malformed Status message talking to audience. Missing Job ID")
			return
		}
		job := JobGet(*outobj.Id)
		jresp := new([2]interface{})
		if nil != job {
			jresp[0] = "OK"
			iresp := NewJsonStatusResponse()
			iresp.Status = job.State
			resnames := JobGetResultNames(*outobj.Id)
			for i := range resnames {
				tr := JobGetResult(*outobj.Id, resnames[i])
				if nil != tr {
					presp := NewJsonPlayerStatus()
					presp.Status = tr.State
					for k, v := range tr.Response {
						presp.Response[k] = v
					}
					iresp.Players[resnames[i]] = presp
				}

			}
			jresp[1] = iresp
		} else {
			jresp[0] = "Error"
			jresp[1] = nil
		}
		enc.Encode(jresp)
		o.Debug("Status...")
	case "queue":
		if nil == outobj.Score {
			o.Warn("Malformed Queue message talking to audience. Missing Score")
			sendQueueFailureResponse("Missing Score", enc)
			return
		}
		if nil == outobj.Scope {
			o.Warn("Malformed Queue message talking to audience. Missing Scope")
			sendQueueFailureResponse("Missing Scope", enc)
			return
		}
		if nil == outobj.Players || len(outobj.Players) < 1 {
			o.Warn("Malformed Queue message talking to audience. Missing Players")
			sendQueueFailureResponse("Missing Players", enc)
			return
		}
		for _, player := range outobj.Players {
			if !HostAuthorised(player) {
				o.Warn("Malformed Queue message - unknown player %s specified.", player)
				sendQueueFailureResponse("Invalid Player", enc)
				return
			}
		}
		job := NewRequest()
		job.Score = *outobj.Score
		job.Scope = *outobj.Scope
		job.Players = outobj.Players
		job.Params = outobj.Params

		QueueJob(job)
		sendQueueSuccessResponse(job, enc)
	default:
		o.Warn("Unknown operation talking to audience: \"%s\"", *(outobj.Op))
		return
	}

	_ = enc
}
Beispiel #8
0
func ProcessingLoop() {
	var conn net.Conn = nil
	var nextRetryResp *TaskResponse = nil
	var taskCompletionChan <-chan *TaskResponse = nil
	var connectDelay time.Duration
	var doScoreReload bool = false
	// kick off a new connection attempt.
	go connectMe(connectDelay)

	// and this is where we spin!
	for {
		var retryDelay time.Duration = 0
		var retryChan <-chan time.Time = nil

		if conn != nil {
			for nextRetryResp == nil {
				nextRetryResp = getNextUnacknowledgedResponse()
				if nil == nextRetryResp {
					break
				}
				retryDelay = nextRetryResp.RetryTime.Sub(time.Now())
				if retryDelay < 0 {
					sendResponse(conn, nextRetryResp)
					nextRetryResp = nil
				}
			}
			if nextRetryResp != nil {
				retryChan = time.After(retryDelay)
			}
		}
		if taskCompletionChan == nil {
			nextTask := getNextPendingTask()
			if nextTask != nil {
				taskCompletionChan = ExecuteTask(nextTask)
			} else {
				if conn != nil && !pendingTaskRequest {
					o.Debug("Asking for trouble")
					p := o.MakeReadyForTask()
					p.Send(conn)
					o.Debug("Sent Request for trouble")
					pendingTaskRequest = true
				}
			}
		}
		select {
		// Currently executing job finishes.
		case newresp := <-taskCompletionChan:
			o.Debug("job%d: Completed with State %s\n", newresp.id, newresp.State)
			// preemptively set a retrytime.
			newresp.RetryTime = time.Now()
			// ENOCONN - sub it in as our next retryresponse, and prepend the old one onto the queue.
			if nil == conn {
				if nil != nextRetryResp {
					prequeueResponse(nextRetryResp)
				}
				o.Debug("job%d: Queuing Initial Response", newresp.id)
				nextRetryResp = newresp
			} else {
				o.Debug("job%d: Sending Initial Response", newresp.id)
				sendResponse(conn, newresp)
			}
			if doScoreReload {
				o.Info("Performing Deferred score reload")
				LoadScores()
				doScoreReload = false
			}
			taskCompletionChan = nil
		// If the current unacknowledged response needs a retry, send it.
		case <-retryChan:
			sendResponse(conn, nextRetryResp)
			nextRetryResp = nil
		// New connection.  Set up the receiver thread and Introduce ourselves.
		case nci := <-newConnection:
			if conn != nil {
				conn.Close()
			}
			conn = nci.conn
			connectDelay = nci.timeout
			pendingTaskRequest = false

			// start the reader
			go Reader(conn)

			/* Introduce ourself */
			p := o.MakeIdentifyClient(LocalHostname, PlayerVersion)
			p.Send(conn)
		// Lost connection.  Shut downt he connection.
		case <-lostConnection:
			o.Warn("Lost Connection to Master")
			conn.Close()
			conn = nil
			// restart the connection attempts
			go connectMe(connectDelay)
		// Message received from master.  Decode and action.
		case p := <-receivedMessage:
			// because the message could possibly be an ACK, push the next retry response back into the queue so acknowledge can find it.
			if nil != nextRetryResp {
				prequeueResponse(nextRetryResp)
				nextRetryResp = nil
			}
			var upkt interface{} = nil
			if p.Length > 0 {
				var err error
				upkt, err = p.Decode()
				o.MightFail(err, "Couldn't decode packet from master")
			}
			handler, exists := dispatcher[p.Type]
			if exists {
				connectDelay = 0
				handler(conn, upkt)
			} else {
				o.Fail("Unhandled Pkt Type %d", p.Type)
			}
		// Reload scores
		case <-reloadScores:
			// fortunately this is actually completely safe as
			// long as nobody's currently executing.
			// who'd have thunk it?
			if taskCompletionChan == nil {
				o.Info("Reloading scores")
				LoadScores()
			} else {
				o.Info("Deferring score reload (execution in progress)")
				doScoreReload = true
			}
		// Keepalive delay expired.  Send Nop.
		case <-time.After(KeepaliveDelay):
			if conn == nil {
				break
			}
			o.Debug("Sending NOP")
			p := o.MakeNop()
			p.Send(conn)
		}
	}
}
Beispiel #9
0
func handleNop(c net.Conn, message interface{}) {
	o.Debug("NOP Received")
}
Beispiel #10
0
func masterDispatch() {
	pq := list.New()
	tq := list.New()

	for {
		select {
		case player := <-playerIdle:
			o.Debug("Dispatch: Player")
			/* first, scan to see if we have anything for this player */
			i := tq.Front()
			for {
				if nil == i {
					/* Out of items! */
					/* Append this player to the waiting players queue */
					pq.PushBack(player)
					break
				}
				t, _ := i.Value.(*TaskRequest)
				if t.IsTarget(player.Player) {
					/* Found a valid job. Send it to the player, and remove it from our pending
					 * list */
					tq.Remove(i)
					player.TaskQ <- t
					break
				}
				i = i.Next()
			}
		case player := <-playerDead:
			o.Debug("Dispatch: Dead Player")
			for i := pq.Front(); i != nil; i = i.Next() {
				p, _ := i.Value.(*ClientInfo)
				if player.Player == p.Player {
					pq.Remove(i)
					break
				}
			}
		case task := <-rqTask:
			o.Debug("Dispatch: Task")
			/* first, scan to see if we have valid pending player for this task */
			i := pq.Front()
			for {
				if nil == i {
					/* Out of players! */
					/* Append this task to the waiting tasks queue */
					tq.PushBack(task)
					break
				}
				p, _ := i.Value.(*ClientInfo)
				if task.IsTarget(p.Player) {
					/* Found it. */
					pq.Remove(i)
					p.TaskQ <- task
					break
				}
				i = i.Next()
			}
		case respChan := <-statusRequest:
			o.Debug("Status!")
			response := new(QueueInformation)
			response.waitingTasks = tq.Len()
			pqLen := pq.Len()
			response.idlePlayers = make([]string, pqLen)

			idx := 0
			for i := pq.Front(); i != nil; i = i.Next() {
				player, _ := i.Value.(*ClientInfo)
				response.idlePlayers[idx] = player.Player
				idx++
			}
			respChan <- response
		}
	}
}
Beispiel #11
0
func handleNop(client *ClientInfo, message interface{}) {
	o.Debug("Client %s: NOP received", client.Name())
}
Beispiel #12
0
func handleAudienceRequest(c net.Conn) {
	defer c.Close()

	c.SetTimeout(0)
	r, _ := c.(io.Reader)
	w, _ := c.(io.Writer)
	dec := json.NewDecoder(r)
	enc := json.NewEncoder(w)

	outobj := new(GenericJsonRequest)
	err := dec.Decode(outobj)
	if err != nil {
		o.Warn("Error decoding JSON talking to audience: %s", err)
		return
	}

	if nil == outobj.Op {
		o.Warn("Malformed JSON message talking to audience.  Missing Op")
		return
	}
	switch *(outobj.Op) {
	case "status":
		if nil == outobj.Id {
			o.Warn("Malformed Status message talking to audience. Missing Job ID")
			return
		}
		job := o.JobGet(*outobj.Id)
		jresp := new([2]interface{})
		if nil != job {
			jresp[0] = "OK"
			iresp := NewJsonStatusResponse()
			switch job.State {
			case o.JOB_PENDING:
				iresp.Status = "PENDING"
			case o.JOB_SUCCESSFUL:
				iresp.Status = "OK"
			case o.JOB_FAILED_PARTIAL:
				iresp.Status = "PARTIAL_FAIL"
			case o.JOB_FAILED:
				iresp.Status = "FAIL"
			default:
				o.Fail("Blargh.  %d is an unknown job state!", job.State)
			}
			resnames := o.JobGetResultNames(*outobj.Id)
			for i := range resnames {
				tr := o.JobGetResult(*outobj.Id, resnames[i])
				if nil != tr {
					presp := NewJsonPlayerStatus()
					switch tr.State {
					case o.RESP_RUNNING:
						presp.Status = "PENDING"
					case o.RESP_FINISHED:
						presp.Status = "OK"
					case o.RESP_FAILED:
						presp.Status = "FAIL"
					case o.RESP_FAILED_UNKNOWN_SCORE:
						presp.Status = "UNK_SCORE"
					case o.RESP_FAILED_HOST_ERROR:
						presp.Status = "HOST_ERROR"
					case o.RESP_FAILED_UNKNOWN:
						presp.Status = "UNKNOWN_FAILURE"
					}
					for k, v := range tr.Response {
						presp.Response[k] = v
					}
					iresp.Players[resnames[i]] = presp
				}

			}
			jresp[1] = iresp
		} else {
			jresp[0] = "Error"
			jresp[1] = nil
		}
		enc.Encode(jresp)
		o.Debug("Status...")
	case "queue":
		if nil == outobj.Score {
			o.Warn("Malformed Queue message talking to audience. Missing Score")
			sendQueueFailureResponse("Missing Score", enc)
			return
		}
		if nil == outobj.Scope {
			o.Warn("Malformed Queue message talking to audience. Missing Scope")
			sendQueueFailureResponse("Missing Scope", enc)
			return
		}
		if nil == outobj.Players || len(outobj.Players) < 1 {
			o.Warn("Malformed Queue message talking to audience. Missing Players")
			sendQueueFailureResponse("Missing Players", enc)
			return
		}
		for _, player := range outobj.Players {
			if !HostAuthorised(player) {
				o.Warn("Malformed Queue message - unknown player %s specified.", player)
				sendQueueFailureResponse("Invalid Player", enc)
				return
			}
		}
		job := NewRequest()
		job.Score = *outobj.Score
		switch *outobj.Scope {
		case "one":
			job.Scope = o.SCOPE_ONEOF
		case "all":
			job.Scope = o.SCOPE_ALLOF
		default:
			sendQueueFailureResponse("Invalid Scope", enc)
			return
		}
		job.Players = outobj.Players
		job.Params = outobj.Params

		QueueJob(job)
		sendQueueSuccessResponse(job, enc)
	default:
		o.Warn("Unknown operation talking to audience: \"%s\"", *(outobj.Op))
		return
	}

	_ = enc
}
Beispiel #13
0
// This takes an unmarshall'd job and stuffs it back into the job state.
func RestoreJobState(job *JobRequest) bool {
	// check the valid players list.
	var playersout []string = nil
	resultsout := make(map[string]*TaskResponse)
	for _, p := range job.Players {
		if HostAuthorised(p) {
			playersout = append(playersout, p)
			// fix the result too.
			resout, exists := job.Results[p]
			if exists && resout != nil {
				resout.id = job.Id
				resultsout[p] = resout
			}
			// remove it so we can sweep it in pass2 for
			// results from old hosts that matter.
			delete(job.Results, p)
		}
	}
	job.Players = playersout
	if len(job.Players) == 0 {
		// If there are no players left at this point, discard
		// the job as corrupt.
		return false
	}
	// now, do pass 2 over the remaining results.
	for k, v := range job.Results {
		if v != nil {
			// if the results indicate completion, we
			// always retain them.
			if v.State.Finished() {
				resultsout[k] = v
				resultsout[k].id = job.Id
			}
		}
	}
	job.Results = resultsout

	// now, check the task data.  ONEOF jobs are allowed to
	// reset tasks that have never been sent.
	var tasksout []*TaskRequest = nil
	for _, t := range job.Tasks {
		// rebuild the return link
		t.job = job
		// finished tasks we don't care about.
		if t.State.Finished() {
			tasksout = append(tasksout, t)
			continue
		}
		if job.Scope == SCOPE_ONEOF {
			if t.Player != "" && (t.State == TASK_QUEUED || !HostAuthorised(t.Player)) {
				t.State = TASK_QUEUED
				t.Player = ""
			}
			tasksout = append(tasksout, t)
			continue
		} else {
			if HostAuthorised(t.Player) {
				tasksout = append(tasksout, t)
			}
		}
	}
	job.Tasks = tasksout
	if len(job.Tasks) == 0 {
		o.Debug("Empty tasks in deserialised job")
		// Tasks should never be empty.
		return false
	}
	// put the job back into the system.
	JobAdd(job)
	JobReviewState(job.Id)
	if !job.State.Finished() {
		// now, redispatch anything that's not actually finished.
		for _, t := range job.Tasks {
			if !t.State.Finished() {
				DispatchTask(t)
			}
		}
	}
	return true
}