func runBuildFile(s *WorkerState) {
var stdout, stderr bytes.Buffer
s.CompileStartTime = time.Now()
_, err := runOutput(CompileTimeout,
nil, /* env */
&stdout,
&stderr,
filepath.Dir(s.BuildFileName),
[]string{s.BuildFileName},
)
s.CompileEndTime = time.Now()
stats.LogTime("Worker", "CompileTime", s.CompileStartTime, s.CompileEndTime)
s.CompileStdout = string(stdout.Bytes())
s.CompileStderr = string(stderr.Bytes())
if err != nil {
switch err.(type) {
case TimeoutError:
s.TimeoutError = true
s.TimeoutValue = err.(TimeoutError).Timeout
stats.Incr("Worker", "RunTimeout")
panic("Failed to compile program")
return
}
}
if err != nil ||
!FileExists(filepath.Join(s.TemporaryDirectory, s.ExecutableFileName)) {
s.CompilationFailed = true
stats.Incr("Worker", "CompilationFailed")
panic("Failed to compile program")
}
}
func (c SecuredApplication) RevertMachineProblem(mpNumString string) revel.Result {
var mp models.MachineProblem
mpNum, err := strconv.Atoi(mpNumString)
if err != nil {
c.Flash.Error("Invalid machine problem")
return c.Render(routes.PublicApplication.Index())
}
user := c.connected()
stats.Incr("App", "ProgramRevert")
if mp, err = models.FindOrCreateMachineProblemByUser(user, mpNum); err != nil {
return c.RenderJson(map[string]interface{}{
"status": "error",
"data": "Could not create mp.",
})
}
if _, err := revertMachineProblemProgram(user, mp); err == nil {
return c.RenderJson(map[string]interface{}{
"status": "success",
})
} else {
return c.RenderJson(map[string]interface{}{
"status": "error",
})
}
}
func (c CourseraApplication) LTIAuthenticate() revel.Result {
var values string
var form string
var userid string
user := c.connected()
if user.Id == 0 {
c.Flash.Error("Must log in before connecting user to coursera")
return c.Redirect(routes.PublicApplication.Login())
}
c.Params.Bind(&values, "Values")
c.Params.Bind(&form, "Form")
c.Params.Bind(&userid, "user_id")
if userid == "" {
c.Flash.Error("Cannot get user identity from coursera!")
stats.Incr("User", "CourseraAuthenticationFailed")
return c.Redirect(routes.PublicApplication.Index())
}
revel.TRACE.Println("identity = ", userid)
stats.Log("User", "Coursera", "Connected to coursera with identity: "+userid)
models.SetUserCourseraCredentials(user, form, values, userid)
c.Flash.Success("Connected to coursera!")
return c.Redirect(routes.PublicApplication.Index())
}
func (c SecuredApplication) SaveProgram(mpNumString string) revel.Result {
var program string
var mp models.MachineProblem
mpNum, err := strconv.Atoi(mpNumString)
if err != nil {
c.Flash.Error("Invalid machine problem")
return c.Render(routes.PublicApplication.Index())
}
user := c.connected()
c.Params.Bind(&program, "program")
stats.Incr("App", "ProgramSave")
if mp, err = models.FindOrCreateMachineProblemByUser(user, mpNum); err != nil {
return c.RenderJson(map[string]interface{}{
"status": "error",
"data": "Could not create mp.",
})
}
if prog, err := models.CreateProgram(mp, program); err != nil {
return c.RenderJson(map[string]interface{}{
"status": "error",
"data": "Could not save program.",
})
} else {
return c.RenderJson(map[string]interface{}{
"status": "success",
"id": prog.Id,
})
}
}
func (c SecuredApplication) Logout() revel.Result {
for k := range c.Session {
delete(c.Session, k)
}
stats.Incr("App", "Logout")
return c.Redirect(routes.PublicApplication.Index())
}
func RegisterWorker(w *WorkerInfo) {
if strings.Contains(w.Address, "%") {
return
}
if _, p := Workers[w.Address]; p == false {
Workers[w.Address] = w
revel.TRACE.Println("Added worker...")
stats.Incr("Master", "Workers")
}
}
func (c PublicApplication) CheckLogin() revel.Result {
var userName string
var password string
c.Params.Bind(&userName, "userName")
c.Params.Bind(&password, "password")
if models.ValidUserNamePassword(userName, password) {
c.Session["user"] = userName
c.Flash.Success("Welcome, " + userName)
stats.Incr("App", "Login")
return c.Redirect(routes.PublicApplication.Index())
} else {
c.Flash.Error("Failed to login")
stats.Incr("App", "LoginFailed")
return c.Redirect(routes.PublicApplication.Login())
}
}
func (c SecuredApplication) SubmitProgram(mpNumString string) (res revel.Result) {
var program string
var datasetId int
mpNum, err := strconv.Atoi(mpNumString)
if err != nil {
c.Flash.Error("Invalid machine problem")
return c.Render(routes.PublicApplication.Index())
}
user := c.connected()
c.Params.Bind(&program, "program")
c.Params.Bind(&datasetId, "datasetId")
mp, err := models.FindOrCreateMachineProblemByUser(user, mpNum)
if err != nil {
return c.RenderJson(map[string]interface{}{
"status": "error",
"title": "Error finding MP record",
"data": "System was not able to save program.",
})
}
if lastAttempt, err := models.FindLastAttemptByMachineProblem(mp); err == nil && time.Since(lastAttempt.Created).Seconds() < 5 {
return c.RenderJson(map[string]interface{}{
"status": "error",
"title": "Attempt limiter",
"data": "Too many attempts. Please wait 10 seconds between attempts.",
})
}
if _, err := models.CreateProgram(mp, program); err != nil {
return c.RenderJson(map[string]interface{}{
"status": "error",
"title": "Error Saving Program",
"data": "System was not able to save program.",
})
}
runId := generateRunId(user)
conf, _ := ReadMachineProblemConfig(mpNum)
stats.Incr("App", "ProgramSubmission")
server.SubmitProgram(mp, program, datasetId, conf.Language, runId, false)
return c.RenderJson(map[string]interface{}{
"status": "success",
"runId": runId,
"attempt": "Attempt submitted",
})
}
func (c SecuredApplication) ComputeGrade(attemptIdString string) revel.Result {
attemptId, err := strconv.Atoi(attemptIdString)
if err != nil {
c.Flash.Error("Invalid attempt Id")
return c.Render(routes.PublicApplication.Index())
}
attempt, err := models.FindAttempt(int64(attemptId))
if err != nil || attempt.Id == 0 {
return c.RenderJson(map[string]interface{}{
"status": "error",
"title": "Failed to grade machine problem",
"data": "Cannot find attempt " + attemptIdString + ".",
})
}
mp, err := models.FindMachineProblem(attempt.MachineProblemInstanceId)
if err != nil || mp.Id == 0 {
return c.RenderJson(map[string]interface{}{
"status": "error",
"title": "Failed to grade",
"data": "Cannot find machine problem instance.",
})
}
user := c.connected()
if mp.UserInstanceId != user.Id {
return c.RenderJson(map[string]interface{}{
"status": "error",
"title": "Failed to grade",
"data": "Invalid user.",
})
}
runId := generateRunId(user)
conf, _ := ReadMachineProblemConfig(mp.Number)
stats.Incr("App", "GradeSubmission")
prog, _ := models.FindProgram(attempt.ProgramInstanceId)
server.SubmitProgram(mp, prog.Text, -1, conf.Language, runId, true)
return c.RenderJson(map[string]interface{}{
"status": "success",
"mpId": strconv.Itoa(int(mp.Id)),
"runId": runId,
"attempt": "Grade submitted",
})
}
func CompileAndRun(req *WorkerRequest) (res []WorkerState) {
stats.Incr("Worker", "Compilations")
conf, _ := ReadMachineProblemConfig(req.MachineProblemNumber)
stats.Log("Worker", "CompileMP", "MP"+
strconv.Itoa(req.MachineProblemNumber))
stats.Log("Worker", "DatasetRun", "MP"+
strconv.Itoa(req.MachineProblemNumber)+"::"+
strconv.Itoa(req.DatasetId))
if req.DatasetId == -1 && len(conf.Datasets) != 0 {
res = make([]WorkerState, len(conf.Datasets))
s := compile(req, 0)
s.OnAllDatasets = true
for i := range conf.Datasets {
res[i] = *s
res[i].DatasetId = i
run(&res[i])
}
/*
if directoryExists(s.TemporaryDirectory) {
go func() {
os.RemoveAll(s.TemporaryDirectory)
}()
}
*/
} else {
res = make([]WorkerState, 1)
s := compile(req, req.DatasetId)
if req.DatasetId == -1 {
s.OnAllDatasets = true
} else {
s.OnAllDatasets = false
}
run(s)
res[0] = *s
/*
if directoryExists(s.TemporaryDirectory) {
go func() {
os.RemoveAll(s.TemporaryDirectory)
}()
}
*/
}
return
}
func (c SecuredApplication) SaveQuestion(mpNumString string, questionNumString string) revel.Result {
var answer string
mpNum, err := strconv.Atoi(mpNumString)
questionNum, err := strconv.Atoi(questionNumString)
c.Params.Bind(&answer, "answer")
if answer == "" {
return c.RenderJson(map[string]interface{}{
"status": "error",
"data": "Blank answer was not saved.",
})
}
user := c.connected()
mp, err := models.FindOrCreateMachineProblemByUser(user, mpNum)
if err != nil {
return c.RenderJson(map[string]interface{}{
"status": "error",
"data": "Cannot find machine problem.",
})
}
conf, _ := ReadMachineProblemConfig(mp.Number)
if MachineProblemCodingDeadlineExpiredQ(conf) {
return c.RenderJson(map[string]interface{}{
"status": "error-deadline",
"data": "Coding deadline has passed.",
})
}
questions, err := models.FindOrCreateQuestionsByMachineProblem(mp)
if err != nil {
return c.RenderJson(map[string]interface{}{
"status": "error",
"data": "Error: create questions.",
})
}
err = models.SaveQuestion(mp, questions, int64(questionNum), answer)
stats.Incr("App", "QuestionSave")
return c.RenderJson(map[string]interface{}{
"status": "success",
"data": answer,
})
}
func (c SecuredApplication) MachineProblem(mpNumString string) revel.Result {
mpNum, err := strconv.Atoi(mpNumString)
if err != nil {
c.Flash.Error("Invalid machine problem")
return c.Render(routes.PublicApplication.Index())
}
user := c.connected()
conf, _ := ReadMachineProblemConfig(mpNum)
if MachineProblemCodingDeadlineExpiredQ(conf) {
c.RenderArgs["coding_expired"] = true
}
if MachineProblemPeerReviewDeadlineExpiredQ(conf) {
c.RenderArgs["peer_review_expired"] = true
}
var mp models.MachineProblem
if mp, err = models.FindOrCreateMachineProblemByUser(user, mpNum); err != nil {
c.Flash.Error("Cannot create machine problem")
return c.Render(routes.PublicApplication.Index())
}
c.RenderArgs["title"] = conf.Name
c.RenderArgs["mp_num"] = mpNumString
c.RenderArgs["mp_description"] = conf.Description
c.RenderArgs["mp_config"] = conf
if prog, err := machineProblemProgram(user, mp); err == nil {
c.RenderArgs["mp_program"] = prog.Text
}
if questions, err := machineProblemQuestions(user, mp); err == nil {
c.RenderArgs["mp_questions"] = questions
}
if history, err := recentPrograms(user, mp, 100); err == nil {
c.RenderArgs["mp_program_history"] = history
}
c.RenderArgs["mp_attempts"] = getAttemptsSummary(user, mp)
stats.Incr("App", "MachineProblemViews")
return c.Render()
}
func (c SecuredApplication) MachineProblemDatasetDownload(
mpNumString string, name string) revel.Result {
if !validMpNumber(mpNumString) {
return c.Redirect("/404")
}
if name != "dataset.zip" && name != "dataset.tar.gz" {
return c.Redirect("/404")
}
path := filepath.Join(MPFileDirectory, mpNumString, name)
if f, err := os.Open(path); err == nil {
stats.Incr("App", "DatasetDownload")
return c.RenderFile(f, "attachement")
}
return c.Redirect("/404")
}
func (c PublicApplication) CreateUser() revel.Result {
var user models.User
c.Params.Bind(&user, "user")
if !validEmail(user.Email) {
c.Flash.Error("Invalid email ... please use the university provided email account")
return c.Redirect(routes.PublicApplication.CreateUser())
}
if models.UserNameExists(user.UserName) {
c.Flash.Error("Username Already taken")
return c.Redirect(routes.PublicApplication.CreateUser())
}
user.Validate(c.Validation)
if c.Validation.HasErrors() {
c.Flash.Error("Cannot signup!")
c.Validation.Keep()
c.FlashParams()
return c.Redirect(routes.PublicApplication.CreateUser())
}
user.Hashed = false
if err := models.CreateUser(user); err != nil {
stats.TRACE.Println("Failed to create user ", err)
return c.Redirect(routes.PublicApplication.CreateUser())
}
c.Session["user"] = user.UserName
c.Flash.Success("Welcome, " + user.UserName)
stats.Incr("App", "Users")
return c.Redirect(routes.PublicApplication.Index())
}
func SubmitJob(ws WorkerRequest) (res *WorkerState, err error) {
worker, err := chooseWorker()
if err != nil {
return
}
js, err := json.Marshal(ws)
if err != nil {
return
}
b := bytes.NewBufferString(string(js))
stats.Incr("Master", "SubmittingJob")
resp, err := http.Post(worker.Address+"/compile", "text/json", b)
if err != nil {
return
}
defer resp.Body.Close()
return
}
// TODO: This should not live here
func CreateAttemptWithStates(states []WorkerState) (attempts []models.Attempt, grade models.Grade, err error) {
defer func() {
if r := recover(); r != nil {
err = errors.New(fmt.Sprint(r))
}
}()
if len(states) == 0 {
err = errors.New("Invalid attempt")
return
}
firstState := states[0]
mp, err := models.FindMachineProblem(firstState.MachineProblemId)
if err != nil {
return
}
prog, err := models.CreateProgram(mp, firstState.Program)
if err != nil {
return
}
toGrade := false
for _, ws := range states {
jsInternalCData, _ := json.Marshal(ws.InternalCData)
attempt := models.Attempt{
MachineProblemInstanceId: mp.Id,
ProgramInstanceId: prog.Id,
RunId: ws.RunId,
DatasetId: ws.DatasetId,
CompilationFailed: ws.CompilationFailed,
CompileStderr: strings.TrimSpace(ws.CompileStderr),
CompileStdout: strings.TrimSpace(ws.CompileStdout),
RunFailed: ws.RunFailed,
RunStdout: strings.TrimSpace(ws.RunStdout),
RunStderr: strings.TrimSpace(ws.RunStderr),
TimeoutError: ws.TimeoutError,
TimeoutValue: ws.TimeoutValue,
Sandboxed: ws.Sandboxed,
SandboxKeyword: ws.SandboxKeyword,
CompileElapsedTime: ws.CompileEndTime.Sub(ws.CompileStartTime).Nanoseconds(),
RunElapsedTime: ws.RunEndTime.Sub(ws.RunStartTime).Nanoseconds(),
CompileStartTime: ws.CompileStartTime,
CompileEndTime: ws.CompileEndTime,
RunStartTime: ws.RunStartTime,
RunEndTime: ws.RunEndTime,
RequestStartTime: ws.RequestStartTime,
RequestEndTime: ws.RequestEndTime,
SolutionCorrect: ws.SolutionCorrect,
SolutionMessage: ws.SolutionMessage,
UserOutput: ws.UserOutput,
Language: ws.Language,
OnAllDatasets: ws.OnAllDatasets,
InternalCData: string(jsInternalCData),
Created: time.Now(),
Updated: time.Now(),
}
if ws.ForGrading {
attempt.GradedQ = true
toGrade = true
}
err = models.DB.Save(&attempt).Error
if err != nil {
revel.TRACE.Println("Failed saving attempt.. ", err)
stats.Incr("Master", "FailedAttemptStore")
} else {
attempts = append(attempts, attempt)
}
}
if toGrade {
stats.Incr("Master", "UpdatingGrade")
//revel.TRACE.Println("Updating grade...")
grade, err = models.UpdateGradeWithAttempts(attempts)
if err != nil {
revel.TRACE.Println("Error updating grade ", err)
}
}
return
}
func doPostCourseraGrade(user models.User, mp models.MachineProblem,
grade models.Grade, toPost string, forceQ bool) error {
type CourseraGrade struct {
UserId string `json:"user_id"`
AssignmentId string `json:"assignment_part_sid"`
Score int64 `json:"score"`
Feedback string `json:"feedback"`
APIKey string `json:"api_key"`
CreateNewSubmission int `json:"create_new_submission"`
}
toPost = strings.ToLower(toPost)
toPostCode := false
toPostPeer := false
if toPost == "all" || toPost == "code" {
toPostCode = true
}
if toPost == "all" || toPost == "peer" {
toPostPeer = true
}
conf, _ := ReadMachineProblemConfig(mp.Number)
idty := models.GetUserIdentity(user)
// Post grade to coursera
postGrade := func(kind string, key string, score int64) error {
reason := ""
if kind == "code" {
reason = grade.Reasons
}
if key == "NONE" {
return errors.New("You are not graded on this MP")
}
t := time.Now().Unix()
if kind == "code" && time.Since(conf.CodingDeadline).Hours() > conf.GracePeriod {
t = conf.CodingDeadline.UTC().Unix()
} else if kind == "peer" && time.Since(conf.PeerReviewDeadline).Hours() > conf.GracePeriod {
t = conf.PeerReviewDeadline.UTC().Unix()
}
vals := url.Values{
"api_key": {CourseraGradeAPIKey},
"user_id": {idty},
"score": {strconv.Itoa(int(score))},
"assignment_part_sid": {key},
"feedback": {reason},
"submission_time": {fmt.Sprint(t)},
}
resp, err := http.PostForm(CourseraGradeURL,
vals)
revel.TRACE.Println("Posting grade for ", idty, " ", CourseraGradeURL, " with key ", key, vals)
if err != nil {
return err
}
models.UpdateCourseraGrade(grade, kind, score)
defer resp.Body.Close()
return err
}
if toPostPeer && grade.PeerReviewScore != 0 && (forceQ == true || grade.PeerReviewScore >= grade.CourseraPeerReviewGrade) {
if err := postGrade("peer", conf.CourseraPeerReviewPostKey, grade.PeerReviewScore); err != nil {
revel.ERROR.Println(err)
return errors.New("Was not able to post coursera grade. Make sure you are connected to coursera first and/or this MP is graded.")
}
}
if toPostCode && grade.CodeScore != 0 && (forceQ == true || grade.CodeScore >= grade.CourseraCodingGrade) {
if err := postGrade("code", conf.CourseraCodePostKey, grade.CodeScore); err != nil {
return errors.New("Was not able to post coursera grade. Make sure you are connected to coursera first and/or this MP is graded.")
}
}
stats.Incr("User", "CourseraPostGrade")
return nil
}
func runProgram(s *WorkerState) {
var stdout, stderr bytes.Buffer
stats.Incr("Worker", "Run")
conf := s.MachineProblemConfig
outputFileExtension := func() string {
if conf.OutputType == "image" {
return "ppm"
} else {
return "raw"
}
}
getInputs := func() []string {
if s.DatasetId < 0 {
return []string{"none"}
} else {
datasetConfig := conf.Datasets[s.DatasetId]
inputs := make([]string, len(datasetConfig.Input))
for i, input := range datasetConfig.Input {
inputs[i] = filepath.Join(s.MachineProblemDirectory, "data", input)
}
return inputs
}
}
getOutput := func() string {
if s.DatasetId < 0 {
return "none"
} else {
datasetConfig := conf.Datasets[s.DatasetId]
output := datasetConfig.Output
return filepath.Join(s.MachineProblemDirectory, "data", output)
}
}
s.UserOutput = filepath.Join(s.TemporaryDirectory, "output."+outputFileExtension())
runCommand := []string{
filepath.Join(s.TemporaryDirectory, s.ExecutableFileName),
"-i ", strings.Join(getInputs(), ","),
//"-o ", s.UserOutput,
"-e ", getOutput(),
"-t ", conf.OutputType,
}
s.RunStartTime = time.Now()
_, err := runOutput(RuntimeTimeout,
nil, /* env */
&stdout,
&stderr,
filepath.Dir(s.BuildFileName),
runCommand,
)
s.RunEndTime = time.Now()
stats.LogTime("Worker", "RunTime", s.RunStartTime, s.RunEndTime)
s.RunStdout = string(stdout.Bytes())
s.RunStderr = string(stderr.Bytes())
removeInternalData := func(s string) string {
ss := strings.Split(s, seperator)
return ss[0]
}
switch err.(type) {
case TimeoutError:
revel.TRACE.Println("Terminated....")
s.TimeoutError = true
s.TimeoutValue = err.(TimeoutError).Timeout
stats.Incr("Worker", "RunTimeout")
panic("Failed to run program")
}
if strings.Contains(s.RunStderr, "<<SANDBOXED>>") {
s.Sandboxed = true
s.SandboxKeyword = "Program sandboxed because of use of " +
removeInternalData(strings.TrimPrefix(s.RunStdout, "<<SANDBOXED>>::")) +
" keyword."
stats.Log("Worker", "RunSandboxed", s.SandboxKeyword)
panic("Failed to run program")
} else if strings.Contains(s.RunStderr, "<<MEMORY>>") {
s.RunFailed = true
s.RunStdout = ""
s.RunStderr = "Program teminated because it is allocating too much memory."
stats.Incr("Worker", "MemoryLimit")
panic("Failed to run program")
} else if err != nil {
s.RunFailed = true
s.RunStdout = removeInternalData(s.RunStdout)
stats.Incr("Worker", "RunFailed")
panic("Failed to run program")
}
ss := strings.Split(s.RunStdout, seperator)
s.RunStdout = ss[0]
var wbData InternalCData
normalizeString := func(s string) string {
res := s
if !utf8.ValidString(s) {
v := make([]rune, 0, len(s))
for i, r := range s {
if r == utf8.RuneError {
if _, size := utf8.DecodeRuneInString(s[i:]); size == 1 {
continue
}
}
v = append(v, r)
}
res = string(v)
}
return res
}
err = json.Unmarshal([]byte(normalizeString(ss[1])), &wbData)
if err != nil {
stats.Incr("Worker", "InternalDataReadError")
s.RunFailed = true
s.RunStdout = s.RunStdout
s.RunStderr = "Failed to read program output. Make sure you do not have special characters in your code."
stats.Log("Worker", "Error", "Failed to read internal data "+fmt.Sprint(err))
return
}
if s.DatasetId < 0 {
wbData.SolutionExists = false
wbData.Solution.CorrectQ = true
wbData.Solution.Message = "No solution expected."
}
s.SolutionCorrect = wbData.Solution.CorrectQ
s.SolutionMessage = wbData.Solution.Message
s.InternalCData = wbData
if !s.SolutionCorrect {
stats.Log("Worker", "IncorrectSolution", 1)
} else {
stats.Log("Worker", "CorrectSolution", 1)
}
}