Пример #1
0
func TestLibinteractive(t *testing.T) {
	minijail := &MinijailSandbox{}
	if testing.Short() {
		t.Skip("skipping test in short mode.")
	}
	if !minijail.Supported() {
		t.Skip("minijail sandbox not supported")
	}
	ctx, err := newRunnerContext()
	if err != nil {
		t.Fatalf("RunnerContext creation failed with %q", err)
	}
	defer ctx.Close()
	if !ctx.Config.Runner.PreserveFiles {
		defer os.RemoveAll(ctx.Config.Runner.RuntimePath)
	}

	inputManager := common.NewInputManager(ctx)
	AplusB, err := common.NewLiteralInputFactory(
		&common.LiteralInput{
			Cases: map[string]common.LiteralCaseSettings{
				"0": {Input: "1 2", ExpectedOutput: "3"},
				"1": {Input: "2 3", ExpectedOutput: "5"},
			},
			Validator: &common.LiteralValidatorSettings{
				Name: "token-numeric",
			},
			Interactive: &common.LiteralInteractiveSettings{
				IDLSource: `
					interface Main {};
					interface AplusB {
						int sum(int a, int b);
					};
					interface Identity {
						int identity(int x);
					};
				`,
				MainSource: `
					#include "AplusB.h"
					#include <iostream>
					using namespace std;
					int main() {
						int A, B;
						cin >> A >> B;
						cout << identity(sum(identity(A), identity(B))) << endl;
					}
				`,
				ModuleName: "AplusB",
				ParentLang: "cpp",
			},
		},
		ctx.Config.Runner.RuntimePath,
	)
	if err != nil {
		t.Fatalf("Failed to create Input: %q", err)
	}
	inputManager.Add(AplusB.Hash(), AplusB)

	input, err := inputManager.Get(AplusB.Hash())
	if err != nil {
		t.Fatalf("Failed to open problem: %q", err)
	}

	runtests := []runnerTestCase{
		{
			"cpp",
			`
				#include <iostream>
				using namespace std;
				int main(int argc, char* argv[]) {
					int A, B;
					cin >> A >> B;
					cerr << argv[1] << endl;
					cout << A + B << endl;
				}
			`,
			"CE",
			0.0,
			expectedResult{
				"",
				`  File "test.py", line 1
	    if
		     ^
				 SyntaxError: invalid syntax`,
				&RunMetadata{ExitStatus: 1, Verdict: "RTE"},
			},
			map[string]expectedResult{},
		},
		{
			"cpp11",
			`
				#include "AplusB.h"
				int sum(int A, int B) {
					return -1;
				}
				int identity(int x) {
					return -1;
				}
			`,
			"WA",
			0.0,
			expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
			map[string]expectedResult{
				"0": {"-1", "", &RunMetadata{Verdict: "OK"}},
				"1": {"-1", "", &RunMetadata{Verdict: "OK"}},
			},
		},
		{
			"cpp",
			`
				#include "AplusB.h"
				int sum(int A, int B) {
					return A + B;
				}
				int identity(int x) {
					return x;
				}
			`,
			"AC",
			1.0,
			expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
			map[string]expectedResult{
				"0": {"3", "", &RunMetadata{Verdict: "OK"}},
				"1": {"5", "", &RunMetadata{Verdict: "OK"}},
			},
		},
		{
			"java",
			`
				class AplusB {
					public static int sum(int A, int B) {
						return A + B;
					}
				}
				class Identity {
					public static int identity(int x) {
						return x;
					}
				}
			`,
			"AC",
			1.0,
			expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
			map[string]expectedResult{
				"0": {"3", "", &RunMetadata{Verdict: "OK"}},
				"1": {"5", "", &RunMetadata{Verdict: "OK"}},
			},
		},
		{
			"py",
			"def sum(A, B):\n  return A + B\ndef identity(x):\n  return x",
			"AC",
			1.0,
			expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
			map[string]expectedResult{
				"0": {"3", "", &RunMetadata{Verdict: "OK"}},
				"1": {"5", "", &RunMetadata{Verdict: "OK"}},
			},
		},
	}
	for idx, rte := range runtests {
		results, err := Grade(
			ctx,
			&bytes.Buffer{},
			&common.Run{
				AttemptID: uint64(idx),
				Language:  rte.language,
				InputHash: input.Hash(),
				Source:    rte.source,
				MaxScore:  1.0,
			},
			input,
			minijail,
		)
		if err != nil {
			t.Errorf("Failed to run %v: %q", rte, err)
			continue
		}
		if results.Verdict != rte.expectedVerdict {
			t.Errorf(
				"results.Verdict = %q, expected %q",
				results.Verdict,
				rte.expectedVerdict,
			)
		}
		if results.Score != rte.expectedScore {
			t.Errorf(
				"results.Score = %v, expected %v",
				results.Score,
				rte.expectedScore,
			)
		}
	}
}
Пример #2
0
func TestQueue(t *testing.T) {
	ctx, err := newGraderContext()
	if err != nil {
		t.Fatalf("GraderContext creation failed with %q", err)
	}
	defer ctx.Close()
	if !ctx.Config.Runner.PreserveFiles {
		defer os.RemoveAll(ctx.Config.Grader.RuntimePath)
	}

	queue, err := ctx.QueueManager.Get("default")
	if err != nil {
		t.Fatalf("default queue not found")
	}

	AplusB, err := common.NewLiteralInputFactory(
		&common.LiteralInput{
			Cases: map[string]common.LiteralCaseSettings{
				"0": {Input: "1 2", ExpectedOutput: "3"},
				"1": {Input: "2 3", ExpectedOutput: "5"},
			},
			Validator: &common.LiteralValidatorSettings{
				Name: "token-numeric",
			},
		},
		ctx.Config.Grader.RuntimePath,
	)
	if err != nil {
		t.Fatalf("Failed to create Input: %q", err)
	}
	ctx.InputManager.Add(AplusB.Hash(), AplusB)
	input, err := ctx.InputManager.Get(AplusB.Hash())
	if err != nil {
		t.Fatalf("Failed to get input back: %q", err)
	}
	runCtx := NewEmptyRunContext(ctx)
	runCtx.Run.InputHash = AplusB.Hash()
	runCtx.Run.Source = "print 3"
	if err = AddRunContext(ctx, runCtx, input); err != nil {
		t.Fatalf("AddRunContext failed with %q", err)
	}
	queue.AddRun(runCtx)
	if len(queue.runs[1]) != 1 {
		t.Fatalf("len(queue.runs[1]) == %d, want %d", len(queue.runs[1]), 1)
	}

	closeNotifier := make(chan bool, 1)

	// Test timeout.
	originalConnectTimeout := ctx.InflightMonitor.connectTimeout
	ctx.InflightMonitor.connectTimeout = 0
	runCtx, timeout, _ := queue.GetRun("test", ctx.InflightMonitor, closeNotifier)
	if len(queue.runs[1]) != 0 {
		t.Fatalf("len(queue.runs[1]) == %d, want %d", len(queue.runs[1]), 0)
	}
	if _, didTimeout := <-timeout; !didTimeout {
		t.Fatalf("expected timeout but did not happen")
	}
	ctx.InflightMonitor.connectTimeout = originalConnectTimeout

	// The run has already been requeued. This time it will be successful.
	if len(queue.runs[0]) != 1 {
		t.Fatalf("len(queue.runs[0]) == %d, want %d", len(queue.runs[0]), 1)
	}
	runCtx, timeout, _ = queue.GetRun("test", ctx.InflightMonitor, closeNotifier)
	if len(queue.runs[0]) != 0 {
		t.Fatalf("len(queue.runs[0]) == %d, want %d", len(queue.runs[0]), 0)
	}
	if _, _, ok := ctx.InflightMonitor.Get(runCtx.Run.AttemptID); !ok {
		t.Fatalf("Run %d not found in the inflight run monitor", runCtx.Run.AttemptID)
	}
	ctx.InflightMonitor.Remove(runCtx.Run.AttemptID)
	if _, didTimeout := <-timeout; didTimeout {
		t.Fatalf("expected run completion, but did not happen")
	}

	// Test the closeNotifier.
	closeNotifier <- true
	if _, _, ok := queue.GetRun(
		"test",
		ctx.InflightMonitor,
		closeNotifier,
	); ok {
		t.Fatalf("Expected closeNotifier to cause no run to be available")
	}
}
Пример #3
0
func runGraderTests(t *testing.T, wrapper sandboxWrapper) {
	ctx, err := newRunnerContext()
	if err != nil {
		t.Fatalf("RunnerContext creation failed with %q", err)
	}
	defer ctx.Close()
	if !ctx.Config.Runner.PreserveFiles {
		defer os.RemoveAll(ctx.Config.Runner.RuntimePath)
	}

	inputManager := common.NewInputManager(ctx)
	AplusB, err := common.NewLiteralInputFactory(
		&common.LiteralInput{
			Cases: map[string]common.LiteralCaseSettings{
				"0": {Input: "1 2", ExpectedOutput: "3"},
				"1": {Input: "2 3", ExpectedOutput: "5"},
			},
			Validator: &common.LiteralValidatorSettings{
				Name: "token-numeric",
			},
		},
		ctx.Config.Runner.RuntimePath,
	)
	if err != nil {
		t.Fatalf("Failed to create Input: %q", err)
	}
	inputManager.Add(AplusB.Hash(), AplusB)

	input, err := inputManager.Get(AplusB.Hash())
	if err != nil {
		t.Fatalf("Failed to open problem: %q", err)
	}

	runtests := []runnerTestCase{
		{
			"py",
			"print sum(map(int, raw_input().strip().split()))",
			"AC",
			1.0,
			expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
			map[string]expectedResult{
				"0": {"3", "", &RunMetadata{Verdict: "OK"}},
				"1": {"5", "", &RunMetadata{Verdict: "OK"}},
			},
		},
		{
			"py",
			"print 3",
			"PA",
			0.5,
			expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
			map[string]expectedResult{
				"0": {"3", "", &RunMetadata{Verdict: "OK"}},
				"1": {"3", "", &RunMetadata{Verdict: "OK"}},
			},
		},
		{
			"py",
			"print 2",
			"WA",
			0.0,
			expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
			map[string]expectedResult{
				"0": {"2", "", &RunMetadata{Verdict: "OK"}},
				"1": {"2", "", &RunMetadata{Verdict: "OK"}},
			},
		},
		{
			"py",
			"if",
			"CE",
			0.0,
			expectedResult{
				"",
				`  File "test.py", line 1
	    if
		     ^
				 SyntaxError: invalid syntax`,
				&RunMetadata{ExitStatus: 1, Verdict: "RTE"},
			},
			map[string]expectedResult{},
		},
		{
			"c",
			"#include <stdio.h>\nint main() { printf(\"3\\n\"); }",
			"PA",
			0.5,
			expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
			map[string]expectedResult{
				"0": {"3", "", &RunMetadata{Verdict: "OK"}},
				"1": {"3", "", &RunMetadata{Verdict: "OK"}},
			},
		},
		{
			"cpp",
			"#include <iostream>\nint main() { std::cout << \"3\\n\"; }",
			"PA",
			0.5,
			expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
			map[string]expectedResult{
				"0": {"3", "", &RunMetadata{Verdict: "OK"}},
				"1": {"3", "", &RunMetadata{Verdict: "OK"}},
			},
		},
		{
			"rb",
			"puts 3",
			"PA",
			0.5,
			expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
			map[string]expectedResult{
				"0": {"3", "", &RunMetadata{Verdict: "OK"}},
				"1": {"3", "", &RunMetadata{Verdict: "OK"}},
			},
		},
		{
			"hs",
			"main = putStrLn \"3\"",
			"PA",
			0.5,
			expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
			map[string]expectedResult{
				"0": {"3", "", &RunMetadata{Verdict: "OK"}},
				"1": {"3", "", &RunMetadata{Verdict: "OK"}},
			},
		},
		{
			"pas",
			`program Main;
			begin
				writeln ('3');
			end.`,
			"PA",
			0.5,
			expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
			map[string]expectedResult{
				"0": {"3", "", &RunMetadata{Verdict: "OK"}},
				"1": {"3", "", &RunMetadata{Verdict: "OK"}},
			},
		},
		{
			"java",
			`class Main {
				public static void main(String[] args) {
					System.out.println('3');
				}
			}`,
			"PA",
			0.5,
			expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
			map[string]expectedResult{
				"0": {"3", "", &RunMetadata{Verdict: "OK"}},
				"1": {"3", "", &RunMetadata{Verdict: "OK"}},
			},
		},
		{
			"cat",
			"data:application/zip;base64,UEsDBAoAAAAAAOWiUUjRnmdVAgAAAAIAAAAFABwAMC" +
				"5vdXRVVAkAAy1HxVYtR8VWdXgLAAEE6AMAAAToAwAAMwpQSwMECgAAAAAA56JRSFc5PQ" +
				"MCAAAAAgAAAAUAHAAxLm91dFVUCQADMUfFVjFHxVZ1eAsAAQToAwAABOgDAAA1ClBLAQ" +
				"IeAwoAAAAAAOWiUUjRnmdVAgAAAAIAAAAFABgAAAAAAAEAAAC0gQAAAAAwLm91dFVUBQ" +
				"ADLUfFVnV4CwABBOgDAAAE6AMAAFBLAQIeAwoAAAAAAOeiUUhXOT0DAgAAAAIAAAAFAB" +
				"gAAAAAAAEAAAC0gUEAAAAxLm91dFVUBQADMUfFVnV4CwABBOgDAAAE6AMAAFBLBQYAAA" +
				"AAAgACAJYAAACCAAAAAAA=",
			"AC",
			1.0,
			expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
			map[string]expectedResult{
				"0": {"3", "", &RunMetadata{Verdict: "OK"}},
				"1": {"5", "", &RunMetadata{Verdict: "OK"}},
			},
		},
	}
	for idx, rte := range runtests {
		results, err := Grade(
			ctx,
			&bytes.Buffer{},
			&common.Run{
				AttemptID: uint64(idx),
				Language:  rte.language,
				InputHash: input.Hash(),
				Source:    rte.source,
				MaxScore:  1.0,
			},
			input,
			wrapper.sandbox(&rte),
		)
		if err != nil {
			t.Errorf("Failed to run %v: %q", rte, err)
			continue
		}
		if results.Verdict != rte.expectedVerdict {
			t.Errorf(
				"results.Verdict = %q, expected %q, test %v",
				results.Verdict,
				rte.expectedVerdict,
				idx,
			)
		}
		if results.Score != rte.expectedScore {
			t.Errorf(
				"results.Score = %v, expected %v",
				results.Score,
				rte.expectedScore,
			)
		}
	}
}