func TestLibinteractive(t *testing.T) {
minijail := &MinijailSandbox{}
if testing.Short() {
t.Skip("skipping test in short mode.")
}
if !minijail.Supported() {
t.Skip("minijail sandbox not supported")
}
ctx, err := newRunnerContext()
if err != nil {
t.Fatalf("RunnerContext creation failed with %q", err)
}
defer ctx.Close()
if !ctx.Config.Runner.PreserveFiles {
defer os.RemoveAll(ctx.Config.Runner.RuntimePath)
}
inputManager := common.NewInputManager(ctx)
AplusB, err := common.NewLiteralInputFactory(
&common.LiteralInput{
Cases: map[string]common.LiteralCaseSettings{
"0": {Input: "1 2", ExpectedOutput: "3"},
"1": {Input: "2 3", ExpectedOutput: "5"},
},
Validator: &common.LiteralValidatorSettings{
Name: "token-numeric",
},
Interactive: &common.LiteralInteractiveSettings{
IDLSource: `
interface Main {};
interface AplusB {
int sum(int a, int b);
};
interface Identity {
int identity(int x);
};
`,
MainSource: `
#include "AplusB.h"
#include <iostream>
using namespace std;
int main() {
int A, B;
cin >> A >> B;
cout << identity(sum(identity(A), identity(B))) << endl;
}
`,
ModuleName: "AplusB",
ParentLang: "cpp",
},
},
ctx.Config.Runner.RuntimePath,
)
if err != nil {
t.Fatalf("Failed to create Input: %q", err)
}
inputManager.Add(AplusB.Hash(), AplusB)
input, err := inputManager.Get(AplusB.Hash())
if err != nil {
t.Fatalf("Failed to open problem: %q", err)
}
runtests := []runnerTestCase{
{
"cpp",
`
#include <iostream>
using namespace std;
int main(int argc, char* argv[]) {
int A, B;
cin >> A >> B;
cerr << argv[1] << endl;
cout << A + B << endl;
}
`,
"CE",
0.0,
expectedResult{
"",
` File "test.py", line 1
if
^
SyntaxError: invalid syntax`,
&RunMetadata{ExitStatus: 1, Verdict: "RTE"},
},
map[string]expectedResult{},
},
{
"cpp11",
`
#include "AplusB.h"
int sum(int A, int B) {
return -1;
}
int identity(int x) {
return -1;
}
`,
"WA",
0.0,
expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
map[string]expectedResult{
"0": {"-1", "", &RunMetadata{Verdict: "OK"}},
"1": {"-1", "", &RunMetadata{Verdict: "OK"}},
},
},
{
"cpp",
`
#include "AplusB.h"
int sum(int A, int B) {
return A + B;
}
int identity(int x) {
return x;
}
`,
"AC",
1.0,
expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
map[string]expectedResult{
"0": {"3", "", &RunMetadata{Verdict: "OK"}},
"1": {"5", "", &RunMetadata{Verdict: "OK"}},
},
},
{
"java",
`
class AplusB {
public static int sum(int A, int B) {
return A + B;
}
}
class Identity {
public static int identity(int x) {
return x;
}
}
`,
"AC",
1.0,
expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
map[string]expectedResult{
"0": {"3", "", &RunMetadata{Verdict: "OK"}},
"1": {"5", "", &RunMetadata{Verdict: "OK"}},
},
},
{
"py",
"def sum(A, B):\n return A + B\ndef identity(x):\n return x",
"AC",
1.0,
expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
map[string]expectedResult{
"0": {"3", "", &RunMetadata{Verdict: "OK"}},
"1": {"5", "", &RunMetadata{Verdict: "OK"}},
},
},
}
for idx, rte := range runtests {
results, err := Grade(
ctx,
&bytes.Buffer{},
&common.Run{
AttemptID: uint64(idx),
Language: rte.language,
InputHash: input.Hash(),
Source: rte.source,
MaxScore: 1.0,
},
input,
minijail,
)
if err != nil {
t.Errorf("Failed to run %v: %q", rte, err)
continue
}
if results.Verdict != rte.expectedVerdict {
t.Errorf(
"results.Verdict = %q, expected %q",
results.Verdict,
rte.expectedVerdict,
)
}
if results.Score != rte.expectedScore {
t.Errorf(
"results.Score = %v, expected %v",
results.Score,
rte.expectedScore,
)
}
}
}
func TestQueue(t *testing.T) {
ctx, err := newGraderContext()
if err != nil {
t.Fatalf("GraderContext creation failed with %q", err)
}
defer ctx.Close()
if !ctx.Config.Runner.PreserveFiles {
defer os.RemoveAll(ctx.Config.Grader.RuntimePath)
}
queue, err := ctx.QueueManager.Get("default")
if err != nil {
t.Fatalf("default queue not found")
}
AplusB, err := common.NewLiteralInputFactory(
&common.LiteralInput{
Cases: map[string]common.LiteralCaseSettings{
"0": {Input: "1 2", ExpectedOutput: "3"},
"1": {Input: "2 3", ExpectedOutput: "5"},
},
Validator: &common.LiteralValidatorSettings{
Name: "token-numeric",
},
},
ctx.Config.Grader.RuntimePath,
)
if err != nil {
t.Fatalf("Failed to create Input: %q", err)
}
ctx.InputManager.Add(AplusB.Hash(), AplusB)
input, err := ctx.InputManager.Get(AplusB.Hash())
if err != nil {
t.Fatalf("Failed to get input back: %q", err)
}
runCtx := NewEmptyRunContext(ctx)
runCtx.Run.InputHash = AplusB.Hash()
runCtx.Run.Source = "print 3"
if err = AddRunContext(ctx, runCtx, input); err != nil {
t.Fatalf("AddRunContext failed with %q", err)
}
queue.AddRun(runCtx)
if len(queue.runs[1]) != 1 {
t.Fatalf("len(queue.runs[1]) == %d, want %d", len(queue.runs[1]), 1)
}
closeNotifier := make(chan bool, 1)
// Test timeout.
originalConnectTimeout := ctx.InflightMonitor.connectTimeout
ctx.InflightMonitor.connectTimeout = 0
runCtx, timeout, _ := queue.GetRun("test", ctx.InflightMonitor, closeNotifier)
if len(queue.runs[1]) != 0 {
t.Fatalf("len(queue.runs[1]) == %d, want %d", len(queue.runs[1]), 0)
}
if _, didTimeout := <-timeout; !didTimeout {
t.Fatalf("expected timeout but did not happen")
}
ctx.InflightMonitor.connectTimeout = originalConnectTimeout
// The run has already been requeued. This time it will be successful.
if len(queue.runs[0]) != 1 {
t.Fatalf("len(queue.runs[0]) == %d, want %d", len(queue.runs[0]), 1)
}
runCtx, timeout, _ = queue.GetRun("test", ctx.InflightMonitor, closeNotifier)
if len(queue.runs[0]) != 0 {
t.Fatalf("len(queue.runs[0]) == %d, want %d", len(queue.runs[0]), 0)
}
if _, _, ok := ctx.InflightMonitor.Get(runCtx.Run.AttemptID); !ok {
t.Fatalf("Run %d not found in the inflight run monitor", runCtx.Run.AttemptID)
}
ctx.InflightMonitor.Remove(runCtx.Run.AttemptID)
if _, didTimeout := <-timeout; didTimeout {
t.Fatalf("expected run completion, but did not happen")
}
// Test the closeNotifier.
closeNotifier <- true
if _, _, ok := queue.GetRun(
"test",
ctx.InflightMonitor,
closeNotifier,
); ok {
t.Fatalf("Expected closeNotifier to cause no run to be available")
}
}
func runGraderTests(t *testing.T, wrapper sandboxWrapper) {
ctx, err := newRunnerContext()
if err != nil {
t.Fatalf("RunnerContext creation failed with %q", err)
}
defer ctx.Close()
if !ctx.Config.Runner.PreserveFiles {
defer os.RemoveAll(ctx.Config.Runner.RuntimePath)
}
inputManager := common.NewInputManager(ctx)
AplusB, err := common.NewLiteralInputFactory(
&common.LiteralInput{
Cases: map[string]common.LiteralCaseSettings{
"0": {Input: "1 2", ExpectedOutput: "3"},
"1": {Input: "2 3", ExpectedOutput: "5"},
},
Validator: &common.LiteralValidatorSettings{
Name: "token-numeric",
},
},
ctx.Config.Runner.RuntimePath,
)
if err != nil {
t.Fatalf("Failed to create Input: %q", err)
}
inputManager.Add(AplusB.Hash(), AplusB)
input, err := inputManager.Get(AplusB.Hash())
if err != nil {
t.Fatalf("Failed to open problem: %q", err)
}
runtests := []runnerTestCase{
{
"py",
"print sum(map(int, raw_input().strip().split()))",
"AC",
1.0,
expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
map[string]expectedResult{
"0": {"3", "", &RunMetadata{Verdict: "OK"}},
"1": {"5", "", &RunMetadata{Verdict: "OK"}},
},
},
{
"py",
"print 3",
"PA",
0.5,
expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
map[string]expectedResult{
"0": {"3", "", &RunMetadata{Verdict: "OK"}},
"1": {"3", "", &RunMetadata{Verdict: "OK"}},
},
},
{
"py",
"print 2",
"WA",
0.0,
expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
map[string]expectedResult{
"0": {"2", "", &RunMetadata{Verdict: "OK"}},
"1": {"2", "", &RunMetadata{Verdict: "OK"}},
},
},
{
"py",
"if",
"CE",
0.0,
expectedResult{
"",
` File "test.py", line 1
if
^
SyntaxError: invalid syntax`,
&RunMetadata{ExitStatus: 1, Verdict: "RTE"},
},
map[string]expectedResult{},
},
{
"c",
"#include <stdio.h>\nint main() { printf(\"3\\n\"); }",
"PA",
0.5,
expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
map[string]expectedResult{
"0": {"3", "", &RunMetadata{Verdict: "OK"}},
"1": {"3", "", &RunMetadata{Verdict: "OK"}},
},
},
{
"cpp",
"#include <iostream>\nint main() { std::cout << \"3\\n\"; }",
"PA",
0.5,
expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
map[string]expectedResult{
"0": {"3", "", &RunMetadata{Verdict: "OK"}},
"1": {"3", "", &RunMetadata{Verdict: "OK"}},
},
},
{
"rb",
"puts 3",
"PA",
0.5,
expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
map[string]expectedResult{
"0": {"3", "", &RunMetadata{Verdict: "OK"}},
"1": {"3", "", &RunMetadata{Verdict: "OK"}},
},
},
{
"hs",
"main = putStrLn \"3\"",
"PA",
0.5,
expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
map[string]expectedResult{
"0": {"3", "", &RunMetadata{Verdict: "OK"}},
"1": {"3", "", &RunMetadata{Verdict: "OK"}},
},
},
{
"pas",
`program Main;
begin
writeln ('3');
end.`,
"PA",
0.5,
expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
map[string]expectedResult{
"0": {"3", "", &RunMetadata{Verdict: "OK"}},
"1": {"3", "", &RunMetadata{Verdict: "OK"}},
},
},
{
"java",
`class Main {
public static void main(String[] args) {
System.out.println('3');
}
}`,
"PA",
0.5,
expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
map[string]expectedResult{
"0": {"3", "", &RunMetadata{Verdict: "OK"}},
"1": {"3", "", &RunMetadata{Verdict: "OK"}},
},
},
{
"cat",
"data:application/zip;base64,UEsDBAoAAAAAAOWiUUjRnmdVAgAAAAIAAAAFABwAMC" +
"5vdXRVVAkAAy1HxVYtR8VWdXgLAAEE6AMAAAToAwAAMwpQSwMECgAAAAAA56JRSFc5PQ" +
"MCAAAAAgAAAAUAHAAxLm91dFVUCQADMUfFVjFHxVZ1eAsAAQToAwAABOgDAAA1ClBLAQ" +
"IeAwoAAAAAAOWiUUjRnmdVAgAAAAIAAAAFABgAAAAAAAEAAAC0gQAAAAAwLm91dFVUBQ" +
"ADLUfFVnV4CwABBOgDAAAE6AMAAFBLAQIeAwoAAAAAAOeiUUhXOT0DAgAAAAIAAAAFAB" +
"gAAAAAAAEAAAC0gUEAAAAxLm91dFVUBQADMUfFVnV4CwABBOgDAAAE6AMAAFBLBQYAAA" +
"AAAgACAJYAAACCAAAAAAA=",
"AC",
1.0,
expectedResult{"", "", &RunMetadata{Verdict: "OK"}},
map[string]expectedResult{
"0": {"3", "", &RunMetadata{Verdict: "OK"}},
"1": {"5", "", &RunMetadata{Verdict: "OK"}},
},
},
}
for idx, rte := range runtests {
results, err := Grade(
ctx,
&bytes.Buffer{},
&common.Run{
AttemptID: uint64(idx),
Language: rte.language,
InputHash: input.Hash(),
Source: rte.source,
MaxScore: 1.0,
},
input,
wrapper.sandbox(&rte),
)
if err != nil {
t.Errorf("Failed to run %v: %q", rte, err)
continue
}
if results.Verdict != rte.expectedVerdict {
t.Errorf(
"results.Verdict = %q, expected %q, test %v",
results.Verdict,
rte.expectedVerdict,
idx,
)
}
if results.Score != rte.expectedScore {
t.Errorf(
"results.Score = %v, expected %v",
results.Score,
rte.expectedScore,
)
}
}
}