func testBlock(t *testing.T, apiServer persist.APIServer) {
	jobInfo, err := apiServer.CreateJobInfo(context.Background(), &persist.JobInfo{})
	require.NoError(t, err)
	jobID := jobInfo.JobId
	go func() {
		_, err := apiServer.CreateJobOutput(
			context.Background(),
			&persist.JobOutput{
				JobId:        jobID,
				OutputCommit: pfsutil.NewCommit("foo", "bar"),
			})
		require.NoError(t, err)
		_, err = apiServer.CreateJobState(
			context.Background(),
			&persist.JobState{
				JobId: jobID,
				State: pps.JobState_JOB_STATE_SUCCESS,
			})
		require.NoError(t, err)
	}()
	_, err = apiServer.InspectJob(
		context.Background(),
		&pps.InspectJobRequest{
			Job:         &pps.Job{Id: jobID},
			BlockOutput: true,
			BlockState:  true,
		},
	)
	require.NoError(t, err)
}
Example #2
0
func testFailures(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
	repositoryName := TestRepositoryName()

	err := pfsutil.InitRepository(apiClient, repositoryName)
	require.NoError(t, err)

	commit, err := pfsutil.Branch(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, commit)
	newCommitID := commit.Id

	err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/b")
	require.NoError(t, err)
	err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/c")
	require.NoError(t, err)

	doWrites(t, apiClient, repositoryName, newCommitID)

	err = pfsutil.Write(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)

	checkWrites(t, apiClient, repositoryName, newCommitID)

	for server := 0; server < testNumReplicas; server++ {
		cluster.Kill(server)
	}
	cluster.WaitForAvailability()

	checkWrites(t, apiClient, repositoryName, newCommitID)
}
Example #3
0
func testFailures(t *testing.T, apiClient pfs.APIClient, cluster Cluster) {
	repoName := "testFailuresRepo"

	err := pfsutil.CreateRepo(apiClient, repoName)
	require.NoError(t, err)

	commit, err := pfsutil.StartCommit(apiClient, repoName, "")
	require.NoError(t, err)
	require.NotNil(t, commit)
	newCommitID := commit.Id

	err = pfsutil.MakeDirectory(apiClient, repoName, newCommitID, "a/b")
	require.NoError(t, err)
	err = pfsutil.MakeDirectory(apiClient, repoName, newCommitID, "a/c")
	require.NoError(t, err)

	doWrites(t, apiClient, repoName, newCommitID)

	err = pfsutil.FinishCommit(apiClient, repoName, newCommitID)
	require.NoError(t, err)

	checkWrites(t, apiClient, repoName, newCommitID)

	cluster.KillRoleAssigner()
	for server := 0; server < testNumReplicas; server++ {
		cluster.Kill(server)
	}
	cluster.RestartRoleAssigner()
	cluster.WaitForAvailability()

	checkWrites(t, apiClient, repoName, newCommitID)
}
Example #4
0
func TestGetNameToNodeInfo(t *testing.T) {
	pipeline, err := parse.NewParser().ParsePipeline("../parse/testdata/basic")
	require.NoError(t, err)
	nodeInfos, err := getNameToNodeInfo(pipeline.NameToNode)
	require.NoError(t, err)
	require.Equal(t, []string{"bar-node"}, nodeInfos["baz-node-bar-in-bar-out-in"].Parents)
}
Example #5
0
func RunTestWithRethinkAPIServer(t *testing.T, testFunc func(t *testing.T, persistAPIServer persist.APIServer)) {
	apiServer, err := NewTestRethinkAPIServer()
	require.NoError(t, err)
	defer func() {
		require.NoError(t, apiServer.Close())
	}()
	testFunc(t, apiServer)
}
Example #6
0
func runTestRethink(t *testing.T, testFunc func(*testing.T, Client)) {
	client, err := getRethinkSession()
	require.NoError(t, err)
	defer func() {
		require.NoError(t, client.Close())
	}()
	testFunc(t, client)
}
Example #7
0
func testMountBig(t *testing.T, apiClient pfs.APIClient, cluster Cluster) {
	repoName := "testMountBigRepo"

	err := pfsutil.CreateRepo(apiClient, repoName)
	require.NoError(t, err)

	directory := "/compile/testMount"
	mounter := fuse.NewMounter("localhost", apiClient)
	ready := make(chan bool)
	go func() {
		err = mounter.Mount(directory, 0, 1, ready)
		require.NoError(t, err)
	}()
	<-ready

	_, err = os.Stat(filepath.Join(directory, repoName))
	require.NoError(t, err)

	_, err = os.Stat(filepath.Join(directory, repoName, "scratch"))
	require.NoError(t, err)

	commit, err := pfsutil.StartCommit(apiClient, repoName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, commit)
	newCommitID := commit.Id

	bigValue := make([]byte, 1024*1024*300)
	for i := 0; i < 1024*1024*300; i++ {
		bigValue[i] = 'a'
	}

	wg := sync.WaitGroup{}
	for j := 0; j < 5; j++ {
		wg.Add(1)
		go func(j int) {
			defer wg.Done()
			err := ioutil.WriteFile(filepath.Join(directory, repoName, newCommitID, fmt.Sprintf("big%d", j)), bigValue, 0666)
			require.NoError(t, err)
		}(j)
	}
	wg.Wait()

	err = pfsutil.FinishCommit(apiClient, repoName, newCommitID)
	require.NoError(t, err)

	wg = sync.WaitGroup{}
	for j := 0; j < 5; j++ {
		wg.Add(1)
		go func(j int) {
			defer wg.Done()
			data, err := ioutil.ReadFile(filepath.Join(directory, repoName, newCommitID, fmt.Sprintf("big%d", j)))
			require.NoError(t, err)
			require.Equal(t, bigValue, data)
		}(j)
	}
	wg.Wait()

	err = mounter.Unmount(directory)
	require.NoError(t, err)
}
Example #8
0
func TestFromCommit(t *testing.T) {
	t.Parallel()
	repo := uniqueString("TestFromCommit")
	pachClient := getPachClient(t)
	seed := time.Now().UnixNano()
	rand := rand.New(rand.NewSource(seed))
	err := pfsutil.CreateRepo(pachClient, repo)
	require.NoError(t, err)
	commit1, err := pfsutil.StartCommit(pachClient, repo, "")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pachClient, repo, commit1.Id, "file", 0, workload.NewReader(rand, KB))
	require.NoError(t, err)
	err = pfsutil.FinishCommit(pachClient, repo, commit1.Id)
	require.NoError(t, err)
	commit2, err := pfsutil.StartCommit(pachClient, repo, commit1.Id)
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pachClient, repo, commit2.Id, "file", 0, workload.NewReader(rand, KB))
	require.NoError(t, err)
	err = pfsutil.FinishCommit(pachClient, repo, commit2.Id)
	require.NoError(t, err)
	var buffer bytes.Buffer
	require.NoError(t, pfsutil.GetFile(pachClient, repo, commit2.Id, "file", 0, 0, commit1.Id, nil, &buffer))
	require.Equal(t, buffer.Len(), KB)
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pachClient, repo, commit2.Id, "file", 0, 0, "", nil, &buffer))
	require.Equal(t, buffer.Len(), 2*KB)
}
Example #9
0
func handleEndJobStatusOutputStream(t *testing.T, jobAPIClient pps.JobAPIClient, jobInfo *pps.JobInfo, outputStream pps.OutputStream) {
	jobAPIGetJobLogsClient, err := jobAPIClient.GetJobLogs(
		context.Background(),
		&pps.GetJobLogsRequest{
			Job:          jobInfo.Job,
			OutputStream: outputStream,
		},
	)
	require.NoError(t, err)
	require.NoError(t, protostream.WriteFromStreamingBytesClient(jobAPIGetJobLogsClient, protolog.Writer()))
}
Example #10
0
func TestGrep(t *testing.T) {
	t.Skip()
	t.Parallel()
	dataRepo := uniqueString("TestGrep.data")
	pfsClient := getPfsClient(t)
	require.NoError(t, pfsutil.CreateRepo(pfsClient, dataRepo))
	commit, err := pfsutil.StartCommit(pfsClient, dataRepo, "")
	require.NoError(t, err)
	for i := 0; i < 100; i++ {
		_, err = pfsutil.PutFile(pfsClient, dataRepo, commit.Id, fmt.Sprintf("file%d", i), 0, strings.NewReader("foo\nbar\nfizz\nbuzz\n"))
		require.NoError(t, err)
	}
	require.NoError(t, pfsutil.FinishCommit(pfsClient, dataRepo, commit.Id))
	ppsClient := getPpsClient(t)
	job1, err := ppsutil.CreateJob(
		ppsClient,
		"",
		[]string{"bash"},
		fmt.Sprintf("grep foo /pfs/%s/* >/pfs/out/foo", dataRepo),
		1,
		[]*pps.JobInput{{Commit: commit}},
		"",
	)
	require.NoError(t, err)
	job2, err := ppsutil.CreateJob(
		ppsClient,
		"",
		[]string{"bash"},
		fmt.Sprintf("grep foo /pfs/%s/* >/pfs/out/foo", dataRepo),
		4,
		[]*pps.JobInput{{Commit: commit}},
		"",
	)
	require.NoError(t, err)
	inspectJobRequest := &pps.InspectJobRequest{
		Job:         job1,
		BlockOutput: true,
		BlockState:  true,
	}
	job1Info, err := ppsClient.InspectJob(context.Background(), inspectJobRequest)
	require.NoError(t, err)
	inspectJobRequest.Job = job2
	job2Info, err := ppsClient.InspectJob(context.Background(), inspectJobRequest)
	require.NoError(t, err)
	repo1Info, err := pfsutil.InspectRepo(pfsClient, job1Info.OutputCommit.Repo.Name)
	require.NoError(t, err)
	repo2Info, err := pfsutil.InspectRepo(pfsClient, job2Info.OutputCommit.Repo.Name)
	require.NoError(t, err)
	require.Equal(t, repo1Info.SizeBytes, repo2Info.SizeBytes)
}
Example #11
0
func TestJob(t *testing.T) {
	t.Parallel()
	dataRepo := uniqueString("TestJob.data")
	pachClient := getPachClient(t)
	require.NoError(t, pfsutil.CreateRepo(pachClient, dataRepo))
	commit, err := pfsutil.StartCommit(pachClient, dataRepo, "")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pachClient, dataRepo, commit.Id, "file", 0, strings.NewReader("foo\n"))
	require.NoError(t, err)
	require.NoError(t, pfsutil.FinishCommit(pachClient, dataRepo, commit.Id))
	job, err := ppsutil.CreateJob(
		pachClient,
		"",
		[]string{"cp", path.Join("/pfs", dataRepo, "file"), "/pfs/out/file"},
		nil,
		1,
		[]*pps.JobInput{{Commit: commit}},
		"",
	)
	require.NoError(t, err)
	inspectJobRequest := &pps.InspectJobRequest{
		Job:         job,
		BlockOutput: true,
		BlockState:  true,
	}
	jobInfo, err := pachClient.InspectJob(context.Background(), inspectJobRequest)
	require.NoError(t, err)
	require.Equal(t, pps.JobState_JOB_STATE_SUCCESS.String(), jobInfo.State.String())
	commitInfo, err := pfsutil.InspectCommit(pachClient, jobInfo.OutputCommit.Repo.Name, jobInfo.OutputCommit.Id)
	require.NoError(t, err)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, commitInfo.CommitType)
	var buffer bytes.Buffer
	require.NoError(t, pfsutil.GetFile(pachClient, jobInfo.OutputCommit.Repo.Name, jobInfo.OutputCommit.Id, "file", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
}
Example #12
0
func testMountBig(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
	repositoryName := TestRepositoryName()

	err := pfsutil.InitRepository(apiClient, repositoryName)
	require.NoError(t, err)

	directory := "/compile/testMount"
	mounter := fuse.NewMounter(apiClient)
	err = mounter.Mount(repositoryName, "", directory, 0, 1)
	require.NoError(t, err)

	_, err = os.Stat(filepath.Join(directory, "scratch"))
	require.NoError(t, err)

	commit, err := pfsutil.Branch(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, commit)
	newCommitID := commit.Id

	bigValue := make([]byte, 1024*1024*300)
	for i := 0; i < 1024*1024*300; i++ {
		bigValue[i] = 'a'
	}

	wg := sync.WaitGroup{}
	for j := 0; j < 5; j++ {
		wg.Add(1)
		go func(j int) {
			defer wg.Done()
			err := ioutil.WriteFile(filepath.Join(directory, newCommitID, fmt.Sprintf("big%d", j)), bigValue, 0666)
			require.NoError(t, err)
		}(j)
	}
	wg.Wait()

	err = pfsutil.Write(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)

	wg = sync.WaitGroup{}
	for j := 0; j < 5; j++ {
		wg.Add(1)
		go func(j int) {
			defer wg.Done()
			data, err := ioutil.ReadFile(filepath.Join(directory, newCommitID, fmt.Sprintf("big%d", j)))
			require.NoError(t, err)
			require.Equal(t, bigValue, data)
		}(j)
	}
	wg.Wait()

	err = mounter.Unmount(directory)
	require.NoError(t, err)
	err = mounter.Wait(directory)
	require.NoError(t, err)
}
Example #13
0
func TestMountBig(t *testing.T) {
	t.Skip()
	t.Parallel()
	apiClient := getPfsClient(t)
	repoName := uniqueString("testMountBigRepo")

	err := pfsutil.CreateRepo(apiClient, repoName)
	require.NoError(t, err)

	directory := "/compile/testMount"
	mounter := fuse.NewMounter("localhost", apiClient)
	ready := make(chan bool)
	go func() {
		err = mounter.Mount(directory, &pfs.Shard{Number: 0, Modulus: 1}, nil, ready)
		require.NoError(t, err)
	}()
	<-ready

	_, err = os.Stat(filepath.Join(directory, repoName))
	require.NoError(t, err)

	commit, err := pfsutil.StartCommit(apiClient, repoName, "")
	require.NoError(t, err)
	require.NotNil(t, commit)
	newCommitID := commit.Id

	bigValue := make([]byte, 1024*1024*300)
	for i := 0; i < 1024*1024*300; i++ {
		bigValue[i] = 'a'
	}

	wg := sync.WaitGroup{}
	for j := 0; j < 5; j++ {
		wg.Add(1)
		go func(j int) {
			defer wg.Done()
			err := ioutil.WriteFile(filepath.Join(directory, repoName, newCommitID, fmt.Sprintf("big%d", j)), bigValue, 0666)
			require.NoError(t, err)
		}(j)
	}
	wg.Wait()

	err = pfsutil.FinishCommit(apiClient, repoName, newCommitID)
	require.NoError(t, err)

	wg = sync.WaitGroup{}
	for j := 0; j < 5; j++ {
		wg.Add(1)
		go func(j int) {
			defer wg.Done()
			data, err := ioutil.ReadFile(filepath.Join(directory, repoName, newCommitID, fmt.Sprintf("big%d", j)))
			require.NoError(t, err)
			require.Equal(t, bigValue, data)
		}(j)
	}
	wg.Wait()

	err = mounter.Unmount(directory)
	require.NoError(t, err)
}
Example #14
0
func waitForJob(t *testing.T, jobAPIClient pps.JobAPIClient, job *pps.Job, timeoutSec int, expectError bool) *pps.JobInfo {
	for i := 0; i < timeoutSec; i++ {
		time.Sleep(1 * time.Second)
		jobInfo, err := jobAPIClient.InspectJob(
			context.Background(),
			&pps.InspectJobRequest{
				Job: job,
			},
		)
		require.NoError(t, err)
		if len(jobInfo.JobStatus) == 0 {
			continue
		}
		jobStatus := jobInfo.JobStatus[0]
		protolog.Infof("status of job %s at %d seconds: %v", job.Id, i+1, jobInfo.JobStatus)
		switch jobStatus.Type {
		case pps.JobStatusType_JOB_STATUS_TYPE_ERROR:
			handleEndJobStatus(t, jobAPIClient, jobInfo)
			if !expectError {
				t.Fatalf("job %s had error", job.Id)
			}
			return jobInfo
		case pps.JobStatusType_JOB_STATUS_TYPE_SUCCESS:
			handleEndJobStatus(t, jobAPIClient, jobInfo)
			if expectError {
				t.Fatalf("job %s did not have error", job.Id)
			}
			return jobInfo
		}
	}
	t.Fatalf("job %s did not finish in %d seconds", job.Id, timeoutSec)
	return nil
}
Example #15
0
func RunTest(
	t *testing.T,
	f func(*testing.T, pfs.ApiClient, pfs.InternalApiClient, Cluster),
) {
	discoveryClient, err := getEtcdClient()
	require.NoError(t, err)
	var cluster Cluster
	prototest.RunT(
		t,
		testNumServers,
		func(servers map[string]*grpc.Server) {
			cluster = registerFunc(t, discoveryClient, servers)
		},
		func(t *testing.T, clientConns map[string]*grpc.ClientConn) {
			var clientConn *grpc.ClientConn
			for _, c := range clientConns {
				clientConn = c
				break
			}
			f(
				t,
				pfs.NewApiClient(
					clientConn,
				),
				pfs.NewInternalApiClient(
					clientConn,
				),
				cluster,
			)
		},
	)
	cluster.Shutdown()
}
Example #16
0
func checkPFSOutput(t *testing.T, pfsAPIClient pfs.APIClient, outputCommit *pfs.Commit, filePathToContent map[string][]byte) {
	for filePath, content := range filePathToContent {
		getContent, err := getPFSContent(pfsAPIClient, outputCommit, filePath)
		require.NoError(t, err)
		require.Equal(t, content, getContent)
	}
}
Example #17
0
func TestWorkload(t *testing.T) {
	t.Parallel()
	pfsClient := getPfsClient(t)
	ppsClient := getPpsClient(t)
	seed := time.Now().UnixNano()
	require.NoError(t, workload.RunWorkload(pfsClient, ppsClient, rand.New(rand.NewSource(seed)), 100))
}
Example #18
0
func RunBench(
	b *testing.B,
	f func(*testing.B, pfs.ApiClient),
) {
	discoveryClient, err := getEtcdClient()
	require.NoError(b, err)
	var cluster Cluster
	prototest.RunB(
		b,
		testNumServers,
		func(servers map[string]*grpc.Server) {
			cluster = registerFunc(b, discoveryClient, servers)
		},
		func(b *testing.B, clientConns map[string]*grpc.ClientConn) {
			var clientConn *grpc.ClientConn
			for _, c := range clientConns {
				clientConn = c
				break
			}
			f(
				b,
				pfs.NewApiClient(
					clientConn,
				),
			)
		},
	)
	cluster.Shutdown()
}
Example #19
0
func RunBench(
	b *testing.B,
	f func(*testing.B, pfs.APIClient),
) {
	discoveryClient, err := getEtcdClient()
	require.NoError(b, err)
	var cluster *cluster
	prototest.RunB(
		b,
		testNumServers,
		func(servers map[string]*grpc.Server) {
			cluster = registerFunc(b, discoveryClient, servers)
		},
		func(b *testing.B, clientConns map[string]*grpc.ClientConn) {
			var clientConn *grpc.ClientConn
			for _, c := range clientConns {
				clientConn = c
				break
			}
			go func() {
				require.Equal(b, cluster.realSharder.AssignRoles(cluster.cancel), shard.ErrCancelled)
			}()
			cluster.WaitForAvailability()
			f(
				b,
				pfs.NewAPIClient(
					clientConn,
				),
			)
		},
	)
	cluster.Shutdown()
}
Example #20
0
func doWrites(tb testing.TB, apiClient pfs.APIClient, repoName string, commitID string) {
	var wg sync.WaitGroup
	defer wg.Wait()
	for i := 0; i < testSize; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			_, iErr := pfsutil.PutFile(apiClient, repoName, commitID,
				fmt.Sprintf("a/b/file%d", i), 0, strings.NewReader(fmt.Sprintf("hello%d", i)))
			require.NoError(tb, iErr)
			_, iErr = pfsutil.PutFile(apiClient, repoName, commitID,
				fmt.Sprintf("a/c/file%d", i), 0, strings.NewReader(fmt.Sprintf("hello%d", i)))
			require.NoError(tb, iErr)
		}()
	}
}
Example #21
0
func TestJob(t *testing.T) {
	dataRepo := uniqueString("TestJob.data")
	pfsClient := getPfsClient(t)
	require.NoError(t, pfsutil.CreateRepo(pfsClient, dataRepo))
	commit, err := pfsutil.StartCommit(pfsClient, dataRepo, "")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pfsClient, dataRepo, commit.Id, "file", 0, strings.NewReader("foo"))
	require.NoError(t, err)
	require.NoError(t, pfsutil.FinishCommit(pfsClient, dataRepo, commit.Id))
	ppsClient := getPpsClient(t)
	job, err := ppsutil.CreateJob(
		ppsClient,
		"",
		[]string{"cp", path.Join("/pfs", dataRepo, "file"), "/pfs/out/file"},
		"",
		1,
		[]*pfs.Commit{commit},
		"",
	)
	require.NoError(t, err)
	inspectJobRequest := &pps.InspectJobRequest{
		Job:         job,
		BlockOutput: true,
		BlockState:  true,
	}
	jobInfo, err := ppsClient.InspectJob(context.Background(), inspectJobRequest)
	require.NoError(t, err)
	var buffer bytes.Buffer
	require.NoError(t, pfsutil.GetFile(pfsClient, jobInfo.OutputCommit.Repo.Name, jobInfo.OutputCommit.Id, "file", 0, 0, nil, &buffer))
	require.Equal(t, "foo", buffer.String())
}
Example #22
0
func testRun(t *testing.T, imageName string, commands []string, expectedStdout string, expectedStderr string) {
	client, err := newTestDockerClient()
	require.NoError(t, err)
	err = client.Pull(imageName, PullOptions{})
	require.NoError(t, err)
	containers, err := client.Create(
		imageName,
		CreateOptions{
			HasCommand: commands != nil,
		},
	)
	require.NoError(t, err)
	require.Equal(t, 1, len(containers))
	container := containers[0]
	err = client.Start(
		container,
		StartOptions{
			Commands: commands,
		},
	)
	require.NoError(t, err)
	err = client.Wait(container, WaitOptions{})
	require.NoError(t, err)
	stdout := bytes.NewBuffer(nil)
	stderr := bytes.NewBuffer(nil)
	err = client.Logs(container, LogsOptions{Stdout: stdout, Stderr: stderr})
	require.NoError(t, err)
	require.Equal(t, expectedStdout, stdout.String())
	require.Equal(t, expectedStderr, stderr.String())
	err = client.Remove(container, RemoveOptions{})
	require.NoError(t, err)
}
Example #23
0
func getPfsClient(tb testing.TB) pfs.APIClient {
	pfsdAddr := os.Getenv("PFSD_PORT_650_TCP_ADDR")
	if pfsdAddr == "" {
		tb.Error("PFSD_PORT_650_TCP_ADDR not set")
	}
	clientConn, err := grpc.Dial(fmt.Sprintf("%s:650", pfsdAddr), grpc.WithInsecure())
	require.NoError(tb, err)
	return pfs.NewAPIClient(clientConn)
}
Example #24
0
func TestBigWrite(t *testing.T) {
	t.Parallel()
	protolog.SetLevel(protolog.Level_LEVEL_DEBUG)
	repo := uniqueString("TestBigWrite")
	pfsClient := getPfsClient(t)
	err := pfsutil.CreateRepo(pfsClient, repo)
	require.NoError(t, err)
	commit, err := pfsutil.StartCommit(pfsClient, repo, "")
	require.NoError(t, err)
	rand := rand.New(rand.NewSource(5))
	_, err = pfsutil.PutFile(pfsClient, repo, commit.Id, "file", 0, workload.NewReader(rand, 10000))
	require.NoError(t, err)
	err = pfsutil.FinishCommit(pfsClient, repo, commit.Id)
	require.NoError(t, err)
	var buffer bytes.Buffer
	err = pfsutil.GetFile(pfsClient, repo, commit.Id, "file", 0, 0, nil, &buffer)
	require.NoError(t, err)
}
Example #25
0
func getPachClient(t *testing.T) *APIClient {
	pachAddr := os.Getenv("PACHD_PORT_650_TCP_ADDR")
	if pachAddr == "" {
		t.Error("PACHD_PORT_650_TCP_ADDR not set")
	}
	clientConn, err := grpc.Dial(fmt.Sprintf("%s:650", pachAddr), grpc.WithInsecure())
	require.NoError(t, err)
	return NewAPIClient(clientConn)
}
Example #26
0
func getPpsClient(t *testing.T) pps.APIClient {
	ppsdAddr := os.Getenv("PPSD_PORT_651_TCP_ADDR")
	if ppsdAddr == "" {
		t.Error("PPSD_PORT_651_TCP_ADDR not set")
	}
	clientConn, err := grpc.Dial(fmt.Sprintf("%s:651", ppsdAddr), grpc.WithInsecure())
	require.NoError(t, err)
	return pps.NewAPIClient(clientConn)
}
Example #27
0
func (c *cluster) WaitForAvailability() {
	// We use address as the id for servers too
	var ids []string
	for _, address := range c.addresses {
		if _, ok := c.cancels[address]; ok {
			ids = append(ids, address)
		}
	}
	require.NoError(c.tb, c.addresser.WaitForAvailability(ids))
}
Example #28
0
func BenchmarkFuse(b *testing.B) {
	apiClient := getPfsClient(b)
	repoName := uniqueString("benchMountRepo")

	if err := pfsutil.CreateRepo(apiClient, repoName); err != nil {
		b.Error(err)
	}

	directory := "/compile/benchMount"
	mounter := fuse.NewMounter("localhost", apiClient)
	ready := make(chan bool)
	go func() {
		err := mounter.Mount(directory, &pfs.Shard{Number: 0, Modulus: 1}, nil, ready)
		require.NoError(b, err)
	}()
	<-ready

	defer func() {
		if err := mounter.Unmount(directory); err != nil {
			b.Error(err)
		}
	}()

	bigValue := make([]byte, 1024*1024)
	for i := 0; i < 1024*1024; i++ {
		bigValue[i] = 'a'
	}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		commit, err := pfsutil.StartCommit(apiClient, repoName, "")
		if err != nil {
			b.Error(err)
		}
		if commit == nil {
			b.Error("nil branch")
		}
		newCommitID := commit.Id
		var wg sync.WaitGroup
		for j := 0; j < 1024; j++ {
			wg.Add(1)
			go func(j int) {
				defer wg.Done()
				if err = ioutil.WriteFile(filepath.Join(directory, repoName, newCommitID, fmt.Sprintf("big%d", j)), bigValue, 0666); err != nil {
					b.Error(err)
				}
			}(j)
		}
		wg.Wait()
		if err := pfsutil.FinishCommit(apiClient, repoName, newCommitID); err != nil {
			b.Error(err)
		}
	}
}
func testBasicRethink(t *testing.T, apiServer persist.APIServer) {
	jobInfo, err := apiServer.CreateJobInfo(
		context.Background(),
		&persist.JobInfo{
			Spec: &persist.JobInfo_PipelineName{
				PipelineName: "foo",
			},
		},
	)
	require.NoError(t, err)
	getJobInfo, err := apiServer.GetJobInfo(
		context.Background(),
		&pps.Job{
			Id: jobInfo.JobId,
		},
	)
	require.NoError(t, err)
	require.Equal(t, jobInfo.JobId, getJobInfo.JobId)
	require.Equal(t, "foo", getJobInfo.GetPipelineName())
}
Example #30
0
func getJobForPipeline(t *testing.T, jobAPIClient pps.JobAPIClient, pipeline *pps.Pipeline) *pps.Job {
	jobInfos, err := jobAPIClient.ListJob(
		context.Background(),
		&pps.ListJobRequest{
			Pipeline: pipeline,
		},
	)
	require.NoError(t, err)
	require.NotNil(t, jobInfos)
	require.Equal(t, 1, len(jobInfos.JobInfo))
	return jobInfos.JobInfo[0].Job
}