Esempio n. 1
0
func testFailures(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
	repositoryName := "testFailuresRepo"

	err := pfsutil.CreateRepo(apiClient, repositoryName)
	require.NoError(t, err)

	commit, err := pfsutil.StartCommit(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, commit)
	newCommitID := commit.Id

	err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/b")
	require.NoError(t, err)
	err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/c")
	require.NoError(t, err)

	doWrites(t, apiClient, repositoryName, newCommitID)

	err = pfsutil.FinishCommit(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)

	checkWrites(t, apiClient, repositoryName, newCommitID)

	cluster.KillRoleAssigner()
	for server := 0; server < testNumReplicas; server++ {
		cluster.Kill(server)
	}
	cluster.RestartRoleAssigner()
	cluster.WaitForAvailability()

	checkWrites(t, apiClient, repositoryName, newCommitID)
}
Esempio n. 2
0
func TestGetNameToNodeInfo(t *testing.T) {
	pipeline, err := parse.NewParser().ParsePipeline("../parse/testdata/basic")
	require.NoError(t, err)
	nodeInfos, err := getNameToNodeInfo(pipeline.NameToNode)
	require.NoError(t, err)
	require.Equal(t, []string{"bar-node"}, nodeInfos["baz-node-bar-in-bar-out-in"].Parents)
}
Esempio n. 3
0
func runTestRethink(t *testing.T, testFunc func(*testing.T, Client)) {
	client, err := getRethinkSession()
	require.NoError(t, err)
	defer func() {
		require.NoError(t, client.Close())
	}()
	testFunc(t, client)
}
Esempio n. 4
0
func testMountBig(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
	repositoryName := "testMountBigRepo"

	err := pfsutil.CreateRepo(apiClient, repositoryName)
	require.NoError(t, err)

	directory := "/compile/testMount"
	mounter := fuse.NewMounter(apiClient)
	err = mounter.Mount(repositoryName, "", directory, 0, 1)
	require.NoError(t, err)

	_, err = os.Stat(filepath.Join(directory, "scratch"))
	require.NoError(t, err)

	commit, err := pfsutil.StartCommit(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, commit)
	newCommitID := commit.Id

	bigValue := make([]byte, 1024*1024*300)
	for i := 0; i < 1024*1024*300; i++ {
		bigValue[i] = 'a'
	}

	wg := sync.WaitGroup{}
	for j := 0; j < 5; j++ {
		wg.Add(1)
		go func(j int) {
			defer wg.Done()
			err := ioutil.WriteFile(filepath.Join(directory, newCommitID, fmt.Sprintf("big%d", j)), bigValue, 0666)
			require.NoError(t, err)
		}(j)
	}
	wg.Wait()

	err = pfsutil.FinishCommit(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)

	wg = sync.WaitGroup{}
	for j := 0; j < 5; j++ {
		wg.Add(1)
		go func(j int) {
			defer wg.Done()
			data, err := ioutil.ReadFile(filepath.Join(directory, newCommitID, fmt.Sprintf("big%d", j)))
			require.NoError(t, err)
			require.Equal(t, bigValue, data)
		}(j)
	}
	wg.Wait()

	err = mounter.Unmount(directory)
	require.NoError(t, err)
	err = mounter.Wait(directory)
	require.NoError(t, err)
}
Esempio n. 5
0
func doWrites(tb testing.TB, apiClient pfs.ApiClient, repositoryName string, commitID string) {
	var wg sync.WaitGroup
	defer wg.Wait()
	for i := 0; i < testSize; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			_, iErr := pfsutil.PutFile(apiClient, repositoryName, commitID,
				fmt.Sprintf("a/b/file%d", i), 0, strings.NewReader(fmt.Sprintf("hello%d", i)))
			require.NoError(tb, iErr)
			_, iErr = pfsutil.PutFile(apiClient, repositoryName, commitID,
				fmt.Sprintf("a/c/file%d", i), 0, strings.NewReader(fmt.Sprintf("hello%d", i)))
			require.NoError(tb, iErr)
		}()
	}
}
Esempio n. 6
0
func testRun(t *testing.T, imageName string, commands []string, expectedStdout string, expectedStderr string) {
	client, err := newTestDockerClient()
	require.NoError(t, err)
	err = client.Pull(imageName, PullOptions{})
	require.NoError(t, err)
	containers, err := client.Create(
		imageName,
		CreateOptions{
			HasCommand: commands != nil,
		},
	)
	require.NoError(t, err)
	require.Equal(t, 1, len(containers))
	container := containers[0]
	err = client.Start(
		container,
		StartOptions{
			Commands: commands,
		},
	)
	require.NoError(t, err)
	err = client.Wait(container, WaitOptions{})
	require.NoError(t, err)
	stdout := bytes.NewBuffer(nil)
	stderr := bytes.NewBuffer(nil)
	err = client.Logs(container, LogsOptions{Stdout: stdout, Stderr: stderr})
	require.NoError(t, err)
	require.Equal(t, expectedStdout, stdout.String())
	require.Equal(t, expectedStderr, stderr.String())
	err = client.Remove(container, RemoveOptions{})
	require.NoError(t, err)
}
Esempio n. 7
0
func TestBasic(t *testing.T) {
	dirPath, err := ioutil.TempDir("", "pachyderm")
	require.NoError(t, err)
	err = GithubClone(
		dirPath,
		"pachyderm",
		"pachyderm",
		"master",
		"11eb4c6e1945beb1e6ce3e878ed2cb6c24ee8bf1",
		"",
	)
	require.NoError(t, err)
	//data, err := ioutil.ReadFile(filepath.Join(dirPath, "README.md"))
	//require.NoError(t, err)
	//require.Equal(
	//t,
	//`pfs
	//===

	//The Pachyderm Filesystem`,
	//string(data),
	//)
}
Esempio n. 8
0
func checkWrites(tb testing.TB, apiClient pfs.ApiClient, repositoryName string, commitID string) {
	var wg sync.WaitGroup
	defer wg.Wait()
	for i := 0; i < testSize; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			buffer := bytes.NewBuffer(nil)
			iErr := pfsutil.GetFile(apiClient, repositoryName, commitID,
				fmt.Sprintf("a/b/file%d", i), 0, math.MaxInt64, buffer)
			require.NoError(tb, iErr)
			require.Equal(tb, fmt.Sprintf("hello%d", i), buffer.String())

			buffer = bytes.NewBuffer(nil)
			iErr = pfsutil.GetFile(apiClient, repositoryName, commitID,
				fmt.Sprintf("a/c/file%d", i), 0, math.MaxInt64, buffer)
			require.NoError(tb, iErr)
			require.Equal(tb, fmt.Sprintf("hello%d", i), buffer.String())

		}()
	}
}
Esempio n. 9
0
func runTest(
	t *testing.T,
	f func(t *testing.T, apiClient pps.ApiClient),
) {
	containerClient, err := getContainerClient()
	require.NoError(t, err)
	storeClient, err := getRethinkClient()
	require.NoError(t, err)
	pfstesting.RunTest(
		t,
		func(t *testing.T, apiClient pfs.ApiClient, internalApiClient pfs.InternalApiClient, cluster pfstesting.Cluster) {
			prototest.RunT(
				t,
				testNumServers,
				func(servers map[string]*grpc.Server) {
					for _, server := range servers {
						pps.RegisterApiServer(server, newAPIServer(apiClient, containerClient, storeClient, timing.NewSystemTimer()))
					}
				},
				func(t *testing.T, clientConns map[string]*grpc.ClientConn) {
					var clientConn *grpc.ClientConn
					for _, c := range clientConns {
						clientConn = c
						break
					}
					f(
						t,
						pps.NewApiClient(
							clientConn,
						),
					)
				},
			)
		},
	)
}
Esempio n. 10
0
func testBasic(t *testing.T, client Client) {
	pipelineRun := &pps.PipelineRun{
		Id:         "id",
		PipelineId: "pipeline_id",
	}
	require.NoError(t, client.CreatePipelineRun(pipelineRun))
	pipelineRunResponse, err := client.GetPipelineRun("id")
	require.NoError(t, err)
	require.Equal(t, pipelineRun, pipelineRunResponse)
	require.NoError(t, client.CreatePipelineRunStatus("id", pps.PipelineRunStatusType_PIPELINE_RUN_STATUS_TYPE_CREATED))
	pipelineRunStatusResponse, err := client.GetAllPipelineRunStatuses("id")
	require.NoError(t, err)
	require.Equal(t, pipelineRunStatusResponse[0].PipelineRunStatusType, pps.PipelineRunStatusType_PIPELINE_RUN_STATUS_TYPE_CREATED)
	require.NoError(t, client.CreatePipelineRunStatus("id", pps.PipelineRunStatusType_PIPELINE_RUN_STATUS_TYPE_SUCCESS))
	pipelineRunStatusResponse, err = client.GetAllPipelineRunStatuses("id")
	require.NoError(t, err)
	require.Equal(t, pipelineRunStatusResponse[0].PipelineRunStatusType, pps.PipelineRunStatusType_PIPELINE_RUN_STATUS_TYPE_SUCCESS)

	require.NoError(t, client.CreatePipelineRunContainers(&pps.PipelineRunContainer{PipelineRunId: "id", ContainerId: "container", Node: "node"}))
	containerIDs, err := client.GetPipelineRunContainers("id")
	require.NoError(t, err)
	require.Equal(t, []*pps.PipelineRunContainer{&pps.PipelineRunContainer{PipelineRunId: "id", ContainerId: "container", Node: "node"}}, containerIDs)
}
Esempio n. 11
0
func checkBlockWrites(tb testing.TB, apiClient pfs.ApiClient, repositoryName string, commitID string) {
	var wg sync.WaitGroup
	defer wg.Wait()
	for i := 0; i < testSize; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			buffer := bytes.NewBuffer(nil)
			sharder := route.NewSharder(testShardsPerServer*testNumServers, testNumReplicas)
			block := sharder.GetBlock([]byte(fmt.Sprintf("hello%d", i)))
			iErr := pfsutil.GetBlock(apiClient, block.Hash, buffer)
			require.NoError(tb, iErr)

			// buffer = bytes.NewBuffer(nil)
			// require.Equal(tb, fmt.Sprintf("hello%d", i), buffer.String())
			// iErr = pfsutil.GetFile(apiClient, repositoryName, commitID,
			// 	fmt.Sprintf("a/d/file%d", i), 0, math.MaxInt64, buffer)
			// require.NoError(tb, iErr)
			// require.Equal(tb, fmt.Sprintf("hello%d", i), buffer.String())
		}()
	}
}
Esempio n. 12
0
func runTest(t *testing.T, client Client) {
	err := client.Set("foo", "one", 0)
	require.NoError(t, err)
	value, err := client.Get("foo")
	require.NoError(t, err)
	require.Equal(t, "one", value)
	//values, err := client.GetAll("foo")
	//require.NoError(t, err)
	//require.Equal(t, map[string]string{"foo": "one"}, values)

	err = client.Set("a/b/foo", "one", 0)
	require.NoError(t, err)
	err = client.Set("a/b/bar", "two", 0)
	require.NoError(t, err)
	values, err := client.GetAll("a/b")
	require.NoError(t, err)
	require.Equal(t, map[string]string{"a/b/foo": "one", "a/b/bar": "two"}, values)

	require.NoError(t, client.Close())
}
Esempio n. 13
0
func TestSimple(t *testing.T) {
	driver1, err := NewDriver(getBtrfsRootDir(t), "drive.TestSimple")
	require.NoError(t, err)
	shards := make(map[uint64]bool)
	shards[0] = true
	repo := &pfs.Repo{Name: "repo1"}
	require.NoError(t, driver1.CreateRepo(repo))
	commit1 := &pfs.Commit{
		Repo: repo,
		Id:   "commit1",
	}
	require.NoError(t, driver1.StartCommit(nil, commit1, shards))
	file1 := &pfs.File{
		Commit: commit1,
		Path:   "foo",
	}
	require.NoError(t, driver1.PutFile(file1, 0, 0, strings.NewReader("foo")))
	require.NoError(t, driver1.FinishCommit(commit1, shards))
	reader, err := driver1.GetFile(file1, 0)
	require.NoError(t, err)
	contents, err := ioutil.ReadAll(reader)
	require.NoError(t, err)
	require.Equal(t, string(contents), "foo")
	commit2 := &pfs.Commit{
		Repo: repo,
		Id:   "commit2",
	}
	require.NoError(t, driver1.StartCommit(commit1, commit2, shards))
	file2 := &pfs.File{
		Commit: commit2,
		Path:   "bar",
	}
	require.NoError(t, driver1.PutFile(file2, 0, 0, strings.NewReader("bar")))
	require.NoError(t, driver1.FinishCommit(commit2, shards))
	changes, err := driver1.ListChange(file2, commit1, 0)
	require.NoError(t, err)
	require.Equal(t, len(changes), 1)
	require.Equal(t, changes[0].File, file2)
	require.Equal(t, changes[0].OffsetBytes, uint64(0))
	require.Equal(t, changes[0].SizeBytes, uint64(3))
	//Replicate repo
	driver2, err := NewDriver(getBtrfsRootDir(t), "drive.TestSimpleReplica")
	require.NoError(t, err)
	require.NoError(t, driver2.CreateRepo(repo))
	var buffer bytes.Buffer
	require.NoError(t, driver1.PullDiff(commit1, 0, &buffer))
	require.NoError(t, driver2.PushDiff(commit1, 0, &buffer))
	buffer = bytes.Buffer{}
	require.NoError(t, driver1.PullDiff(commit2, 0, &buffer))
	require.NoError(t, driver2.PushDiff(commit2, 0, &buffer))
	reader, err = driver2.GetFile(file1, 0)
	require.NoError(t, err)
	contents, err = ioutil.ReadAll(reader)
	require.NoError(t, err)
	require.Equal(t, string(contents), "foo")
	changes, err = driver2.ListChange(file2, commit1, 0)
	require.NoError(t, err)
	require.Equal(t, len(changes), 1)
	require.Equal(t, changes[0].File, file2)
	require.Equal(t, changes[0].OffsetBytes, uint64(0))
	require.Equal(t, changes[0].SizeBytes, uint64(3))
}
Esempio n. 14
0
func TestMasterReplica(t *testing.T) {
	client, err := getEtcdClient()
	require.NoError(t, err)
	runMasterReplicaTest(t, client)
}
Esempio n. 15
0
func TestEtcdWatch(t *testing.T) {
	t.Parallel()
	client, err := getEtcdClient()
	require.NoError(t, err)
	runWatchTest(t, client)
}
Esempio n. 16
0
func TestBasic(t *testing.T) {
	pipeline, err := NewParser().ParsePipeline("testdata/basic")
	require.NoError(t, err)
	fmt.Println(pipeline)
}
Esempio n. 17
0
func testBasic(t *testing.T, apiClient pps.ApiClient) {
	_ = os.RemoveAll("/tmp/pachyderm-test")
	pipelineSource, err := apiClient.CreatePipelineSource(
		context.Background(),
		&pps.CreatePipelineSourceRequest{
			PipelineSource: &pps.PipelineSource{
				TypedPipelineSource: &pps.PipelineSource_GithubPipelineSource{
					GithubPipelineSource: &pps.GithubPipelineSource{
						ContextDir: "src/pps/server/testdata/basic",
						User:       "******",
						Repository: "pachyderm",
						Branch:     "master",
					},
				},
			},
		},
	)
	require.NoError(t, err)
	pipeline, err := apiClient.CreateAndGetPipeline(
		context.Background(),
		&pps.CreateAndGetPipelineRequest{
			PipelineSourceId: pipelineSource.Id,
		},
	)
	require.NoError(t, err)
	pipelineRun, err := apiClient.CreatePipelineRun(
		context.Background(),
		&pps.CreatePipelineRunRequest{
			PipelineId: pipeline.Id,
		},
	)
	require.NoError(t, err)
	_, err = apiClient.StartPipelineRun(
		context.Background(),
		&pps.StartPipelineRunRequest{
			PipelineRunId: pipelineRun.Id,
		},
	)
	require.NoError(t, err)
	pipelineRunID := pipelineRun.Id
	pipelineRunStatus, err := getFinalPipelineRunStatus(apiClient, pipelineRunID)
	require.NoError(t, err)
	require.Equal(t, pps.PipelineRunStatusType_PIPELINE_RUN_STATUS_TYPE_SUCCESS, pipelineRunStatus.PipelineRunStatusType)
	matches, err := filepath.Glob("/tmp/pachyderm-test/1-out/*")
	require.NoError(t, err)
	require.Equal(
		t,
		[]string{
			"/tmp/pachyderm-test/1-out/1.txt",
			"/tmp/pachyderm-test/1-out/10.txt",
			"/tmp/pachyderm-test/1-out/2.txt",
			"/tmp/pachyderm-test/1-out/20.txt",
			"/tmp/pachyderm-test/1-out/3.txt",
			"/tmp/pachyderm-test/1-out/30.txt",
			"/tmp/pachyderm-test/1-out/4.txt",
			"/tmp/pachyderm-test/1-out/40.txt",
			"/tmp/pachyderm-test/1-out/5.txt",
			"/tmp/pachyderm-test/1-out/50.txt",
		},
		matches,
	)
	matches, err = filepath.Glob("/tmp/pachyderm-test/2-out/*")
	require.NoError(t, err)
	require.Equal(
		t,
		[]string{
			"/tmp/pachyderm-test/2-out/1.txt.copy",
			"/tmp/pachyderm-test/2-out/10.txt.copy",
			"/tmp/pachyderm-test/2-out/2.txt.copy",
			"/tmp/pachyderm-test/2-out/20.txt.copy",
			"/tmp/pachyderm-test/2-out/3.txt.copy",
			"/tmp/pachyderm-test/2-out/30.txt.copy",
			"/tmp/pachyderm-test/2-out/4.txt.copy",
			"/tmp/pachyderm-test/2-out/40.txt.copy",
			"/tmp/pachyderm-test/2-out/5.txt.copy",
			"/tmp/pachyderm-test/2-out/50.txt.copy",
		},
		matches,
	)
	matches, err = filepath.Glob("/tmp/pachyderm-test/3-out/*")
	require.NoError(t, err)
	require.Equal(
		t,
		[]string{
			"/tmp/pachyderm-test/3-out/1.txt.copy3",
			"/tmp/pachyderm-test/3-out/10.txt.copy3",
			"/tmp/pachyderm-test/3-out/2.txt.copy3",
			"/tmp/pachyderm-test/3-out/20.txt.copy3",
			"/tmp/pachyderm-test/3-out/3.txt.copy3",
			"/tmp/pachyderm-test/3-out/30.txt.copy3",
			"/tmp/pachyderm-test/3-out/4.txt.copy3",
			"/tmp/pachyderm-test/3-out/40.txt.copy3",
			"/tmp/pachyderm-test/3-out/5.txt.copy3",
			"/tmp/pachyderm-test/3-out/50.txt.copy3",
		},
		matches,
	)
	matches, err = filepath.Glob("/tmp/pachyderm-test/4-out/*")
	require.NoError(t, err)
	require.Equal(
		t,
		[]string{
			"/tmp/pachyderm-test/4-out/1.txt.copy4",
			"/tmp/pachyderm-test/4-out/10.txt.copy4",
			"/tmp/pachyderm-test/4-out/2.txt.copy4",
			"/tmp/pachyderm-test/4-out/20.txt.copy4",
			"/tmp/pachyderm-test/4-out/3.txt.copy4",
			"/tmp/pachyderm-test/4-out/30.txt.copy4",
			"/tmp/pachyderm-test/4-out/4.txt.copy4",
			"/tmp/pachyderm-test/4-out/40.txt.copy4",
			"/tmp/pachyderm-test/4-out/5.txt.copy4",
			"/tmp/pachyderm-test/4-out/50.txt.copy4",
			"/tmp/pachyderm-test/4-out/build-file.txt4",
		},
		matches,
	)
	matches, err = filepath.Glob("/tmp/pachyderm-test/5-out/*")
	require.NoError(t, err)
	require.Equal(
		t,
		[]string{
			"/tmp/pachyderm-test/5-out/1.txt.copy3",
			"/tmp/pachyderm-test/5-out/1.txt.copy4",
			"/tmp/pachyderm-test/5-out/10.txt.copy3",
			"/tmp/pachyderm-test/5-out/10.txt.copy4",
			"/tmp/pachyderm-test/5-out/2.txt.copy3",
			"/tmp/pachyderm-test/5-out/2.txt.copy4",
			"/tmp/pachyderm-test/5-out/20.txt.copy3",
			"/tmp/pachyderm-test/5-out/20.txt.copy4",
			"/tmp/pachyderm-test/5-out/3.txt.copy3",
			"/tmp/pachyderm-test/5-out/3.txt.copy4",
			"/tmp/pachyderm-test/5-out/30.txt.copy3",
			"/tmp/pachyderm-test/5-out/30.txt.copy4",
			"/tmp/pachyderm-test/5-out/4.txt.copy3",
			"/tmp/pachyderm-test/5-out/4.txt.copy4",
			"/tmp/pachyderm-test/5-out/40.txt.copy3",
			"/tmp/pachyderm-test/5-out/40.txt.copy4",
			"/tmp/pachyderm-test/5-out/5.txt.copy3",
			"/tmp/pachyderm-test/5-out/5.txt.copy4",
			"/tmp/pachyderm-test/5-out/50.txt.copy3",
			"/tmp/pachyderm-test/5-out/50.txt.copy4",
			"/tmp/pachyderm-test/5-out/build-file.txt4",
			"/tmp/pachyderm-test/5-out/build-file2.txt",
		},
		matches,
	)
}
Esempio n. 18
0
func TestBuildWithError(t *testing.T) {
	intC := make(chan int, 5)
	nameToNodeInfo := map[string]*NodeInfo{
		"1": &NodeInfo{
			Parents: []string{},
		},
		"2": &NodeInfo{
			Parents: []string{},
		},
		"3-1": &NodeInfo{
			Parents: []string{
				"1",
				"2",
			},
		},
		"3-2": &NodeInfo{
			Parents: []string{
				"1",
				"2",
			},
		},
		"3-3": &NodeInfo{
			Parents: []string{
				"1",
				"2",
			},
		},
		"4-1": &NodeInfo{
			Parents: []string{
				"3-1",
				"3-2",
				"3-3",
			},
		},
		"4-2": &NodeInfo{
			Parents: []string{
				"3-1",
				"3-2",
				"3-3",
			},
		},
		"5": &NodeInfo{
			Parents: []string{
				"4-1",
				"4-2",
			},
		},
	}
	counter := int32(0)
	nameToNodeFunc := map[string]func() error{
		"1":   testNodeFunc(&counter, intC, "1", 1, ""),
		"2":   testNodeFunc(&counter, intC, "2", 2, ""),
		"3-1": testNodeFunc(&counter, intC, "3-1", 3, "3-1:error"),
		"3-2": testNodeFunc(&counter, intC, "3-2", 4, ""),
		"3-3": testNodeFunc(&counter, intC, "3-3", 5, ""),
		"4-1": testNodeFunc(&counter, intC, "4-1", 6, ""),
		"4-2": testNodeFunc(&counter, intC, "4-2", 7, ""),
		"5":   testNodeFunc(&counter, intC, "5", 8, ""),
	}

	run, err := build(nameToNodeInfo, nameToNodeFunc)
	require.NoError(t, err)
	err = run.Do()
	require.NotNil(t, err)
	require.Equal(t, "3-1:error", err.Error())

	require.Equal(t, int32(5), counter)
	i := <-intC
	require.True(t, i == 1 || i == 2)
	i = <-intC
	require.True(t, i == 1 || i == 2)
	i = <-intC
	require.True(t, i == 3 || i == 4 || i == 5)
	i = <-intC
	require.True(t, i == 3 || i == 4 || i == 5)
	i = <-intC
	require.True(t, i == 3 || i == 4 || i == 5)
}
Esempio n. 19
0
func testMount(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
	repositoryName := "testMountRepo"

	err := pfsutil.CreateRepo(apiClient, repositoryName)
	require.NoError(t, err)

	directory := "/compile/testMount"
	mounter := fuse.NewMounter(apiClient)
	err = mounter.Mount(repositoryName, directory, "", 0, 1)
	require.NoError(t, err)

	_, err = os.Stat(filepath.Join(directory, "scratch"))
	require.NoError(t, err)

	commit, err := pfsutil.StartCommit(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, commit)
	newCommitID := commit.Id

	_, err = os.Stat(filepath.Join(directory, newCommitID))
	require.NoError(t, err)

	err = ioutil.WriteFile(filepath.Join(directory, newCommitID, "foo"), []byte("foo"), 0666)
	require.NoError(t, err)

	_, err = pfsutil.PutFile(apiClient, repositoryName, newCommitID, "bar", 0, strings.NewReader("bar"))
	require.NoError(t, err)

	bigValue := make([]byte, 1024*1024)
	for i := 0; i < 1024*1024; i++ {
		bigValue[i] = 'a'
	}

	err = ioutil.WriteFile(filepath.Join(directory, newCommitID, "big1"), bigValue, 0666)
	require.NoError(t, err)

	_, err = pfsutil.PutFile(apiClient, repositoryName, newCommitID, "big2", 0, bytes.NewReader(bigValue))
	require.NoError(t, err)

	err = pfsutil.FinishCommit(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)

	fInfo, err := os.Stat(filepath.Join(directory, newCommitID, "foo"))
	require.NoError(t, err)
	require.Equal(t, int64(3), fInfo.Size())

	data, err := ioutil.ReadFile(filepath.Join(directory, newCommitID, "foo"))
	require.NoError(t, err)
	require.Equal(t, "foo", string(data))

	data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "bar"))
	require.NoError(t, err)
	require.Equal(t, "bar", string(data))

	data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "big1"))
	require.NoError(t, err)
	require.Equal(t, bigValue, data)

	data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "big2"))
	require.NoError(t, err)
	require.Equal(t, bigValue, data)

	err = mounter.Unmount(directory)
	require.NoError(t, err)
	err = mounter.Wait(directory)
	require.NoError(t, err)
}
Esempio n. 20
0
func testSimple(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
	repositoryName := "testSimpleRepo"

	err := pfsutil.CreateRepo(apiClient, repositoryName)
	require.NoError(t, err)

	scratchCommitInfo, err := pfsutil.InspectCommit(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, scratchCommitInfo)
	require.Equal(t, "scratch", scratchCommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, scratchCommitInfo.CommitType)
	require.Nil(t, scratchCommitInfo.ParentCommit)

	commitInfos, err := pfsutil.ListCommit(apiClient, repositoryName)
	require.NoError(t, err)
	require.Equal(t, 1, len(commitInfos))
	require.Equal(t, scratchCommitInfo.Commit, commitInfos[0].Commit)

	commit, err := pfsutil.StartCommit(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, commit)
	newCommitID := commit.Id

	newCommitInfo, err := pfsutil.InspectCommit(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)
	require.NotNil(t, newCommitInfo)
	require.Equal(t, newCommitID, newCommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_WRITE, newCommitInfo.CommitType)
	require.Equal(t, "scratch", newCommitInfo.ParentCommit.Id)

	commitInfos, err = pfsutil.ListCommit(apiClient, repositoryName)
	require.NoError(t, err)
	require.Equal(t, 2, len(commitInfos))
	require.Equal(t, newCommitInfo.Commit, commitInfos[0].Commit)
	require.Equal(t, scratchCommitInfo.Commit, commitInfos[1].Commit)

	err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/b")
	require.NoError(t, err)
	err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/c")
	require.NoError(t, err)
	err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/d")
	require.NoError(t, err)

	doWrites(t, apiClient, repositoryName, newCommitID)
	doBlockWrites(t, apiClient, repositoryName, newCommitID)

	err = pfsutil.FinishCommit(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)

	newCommitInfo, err = pfsutil.InspectCommit(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)
	require.NotNil(t, newCommitInfo)
	require.Equal(t, newCommitID, newCommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, newCommitInfo.CommitType)
	require.Equal(t, "scratch", newCommitInfo.ParentCommit.Id)

	checkWrites(t, apiClient, repositoryName, newCommitID)
	checkBlockWrites(t, apiClient, repositoryName, newCommitID)

	fileInfos, err := pfsutil.ListFile(apiClient, repositoryName, newCommitID, "a/b", 0, 1)
	require.NoError(t, err)
	require.Equal(t, testSize, len(fileInfos))
	fileInfos, err = pfsutil.ListFile(apiClient, repositoryName, newCommitID, "a/c", 0, 1)
	require.NoError(t, err)
	require.Equal(t, testSize, len(fileInfos))

	var fileInfos2 [7][]*pfs.FileInfo
	var wg sync.WaitGroup
	for i := 0; i < 7; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			fileInfos3, iErr := pfsutil.ListFile(apiClient, repositoryName, newCommitID, "a/b", uint64(i), 7)
			require.NoError(t, iErr)
			fileInfos2[i] = fileInfos3
		}()
	}
	wg.Wait()
	count := 0
	for i := 0; i < 7; i++ {
		count += len(fileInfos2[i])
	}
	require.Equal(t, testSize, count)
}