コード例 #1
0
ファイル: testing_test.go プロジェクト: bereal/pachyderm
func testFailures(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
	repositoryName := "testFailuresRepo"

	err := pfsutil.CreateRepo(apiClient, repositoryName)
	require.NoError(t, err)

	commit, err := pfsutil.StartCommit(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, commit)
	newCommitID := commit.Id

	err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/b")
	require.NoError(t, err)
	err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/c")
	require.NoError(t, err)

	doWrites(t, apiClient, repositoryName, newCommitID)

	err = pfsutil.FinishCommit(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)

	checkWrites(t, apiClient, repositoryName, newCommitID)

	cluster.KillRoleAssigner()
	for server := 0; server < testNumReplicas; server++ {
		cluster.Kill(server)
	}
	cluster.RestartRoleAssigner()
	cluster.WaitForAvailability()

	checkWrites(t, apiClient, repositoryName, newCommitID)
}
コード例 #2
0
ファイル: testing_test.go プロジェクト: bereal/pachyderm
func testMountBig(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
	repositoryName := "testMountBigRepo"

	err := pfsutil.CreateRepo(apiClient, repositoryName)
	require.NoError(t, err)

	directory := "/compile/testMount"
	mounter := fuse.NewMounter(apiClient)
	err = mounter.Mount(repositoryName, "", directory, 0, 1)
	require.NoError(t, err)

	_, err = os.Stat(filepath.Join(directory, "scratch"))
	require.NoError(t, err)

	commit, err := pfsutil.StartCommit(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, commit)
	newCommitID := commit.Id

	bigValue := make([]byte, 1024*1024*300)
	for i := 0; i < 1024*1024*300; i++ {
		bigValue[i] = 'a'
	}

	wg := sync.WaitGroup{}
	for j := 0; j < 5; j++ {
		wg.Add(1)
		go func(j int) {
			defer wg.Done()
			err := ioutil.WriteFile(filepath.Join(directory, newCommitID, fmt.Sprintf("big%d", j)), bigValue, 0666)
			require.NoError(t, err)
		}(j)
	}
	wg.Wait()

	err = pfsutil.FinishCommit(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)

	wg = sync.WaitGroup{}
	for j := 0; j < 5; j++ {
		wg.Add(1)
		go func(j int) {
			defer wg.Done()
			data, err := ioutil.ReadFile(filepath.Join(directory, newCommitID, fmt.Sprintf("big%d", j)))
			require.NoError(t, err)
			require.Equal(t, bigValue, data)
		}(j)
	}
	wg.Wait()

	err = mounter.Unmount(directory)
	require.NoError(t, err)
	err = mounter.Wait(directory)
	require.NoError(t, err)
}
コード例 #3
0
ファイル: testing_test.go プロジェクト: bereal/pachyderm
func testSimple(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
	repositoryName := "testSimpleRepo"

	err := pfsutil.CreateRepo(apiClient, repositoryName)
	require.NoError(t, err)

	scratchCommitInfo, err := pfsutil.InspectCommit(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, scratchCommitInfo)
	require.Equal(t, "scratch", scratchCommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, scratchCommitInfo.CommitType)
	require.Nil(t, scratchCommitInfo.ParentCommit)

	commitInfos, err := pfsutil.ListCommit(apiClient, repositoryName)
	require.NoError(t, err)
	require.Equal(t, 1, len(commitInfos))
	require.Equal(t, scratchCommitInfo.Commit, commitInfos[0].Commit)

	commit, err := pfsutil.StartCommit(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, commit)
	newCommitID := commit.Id

	newCommitInfo, err := pfsutil.InspectCommit(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)
	require.NotNil(t, newCommitInfo)
	require.Equal(t, newCommitID, newCommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_WRITE, newCommitInfo.CommitType)
	require.Equal(t, "scratch", newCommitInfo.ParentCommit.Id)

	commitInfos, err = pfsutil.ListCommit(apiClient, repositoryName)
	require.NoError(t, err)
	require.Equal(t, 2, len(commitInfos))
	require.Equal(t, newCommitInfo.Commit, commitInfos[0].Commit)
	require.Equal(t, scratchCommitInfo.Commit, commitInfos[1].Commit)

	err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/b")
	require.NoError(t, err)
	err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/c")
	require.NoError(t, err)
	err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/d")
	require.NoError(t, err)

	doWrites(t, apiClient, repositoryName, newCommitID)
	doBlockWrites(t, apiClient, repositoryName, newCommitID)

	err = pfsutil.FinishCommit(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)

	newCommitInfo, err = pfsutil.InspectCommit(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)
	require.NotNil(t, newCommitInfo)
	require.Equal(t, newCommitID, newCommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, newCommitInfo.CommitType)
	require.Equal(t, "scratch", newCommitInfo.ParentCommit.Id)

	checkWrites(t, apiClient, repositoryName, newCommitID)
	checkBlockWrites(t, apiClient, repositoryName, newCommitID)

	fileInfos, err := pfsutil.ListFile(apiClient, repositoryName, newCommitID, "a/b", 0, 1)
	require.NoError(t, err)
	require.Equal(t, testSize, len(fileInfos))
	fileInfos, err = pfsutil.ListFile(apiClient, repositoryName, newCommitID, "a/c", 0, 1)
	require.NoError(t, err)
	require.Equal(t, testSize, len(fileInfos))

	var fileInfos2 [7][]*pfs.FileInfo
	var wg sync.WaitGroup
	for i := 0; i < 7; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			fileInfos3, iErr := pfsutil.ListFile(apiClient, repositoryName, newCommitID, "a/b", uint64(i), 7)
			require.NoError(t, iErr)
			fileInfos2[i] = fileInfos3
		}()
	}
	wg.Wait()
	count := 0
	for i := 0; i < 7; i++ {
		count += len(fileInfos2[i])
	}
	require.Equal(t, testSize, count)
}
コード例 #4
0
ファイル: testing_test.go プロジェクト: bereal/pachyderm
func testMount(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
	repositoryName := "testMountRepo"

	err := pfsutil.CreateRepo(apiClient, repositoryName)
	require.NoError(t, err)

	directory := "/compile/testMount"
	mounter := fuse.NewMounter(apiClient)
	err = mounter.Mount(repositoryName, directory, "", 0, 1)
	require.NoError(t, err)

	_, err = os.Stat(filepath.Join(directory, "scratch"))
	require.NoError(t, err)

	commit, err := pfsutil.StartCommit(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, commit)
	newCommitID := commit.Id

	_, err = os.Stat(filepath.Join(directory, newCommitID))
	require.NoError(t, err)

	err = ioutil.WriteFile(filepath.Join(directory, newCommitID, "foo"), []byte("foo"), 0666)
	require.NoError(t, err)

	_, err = pfsutil.PutFile(apiClient, repositoryName, newCommitID, "bar", 0, strings.NewReader("bar"))
	require.NoError(t, err)

	bigValue := make([]byte, 1024*1024)
	for i := 0; i < 1024*1024; i++ {
		bigValue[i] = 'a'
	}

	err = ioutil.WriteFile(filepath.Join(directory, newCommitID, "big1"), bigValue, 0666)
	require.NoError(t, err)

	_, err = pfsutil.PutFile(apiClient, repositoryName, newCommitID, "big2", 0, bytes.NewReader(bigValue))
	require.NoError(t, err)

	err = pfsutil.FinishCommit(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)

	fInfo, err := os.Stat(filepath.Join(directory, newCommitID, "foo"))
	require.NoError(t, err)
	require.Equal(t, int64(3), fInfo.Size())

	data, err := ioutil.ReadFile(filepath.Join(directory, newCommitID, "foo"))
	require.NoError(t, err)
	require.Equal(t, "foo", string(data))

	data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "bar"))
	require.NoError(t, err)
	require.Equal(t, "bar", string(data))

	data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "big1"))
	require.NoError(t, err)
	require.Equal(t, bigValue, data)

	data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "big2"))
	require.NoError(t, err)
	require.Equal(t, bigValue, data)

	err = mounter.Unmount(directory)
	require.NoError(t, err)
	err = mounter.Wait(directory)
	require.NoError(t, err)
}
コード例 #5
0
ファイル: graph_test.go プロジェクト: wxdublin/pachyderm
func TestBuildWithError(t *testing.T) {
	intC := make(chan int, 5)
	nameToNodeInfo := map[string]*NodeInfo{
		"1": &NodeInfo{
			Parents: []string{},
		},
		"2": &NodeInfo{
			Parents: []string{},
		},
		"3-1": &NodeInfo{
			Parents: []string{
				"1",
				"2",
			},
		},
		"3-2": &NodeInfo{
			Parents: []string{
				"1",
				"2",
			},
		},
		"3-3": &NodeInfo{
			Parents: []string{
				"1",
				"2",
			},
		},
		"4-1": &NodeInfo{
			Parents: []string{
				"3-1",
				"3-2",
				"3-3",
			},
		},
		"4-2": &NodeInfo{
			Parents: []string{
				"3-1",
				"3-2",
				"3-3",
			},
		},
		"5": &NodeInfo{
			Parents: []string{
				"4-1",
				"4-2",
			},
		},
	}
	counter := int32(0)
	nameToNodeFunc := map[string]func() error{
		"1":   testNodeFunc(&counter, intC, "1", 1, ""),
		"2":   testNodeFunc(&counter, intC, "2", 2, ""),
		"3-1": testNodeFunc(&counter, intC, "3-1", 3, "3-1:error"),
		"3-2": testNodeFunc(&counter, intC, "3-2", 4, ""),
		"3-3": testNodeFunc(&counter, intC, "3-3", 5, ""),
		"4-1": testNodeFunc(&counter, intC, "4-1", 6, ""),
		"4-2": testNodeFunc(&counter, intC, "4-2", 7, ""),
		"5":   testNodeFunc(&counter, intC, "5", 8, ""),
	}

	run, err := build(nameToNodeInfo, nameToNodeFunc)
	require.NoError(t, err)
	err = run.Do()
	require.NotNil(t, err)
	require.Equal(t, "3-1:error", err.Error())

	require.Equal(t, int32(5), counter)
	i := <-intC
	require.True(t, i == 1 || i == 2)
	i = <-intC
	require.True(t, i == 1 || i == 2)
	i = <-intC
	require.True(t, i == 3 || i == 4 || i == 5)
	i = <-intC
	require.True(t, i == 3 || i == 4 || i == 5)
	i = <-intC
	require.True(t, i == 3 || i == 4 || i == 5)
}