func testFailures(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
repositoryName := "testFailuresRepo"
err := pfsutil.CreateRepo(apiClient, repositoryName)
require.NoError(t, err)
commit, err := pfsutil.StartCommit(apiClient, repositoryName, "scratch")
require.NoError(t, err)
require.NotNil(t, commit)
newCommitID := commit.Id
err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/b")
require.NoError(t, err)
err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/c")
require.NoError(t, err)
doWrites(t, apiClient, repositoryName, newCommitID)
err = pfsutil.FinishCommit(apiClient, repositoryName, newCommitID)
require.NoError(t, err)
checkWrites(t, apiClient, repositoryName, newCommitID)
cluster.KillRoleAssigner()
for server := 0; server < testNumReplicas; server++ {
cluster.Kill(server)
}
cluster.RestartRoleAssigner()
cluster.WaitForAvailability()
checkWrites(t, apiClient, repositoryName, newCommitID)
}
func testMountBig(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
repositoryName := "testMountBigRepo"
err := pfsutil.CreateRepo(apiClient, repositoryName)
require.NoError(t, err)
directory := "/compile/testMount"
mounter := fuse.NewMounter(apiClient)
err = mounter.Mount(repositoryName, "", directory, 0, 1)
require.NoError(t, err)
_, err = os.Stat(filepath.Join(directory, "scratch"))
require.NoError(t, err)
commit, err := pfsutil.StartCommit(apiClient, repositoryName, "scratch")
require.NoError(t, err)
require.NotNil(t, commit)
newCommitID := commit.Id
bigValue := make([]byte, 1024*1024*300)
for i := 0; i < 1024*1024*300; i++ {
bigValue[i] = 'a'
}
wg := sync.WaitGroup{}
for j := 0; j < 5; j++ {
wg.Add(1)
go func(j int) {
defer wg.Done()
err := ioutil.WriteFile(filepath.Join(directory, newCommitID, fmt.Sprintf("big%d", j)), bigValue, 0666)
require.NoError(t, err)
}(j)
}
wg.Wait()
err = pfsutil.FinishCommit(apiClient, repositoryName, newCommitID)
require.NoError(t, err)
wg = sync.WaitGroup{}
for j := 0; j < 5; j++ {
wg.Add(1)
go func(j int) {
defer wg.Done()
data, err := ioutil.ReadFile(filepath.Join(directory, newCommitID, fmt.Sprintf("big%d", j)))
require.NoError(t, err)
require.Equal(t, bigValue, data)
}(j)
}
wg.Wait()
err = mounter.Unmount(directory)
require.NoError(t, err)
err = mounter.Wait(directory)
require.NoError(t, err)
}
func testSimple(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
repositoryName := "testSimpleRepo"
err := pfsutil.CreateRepo(apiClient, repositoryName)
require.NoError(t, err)
scratchCommitInfo, err := pfsutil.InspectCommit(apiClient, repositoryName, "scratch")
require.NoError(t, err)
require.NotNil(t, scratchCommitInfo)
require.Equal(t, "scratch", scratchCommitInfo.Commit.Id)
require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, scratchCommitInfo.CommitType)
require.Nil(t, scratchCommitInfo.ParentCommit)
commitInfos, err := pfsutil.ListCommit(apiClient, repositoryName)
require.NoError(t, err)
require.Equal(t, 1, len(commitInfos))
require.Equal(t, scratchCommitInfo.Commit, commitInfos[0].Commit)
commit, err := pfsutil.StartCommit(apiClient, repositoryName, "scratch")
require.NoError(t, err)
require.NotNil(t, commit)
newCommitID := commit.Id
newCommitInfo, err := pfsutil.InspectCommit(apiClient, repositoryName, newCommitID)
require.NoError(t, err)
require.NotNil(t, newCommitInfo)
require.Equal(t, newCommitID, newCommitInfo.Commit.Id)
require.Equal(t, pfs.CommitType_COMMIT_TYPE_WRITE, newCommitInfo.CommitType)
require.Equal(t, "scratch", newCommitInfo.ParentCommit.Id)
commitInfos, err = pfsutil.ListCommit(apiClient, repositoryName)
require.NoError(t, err)
require.Equal(t, 2, len(commitInfos))
require.Equal(t, newCommitInfo.Commit, commitInfos[0].Commit)
require.Equal(t, scratchCommitInfo.Commit, commitInfos[1].Commit)
err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/b")
require.NoError(t, err)
err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/c")
require.NoError(t, err)
err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/d")
require.NoError(t, err)
doWrites(t, apiClient, repositoryName, newCommitID)
doBlockWrites(t, apiClient, repositoryName, newCommitID)
err = pfsutil.FinishCommit(apiClient, repositoryName, newCommitID)
require.NoError(t, err)
newCommitInfo, err = pfsutil.InspectCommit(apiClient, repositoryName, newCommitID)
require.NoError(t, err)
require.NotNil(t, newCommitInfo)
require.Equal(t, newCommitID, newCommitInfo.Commit.Id)
require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, newCommitInfo.CommitType)
require.Equal(t, "scratch", newCommitInfo.ParentCommit.Id)
checkWrites(t, apiClient, repositoryName, newCommitID)
checkBlockWrites(t, apiClient, repositoryName, newCommitID)
fileInfos, err := pfsutil.ListFile(apiClient, repositoryName, newCommitID, "a/b", 0, 1)
require.NoError(t, err)
require.Equal(t, testSize, len(fileInfos))
fileInfos, err = pfsutil.ListFile(apiClient, repositoryName, newCommitID, "a/c", 0, 1)
require.NoError(t, err)
require.Equal(t, testSize, len(fileInfos))
var fileInfos2 [7][]*pfs.FileInfo
var wg sync.WaitGroup
for i := 0; i < 7; i++ {
i := i
wg.Add(1)
go func() {
defer wg.Done()
fileInfos3, iErr := pfsutil.ListFile(apiClient, repositoryName, newCommitID, "a/b", uint64(i), 7)
require.NoError(t, iErr)
fileInfos2[i] = fileInfos3
}()
}
wg.Wait()
count := 0
for i := 0; i < 7; i++ {
count += len(fileInfos2[i])
}
require.Equal(t, testSize, count)
}
func testMount(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
repositoryName := "testMountRepo"
err := pfsutil.CreateRepo(apiClient, repositoryName)
require.NoError(t, err)
directory := "/compile/testMount"
mounter := fuse.NewMounter(apiClient)
err = mounter.Mount(repositoryName, directory, "", 0, 1)
require.NoError(t, err)
_, err = os.Stat(filepath.Join(directory, "scratch"))
require.NoError(t, err)
commit, err := pfsutil.StartCommit(apiClient, repositoryName, "scratch")
require.NoError(t, err)
require.NotNil(t, commit)
newCommitID := commit.Id
_, err = os.Stat(filepath.Join(directory, newCommitID))
require.NoError(t, err)
err = ioutil.WriteFile(filepath.Join(directory, newCommitID, "foo"), []byte("foo"), 0666)
require.NoError(t, err)
_, err = pfsutil.PutFile(apiClient, repositoryName, newCommitID, "bar", 0, strings.NewReader("bar"))
require.NoError(t, err)
bigValue := make([]byte, 1024*1024)
for i := 0; i < 1024*1024; i++ {
bigValue[i] = 'a'
}
err = ioutil.WriteFile(filepath.Join(directory, newCommitID, "big1"), bigValue, 0666)
require.NoError(t, err)
_, err = pfsutil.PutFile(apiClient, repositoryName, newCommitID, "big2", 0, bytes.NewReader(bigValue))
require.NoError(t, err)
err = pfsutil.FinishCommit(apiClient, repositoryName, newCommitID)
require.NoError(t, err)
fInfo, err := os.Stat(filepath.Join(directory, newCommitID, "foo"))
require.NoError(t, err)
require.Equal(t, int64(3), fInfo.Size())
data, err := ioutil.ReadFile(filepath.Join(directory, newCommitID, "foo"))
require.NoError(t, err)
require.Equal(t, "foo", string(data))
data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "bar"))
require.NoError(t, err)
require.Equal(t, "bar", string(data))
data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "big1"))
require.NoError(t, err)
require.Equal(t, bigValue, data)
data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "big2"))
require.NoError(t, err)
require.Equal(t, bigValue, data)
err = mounter.Unmount(directory)
require.NoError(t, err)
err = mounter.Wait(directory)
require.NoError(t, err)
}
func TestBuildWithError(t *testing.T) {
intC := make(chan int, 5)
nameToNodeInfo := map[string]*NodeInfo{
"1": &NodeInfo{
Parents: []string{},
},
"2": &NodeInfo{
Parents: []string{},
},
"3-1": &NodeInfo{
Parents: []string{
"1",
"2",
},
},
"3-2": &NodeInfo{
Parents: []string{
"1",
"2",
},
},
"3-3": &NodeInfo{
Parents: []string{
"1",
"2",
},
},
"4-1": &NodeInfo{
Parents: []string{
"3-1",
"3-2",
"3-3",
},
},
"4-2": &NodeInfo{
Parents: []string{
"3-1",
"3-2",
"3-3",
},
},
"5": &NodeInfo{
Parents: []string{
"4-1",
"4-2",
},
},
}
counter := int32(0)
nameToNodeFunc := map[string]func() error{
"1": testNodeFunc(&counter, intC, "1", 1, ""),
"2": testNodeFunc(&counter, intC, "2", 2, ""),
"3-1": testNodeFunc(&counter, intC, "3-1", 3, "3-1:error"),
"3-2": testNodeFunc(&counter, intC, "3-2", 4, ""),
"3-3": testNodeFunc(&counter, intC, "3-3", 5, ""),
"4-1": testNodeFunc(&counter, intC, "4-1", 6, ""),
"4-2": testNodeFunc(&counter, intC, "4-2", 7, ""),
"5": testNodeFunc(&counter, intC, "5", 8, ""),
}
run, err := build(nameToNodeInfo, nameToNodeFunc)
require.NoError(t, err)
err = run.Do()
require.NotNil(t, err)
require.Equal(t, "3-1:error", err.Error())
require.Equal(t, int32(5), counter)
i := <-intC
require.True(t, i == 1 || i == 2)
i = <-intC
require.True(t, i == 1 || i == 2)
i = <-intC
require.True(t, i == 3 || i == 4 || i == 5)
i = <-intC
require.True(t, i == 3 || i == 4 || i == 5)
i = <-intC
require.True(t, i == 3 || i == 4 || i == 5)
}