func TestMountBig(t *testing.T) {
t.Skip()
t.Parallel()
apiClient := getPfsClient(t)
repoName := uniqueString("testMountBigRepo")
err := pfsutil.CreateRepo(apiClient, repoName)
require.NoError(t, err)
directory := "/compile/testMount"
mounter := fuse.NewMounter("localhost", apiClient)
ready := make(chan bool)
go func() {
err = mounter.Mount(directory, &pfs.Shard{Number: 0, Modulus: 1}, nil, ready)
require.NoError(t, err)
}()
<-ready
_, err = os.Stat(filepath.Join(directory, repoName))
require.NoError(t, err)
commit, err := pfsutil.StartCommit(apiClient, repoName, "")
require.NoError(t, err)
require.NotNil(t, commit)
newCommitID := commit.Id
bigValue := make([]byte, 1024*1024*300)
for i := 0; i < 1024*1024*300; i++ {
bigValue[i] = 'a'
}
wg := sync.WaitGroup{}
for j := 0; j < 5; j++ {
wg.Add(1)
go func(j int) {
defer wg.Done()
err := ioutil.WriteFile(filepath.Join(directory, repoName, newCommitID, fmt.Sprintf("big%d", j)), bigValue, 0666)
require.NoError(t, err)
}(j)
}
wg.Wait()
err = pfsutil.FinishCommit(apiClient, repoName, newCommitID)
require.NoError(t, err)
wg = sync.WaitGroup{}
for j := 0; j < 5; j++ {
wg.Add(1)
go func(j int) {
defer wg.Done()
data, err := ioutil.ReadFile(filepath.Join(directory, repoName, newCommitID, fmt.Sprintf("big%d", j)))
require.NoError(t, err)
require.Equal(t, bigValue, data)
}(j)
}
wg.Wait()
err = mounter.Unmount(directory)
require.NoError(t, err)
}
func testFailures(t *testing.T, apiClient pfs.APIClient, cluster Cluster) {
repoName := "testFailuresRepo"
err := pfsutil.CreateRepo(apiClient, repoName)
require.NoError(t, err)
commit, err := pfsutil.StartCommit(apiClient, repoName, "")
require.NoError(t, err)
require.NotNil(t, commit)
newCommitID := commit.Id
err = pfsutil.MakeDirectory(apiClient, repoName, newCommitID, "a/b")
require.NoError(t, err)
err = pfsutil.MakeDirectory(apiClient, repoName, newCommitID, "a/c")
require.NoError(t, err)
doWrites(t, apiClient, repoName, newCommitID)
err = pfsutil.FinishCommit(apiClient, repoName, newCommitID)
require.NoError(t, err)
checkWrites(t, apiClient, repoName, newCommitID)
cluster.KillRoleAssigner()
for server := 0; server < testNumReplicas; server++ {
cluster.Kill(server)
}
cluster.RestartRoleAssigner()
cluster.WaitForAvailability()
checkWrites(t, apiClient, repoName, newCommitID)
}
func testFailures(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
repositoryName := TestRepositoryName()
err := pfsutil.InitRepository(apiClient, repositoryName)
require.NoError(t, err)
commit, err := pfsutil.Branch(apiClient, repositoryName, "scratch")
require.NoError(t, err)
require.NotNil(t, commit)
newCommitID := commit.Id
err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/b")
require.NoError(t, err)
err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/c")
require.NoError(t, err)
doWrites(t, apiClient, repositoryName, newCommitID)
err = pfsutil.Write(apiClient, repositoryName, newCommitID)
require.NoError(t, err)
checkWrites(t, apiClient, repositoryName, newCommitID)
for server := 0; server < testNumReplicas; server++ {
cluster.Kill(server)
}
cluster.WaitForAvailability()
checkWrites(t, apiClient, repositoryName, newCommitID)
}
func testMountBig(t *testing.T, apiClient pfs.APIClient, cluster Cluster) {
repoName := "testMountBigRepo"
err := pfsutil.CreateRepo(apiClient, repoName)
require.NoError(t, err)
directory := "/compile/testMount"
mounter := fuse.NewMounter("localhost", apiClient)
ready := make(chan bool)
go func() {
err = mounter.Mount(directory, 0, 1, ready)
require.NoError(t, err)
}()
<-ready
_, err = os.Stat(filepath.Join(directory, repoName))
require.NoError(t, err)
_, err = os.Stat(filepath.Join(directory, repoName, "scratch"))
require.NoError(t, err)
commit, err := pfsutil.StartCommit(apiClient, repoName, "scratch")
require.NoError(t, err)
require.NotNil(t, commit)
newCommitID := commit.Id
bigValue := make([]byte, 1024*1024*300)
for i := 0; i < 1024*1024*300; i++ {
bigValue[i] = 'a'
}
wg := sync.WaitGroup{}
for j := 0; j < 5; j++ {
wg.Add(1)
go func(j int) {
defer wg.Done()
err := ioutil.WriteFile(filepath.Join(directory, repoName, newCommitID, fmt.Sprintf("big%d", j)), bigValue, 0666)
require.NoError(t, err)
}(j)
}
wg.Wait()
err = pfsutil.FinishCommit(apiClient, repoName, newCommitID)
require.NoError(t, err)
wg = sync.WaitGroup{}
for j := 0; j < 5; j++ {
wg.Add(1)
go func(j int) {
defer wg.Done()
data, err := ioutil.ReadFile(filepath.Join(directory, repoName, newCommitID, fmt.Sprintf("big%d", j)))
require.NoError(t, err)
require.Equal(t, bigValue, data)
}(j)
}
wg.Wait()
err = mounter.Unmount(directory)
require.NoError(t, err)
}
func testMountBig(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
repositoryName := TestRepositoryName()
err := pfsutil.InitRepository(apiClient, repositoryName)
require.NoError(t, err)
directory := "/compile/testMount"
mounter := fuse.NewMounter(apiClient)
err = mounter.Mount(repositoryName, "", directory, 0, 1)
require.NoError(t, err)
_, err = os.Stat(filepath.Join(directory, "scratch"))
require.NoError(t, err)
commit, err := pfsutil.Branch(apiClient, repositoryName, "scratch")
require.NoError(t, err)
require.NotNil(t, commit)
newCommitID := commit.Id
bigValue := make([]byte, 1024*1024*300)
for i := 0; i < 1024*1024*300; i++ {
bigValue[i] = 'a'
}
wg := sync.WaitGroup{}
for j := 0; j < 5; j++ {
wg.Add(1)
go func(j int) {
defer wg.Done()
err := ioutil.WriteFile(filepath.Join(directory, newCommitID, fmt.Sprintf("big%d", j)), bigValue, 0666)
require.NoError(t, err)
}(j)
}
wg.Wait()
err = pfsutil.Write(apiClient, repositoryName, newCommitID)
require.NoError(t, err)
wg = sync.WaitGroup{}
for j := 0; j < 5; j++ {
wg.Add(1)
go func(j int) {
defer wg.Done()
data, err := ioutil.ReadFile(filepath.Join(directory, newCommitID, fmt.Sprintf("big%d", j)))
require.NoError(t, err)
require.Equal(t, bigValue, data)
}(j)
}
wg.Wait()
err = mounter.Unmount(directory)
require.NoError(t, err)
err = mounter.Wait(directory)
require.NoError(t, err)
}
func getJobForPipeline(t *testing.T, jobAPIClient pps.JobAPIClient, pipeline *pps.Pipeline) *pps.Job {
jobInfos, err := jobAPIClient.ListJob(
context.Background(),
&pps.ListJobRequest{
Pipeline: pipeline,
},
)
require.NoError(t, err)
require.NotNil(t, jobInfos)
require.Equal(t, 1, len(jobInfos.JobInfo))
return jobInfos.JobInfo[0].Job
}
func testBasicRethink(t *testing.T, apiServer persist.APIServer) {
jobInfo, err := apiServer.CreateJobInfo(
context.Background(),
&persist.JobInfo{
Spec: &persist.JobInfo_PipelineName{
PipelineName: "foo",
},
},
)
require.NoError(t, err)
getJobInfo, err := apiServer.GetJobInfo(
context.Background(),
&pps.Job{
Id: jobInfo.JobId,
},
)
require.NoError(t, err)
require.Equal(t, jobInfo.JobId, getJobInfo.JobId)
require.Equal(t, "foo", getJobInfo.GetPipelineName())
require.NotNil(t, getJobInfo.CreatedAt)
}
func testSimple(t *testing.T, apiClient pfs.APIClient, cluster Cluster) {
repoName := "testSimpleRepo"
err := pfsutil.CreateRepo(apiClient, repoName)
require.NoError(t, err)
commit, err := pfsutil.StartCommit(apiClient, repoName, "")
require.NoError(t, err)
require.NotNil(t, commit)
newCommitID := commit.Id
newCommitInfo, err := pfsutil.InspectCommit(apiClient, repoName, newCommitID)
require.NoError(t, err)
require.NotNil(t, newCommitInfo)
require.Equal(t, newCommitID, newCommitInfo.Commit.Id)
require.Equal(t, pfs.CommitType_COMMIT_TYPE_WRITE, newCommitInfo.CommitType)
require.Nil(t, newCommitInfo.ParentCommit)
commitInfos, err := pfsutil.ListCommit(apiClient, repoName)
require.NoError(t, err)
require.Equal(t, 1, len(commitInfos))
require.Equal(t, newCommitInfo.Commit, commitInfos[0].Commit)
err = pfsutil.MakeDirectory(apiClient, repoName, newCommitID, "a/b")
require.NoError(t, err)
err = pfsutil.MakeDirectory(apiClient, repoName, newCommitID, "a/c")
require.NoError(t, err)
err = pfsutil.MakeDirectory(apiClient, repoName, newCommitID, "a/d")
require.NoError(t, err)
doWrites(t, apiClient, repoName, newCommitID)
doBlockWrites(t, apiClient, repoName, newCommitID)
err = pfsutil.FinishCommit(apiClient, repoName, newCommitID)
require.NoError(t, err)
newCommitInfo, err = pfsutil.InspectCommit(apiClient, repoName, newCommitID)
require.NoError(t, err)
require.NotNil(t, newCommitInfo)
require.Equal(t, newCommitID, newCommitInfo.Commit.Id)
require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, newCommitInfo.CommitType)
require.Nil(t, newCommitInfo.ParentCommit)
checkWrites(t, apiClient, repoName, newCommitID)
checkBlockWrites(t, apiClient, repoName, newCommitID)
fileInfos, err := pfsutil.ListFile(apiClient, repoName, newCommitID, "a/b", &pfs.Shard{Number: 0, Modulus: 1})
require.NoError(t, err)
require.Equal(t, testSize, len(fileInfos))
fileInfos, err = pfsutil.ListFile(apiClient, repoName, newCommitID, "a/c", &pfs.Shard{Number: 0, Modulus: 1})
require.NoError(t, err)
require.Equal(t, testSize, len(fileInfos))
var fileInfos2 [7][]*pfs.FileInfo
var wg sync.WaitGroup
for i := 0; i < 7; i++ {
i := i
wg.Add(1)
go func() {
defer wg.Done()
fileInfos3, iErr := pfsutil.ListFile(apiClient, repoName, newCommitID, "a/b", &pfs.Shard{Number: uint64(i), Modulus: 7})
require.NoError(t, iErr)
fileInfos2[i] = fileInfos3
}()
}
wg.Wait()
count := 0
for i := 0; i < 7; i++ {
count += len(fileInfos2[i])
}
require.Equal(t, testSize, count)
}
func testMount(t *testing.T, apiClient pfs.APIClient, cluster Cluster) {
repoName := "testMountRepo"
err := pfsutil.CreateRepo(apiClient, repoName)
require.NoError(t, err)
directory := "/compile/testMount"
mounter := fuse.NewMounter("localhost", apiClient)
ready := make(chan bool)
go func() {
err = mounter.Mount(directory, &pfs.Shard{Number: 0, Modulus: 1}, nil, ready)
require.NoError(t, err)
}()
<-ready
_, err = os.Stat(filepath.Join(directory, repoName))
require.NoError(t, err)
commit, err := pfsutil.StartCommit(apiClient, repoName, "")
require.NoError(t, err)
require.NotNil(t, commit)
newCommitID := commit.Id
_, err = os.Stat(filepath.Join(directory, repoName, newCommitID))
require.NoError(t, err)
err = ioutil.WriteFile(filepath.Join(directory, repoName, newCommitID, "foo"), []byte("foo"), 0666)
require.NoError(t, err)
_, err = pfsutil.PutFile(apiClient, repoName, newCommitID, "bar", 0, strings.NewReader("bar"))
require.NoError(t, err)
bigValue := make([]byte, 1024*1024)
for i := 0; i < 1024*1024; i++ {
bigValue[i] = 'a'
}
err = ioutil.WriteFile(filepath.Join(directory, repoName, newCommitID, "big1"), bigValue, 0666)
require.NoError(t, err)
_, err = pfsutil.PutFile(apiClient, repoName, newCommitID, "big2", 0, bytes.NewReader(bigValue))
require.NoError(t, err)
err = pfsutil.FinishCommit(apiClient, repoName, newCommitID)
require.NoError(t, err)
fInfo, err := os.Stat(filepath.Join(directory, repoName, newCommitID, "foo"))
require.NoError(t, err)
require.Equal(t, int64(3), fInfo.Size())
data, err := ioutil.ReadFile(filepath.Join(directory, repoName, newCommitID, "foo"))
require.NoError(t, err)
require.Equal(t, "foo", string(data))
data, err = ioutil.ReadFile(filepath.Join(directory, repoName, newCommitID, "bar"))
require.NoError(t, err)
require.Equal(t, "bar", string(data))
data, err = ioutil.ReadFile(filepath.Join(directory, repoName, newCommitID, "big1"))
require.NoError(t, err)
require.Equal(t, bigValue, data)
data, err = ioutil.ReadFile(filepath.Join(directory, repoName, newCommitID, "big2"))
require.NoError(t, err)
require.Equal(t, bigValue, data)
err = mounter.Unmount(directory)
require.NoError(t, err)
}
func testBlockListCommits(t *testing.T, apiClient pfs.APIClient, cluster Cluster) {
repoName := "testBlockListCommitsRepo"
err := pfsutil.CreateRepo(apiClient, repoName)
require.NoError(t, err)
baseCommit, err := pfsutil.StartCommit(apiClient, repoName, "")
require.NoError(t, err)
err = pfsutil.FinishCommit(apiClient, repoName, baseCommit.Id)
require.NoError(t, err)
repo := &pfs.Repo{
Name: repoName,
}
listCommitRequest := &pfs.ListCommitRequest{
Repo: repo,
From: baseCommit,
}
commitInfos, err := apiClient.ListCommit(
context.Background(),
listCommitRequest,
)
require.NoError(t, err)
require.Equal(t, len(commitInfos.CommitInfo), 0)
var newCommit *pfs.Commit
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
time.Sleep(1)
commit, err := pfsutil.StartCommit(apiClient, repoName, baseCommit.Id)
require.NoError(t, err)
require.NotNil(t, commit)
newCommit = commit
}()
listCommitRequest.Block = true
listCommitRequest.CommitType = pfs.CommitType_COMMIT_TYPE_WRITE
commitInfos, err = apiClient.ListCommit(
context.Background(),
listCommitRequest,
)
wg.Wait()
require.NoError(t, err)
require.Equal(t, len(commitInfos.CommitInfo), 1)
require.Equal(t, newCommit, commitInfos.CommitInfo[0].Commit)
wg = sync.WaitGroup{}
wg.Add(1)
go func() {
defer wg.Done()
time.Sleep(1)
err := pfsutil.FinishCommit(apiClient, repoName, newCommit.Id)
require.NoError(t, err)
}()
listCommitRequest.Block = true
listCommitRequest.CommitType = pfs.CommitType_COMMIT_TYPE_READ
commitInfos, err = apiClient.ListCommit(
context.Background(),
listCommitRequest,
)
wg.Wait()
require.NoError(t, err)
require.Equal(t, len(commitInfos.CommitInfo), 1)
require.Equal(t, newCommit, commitInfos.CommitInfo[0].Commit)
}
func TestBlockListCommits(t *testing.T) {
t.Parallel()
apiClient := getPfsClient(t)
repoName := uniqueString("testBlockListCommitsRepo")
err := pfsutil.CreateRepo(apiClient, repoName)
require.NoError(t, err)
baseCommit, err := pfsutil.StartCommit(apiClient, repoName, "")
require.NoError(t, err)
err = pfsutil.FinishCommit(apiClient, repoName, baseCommit.Id)
require.NoError(t, err)
repo := &pfs.Repo{
Name: repoName,
}
listCommitRequest := &pfs.ListCommitRequest{
Repo: []*pfs.Repo{repo},
FromCommit: []*pfs.Commit{baseCommit},
}
commitInfos, err := apiClient.ListCommit(
context.Background(),
listCommitRequest,
)
require.NoError(t, err)
require.Equal(t, len(commitInfos.CommitInfo), 0)
var newCommit *pfs.Commit
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
time.Sleep(1)
commit, err := pfsutil.StartCommit(apiClient, repoName, baseCommit.Id)
require.NoError(t, err)
require.NotNil(t, commit)
newCommit = commit
}()
listCommitRequest.Block = true
listCommitRequest.CommitType = pfs.CommitType_COMMIT_TYPE_WRITE
commitInfos, err = apiClient.ListCommit(
context.Background(),
listCommitRequest,
)
wg.Wait()
require.NoError(t, err)
require.Equal(t, len(commitInfos.CommitInfo), 1)
require.Equal(t, newCommit, commitInfos.CommitInfo[0].Commit)
wg = sync.WaitGroup{}
wg.Add(1)
go func() {
defer wg.Done()
time.Sleep(1)
err := pfsutil.FinishCommit(apiClient, repoName, newCommit.Id)
require.NoError(t, err)
}()
listCommitRequest.Block = true
listCommitRequest.CommitType = pfs.CommitType_COMMIT_TYPE_READ
commitInfos, err = apiClient.ListCommit(
context.Background(),
listCommitRequest,
)
wg.Wait()
require.NoError(t, err)
require.Equal(t, len(commitInfos.CommitInfo), 1)
require.Equal(t, newCommit, commitInfos.CommitInfo[0].Commit)
}
func TestBuildWithError(t *testing.T) {
intC := make(chan int, 5)
nameToNodeInfo := map[string]*NodeInfo{
"1": &NodeInfo{
Parents: []string{},
},
"2": &NodeInfo{
Parents: []string{},
},
"3-1": &NodeInfo{
Parents: []string{
"1",
"2",
},
},
"3-2": &NodeInfo{
Parents: []string{
"1",
"2",
},
},
"3-3": &NodeInfo{
Parents: []string{
"1",
"2",
},
},
"4-1": &NodeInfo{
Parents: []string{
"3-1",
"3-2",
"3-3",
},
},
"4-2": &NodeInfo{
Parents: []string{
"3-1",
"3-2",
"3-3",
},
},
"5": &NodeInfo{
Parents: []string{
"4-1",
"4-2",
},
},
}
counter := int32(0)
nameToNodeFunc := map[string]func() error{
"1": testNodeFunc(&counter, intC, "1", 1, ""),
"2": testNodeFunc(&counter, intC, "2", 2, ""),
"3-1": testNodeFunc(&counter, intC, "3-1", 3, "3-1:error"),
"3-2": testNodeFunc(&counter, intC, "3-2", 4, ""),
"3-3": testNodeFunc(&counter, intC, "3-3", 5, ""),
"4-1": testNodeFunc(&counter, intC, "4-1", 6, ""),
"4-2": testNodeFunc(&counter, intC, "4-2", 7, ""),
"5": testNodeFunc(&counter, intC, "5", 8, ""),
}
run, err := build(nameToNodeInfo, nameToNodeFunc)
require.NoError(t, err)
err = run.Do()
require.NotNil(t, err)
require.Equal(t, "3-1:error", err.Error())
require.Equal(t, int32(5), counter)
i := <-intC
require.True(t, i == 1 || i == 2)
i = <-intC
require.True(t, i == 1 || i == 2)
i = <-intC
require.True(t, i == 3 || i == 4 || i == 5)
i = <-intC
require.True(t, i == 3 || i == 4 || i == 5)
i = <-intC
require.True(t, i == 3 || i == 4 || i == 5)
}
func TestPipeline(t *testing.T) {
t.Parallel()
pachClient := getPachClient(t)
// create repos
dataRepo := uniqueString("TestPipeline.data")
require.NoError(t, pfsutil.CreateRepo(pachClient, dataRepo))
// create pipeline
pipelineName := uniqueString("pipeline")
outRepo := pps.PipelineRepo(ppsutil.NewPipeline(pipelineName))
require.NoError(t, ppsutil.CreatePipeline(
pachClient,
pipelineName,
"",
[]string{"cp", path.Join("/pfs", dataRepo, "file"), "/pfs/out/file"},
nil,
1,
[]*pps.PipelineInput{{Repo: &pfs.Repo{Name: dataRepo}}},
))
// Do first commit to repo
commit1, err := pfsutil.StartCommit(pachClient, dataRepo, "")
require.NoError(t, err)
_, err = pfsutil.PutFile(pachClient, dataRepo, commit1.Id, "file", 0, strings.NewReader("foo\n"))
require.NoError(t, err)
require.NoError(t, pfsutil.FinishCommit(pachClient, dataRepo, commit1.Id))
listCommitRequest := &pfs.ListCommitRequest{
Repo: []*pfs.Repo{outRepo},
CommitType: pfs.CommitType_COMMIT_TYPE_READ,
Block: true,
}
listCommitResponse, err := pachClient.ListCommit(
context.Background(),
listCommitRequest,
)
require.NoError(t, err)
outCommits := listCommitResponse.CommitInfo
require.Equal(t, 1, len(outCommits))
var buffer bytes.Buffer
require.NoError(t, pfsutil.GetFile(pachClient, outRepo.Name, outCommits[0].Commit.Id, "file", 0, 0, "", nil, &buffer))
require.Equal(t, "foo\n", buffer.String())
// Do second commit to repo
commit2, err := pfsutil.StartCommit(pachClient, dataRepo, commit1.Id)
require.NoError(t, err)
_, err = pfsutil.PutFile(pachClient, dataRepo, commit2.Id, "file", 0, strings.NewReader("bar\n"))
require.NoError(t, err)
require.NoError(t, pfsutil.FinishCommit(pachClient, dataRepo, commit2.Id))
listCommitRequest = &pfs.ListCommitRequest{
Repo: []*pfs.Repo{outRepo},
FromCommit: []*pfs.Commit{outCommits[0].Commit},
CommitType: pfs.CommitType_COMMIT_TYPE_READ,
Block: true,
}
listCommitResponse, err = pachClient.ListCommit(
context.Background(),
listCommitRequest,
)
require.NoError(t, err)
require.NotNil(t, listCommitResponse.CommitInfo[0].ParentCommit)
require.Equal(t, outCommits[0].Commit.Id, listCommitResponse.CommitInfo[0].ParentCommit.Id)
outCommits = listCommitResponse.CommitInfo
require.Equal(t, 1, len(outCommits))
buffer = bytes.Buffer{}
require.NoError(t, pfsutil.GetFile(pachClient, outRepo.Name, outCommits[0].Commit.Id, "file", 0, 0, "", nil, &buffer))
require.Equal(t, "foo\nbar\n", buffer.String())
}
func testMount(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
repositoryName := "testMountRepo"
err := pfsutil.CreateRepo(apiClient, repositoryName)
require.NoError(t, err)
directory := "/compile/testMount"
mounter := fuse.NewMounter(apiClient)
err = mounter.Mount(repositoryName, directory, "", 0, 1)
require.NoError(t, err)
_, err = os.Stat(filepath.Join(directory, "scratch"))
require.NoError(t, err)
commit, err := pfsutil.StartCommit(apiClient, repositoryName, "scratch")
require.NoError(t, err)
require.NotNil(t, commit)
newCommitID := commit.Id
_, err = os.Stat(filepath.Join(directory, newCommitID))
require.NoError(t, err)
err = ioutil.WriteFile(filepath.Join(directory, newCommitID, "foo"), []byte("foo"), 0666)
require.NoError(t, err)
_, err = pfsutil.PutFile(apiClient, repositoryName, newCommitID, "bar", 0, strings.NewReader("bar"))
require.NoError(t, err)
bigValue := make([]byte, 1024*1024)
for i := 0; i < 1024*1024; i++ {
bigValue[i] = 'a'
}
err = ioutil.WriteFile(filepath.Join(directory, newCommitID, "big1"), bigValue, 0666)
require.NoError(t, err)
_, err = pfsutil.PutFile(apiClient, repositoryName, newCommitID, "big2", 0, bytes.NewReader(bigValue))
require.NoError(t, err)
err = pfsutil.FinishCommit(apiClient, repositoryName, newCommitID)
require.NoError(t, err)
fInfo, err := os.Stat(filepath.Join(directory, newCommitID, "foo"))
require.NoError(t, err)
require.Equal(t, int64(3), fInfo.Size())
data, err := ioutil.ReadFile(filepath.Join(directory, newCommitID, "foo"))
require.NoError(t, err)
require.Equal(t, "foo", string(data))
data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "bar"))
require.NoError(t, err)
require.Equal(t, "bar", string(data))
data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "big1"))
require.NoError(t, err)
require.Equal(t, bigValue, data)
data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "big2"))
require.NoError(t, err)
require.Equal(t, bigValue, data)
err = mounter.Unmount(directory)
require.NoError(t, err)
err = mounter.Wait(directory)
require.NoError(t, err)
}
func testSimple(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
repositoryName := TestRepositoryName()
err := pfsutil.InitRepository(apiClient, repositoryName)
require.NoError(t, err)
commitInfo, err := pfsutil.GetCommitInfo(apiClient, repositoryName, "scratch")
require.NoError(t, err)
require.NotNil(t, commitInfo)
require.Equal(t, "scratch", commitInfo.Commit.Id)
require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, commitInfo.CommitType)
require.Nil(t, commitInfo.ParentCommit)
scratchCommitInfo := commitInfo
commitInfos, err := pfsutil.ListCommits(apiClient, repositoryName)
require.NoError(t, err)
require.Equal(t, 1, len(commitInfos.CommitInfo))
require.Equal(t, scratchCommitInfo, commitInfos.CommitInfo[0])
commit, err := pfsutil.Branch(apiClient, repositoryName, "scratch")
require.NoError(t, err)
require.NotNil(t, commit)
newCommitID := commit.Id
commitInfo, err = pfsutil.GetCommitInfo(apiClient, repositoryName, newCommitID)
require.NoError(t, err)
require.NotNil(t, commitInfo)
require.Equal(t, newCommitID, commitInfo.Commit.Id)
require.Equal(t, pfs.CommitType_COMMIT_TYPE_WRITE, commitInfo.CommitType)
require.Equal(t, "scratch", commitInfo.ParentCommit.Id)
newCommitInfo := commitInfo
commitInfos, err = pfsutil.ListCommits(apiClient, repositoryName)
require.NoError(t, err)
require.Equal(t, 2, len(commitInfos.CommitInfo))
require.Equal(t, newCommitInfo, commitInfos.CommitInfo[0])
require.Equal(t, scratchCommitInfo, commitInfos.CommitInfo[1])
err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/b")
require.NoError(t, err)
err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/c")
require.NoError(t, err)
doWrites(t, apiClient, repositoryName, newCommitID)
err = pfsutil.Write(apiClient, repositoryName, newCommitID)
require.NoError(t, err)
commitInfo, err = pfsutil.GetCommitInfo(apiClient, repositoryName, newCommitID)
require.NoError(t, err)
require.NotNil(t, commitInfo)
require.Equal(t, newCommitID, commitInfo.Commit.Id)
require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, commitInfo.CommitType)
require.Equal(t, "scratch", commitInfo.ParentCommit.Id)
checkWrites(t, apiClient, repositoryName, newCommitID)
fileInfos, err := pfsutil.ListFiles(apiClient, repositoryName, newCommitID, "a/b", 0, 1)
require.NoError(t, err)
require.Equal(t, testSize, len(fileInfos.FileInfo))
fileInfos, err = pfsutil.ListFiles(apiClient, repositoryName, newCommitID, "a/c", 0, 1)
require.NoError(t, err)
require.Equal(t, testSize, len(fileInfos.FileInfo))
var fileInfos2 [7][]*pfs.FileInfo
var wg sync.WaitGroup
for i := 0; i < 7; i++ {
i := i
wg.Add(1)
go func() {
defer wg.Done()
fileInfos3, iErr := pfsutil.ListFiles(apiClient, repositoryName, newCommitID, "a/b", uint64(i), 7)
require.NoError(t, iErr)
fileInfos2[i] = fileInfos3.FileInfo
}()
}
wg.Wait()
count := 0
for i := 0; i < 7; i++ {
count += len(fileInfos2[i])
}
require.Equal(t, testSize, count)
}