func testFailures(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
repositoryName := "testFailuresRepo"
err := pfsutil.CreateRepo(apiClient, repositoryName)
require.NoError(t, err)
commit, err := pfsutil.StartCommit(apiClient, repositoryName, "scratch")
require.NoError(t, err)
require.NotNil(t, commit)
newCommitID := commit.Id
err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/b")
require.NoError(t, err)
err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/c")
require.NoError(t, err)
doWrites(t, apiClient, repositoryName, newCommitID)
err = pfsutil.FinishCommit(apiClient, repositoryName, newCommitID)
require.NoError(t, err)
checkWrites(t, apiClient, repositoryName, newCommitID)
cluster.KillRoleAssigner()
for server := 0; server < testNumReplicas; server++ {
cluster.Kill(server)
}
cluster.RestartRoleAssigner()
cluster.WaitForAvailability()
checkWrites(t, apiClient, repositoryName, newCommitID)
}
func TestGetNameToNodeInfo(t *testing.T) {
pipeline, err := parse.NewParser().ParsePipeline("../parse/testdata/basic")
require.NoError(t, err)
nodeInfos, err := getNameToNodeInfo(pipeline.NameToNode)
require.NoError(t, err)
require.Equal(t, []string{"bar-node"}, nodeInfos["baz-node-bar-in-bar-out-in"].Parents)
}
func runTestRethink(t *testing.T, testFunc func(*testing.T, Client)) {
client, err := getRethinkSession()
require.NoError(t, err)
defer func() {
require.NoError(t, client.Close())
}()
testFunc(t, client)
}
func testMountBig(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
repositoryName := "testMountBigRepo"
err := pfsutil.CreateRepo(apiClient, repositoryName)
require.NoError(t, err)
directory := "/compile/testMount"
mounter := fuse.NewMounter(apiClient)
err = mounter.Mount(repositoryName, "", directory, 0, 1)
require.NoError(t, err)
_, err = os.Stat(filepath.Join(directory, "scratch"))
require.NoError(t, err)
commit, err := pfsutil.StartCommit(apiClient, repositoryName, "scratch")
require.NoError(t, err)
require.NotNil(t, commit)
newCommitID := commit.Id
bigValue := make([]byte, 1024*1024*300)
for i := 0; i < 1024*1024*300; i++ {
bigValue[i] = 'a'
}
wg := sync.WaitGroup{}
for j := 0; j < 5; j++ {
wg.Add(1)
go func(j int) {
defer wg.Done()
err := ioutil.WriteFile(filepath.Join(directory, newCommitID, fmt.Sprintf("big%d", j)), bigValue, 0666)
require.NoError(t, err)
}(j)
}
wg.Wait()
err = pfsutil.FinishCommit(apiClient, repositoryName, newCommitID)
require.NoError(t, err)
wg = sync.WaitGroup{}
for j := 0; j < 5; j++ {
wg.Add(1)
go func(j int) {
defer wg.Done()
data, err := ioutil.ReadFile(filepath.Join(directory, newCommitID, fmt.Sprintf("big%d", j)))
require.NoError(t, err)
require.Equal(t, bigValue, data)
}(j)
}
wg.Wait()
err = mounter.Unmount(directory)
require.NoError(t, err)
err = mounter.Wait(directory)
require.NoError(t, err)
}
func doWrites(tb testing.TB, apiClient pfs.ApiClient, repositoryName string, commitID string) {
var wg sync.WaitGroup
defer wg.Wait()
for i := 0; i < testSize; i++ {
i := i
wg.Add(1)
go func() {
defer wg.Done()
_, iErr := pfsutil.PutFile(apiClient, repositoryName, commitID,
fmt.Sprintf("a/b/file%d", i), 0, strings.NewReader(fmt.Sprintf("hello%d", i)))
require.NoError(tb, iErr)
_, iErr = pfsutil.PutFile(apiClient, repositoryName, commitID,
fmt.Sprintf("a/c/file%d", i), 0, strings.NewReader(fmt.Sprintf("hello%d", i)))
require.NoError(tb, iErr)
}()
}
}
func testRun(t *testing.T, imageName string, commands []string, expectedStdout string, expectedStderr string) {
client, err := newTestDockerClient()
require.NoError(t, err)
err = client.Pull(imageName, PullOptions{})
require.NoError(t, err)
containers, err := client.Create(
imageName,
CreateOptions{
HasCommand: commands != nil,
},
)
require.NoError(t, err)
require.Equal(t, 1, len(containers))
container := containers[0]
err = client.Start(
container,
StartOptions{
Commands: commands,
},
)
require.NoError(t, err)
err = client.Wait(container, WaitOptions{})
require.NoError(t, err)
stdout := bytes.NewBuffer(nil)
stderr := bytes.NewBuffer(nil)
err = client.Logs(container, LogsOptions{Stdout: stdout, Stderr: stderr})
require.NoError(t, err)
require.Equal(t, expectedStdout, stdout.String())
require.Equal(t, expectedStderr, stderr.String())
err = client.Remove(container, RemoveOptions{})
require.NoError(t, err)
}
func TestBasic(t *testing.T) {
dirPath, err := ioutil.TempDir("", "pachyderm")
require.NoError(t, err)
err = GithubClone(
dirPath,
"pachyderm",
"pachyderm",
"master",
"11eb4c6e1945beb1e6ce3e878ed2cb6c24ee8bf1",
"",
)
require.NoError(t, err)
//data, err := ioutil.ReadFile(filepath.Join(dirPath, "README.md"))
//require.NoError(t, err)
//require.Equal(
//t,
//`pfs
//===
//The Pachyderm Filesystem`,
//string(data),
//)
}
func checkWrites(tb testing.TB, apiClient pfs.ApiClient, repositoryName string, commitID string) {
var wg sync.WaitGroup
defer wg.Wait()
for i := 0; i < testSize; i++ {
i := i
wg.Add(1)
go func() {
defer wg.Done()
buffer := bytes.NewBuffer(nil)
iErr := pfsutil.GetFile(apiClient, repositoryName, commitID,
fmt.Sprintf("a/b/file%d", i), 0, math.MaxInt64, buffer)
require.NoError(tb, iErr)
require.Equal(tb, fmt.Sprintf("hello%d", i), buffer.String())
buffer = bytes.NewBuffer(nil)
iErr = pfsutil.GetFile(apiClient, repositoryName, commitID,
fmt.Sprintf("a/c/file%d", i), 0, math.MaxInt64, buffer)
require.NoError(tb, iErr)
require.Equal(tb, fmt.Sprintf("hello%d", i), buffer.String())
}()
}
}
func runTest(
t *testing.T,
f func(t *testing.T, apiClient pps.ApiClient),
) {
containerClient, err := getContainerClient()
require.NoError(t, err)
storeClient, err := getRethinkClient()
require.NoError(t, err)
pfstesting.RunTest(
t,
func(t *testing.T, apiClient pfs.ApiClient, internalApiClient pfs.InternalApiClient, cluster pfstesting.Cluster) {
prototest.RunT(
t,
testNumServers,
func(servers map[string]*grpc.Server) {
for _, server := range servers {
pps.RegisterApiServer(server, newAPIServer(apiClient, containerClient, storeClient, timing.NewSystemTimer()))
}
},
func(t *testing.T, clientConns map[string]*grpc.ClientConn) {
var clientConn *grpc.ClientConn
for _, c := range clientConns {
clientConn = c
break
}
f(
t,
pps.NewApiClient(
clientConn,
),
)
},
)
},
)
}
func testBasic(t *testing.T, client Client) {
pipelineRun := &pps.PipelineRun{
Id: "id",
PipelineId: "pipeline_id",
}
require.NoError(t, client.CreatePipelineRun(pipelineRun))
pipelineRunResponse, err := client.GetPipelineRun("id")
require.NoError(t, err)
require.Equal(t, pipelineRun, pipelineRunResponse)
require.NoError(t, client.CreatePipelineRunStatus("id", pps.PipelineRunStatusType_PIPELINE_RUN_STATUS_TYPE_CREATED))
pipelineRunStatusResponse, err := client.GetAllPipelineRunStatuses("id")
require.NoError(t, err)
require.Equal(t, pipelineRunStatusResponse[0].PipelineRunStatusType, pps.PipelineRunStatusType_PIPELINE_RUN_STATUS_TYPE_CREATED)
require.NoError(t, client.CreatePipelineRunStatus("id", pps.PipelineRunStatusType_PIPELINE_RUN_STATUS_TYPE_SUCCESS))
pipelineRunStatusResponse, err = client.GetAllPipelineRunStatuses("id")
require.NoError(t, err)
require.Equal(t, pipelineRunStatusResponse[0].PipelineRunStatusType, pps.PipelineRunStatusType_PIPELINE_RUN_STATUS_TYPE_SUCCESS)
require.NoError(t, client.CreatePipelineRunContainers(&pps.PipelineRunContainer{PipelineRunId: "id", ContainerId: "container", Node: "node"}))
containerIDs, err := client.GetPipelineRunContainers("id")
require.NoError(t, err)
require.Equal(t, []*pps.PipelineRunContainer{&pps.PipelineRunContainer{PipelineRunId: "id", ContainerId: "container", Node: "node"}}, containerIDs)
}
func checkBlockWrites(tb testing.TB, apiClient pfs.ApiClient, repositoryName string, commitID string) {
var wg sync.WaitGroup
defer wg.Wait()
for i := 0; i < testSize; i++ {
i := i
wg.Add(1)
go func() {
defer wg.Done()
buffer := bytes.NewBuffer(nil)
sharder := route.NewSharder(testShardsPerServer*testNumServers, testNumReplicas)
block := sharder.GetBlock([]byte(fmt.Sprintf("hello%d", i)))
iErr := pfsutil.GetBlock(apiClient, block.Hash, buffer)
require.NoError(tb, iErr)
// buffer = bytes.NewBuffer(nil)
// require.Equal(tb, fmt.Sprintf("hello%d", i), buffer.String())
// iErr = pfsutil.GetFile(apiClient, repositoryName, commitID,
// fmt.Sprintf("a/d/file%d", i), 0, math.MaxInt64, buffer)
// require.NoError(tb, iErr)
// require.Equal(tb, fmt.Sprintf("hello%d", i), buffer.String())
}()
}
}
func runTest(t *testing.T, client Client) {
err := client.Set("foo", "one", 0)
require.NoError(t, err)
value, err := client.Get("foo")
require.NoError(t, err)
require.Equal(t, "one", value)
//values, err := client.GetAll("foo")
//require.NoError(t, err)
//require.Equal(t, map[string]string{"foo": "one"}, values)
err = client.Set("a/b/foo", "one", 0)
require.NoError(t, err)
err = client.Set("a/b/bar", "two", 0)
require.NoError(t, err)
values, err := client.GetAll("a/b")
require.NoError(t, err)
require.Equal(t, map[string]string{"a/b/foo": "one", "a/b/bar": "two"}, values)
require.NoError(t, client.Close())
}
func TestSimple(t *testing.T) {
driver1, err := NewDriver(getBtrfsRootDir(t), "drive.TestSimple")
require.NoError(t, err)
shards := make(map[uint64]bool)
shards[0] = true
repo := &pfs.Repo{Name: "repo1"}
require.NoError(t, driver1.CreateRepo(repo))
commit1 := &pfs.Commit{
Repo: repo,
Id: "commit1",
}
require.NoError(t, driver1.StartCommit(nil, commit1, shards))
file1 := &pfs.File{
Commit: commit1,
Path: "foo",
}
require.NoError(t, driver1.PutFile(file1, 0, 0, strings.NewReader("foo")))
require.NoError(t, driver1.FinishCommit(commit1, shards))
reader, err := driver1.GetFile(file1, 0)
require.NoError(t, err)
contents, err := ioutil.ReadAll(reader)
require.NoError(t, err)
require.Equal(t, string(contents), "foo")
commit2 := &pfs.Commit{
Repo: repo,
Id: "commit2",
}
require.NoError(t, driver1.StartCommit(commit1, commit2, shards))
file2 := &pfs.File{
Commit: commit2,
Path: "bar",
}
require.NoError(t, driver1.PutFile(file2, 0, 0, strings.NewReader("bar")))
require.NoError(t, driver1.FinishCommit(commit2, shards))
changes, err := driver1.ListChange(file2, commit1, 0)
require.NoError(t, err)
require.Equal(t, len(changes), 1)
require.Equal(t, changes[0].File, file2)
require.Equal(t, changes[0].OffsetBytes, uint64(0))
require.Equal(t, changes[0].SizeBytes, uint64(3))
//Replicate repo
driver2, err := NewDriver(getBtrfsRootDir(t), "drive.TestSimpleReplica")
require.NoError(t, err)
require.NoError(t, driver2.CreateRepo(repo))
var buffer bytes.Buffer
require.NoError(t, driver1.PullDiff(commit1, 0, &buffer))
require.NoError(t, driver2.PushDiff(commit1, 0, &buffer))
buffer = bytes.Buffer{}
require.NoError(t, driver1.PullDiff(commit2, 0, &buffer))
require.NoError(t, driver2.PushDiff(commit2, 0, &buffer))
reader, err = driver2.GetFile(file1, 0)
require.NoError(t, err)
contents, err = ioutil.ReadAll(reader)
require.NoError(t, err)
require.Equal(t, string(contents), "foo")
changes, err = driver2.ListChange(file2, commit1, 0)
require.NoError(t, err)
require.Equal(t, len(changes), 1)
require.Equal(t, changes[0].File, file2)
require.Equal(t, changes[0].OffsetBytes, uint64(0))
require.Equal(t, changes[0].SizeBytes, uint64(3))
}
func TestMasterReplica(t *testing.T) {
client, err := getEtcdClient()
require.NoError(t, err)
runMasterReplicaTest(t, client)
}
func TestEtcdWatch(t *testing.T) {
t.Parallel()
client, err := getEtcdClient()
require.NoError(t, err)
runWatchTest(t, client)
}
func TestBasic(t *testing.T) {
pipeline, err := NewParser().ParsePipeline("testdata/basic")
require.NoError(t, err)
fmt.Println(pipeline)
}
func testBasic(t *testing.T, apiClient pps.ApiClient) {
_ = os.RemoveAll("/tmp/pachyderm-test")
pipelineSource, err := apiClient.CreatePipelineSource(
context.Background(),
&pps.CreatePipelineSourceRequest{
PipelineSource: &pps.PipelineSource{
TypedPipelineSource: &pps.PipelineSource_GithubPipelineSource{
GithubPipelineSource: &pps.GithubPipelineSource{
ContextDir: "src/pps/server/testdata/basic",
User: "******",
Repository: "pachyderm",
Branch: "master",
},
},
},
},
)
require.NoError(t, err)
pipeline, err := apiClient.CreateAndGetPipeline(
context.Background(),
&pps.CreateAndGetPipelineRequest{
PipelineSourceId: pipelineSource.Id,
},
)
require.NoError(t, err)
pipelineRun, err := apiClient.CreatePipelineRun(
context.Background(),
&pps.CreatePipelineRunRequest{
PipelineId: pipeline.Id,
},
)
require.NoError(t, err)
_, err = apiClient.StartPipelineRun(
context.Background(),
&pps.StartPipelineRunRequest{
PipelineRunId: pipelineRun.Id,
},
)
require.NoError(t, err)
pipelineRunID := pipelineRun.Id
pipelineRunStatus, err := getFinalPipelineRunStatus(apiClient, pipelineRunID)
require.NoError(t, err)
require.Equal(t, pps.PipelineRunStatusType_PIPELINE_RUN_STATUS_TYPE_SUCCESS, pipelineRunStatus.PipelineRunStatusType)
matches, err := filepath.Glob("/tmp/pachyderm-test/1-out/*")
require.NoError(t, err)
require.Equal(
t,
[]string{
"/tmp/pachyderm-test/1-out/1.txt",
"/tmp/pachyderm-test/1-out/10.txt",
"/tmp/pachyderm-test/1-out/2.txt",
"/tmp/pachyderm-test/1-out/20.txt",
"/tmp/pachyderm-test/1-out/3.txt",
"/tmp/pachyderm-test/1-out/30.txt",
"/tmp/pachyderm-test/1-out/4.txt",
"/tmp/pachyderm-test/1-out/40.txt",
"/tmp/pachyderm-test/1-out/5.txt",
"/tmp/pachyderm-test/1-out/50.txt",
},
matches,
)
matches, err = filepath.Glob("/tmp/pachyderm-test/2-out/*")
require.NoError(t, err)
require.Equal(
t,
[]string{
"/tmp/pachyderm-test/2-out/1.txt.copy",
"/tmp/pachyderm-test/2-out/10.txt.copy",
"/tmp/pachyderm-test/2-out/2.txt.copy",
"/tmp/pachyderm-test/2-out/20.txt.copy",
"/tmp/pachyderm-test/2-out/3.txt.copy",
"/tmp/pachyderm-test/2-out/30.txt.copy",
"/tmp/pachyderm-test/2-out/4.txt.copy",
"/tmp/pachyderm-test/2-out/40.txt.copy",
"/tmp/pachyderm-test/2-out/5.txt.copy",
"/tmp/pachyderm-test/2-out/50.txt.copy",
},
matches,
)
matches, err = filepath.Glob("/tmp/pachyderm-test/3-out/*")
require.NoError(t, err)
require.Equal(
t,
[]string{
"/tmp/pachyderm-test/3-out/1.txt.copy3",
"/tmp/pachyderm-test/3-out/10.txt.copy3",
"/tmp/pachyderm-test/3-out/2.txt.copy3",
"/tmp/pachyderm-test/3-out/20.txt.copy3",
"/tmp/pachyderm-test/3-out/3.txt.copy3",
"/tmp/pachyderm-test/3-out/30.txt.copy3",
"/tmp/pachyderm-test/3-out/4.txt.copy3",
"/tmp/pachyderm-test/3-out/40.txt.copy3",
"/tmp/pachyderm-test/3-out/5.txt.copy3",
"/tmp/pachyderm-test/3-out/50.txt.copy3",
},
matches,
)
matches, err = filepath.Glob("/tmp/pachyderm-test/4-out/*")
require.NoError(t, err)
require.Equal(
t,
[]string{
"/tmp/pachyderm-test/4-out/1.txt.copy4",
"/tmp/pachyderm-test/4-out/10.txt.copy4",
"/tmp/pachyderm-test/4-out/2.txt.copy4",
"/tmp/pachyderm-test/4-out/20.txt.copy4",
"/tmp/pachyderm-test/4-out/3.txt.copy4",
"/tmp/pachyderm-test/4-out/30.txt.copy4",
"/tmp/pachyderm-test/4-out/4.txt.copy4",
"/tmp/pachyderm-test/4-out/40.txt.copy4",
"/tmp/pachyderm-test/4-out/5.txt.copy4",
"/tmp/pachyderm-test/4-out/50.txt.copy4",
"/tmp/pachyderm-test/4-out/build-file.txt4",
},
matches,
)
matches, err = filepath.Glob("/tmp/pachyderm-test/5-out/*")
require.NoError(t, err)
require.Equal(
t,
[]string{
"/tmp/pachyderm-test/5-out/1.txt.copy3",
"/tmp/pachyderm-test/5-out/1.txt.copy4",
"/tmp/pachyderm-test/5-out/10.txt.copy3",
"/tmp/pachyderm-test/5-out/10.txt.copy4",
"/tmp/pachyderm-test/5-out/2.txt.copy3",
"/tmp/pachyderm-test/5-out/2.txt.copy4",
"/tmp/pachyderm-test/5-out/20.txt.copy3",
"/tmp/pachyderm-test/5-out/20.txt.copy4",
"/tmp/pachyderm-test/5-out/3.txt.copy3",
"/tmp/pachyderm-test/5-out/3.txt.copy4",
"/tmp/pachyderm-test/5-out/30.txt.copy3",
"/tmp/pachyderm-test/5-out/30.txt.copy4",
"/tmp/pachyderm-test/5-out/4.txt.copy3",
"/tmp/pachyderm-test/5-out/4.txt.copy4",
"/tmp/pachyderm-test/5-out/40.txt.copy3",
"/tmp/pachyderm-test/5-out/40.txt.copy4",
"/tmp/pachyderm-test/5-out/5.txt.copy3",
"/tmp/pachyderm-test/5-out/5.txt.copy4",
"/tmp/pachyderm-test/5-out/50.txt.copy3",
"/tmp/pachyderm-test/5-out/50.txt.copy4",
"/tmp/pachyderm-test/5-out/build-file.txt4",
"/tmp/pachyderm-test/5-out/build-file2.txt",
},
matches,
)
}
func TestBuildWithError(t *testing.T) {
intC := make(chan int, 5)
nameToNodeInfo := map[string]*NodeInfo{
"1": &NodeInfo{
Parents: []string{},
},
"2": &NodeInfo{
Parents: []string{},
},
"3-1": &NodeInfo{
Parents: []string{
"1",
"2",
},
},
"3-2": &NodeInfo{
Parents: []string{
"1",
"2",
},
},
"3-3": &NodeInfo{
Parents: []string{
"1",
"2",
},
},
"4-1": &NodeInfo{
Parents: []string{
"3-1",
"3-2",
"3-3",
},
},
"4-2": &NodeInfo{
Parents: []string{
"3-1",
"3-2",
"3-3",
},
},
"5": &NodeInfo{
Parents: []string{
"4-1",
"4-2",
},
},
}
counter := int32(0)
nameToNodeFunc := map[string]func() error{
"1": testNodeFunc(&counter, intC, "1", 1, ""),
"2": testNodeFunc(&counter, intC, "2", 2, ""),
"3-1": testNodeFunc(&counter, intC, "3-1", 3, "3-1:error"),
"3-2": testNodeFunc(&counter, intC, "3-2", 4, ""),
"3-3": testNodeFunc(&counter, intC, "3-3", 5, ""),
"4-1": testNodeFunc(&counter, intC, "4-1", 6, ""),
"4-2": testNodeFunc(&counter, intC, "4-2", 7, ""),
"5": testNodeFunc(&counter, intC, "5", 8, ""),
}
run, err := build(nameToNodeInfo, nameToNodeFunc)
require.NoError(t, err)
err = run.Do()
require.NotNil(t, err)
require.Equal(t, "3-1:error", err.Error())
require.Equal(t, int32(5), counter)
i := <-intC
require.True(t, i == 1 || i == 2)
i = <-intC
require.True(t, i == 1 || i == 2)
i = <-intC
require.True(t, i == 3 || i == 4 || i == 5)
i = <-intC
require.True(t, i == 3 || i == 4 || i == 5)
i = <-intC
require.True(t, i == 3 || i == 4 || i == 5)
}
func testMount(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
repositoryName := "testMountRepo"
err := pfsutil.CreateRepo(apiClient, repositoryName)
require.NoError(t, err)
directory := "/compile/testMount"
mounter := fuse.NewMounter(apiClient)
err = mounter.Mount(repositoryName, directory, "", 0, 1)
require.NoError(t, err)
_, err = os.Stat(filepath.Join(directory, "scratch"))
require.NoError(t, err)
commit, err := pfsutil.StartCommit(apiClient, repositoryName, "scratch")
require.NoError(t, err)
require.NotNil(t, commit)
newCommitID := commit.Id
_, err = os.Stat(filepath.Join(directory, newCommitID))
require.NoError(t, err)
err = ioutil.WriteFile(filepath.Join(directory, newCommitID, "foo"), []byte("foo"), 0666)
require.NoError(t, err)
_, err = pfsutil.PutFile(apiClient, repositoryName, newCommitID, "bar", 0, strings.NewReader("bar"))
require.NoError(t, err)
bigValue := make([]byte, 1024*1024)
for i := 0; i < 1024*1024; i++ {
bigValue[i] = 'a'
}
err = ioutil.WriteFile(filepath.Join(directory, newCommitID, "big1"), bigValue, 0666)
require.NoError(t, err)
_, err = pfsutil.PutFile(apiClient, repositoryName, newCommitID, "big2", 0, bytes.NewReader(bigValue))
require.NoError(t, err)
err = pfsutil.FinishCommit(apiClient, repositoryName, newCommitID)
require.NoError(t, err)
fInfo, err := os.Stat(filepath.Join(directory, newCommitID, "foo"))
require.NoError(t, err)
require.Equal(t, int64(3), fInfo.Size())
data, err := ioutil.ReadFile(filepath.Join(directory, newCommitID, "foo"))
require.NoError(t, err)
require.Equal(t, "foo", string(data))
data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "bar"))
require.NoError(t, err)
require.Equal(t, "bar", string(data))
data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "big1"))
require.NoError(t, err)
require.Equal(t, bigValue, data)
data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "big2"))
require.NoError(t, err)
require.Equal(t, bigValue, data)
err = mounter.Unmount(directory)
require.NoError(t, err)
err = mounter.Wait(directory)
require.NoError(t, err)
}
func testSimple(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
repositoryName := "testSimpleRepo"
err := pfsutil.CreateRepo(apiClient, repositoryName)
require.NoError(t, err)
scratchCommitInfo, err := pfsutil.InspectCommit(apiClient, repositoryName, "scratch")
require.NoError(t, err)
require.NotNil(t, scratchCommitInfo)
require.Equal(t, "scratch", scratchCommitInfo.Commit.Id)
require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, scratchCommitInfo.CommitType)
require.Nil(t, scratchCommitInfo.ParentCommit)
commitInfos, err := pfsutil.ListCommit(apiClient, repositoryName)
require.NoError(t, err)
require.Equal(t, 1, len(commitInfos))
require.Equal(t, scratchCommitInfo.Commit, commitInfos[0].Commit)
commit, err := pfsutil.StartCommit(apiClient, repositoryName, "scratch")
require.NoError(t, err)
require.NotNil(t, commit)
newCommitID := commit.Id
newCommitInfo, err := pfsutil.InspectCommit(apiClient, repositoryName, newCommitID)
require.NoError(t, err)
require.NotNil(t, newCommitInfo)
require.Equal(t, newCommitID, newCommitInfo.Commit.Id)
require.Equal(t, pfs.CommitType_COMMIT_TYPE_WRITE, newCommitInfo.CommitType)
require.Equal(t, "scratch", newCommitInfo.ParentCommit.Id)
commitInfos, err = pfsutil.ListCommit(apiClient, repositoryName)
require.NoError(t, err)
require.Equal(t, 2, len(commitInfos))
require.Equal(t, newCommitInfo.Commit, commitInfos[0].Commit)
require.Equal(t, scratchCommitInfo.Commit, commitInfos[1].Commit)
err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/b")
require.NoError(t, err)
err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/c")
require.NoError(t, err)
err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/d")
require.NoError(t, err)
doWrites(t, apiClient, repositoryName, newCommitID)
doBlockWrites(t, apiClient, repositoryName, newCommitID)
err = pfsutil.FinishCommit(apiClient, repositoryName, newCommitID)
require.NoError(t, err)
newCommitInfo, err = pfsutil.InspectCommit(apiClient, repositoryName, newCommitID)
require.NoError(t, err)
require.NotNil(t, newCommitInfo)
require.Equal(t, newCommitID, newCommitInfo.Commit.Id)
require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, newCommitInfo.CommitType)
require.Equal(t, "scratch", newCommitInfo.ParentCommit.Id)
checkWrites(t, apiClient, repositoryName, newCommitID)
checkBlockWrites(t, apiClient, repositoryName, newCommitID)
fileInfos, err := pfsutil.ListFile(apiClient, repositoryName, newCommitID, "a/b", 0, 1)
require.NoError(t, err)
require.Equal(t, testSize, len(fileInfos))
fileInfos, err = pfsutil.ListFile(apiClient, repositoryName, newCommitID, "a/c", 0, 1)
require.NoError(t, err)
require.Equal(t, testSize, len(fileInfos))
var fileInfos2 [7][]*pfs.FileInfo
var wg sync.WaitGroup
for i := 0; i < 7; i++ {
i := i
wg.Add(1)
go func() {
defer wg.Done()
fileInfos3, iErr := pfsutil.ListFile(apiClient, repositoryName, newCommitID, "a/b", uint64(i), 7)
require.NoError(t, iErr)
fileInfos2[i] = fileInfos3
}()
}
wg.Wait()
count := 0
for i := 0; i < 7; i++ {
count += len(fileInfos2[i])
}
require.Equal(t, testSize, count)
}