func TestMountCachingViaShell(t *testing.T) {
if testing.Short() {
t.Skip("Skipped because of short mode")
}
testFuse(t, func(c client.APIClient, mountpoint string) {
repo1 := "foo"
require.NoError(t, c.CreateRepo(repo1))
// Now if we walk the FS we should see the file.
// This first act of 'walking' should also initiate the cache
ls := exec.Command("ls")
ls.Dir = mountpoint
out, err := ls.Output()
require.NoError(t, err)
require.Equal(t, "foo\n", string(out))
// Now create another repo, and look for it under the mount point
repo2 := "bar"
require.NoError(t, c.CreateRepo(repo2))
// Now if we walk the FS we should see the new file.
// This second ls on mac doesn't report the file!
ls = exec.Command("ls")
ls.Dir = mountpoint
out, err = ls.Output()
require.NoError(t, err)
require.Equal(t, true, "foo\nbar\n" == string(out) || "bar\nfoo\n" == string(out))
})
}
func testBlock(t *testing.T, apiServer persist.APIServer) {
jobInfo, err := apiServer.CreateJobInfo(context.Background(), &persist.JobInfo{
JobID: uuid.NewWithoutDashes(),
})
require.NoError(t, err)
jobID := jobInfo.JobID
go func() {
_, err := apiServer.CreateJobOutput(
context.Background(),
&persist.JobOutput{
JobID: jobID,
OutputCommit: client.NewCommit("foo", "bar"),
})
require.NoError(t, err)
_, err = apiServer.CreateJobState(
context.Background(),
&persist.JobState{
JobID: jobID,
State: ppsclient.JobState_JOB_SUCCESS,
})
require.NoError(t, err)
}()
_, err = apiServer.InspectJob(
context.Background(),
&ppsclient.InspectJobRequest{
Job: &ppsclient.Job{ID: jobID},
BlockState: true,
},
)
require.NoError(t, err)
}
func TestCommitOpenReadDir(t *testing.T) {
if testing.Short() {
t.Skip("Skipped because of short mode")
}
testFuse(t, func(c client.APIClient, mountpoint string) {
repoName := "foo"
require.NoError(t, c.CreateRepo(repoName))
commit, err := c.StartCommit(repoName, "", "")
require.NoError(t, err)
t.Logf("open commit %v", commit.ID)
const (
greetingName = "greeting"
greeting = "Hello, world\n"
greetingPerm = 0644
)
require.NoError(t, ioutil.WriteFile(filepath.Join(mountpoint, repoName, commit.ID, greetingName), []byte(greeting), greetingPerm))
const (
scriptName = "script"
script = "#!/bin/sh\necho foo\n"
scriptPerm = 0750
)
require.NoError(t, ioutil.WriteFile(filepath.Join(mountpoint, repoName, commit.ID, scriptName), []byte(script), scriptPerm))
require.NoError(t, fstestutil.CheckDir(filepath.Join(mountpoint, repoName, commit.ID), map[string]fstestutil.FileInfoCheck{
greetingName: func(fi os.FileInfo) error {
// TODO respect greetingPerm
if g, e := fi.Mode(), os.FileMode(0666); g != e {
return fmt.Errorf("wrong mode: %v != %v", g, e)
}
if g, e := fi.Size(), int64(len(greeting)); g != e {
t.Errorf("wrong size: %v != %v", g, e)
}
// TODO show fileModTime as mtime
// if g, e := fi.ModTime().UTC(), fileModTime; g != e {
// t.Errorf("wrong mtime: %v != %v", g, e)
// }
return nil
},
scriptName: func(fi os.FileInfo) error {
// TODO respect scriptPerm
if g, e := fi.Mode(), os.FileMode(0666); g != e {
return fmt.Errorf("wrong mode: %v != %v", g, e)
}
if g, e := fi.Size(), int64(len(script)); g != e {
t.Errorf("wrong size: %v != %v", g, e)
}
// TODO show fileModTime as mtime
// if g, e := fi.ModTime().UTC(), fileModTime; g != e {
// t.Errorf("wrong mtime: %v != %v", g, e)
// }
return nil
},
}))
})
}
func TestSeekRead(t *testing.T) {
if testing.Short() {
t.Skip("Skipped because of short mode")
}
testFuse(t, func(c client.APIClient, mountpoint string) {
repo := "test"
require.NoError(t, c.CreateRepo(repo))
commit, err := c.StartCommit(repo, "", "")
require.NoError(t, err)
path := filepath.Join(mountpoint, repo, commit.ID, "file")
file, err := os.Create(path)
require.NoError(t, err)
_, err = file.Write([]byte("foobarbaz"))
require.NoError(t, err)
require.NoError(t, file.Close())
require.NoError(t, c.FinishCommit(repo, commit.ID))
fmt.Printf("==== Finished commit\n")
file, err = os.Open(path)
defer file.Close()
require.NoError(t, err)
word1 := make([]byte, 3)
n1, err := file.Read(word1)
require.NoError(t, err)
require.Equal(t, 3, n1)
require.Equal(t, "foo", string(word1))
fmt.Printf("==== %v - Read word len %v : %v\n", time.Now(), n1, string(word1))
offset, err := file.Seek(6, 0)
fmt.Printf("==== %v - err (%v)\n", time.Now(), err)
fmt.Printf("==== %v - offset (%v)\n", time.Now(), offset)
// MAC OS X BEHAVIOR - it allows seek
require.NoError(t, err)
require.Equal(t, int64(6), offset)
fmt.Printf("==== Seeked to %v\n", offset)
/* Leaving in place so the test's intention is clear / for repro'ing manually for mac */
word2 := make([]byte, 3)
n2, err := file.Read(word2)
require.NoError(t, err)
require.Equal(t, 3, n2)
require.Equal(t, "baz", string(word2))
fmt.Printf("==== Read word len %v : %v\n", n2, string(word2))
})
}
func Test296(t *testing.T) {
lion.SetLevel(lion.LevelDebug)
if testing.Short() {
t.Skip("Skipped because of short mode")
}
testFuse(t, func(c client.APIClient, mountpoint string) {
repo := "test"
require.NoError(t, c.CreateRepo(repo))
commit, err := c.StartCommit(repo, "", "")
require.NoError(t, err)
path := filepath.Join(mountpoint, repo, commit.ID, "file")
stdin := strings.NewReader(fmt.Sprintf("echo 1 >%s", path))
require.NoError(t, pkgexec.RunStdin(stdin, "sh"))
stdin = strings.NewReader(fmt.Sprintf("echo 2 >%s", path))
require.NoError(t, pkgexec.RunStdin(stdin, "sh"))
require.NoError(t, c.FinishCommit(repo, commit.ID))
commit2, err := c.StartCommit(repo, commit.ID, "")
require.NoError(t, err)
path = filepath.Join(mountpoint, repo, commit2.ID, "file")
stdin = strings.NewReader(fmt.Sprintf("echo 3 >%s", path))
require.NoError(t, pkgexec.RunStdin(stdin, "sh"))
require.NoError(t, c.FinishCommit(repo, commit2.ID))
data, err := ioutil.ReadFile(path)
require.NoError(t, err)
require.Equal(t, "3\n", string(data))
})
}
func TestOpenAndWriteFile(t *testing.T) {
if testing.Short() {
t.Skip("Skipped because of short mode")
}
testFuse(t, func(c client.APIClient, mountpoint string) {
require.NoError(t, c.CreateRepo("repo"))
commit1, err := c.StartCommit("repo", "", "")
require.NoError(t, err)
filePath := filepath.Join(mountpoint, "repo", commit1.ID, "foo")
f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
require.NoError(t, err)
defer func() {
err = f.Close()
require.NoError(t, err)
}()
data1 := []byte("something\nis\nrotten\n")
_, err = f.Write(data1)
require.NoError(t, err)
data2 := []byte("in\nthe\nstate\nof\nDenmark\n")
_, err = f.Write(data2)
require.NoError(t, err)
require.NoError(t, f.Sync())
require.NoError(t, c.FinishCommit("repo", commit1.ID))
result, err := ioutil.ReadFile(filePath)
require.NoError(t, err)
require.Equal(t, fmt.Sprintf("%v%v", string(data1), string(data2)), string(result))
})
}
func RunTestWithRethinkAPIServer(t *testing.T, testFunc func(t *testing.T, persistAPIServer persist.APIServer)) {
if testing.Short() {
t.Skip("Skipping test because of short mode.")
}
apiServer, err := NewTestRethinkAPIServer()
require.NoError(t, err)
defer func() {
require.NoError(t, apiServer.Close())
}()
testFunc(t, apiServer)
}
func TestRepoReadDir(t *testing.T) {
if testing.Short() {
t.Skip("Skipped because of short mode")
}
testFuse(t, func(c client.APIClient, mountpoint string) {
repoName := "foo"
require.NoError(t, c.CreateRepo(repoName))
commitA, err := c.StartCommit(repoName, "", "")
require.NoError(t, err)
require.NoError(t, c.FinishCommit(repoName, commitA.ID))
t.Logf("finished commit %v", commitA.ID)
commitB, err := c.StartCommit(repoName, "", "")
require.NoError(t, err)
t.Logf("open commit %v", commitB.ID)
require.NoError(t, fstestutil.CheckDir(filepath.Join(mountpoint, repoName), map[string]fstestutil.FileInfoCheck{
commitA.ID: func(fi os.FileInfo) error {
if g, e := fi.Mode(), os.ModeDir|0555; g != e {
return fmt.Errorf("wrong mode: %v != %v", g, e)
}
// TODO show commitSize in commit stat?
if g, e := fi.Size(), int64(0); g != e {
t.Errorf("wrong size: %v != %v", g, e)
}
// TODO show CommitInfo.StartTime as ctime, CommitInfo.Finished as mtime
// TODO test ctime via .Sys
// if g, e := fi.ModTime().UTC(), commitFinishTime; g != e {
// t.Errorf("wrong mtime: %v != %v", g, e)
// }
return nil
},
commitB.ID: func(fi os.FileInfo) error {
if g, e := fi.Mode(), os.ModeDir|0775; g != e {
return fmt.Errorf("wrong mode: %v != %v", g, e)
}
// TODO show commitSize in commit stat?
if g, e := fi.Size(), int64(0); g != e {
t.Errorf("wrong size: %v != %v", g, e)
}
// TODO show CommitInfo.StartTime as ctime, ??? as mtime
// TODO test ctime via .Sys
// if g, e := fi.ModTime().UTC(), commitFinishTime; g != e {
// t.Errorf("wrong mtime: %v != %v", g, e)
// }
return nil
},
}))
})
}
func TestCreateFileInDir(t *testing.T) {
lion.SetLevel(lion.LevelDebug)
if testing.Short() {
t.Skip("Skipped because of short mode")
}
testFuse(t, func(c client.APIClient, mountpoint string) {
require.NoError(t, c.CreateRepo("repo"))
commit, err := c.StartCommit("repo", "", "")
require.NoError(t, err)
require.NoError(t, os.Mkdir(filepath.Join(mountpoint, "repo", commit.ID, "dir"), 0700))
require.NoError(t, ioutil.WriteFile(filepath.Join(mountpoint, "repo", commit.ID, "dir", "file"), []byte("foo"), 0644))
require.NoError(t, c.FinishCommit("repo", commit.ID))
})
}
func testRun(t *testing.T, imageName string, commands []string, expectedStdout string, expectedStderr string) {
if testing.Short() {
t.Skip("Skipping integration tests in short mode")
}
client, err := newTestDockerClient()
require.NoError(t, err)
err = client.Pull(imageName, PullOptions{})
require.NoError(t, err)
container, err := client.Create(
imageName,
CreateOptions{
HasCommand: commands != nil,
},
)
require.NoError(t, err)
err = client.Start(
container,
StartOptions{
Commands: commands,
},
)
require.NoError(t, err)
err = client.Wait(container, WaitOptions{})
require.NoError(t, err)
stdout := bytes.NewBuffer(nil)
stderr := bytes.NewBuffer(nil)
err = client.Logs(container, LogsOptions{Stdout: stdout, Stderr: stderr})
require.NoError(t, err)
require.Equal(t, expectedStdout, stdout.String())
require.Equal(t, expectedStderr, stderr.String())
err = client.Remove(container, RemoveOptions{})
require.NoError(t, err)
}
func TestSeekWriteBackwards(t *testing.T) {
if testing.Short() {
t.Skip("Skipped because of short mode")
}
testFuse(t, func(c client.APIClient, mountpoint string) {
repo := "test"
require.NoError(t, c.CreateRepo(repo))
commit, err := c.StartCommit(repo, "", "")
require.NoError(t, err)
path := filepath.Join(mountpoint, repo, commit.ID, "file")
file, err := os.Create(path)
require.NoError(t, err)
defer func() {
// MAC OS X BEHAVIOR - it allows seek but this is an invalid write
require.YesError(t, file.Close())
}()
_, err = file.Write([]byte("foofoofoo"))
require.NoError(t, err)
err = file.Sync()
require.NoError(t, err)
offset, err := file.Seek(3, 0)
fmt.Printf("==== %v - err (%v)\n", time.Now(), err)
fmt.Printf("==== %v - offset (%v)\n", time.Now(), offset)
// MAC OS X BEHAVIOR - it allows seek
require.NoError(t, err)
require.Equal(t, int64(3), offset)
/* Leaving in place so the test's intention is clear / for repro'ing manually for mac */
err = file.Sync()
require.NoError(t, err)
n1, err := file.Write([]byte("bar"))
// MAC OS X BEHAVIOR - it allows seek
require.NoError(t, err)
require.Equal(t, 3, n1)
fmt.Printf("==== %v - write word len %v\n", time.Now(), n1)
require.NoError(t, c.FinishCommit(repo, commit.ID))
})
}
func TestBigWrite(t *testing.T) {
if testing.Short() {
t.Skip("Skipped because of short mode")
}
testFuse(t, func(c client.APIClient, mountpoint string) {
repo := "test"
require.NoError(t, c.CreateRepo(repo))
commit, err := c.StartCommit(repo, "", "")
require.NoError(t, err)
path := filepath.Join(mountpoint, repo, commit.ID, "file1")
stdin := strings.NewReader(fmt.Sprintf("yes | tr -d '\\n' | head -c 1000000 > %s", path))
require.NoError(t, pkgexec.RunStdin(stdin, "sh"))
require.NoError(t, c.FinishCommit(repo, commit.ID))
data, err := ioutil.ReadFile(path)
require.NoError(t, err)
require.Equal(t, bytes.Repeat([]byte{'y'}, 1000000), data)
})
}
func TestEtcdWatch(t *testing.T) {
if os.Getenv("ETCD_PORT_2379_TCP_ADDR") == "" {
t.Skip("skipping test; $ETCD_PORT_2379_TCP_ADDR not set")
}
t.Parallel()
client, err := getEtcdClient()
require.NoError(t, err)
runWatchTest(t, client)
}
func TestRootReadDir(t *testing.T) {
if testing.Short() {
t.Skip("Skipped because of short mode")
}
testFuse(t, func(c client.APIClient, mountpoint string) {
require.NoError(t, c.CreateRepo("one"))
require.NoError(t, c.CreateRepo("two"))
require.NoError(t, fstestutil.CheckDir(mountpoint, map[string]fstestutil.FileInfoCheck{
"one": func(fi os.FileInfo) error {
if g, e := fi.Mode(), os.ModeDir|0555; g != e {
return fmt.Errorf("wrong mode: %v != %v", g, e)
}
// TODO show repoSize in repo stat?
if g, e := fi.Size(), int64(0); g != e {
t.Errorf("wrong size: %v != %v", g, e)
}
// TODO show RepoInfo.Created as time
// if g, e := fi.ModTime().UTC(), repoModTime; g != e {
// t.Errorf("wrong mtime: %v != %v", g, e)
// }
return nil
},
"two": func(fi os.FileInfo) error {
if g, e := fi.Mode(), os.ModeDir|0555; g != e {
return fmt.Errorf("wrong mode: %v != %v", g, e)
}
// TODO show repoSize in repo stat?
if g, e := fi.Size(), int64(0); g != e {
t.Errorf("wrong size: %v != %v", g, e)
}
// TODO show RepoInfo.Created as time
// if g, e := fi.ModTime().UTC(), repoModTime; g != e {
// t.Errorf("wrong mtime: %v != %v", g, e)
// }
return nil
},
}))
})
}
func TestMountCachingViaWalk(t *testing.T) {
if testing.Short() {
t.Skip("Skipped because of short mode")
}
testFuse(t, func(c client.APIClient, mountpoint string) {
repo1 := "foo"
require.NoError(t, c.CreateRepo(repo1))
// Now if we walk the FS we should see the file.
// This first act of 'walking' should also initiate the cache
var filesSeen []interface{}
walkCallback := func(path string, info os.FileInfo, err error) error {
tokens := strings.Split(path, "/mnt")
filesSeen = append(filesSeen, tokens[len(tokens)-1])
return nil
}
err := filepath.Walk(mountpoint, walkCallback)
require.NoError(t, err)
require.OneOfEquals(t, "/foo", filesSeen)
// Now create another repo, and look for it under the mount point
repo2 := "bar"
require.NoError(t, c.CreateRepo(repo2))
// Now if we walk the FS we should see the new file.
// This now works. But originally (issue #205) this second ls on mac doesn't report the file!
filesSeen = make([]interface{}, 0)
err = filepath.Walk(mountpoint, walkCallback)
require.NoError(t, err)
require.OneOfEquals(t, "/foo", filesSeen)
require.OneOfEquals(t, "/bar", filesSeen)
})
}
func TestFruitStand(t *testing.T) {
require.NoError(t, exec.Command("pachctl", "create-repo", "data").Run())
require.NoError(t, exec.Command("pachctl", "start-commit", "data", "master").Run())
putFileCmd := exec.Command("pachctl", "put-file", "data", "master", "sales")
set1, err := os.Open("set1.txt")
require.NoError(t, err)
putFileCmd.Stdin = set1
require.NoError(t, putFileCmd.Run())
require.NoError(t, exec.Command("pachctl", "finish-commit", "data", "master").Run())
require.NoError(t, exec.Command("pachctl", "create-pipeline", "-f", "pipeline.json").Run())
time.Sleep(5 * time.Second)
require.NoError(t, exec.Command("pachctl", "flush-commit", "data/master").Run())
}
func TestScraper(t *testing.T) {
require.NoError(t, exec.Command("pachctl", "create-repo", "urls").Run())
require.NoError(t, exec.Command("pachctl", "start-commit", "urls", "master").Run())
putFileCmd := exec.Command("pachctl", "put-file", "urls", "master", "urls")
urls, err := os.Open("urls")
require.NoError(t, err)
putFileCmd.Stdin = urls
require.NoError(t, putFileCmd.Run())
require.NoError(t, exec.Command("pachctl", "finish-commit", "urls", "master").Run())
require.NoError(t, exec.Command("pachctl", "create-pipeline", "-f", "scraper.json").Run())
time.Sleep(5 * time.Second)
require.NoError(t, exec.Command("pachctl", "flush-commit", "urls/master").Run())
}
func TestOverwriteFile(t *testing.T) {
if testing.Short() {
t.Skip("Skipped because of short mode")
}
testFuse(t, func(c client.APIClient, mountpoint string) {
require.NoError(t, c.CreateRepo("repo"))
commit1, err := c.StartCommit("repo", "", "")
require.NoError(t, err)
_, err = c.PutFile("repo", commit1.ID, "file", strings.NewReader("foo\n"))
require.NoError(t, err)
require.NoError(t, c.FinishCommit("repo", commit1.ID))
commit2, err := c.StartCommit("repo", commit1.ID, "")
require.NoError(t, err)
path := filepath.Join(mountpoint, "repo", commit2.ID, "file")
stdin := strings.NewReader(fmt.Sprintf("echo bar >%s", path))
require.NoError(t, pkgexec.RunStdin(stdin, "sh"))
require.NoError(t, c.FinishCommit("repo", commit2.ID))
result, err := ioutil.ReadFile(path)
require.NoError(t, err)
require.Equal(t, "bar\n", string(result))
})
}
func TestSpacedWrites(t *testing.T) {
if testing.Short() {
t.Skip("Skipped because of short mode")
}
testFuse(t, func(c client.APIClient, mountpoint string) {
repo := "test"
require.NoError(t, c.CreateRepo(repo))
commit, err := c.StartCommit(repo, "", "")
require.NoError(t, err)
path := filepath.Join(mountpoint, repo, commit.ID, "file")
file, err := os.Create(path)
require.NoError(t, err)
_, err = file.Write([]byte("foo"))
require.NoError(t, err)
_, err = file.Write([]byte("foo"))
require.NoError(t, err)
require.NoError(t, file.Close())
require.NoError(t, c.FinishCommit(repo, commit.ID))
data, err := ioutil.ReadFile(path)
require.NoError(t, err)
require.Equal(t, "foofoo", string(data))
})
}
func TestSeekWriteGap(t *testing.T) {
if testing.Short() {
t.Skip("Skipped because of short mode")
}
testFuse(t, func(c client.APIClient, mountpoint string) {
repo := "test"
require.NoError(t, c.CreateRepo(repo))
commit, err := c.StartCommit(repo, "", "")
require.NoError(t, err)
path := filepath.Join(mountpoint, repo, commit.ID, "file")
file, err := os.Create(path)
require.NoError(t, err)
defer func() {
require.NoError(t, file.Close())
}()
_, err = file.Write([]byte("foo"))
require.NoError(t, err)
err = file.Sync()
require.NoError(t, err)
offset, err := file.Seek(6, 0)
fmt.Printf("==== %v - err (%v)\n", time.Now(), err)
fmt.Printf("==== %v - offset (%v)\n", time.Now(), offset)
require.YesError(t, err)
require.Equal(t, int64(0), offset)
/* Leaving in place so the test's intention is clear / for repro'ing manually for mac
err = file.Sync()
require.NoError(t, err)
n1, err := file.Write([]byte("baz"))
require.YesError(t, err)
require.Equal(t, 3, n1)
require.NoError(t, c.FinishCommit(repo, commit.ID)) */
})
}
func runTest(t *testing.T, client Client) {
err := client.Set("foo", "one", 0)
require.NoError(t, err)
value, err := client.Get("foo")
require.NoError(t, err)
require.Equal(t, "one", value)
//values, err := client.GetAll("foo")
//require.NoError(t, err)
//require.Equal(t, map[string]string{"foo": "one"}, values)
err = client.Set("a/b/foo", "one", 0)
require.NoError(t, err)
err = client.Set("a/b/bar", "two", 0)
require.NoError(t, err)
values, err := client.GetAll("a/b")
require.NoError(t, err)
require.Equal(t, map[string]string{"a/b/foo": "one", "a/b/bar": "two"}, values)
require.NoError(t, client.Close())
}
func TestWriteAndRead(t *testing.T) {
if testing.Short() {
t.Skip("Skipped because of short mode")
}
testFuse(t, func(c client.APIClient, mountpoint string) {
repoName := "foo"
require.NoError(t, c.CreateRepo(repoName))
commit, err := c.StartCommit(repoName, "", "")
require.NoError(t, err)
greeting := "Hello, world\n"
filePath := filepath.Join(mountpoint, repoName, commit.ID, "greeting")
require.NoError(t, ioutil.WriteFile(filePath, []byte(greeting), 0644))
readGreeting, err := ioutil.ReadFile(filePath)
require.NoError(t, err)
require.Equal(t, greeting, string(readGreeting))
require.NoError(t, c.FinishCommit(repoName, commit.ID))
data, err := ioutil.ReadFile(filePath)
require.NoError(t, err)
require.Equal(t, []byte(greeting), data)
})
}
func testBasicRethink(t *testing.T, apiServer persist.APIServer) {
_, err := apiServer.CreatePipelineInfo(
context.Background(),
&persist.PipelineInfo{
PipelineName: "foo",
},
)
require.NoError(t, err)
pipelineInfo, err := apiServer.GetPipelineInfo(
context.Background(),
&ppsclient.Pipeline{Name: "foo"},
)
require.NoError(t, err)
require.Equal(t, pipelineInfo.PipelineName, "foo")
input := &ppsclient.JobInput{Commit: client.NewCommit("bar", uuid.NewWithoutDashes())}
jobInfo, err := apiServer.CreateJobInfo(
context.Background(),
&persist.JobInfo{
JobID: uuid.NewWithoutDashes(),
PipelineName: "foo",
Inputs: []*ppsclient.JobInput{input},
},
)
jobID := jobInfo.JobID
input2 := &ppsclient.JobInput{Commit: client.NewCommit("fizz", uuid.NewWithoutDashes())}
_, err = apiServer.CreateJobInfo(
context.Background(),
&persist.JobInfo{
JobID: uuid.NewWithoutDashes(),
PipelineName: "buzz",
Inputs: []*ppsclient.JobInput{input2},
},
)
require.NoError(t, err)
jobInfo, err = apiServer.InspectJob(
context.Background(),
&ppsclient.InspectJobRequest{
Job: &ppsclient.Job{
ID: jobInfo.JobID,
},
},
)
require.NoError(t, err)
require.Equal(t, jobInfo.JobID, jobID)
require.Equal(t, "foo", jobInfo.PipelineName)
jobInfos, err := apiServer.ListJobInfos(
context.Background(),
&ppsclient.ListJobRequest{
Pipeline: &ppsclient.Pipeline{Name: "foo"},
},
)
require.NoError(t, err)
require.Equal(t, len(jobInfos.JobInfo), 1)
require.Equal(t, jobInfos.JobInfo[0].JobID, jobID)
jobInfos, err = apiServer.ListJobInfos(
context.Background(),
&ppsclient.ListJobRequest{
InputCommit: []*pfsclient.Commit{input.Commit},
},
)
require.NoError(t, err)
require.Equal(t, len(jobInfos.JobInfo), 1)
require.Equal(t, jobInfos.JobInfo[0].JobID, jobID)
jobInfos, err = apiServer.ListJobInfos(
context.Background(),
&ppsclient.ListJobRequest{
Pipeline: &ppsclient.Pipeline{Name: "foo"},
InputCommit: []*pfsclient.Commit{input.Commit},
},
)
require.NoError(t, err)
require.Equal(t, len(jobInfos.JobInfo), 1)
require.Equal(t, jobInfos.JobInfo[0].JobID, jobID)
}
func testFuse(
t *testing.T,
test func(client client.APIClient, mountpoint string),
) {
// don't leave goroutines running
var wg sync.WaitGroup
defer wg.Wait()
tmp, err := ioutil.TempDir("", "pachyderm-test-")
require.NoError(t, err)
defer func() {
_ = os.RemoveAll(tmp)
}()
// closed on successful termination
quit := make(chan struct{})
defer close(quit)
listener, err := net.Listen("tcp", "localhost:0")
require.NoError(t, err)
defer func() {
_ = listener.Close()
}()
// TODO try to share more of this setup code with various main
// functions
localAddress := listener.Addr().String()
srv := grpc.NewServer()
const (
numShards = 1
)
sharder := shard.NewLocalSharder([]string{localAddress}, numShards)
hasher := pfsserver.NewHasher(numShards, 1)
router := shard.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
localAddress,
)
blockDir := filepath.Join(tmp, "blocks")
blockServer, err := server.NewLocalBlockAPIServer(blockDir)
require.NoError(t, err)
pfsclient.RegisterBlockAPIServer(srv, blockServer)
driver, err := drive.NewDriver(localAddress)
require.NoError(t, err)
apiServer := server.NewAPIServer(
hasher,
router,
)
pfsclient.RegisterAPIServer(srv, apiServer)
internalAPIServer := server.NewInternalAPIServer(
hasher,
router,
driver,
)
pfsclient.RegisterInternalAPIServer(srv, internalAPIServer)
wg.Add(1)
go func() {
defer wg.Done()
if err := srv.Serve(listener); err != nil {
select {
case <-quit:
// orderly shutdown
return
default:
t.Errorf("grpc serve: %v", err)
}
}
}()
clientConn, err := grpc.Dial(localAddress, grpc.WithInsecure())
require.NoError(t, err)
apiClient := pfsclient.NewAPIClient(clientConn)
mounter := fuse.NewMounter(localAddress, apiClient)
mountpoint := filepath.Join(tmp, "mnt")
require.NoError(t, os.Mkdir(mountpoint, 0700))
ready := make(chan bool)
wg.Add(1)
go func() {
defer wg.Done()
require.NoError(t, mounter.MountAndCreate(mountpoint, nil, nil, ready, true))
}()
<-ready
defer func() {
_ = mounter.Unmount(mountpoint)
}()
test(client.APIClient{PfsAPIClient: apiClient}, mountpoint)
}
func TestNoDelimiter(t *testing.T) {
if testing.Short() {
t.Skip("Skipped because of short mode")
}
testFuse(t, func(c client.APIClient, mountpoint string) {
repo := "test"
name := "foo.bin"
require.NoError(t, c.CreateRepo(repo))
commit1, err := c.StartCommit(repo, "", "")
require.NoError(t, err)
rawMessage := "Some\ncontent\nthat\nshouldnt\nbe\nline\ndelimited.\n"
filePath := filepath.Join(mountpoint, repo, commit1.ID, name)
// Write a big blob that would normally not fit in a block
var expectedOutputA []byte
for !(len(expectedOutputA) > 9*1024*1024) {
expectedOutputA = append(expectedOutputA, []byte(rawMessage)...)
}
require.NoError(t, ioutil.WriteFile(filePath, expectedOutputA, 0644))
// Write another big block
var expectedOutputB []byte
for !(len(expectedOutputB) > 10*1024*1024) {
expectedOutputB = append(expectedOutputB, []byte(rawMessage)...)
}
require.NoError(t, ioutil.WriteFile("/tmp/b", expectedOutputB, 0644))
stdin := strings.NewReader(fmt.Sprintf("cat /tmp/b >>%s", filePath))
require.NoError(t, pkgexec.RunStdin(stdin, "sh"))
// Finish the commit so I can read the data
require.NoError(t, c.FinishCommit(repo, commit1.ID))
// Make sure all the content is there
var buffer bytes.Buffer
require.NoError(t, c.GetFile(repo, commit1.ID, name, 0, 0, "", nil, &buffer))
require.Equal(t, len(expectedOutputA)+len(expectedOutputB), buffer.Len())
require.Equal(t, string(append(expectedOutputA, expectedOutputB...)), buffer.String())
// Now verify that each block only contains objects of the size we've written
bigModulus := 10 // Make it big to make it less likely that I return both blocks together
blockLengths := []interface{}{len(expectedOutputA), len(expectedOutputB)}
for b := 0; b < bigModulus; b++ {
blockFilter := &pfsclient.Shard{
BlockNumber: uint64(b),
BlockModulus: uint64(bigModulus),
}
buffer.Reset()
require.NoError(t, c.GetFile(repo, commit1.ID, name, 0, 0, "", blockFilter, &buffer))
// If any single block returns content of size equal to the total, we
// got a block collision and we're not testing anything
require.NotEqual(t, len(expectedOutputA)+len(expectedOutputB), buffer.Len())
if buffer.Len() == 0 {
continue
}
require.EqualOneOf(t, blockLengths, buffer.Len())
}
})
}
func TestDelimitJSON(t *testing.T) {
if testing.Short() {
t.Skip("Skipped because of short mode")
}
testFuse(t, func(c client.APIClient, mountpoint string) {
repo := "abba"
require.NoError(t, c.CreateRepo(repo))
commit, err := c.StartCommit(repo, "", "")
require.NoError(t, err)
var expectedOutput []byte
rawMessage := `{
"level":"debug",
"timestamp":"345",
"message":{
"thing":"foo"
},
"timing":[1,3,34,6,7]
}`
for !(len(expectedOutput) > 9*1024*1024) {
expectedOutput = append(expectedOutput, []byte(rawMessage)...)
}
filePath := filepath.Join(mountpoint, repo, commit.ID, "foo.json")
require.NoError(t, ioutil.WriteFile(filePath, expectedOutput, 0644))
require.NoError(t, c.FinishCommit(repo, commit.ID))
// Make sure all the content is there
var buffer bytes.Buffer
require.NoError(t, c.GetFile(repo, commit.ID, "foo.json", 0, 0, "", nil, &buffer))
require.Equal(t, len(expectedOutput), buffer.Len())
require.Equal(t, string(expectedOutput), buffer.String())
// Now verify that each block contains only valid JSON objects
bigModulus := 10 // Make it big to make it less likely that I return both blocks together
for b := 0; b < bigModulus; b++ {
blockFilter := &pfsclient.Shard{
BlockNumber: uint64(b),
BlockModulus: uint64(bigModulus),
}
buffer.Reset()
require.NoError(t, c.GetFile(repo, commit.ID, "foo.json", 0, 0, "", blockFilter, &buffer))
// If any single block returns content of size equal to the total, we
// got a block collision and we're not testing anything
require.NotEqual(t, buffer.Len(), len(expectedOutput))
var value json.RawMessage
decoder := json.NewDecoder(&buffer)
for {
err = decoder.Decode(&value)
if err != nil {
if err == io.EOF {
break
} else {
require.NoError(t, err)
}
}
require.Equal(t, rawMessage, string(value))
}
}
})
}