func runWatchTest(t *testing.T, client Client) { cancel := make(chan bool) err := client.Watch( "watch/foo", cancel, func(value string) error { if value == "" { return client.Set("watch/foo", "bar", 0) } require.Equal(t, "bar", value) close(cancel) return nil }, ) require.Equal(t, ErrCancelled, err) cancel = make(chan bool) err = client.WatchAll( "watchAll/foo", cancel, func(value map[string]string) error { if value == nil { return client.Set("watchAll/foo/bar", "quux", 0) } require.Equal(t, map[string]string{"watchAll/foo/bar": "quux"}, value) close(cancel) return nil }, ) require.Equal(t, ErrCancelled, err) }
func testRun(t *testing.T, imageName string, commands []string, expectedStdout string, expectedStderr string) { if testing.Short() { t.Skip("Skipping integration tests in short mode") } client, err := newTestDockerClient() require.NoError(t, err) err = client.Pull(imageName, PullOptions{}) require.NoError(t, err) container, err := client.Create( imageName, CreateOptions{ HasCommand: commands != nil, }, ) require.NoError(t, err) err = client.Start( container, StartOptions{ Commands: commands, }, ) require.NoError(t, err) err = client.Wait(container, WaitOptions{}) require.NoError(t, err) stdout := bytes.NewBuffer(nil) stderr := bytes.NewBuffer(nil) err = client.Logs(container, LogsOptions{Stdout: stdout, Stderr: stderr}) require.NoError(t, err) require.Equal(t, expectedStdout, stdout.String()) require.Equal(t, expectedStderr, stderr.String()) err = client.Remove(container, RemoveOptions{}) require.NoError(t, err) }
func TestMountCachingViaShell(t *testing.T) { if testing.Short() { t.Skip("Skipped because of short mode") } testFuse(t, func(c client.APIClient, mountpoint string) { repo1 := "foo" require.NoError(t, c.CreateRepo(repo1)) // Now if we walk the FS we should see the file. // This first act of 'walking' should also initiate the cache ls := exec.Command("ls") ls.Dir = mountpoint out, err := ls.Output() require.NoError(t, err) require.Equal(t, "foo\n", string(out)) // Now create another repo, and look for it under the mount point repo2 := "bar" require.NoError(t, c.CreateRepo(repo2)) // Now if we walk the FS we should see the new file. // This second ls on mac doesn't report the file! ls = exec.Command("ls") ls.Dir = mountpoint out, err = ls.Output() require.NoError(t, err) require.Equal(t, true, "foo\nbar\n" == string(out) || "bar\nfoo\n" == string(out)) }) }
func TestSeekRead(t *testing.T) { if testing.Short() { t.Skip("Skipped because of short mode") } testFuse(t, func(c client.APIClient, mountpoint string) { repo := "test" require.NoError(t, c.CreateRepo(repo)) commit, err := c.StartCommit(repo, "", "") require.NoError(t, err) path := filepath.Join(mountpoint, repo, commit.ID, "file") file, err := os.Create(path) require.NoError(t, err) _, err = file.Write([]byte("foobarbaz")) require.NoError(t, err) require.NoError(t, file.Close()) require.NoError(t, c.FinishCommit(repo, commit.ID)) fmt.Printf("==== Finished commit\n") file, err = os.Open(path) defer file.Close() require.NoError(t, err) word1 := make([]byte, 3) n1, err := file.Read(word1) require.NoError(t, err) require.Equal(t, 3, n1) require.Equal(t, "foo", string(word1)) fmt.Printf("==== %v - Read word len %v : %v\n", time.Now(), n1, string(word1)) offset, err := file.Seek(6, 0) fmt.Printf("==== %v - err (%v)\n", time.Now(), err) fmt.Printf("==== %v - offset (%v)\n", time.Now(), offset) // MAC OS X BEHAVIOR - it allows seek require.NoError(t, err) require.Equal(t, int64(6), offset) fmt.Printf("==== Seeked to %v\n", offset) /* Leaving in place so the test's intention is clear / for repro'ing manually for mac */ word2 := make([]byte, 3) n2, err := file.Read(word2) require.NoError(t, err) require.Equal(t, 3, n2) require.Equal(t, "baz", string(word2)) fmt.Printf("==== Read word len %v : %v\n", n2, string(word2)) }) }
func TestSeekWriteBackwards(t *testing.T) { if testing.Short() { t.Skip("Skipped because of short mode") } testFuse(t, func(c client.APIClient, mountpoint string) { repo := "test" require.NoError(t, c.CreateRepo(repo)) commit, err := c.StartCommit(repo, "", "") require.NoError(t, err) path := filepath.Join(mountpoint, repo, commit.ID, "file") file, err := os.Create(path) require.NoError(t, err) defer func() { // MAC OS X BEHAVIOR - it allows seek but this is an invalid write require.YesError(t, file.Close()) }() _, err = file.Write([]byte("foofoofoo")) require.NoError(t, err) err = file.Sync() require.NoError(t, err) offset, err := file.Seek(3, 0) fmt.Printf("==== %v - err (%v)\n", time.Now(), err) fmt.Printf("==== %v - offset (%v)\n", time.Now(), offset) // MAC OS X BEHAVIOR - it allows seek require.NoError(t, err) require.Equal(t, int64(3), offset) /* Leaving in place so the test's intention is clear / for repro'ing manually for mac */ err = file.Sync() require.NoError(t, err) n1, err := file.Write([]byte("bar")) // MAC OS X BEHAVIOR - it allows seek require.NoError(t, err) require.Equal(t, 3, n1) fmt.Printf("==== %v - write word len %v\n", time.Now(), n1) require.NoError(t, c.FinishCommit(repo, commit.ID)) }) }
func Test296(t *testing.T) { lion.SetLevel(lion.LevelDebug) if testing.Short() { t.Skip("Skipped because of short mode") } testFuse(t, func(c client.APIClient, mountpoint string) { repo := "test" require.NoError(t, c.CreateRepo(repo)) commit, err := c.StartCommit(repo, "", "") require.NoError(t, err) path := filepath.Join(mountpoint, repo, commit.ID, "file") stdin := strings.NewReader(fmt.Sprintf("echo 1 >%s", path)) require.NoError(t, pkgexec.RunStdin(stdin, "sh")) stdin = strings.NewReader(fmt.Sprintf("echo 2 >%s", path)) require.NoError(t, pkgexec.RunStdin(stdin, "sh")) require.NoError(t, c.FinishCommit(repo, commit.ID)) commit2, err := c.StartCommit(repo, commit.ID, "") require.NoError(t, err) path = filepath.Join(mountpoint, repo, commit2.ID, "file") stdin = strings.NewReader(fmt.Sprintf("echo 3 >%s", path)) require.NoError(t, pkgexec.RunStdin(stdin, "sh")) require.NoError(t, c.FinishCommit(repo, commit2.ID)) data, err := ioutil.ReadFile(path) require.NoError(t, err) require.Equal(t, "3\n", string(data)) }) }
func TestOpenAndWriteFile(t *testing.T) { if testing.Short() { t.Skip("Skipped because of short mode") } testFuse(t, func(c client.APIClient, mountpoint string) { require.NoError(t, c.CreateRepo("repo")) commit1, err := c.StartCommit("repo", "", "") require.NoError(t, err) filePath := filepath.Join(mountpoint, "repo", commit1.ID, "foo") f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) require.NoError(t, err) defer func() { err = f.Close() require.NoError(t, err) }() data1 := []byte("something\nis\nrotten\n") _, err = f.Write(data1) require.NoError(t, err) data2 := []byte("in\nthe\nstate\nof\nDenmark\n") _, err = f.Write(data2) require.NoError(t, err) require.NoError(t, f.Sync()) require.NoError(t, c.FinishCommit("repo", commit1.ID)) result, err := ioutil.ReadFile(filePath) require.NoError(t, err) require.Equal(t, fmt.Sprintf("%v%v", string(data1), string(data2)), string(result)) }) }
func TestDiamond(t *testing.T) { d := NewDAG(map[string][]string{ "1": {}, "2": {"1"}, "3": {"1"}, "4": {"2", "3"}, }) require.EqualOneOf( t, []interface{}{ []string{"1", "2", "3", "4"}, []string{"1", "3", "2", "4"}, }, d.Sorted(), ) require.EqualOneOf( t, []interface{}{ []string{"1", "2", "3", "4"}, []string{"1", "3", "2", "4"}, }, d.Ancestors("4", nil), ) require.EqualOneOf( t, []interface{}{ []string{"2", "3", "4"}, []string{"3", "2", "4"}, }, d.Ancestors("4", []string{"1"}), ) require.EqualOneOf( t, []interface{}{ []string{"1", "2", "3"}, []string{"1", "3", "2"}, }, d.Descendants("1", []string{"4"}), ) require.Equal(t, []string{"4"}, d.Leaves()) require.Equal(t, 0, len(d.Ghosts())) }
func runTest(t *testing.T, client Client) { err := client.Set("foo", "one", 0) require.NoError(t, err) value, err := client.Get("foo") require.NoError(t, err) require.Equal(t, "one", value) //values, err := client.GetAll("foo") //require.NoError(t, err) //require.Equal(t, map[string]string{"foo": "one"}, values) err = client.Set("a/b/foo", "one", 0) require.NoError(t, err) err = client.Set("a/b/bar", "two", 0) require.NoError(t, err) values, err := client.GetAll("a/b") require.NoError(t, err) require.Equal(t, map[string]string{"a/b/foo": "one", "a/b/bar": "two"}, values) require.NoError(t, client.Close()) }
func TestLine(t *testing.T) { d := NewDAG(map[string][]string{ "1": {}, "2": {"1"}, "3": {"2"}, "4": {"3"}, }) require.Equal(t, []string{"1", "2", "3", "4"}, d.Sorted()) require.Equal(t, []string{"1", "2", "3", "4"}, d.Ancestors("4", nil)) require.Equal(t, []string{"3", "4"}, d.Ancestors("4", []string{"2"})) require.Equal(t, []string{"1", "2"}, d.Ancestors("2", nil)) require.Equal(t, []string{"1", "2", "3", "4"}, d.Descendants("1", nil)) require.Equal(t, []string{"3", "4"}, d.Descendants("3", nil)) require.Equal(t, []string{"1", "2"}, d.Descendants("1", []string{"3"})) require.Equal(t, []string{"4"}, d.Leaves()) require.Equal(t, 0, len(d.Ghosts())) }
func TestWriteAndRead(t *testing.T) { if testing.Short() { t.Skip("Skipped because of short mode") } testFuse(t, func(c client.APIClient, mountpoint string) { repoName := "foo" require.NoError(t, c.CreateRepo(repoName)) commit, err := c.StartCommit(repoName, "", "") require.NoError(t, err) greeting := "Hello, world\n" filePath := filepath.Join(mountpoint, repoName, commit.ID, "greeting") require.NoError(t, ioutil.WriteFile(filePath, []byte(greeting), 0644)) readGreeting, err := ioutil.ReadFile(filePath) require.NoError(t, err) require.Equal(t, greeting, string(readGreeting)) require.NoError(t, c.FinishCommit(repoName, commit.ID)) data, err := ioutil.ReadFile(filePath) require.NoError(t, err) require.Equal(t, []byte(greeting), data) }) }
func TestJSONSyntaxErrorsReportedCreatePipelineFromStdin(t *testing.T) { descriptiveOutput := `Reading from stdin. Error parsing pipeline spec: Syntax Error on line 5: "c": {a ^ invalid character 'a' looking for beginning of object key string ` rawCmd := []string{"pachctl", "create-pipeline"} testName := "TestJSONSyntaxErrorsReportedCreatePipelineFromStdin" if os.Getenv("BE_CRASHER") == "1" { address := "0.0.0.0:30650" rootCmd := &cobra.Command{ Use: os.Args[0], Long: `Access the Pachyderm API. Envronment variables: ADDRESS=0.0.0.0:30650, the server to connect to. `, } cmds, _ := Cmds(address) for _, cmd := range cmds { rootCmd.AddCommand(cmd) } os.Args = rawCmd ioutil.WriteFile("bad2.json", []byte(badJSON2), 0644) os.Stdin, _ = os.Open("bad2.json") rootCmd.Execute() return } cmd := exec.Command(os.Args[0], fmt.Sprintf("-test.run=%v", testName)) cmd.Env = append(os.Environ(), "BE_CRASHER=1") out, err := cmd.CombinedOutput() // Put our cleanup here, since we have an exit 1 in the actual run: wd, _ := os.Getwd() os.Remove(filepath.Join(wd, "bad2.json")) require.YesError(t, err) if e, ok := err.(*exec.ExitError); ok && !e.Success() { require.Equal(t, descriptiveOutput, string(out)) return } t.Fatalf("process ran with err %v, want exit status 1", err) }
func TestBigWrite(t *testing.T) { if testing.Short() { t.Skip("Skipped because of short mode") } testFuse(t, func(c client.APIClient, mountpoint string) { repo := "test" require.NoError(t, c.CreateRepo(repo)) commit, err := c.StartCommit(repo, "", "") require.NoError(t, err) path := filepath.Join(mountpoint, repo, commit.ID, "file1") stdin := strings.NewReader(fmt.Sprintf("yes | tr -d '\\n' | head -c 1000000 > %s", path)) require.NoError(t, pkgexec.RunStdin(stdin, "sh")) require.NoError(t, c.FinishCommit(repo, commit.ID)) data, err := ioutil.ReadFile(path) require.NoError(t, err) require.Equal(t, bytes.Repeat([]byte{'y'}, 1000000), data) }) }
func TestSeekWriteGap(t *testing.T) { if testing.Short() { t.Skip("Skipped because of short mode") } testFuse(t, func(c client.APIClient, mountpoint string) { repo := "test" require.NoError(t, c.CreateRepo(repo)) commit, err := c.StartCommit(repo, "", "") require.NoError(t, err) path := filepath.Join(mountpoint, repo, commit.ID, "file") file, err := os.Create(path) require.NoError(t, err) defer func() { require.NoError(t, file.Close()) }() _, err = file.Write([]byte("foo")) require.NoError(t, err) err = file.Sync() require.NoError(t, err) offset, err := file.Seek(6, 0) fmt.Printf("==== %v - err (%v)\n", time.Now(), err) fmt.Printf("==== %v - offset (%v)\n", time.Now(), offset) require.YesError(t, err) require.Equal(t, int64(0), offset) /* Leaving in place so the test's intention is clear / for repro'ing manually for mac err = file.Sync() require.NoError(t, err) n1, err := file.Write([]byte("baz")) require.YesError(t, err) require.Equal(t, 3, n1) require.NoError(t, c.FinishCommit(repo, commit.ID)) */ }) }
func TestOverwriteFile(t *testing.T) { if testing.Short() { t.Skip("Skipped because of short mode") } testFuse(t, func(c client.APIClient, mountpoint string) { require.NoError(t, c.CreateRepo("repo")) commit1, err := c.StartCommit("repo", "", "") require.NoError(t, err) _, err = c.PutFile("repo", commit1.ID, "file", strings.NewReader("foo\n")) require.NoError(t, err) require.NoError(t, c.FinishCommit("repo", commit1.ID)) commit2, err := c.StartCommit("repo", commit1.ID, "") require.NoError(t, err) path := filepath.Join(mountpoint, "repo", commit2.ID, "file") stdin := strings.NewReader(fmt.Sprintf("echo bar >%s", path)) require.NoError(t, pkgexec.RunStdin(stdin, "sh")) require.NoError(t, c.FinishCommit("repo", commit2.ID)) result, err := ioutil.ReadFile(path) require.NoError(t, err) require.Equal(t, "bar\n", string(result)) }) }
func TestSpacedWrites(t *testing.T) { if testing.Short() { t.Skip("Skipped because of short mode") } testFuse(t, func(c client.APIClient, mountpoint string) { repo := "test" require.NoError(t, c.CreateRepo(repo)) commit, err := c.StartCommit(repo, "", "") require.NoError(t, err) path := filepath.Join(mountpoint, repo, commit.ID, "file") file, err := os.Create(path) require.NoError(t, err) _, err = file.Write([]byte("foo")) require.NoError(t, err) _, err = file.Write([]byte("foo")) require.NoError(t, err) require.NoError(t, file.Close()) require.NoError(t, c.FinishCommit(repo, commit.ID)) data, err := ioutil.ReadFile(path) require.NoError(t, err) require.Equal(t, "foofoo", string(data)) }) }
func testBadJSON(t *testing.T, testName string, inputFile string, inputFileValue string, inputCommand []string, expectedOutput string) { if os.Getenv("BE_CRASHER") == "1" { testJSONSyntaxErrorsReported(inputFile, inputFileValue, inputCommand) return } cmd := exec.Command(os.Args[0], fmt.Sprintf("-test.run=%v", testName)) cmd.Env = append(os.Environ(), "BE_CRASHER=1") out, err := cmd.CombinedOutput() // Do our cleanup here, since we have an exit 1 in the actual run: wd, _ := os.Getwd() fileName := filepath.Join(wd, inputFile) os.Remove(fileName) require.YesError(t, err) if e, ok := err.(*exec.ExitError); ok && !e.Success() { require.Equal(t, expectedOutput, string(out)) return } t.Fatalf("process ran with err %v, want exit status 1", err) }
func TestNoDelimiter(t *testing.T) { if testing.Short() { t.Skip("Skipped because of short mode") } testFuse(t, func(c client.APIClient, mountpoint string) { repo := "test" name := "foo.bin" require.NoError(t, c.CreateRepo(repo)) commit1, err := c.StartCommit(repo, "", "") require.NoError(t, err) rawMessage := "Some\ncontent\nthat\nshouldnt\nbe\nline\ndelimited.\n" filePath := filepath.Join(mountpoint, repo, commit1.ID, name) // Write a big blob that would normally not fit in a block var expectedOutputA []byte for !(len(expectedOutputA) > 9*1024*1024) { expectedOutputA = append(expectedOutputA, []byte(rawMessage)...) } require.NoError(t, ioutil.WriteFile(filePath, expectedOutputA, 0644)) // Write another big block var expectedOutputB []byte for !(len(expectedOutputB) > 10*1024*1024) { expectedOutputB = append(expectedOutputB, []byte(rawMessage)...) } require.NoError(t, ioutil.WriteFile("/tmp/b", expectedOutputB, 0644)) stdin := strings.NewReader(fmt.Sprintf("cat /tmp/b >>%s", filePath)) require.NoError(t, pkgexec.RunStdin(stdin, "sh")) // Finish the commit so I can read the data require.NoError(t, c.FinishCommit(repo, commit1.ID)) // Make sure all the content is there var buffer bytes.Buffer require.NoError(t, c.GetFile(repo, commit1.ID, name, 0, 0, "", nil, &buffer)) require.Equal(t, len(expectedOutputA)+len(expectedOutputB), buffer.Len()) require.Equal(t, string(append(expectedOutputA, expectedOutputB...)), buffer.String()) // Now verify that each block only contains objects of the size we've written bigModulus := 10 // Make it big to make it less likely that I return both blocks together blockLengths := []interface{}{len(expectedOutputA), len(expectedOutputB)} for b := 0; b < bigModulus; b++ { blockFilter := &pfsclient.Shard{ BlockNumber: uint64(b), BlockModulus: uint64(bigModulus), } buffer.Reset() require.NoError(t, c.GetFile(repo, commit1.ID, name, 0, 0, "", blockFilter, &buffer)) // If any single block returns content of size equal to the total, we // got a block collision and we're not testing anything require.NotEqual(t, len(expectedOutputA)+len(expectedOutputB), buffer.Len()) if buffer.Len() == 0 { continue } require.EqualOneOf(t, blockLengths, buffer.Len()) } }) }
func TestDelimitJSON(t *testing.T) { if testing.Short() { t.Skip("Skipped because of short mode") } testFuse(t, func(c client.APIClient, mountpoint string) { repo := "abba" require.NoError(t, c.CreateRepo(repo)) commit, err := c.StartCommit(repo, "", "") require.NoError(t, err) var expectedOutput []byte rawMessage := `{ "level":"debug", "timestamp":"345", "message":{ "thing":"foo" }, "timing":[1,3,34,6,7] }` for !(len(expectedOutput) > 9*1024*1024) { expectedOutput = append(expectedOutput, []byte(rawMessage)...) } filePath := filepath.Join(mountpoint, repo, commit.ID, "foo.json") require.NoError(t, ioutil.WriteFile(filePath, expectedOutput, 0644)) require.NoError(t, c.FinishCommit(repo, commit.ID)) // Make sure all the content is there var buffer bytes.Buffer require.NoError(t, c.GetFile(repo, commit.ID, "foo.json", 0, 0, "", nil, &buffer)) require.Equal(t, len(expectedOutput), buffer.Len()) require.Equal(t, string(expectedOutput), buffer.String()) // Now verify that each block contains only valid JSON objects bigModulus := 10 // Make it big to make it less likely that I return both blocks together for b := 0; b < bigModulus; b++ { blockFilter := &pfsclient.Shard{ BlockNumber: uint64(b), BlockModulus: uint64(bigModulus), } buffer.Reset() require.NoError(t, c.GetFile(repo, commit.ID, "foo.json", 0, 0, "", blockFilter, &buffer)) // If any single block returns content of size equal to the total, we // got a block collision and we're not testing anything require.NotEqual(t, buffer.Len(), len(expectedOutput)) var value json.RawMessage decoder := json.NewDecoder(&buffer) for { err = decoder.Decode(&value) if err != nil { if err == io.EOF { break } else { require.NoError(t, err) } } require.Equal(t, rawMessage, string(value)) } } }) }
func testBasicRethink(t *testing.T, apiServer persist.APIServer) { _, err := apiServer.CreatePipelineInfo( context.Background(), &persist.PipelineInfo{ PipelineName: "foo", }, ) require.NoError(t, err) pipelineInfo, err := apiServer.GetPipelineInfo( context.Background(), &ppsclient.Pipeline{Name: "foo"}, ) require.NoError(t, err) require.Equal(t, pipelineInfo.PipelineName, "foo") input := &ppsclient.JobInput{Commit: client.NewCommit("bar", uuid.NewWithoutDashes())} jobInfo, err := apiServer.CreateJobInfo( context.Background(), &persist.JobInfo{ JobID: uuid.NewWithoutDashes(), PipelineName: "foo", Inputs: []*ppsclient.JobInput{input}, }, ) jobID := jobInfo.JobID input2 := &ppsclient.JobInput{Commit: client.NewCommit("fizz", uuid.NewWithoutDashes())} _, err = apiServer.CreateJobInfo( context.Background(), &persist.JobInfo{ JobID: uuid.NewWithoutDashes(), PipelineName: "buzz", Inputs: []*ppsclient.JobInput{input2}, }, ) require.NoError(t, err) jobInfo, err = apiServer.InspectJob( context.Background(), &ppsclient.InspectJobRequest{ Job: &ppsclient.Job{ ID: jobInfo.JobID, }, }, ) require.NoError(t, err) require.Equal(t, jobInfo.JobID, jobID) require.Equal(t, "foo", jobInfo.PipelineName) jobInfos, err := apiServer.ListJobInfos( context.Background(), &ppsclient.ListJobRequest{ Pipeline: &ppsclient.Pipeline{Name: "foo"}, }, ) require.NoError(t, err) require.Equal(t, len(jobInfos.JobInfo), 1) require.Equal(t, jobInfos.JobInfo[0].JobID, jobID) jobInfos, err = apiServer.ListJobInfos( context.Background(), &ppsclient.ListJobRequest{ InputCommit: []*pfsclient.Commit{input.Commit}, }, ) require.NoError(t, err) require.Equal(t, len(jobInfos.JobInfo), 1) require.Equal(t, jobInfos.JobInfo[0].JobID, jobID) jobInfos, err = apiServer.ListJobInfos( context.Background(), &ppsclient.ListJobRequest{ Pipeline: &ppsclient.Pipeline{Name: "foo"}, InputCommit: []*pfsclient.Commit{input.Commit}, }, ) require.NoError(t, err) require.Equal(t, len(jobInfos.JobInfo), 1) require.Equal(t, jobInfos.JobInfo[0].JobID, jobID) }