コード例 #1
0
func TestMountCachingViaShell(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipped because of short mode")
	}

	testFuse(t, func(c client.APIClient, mountpoint string) {
		repo1 := "foo"
		require.NoError(t, c.CreateRepo(repo1))

		// Now if we walk the FS we should see the file.
		// This first act of 'walking' should also initiate the cache

		ls := exec.Command("ls")
		ls.Dir = mountpoint
		out, err := ls.Output()
		require.NoError(t, err)

		require.Equal(t, "foo\n", string(out))

		// Now create another repo, and look for it under the mount point
		repo2 := "bar"
		require.NoError(t, c.CreateRepo(repo2))

		// Now if we walk the FS we should see the new file.
		// This second ls on mac doesn't report the file!

		ls = exec.Command("ls")
		ls.Dir = mountpoint
		out, err = ls.Output()
		require.NoError(t, err)

		require.Equal(t, true, "foo\nbar\n" == string(out) || "bar\nfoo\n" == string(out))

	})
}
コード例 #2
0
func testBlock(t *testing.T, apiServer persist.APIServer) {
	jobInfo, err := apiServer.CreateJobInfo(context.Background(), &persist.JobInfo{
		JobID: uuid.NewWithoutDashes(),
	})
	require.NoError(t, err)
	jobID := jobInfo.JobID
	go func() {
		_, err := apiServer.CreateJobOutput(
			context.Background(),
			&persist.JobOutput{
				JobID:        jobID,
				OutputCommit: client.NewCommit("foo", "bar"),
			})
		require.NoError(t, err)
		_, err = apiServer.CreateJobState(
			context.Background(),
			&persist.JobState{
				JobID: jobID,
				State: ppsclient.JobState_JOB_SUCCESS,
			})
		require.NoError(t, err)
	}()
	_, err = apiServer.InspectJob(
		context.Background(),
		&ppsclient.InspectJobRequest{
			Job:        &ppsclient.Job{ID: jobID},
			BlockState: true,
		},
	)
	require.NoError(t, err)
}
コード例 #3
0
func TestCommitOpenReadDir(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipped because of short mode")
	}

	testFuse(t, func(c client.APIClient, mountpoint string) {
		repoName := "foo"
		require.NoError(t, c.CreateRepo(repoName))
		commit, err := c.StartCommit(repoName, "", "")
		require.NoError(t, err)
		t.Logf("open commit %v", commit.ID)

		const (
			greetingName = "greeting"
			greeting     = "Hello, world\n"
			greetingPerm = 0644
		)
		require.NoError(t, ioutil.WriteFile(filepath.Join(mountpoint, repoName, commit.ID, greetingName), []byte(greeting), greetingPerm))
		const (
			scriptName = "script"
			script     = "#!/bin/sh\necho foo\n"
			scriptPerm = 0750
		)
		require.NoError(t, ioutil.WriteFile(filepath.Join(mountpoint, repoName, commit.ID, scriptName), []byte(script), scriptPerm))

		require.NoError(t, fstestutil.CheckDir(filepath.Join(mountpoint, repoName, commit.ID), map[string]fstestutil.FileInfoCheck{
			greetingName: func(fi os.FileInfo) error {
				// TODO respect greetingPerm
				if g, e := fi.Mode(), os.FileMode(0666); g != e {
					return fmt.Errorf("wrong mode: %v != %v", g, e)
				}
				if g, e := fi.Size(), int64(len(greeting)); g != e {
					t.Errorf("wrong size: %v != %v", g, e)
				}
				// TODO show fileModTime as mtime
				// if g, e := fi.ModTime().UTC(), fileModTime; g != e {
				// 	t.Errorf("wrong mtime: %v != %v", g, e)
				// }
				return nil
			},
			scriptName: func(fi os.FileInfo) error {
				// TODO respect scriptPerm
				if g, e := fi.Mode(), os.FileMode(0666); g != e {
					return fmt.Errorf("wrong mode: %v != %v", g, e)
				}
				if g, e := fi.Size(), int64(len(script)); g != e {
					t.Errorf("wrong size: %v != %v", g, e)
				}
				// TODO show fileModTime as mtime
				// if g, e := fi.ModTime().UTC(), fileModTime; g != e {
				// 	t.Errorf("wrong mtime: %v != %v", g, e)
				// }
				return nil
			},
		}))
	})
}
コード例 #4
0
func TestSeekRead(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipped because of short mode")
	}

	testFuse(t, func(c client.APIClient, mountpoint string) {
		repo := "test"
		require.NoError(t, c.CreateRepo(repo))
		commit, err := c.StartCommit(repo, "", "")
		require.NoError(t, err)
		path := filepath.Join(mountpoint, repo, commit.ID, "file")
		file, err := os.Create(path)
		require.NoError(t, err)
		_, err = file.Write([]byte("foobarbaz"))

		require.NoError(t, err)
		require.NoError(t, file.Close())
		require.NoError(t, c.FinishCommit(repo, commit.ID))

		fmt.Printf("==== Finished commit\n")

		file, err = os.Open(path)
		defer file.Close()
		require.NoError(t, err)

		word1 := make([]byte, 3)
		n1, err := file.Read(word1)
		require.NoError(t, err)
		require.Equal(t, 3, n1)
		require.Equal(t, "foo", string(word1))

		fmt.Printf("==== %v - Read word len %v : %v\n", time.Now(), n1, string(word1))

		offset, err := file.Seek(6, 0)
		fmt.Printf("==== %v - err (%v)\n", time.Now(), err)

		fmt.Printf("==== %v - offset (%v)\n", time.Now(), offset)

		// MAC OS X BEHAVIOR - it allows seek
		require.NoError(t, err)
		require.Equal(t, int64(6), offset)

		fmt.Printf("==== Seeked to %v\n", offset)

		/* Leaving in place so the test's intention is clear / for repro'ing manually for mac */

		word2 := make([]byte, 3)
		n2, err := file.Read(word2)
		require.NoError(t, err)
		require.Equal(t, 3, n2)
		require.Equal(t, "baz", string(word2))

		fmt.Printf("==== Read word len %v : %v\n", n2, string(word2))

	})
}
コード例 #5
0
func Test296(t *testing.T) {
	lion.SetLevel(lion.LevelDebug)
	if testing.Short() {
		t.Skip("Skipped because of short mode")
	}

	testFuse(t, func(c client.APIClient, mountpoint string) {
		repo := "test"
		require.NoError(t, c.CreateRepo(repo))
		commit, err := c.StartCommit(repo, "", "")
		require.NoError(t, err)
		path := filepath.Join(mountpoint, repo, commit.ID, "file")
		stdin := strings.NewReader(fmt.Sprintf("echo 1 >%s", path))
		require.NoError(t, pkgexec.RunStdin(stdin, "sh"))
		stdin = strings.NewReader(fmt.Sprintf("echo 2 >%s", path))
		require.NoError(t, pkgexec.RunStdin(stdin, "sh"))
		require.NoError(t, c.FinishCommit(repo, commit.ID))
		commit2, err := c.StartCommit(repo, commit.ID, "")
		require.NoError(t, err)
		path = filepath.Join(mountpoint, repo, commit2.ID, "file")
		stdin = strings.NewReader(fmt.Sprintf("echo 3 >%s", path))
		require.NoError(t, pkgexec.RunStdin(stdin, "sh"))
		require.NoError(t, c.FinishCommit(repo, commit2.ID))
		data, err := ioutil.ReadFile(path)
		require.NoError(t, err)
		require.Equal(t, "3\n", string(data))
	})
}
コード例 #6
0
func TestOpenAndWriteFile(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipped because of short mode")
	}
	testFuse(t, func(c client.APIClient, mountpoint string) {
		require.NoError(t, c.CreateRepo("repo"))
		commit1, err := c.StartCommit("repo", "", "")
		require.NoError(t, err)
		filePath := filepath.Join(mountpoint, "repo", commit1.ID, "foo")
		f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
		require.NoError(t, err)
		defer func() {
			err = f.Close()
			require.NoError(t, err)
		}()
		data1 := []byte("something\nis\nrotten\n")
		_, err = f.Write(data1)
		require.NoError(t, err)
		data2 := []byte("in\nthe\nstate\nof\nDenmark\n")
		_, err = f.Write(data2)
		require.NoError(t, err)
		require.NoError(t, f.Sync())
		require.NoError(t, c.FinishCommit("repo", commit1.ID))
		result, err := ioutil.ReadFile(filePath)
		require.NoError(t, err)
		require.Equal(t, fmt.Sprintf("%v%v", string(data1), string(data2)), string(result))
	})
}
コード例 #7
0
ファイル: testing.go プロジェクト: saakaifoundry/pachyderm
func RunTestWithRethinkAPIServer(t *testing.T, testFunc func(t *testing.T, persistAPIServer persist.APIServer)) {
	if testing.Short() {
		t.Skip("Skipping test because of short mode.")
	}

	apiServer, err := NewTestRethinkAPIServer()
	require.NoError(t, err)
	defer func() {
		require.NoError(t, apiServer.Close())
	}()
	testFunc(t, apiServer)
}
コード例 #8
0
func TestRepoReadDir(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipped because of short mode")
	}

	testFuse(t, func(c client.APIClient, mountpoint string) {
		repoName := "foo"
		require.NoError(t, c.CreateRepo(repoName))
		commitA, err := c.StartCommit(repoName, "", "")
		require.NoError(t, err)
		require.NoError(t, c.FinishCommit(repoName, commitA.ID))
		t.Logf("finished commit %v", commitA.ID)

		commitB, err := c.StartCommit(repoName, "", "")
		require.NoError(t, err)
		t.Logf("open commit %v", commitB.ID)

		require.NoError(t, fstestutil.CheckDir(filepath.Join(mountpoint, repoName), map[string]fstestutil.FileInfoCheck{
			commitA.ID: func(fi os.FileInfo) error {
				if g, e := fi.Mode(), os.ModeDir|0555; g != e {
					return fmt.Errorf("wrong mode: %v != %v", g, e)
				}
				// TODO show commitSize in commit stat?
				if g, e := fi.Size(), int64(0); g != e {
					t.Errorf("wrong size: %v != %v", g, e)
				}
				// TODO show CommitInfo.StartTime as ctime, CommitInfo.Finished as mtime
				// TODO test ctime via .Sys
				// if g, e := fi.ModTime().UTC(), commitFinishTime; g != e {
				// 	t.Errorf("wrong mtime: %v != %v", g, e)
				// }
				return nil
			},
			commitB.ID: func(fi os.FileInfo) error {
				if g, e := fi.Mode(), os.ModeDir|0775; g != e {
					return fmt.Errorf("wrong mode: %v != %v", g, e)
				}
				// TODO show commitSize in commit stat?
				if g, e := fi.Size(), int64(0); g != e {
					t.Errorf("wrong size: %v != %v", g, e)
				}
				// TODO show CommitInfo.StartTime as ctime, ??? as mtime
				// TODO test ctime via .Sys
				// if g, e := fi.ModTime().UTC(), commitFinishTime; g != e {
				// 	t.Errorf("wrong mtime: %v != %v", g, e)
				// }
				return nil
			},
		}))
	})
}
コード例 #9
0
func TestCreateFileInDir(t *testing.T) {
	lion.SetLevel(lion.LevelDebug)
	if testing.Short() {
		t.Skip("Skipped because of short mode")
	}
	testFuse(t, func(c client.APIClient, mountpoint string) {
		require.NoError(t, c.CreateRepo("repo"))
		commit, err := c.StartCommit("repo", "", "")
		require.NoError(t, err)

		require.NoError(t, os.Mkdir(filepath.Join(mountpoint, "repo", commit.ID, "dir"), 0700))
		require.NoError(t, ioutil.WriteFile(filepath.Join(mountpoint, "repo", commit.ID, "dir", "file"), []byte("foo"), 0644))
		require.NoError(t, c.FinishCommit("repo", commit.ID))
	})
}
コード例 #10
0
func testRun(t *testing.T, imageName string, commands []string, expectedStdout string, expectedStderr string) {

	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	client, err := newTestDockerClient()
	require.NoError(t, err)
	err = client.Pull(imageName, PullOptions{})
	require.NoError(t, err)
	container, err := client.Create(
		imageName,
		CreateOptions{
			HasCommand: commands != nil,
		},
	)
	require.NoError(t, err)
	err = client.Start(
		container,
		StartOptions{
			Commands: commands,
		},
	)
	require.NoError(t, err)
	err = client.Wait(container, WaitOptions{})
	require.NoError(t, err)
	stdout := bytes.NewBuffer(nil)
	stderr := bytes.NewBuffer(nil)
	err = client.Logs(container, LogsOptions{Stdout: stdout, Stderr: stderr})
	require.NoError(t, err)
	require.Equal(t, expectedStdout, stdout.String())
	require.Equal(t, expectedStderr, stderr.String())
	err = client.Remove(container, RemoveOptions{})
	require.NoError(t, err)
}
コード例 #11
0
func TestSeekWriteBackwards(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipped because of short mode")
	}

	testFuse(t, func(c client.APIClient, mountpoint string) {
		repo := "test"
		require.NoError(t, c.CreateRepo(repo))
		commit, err := c.StartCommit(repo, "", "")
		require.NoError(t, err)
		path := filepath.Join(mountpoint, repo, commit.ID, "file")
		file, err := os.Create(path)
		require.NoError(t, err)
		defer func() {
			// MAC OS X BEHAVIOR - it allows seek but this is an invalid write
			require.YesError(t, file.Close())
		}()

		_, err = file.Write([]byte("foofoofoo"))
		require.NoError(t, err)

		err = file.Sync()
		require.NoError(t, err)

		offset, err := file.Seek(3, 0)

		fmt.Printf("==== %v - err (%v)\n", time.Now(), err)
		fmt.Printf("==== %v - offset (%v)\n", time.Now(), offset)

		// MAC OS X BEHAVIOR - it allows seek
		require.NoError(t, err)
		require.Equal(t, int64(3), offset)

		/* Leaving in place so the test's intention is clear / for repro'ing manually for mac */

		err = file.Sync()
		require.NoError(t, err)

		n1, err := file.Write([]byte("bar"))

		// MAC OS X BEHAVIOR - it allows seek
		require.NoError(t, err)
		require.Equal(t, 3, n1)

		fmt.Printf("==== %v - write word len %v\n", time.Now(), n1)

		require.NoError(t, c.FinishCommit(repo, commit.ID))
	})
}
コード例 #12
0
func TestBigWrite(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipped because of short mode")
	}

	testFuse(t, func(c client.APIClient, mountpoint string) {
		repo := "test"
		require.NoError(t, c.CreateRepo(repo))
		commit, err := c.StartCommit(repo, "", "")
		require.NoError(t, err)
		path := filepath.Join(mountpoint, repo, commit.ID, "file1")
		stdin := strings.NewReader(fmt.Sprintf("yes | tr -d '\\n' | head -c 1000000 > %s", path))
		require.NoError(t, pkgexec.RunStdin(stdin, "sh"))
		require.NoError(t, c.FinishCommit(repo, commit.ID))
		data, err := ioutil.ReadFile(path)
		require.NoError(t, err)
		require.Equal(t, bytes.Repeat([]byte{'y'}, 1000000), data)
	})
}
コード例 #13
0
func TestEtcdWatch(t *testing.T) {

	if os.Getenv("ETCD_PORT_2379_TCP_ADDR") == "" {
		t.Skip("skipping test; $ETCD_PORT_2379_TCP_ADDR not set")
	}

	t.Parallel()
	client, err := getEtcdClient()
	require.NoError(t, err)
	runWatchTest(t, client)
}
コード例 #14
0
func TestRootReadDir(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipped because of short mode")
	}

	testFuse(t, func(c client.APIClient, mountpoint string) {
		require.NoError(t, c.CreateRepo("one"))
		require.NoError(t, c.CreateRepo("two"))

		require.NoError(t, fstestutil.CheckDir(mountpoint, map[string]fstestutil.FileInfoCheck{
			"one": func(fi os.FileInfo) error {
				if g, e := fi.Mode(), os.ModeDir|0555; g != e {
					return fmt.Errorf("wrong mode: %v != %v", g, e)
				}
				// TODO show repoSize in repo stat?
				if g, e := fi.Size(), int64(0); g != e {
					t.Errorf("wrong size: %v != %v", g, e)
				}
				// TODO show RepoInfo.Created as time
				// if g, e := fi.ModTime().UTC(), repoModTime; g != e {
				// 	t.Errorf("wrong mtime: %v != %v", g, e)
				// }
				return nil
			},
			"two": func(fi os.FileInfo) error {
				if g, e := fi.Mode(), os.ModeDir|0555; g != e {
					return fmt.Errorf("wrong mode: %v != %v", g, e)
				}
				// TODO show repoSize in repo stat?
				if g, e := fi.Size(), int64(0); g != e {
					t.Errorf("wrong size: %v != %v", g, e)
				}
				// TODO show RepoInfo.Created as time
				// if g, e := fi.ModTime().UTC(), repoModTime; g != e {
				// 	t.Errorf("wrong mtime: %v != %v", g, e)
				// }
				return nil
			},
		}))
	})
}
コード例 #15
0
func TestMountCachingViaWalk(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipped because of short mode")
	}

	testFuse(t, func(c client.APIClient, mountpoint string) {
		repo1 := "foo"
		require.NoError(t, c.CreateRepo(repo1))

		// Now if we walk the FS we should see the file.
		// This first act of 'walking' should also initiate the cache

		var filesSeen []interface{}
		walkCallback := func(path string, info os.FileInfo, err error) error {
			tokens := strings.Split(path, "/mnt")
			filesSeen = append(filesSeen, tokens[len(tokens)-1])
			return nil
		}

		err := filepath.Walk(mountpoint, walkCallback)
		require.NoError(t, err)
		require.OneOfEquals(t, "/foo", filesSeen)

		// Now create another repo, and look for it under the mount point
		repo2 := "bar"
		require.NoError(t, c.CreateRepo(repo2))

		// Now if we walk the FS we should see the new file.
		// This now works. But originally (issue #205) this second ls on mac doesn't report the file!

		filesSeen = make([]interface{}, 0)
		err = filepath.Walk(mountpoint, walkCallback)
		require.NoError(t, err)
		require.OneOfEquals(t, "/foo", filesSeen)
		require.OneOfEquals(t, "/bar", filesSeen)

	})
}
コード例 #16
0
func TestFruitStand(t *testing.T) {
	require.NoError(t, exec.Command("pachctl", "create-repo", "data").Run())
	require.NoError(t, exec.Command("pachctl", "start-commit", "data", "master").Run())
	putFileCmd := exec.Command("pachctl", "put-file", "data", "master", "sales")
	set1, err := os.Open("set1.txt")
	require.NoError(t, err)
	putFileCmd.Stdin = set1
	require.NoError(t, putFileCmd.Run())
	require.NoError(t, exec.Command("pachctl", "finish-commit", "data", "master").Run())
	require.NoError(t, exec.Command("pachctl", "create-pipeline", "-f", "pipeline.json").Run())
	time.Sleep(5 * time.Second)
	require.NoError(t, exec.Command("pachctl", "flush-commit", "data/master").Run())
}
コード例 #17
0
func TestScraper(t *testing.T) {
	require.NoError(t, exec.Command("pachctl", "create-repo", "urls").Run())
	require.NoError(t, exec.Command("pachctl", "start-commit", "urls", "master").Run())
	putFileCmd := exec.Command("pachctl", "put-file", "urls", "master", "urls")
	urls, err := os.Open("urls")
	require.NoError(t, err)
	putFileCmd.Stdin = urls
	require.NoError(t, putFileCmd.Run())
	require.NoError(t, exec.Command("pachctl", "finish-commit", "urls", "master").Run())
	require.NoError(t, exec.Command("pachctl", "create-pipeline", "-f", "scraper.json").Run())
	time.Sleep(5 * time.Second)
	require.NoError(t, exec.Command("pachctl", "flush-commit", "urls/master").Run())
}
コード例 #18
0
func TestOverwriteFile(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipped because of short mode")
	}
	testFuse(t, func(c client.APIClient, mountpoint string) {
		require.NoError(t, c.CreateRepo("repo"))
		commit1, err := c.StartCommit("repo", "", "")
		require.NoError(t, err)
		_, err = c.PutFile("repo", commit1.ID, "file", strings.NewReader("foo\n"))
		require.NoError(t, err)
		require.NoError(t, c.FinishCommit("repo", commit1.ID))
		commit2, err := c.StartCommit("repo", commit1.ID, "")
		require.NoError(t, err)
		path := filepath.Join(mountpoint, "repo", commit2.ID, "file")
		stdin := strings.NewReader(fmt.Sprintf("echo bar >%s", path))
		require.NoError(t, pkgexec.RunStdin(stdin, "sh"))
		require.NoError(t, c.FinishCommit("repo", commit2.ID))
		result, err := ioutil.ReadFile(path)
		require.NoError(t, err)
		require.Equal(t, "bar\n", string(result))
	})
}
コード例 #19
0
func TestSpacedWrites(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipped because of short mode")
	}

	testFuse(t, func(c client.APIClient, mountpoint string) {
		repo := "test"
		require.NoError(t, c.CreateRepo(repo))
		commit, err := c.StartCommit(repo, "", "")
		require.NoError(t, err)
		path := filepath.Join(mountpoint, repo, commit.ID, "file")
		file, err := os.Create(path)
		require.NoError(t, err)
		_, err = file.Write([]byte("foo"))
		require.NoError(t, err)
		_, err = file.Write([]byte("foo"))
		require.NoError(t, err)
		require.NoError(t, file.Close())
		require.NoError(t, c.FinishCommit(repo, commit.ID))
		data, err := ioutil.ReadFile(path)
		require.NoError(t, err)
		require.Equal(t, "foofoo", string(data))
	})
}
コード例 #20
0
func TestSeekWriteGap(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipped because of short mode")
	}

	testFuse(t, func(c client.APIClient, mountpoint string) {
		repo := "test"
		require.NoError(t, c.CreateRepo(repo))
		commit, err := c.StartCommit(repo, "", "")
		require.NoError(t, err)
		path := filepath.Join(mountpoint, repo, commit.ID, "file")
		file, err := os.Create(path)
		require.NoError(t, err)
		defer func() {
			require.NoError(t, file.Close())
		}()

		_, err = file.Write([]byte("foo"))
		require.NoError(t, err)

		err = file.Sync()
		require.NoError(t, err)

		offset, err := file.Seek(6, 0)

		fmt.Printf("==== %v - err (%v)\n", time.Now(), err)
		fmt.Printf("==== %v - offset (%v)\n", time.Now(), offset)
		require.YesError(t, err)
		require.Equal(t, int64(0), offset)

		/* Leaving in place so the test's intention is clear / for repro'ing manually for mac

		err = file.Sync()
		require.NoError(t, err)

		n1, err := file.Write([]byte("baz"))
		require.YesError(t, err)
		require.Equal(t, 3, n1)

		require.NoError(t, c.FinishCommit(repo, commit.ID)) */
	})
}
コード例 #21
0
func runTest(t *testing.T, client Client) {
	err := client.Set("foo", "one", 0)
	require.NoError(t, err)
	value, err := client.Get("foo")
	require.NoError(t, err)
	require.Equal(t, "one", value)
	//values, err := client.GetAll("foo")
	//require.NoError(t, err)
	//require.Equal(t, map[string]string{"foo": "one"}, values)

	err = client.Set("a/b/foo", "one", 0)
	require.NoError(t, err)
	err = client.Set("a/b/bar", "two", 0)
	require.NoError(t, err)
	values, err := client.GetAll("a/b")
	require.NoError(t, err)
	require.Equal(t, map[string]string{"a/b/foo": "one", "a/b/bar": "two"}, values)

	require.NoError(t, client.Close())
}
コード例 #22
0
func TestWriteAndRead(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipped because of short mode")
	}

	testFuse(t, func(c client.APIClient, mountpoint string) {
		repoName := "foo"
		require.NoError(t, c.CreateRepo(repoName))
		commit, err := c.StartCommit(repoName, "", "")
		require.NoError(t, err)
		greeting := "Hello, world\n"
		filePath := filepath.Join(mountpoint, repoName, commit.ID, "greeting")
		require.NoError(t, ioutil.WriteFile(filePath, []byte(greeting), 0644))
		readGreeting, err := ioutil.ReadFile(filePath)
		require.NoError(t, err)
		require.Equal(t, greeting, string(readGreeting))
		require.NoError(t, c.FinishCommit(repoName, commit.ID))
		data, err := ioutil.ReadFile(filePath)
		require.NoError(t, err)
		require.Equal(t, []byte(greeting), data)
	})
}
コード例 #23
0
func testBasicRethink(t *testing.T, apiServer persist.APIServer) {
	_, err := apiServer.CreatePipelineInfo(
		context.Background(),
		&persist.PipelineInfo{
			PipelineName: "foo",
		},
	)
	require.NoError(t, err)
	pipelineInfo, err := apiServer.GetPipelineInfo(
		context.Background(),
		&ppsclient.Pipeline{Name: "foo"},
	)
	require.NoError(t, err)
	require.Equal(t, pipelineInfo.PipelineName, "foo")
	input := &ppsclient.JobInput{Commit: client.NewCommit("bar", uuid.NewWithoutDashes())}
	jobInfo, err := apiServer.CreateJobInfo(
		context.Background(),
		&persist.JobInfo{
			JobID:        uuid.NewWithoutDashes(),
			PipelineName: "foo",
			Inputs:       []*ppsclient.JobInput{input},
		},
	)
	jobID := jobInfo.JobID
	input2 := &ppsclient.JobInput{Commit: client.NewCommit("fizz", uuid.NewWithoutDashes())}

	_, err = apiServer.CreateJobInfo(
		context.Background(),
		&persist.JobInfo{
			JobID:        uuid.NewWithoutDashes(),
			PipelineName: "buzz",
			Inputs:       []*ppsclient.JobInput{input2},
		},
	)
	require.NoError(t, err)
	jobInfo, err = apiServer.InspectJob(
		context.Background(),
		&ppsclient.InspectJobRequest{
			Job: &ppsclient.Job{
				ID: jobInfo.JobID,
			},
		},
	)
	require.NoError(t, err)
	require.Equal(t, jobInfo.JobID, jobID)
	require.Equal(t, "foo", jobInfo.PipelineName)
	jobInfos, err := apiServer.ListJobInfos(
		context.Background(),
		&ppsclient.ListJobRequest{
			Pipeline: &ppsclient.Pipeline{Name: "foo"},
		},
	)
	require.NoError(t, err)
	require.Equal(t, len(jobInfos.JobInfo), 1)
	require.Equal(t, jobInfos.JobInfo[0].JobID, jobID)
	jobInfos, err = apiServer.ListJobInfos(
		context.Background(),
		&ppsclient.ListJobRequest{
			InputCommit: []*pfsclient.Commit{input.Commit},
		},
	)
	require.NoError(t, err)
	require.Equal(t, len(jobInfos.JobInfo), 1)
	require.Equal(t, jobInfos.JobInfo[0].JobID, jobID)
	jobInfos, err = apiServer.ListJobInfos(
		context.Background(),
		&ppsclient.ListJobRequest{
			Pipeline:    &ppsclient.Pipeline{Name: "foo"},
			InputCommit: []*pfsclient.Commit{input.Commit},
		},
	)
	require.NoError(t, err)
	require.Equal(t, len(jobInfos.JobInfo), 1)
	require.Equal(t, jobInfos.JobInfo[0].JobID, jobID)
}
コード例 #24
0
func testFuse(
	t *testing.T,
	test func(client client.APIClient, mountpoint string),
) {
	// don't leave goroutines running
	var wg sync.WaitGroup
	defer wg.Wait()

	tmp, err := ioutil.TempDir("", "pachyderm-test-")
	require.NoError(t, err)
	defer func() {
		_ = os.RemoveAll(tmp)
	}()

	// closed on successful termination
	quit := make(chan struct{})
	defer close(quit)
	listener, err := net.Listen("tcp", "localhost:0")
	require.NoError(t, err)
	defer func() {
		_ = listener.Close()
	}()

	// TODO try to share more of this setup code with various main
	// functions
	localAddress := listener.Addr().String()
	srv := grpc.NewServer()
	const (
		numShards = 1
	)
	sharder := shard.NewLocalSharder([]string{localAddress}, numShards)
	hasher := pfsserver.NewHasher(numShards, 1)
	router := shard.NewRouter(
		sharder,
		grpcutil.NewDialer(
			grpc.WithInsecure(),
		),
		localAddress,
	)

	blockDir := filepath.Join(tmp, "blocks")
	blockServer, err := server.NewLocalBlockAPIServer(blockDir)
	require.NoError(t, err)
	pfsclient.RegisterBlockAPIServer(srv, blockServer)

	driver, err := drive.NewDriver(localAddress)
	require.NoError(t, err)

	apiServer := server.NewAPIServer(
		hasher,
		router,
	)
	pfsclient.RegisterAPIServer(srv, apiServer)

	internalAPIServer := server.NewInternalAPIServer(
		hasher,
		router,
		driver,
	)
	pfsclient.RegisterInternalAPIServer(srv, internalAPIServer)

	wg.Add(1)
	go func() {
		defer wg.Done()
		if err := srv.Serve(listener); err != nil {
			select {
			case <-quit:
				// orderly shutdown
				return
			default:
				t.Errorf("grpc serve: %v", err)
			}
		}
	}()

	clientConn, err := grpc.Dial(localAddress, grpc.WithInsecure())
	require.NoError(t, err)
	apiClient := pfsclient.NewAPIClient(clientConn)
	mounter := fuse.NewMounter(localAddress, apiClient)
	mountpoint := filepath.Join(tmp, "mnt")
	require.NoError(t, os.Mkdir(mountpoint, 0700))
	ready := make(chan bool)
	wg.Add(1)
	go func() {
		defer wg.Done()
		require.NoError(t, mounter.MountAndCreate(mountpoint, nil, nil, ready, true))
	}()

	<-ready

	defer func() {
		_ = mounter.Unmount(mountpoint)
	}()
	test(client.APIClient{PfsAPIClient: apiClient}, mountpoint)
}
コード例 #25
0
func TestNoDelimiter(t *testing.T) {

	if testing.Short() {
		t.Skip("Skipped because of short mode")
	}
	testFuse(t, func(c client.APIClient, mountpoint string) {
		repo := "test"
		name := "foo.bin"
		require.NoError(t, c.CreateRepo(repo))
		commit1, err := c.StartCommit(repo, "", "")
		require.NoError(t, err)

		rawMessage := "Some\ncontent\nthat\nshouldnt\nbe\nline\ndelimited.\n"
		filePath := filepath.Join(mountpoint, repo, commit1.ID, name)

		// Write a big blob that would normally not fit in a block
		var expectedOutputA []byte
		for !(len(expectedOutputA) > 9*1024*1024) {
			expectedOutputA = append(expectedOutputA, []byte(rawMessage)...)
		}
		require.NoError(t, ioutil.WriteFile(filePath, expectedOutputA, 0644))

		// Write another big block
		var expectedOutputB []byte
		for !(len(expectedOutputB) > 10*1024*1024) {
			expectedOutputB = append(expectedOutputB, []byte(rawMessage)...)
		}
		require.NoError(t, ioutil.WriteFile("/tmp/b", expectedOutputB, 0644))
		stdin := strings.NewReader(fmt.Sprintf("cat /tmp/b >>%s", filePath))
		require.NoError(t, pkgexec.RunStdin(stdin, "sh"))

		// Finish the commit so I can read the data
		require.NoError(t, c.FinishCommit(repo, commit1.ID))

		// Make sure all the content is there
		var buffer bytes.Buffer
		require.NoError(t, c.GetFile(repo, commit1.ID, name, 0, 0, "", nil, &buffer))
		require.Equal(t, len(expectedOutputA)+len(expectedOutputB), buffer.Len())
		require.Equal(t, string(append(expectedOutputA, expectedOutputB...)), buffer.String())

		// Now verify that each block only contains objects of the size we've written
		bigModulus := 10 // Make it big to make it less likely that I return both blocks together
		blockLengths := []interface{}{len(expectedOutputA), len(expectedOutputB)}
		for b := 0; b < bigModulus; b++ {
			blockFilter := &pfsclient.Shard{
				BlockNumber:  uint64(b),
				BlockModulus: uint64(bigModulus),
			}

			buffer.Reset()
			require.NoError(t, c.GetFile(repo, commit1.ID, name, 0, 0, "", blockFilter, &buffer))

			// If any single block returns content of size equal to the total, we
			// got a block collision and we're not testing anything
			require.NotEqual(t, len(expectedOutputA)+len(expectedOutputB), buffer.Len())
			if buffer.Len() == 0 {
				continue
			}
			require.EqualOneOf(t, blockLengths, buffer.Len())
		}
	})
}
コード例 #26
0
func TestDelimitJSON(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipped because of short mode")
	}
	testFuse(t, func(c client.APIClient, mountpoint string) {
		repo := "abba"
		require.NoError(t, c.CreateRepo(repo))
		commit, err := c.StartCommit(repo, "", "")
		require.NoError(t, err)
		var expectedOutput []byte
		rawMessage := `{
		"level":"debug",
		"timestamp":"345",
		"message":{
			"thing":"foo"
		},
		"timing":[1,3,34,6,7]
	}`
		for !(len(expectedOutput) > 9*1024*1024) {
			expectedOutput = append(expectedOutput, []byte(rawMessage)...)
		}
		filePath := filepath.Join(mountpoint, repo, commit.ID, "foo.json")
		require.NoError(t, ioutil.WriteFile(filePath, expectedOutput, 0644))
		require.NoError(t, c.FinishCommit(repo, commit.ID))
		// Make sure all the content is there
		var buffer bytes.Buffer
		require.NoError(t, c.GetFile(repo, commit.ID, "foo.json", 0, 0, "", nil, &buffer))
		require.Equal(t, len(expectedOutput), buffer.Len())
		require.Equal(t, string(expectedOutput), buffer.String())

		// Now verify that each block contains only valid JSON objects
		bigModulus := 10 // Make it big to make it less likely that I return both blocks together
		for b := 0; b < bigModulus; b++ {
			blockFilter := &pfsclient.Shard{
				BlockNumber:  uint64(b),
				BlockModulus: uint64(bigModulus),
			}

			buffer.Reset()
			require.NoError(t, c.GetFile(repo, commit.ID, "foo.json", 0, 0, "", blockFilter, &buffer))

			// If any single block returns content of size equal to the total, we
			// got a block collision and we're not testing anything
			require.NotEqual(t, buffer.Len(), len(expectedOutput))

			var value json.RawMessage
			decoder := json.NewDecoder(&buffer)
			for {
				err = decoder.Decode(&value)
				if err != nil {
					if err == io.EOF {
						break
					} else {
						require.NoError(t, err)
					}
				}
				require.Equal(t, rawMessage, string(value))
			}
		}
	})
}