Exemple #1
0
func TestFromCommit(t *testing.T) {
	t.Parallel()
	repo := uniqueString("TestFromCommit")
	pachClient := getPachClient(t)
	seed := time.Now().UnixNano()
	rand := rand.New(rand.NewSource(seed))
	err := pfsutil.CreateRepo(pachClient, repo)
	require.NoError(t, err)
	commit1, err := pfsutil.StartCommit(pachClient, repo, "")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pachClient, repo, commit1.Id, "file", 0, workload.NewReader(rand, KB))
	require.NoError(t, err)
	err = pfsutil.FinishCommit(pachClient, repo, commit1.Id)
	require.NoError(t, err)
	commit2, err := pfsutil.StartCommit(pachClient, repo, commit1.Id)
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pachClient, repo, commit2.Id, "file", 0, workload.NewReader(rand, KB))
	require.NoError(t, err)
	err = pfsutil.FinishCommit(pachClient, repo, commit2.Id)
	require.NoError(t, err)
	var buffer bytes.Buffer
	require.NoError(t, pfsutil.GetFile(pachClient, repo, commit2.Id, "file", 0, 0, commit1.Id, nil, &buffer))
	require.Equal(t, buffer.Len(), KB)
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pachClient, repo, commit2.Id, "file", 0, 0, "", nil, &buffer))
	require.Equal(t, buffer.Len(), 2*KB)
}
Exemple #2
0
func TestJob(t *testing.T) {
	t.Parallel()
	dataRepo := uniqueString("TestJob.data")
	pachClient := getPachClient(t)
	require.NoError(t, pfsutil.CreateRepo(pachClient, dataRepo))
	commit, err := pfsutil.StartCommit(pachClient, dataRepo, "")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pachClient, dataRepo, commit.Id, "file", 0, strings.NewReader("foo\n"))
	require.NoError(t, err)
	require.NoError(t, pfsutil.FinishCommit(pachClient, dataRepo, commit.Id))
	job, err := ppsutil.CreateJob(
		pachClient,
		"",
		[]string{"cp", path.Join("/pfs", dataRepo, "file"), "/pfs/out/file"},
		nil,
		1,
		[]*pps.JobInput{{Commit: commit}},
		"",
	)
	require.NoError(t, err)
	inspectJobRequest := &pps.InspectJobRequest{
		Job:         job,
		BlockOutput: true,
		BlockState:  true,
	}
	jobInfo, err := pachClient.InspectJob(context.Background(), inspectJobRequest)
	require.NoError(t, err)
	require.Equal(t, pps.JobState_JOB_STATE_SUCCESS.String(), jobInfo.State.String())
	commitInfo, err := pfsutil.InspectCommit(pachClient, jobInfo.OutputCommit.Repo.Name, jobInfo.OutputCommit.Id)
	require.NoError(t, err)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, commitInfo.CommitType)
	var buffer bytes.Buffer
	require.NoError(t, pfsutil.GetFile(pachClient, jobInfo.OutputCommit.Repo.Name, jobInfo.OutputCommit.Id, "file", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
}
Exemple #3
0
func (s *serverGroup) run(t *testing.T) {
	var wg sync.WaitGroup
	defer wg.Wait()
	for i, server := range s.servers {
		wg.Add(1)
		i := i
		server := server
		go func() {
			defer wg.Done()
			require.Equal(
				t,
				shard.ErrCancelled,
				s.sharder.Register(s.cancel, fmt.Sprintf("address-%d", i+s.offset), server),
			)
		}()
	}
	for i, frontend := range s.frontends {
		wg.Add(1)
		i := i
		frontend := frontend
		go func() {
			defer wg.Done()
			require.Equal(
				t,
				shard.ErrCancelled,
				s.sharder.RegisterFrontend(s.cancel, fmt.Sprintf("address-%d", i+s.offset), frontend),
			)
		}()
	}
}
Exemple #4
0
func testRun(t *testing.T, imageName string, commands []string, expectedStdout string, expectedStderr string) {
	client, err := newTestDockerClient()
	require.NoError(t, err)
	err = client.Pull(imageName, PullOptions{})
	require.NoError(t, err)
	containers, err := client.Create(
		imageName,
		CreateOptions{
			HasCommand: commands != nil,
		},
	)
	require.NoError(t, err)
	require.Equal(t, 1, len(containers))
	container := containers[0]
	err = client.Start(
		container,
		StartOptions{
			Commands: commands,
		},
	)
	require.NoError(t, err)
	err = client.Wait(container, WaitOptions{})
	require.NoError(t, err)
	stdout := bytes.NewBuffer(nil)
	stderr := bytes.NewBuffer(nil)
	err = client.Logs(container, LogsOptions{Stdout: stdout, Stderr: stderr})
	require.NoError(t, err)
	require.Equal(t, expectedStdout, stdout.String())
	require.Equal(t, expectedStderr, stderr.String())
	err = client.Remove(container, RemoveOptions{})
	require.NoError(t, err)
}
func runWatchTest(t *testing.T, client Client) {
	cancel := make(chan bool)
	err := client.Watch(
		"watch/foo",
		cancel,
		func(value string) error {
			if value == "" {
				return client.Set("watch/foo", "bar", 0)
			}
			require.Equal(t, "bar", value)
			close(cancel)
			return nil
		},
	)
	require.Equal(t, ErrCancelled, err)

	cancel = make(chan bool)
	err = client.WatchAll(
		"watchAll/foo",
		cancel,
		func(value map[string]string) error {
			if value == nil {
				return client.Set("watchAll/foo/bar", "quux", 0)
			}
			require.Equal(t, map[string]string{"watchAll/foo/bar": "quux"}, value)
			close(cancel)
			return nil
		},
	)
	require.Equal(t, ErrCancelled, err)
}
Exemple #6
0
func newCluster(tb testing.TB, discoveryClient discovery.Client, servers map[string]*grpc.Server) *cluster {
	realSharder := shard.NewTestSharder(
		discoveryClient,
		testShardsPerServer*testNumServers,
		testNumReplicas,
		testNamespace(),
	)
	sharder := route.NewSharder(
		testShardsPerServer*testNumServers,
		testNumReplicas,
	)
	cluster := cluster{
		servers:         make(map[string]server.APIServer),
		internalServers: make(map[string]server.InternalAPIServer),
		cancels:         make(map[string]chan bool),
		internalCancels: make(map[string]chan bool),
		cancel:          make(chan bool),
		realSharder:     realSharder,
		sharder:         sharder,
		tb:              tb,
	}
	for address, s := range servers {
		cluster.addresses = append(cluster.addresses, address)
		router := route.NewRouter(
			cluster.realSharder,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		)
		apiServer := server.NewAPIServer(
			cluster.sharder,
			router,
		)
		cluster.servers[address] = apiServer
		cluster.cancels[address] = make(chan bool)
		go func(address string) {
			require.Equal(tb, cluster.realSharder.RegisterFrontend(cluster.cancels[address], address, cluster.servers[address]), shard.ErrCancelled)
		}(address)
		pfs.RegisterAPIServer(s, apiServer)
		internalAPIServer := server.NewInternalAPIServer(
			cluster.sharder,
			router,
			getDriver(tb, address),
		)
		pfs.RegisterInternalAPIServer(s, internalAPIServer)
		cluster.internalServers[address] = internalAPIServer
		cluster.internalCancels[address] = make(chan bool)
		go func(address string) {
			require.Equal(tb, cluster.realSharder.Register(cluster.internalCancels[address], address, cluster.internalServers[address]), shard.ErrCancelled)
		}(address)
	}
	return &cluster
}
func TestMountBig(t *testing.T) {
	t.Skip()
	t.Parallel()
	apiClient := getPfsClient(t)
	repoName := uniqueString("testMountBigRepo")

	err := pfsutil.CreateRepo(apiClient, repoName)
	require.NoError(t, err)

	directory := "/compile/testMount"
	mounter := fuse.NewMounter("localhost", apiClient)
	ready := make(chan bool)
	go func() {
		err = mounter.Mount(directory, &pfs.Shard{Number: 0, Modulus: 1}, nil, ready)
		require.NoError(t, err)
	}()
	<-ready

	_, err = os.Stat(filepath.Join(directory, repoName))
	require.NoError(t, err)

	commit, err := pfsutil.StartCommit(apiClient, repoName, "")
	require.NoError(t, err)
	require.NotNil(t, commit)
	newCommitID := commit.Id

	bigValue := make([]byte, 1024*1024*300)
	for i := 0; i < 1024*1024*300; i++ {
		bigValue[i] = 'a'
	}

	wg := sync.WaitGroup{}
	for j := 0; j < 5; j++ {
		wg.Add(1)
		go func(j int) {
			defer wg.Done()
			err := ioutil.WriteFile(filepath.Join(directory, repoName, newCommitID, fmt.Sprintf("big%d", j)), bigValue, 0666)
			require.NoError(t, err)
		}(j)
	}
	wg.Wait()

	err = pfsutil.FinishCommit(apiClient, repoName, newCommitID)
	require.NoError(t, err)

	wg = sync.WaitGroup{}
	for j := 0; j < 5; j++ {
		wg.Add(1)
		go func(j int) {
			defer wg.Done()
			data, err := ioutil.ReadFile(filepath.Join(directory, repoName, newCommitID, fmt.Sprintf("big%d", j)))
			require.NoError(t, err)
			require.Equal(t, bigValue, data)
		}(j)
	}
	wg.Wait()

	err = mounter.Unmount(directory)
	require.NoError(t, err)
}
Exemple #8
0
func (c *cluster) Restart(server int) {
	address := c.addresses[server]
	c.cancels[address] = make(chan bool)
	go func() {
		require.Equal(c.tb, c.addresser.Register(c.cancels[address], address, address, c.internalServers[address]), route.ErrCancelled)
	}()
}
Exemple #9
0
func TestGetNameToNodeInfo(t *testing.T) {
	pipeline, err := parse.NewParser().ParsePipeline("../parse/testdata/basic")
	require.NoError(t, err)
	nodeInfos, err := getNameToNodeInfo(pipeline.NameToNode)
	require.NoError(t, err)
	require.Equal(t, []string{"bar-node"}, nodeInfos["baz-node-bar-in-bar-out-in"].Parents)
}
Exemple #10
0
func TestJob(t *testing.T) {
	dataRepo := uniqueString("TestJob.data")
	pfsClient := getPfsClient(t)
	require.NoError(t, pfsutil.CreateRepo(pfsClient, dataRepo))
	commit, err := pfsutil.StartCommit(pfsClient, dataRepo, "")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pfsClient, dataRepo, commit.Id, "file", 0, strings.NewReader("foo"))
	require.NoError(t, err)
	require.NoError(t, pfsutil.FinishCommit(pfsClient, dataRepo, commit.Id))
	ppsClient := getPpsClient(t)
	job, err := ppsutil.CreateJob(
		ppsClient,
		"",
		[]string{"cp", path.Join("/pfs", dataRepo, "file"), "/pfs/out/file"},
		"",
		1,
		[]*pfs.Commit{commit},
		"",
	)
	require.NoError(t, err)
	inspectJobRequest := &pps.InspectJobRequest{
		Job:         job,
		BlockOutput: true,
		BlockState:  true,
	}
	jobInfo, err := ppsClient.InspectJob(context.Background(), inspectJobRequest)
	require.NoError(t, err)
	var buffer bytes.Buffer
	require.NoError(t, pfsutil.GetFile(pfsClient, jobInfo.OutputCommit.Repo.Name, jobInfo.OutputCommit.Id, "file", 0, 0, nil, &buffer))
	require.Equal(t, "foo", buffer.String())
}
Exemple #11
0
func RunBench(
	b *testing.B,
	f func(*testing.B, pfs.APIClient),
) {
	discoveryClient, err := getEtcdClient()
	require.NoError(b, err)
	var cluster *cluster
	prototest.RunB(
		b,
		testNumServers,
		func(servers map[string]*grpc.Server) {
			cluster = registerFunc(b, discoveryClient, servers)
		},
		func(b *testing.B, clientConns map[string]*grpc.ClientConn) {
			var clientConn *grpc.ClientConn
			for _, c := range clientConns {
				clientConn = c
				break
			}
			go func() {
				require.Equal(b, cluster.realSharder.AssignRoles(cluster.cancel), shard.ErrCancelled)
			}()
			cluster.WaitForAvailability()
			f(
				b,
				pfs.NewAPIClient(
					clientConn,
				),
			)
		},
	)
	cluster.Shutdown()
}
Exemple #12
0
func newCluster(tb testing.TB, discoveryClient discovery.Client, servers map[string]*grpc.Server) Cluster {
	cluster := cluster{
		rolers:  make(map[string]role.Roler),
		servers: make(map[string]server.CombinedAPIServer),
		addresser: route.NewDiscoveryAddresser(
			discoveryClient,
			testNamespace(),
		),
		sharder: route.NewSharder(
			testShardsPerServer * testNumServers,
		),
		tb: tb,
	}
	for address, s := range servers {
		combinedAPIServer := server.NewCombinedAPIServer(
			cluster.sharder,
			route.NewRouter(
				cluster.addresser,
				grpcutil.NewDialer(
					grpc.WithInsecure(),
				),
				address,
			),
			getDriver(tb, address),
		)
		pfs.RegisterApiServer(s, combinedAPIServer)
		pfs.RegisterInternalApiServer(s, combinedAPIServer)
		roler := role.NewRoler(cluster.addresser, cluster.sharder, combinedAPIServer, address, testNumReplicas)
		go func() { require.Equal(tb, roler.Run(), discovery.ErrCancelled) }()
		cluster.addresses = append(cluster.addresses, address)
		cluster.rolers[address] = roler
		cluster.servers[address] = combinedAPIServer
	}
	return &cluster
}
Exemple #13
0
func checkPFSOutput(t *testing.T, pfsAPIClient pfs.APIClient, outputCommit *pfs.Commit, filePathToContent map[string][]byte) {
	for filePath, content := range filePathToContent {
		getContent, err := getPFSContent(pfsAPIClient, outputCommit, filePath)
		require.NoError(t, err)
		require.Equal(t, content, getContent)
	}
}
Exemple #14
0
func testMountBig(t *testing.T, apiClient pfs.APIClient, cluster Cluster) {
	repoName := "testMountBigRepo"

	err := pfsutil.CreateRepo(apiClient, repoName)
	require.NoError(t, err)

	directory := "/compile/testMount"
	mounter := fuse.NewMounter("localhost", apiClient)
	ready := make(chan bool)
	go func() {
		err = mounter.Mount(directory, 0, 1, ready)
		require.NoError(t, err)
	}()
	<-ready

	_, err = os.Stat(filepath.Join(directory, repoName))
	require.NoError(t, err)

	_, err = os.Stat(filepath.Join(directory, repoName, "scratch"))
	require.NoError(t, err)

	commit, err := pfsutil.StartCommit(apiClient, repoName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, commit)
	newCommitID := commit.Id

	bigValue := make([]byte, 1024*1024*300)
	for i := 0; i < 1024*1024*300; i++ {
		bigValue[i] = 'a'
	}

	wg := sync.WaitGroup{}
	for j := 0; j < 5; j++ {
		wg.Add(1)
		go func(j int) {
			defer wg.Done()
			err := ioutil.WriteFile(filepath.Join(directory, repoName, newCommitID, fmt.Sprintf("big%d", j)), bigValue, 0666)
			require.NoError(t, err)
		}(j)
	}
	wg.Wait()

	err = pfsutil.FinishCommit(apiClient, repoName, newCommitID)
	require.NoError(t, err)

	wg = sync.WaitGroup{}
	for j := 0; j < 5; j++ {
		wg.Add(1)
		go func(j int) {
			defer wg.Done()
			data, err := ioutil.ReadFile(filepath.Join(directory, repoName, newCommitID, fmt.Sprintf("big%d", j)))
			require.NoError(t, err)
			require.Equal(t, bigValue, data)
		}(j)
	}
	wg.Wait()

	err = mounter.Unmount(directory)
	require.NoError(t, err)
}
Exemple #15
0
func testMountBig(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
	repositoryName := TestRepositoryName()

	err := pfsutil.InitRepository(apiClient, repositoryName)
	require.NoError(t, err)

	directory := "/compile/testMount"
	mounter := fuse.NewMounter(apiClient)
	err = mounter.Mount(repositoryName, "", directory, 0, 1)
	require.NoError(t, err)

	_, err = os.Stat(filepath.Join(directory, "scratch"))
	require.NoError(t, err)

	commit, err := pfsutil.Branch(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, commit)
	newCommitID := commit.Id

	bigValue := make([]byte, 1024*1024*300)
	for i := 0; i < 1024*1024*300; i++ {
		bigValue[i] = 'a'
	}

	wg := sync.WaitGroup{}
	for j := 0; j < 5; j++ {
		wg.Add(1)
		go func(j int) {
			defer wg.Done()
			err := ioutil.WriteFile(filepath.Join(directory, newCommitID, fmt.Sprintf("big%d", j)), bigValue, 0666)
			require.NoError(t, err)
		}(j)
	}
	wg.Wait()

	err = pfsutil.Write(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)

	wg = sync.WaitGroup{}
	for j := 0; j < 5; j++ {
		wg.Add(1)
		go func(j int) {
			defer wg.Done()
			data, err := ioutil.ReadFile(filepath.Join(directory, newCommitID, fmt.Sprintf("big%d", j)))
			require.NoError(t, err)
			require.Equal(t, bigValue, data)
		}(j)
	}
	wg.Wait()

	err = mounter.Unmount(directory)
	require.NoError(t, err)
	err = mounter.Wait(directory)
	require.NoError(t, err)
}
func testBasicRethink(t *testing.T, apiServer persist.APIServer) {
	jobInfo, err := apiServer.CreateJobInfo(
		context.Background(),
		&persist.JobInfo{
			Spec: &persist.JobInfo_PipelineName{
				PipelineName: "foo",
			},
		},
	)
	require.NoError(t, err)
	getJobInfo, err := apiServer.GetJobInfo(
		context.Background(),
		&pps.Job{
			Id: jobInfo.JobId,
		},
	)
	require.NoError(t, err)
	require.Equal(t, jobInfo.JobId, getJobInfo.JobId)
	require.Equal(t, "foo", getJobInfo.GetPipelineName())
}
Exemple #17
0
func (s *serverGroup) run(t *testing.T) {
	var wg sync.WaitGroup
	defer wg.Wait()
	for _, roler := range s.rolers {
		wg.Add(1)
		go func(roler Roler) {
			defer wg.Done()
			require.Equal(t, discovery.ErrCancelled, roler.Run())
		}(roler)
	}
}
func runTest(t *testing.T, client Client) {
	err := client.Set("foo", "one", 0)
	require.NoError(t, err)
	value, err := client.Get("foo")
	require.NoError(t, err)
	require.Equal(t, "one", value)
	//values, err := client.GetAll("foo")
	//require.NoError(t, err)
	//require.Equal(t, map[string]string{"foo": "one"}, values)

	err = client.Set("a/b/foo", "one", 0)
	require.NoError(t, err)
	err = client.Set("a/b/bar", "two", 0)
	require.NoError(t, err)
	values, err := client.GetAll("a/b")
	require.NoError(t, err)
	require.Equal(t, map[string]string{"a/b/foo": "one", "a/b/bar": "two"}, values)

	require.NoError(t, client.Close())
}
Exemple #19
0
func TestSimple(t *testing.T) {
	driver, err := NewDriver(getBtrfsRootDir(t), "drive.TestSimple")
	require.NoError(t, err)
	shards := make(map[uint64]bool)
	shards[0] = true
	repo := &pfs.Repo{Name: "drive.TestSimple"}
	require.NoError(t, driver.CreateRepo(repo))
	commit1 := &pfs.Commit{
		Repo: repo,
		Id:   "commit1",
	}
	require.NoError(t, driver.StartCommit(nil, commit1, shards))
	file1 := &pfs.File{
		Commit: commit1,
		Path:   "foo",
	}
	require.NoError(t, driver.PutFile(file1, 0, 0, strings.NewReader("foo")))
	require.NoError(t, driver.FinishCommit(commit1, shards))
	reader, err := driver.GetFile(file1, 0)
	require.NoError(t, err)
	contents, err := ioutil.ReadAll(reader)
	require.NoError(t, err)
	require.Equal(t, string(contents), "foo")
	commit2 := &pfs.Commit{
		Repo: repo,
		Id:   "commit2",
	}
	require.NoError(t, driver.StartCommit(commit1, commit2, shards))
	file2 := &pfs.File{
		Commit: commit2,
		Path:   "bar",
	}
	require.NoError(t, driver.PutFile(file2, 0, 0, strings.NewReader("bar")))
	require.NoError(t, driver.FinishCommit(commit2, shards))
	changes, err := driver.ListChange(file2, commit1, 0)
	require.NoError(t, err)
	require.Equal(t, len(changes), 1)
	require.Equal(t, changes[0].File, file2)
	require.Equal(t, changes[0].OffsetBytes, uint64(0))
	require.Equal(t, changes[0].SizeBytes, uint64(3))
}
Exemple #20
0
func getJobForPipeline(t *testing.T, jobAPIClient pps.JobAPIClient, pipeline *pps.Pipeline) *pps.Job {
	jobInfos, err := jobAPIClient.ListJob(
		context.Background(),
		&pps.ListJobRequest{
			Pipeline: pipeline,
		},
	)
	require.NoError(t, err)
	require.NotNil(t, jobInfos)
	require.Equal(t, 1, len(jobInfos.JobInfo))
	return jobInfos.JobInfo[0].Job
}
Exemple #21
0
func checkWrites(tb testing.TB, apiClient pfs.ApiClient, repositoryName string, commitID string) {
	var wg sync.WaitGroup
	defer wg.Wait()
	for i := 0; i < testSize; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			buffer := bytes.NewBuffer(nil)
			iErr := pfsutil.GetFile(apiClient, repositoryName, commitID,
				fmt.Sprintf("a/b/file%d", i), 0, pfsutil.GetAll, buffer)
			require.NoError(tb, iErr)
			require.Equal(tb, fmt.Sprintf("hello%d", i), buffer.String())
			buffer = bytes.NewBuffer(nil)
			iErr = pfsutil.GetFile(apiClient, repositoryName, commitID,
				fmt.Sprintf("a/c/file%d", i), 0, pfsutil.GetAll, buffer)
			require.NoError(tb, iErr)
			require.Equal(tb, fmt.Sprintf("hello%d", i), buffer.String())
		}()
	}
}
Exemple #22
0
func checkWrites(tb testing.TB, apiClient pfs.APIClient, repoName string, commitID string) {
	var wg sync.WaitGroup
	defer wg.Wait()
	for i := 0; i < testSize; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			buffer := bytes.NewBuffer(nil)
			iErr := pfsutil.GetFile(
				apiClient,
				repoName,
				commitID,
				fmt.Sprintf("a/b/file%d", i),
				0,
				math.MaxInt64,
				&pfs.Shard{Number: 0, Modulus: 1},
				buffer,
			)
			require.NoError(tb, iErr)
			require.Equal(tb, fmt.Sprintf("hello%d", i), buffer.String())

			buffer = bytes.NewBuffer(nil)
			iErr = pfsutil.GetFile(
				apiClient,
				repoName,
				commitID,
				fmt.Sprintf("a/c/file%d", i),
				0,
				math.MaxInt64,
				&pfs.Shard{Number: 0, Modulus: 1},
				buffer,
			)
			require.NoError(tb, iErr)
			require.Equal(tb, fmt.Sprintf("hello%d", i), buffer.String())

		}()
	}
}
Exemple #23
0
func TestGrep(t *testing.T) {
	t.Skip()
	t.Parallel()
	dataRepo := uniqueString("TestGrep.data")
	pfsClient := getPfsClient(t)
	require.NoError(t, pfsutil.CreateRepo(pfsClient, dataRepo))
	commit, err := pfsutil.StartCommit(pfsClient, dataRepo, "")
	require.NoError(t, err)
	for i := 0; i < 100; i++ {
		_, err = pfsutil.PutFile(pfsClient, dataRepo, commit.Id, fmt.Sprintf("file%d", i), 0, strings.NewReader("foo\nbar\nfizz\nbuzz\n"))
		require.NoError(t, err)
	}
	require.NoError(t, pfsutil.FinishCommit(pfsClient, dataRepo, commit.Id))
	ppsClient := getPpsClient(t)
	job1, err := ppsutil.CreateJob(
		ppsClient,
		"",
		[]string{"bash"},
		fmt.Sprintf("grep foo /pfs/%s/* >/pfs/out/foo", dataRepo),
		1,
		[]*pps.JobInput{{Commit: commit}},
		"",
	)
	require.NoError(t, err)
	job2, err := ppsutil.CreateJob(
		ppsClient,
		"",
		[]string{"bash"},
		fmt.Sprintf("grep foo /pfs/%s/* >/pfs/out/foo", dataRepo),
		4,
		[]*pps.JobInput{{Commit: commit}},
		"",
	)
	require.NoError(t, err)
	inspectJobRequest := &pps.InspectJobRequest{
		Job:         job1,
		BlockOutput: true,
		BlockState:  true,
	}
	job1Info, err := ppsClient.InspectJob(context.Background(), inspectJobRequest)
	require.NoError(t, err)
	inspectJobRequest.Job = job2
	job2Info, err := ppsClient.InspectJob(context.Background(), inspectJobRequest)
	require.NoError(t, err)
	repo1Info, err := pfsutil.InspectRepo(pfsClient, job1Info.OutputCommit.Repo.Name)
	require.NoError(t, err)
	repo2Info, err := pfsutil.InspectRepo(pfsClient, job2Info.OutputCommit.Repo.Name)
	require.NoError(t, err)
	require.Equal(t, repo1Info.SizeBytes, repo2Info.SizeBytes)
}
Exemple #24
0
func testBasic(t *testing.T, client Client) {
	pipelineRun := &pps.PipelineRun{
		Id:         "id",
		PipelineId: "pipeline_id",
	}
	require.NoError(t, client.CreatePipelineRun(pipelineRun))
	pipelineRunResponse, err := client.GetPipelineRun("id")
	require.NoError(t, err)
	require.Equal(t, pipelineRun, pipelineRunResponse)
	require.NoError(t, client.CreatePipelineRunStatus("id", pps.PipelineRunStatusType_PIPELINE_RUN_STATUS_TYPE_CREATED))
	pipelineRunStatusResponse, err := client.GetAllPipelineRunStatuses("id")
	require.NoError(t, err)
	require.Equal(t, pipelineRunStatusResponse[0].PipelineRunStatusType, pps.PipelineRunStatusType_PIPELINE_RUN_STATUS_TYPE_CREATED)
	require.NoError(t, client.CreatePipelineRunStatus("id", pps.PipelineRunStatusType_PIPELINE_RUN_STATUS_TYPE_SUCCESS))
	pipelineRunStatusResponse, err = client.GetAllPipelineRunStatuses("id")
	require.NoError(t, err)
	require.Equal(t, pipelineRunStatusResponse[0].PipelineRunStatusType, pps.PipelineRunStatusType_PIPELINE_RUN_STATUS_TYPE_SUCCESS)

	require.NoError(t, client.CreatePipelineRunContainers(&pps.PipelineRunContainer{PipelineRunId: "id", ContainerId: "container", Node: "node"}))
	containerIDs, err := client.GetPipelineRunContainers("id")
	require.NoError(t, err)
	require.Equal(t, []*pps.PipelineRunContainer{&pps.PipelineRunContainer{PipelineRunId: "id", ContainerId: "container", Node: "node"}}, containerIDs)
}
Exemple #25
0
func (s *serverGroup) run(t *testing.T) {
	var wg sync.WaitGroup
	defer wg.Wait()
	for i, server := range s.servers {
		wg.Add(1)
		go func(i int, server Server) {
			defer wg.Done()
			require.Equal(
				t,
				ErrCancelled,
				s.addresser.Register(s.cancel, fmt.Sprintf("server-%d", i+s.offset), fmt.Sprintf("address-%d", i+s.offset), server),
			)
		}(i, server)
	}
}
Exemple #26
0
func TestSharding(t *testing.T) {
	t.Parallel()
	repo := uniqueString("TestSharding")
	pachClient := getPachClient(t)
	err := pfsutil.CreateRepo(pachClient, repo)
	require.NoError(t, err)
	commit, err := pfsutil.StartCommit(pachClient, repo, "")
	require.NoError(t, err)
	var wg sync.WaitGroup
	for i := 0; i < NUMFILES; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			rand := rand.New(rand.NewSource(int64(i)))
			_, err = pfsutil.PutFile(pachClient, repo, commit.Id, fmt.Sprintf("file%d", i), 0, workload.NewReader(rand, KB))
			require.NoError(t, err)
		}()
	}
	wg.Wait()
	err = pfsutil.FinishCommit(pachClient, repo, commit.Id)
	require.NoError(t, err)
	wg = sync.WaitGroup{}
	for i := 0; i < NUMFILES; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			var buffer1Shard bytes.Buffer
			var buffer4Shard bytes.Buffer
			shard := &pfs.Shard{FileModulus: 1, BlockModulus: 1}
			err := pfsutil.GetFile(pachClient, repo, commit.Id,
				fmt.Sprintf("file%d", i), 0, 0, "", shard, &buffer1Shard)
			require.NoError(t, err)
			shard.BlockModulus = 4
			for blockNumber := uint64(0); blockNumber < 4; blockNumber++ {
				shard.BlockNumber = blockNumber
				err := pfsutil.GetFile(pachClient, repo, commit.Id,
					fmt.Sprintf("file%d", i), 0, 0, "", shard, &buffer4Shard)
				require.NoError(t, err)
			}
			require.Equal(t, buffer1Shard.Len(), buffer4Shard.Len())
		}()
	}
	wg.Wait()
}
Exemple #27
0
func TestGetAllFilePaths(t *testing.T) {
	files, err := getAllFilePaths("testdata/basic", "", []string{}, []string{"other", "root/ignore", "ignore-me.yml"})
	require.NoError(t, err)
	require.Equal(
		t,
		[]string{
			"root/foo-node.yml",
			"root/foo-service.yml",
			"root/include/bar-node.yml",
			"root/include/bar-service.yml",
			"root/include/bat-node.yml",
			"root/include/baz-node.yml",
			"root/include/baz-service.yml",
		},
		files,
	)
}
Exemple #28
0
func (c *cluster) Restart(index int) {
	address := c.addresses[index]
	c.cancels[address] = make(chan bool)
	internalAPIServer := server.NewInternalAPIServer(
		c.sharder,
		route.NewRouter(
			c.realSharder,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		),
		getDriver(c.tb, address),
	)
	c.internalServers[address] = internalAPIServer
	go func() {
		require.Equal(c.tb, c.realSharder.Register(c.cancels[address], address, c.internalServers[address]), shard.ErrCancelled)
	}()
}
Exemple #29
0
func (c *cluster) WaitForAvailability() {
	cancel := make(chan bool)
	time.AfterFunc(45*time.Second, func() { close(cancel) })
	var _shardToMasterAddress map[int]route.Address
	var _shardToReplicaAddress map[int]map[int]route.Address
	err := c.addresser.WatchShardToAddress(cancel, func(shardToMasterAddress map[int]route.Address, shardToReplicaAddress map[int]map[int]route.Address) (uint64, error) {
		_shardToMasterAddress = shardToMasterAddress
		_shardToReplicaAddress = shardToReplicaAddress
		if len(shardToMasterAddress) != testShardsPerServer*testNumServers {
			return 0, nil
		}
		if len(shardToReplicaAddress) != testShardsPerServer*testNumServers {
			return 0, nil
		}
		for _, addresses := range shardToReplicaAddress {
			if len(addresses) != testNumReplicas {
				return 0, nil
			}
		}
		for _, address := range shardToMasterAddress {
			if address.Backfilling {
				return 0, nil
			}
			if _, ok := c.rolers[address.Address]; !ok {
				return 0, nil
			}
		}
		for _, addresses := range shardToReplicaAddress {
			for _, address := range addresses {
				if address.Backfilling {
					return 0, nil
				}
				if _, ok := c.rolers[address.Address]; !ok {
					return 0, nil
				}
			}
		}
		return 0, fmt.Errorf("Complete")
	})
	require.Equal(c.tb, err.Error(), "Complete")
}
Exemple #30
0
func runMasterReplicaTest(t *testing.T, client discovery.Client) {
	sharder := NewSharder(testNumShards, testNumReplicas)
	addresser := NewDiscoveryAddresser(client, sharder, "TestMasterReplica")
	cancel := make(chan bool)
	go func() {
		require.Equal(t, ErrCancelled, addresser.AssignRoles(cancel))
	}()
	defer func() {
		close(cancel)
	}()
	serverGroup1 := NewServerGroup(t, addresser, testNumServers/2, 0)
	go serverGroup1.run(t)
	start := time.Now()
	for !serverGroup1.satisfied((testNumShards * (testNumReplicas + 1)) / (testNumServers / 2)) {
		time.Sleep(500 * time.Millisecond)
		if time.Since(start) > time.Second*time.Duration(30) {
			t.Fatal("test timed out")
		}
	}

	serverGroup2 := NewServerGroup(t, addresser, testNumServers/2, testNumServers/2)
	go serverGroup2.run(t)
	start = time.Now()
	for !serverGroup1.satisfied((testNumShards*(testNumReplicas+1))/testNumServers) ||
		!serverGroup2.satisfied((testNumShards*(testNumReplicas+1))/testNumServers) {
		time.Sleep(time.Second)
		if time.Since(start) > time.Second*time.Duration(60) {
			t.Fatal("test timed out")
		}
	}

	close(serverGroup1.cancel)
	for !serverGroup2.satisfied((testNumShards * (testNumReplicas + 1)) / (testNumServers / 2)) {
		time.Sleep(500 * time.Millisecond)
		if time.Since(start) > time.Second*time.Duration(60) {
			t.Fatal("test timed out")
		}
	}
}