Example #1
0
func TestFromCommit(t *testing.T) {
	t.Parallel()
	repo := uniqueString("TestFromCommit")
	pachClient := getPachClient(t)
	seed := time.Now().UnixNano()
	rand := rand.New(rand.NewSource(seed))
	err := pfsutil.CreateRepo(pachClient, repo)
	require.NoError(t, err)
	commit1, err := pfsutil.StartCommit(pachClient, repo, "")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pachClient, repo, commit1.Id, "file", 0, workload.NewReader(rand, KB))
	require.NoError(t, err)
	err = pfsutil.FinishCommit(pachClient, repo, commit1.Id)
	require.NoError(t, err)
	commit2, err := pfsutil.StartCommit(pachClient, repo, commit1.Id)
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pachClient, repo, commit2.Id, "file", 0, workload.NewReader(rand, KB))
	require.NoError(t, err)
	err = pfsutil.FinishCommit(pachClient, repo, commit2.Id)
	require.NoError(t, err)
	var buffer bytes.Buffer
	require.NoError(t, pfsutil.GetFile(pachClient, repo, commit2.Id, "file", 0, 0, commit1.Id, nil, &buffer))
	require.Equal(t, buffer.Len(), KB)
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pachClient, repo, commit2.Id, "file", 0, 0, "", nil, &buffer))
	require.Equal(t, buffer.Len(), 2*KB)
}
Example #2
0
func TestSharding(t *testing.T) {
	t.Parallel()
	repo := uniqueString("TestSharding")
	pfsClient := getPfsClient(t)
	err := pfsutil.CreateRepo(pfsClient, repo)
	require.NoError(t, err)
	commit, err := pfsutil.StartCommit(pfsClient, repo, "")
	require.NoError(t, err)
	var wg sync.WaitGroup
	for i := 0; i < NUMFILES; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			rand := rand.New(rand.NewSource(int64(i)))
			_, err = pfsutil.PutFile(pfsClient, repo, commit.Id, fmt.Sprintf("file%d", i), 0, workload.NewReader(rand, 1024*1024))
			require.NoError(t, err)
		}()
	}
	wg.Wait()
	err = pfsutil.FinishCommit(pfsClient, repo, commit.Id)
	require.NoError(t, err)
	wg = sync.WaitGroup{}
	for i := 0; i < NUMFILES; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			var buffer1Shard bytes.Buffer
			var buffer4Shard bytes.Buffer
			shard := &pfs.Shard{FileModulus: 1, BlockModulus: 1}
			err := pfsutil.GetFile(pfsClient, repo, commit.Id,
				fmt.Sprintf("file%d", i), 0, 0, shard, &buffer1Shard)
			require.NoError(t, err)
			shard.BlockModulus = 4
			for blockNumber := uint64(0); blockNumber < 4; blockNumber++ {
				shard.BlockNumber = blockNumber
				err := pfsutil.GetFile(pfsClient, repo, commit.Id,
					fmt.Sprintf("file%d", i), 0, 0, shard, &buffer4Shard)
				require.NoError(t, err)
			}
			require.Equal(t, buffer1Shard.Len(), buffer4Shard.Len())
		}()
	}
	wg.Wait()
}
Example #3
0
func TestBigWrite(t *testing.T) {
	t.Parallel()
	protolog.SetLevel(protolog.Level_LEVEL_DEBUG)
	repo := uniqueString("TestBigWrite")
	pfsClient := getPfsClient(t)
	err := pfsutil.CreateRepo(pfsClient, repo)
	require.NoError(t, err)
	commit, err := pfsutil.StartCommit(pfsClient, repo, "")
	require.NoError(t, err)
	rand := rand.New(rand.NewSource(5))
	_, err = pfsutil.PutFile(pfsClient, repo, commit.Id, "file", 0, workload.NewReader(rand, 10000))
	require.NoError(t, err)
	err = pfsutil.FinishCommit(pfsClient, repo, commit.Id)
	require.NoError(t, err)
	var buffer bytes.Buffer
	err = pfsutil.GetFile(pfsClient, repo, commit.Id, "file", 0, 0, nil, &buffer)
	require.NoError(t, err)
}
Example #4
0
func TestSharding(t *testing.T) {
	t.Parallel()
	repo := uniqueString("TestSharding")
	pfsClient := getPfsClient(t)
	err := pfsutil.CreateRepo(pfsClient, repo)
	require.NoError(t, err)
	commit, err := pfsutil.StartCommit(pfsClient, repo, "")
	require.NoError(t, err)
	var wg sync.WaitGroup
	for i := 0; i < NUMFILES; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			rand := rand.New(rand.NewSource(int64(i)))
			_, err = pfsutil.PutFile(pfsClient, repo, commit.Id, fmt.Sprintf("file%d", i), 0, workload.NewReader(rand, 128*1024*1024))
			require.NoError(t, err)
		}()
	}
	wg.Wait()
	err = pfsutil.FinishCommit(pfsClient, repo, commit.Id)
	require.NoError(t, err)
}