예제 #1
0
func TestNoDelimiter(t *testing.T) {

	if testing.Short() {
		t.Skip("Skipped because of short mode")
	}
	testFuse(t, func(c client.APIClient, mountpoint string) {
		repo := "test"
		name := "foo.bin"
		require.NoError(t, c.CreateRepo(repo))
		commit1, err := c.StartCommit(repo, "", "")
		require.NoError(t, err)

		rawMessage := "Some\ncontent\nthat\nshouldnt\nbe\nline\ndelimited.\n"
		filePath := filepath.Join(mountpoint, repo, commit1.ID, name)

		// Write a big blob that would normally not fit in a block
		var expectedOutputA []byte
		for !(len(expectedOutputA) > 9*1024*1024) {
			expectedOutputA = append(expectedOutputA, []byte(rawMessage)...)
		}
		require.NoError(t, ioutil.WriteFile(filePath, expectedOutputA, 0644))

		// Write another big block
		var expectedOutputB []byte
		for !(len(expectedOutputB) > 10*1024*1024) {
			expectedOutputB = append(expectedOutputB, []byte(rawMessage)...)
		}
		require.NoError(t, ioutil.WriteFile("/tmp/b", expectedOutputB, 0644))
		stdin := strings.NewReader(fmt.Sprintf("cat /tmp/b >>%s", filePath))
		require.NoError(t, pkgexec.RunStdin(stdin, "sh"))

		// Finish the commit so I can read the data
		require.NoError(t, c.FinishCommit(repo, commit1.ID))

		// Make sure all the content is there
		var buffer bytes.Buffer
		require.NoError(t, c.GetFile(repo, commit1.ID, name, 0, 0, "", nil, &buffer))
		require.Equal(t, len(expectedOutputA)+len(expectedOutputB), buffer.Len())
		require.Equal(t, string(append(expectedOutputA, expectedOutputB...)), buffer.String())

		// Now verify that each block only contains objects of the size we've written
		bigModulus := 10 // Make it big to make it less likely that I return both blocks together
		blockLengths := []interface{}{len(expectedOutputA), len(expectedOutputB)}
		for b := 0; b < bigModulus; b++ {
			blockFilter := &pfsclient.Shard{
				BlockNumber:  uint64(b),
				BlockModulus: uint64(bigModulus),
			}

			buffer.Reset()
			require.NoError(t, c.GetFile(repo, commit1.ID, name, 0, 0, "", blockFilter, &buffer))

			// If any single block returns content of size equal to the total, we
			// got a block collision and we're not testing anything
			require.NotEqual(t, len(expectedOutputA)+len(expectedOutputB), buffer.Len())
			if buffer.Len() == 0 {
				continue
			}
			require.EqualOneOf(t, blockLengths, buffer.Len())
		}
	})
}
예제 #2
0
func TestDelimitJSON(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipped because of short mode")
	}
	testFuse(t, func(c client.APIClient, mountpoint string) {
		repo := "abba"
		require.NoError(t, c.CreateRepo(repo))
		commit, err := c.StartCommit(repo, "", "")
		require.NoError(t, err)
		var expectedOutput []byte
		rawMessage := `{
		"level":"debug",
		"timestamp":"345",
		"message":{
			"thing":"foo"
		},
		"timing":[1,3,34,6,7]
	}`
		for !(len(expectedOutput) > 9*1024*1024) {
			expectedOutput = append(expectedOutput, []byte(rawMessage)...)
		}
		filePath := filepath.Join(mountpoint, repo, commit.ID, "foo.json")
		require.NoError(t, ioutil.WriteFile(filePath, expectedOutput, 0644))
		require.NoError(t, c.FinishCommit(repo, commit.ID))
		// Make sure all the content is there
		var buffer bytes.Buffer
		require.NoError(t, c.GetFile(repo, commit.ID, "foo.json", 0, 0, "", nil, &buffer))
		require.Equal(t, len(expectedOutput), buffer.Len())
		require.Equal(t, string(expectedOutput), buffer.String())

		// Now verify that each block contains only valid JSON objects
		bigModulus := 10 // Make it big to make it less likely that I return both blocks together
		for b := 0; b < bigModulus; b++ {
			blockFilter := &pfsclient.Shard{
				BlockNumber:  uint64(b),
				BlockModulus: uint64(bigModulus),
			}

			buffer.Reset()
			require.NoError(t, c.GetFile(repo, commit.ID, "foo.json", 0, 0, "", blockFilter, &buffer))

			// If any single block returns content of size equal to the total, we
			// got a block collision and we're not testing anything
			require.NotEqual(t, buffer.Len(), len(expectedOutput))

			var value json.RawMessage
			decoder := json.NewDecoder(&buffer)
			for {
				err = decoder.Decode(&value)
				if err != nil {
					if err == io.EOF {
						break
					} else {
						require.NoError(t, err)
					}
				}
				require.Equal(t, rawMessage, string(value))
			}
		}
	})
}