func TestNoDelimiter(t *testing.T) {
if testing.Short() {
t.Skip("Skipped because of short mode")
}
testFuse(t, func(c client.APIClient, mountpoint string) {
repo := "test"
name := "foo.bin"
require.NoError(t, c.CreateRepo(repo))
commit1, err := c.StartCommit(repo, "", "")
require.NoError(t, err)
rawMessage := "Some\ncontent\nthat\nshouldnt\nbe\nline\ndelimited.\n"
filePath := filepath.Join(mountpoint, repo, commit1.ID, name)
// Write a big blob that would normally not fit in a block
var expectedOutputA []byte
for !(len(expectedOutputA) > 9*1024*1024) {
expectedOutputA = append(expectedOutputA, []byte(rawMessage)...)
}
require.NoError(t, ioutil.WriteFile(filePath, expectedOutputA, 0644))
// Write another big block
var expectedOutputB []byte
for !(len(expectedOutputB) > 10*1024*1024) {
expectedOutputB = append(expectedOutputB, []byte(rawMessage)...)
}
require.NoError(t, ioutil.WriteFile("/tmp/b", expectedOutputB, 0644))
stdin := strings.NewReader(fmt.Sprintf("cat /tmp/b >>%s", filePath))
require.NoError(t, pkgexec.RunStdin(stdin, "sh"))
// Finish the commit so I can read the data
require.NoError(t, c.FinishCommit(repo, commit1.ID))
// Make sure all the content is there
var buffer bytes.Buffer
require.NoError(t, c.GetFile(repo, commit1.ID, name, 0, 0, "", nil, &buffer))
require.Equal(t, len(expectedOutputA)+len(expectedOutputB), buffer.Len())
require.Equal(t, string(append(expectedOutputA, expectedOutputB...)), buffer.String())
// Now verify that each block only contains objects of the size we've written
bigModulus := 10 // Make it big to make it less likely that I return both blocks together
blockLengths := []interface{}{len(expectedOutputA), len(expectedOutputB)}
for b := 0; b < bigModulus; b++ {
blockFilter := &pfsclient.Shard{
BlockNumber: uint64(b),
BlockModulus: uint64(bigModulus),
}
buffer.Reset()
require.NoError(t, c.GetFile(repo, commit1.ID, name, 0, 0, "", blockFilter, &buffer))
// If any single block returns content of size equal to the total, we
// got a block collision and we're not testing anything
require.NotEqual(t, len(expectedOutputA)+len(expectedOutputB), buffer.Len())
if buffer.Len() == 0 {
continue
}
require.EqualOneOf(t, blockLengths, buffer.Len())
}
})
}
func TestDelimitJSON(t *testing.T) {
if testing.Short() {
t.Skip("Skipped because of short mode")
}
testFuse(t, func(c client.APIClient, mountpoint string) {
repo := "abba"
require.NoError(t, c.CreateRepo(repo))
commit, err := c.StartCommit(repo, "", "")
require.NoError(t, err)
var expectedOutput []byte
rawMessage := `{
"level":"debug",
"timestamp":"345",
"message":{
"thing":"foo"
},
"timing":[1,3,34,6,7]
}`
for !(len(expectedOutput) > 9*1024*1024) {
expectedOutput = append(expectedOutput, []byte(rawMessage)...)
}
filePath := filepath.Join(mountpoint, repo, commit.ID, "foo.json")
require.NoError(t, ioutil.WriteFile(filePath, expectedOutput, 0644))
require.NoError(t, c.FinishCommit(repo, commit.ID))
// Make sure all the content is there
var buffer bytes.Buffer
require.NoError(t, c.GetFile(repo, commit.ID, "foo.json", 0, 0, "", nil, &buffer))
require.Equal(t, len(expectedOutput), buffer.Len())
require.Equal(t, string(expectedOutput), buffer.String())
// Now verify that each block contains only valid JSON objects
bigModulus := 10 // Make it big to make it less likely that I return both blocks together
for b := 0; b < bigModulus; b++ {
blockFilter := &pfsclient.Shard{
BlockNumber: uint64(b),
BlockModulus: uint64(bigModulus),
}
buffer.Reset()
require.NoError(t, c.GetFile(repo, commit.ID, "foo.json", 0, 0, "", blockFilter, &buffer))
// If any single block returns content of size equal to the total, we
// got a block collision and we're not testing anything
require.NotEqual(t, buffer.Len(), len(expectedOutput))
var value json.RawMessage
decoder := json.NewDecoder(&buffer)
for {
err = decoder.Decode(&value)
if err != nil {
if err == io.EOF {
break
} else {
require.NoError(t, err)
}
}
require.Equal(t, rawMessage, string(value))
}
}
})
}
