func TestGetLocalResultFileLocations(t *testing.T) {
testutils.SkipIfShort(t)
err := testutils.DownloadTestDataArchive(t, TEST_DATA_STORAGE_PATH, TEST_DATA_DIR)
assert.Nil(t, err)
startTS := time.Date(2015, time.May, 5, 0, 0, 0, 0, time.UTC).Unix()
endTS := time.Date(2015, time.May, 17, 23, 59, 59, 0, time.UTC).Unix()
resultFiles, err := getLocalResultsFileLocations(startTS, endTS, filepath.Join(TEST_DATA_DIR, "nano-json-v1"))
assert.Nil(t, err)
// Read the expected list of files and compare them.
content, err := ioutil.ReadFile("./testdata/local_ingest_files.txt")
assert.Nil(t, err)
lines := strings.Split(strings.TrimSpace(string(content)), "\n")
sort.Strings(lines)
resultNames := make([]string, len(resultFiles))
for idx, rf := range resultFiles {
resultNames[idx] = rf.Name
}
sort.Strings(resultNames)
assert.Equal(t, len(lines), len(resultNames))
assert.Equal(t, lines, resultNames)
}
func TestFileSystemResultFileLocations(t *testing.T) {
testutils.SkipIfShort(t)
err := testutils.DownloadTestDataArchive(t, TEST_DATA_STORAGE_PATH, TEST_DATA_DIR)
assert.Nil(t, err)
defer testutils.RemoveAll(t, TEST_DATA_DIR)
src, err := NewFileSystemSource("test-fs-source", TEST_DATA_DIR)
assert.Nil(t, err)
testSource(t, src)
}
func TestCompareSources(t *testing.T) {
testutils.SkipIfShort(t)
gsSource, err := NewGoogleStorageSource("gs-test-src", TEST_GS_BUCKET, TEST_GS_DIR, http.DefaultClient)
assert.Nil(t, err)
err = testutils.DownloadTestDataArchive(t, TEST_DATA_STORAGE_PATH, TEST_DATA_DIR)
assert.Nil(t, err)
defer testutils.RemoveAll(t, TEST_DATA_DIR)
fsSource, err := NewFileSystemSource("test-fs-source", TEST_DATA_DIR)
assert.Nil(t, err)
gsResults, err := gsSource.Poll(START_TIME, END_TIME)
assert.Nil(t, err)
fsResults, err := fsSource.Poll(START_TIME, END_TIME)
assert.Nil(t, err)
assert.Equal(t, len(gsResults), len(fsResults))
sort.Sort(rflSlice(gsResults))
sort.Sort(rflSlice(fsResults))
// Only compare a subset of all files to limit the runtime of this test.
gsResults = gsResults[:10]
fsResults = fsResults[:10]
for idx, result := range gsResults {
// Check if the MD5 from GS and the file system match.
assert.Equal(t, result.MD5(), fsResults[idx].MD5())
// Check if the content is the same for both sources.
file, err := result.Open()
assert.Nil(t, err)
c1, err := ioutil.ReadAll(file)
assert.Nil(t, err)
file, err = fsResults[idx].Open()
c2, err := ioutil.ReadAll(file)
assert.Nil(t, err)
assert.Equal(t, string(c1), string(c2))
}
}
func BenchmarkBigDataset(b *testing.B) {
// Download the testdata and remove the testdata directory at the end.
err := testutils.DownloadTestDataArchive(b, TEST_DATA_STORAGE_PATH, TEST_DATA_DIR)
assert.Nil(b, err, "Unable to download testdata.")
defer func() {
util.LogErr(os.RemoveAll(TEST_DATA_DIR))
}()
// Load the data
fileInfos, err := ioutil.ReadDir(TEST_DATA_DIR)
assert.Nil(b, err)
results := make(chan interface{}, len(fileInfos))
var codec TestStructCodec
counter := 0
for _, fi := range fileInfos {
if strings.HasSuffix(fi.Name(), ".json") {
go func(fName string) {
f, err := os.Open(fName)
if err != nil {
glog.Fatalf("Unable to open file %s", fName)
}
content, err := ioutil.ReadAll(f)
if err != nil {
glog.Fatalf("Unable to read file %s", fName)
}
v, err := codec.Decode(content)
if err != nil {
glog.Fatalf("Unable to decode file %s", fName)
}
if _, ok := v.(*TestStruct); !ok {
glog.Fatalln("Expected to get instance of TestStruct")
}
// Store the filepath in this field to use as cache key.
ts := v.(*TestStruct)
ts.PixelDiffFilePath = fName
results <- ts
}(filepath.Join("./benchdata", fi.Name()))
counter++
}
}
groundTruth := make(map[string]interface{}, counter)
cache := NewRedisLRUCache("localhost:6379", 1, "di", TestStructCodec(0))
rlruCache := cache.(*RedisLRUCache)
rlruCache.Purge()
for i := 0; i < counter; i++ {
ret := <-results
ts := ret.(*TestStruct)
groundTruth[ts.PixelDiffFilePath] = ret
}
glog.Infof("Done importing %d files. Starting bench.", len(groundTruth))
b.ResetTimer()
for k, v := range groundTruth {
cache.Add(k, v)
}
assert.Equal(b, len(groundTruth), cache.Len())
counter = 0
for k, v := range groundTruth {
found, ok := cache.Get(k)
assert.True(b, ok)
assert.Equal(b, v, found)
counter++
// if (counter % 1000) == 0 {
// glog.Infof("Checked %d records.", counter)
// }
}
b.StopTimer()
// Cleanup code that should not be timed but deserves to be tested.
rlruCache.Purge()
assert.Equal(b, 0, cache.Len())
}
