func TestGetLocalResultFileLocations(t *testing.T) { testutils.SkipIfShort(t) err := testutils.DownloadTestDataArchive(t, TEST_DATA_STORAGE_PATH, TEST_DATA_DIR) assert.Nil(t, err) startTS := time.Date(2015, time.May, 5, 0, 0, 0, 0, time.UTC).Unix() endTS := time.Date(2015, time.May, 17, 23, 59, 59, 0, time.UTC).Unix() resultFiles, err := getLocalResultsFileLocations(startTS, endTS, filepath.Join(TEST_DATA_DIR, "nano-json-v1")) assert.Nil(t, err) // Read the expected list of files and compare them. content, err := ioutil.ReadFile("./testdata/local_ingest_files.txt") assert.Nil(t, err) lines := strings.Split(strings.TrimSpace(string(content)), "\n") sort.Strings(lines) resultNames := make([]string, len(resultFiles)) for idx, rf := range resultFiles { resultNames[idx] = rf.Name } sort.Strings(resultNames) assert.Equal(t, len(lines), len(resultNames)) assert.Equal(t, lines, resultNames) }
func TestFileSystemResultFileLocations(t *testing.T) { testutils.SkipIfShort(t) err := testutils.DownloadTestDataArchive(t, TEST_DATA_STORAGE_PATH, TEST_DATA_DIR) assert.Nil(t, err) defer testutils.RemoveAll(t, TEST_DATA_DIR) src, err := NewFileSystemSource("test-fs-source", TEST_DATA_DIR) assert.Nil(t, err) testSource(t, src) }
func TestCompareSources(t *testing.T) { testutils.SkipIfShort(t) gsSource, err := NewGoogleStorageSource("gs-test-src", TEST_GS_BUCKET, TEST_GS_DIR, http.DefaultClient) assert.Nil(t, err) err = testutils.DownloadTestDataArchive(t, TEST_DATA_STORAGE_PATH, TEST_DATA_DIR) assert.Nil(t, err) defer testutils.RemoveAll(t, TEST_DATA_DIR) fsSource, err := NewFileSystemSource("test-fs-source", TEST_DATA_DIR) assert.Nil(t, err) gsResults, err := gsSource.Poll(START_TIME, END_TIME) assert.Nil(t, err) fsResults, err := fsSource.Poll(START_TIME, END_TIME) assert.Nil(t, err) assert.Equal(t, len(gsResults), len(fsResults)) sort.Sort(rflSlice(gsResults)) sort.Sort(rflSlice(fsResults)) // Only compare a subset of all files to limit the runtime of this test. gsResults = gsResults[:10] fsResults = fsResults[:10] for idx, result := range gsResults { // Check if the MD5 from GS and the file system match. assert.Equal(t, result.MD5(), fsResults[idx].MD5()) // Check if the content is the same for both sources. file, err := result.Open() assert.Nil(t, err) c1, err := ioutil.ReadAll(file) assert.Nil(t, err) file, err = fsResults[idx].Open() c2, err := ioutil.ReadAll(file) assert.Nil(t, err) assert.Equal(t, string(c1), string(c2)) } }
func BenchmarkBigDataset(b *testing.B) { // Download the testdata and remove the testdata directory at the end. err := testutils.DownloadTestDataArchive(b, TEST_DATA_STORAGE_PATH, TEST_DATA_DIR) assert.Nil(b, err, "Unable to download testdata.") defer func() { util.LogErr(os.RemoveAll(TEST_DATA_DIR)) }() // Load the data fileInfos, err := ioutil.ReadDir(TEST_DATA_DIR) assert.Nil(b, err) results := make(chan interface{}, len(fileInfos)) var codec TestStructCodec counter := 0 for _, fi := range fileInfos { if strings.HasSuffix(fi.Name(), ".json") { go func(fName string) { f, err := os.Open(fName) if err != nil { glog.Fatalf("Unable to open file %s", fName) } content, err := ioutil.ReadAll(f) if err != nil { glog.Fatalf("Unable to read file %s", fName) } v, err := codec.Decode(content) if err != nil { glog.Fatalf("Unable to decode file %s", fName) } if _, ok := v.(*TestStruct); !ok { glog.Fatalln("Expected to get instance of TestStruct") } // Store the filepath in this field to use as cache key. ts := v.(*TestStruct) ts.PixelDiffFilePath = fName results <- ts }(filepath.Join("./benchdata", fi.Name())) counter++ } } groundTruth := make(map[string]interface{}, counter) cache := NewRedisLRUCache("localhost:6379", 1, "di", TestStructCodec(0)) rlruCache := cache.(*RedisLRUCache) rlruCache.Purge() for i := 0; i < counter; i++ { ret := <-results ts := ret.(*TestStruct) groundTruth[ts.PixelDiffFilePath] = ret } glog.Infof("Done importing %d files. Starting bench.", len(groundTruth)) b.ResetTimer() for k, v := range groundTruth { cache.Add(k, v) } assert.Equal(b, len(groundTruth), cache.Len()) counter = 0 for k, v := range groundTruth { found, ok := cache.Get(k) assert.True(b, ok) assert.Equal(b, v, found) counter++ // if (counter % 1000) == 0 { // glog.Infof("Checked %d records.", counter) // } } b.StopTimer() // Cleanup code that should not be timed but deserves to be tested. rlruCache.Purge() assert.Equal(b, 0, cache.Len()) }