Ejemplo n.º 1
0
func TestGetLocalResultFileLocations(t *testing.T) {
	testutils.SkipIfShort(t)

	err := testutils.DownloadTestDataArchive(t, TEST_DATA_STORAGE_PATH, TEST_DATA_DIR)
	assert.Nil(t, err)

	startTS := time.Date(2015, time.May, 5, 0, 0, 0, 0, time.UTC).Unix()
	endTS := time.Date(2015, time.May, 17, 23, 59, 59, 0, time.UTC).Unix()

	resultFiles, err := getLocalResultsFileLocations(startTS, endTS, filepath.Join(TEST_DATA_DIR, "nano-json-v1"))
	assert.Nil(t, err)

	// Read the expected list of files and compare them.
	content, err := ioutil.ReadFile("./testdata/local_ingest_files.txt")
	assert.Nil(t, err)
	lines := strings.Split(strings.TrimSpace(string(content)), "\n")
	sort.Strings(lines)

	resultNames := make([]string, len(resultFiles))
	for idx, rf := range resultFiles {
		resultNames[idx] = rf.Name
	}
	sort.Strings(resultNames)
	assert.Equal(t, len(lines), len(resultNames))
	assert.Equal(t, lines, resultNames)
}
Ejemplo n.º 2
0
func TestFileSystemResultFileLocations(t *testing.T) {
	testutils.SkipIfShort(t)

	err := testutils.DownloadTestDataArchive(t, TEST_DATA_STORAGE_PATH, TEST_DATA_DIR)
	assert.Nil(t, err)
	defer testutils.RemoveAll(t, TEST_DATA_DIR)

	src, err := NewFileSystemSource("test-fs-source", TEST_DATA_DIR)
	assert.Nil(t, err)
	testSource(t, src)
}
Ejemplo n.º 3
0
func TestCompareSources(t *testing.T) {
	testutils.SkipIfShort(t)

	gsSource, err := NewGoogleStorageSource("gs-test-src", TEST_GS_BUCKET, TEST_GS_DIR, http.DefaultClient)
	assert.Nil(t, err)

	err = testutils.DownloadTestDataArchive(t, TEST_DATA_STORAGE_PATH, TEST_DATA_DIR)
	assert.Nil(t, err)
	defer testutils.RemoveAll(t, TEST_DATA_DIR)

	fsSource, err := NewFileSystemSource("test-fs-source", TEST_DATA_DIR)
	assert.Nil(t, err)

	gsResults, err := gsSource.Poll(START_TIME, END_TIME)
	assert.Nil(t, err)

	fsResults, err := fsSource.Poll(START_TIME, END_TIME)
	assert.Nil(t, err)

	assert.Equal(t, len(gsResults), len(fsResults))
	sort.Sort(rflSlice(gsResults))
	sort.Sort(rflSlice(fsResults))

	// Only compare a subset of all files to limit the runtime of this test.
	gsResults = gsResults[:10]
	fsResults = fsResults[:10]

	for idx, result := range gsResults {
		// Check if the MD5 from GS and the file system match.
		assert.Equal(t, result.MD5(), fsResults[idx].MD5())

		// Check if the content is the same for both sources.
		file, err := result.Open()
		assert.Nil(t, err)
		c1, err := ioutil.ReadAll(file)
		assert.Nil(t, err)
		file, err = fsResults[idx].Open()
		c2, err := ioutil.ReadAll(file)
		assert.Nil(t, err)
		assert.Equal(t, string(c1), string(c2))
	}
}
Ejemplo n.º 4
0
func BenchmarkBigDataset(b *testing.B) {
	// Download the testdata and remove the testdata directory at the end.
	err := testutils.DownloadTestDataArchive(b, TEST_DATA_STORAGE_PATH, TEST_DATA_DIR)
	assert.Nil(b, err, "Unable to download testdata.")
	defer func() {
		util.LogErr(os.RemoveAll(TEST_DATA_DIR))
	}()

	// Load the data
	fileInfos, err := ioutil.ReadDir(TEST_DATA_DIR)
	assert.Nil(b, err)

	results := make(chan interface{}, len(fileInfos))
	var codec TestStructCodec
	counter := 0
	for _, fi := range fileInfos {
		if strings.HasSuffix(fi.Name(), ".json") {
			go func(fName string) {
				f, err := os.Open(fName)
				if err != nil {
					glog.Fatalf("Unable to open file %s", fName)
				}

				content, err := ioutil.ReadAll(f)
				if err != nil {
					glog.Fatalf("Unable to read file %s", fName)
				}

				v, err := codec.Decode(content)
				if err != nil {
					glog.Fatalf("Unable to decode file %s", fName)
				}
				if _, ok := v.(*TestStruct); !ok {
					glog.Fatalln("Expected to get instance of TestStruct")
				}

				// Store the filepath in this field to use as cache key.
				ts := v.(*TestStruct)
				ts.PixelDiffFilePath = fName
				results <- ts
			}(filepath.Join("./benchdata", fi.Name()))
			counter++
		}
	}

	groundTruth := make(map[string]interface{}, counter)
	cache := NewRedisLRUCache("localhost:6379", 1, "di", TestStructCodec(0))
	rlruCache := cache.(*RedisLRUCache)
	rlruCache.Purge()

	for i := 0; i < counter; i++ {
		ret := <-results
		ts := ret.(*TestStruct)
		groundTruth[ts.PixelDiffFilePath] = ret
	}

	glog.Infof("Done importing %d files. Starting bench.", len(groundTruth))

	b.ResetTimer()
	for k, v := range groundTruth {
		cache.Add(k, v)
	}

	assert.Equal(b, len(groundTruth), cache.Len())
	counter = 0
	for k, v := range groundTruth {
		found, ok := cache.Get(k)
		assert.True(b, ok)
		assert.Equal(b, v, found)
		counter++
		// if (counter % 1000) == 0 {
		// 	glog.Infof("Checked %d records.", counter)
		// }
	}
	b.StopTimer()

	// Cleanup code that should not be timed but deserves to be tested.
	rlruCache.Purge()
	assert.Equal(b, 0, cache.Len())
}