func TestRawResponseData(t *testing.T) {
// nodeGetHandler is defined in dpn_rest_client_test.go
testServer := httptest.NewServer(http.HandlerFunc(nodeGetHandler))
defer testServer.Close()
// configFile is defined in dpn_rest_client_test.go
config, err := models.LoadConfigFile(configFile)
require.Nil(t, err)
client, err := network.NewDPNRestClient(
testServer.URL,
"",
"",
config.DPN.LocalNode,
config.DPN)
if err != nil {
t.Error(err)
return
}
resp := client.NodeGet("luna")
// Should be able to call repeatedly without error.
// Incorrect implementation would try to read from
// closed network socket.
for i := 0; i < 3; i++ {
bytes, err := resp.RawResponseData()
assert.NotNil(t, bytes)
assert.NotEmpty(t, bytes)
assert.Nil(t, err)
}
}
func main() {
pathToConfigFile := parseCommandLine()
config, err := models.LoadConfigFile(pathToConfigFile)
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
_context := context.NewContext(config)
_context.MessageLog.Info("Connecting to NSQLookupd at %s", _context.Config.NsqLookupd)
_context.MessageLog.Info("NSQDHttpAddress is %s", _context.Config.NsqdHttpAddress)
consumer, err := apt_workers.CreateNsqConsumer(_context.Config, &_context.Config.DPN.DPNIngestStoreWorker)
if err != nil {
_context.MessageLog.Fatalf(err.Error())
}
storer, err := workers.NewDPNIngestStorer(_context)
if err != nil {
_context.MessageLog.Error(err.Error())
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
_context.MessageLog.Info("dpn_ingest_store started")
consumer.AddHandler(storer)
consumer.ConnectToNSQLookupd(_context.Config.NsqLookupd)
// This reader blocks until we get an interrupt, so our program does not exit.
<-consumer.StopChan
}
func TestEnsurePharosConfig(t *testing.T) {
configFile := filepath.Join("config", "test.json")
config, err := models.LoadConfigFile(configFile)
require.Nil(t, err)
url := config.PharosURL
config.PharosURL = ""
err = config.EnsurePharosConfig()
assert.Equal(t, "PharosUrl is missing from config file", err.Error())
config.PharosURL = url
apiUser := os.Getenv("PHAROS_API_USER")
os.Setenv("PHAROS_API_USER", "")
err = config.EnsurePharosConfig()
assert.Equal(t, "Environment variable PHAROS_API_USER is not set", err.Error())
os.Setenv("PHAROS_API_USER", "Bogus value set by config_test.go")
apiKey := os.Getenv("PHAROS_API_KEY")
os.Setenv("PHAROS_API_KEY", "")
err = config.EnsurePharosConfig()
assert.Equal(t, "Environment variable PHAROS_API_KEY is not set", err.Error())
os.Setenv("PHAROS_API_USER", apiUser)
os.Setenv("PHAROS_API_KEY", apiKey)
}
// GetContext returns a context object initialized with the specified
// config file. Param configFile should be the name of a JSON config
// file in the exchange/config directory. For example, "test.json"
// or "integration.json".
func GetContext(configFile string) (*context.Context, error) {
configPath := filepath.Join("config", configFile)
config, err := models.LoadConfigFile(configPath)
if err != nil {
return nil, err
}
config.ExpandFilePaths()
return context.NewContext(config), nil
}
func TestGetRemoteClient(t *testing.T) {
if runRestTests(t) == false {
return
}
config, err := apt_models.LoadConfigFile(configFile)
require.Nil(t, err)
client := getClient(t)
nodes := []string{"chron", "hathi", "sdr", "tdr"}
for _, node := range nodes {
_, err := client.GetRemoteClient(node, config.DPN)
assert.Nil(t, err)
}
}
func getPostTestClient(t *testing.T) *network.DPNRestClient {
// If you want to debug, change ioutil.Discard to os.Stdout
// to see log output from the client.
config, err := apt_models.LoadConfigFile(filepath.Join("config", "integration.json"))
require.Nil(t, err)
client, err := network.NewDPNRestClient(
config.DPN.RestClient.LocalServiceURL,
config.DPN.RestClient.LocalAPIRoot,
config.DPN.RestClient.LocalAuthToken,
config.DPN.LocalNode,
config.DPN)
require.Nil(t, err)
return client
}
func main() {
pathToConfigFile := parseCommandLine()
config, err := models.LoadConfigFile(pathToConfigFile)
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
_context := context.NewContext(config)
dpnSync, err := workers.NewDPNSync(_context)
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
dpnSync.Run()
}
func createBagBuilder(t *testing.T) (builder *util.BagBuilder) {
obj := intelObj(t)
configFile := filepath.Join("config", "test.json")
config, err := models.LoadConfigFile(configFile)
require.Nil(t, err)
builder, err = util.NewBagBuilder(testBagPath(), obj, config.DPN.DefaultMetadata)
if err != nil {
tearDown()
t.Errorf("Could not create bag builder: %s", err.Error())
return nil
}
builder.Bag.Save()
return builder
}
func TestLoad(t *testing.T) {
configFile := filepath.Join("config", "test.json")
config, err := models.LoadConfigFile(configFile)
require.Nil(t, err)
// Spot check a few settings.
assert.Equal(t, 60, config.MaxDaysSinceFixityCheck)
assert.Equal(t, "http://localhost:3000", config.PharosURL)
assert.Equal(t, "10s", config.FetchWorker.HeartbeatInterval)
assert.Equal(t, 18, len(config.ReceivingBuckets))
assert.Equal(t, configFile, config.ActiveConfig)
assert.Equal(t, 24, config.BucketReaderCacheHours)
assert.Equal(t, "api-v2", config.DPN.DPNAPIVersion)
assert.Equal(t, "us-east-1", config.DPN.DPNGlacierRegion)
}
func runRestTests(t *testing.T) bool {
config, err := apt_models.LoadConfigFile(configFile)
require.Nil(t, err)
_, err = http.Get(config.DPN.RestClient.LocalServiceURL)
if err != nil {
if skipRestMessagePrinted == false {
skipRestMessagePrinted = true
fmt.Printf("Skipping DPN REST integration tests: "+
"DPN REST server is not running at %s\n",
config.DPN.RestClient.LocalServiceURL)
}
return false
}
return true
}
// See printUsage for a description.
func main() {
pathToConfigFile := parseCommandLine()
config, err := models.LoadConfigFile(pathToConfigFile)
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
_context := context.NewContext(config)
_context.MessageLog.Info("apt_volume_service started")
volumeService := service.NewVolumeService(
_context.Config.VolumeServicePort,
_context.MessageLog)
volumeService.Serve()
}
func main() {
pathToConfigFile, hours := parseCommandLine()
config, err := models.LoadConfigFile(pathToConfigFile)
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
_context := context.NewContext(config)
dpnQueue, err := workers.NewDPNQueue(_context, hours)
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
dpnQueue.Run()
}
func getClient(t *testing.T) *network.DPNRestClient {
// If you want to debug, change ioutil.Discard to os.Stdout
// to see log output from the client.
config, err := apt_models.LoadConfigFile(configFile)
require.Nil(t, err)
client, err := network.NewDPNRestClient(
config.DPN.RestClient.LocalServiceURL,
config.DPN.RestClient.LocalAPIRoot,
config.DPN.RestClient.LocalAuthToken,
config.DPN.LocalNode,
config.DPN)
if err != nil {
t.Errorf("Error constructing DPN REST client: %v", err)
}
return client
}
func TestBuildUrl(t *testing.T) {
config, err := apt_models.LoadConfigFile(configFile)
require.Nil(t, err)
client := getClient(t)
require.NotNil(t, client)
relativeUrl := "/api-v1/popeye/olive/oyl/"
expectedUrl := config.DPN.RestClient.LocalServiceURL + relativeUrl
if client.BuildUrl(relativeUrl, nil) != expectedUrl {
t.Errorf("BuildUrl returned '%s', expected '%s'",
client.BuildUrl(relativeUrl, nil), expectedUrl)
}
params := url.Values{}
params.Set("color", "blue")
params.Set("material", "cotton")
params.Set("size", "extra medium")
actualUrl := client.BuildUrl(relativeUrl, params)
expectedUrl = expectedUrl + "?color=blue&material=cotton&size=extra+medium"
assert.Equal(t, expectedUrl, actualUrl)
}
// test_push_to_dpn is for integration testing only.
// It creates a few WorkItems in Pharos asking that
// a handful of bags be pushed to DPN. Subsequent
// integration tests (such as dpn_queue) depend on
// the WorkItems created by this test.
func TestPushToDPN(t *testing.T) {
if !testutil.ShouldRunIntegrationTests() {
t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
}
configFile := filepath.Join("config", "integration.json")
config, err := models.LoadConfigFile(configFile)
require.Nil(t, err)
_context := context.NewContext(config)
for _, s3Key := range testutil.INTEGRATION_GOOD_BAGS[0:7] {
identifier := strings.Replace(s3Key, "aptrust.receiving.test.", "", 1)
identifier = strings.Replace(identifier, ".tar", "", 1)
resp := _context.PharosClient.IntellectualObjectPushToDPN(identifier)
workItem := resp.WorkItem()
require.Nil(t, resp.Error)
require.NotNil(t, workItem)
_context.MessageLog.Info("Created DPN work item #%d for %s",
workItem.Id, workItem.ObjectIdentifier)
}
}
func TestFetchResults(t *testing.T) {
if !testutil.ShouldRunIntegrationTests() {
t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
}
// Load config
configFile := filepath.Join("config", "integration.json")
config, err := models.LoadConfigFile(configFile)
require.Nil(t, err)
config.ExpandFilePaths()
// Find the log file that apt_fetch created when it was running
// with the "config/integration.json" config options. We'll read
// that file.
pathToJsonLog := filepath.Join(config.LogDirectory, "apt_fetch.json")
for _, bagName := range testutil.INTEGRATION_GOOD_BAGS {
ingestManifest, err := testutil.FindIngestManifestInLog(pathToJsonLog, bagName)
assert.Nil(t, err)
if err != nil {
continue
}
fetcherTestCommon(t, bagName, ingestManifest)
// TODO: Test WorkItem (stage, status, etc.) below
fetcherTestGoodBagResult(t, bagName, ingestManifest)
if bagName == "aptrust.receiving.test.test.edu/example.edu.tagsample_good.tar" {
fetcherTestSpecifics(t, ingestManifest)
}
}
for _, bagName := range testutil.INTEGRATION_BAD_BAGS {
ingestManifest, err := testutil.FindIngestManifestInLog(pathToJsonLog, bagName)
assert.Nil(t, err)
if err != nil {
continue
}
fetcherTestCommon(t, bagName, ingestManifest)
// TODO: Test WorkItem (stage, status, etc.) below
fetcherTestBadBagResult(t, bagName, ingestManifest)
}
}
func TestEnsureDPNConfig(t *testing.T) {
configFile := filepath.Join("config", "test.json")
config, err := models.LoadConfigFile(configFile)
require.Nil(t, err)
assert.NotEmpty(t, config.DPN.DefaultMetadata.BagItVersion)
assert.NotEmpty(t, config.DPN.DefaultMetadata.BagItEncoding)
assert.NotEmpty(t, config.DPN.DefaultMetadata.IngestNodeName)
assert.NotEmpty(t, config.DPN.DefaultMetadata.IngestNodeAddress)
assert.NotEmpty(t, config.DPN.DefaultMetadata.IngestNodeContactName)
assert.NotEmpty(t, config.DPN.DefaultMetadata.IngestNodeContactEmail)
assert.NotEmpty(t, config.DPN.RestClient.LocalServiceURL)
assert.NotEmpty(t, config.DPN.RestClient.LocalAPIRoot)
assert.NotEmpty(t, config.DPN.RestClient.LocalAuthToken)
assert.EqualValues(t, 3, config.DPN.DPNValidationWorker.MaxAttempts)
assert.EqualValues(t, 3, config.DPN.DPNCopyWorker.MaxAttempts)
assert.Equal(t, "chron_token", config.DPN.RemoteNodeAdminTokensForTesting["chron"])
assert.Equal(t, "aptrust_token", config.DPN.RemoteNodeTokens["chron"])
assert.Equal(t, "http://localhost:3002", config.DPN.RemoteNodeURLs["chron"])
}
// Returns two Stats objects: the expected stats, from our test data dir,
// and the actual stats, from the JSON file that the bucket reader dumped
// out last time it ran
func getBucketReaderOutputs(t *testing.T) (expected *stats.APTBucketReaderStats, actual *stats.APTBucketReaderStats) {
configFile := filepath.Join("config", "integration.json")
appConfig, err := models.LoadConfigFile(configFile)
require.Nil(t, err)
// This JSON file is part of our code repo.
// It contains the output we expect from a success test run.
pathToExpectedJson, err := fileutil.RelativeToAbsPath(filepath.Join("testdata",
"integration_results", "bucket_reader_stats.json"))
require.Nil(t, err)
expected, err = stats.APTBucketReaderStatsLoadFromFile(pathToExpectedJson)
require.Nil(t, err)
// This JSON file is recreated every time we run apt_bucket_reader
// as part of the integration tests. It contains the output of the
// actual test run.
pathToActualJson, err := fileutil.ExpandTilde(filepath.Join(appConfig.LogDirectory, "bucket_reader_stats.json"))
require.Nil(t, err)
actual, err = stats.APTBucketReaderStatsLoadFromFile(pathToActualJson)
require.Nil(t, err)
return expected, actual
}
func TestNewContext(t *testing.T) {
configFile := filepath.Join("config", "test.json")
appConfig, err := models.LoadConfigFile(configFile)
require.Nil(t, err)
// In some tests we want to log to STDERR, but in this case, if it
// happens to be turned on, it just creates useless, annoying output.
appConfig.LogToStderr = false
_context := context.NewContext(appConfig)
require.NotNil(t, _context)
expectedPathToLogFile := filepath.Join(_context.Config.AbsLogDirectory(), path.Base(os.Args[0])+".log")
expectedPathToJsonLog := filepath.Join(_context.Config.AbsLogDirectory(), path.Base(os.Args[0])+".json")
assert.NotNil(t, _context.Config)
assert.NotNil(t, _context.NSQClient)
assert.NotNil(t, _context.PharosClient)
assert.NotNil(t, _context.MessageLog)
assert.NotNil(t, _context.JsonLog)
assert.Equal(t, expectedPathToLogFile, _context.PathToLogFile())
assert.Equal(t, expectedPathToJsonLog, _context.PathToJsonLog())
assert.Equal(t, int64(0), _context.Succeeded())
assert.Equal(t, int64(0), _context.Failed())
assert.NotPanics(t, func() { _context.MessageLog.Info("Test INFO log message") })
assert.NotPanics(t, func() { _context.MessageLog.Debug("Test DEBUG log message") })
assert.NotPanics(t, func() { _context.JsonLog.Println(`{"message": "Test JSON log message"}`) })
// Cleanup, but only if context was successfully created
if _context != nil && _context.PathToLogFile() != "" {
os.Remove(_context.PathToLogFile())
}
if _context != nil && _context.PathToJsonLog() != "" {
os.Remove(_context.PathToJsonLog())
}
}
func TestStoreResults(t *testing.T) {
if !testutil.ShouldRunIntegrationTests() {
t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
}
// Load config
configFile := filepath.Join("config", "integration.json")
config, err := models.LoadConfigFile(configFile)
require.Nil(t, err)
config.ExpandFilePaths()
// Find the log file that apt_store created when it was running
// with the "config/integration.json" config options. We'll read
// that file.
pathToJsonLog := filepath.Join(config.LogDirectory, "apt_store.json")
for _, bagName := range testutil.INTEGRATION_GOOD_BAGS {
ingestManifest, err := testutil.FindIngestManifestInLog(pathToJsonLog, bagName)
assert.Nil(t, err)
if err != nil {
continue
}
// TODO: Test WorkItem (stage, status, etc.) below
storeTestCommon(t, bagName, ingestManifest, config)
}
}
// apt_record records IntellectualObjects, GenericFiles, PremisEvents
// and Checksums in Pharos. Those items will have already been stored
// in S3/Glacier by apt_store. This is the third and last step in the
// ingest process.
func main() {
pathToConfigFile := parseCommandLine()
config, err := models.LoadConfigFile(pathToConfigFile)
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
_context := context.NewContext(config)
_context.MessageLog.Info("Connecting to NSQLookupd at %s", _context.Config.NsqLookupd)
_context.MessageLog.Info("NSQDHttpAddress is %s", _context.Config.NsqdHttpAddress)
consumer, err := workers.CreateNsqConsumer(_context.Config, &_context.Config.RecordWorker)
if err != nil {
_context.MessageLog.Fatalf(err.Error())
}
_context.MessageLog.Info("apt_record started with config %s", _context.Config.ActiveConfig)
_context.MessageLog.Info("DeleteOnSuccess is set to %t", _context.Config.DeleteOnSuccess)
recorder := workers.NewAPTRecorder(_context)
consumer.AddHandler(recorder)
consumer.ConnectToNSQLookupd(_context.Config.NsqLookupd)
// This reader blocks until we get an interrupt, so our program does not exit.
<-consumer.StopChan
}
func loadConfig(t *testing.T) *apt_models.Config {
configFile := filepath.Join("config", "integration.json")
config, err := apt_models.LoadConfigFile(configFile)
require.Nil(t, err)
return config
}