Exemple #1
0
func main() {
	pathToConfigFile := parseCommandLine()
	config, err := models.LoadConfigFile(pathToConfigFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	_context := context.NewContext(config)
	_context.MessageLog.Info("Connecting to NSQLookupd at %s", _context.Config.NsqLookupd)
	_context.MessageLog.Info("NSQDHttpAddress is %s", _context.Config.NsqdHttpAddress)
	consumer, err := apt_workers.CreateNsqConsumer(_context.Config, &_context.Config.DPN.DPNIngestStoreWorker)
	if err != nil {
		_context.MessageLog.Fatalf(err.Error())
	}
	storer, err := workers.NewDPNIngestStorer(_context)
	if err != nil {
		_context.MessageLog.Error(err.Error())
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	_context.MessageLog.Info("dpn_ingest_store started")
	consumer.AddHandler(storer)
	consumer.ConnectToNSQLookupd(_context.Config.NsqLookupd)

	// This reader blocks until we get an interrupt, so our program does not exit.
	<-consumer.StopChan
}
Exemple #2
0
// GetContext returns a context object initialized with the specified
// config file. Param configFile should be the name of a JSON config
// file in the exchange/config directory. For example, "test.json"
// or "integration.json".
func GetContext(configFile string) (*context.Context, error) {
	configPath := filepath.Join("config", configFile)
	config, err := models.LoadConfigFile(configPath)
	if err != nil {
		return nil, err
	}
	config.ExpandFilePaths()
	return context.NewContext(config), nil
}
func newDPNSync(t *testing.T) *workers.DPNSync {
	config := loadConfig(t)
	_context := context.NewContext(config)
	dpnSync, err := workers.NewDPNSync(_context)
	require.Nil(t, err)
	for namespace := range config.DPN.RemoteNodeTokens {
		require.NotNil(t, dpnSync.RemoteClients[namespace], namespace)
	}
	return dpnSync
}
Exemple #4
0
func main() {
	pathToConfigFile := parseCommandLine()
	config, err := models.LoadConfigFile(pathToConfigFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	_context := context.NewContext(config)
	dpnSync, err := workers.NewDPNSync(_context)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	dpnSync.Run()
}
// See printUsage for a description.
func main() {
	pathToConfigFile := parseCommandLine()
	config, err := models.LoadConfigFile(pathToConfigFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	_context := context.NewContext(config)
	_context.MessageLog.Info("apt_volume_service started")

	volumeService := service.NewVolumeService(
		_context.Config.VolumeServicePort,
		_context.MessageLog)
	volumeService.Serve()
}
Exemple #6
0
func main() {
	pathToConfigFile, hours := parseCommandLine()
	config, err := models.LoadConfigFile(pathToConfigFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	_context := context.NewContext(config)
	dpnQueue, err := workers.NewDPNQueue(_context, hours)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	dpnQueue.Run()
}
// test_push_to_dpn is for integration testing only.
// It creates a few WorkItems in Pharos asking that
// a handful of bags be pushed to DPN. Subsequent
// integration tests (such as dpn_queue) depend on
// the WorkItems created by this test.
func TestPushToDPN(t *testing.T) {
	if !testutil.ShouldRunIntegrationTests() {
		t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
	}
	configFile := filepath.Join("config", "integration.json")
	config, err := models.LoadConfigFile(configFile)
	require.Nil(t, err)
	_context := context.NewContext(config)
	for _, s3Key := range testutil.INTEGRATION_GOOD_BAGS[0:7] {
		identifier := strings.Replace(s3Key, "aptrust.receiving.test.", "", 1)
		identifier = strings.Replace(identifier, ".tar", "", 1)
		resp := _context.PharosClient.IntellectualObjectPushToDPN(identifier)
		workItem := resp.WorkItem()
		require.Nil(t, resp.Error)
		require.NotNil(t, workItem)
		_context.MessageLog.Info("Created DPN work item #%d for %s",
			workItem.Id, workItem.ObjectIdentifier)
	}
}
Exemple #8
0
func TestNewContext(t *testing.T) {
	configFile := filepath.Join("config", "test.json")
	appConfig, err := models.LoadConfigFile(configFile)
	require.Nil(t, err)

	// In some tests we want to log to STDERR, but in this case, if it
	// happens to be turned on, it just creates useless, annoying output.
	appConfig.LogToStderr = false

	_context := context.NewContext(appConfig)
	require.NotNil(t, _context)

	expectedPathToLogFile := filepath.Join(_context.Config.AbsLogDirectory(), path.Base(os.Args[0])+".log")
	expectedPathToJsonLog := filepath.Join(_context.Config.AbsLogDirectory(), path.Base(os.Args[0])+".json")

	assert.NotNil(t, _context.Config)
	assert.NotNil(t, _context.NSQClient)
	assert.NotNil(t, _context.PharosClient)
	assert.NotNil(t, _context.MessageLog)
	assert.NotNil(t, _context.JsonLog)
	assert.Equal(t, expectedPathToLogFile, _context.PathToLogFile())
	assert.Equal(t, expectedPathToJsonLog, _context.PathToJsonLog())
	assert.Equal(t, int64(0), _context.Succeeded())
	assert.Equal(t, int64(0), _context.Failed())

	assert.NotPanics(t, func() { _context.MessageLog.Info("Test INFO log message") })
	assert.NotPanics(t, func() { _context.MessageLog.Debug("Test DEBUG log message") })
	assert.NotPanics(t, func() { _context.JsonLog.Println(`{"message": "Test JSON log message"}`) })

	// Cleanup, but only if context was successfully created
	if _context != nil && _context.PathToLogFile() != "" {
		os.Remove(_context.PathToLogFile())
	}
	if _context != nil && _context.PathToJsonLog() != "" {
		os.Remove(_context.PathToJsonLog())
	}
}
Exemple #9
0
// apt_record records IntellectualObjects, GenericFiles, PremisEvents
// and Checksums in Pharos. Those items will have already been stored
// in S3/Glacier by apt_store. This is the third and last step in the
// ingest process.
func main() {
	pathToConfigFile := parseCommandLine()
	config, err := models.LoadConfigFile(pathToConfigFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	_context := context.NewContext(config)
	_context.MessageLog.Info("Connecting to NSQLookupd at %s", _context.Config.NsqLookupd)
	_context.MessageLog.Info("NSQDHttpAddress is %s", _context.Config.NsqdHttpAddress)
	consumer, err := workers.CreateNsqConsumer(_context.Config, &_context.Config.RecordWorker)
	if err != nil {
		_context.MessageLog.Fatalf(err.Error())
	}
	_context.MessageLog.Info("apt_record started with config %s", _context.Config.ActiveConfig)
	_context.MessageLog.Info("DeleteOnSuccess is set to %t", _context.Config.DeleteOnSuccess)

	recorder := workers.NewAPTRecorder(_context)
	consumer.AddHandler(recorder)
	consumer.ConnectToNSQLookupd(_context.Config.NsqLookupd)

	// This reader blocks until we get an interrupt, so our program does not exit.
	<-consumer.StopChan
}