func main() {
pathToConfigFile := parseCommandLine()
config, err := models.LoadConfigFile(pathToConfigFile)
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
_context := context.NewContext(config)
_context.MessageLog.Info("Connecting to NSQLookupd at %s", _context.Config.NsqLookupd)
_context.MessageLog.Info("NSQDHttpAddress is %s", _context.Config.NsqdHttpAddress)
consumer, err := apt_workers.CreateNsqConsumer(_context.Config, &_context.Config.DPN.DPNIngestStoreWorker)
if err != nil {
_context.MessageLog.Fatalf(err.Error())
}
storer, err := workers.NewDPNIngestStorer(_context)
if err != nil {
_context.MessageLog.Error(err.Error())
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
_context.MessageLog.Info("dpn_ingest_store started")
consumer.AddHandler(storer)
consumer.ConnectToNSQLookupd(_context.Config.NsqLookupd)
// This reader blocks until we get an interrupt, so our program does not exit.
<-consumer.StopChan
}
// GetContext returns a context object initialized with the specified
// config file. Param configFile should be the name of a JSON config
// file in the exchange/config directory. For example, "test.json"
// or "integration.json".
func GetContext(configFile string) (*context.Context, error) {
configPath := filepath.Join("config", configFile)
config, err := models.LoadConfigFile(configPath)
if err != nil {
return nil, err
}
config.ExpandFilePaths()
return context.NewContext(config), nil
}
func newDPNSync(t *testing.T) *workers.DPNSync {
config := loadConfig(t)
_context := context.NewContext(config)
dpnSync, err := workers.NewDPNSync(_context)
require.Nil(t, err)
for namespace := range config.DPN.RemoteNodeTokens {
require.NotNil(t, dpnSync.RemoteClients[namespace], namespace)
}
return dpnSync
}
func main() {
pathToConfigFile := parseCommandLine()
config, err := models.LoadConfigFile(pathToConfigFile)
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
_context := context.NewContext(config)
dpnSync, err := workers.NewDPNSync(_context)
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
dpnSync.Run()
}
// See printUsage for a description.
func main() {
pathToConfigFile := parseCommandLine()
config, err := models.LoadConfigFile(pathToConfigFile)
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
_context := context.NewContext(config)
_context.MessageLog.Info("apt_volume_service started")
volumeService := service.NewVolumeService(
_context.Config.VolumeServicePort,
_context.MessageLog)
volumeService.Serve()
}
func main() {
pathToConfigFile, hours := parseCommandLine()
config, err := models.LoadConfigFile(pathToConfigFile)
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
_context := context.NewContext(config)
dpnQueue, err := workers.NewDPNQueue(_context, hours)
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
dpnQueue.Run()
}
// test_push_to_dpn is for integration testing only.
// It creates a few WorkItems in Pharos asking that
// a handful of bags be pushed to DPN. Subsequent
// integration tests (such as dpn_queue) depend on
// the WorkItems created by this test.
func TestPushToDPN(t *testing.T) {
if !testutil.ShouldRunIntegrationTests() {
t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
}
configFile := filepath.Join("config", "integration.json")
config, err := models.LoadConfigFile(configFile)
require.Nil(t, err)
_context := context.NewContext(config)
for _, s3Key := range testutil.INTEGRATION_GOOD_BAGS[0:7] {
identifier := strings.Replace(s3Key, "aptrust.receiving.test.", "", 1)
identifier = strings.Replace(identifier, ".tar", "", 1)
resp := _context.PharosClient.IntellectualObjectPushToDPN(identifier)
workItem := resp.WorkItem()
require.Nil(t, resp.Error)
require.NotNil(t, workItem)
_context.MessageLog.Info("Created DPN work item #%d for %s",
workItem.Id, workItem.ObjectIdentifier)
}
}
func TestNewContext(t *testing.T) {
configFile := filepath.Join("config", "test.json")
appConfig, err := models.LoadConfigFile(configFile)
require.Nil(t, err)
// In some tests we want to log to STDERR, but in this case, if it
// happens to be turned on, it just creates useless, annoying output.
appConfig.LogToStderr = false
_context := context.NewContext(appConfig)
require.NotNil(t, _context)
expectedPathToLogFile := filepath.Join(_context.Config.AbsLogDirectory(), path.Base(os.Args[0])+".log")
expectedPathToJsonLog := filepath.Join(_context.Config.AbsLogDirectory(), path.Base(os.Args[0])+".json")
assert.NotNil(t, _context.Config)
assert.NotNil(t, _context.NSQClient)
assert.NotNil(t, _context.PharosClient)
assert.NotNil(t, _context.MessageLog)
assert.NotNil(t, _context.JsonLog)
assert.Equal(t, expectedPathToLogFile, _context.PathToLogFile())
assert.Equal(t, expectedPathToJsonLog, _context.PathToJsonLog())
assert.Equal(t, int64(0), _context.Succeeded())
assert.Equal(t, int64(0), _context.Failed())
assert.NotPanics(t, func() { _context.MessageLog.Info("Test INFO log message") })
assert.NotPanics(t, func() { _context.MessageLog.Debug("Test DEBUG log message") })
assert.NotPanics(t, func() { _context.JsonLog.Println(`{"message": "Test JSON log message"}`) })
// Cleanup, but only if context was successfully created
if _context != nil && _context.PathToLogFile() != "" {
os.Remove(_context.PathToLogFile())
}
if _context != nil && _context.PathToJsonLog() != "" {
os.Remove(_context.PathToJsonLog())
}
}
// apt_record records IntellectualObjects, GenericFiles, PremisEvents
// and Checksums in Pharos. Those items will have already been stored
// in S3/Glacier by apt_store. This is the third and last step in the
// ingest process.
func main() {
pathToConfigFile := parseCommandLine()
config, err := models.LoadConfigFile(pathToConfigFile)
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
_context := context.NewContext(config)
_context.MessageLog.Info("Connecting to NSQLookupd at %s", _context.Config.NsqLookupd)
_context.MessageLog.Info("NSQDHttpAddress is %s", _context.Config.NsqdHttpAddress)
consumer, err := workers.CreateNsqConsumer(_context.Config, &_context.Config.RecordWorker)
if err != nil {
_context.MessageLog.Fatalf(err.Error())
}
_context.MessageLog.Info("apt_record started with config %s", _context.Config.ActiveConfig)
_context.MessageLog.Info("DeleteOnSuccess is set to %t", _context.Config.DeleteOnSuccess)
recorder := workers.NewAPTRecorder(_context)
consumer.AddHandler(recorder)
consumer.ConnectToNSQLookupd(_context.Config.NsqLookupd)
// This reader blocks until we get an interrupt, so our program does not exit.
<-consumer.StopChan
}
