func main() { pathToConfigFile := parseCommandLine() config, err := models.LoadConfigFile(pathToConfigFile) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) } _context := context.NewContext(config) _context.MessageLog.Info("Connecting to NSQLookupd at %s", _context.Config.NsqLookupd) _context.MessageLog.Info("NSQDHttpAddress is %s", _context.Config.NsqdHttpAddress) consumer, err := apt_workers.CreateNsqConsumer(_context.Config, &_context.Config.DPN.DPNIngestStoreWorker) if err != nil { _context.MessageLog.Fatalf(err.Error()) } storer, err := workers.NewDPNIngestStorer(_context) if err != nil { _context.MessageLog.Error(err.Error()) fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) } _context.MessageLog.Info("dpn_ingest_store started") consumer.AddHandler(storer) consumer.ConnectToNSQLookupd(_context.Config.NsqLookupd) // This reader blocks until we get an interrupt, so our program does not exit. <-consumer.StopChan }
// GetContext returns a context object initialized with the specified // config file. Param configFile should be the name of a JSON config // file in the exchange/config directory. For example, "test.json" // or "integration.json". func GetContext(configFile string) (*context.Context, error) { configPath := filepath.Join("config", configFile) config, err := models.LoadConfigFile(configPath) if err != nil { return nil, err } config.ExpandFilePaths() return context.NewContext(config), nil }
func newDPNSync(t *testing.T) *workers.DPNSync { config := loadConfig(t) _context := context.NewContext(config) dpnSync, err := workers.NewDPNSync(_context) require.Nil(t, err) for namespace := range config.DPN.RemoteNodeTokens { require.NotNil(t, dpnSync.RemoteClients[namespace], namespace) } return dpnSync }
func main() { pathToConfigFile := parseCommandLine() config, err := models.LoadConfigFile(pathToConfigFile) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) } _context := context.NewContext(config) dpnSync, err := workers.NewDPNSync(_context) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) } dpnSync.Run() }
// See printUsage for a description. func main() { pathToConfigFile := parseCommandLine() config, err := models.LoadConfigFile(pathToConfigFile) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) } _context := context.NewContext(config) _context.MessageLog.Info("apt_volume_service started") volumeService := service.NewVolumeService( _context.Config.VolumeServicePort, _context.MessageLog) volumeService.Serve() }
func main() { pathToConfigFile, hours := parseCommandLine() config, err := models.LoadConfigFile(pathToConfigFile) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) } _context := context.NewContext(config) dpnQueue, err := workers.NewDPNQueue(_context, hours) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) } dpnQueue.Run() }
// test_push_to_dpn is for integration testing only. // It creates a few WorkItems in Pharos asking that // a handful of bags be pushed to DPN. Subsequent // integration tests (such as dpn_queue) depend on // the WorkItems created by this test. func TestPushToDPN(t *testing.T) { if !testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } configFile := filepath.Join("config", "integration.json") config, err := models.LoadConfigFile(configFile) require.Nil(t, err) _context := context.NewContext(config) for _, s3Key := range testutil.INTEGRATION_GOOD_BAGS[0:7] { identifier := strings.Replace(s3Key, "aptrust.receiving.test.", "", 1) identifier = strings.Replace(identifier, ".tar", "", 1) resp := _context.PharosClient.IntellectualObjectPushToDPN(identifier) workItem := resp.WorkItem() require.Nil(t, resp.Error) require.NotNil(t, workItem) _context.MessageLog.Info("Created DPN work item #%d for %s", workItem.Id, workItem.ObjectIdentifier) } }
func TestNewContext(t *testing.T) { configFile := filepath.Join("config", "test.json") appConfig, err := models.LoadConfigFile(configFile) require.Nil(t, err) // In some tests we want to log to STDERR, but in this case, if it // happens to be turned on, it just creates useless, annoying output. appConfig.LogToStderr = false _context := context.NewContext(appConfig) require.NotNil(t, _context) expectedPathToLogFile := filepath.Join(_context.Config.AbsLogDirectory(), path.Base(os.Args[0])+".log") expectedPathToJsonLog := filepath.Join(_context.Config.AbsLogDirectory(), path.Base(os.Args[0])+".json") assert.NotNil(t, _context.Config) assert.NotNil(t, _context.NSQClient) assert.NotNil(t, _context.PharosClient) assert.NotNil(t, _context.MessageLog) assert.NotNil(t, _context.JsonLog) assert.Equal(t, expectedPathToLogFile, _context.PathToLogFile()) assert.Equal(t, expectedPathToJsonLog, _context.PathToJsonLog()) assert.Equal(t, int64(0), _context.Succeeded()) assert.Equal(t, int64(0), _context.Failed()) assert.NotPanics(t, func() { _context.MessageLog.Info("Test INFO log message") }) assert.NotPanics(t, func() { _context.MessageLog.Debug("Test DEBUG log message") }) assert.NotPanics(t, func() { _context.JsonLog.Println(`{"message": "Test JSON log message"}`) }) // Cleanup, but only if context was successfully created if _context != nil && _context.PathToLogFile() != "" { os.Remove(_context.PathToLogFile()) } if _context != nil && _context.PathToJsonLog() != "" { os.Remove(_context.PathToJsonLog()) } }
// apt_record records IntellectualObjects, GenericFiles, PremisEvents // and Checksums in Pharos. Those items will have already been stored // in S3/Glacier by apt_store. This is the third and last step in the // ingest process. func main() { pathToConfigFile := parseCommandLine() config, err := models.LoadConfigFile(pathToConfigFile) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) } _context := context.NewContext(config) _context.MessageLog.Info("Connecting to NSQLookupd at %s", _context.Config.NsqLookupd) _context.MessageLog.Info("NSQDHttpAddress is %s", _context.Config.NsqdHttpAddress) consumer, err := workers.CreateNsqConsumer(_context.Config, &_context.Config.RecordWorker) if err != nil { _context.MessageLog.Fatalf(err.Error()) } _context.MessageLog.Info("apt_record started with config %s", _context.Config.ActiveConfig) _context.MessageLog.Info("DeleteOnSuccess is set to %t", _context.Config.DeleteOnSuccess) recorder := workers.NewAPTRecorder(_context) consumer.AddHandler(recorder) consumer.ConnectToNSQLookupd(_context.Config.NsqLookupd) // This reader blocks until we get an interrupt, so our program does not exit. <-consumer.StopChan }