Ejemplo n.º 1
0
Archivo: app.go Proyecto: decebal/kit
// Init is called to initialize the application.
func Init(configKey string) {

	// Init the configuration system.
	if err := cfg.Init(cfg.EnvProvider{Namespace: configKey}); err != nil {
		fmt.Println("Error initalizing configuration system", err)
		os.Exit(1)
	}

	// Init the log system.
	logLevel := func() int {
		ll, err := cfg.Int(cfgLoggingLevel)
		if err != nil {
			return log.USER
		}
		return ll
	}
	log.Init(os.Stderr, logLevel)

	// Log all the configuration options
	log.User("startup", "Init", "\n\nConfig Settings: %s\n%s\n", configKey, cfg.Log())

	// Load user defined custom headers. HEADERS should be key:value,key:value
	if hs, err := cfg.String("HEADERS"); err == nil {
		hdrs := strings.Split(hs, ",")
		for _, hdr := range hdrs {
			if kv := strings.Split(hdr, ":"); len(kv) == 2 {
				log.User("startup", "Init", "User Headers : %s:%s", kv[0], kv[1])
				app.userHeaders[kv[0]] = kv[1]
			}
		}
	}
}
Ejemplo n.º 2
0
// Import gets data from report on failed import, transform it and send it to pillar
func importOnlyFailedRecords() { //dbsource source.Sourcer, limit int, offset int, orderby string, thisStrategy string, reportOnFailedRecords bool) {

	log.User(uuid, "sponge.importOnlyFailedRecords", "### Reading file of data to import.")

	// get the data that needs to be imported
	tables, err := report.ReadReport(options.Reportdbfile) //map[model]map[id]interface{}
	if err != nil {
		log.Error(uuid, "sponge.importOnlyFailedRecords", err, "Getting the rows that will be imported")
	}

	var data []map[string]interface{}

	for table, ids := range tables {

		if len(ids) < 1 { // only one ID
			// Get the data
			log.User(uuid, "sponge.importOnlyFailedRecords", "### Reading data for entity '%s'. \n", table)
			data, err = dbsource.GetData(table, &options) //options.offset, options.limit, options.orderby, options.query)
		} else {
			log.User(uuid, "sponge.importOnlyFailedRecords", "### Reading data for entity '%s', quering '%s'. \n", table, ids)
			data, err = dbsource.GetQueryData(table, &options, ids)
		}
		if err != nil && options.ReportOnFailedRecords {
			report.Record(table, ids, "Failing getting data", err)
		}

		// transform and get data into pillar
		process(table, data)
	}
}
Ejemplo n.º 3
0
func importFromDB(collections []string) {
	// var data []map[string]interface{}

	for _, name := range collections { // Reads through all the collections whose transformations are in the strategy configuration file

		foreignEntity := source.GetForeignEntity(name)

		log.User(uuid, "sponge.importAll", "### Reading data to import from %s into collection '%s'. \n", foreignEntity, name)

		// Get the data
		data, err := dbsource.GetData(name, &options) //options.offset, options.limit, options.orderby, "")
		if err != nil {
			log.Error(uuid, "sponge.importAll", err, "Get external data for collection %s.", name)
			//RECORD to report about failing modelName
			if options.ReportOnFailedRecords {
				report.Record(name, "", "Failing to get data.", err)
			}
			continue
		}

		log.User(uuid, "sponge.importAll", "### Transforming data and sending it to Coral. \n")
		//transform and send to pillar the data
		process(name, data)
	}
}
Ejemplo n.º 4
0
func process(coralName string, data []map[string]interface{}) {

	// Transform the data row by row
	log.User(uuid, "sponge.process", "# Transforming data to the coral schema.\n")
	log.User(uuid, "sponge.process", "# And importing %v documents.", len(data))

	// Initialize benchmarking for current table
	start := time.Now()
	blockStart := time.Now()
	blockSize := int64(1000) // number of documents between each report
	documents := int64(0)
	totalDocuments := int64(len(data))

	for _, row := range data {

		// output benchmarking for each block of documents
		if documents%blockSize == 0 && documents > 0 {

			// calculate stats
			percentComplete := float64(documents) / float64(totalDocuments) * float64(100)
			msSinceStart := time.Since(start).Nanoseconds() / int64(1000000)
			msSinceBlock := time.Since(blockStart).Nanoseconds() / int64(1000000)
			timeRemaining := int64(float64(time.Since(start).Seconds()) / float64(percentComplete) * float64(100))

			//log stats
			log.User(uuid, "sponge.process", "%v%% (%v/%v imported) %vms, %vms avg - last %v in %vms, %vms avg -- est time remaining %vs\n", int64(percentComplete), documents, totalDocuments, msSinceStart, msSinceStart/documents, blockSize, msSinceBlock, msSinceBlock/blockSize, int64(timeRemaining))
			blockStart = time.Now()

		}
		documents = documents + 1

		// transform the row
		id, newRows, err := fiddler.TransformRow(row, coralName)
		if err != nil {
			log.Error(uuid, "sponge.process", err, "Error when transforming the row %s.", row)
			//RECORD to report about failing transformation
			if options.ReportOnFailedRecords {
				report.Record(coralName, id, "Failing transform data", err)
			}
		}

		// Usually newRows only will have a document but in the case that we have subcollections
		// we may get more than one document from a transformation
		for _, newRow := range newRows {

			log.Dev(uuid, "sponge.process", "Transforming: %v into %v.", row, newRow)

			// send the row to pillar
			err = coral.AddRow(newRow, coralName)
			if err != nil {
				log.Error(uuid, "sponge.process", err, "Error when adding a row") // thae row %v to %s.", string(newRow), modelName)
				//RECORD to report about failing adding row to coral db
				if options.ReportOnFailedRecords {
					report.Record(coralName, id, "Failing add row to coral", err)
				}
			}
		}
	}
}
Ejemplo n.º 5
0
// Job implements the Jobber interface so task can be managed.
func (t *Task) Job(context interface{}) error {
	log.User(context, "Job", "Started : **********")

	time.Sleep(time.Second)

	log.User(context, "Job", "Completed : **********")
	return nil
}
Ejemplo n.º 6
0
// Run performs the execution of the specified job.
func Run(context interface{}, timeout time.Duration, job Jobber) error {
	log.User(context, "Run", "Started : Timeout[%v]", timeout)

	// Init the runner for use.
	if initRunner(timeout) {
		return errors.New("Already running")
	}

	// When the task is done reset everything.
	defer resetRunner()

	// We want to receive all interrupt based signals.
	signal.Notify(runner.sigChan, os.Interrupt)

	// Launch the processor.
	go processor(context, job)

	for {
		select {
		case <-runner.sigChan:

			// Interrupt event signaled by the operation system.
			log.User(context, "Run", "Interrupt Received")

			// Flag we received the signal.
			runner.recvShutdown = true

			// Close the channel to signal to the processor
			// it needs to shutdown.
			close(runner.shutdown)

			// No need to process anymore events.
			signal.Stop(runner.sigChan)

		case <-runner.kill:

			// We have taken too much time. Kill the app hard.
			log.User(context, "Run", "Completed : Timedout")
			return ErrTimeout

		case err := <-runner.complete:

			// Everything completed within the time given.
			log.User(context, "Run", "Completed : Task Result : %v", err)

			if runner.recvShutdown {
				return ErrSignaled
			}

			return err
		}
	}
}
Ejemplo n.º 7
0
// Handle is our mechanism for mounting Handlers for a given HTTP verb and path
// pair, this makes for really easy, convenient routing.
func (a *App) Handle(verb, path string, handler Handler, mw ...Middleware) {

	// The function to execute for each request.
	h := func(w http.ResponseWriter, r *http.Request, p map[string]string) {
		start := time.Now()

		var dbConn *db.DB
		if app.useMongo {
			dbConn = db.NewMGO()
		}

		c := Context{
			DB:             dbConn,
			ResponseWriter: w,
			Request:        r,
			Params:         p,
			SessionID:      uuid.New(),
		}

		if app.useMongo {
			defer c.DB.CloseMGO()
		}

		log.User(c.SessionID, "Request", "Started : Method[%s] URL[%s] RADDR[%s]", c.Request.Method, c.Request.URL.Path, c.Request.RemoteAddr)

		// Wrap the handler in all associated middleware.
		wrap := func(h Handler) Handler {
			// Wrap up the application-wide first...
			for i := len(a.mw) - 1; i >= 0; i-- {
				h = a.mw[i](h)
			}

			// Then wrap with our route specific ones.
			for i := len(mw) - 1; i >= 0; i-- {
				h = mw[i](h)
			}

			return h
		}

		// Call the wrapped handler and handle any possible error.
		if err := wrap(handler)(&c); err != nil {
			c.Error(err)
		}

		log.User(c.SessionID, "Request", "Completed : Status[%d] Duration[%s]", c.Status, time.Since(start))
	}

	// Add this handler for the specified verb and route.
	a.TreeMux.Handle(verb, path, h)
}
Ejemplo n.º 8
0
// Init is called to initialize the application.
func Init(configKey string) {

	// Init the configuration system.
	if err := cfg.Init(cfg.EnvProvider{Namespace: configKey}); err != nil {
		fmt.Println("Error initalizing configuration system", err)
		os.Exit(1)
	}

	// Init the log system.
	logLevel := func() int {
		ll, err := cfg.Int(cfgLoggingLevel)
		if err != nil {
			return log.USER
		}
		return ll
	}
	log.Init(os.Stderr, logLevel)

	// Log all the configuration options
	log.User("startup", "Init", "\n\nConfig Settings: %s\n%s\n", configKey, cfg.Log())

	// Init MongoDB if configured.
	if _, err := cfg.String(cfgMongoHost); err == nil {
		app.useMongo = true

		cfg := mongo.Config{
			Host:     cfg.MustString(cfgMongoHost),
			AuthDB:   cfg.MustString(cfgMongoAuthDB),
			DB:       cfg.MustString(cfgMongoDB),
			User:     cfg.MustString(cfgMongoUser),
			Password: cfg.MustString(cfgMongoPassword),
		}

		if err := mongo.Init(cfg); err != nil {
			log.Error("startup", "Init", err, "Initializing MongoDB")
			os.Exit(1)
		}
	}

	// Load user defined custom headers. HEADERS should be key:value,key:value
	if hs, err := cfg.String("HEADERS"); err == nil {
		hdrs := strings.Split(hs, ",")
		for _, hdr := range hdrs {
			if kv := strings.Split(hdr, ":"); len(kv) == 2 {
				log.User("startup", "Init", "User Headers : %s:%s", kv[0], kv[1])
				app.userHeaders[kv[0]] = kv[1]
			}
		}
	}
}
Ejemplo n.º 9
0
func main() {
	const context = "startup"

	// Create the configuration.
	cfg := udp.Config{
		NetType: "udp4",
		Addr:    ":6000",

		ConnHandler: udpConnHandler{},
		ReqHandler:  udpReqHandler{},
		RespHandler: udpRespHandler{},

		OptIntPool: udp.OptIntPool{
			RecvMinPoolSize: func() int { return 2 },
			RecvMaxPoolSize: func() int { return 100 },
			SendMinPoolSize: func() int { return 2 },
			SendMaxPoolSize: func() int { return 100 },
		},
	}

	// Create a new UDP value.
	u, err := udp.New(context, "Sample", cfg)
	if err != nil {
		log.Error(context, "main", err, "Creating udp")
		return
	}

	// Start accepting client data.
	if err := u.Start(context); err != nil {
		log.Error(context, "main", err, "Starting udp")
		return
	}

	// Defer the stop on shutdown.
	defer u.Stop(context)

	log.User(context, "main", "Waiting for data on: %s", u.Addr())

	// Listen for an interrupt signal from the OS.
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt)
	<-sigChan

	// Use netcat to test the server.
	// nc -4u localhost 6000 < test.hex

	log.User(context, "main", "Shutting down")
}
Ejemplo n.º 10
0
// GetData returns the raw data from the external source, entityname entity
func (m MongoDB) GetData(entityname string, options *Options) ([]map[string]interface{}, error) { // offset int, limit int, orderby string, query string) ([]map[string]interface{}, bool, error) { //(*sql.Rows, error) {

	var data []map[string]interface{}

	//fields := strategy.GetEntityForeignFields(entityname) //[]]map[string]string

	// open a connection
	session, err := m.initSession()
	if err != nil {
		log.Error(uuid, "mongodb.getdata", err, "Initializing mongo session.")
		return nil, err
	}
	defer m.closeSession(session)

	credentialD, ok := credential.(str.CredentialDatabase)
	if !ok {
		err = fmt.Errorf("Error asserting type CredentialDatabase from interface Credential.")
		log.Error(uuid, "mongodb.getdata", err, "Asserting Type CredentialDatabase")
		return nil, err
	}
	cred := mgo.Credential{
		Username: credentialD.Username,
		Password: credentialD.Password,
	}

	err = session.Login(&cred)
	if err != nil {
		log.Error(uuid, "mongodb.getdata", err, "Login mongo session.")
		return nil, err
	}

	db := session.DB(credentialD.Database)
	col := db.C(entityname)

	var mquery map[string]interface{}
	if options.Query != "" {
		err = json.Unmarshal([]byte(options.Query), &mquery)
		if err != nil {
			log.Error(uuid, "mongodb.getdata", err, "Unmarshalling query %v", options.Query)
			return nil, err
		}
	}

	//.Select(fieldsToGet) <--- I'm not using Select because SOME FIELDS IN THE TRANSLATION FILE ARE NOT THE RIGHT ONES TO DO THE SELECT. For example: context.object.0.uri
	err = col.Find(mquery).Skip(options.Offset).Limit(options.Limit).All(&data)
	if err != nil {
		log.Error(uuid, "mongodb.getdata", err, "Getting collection %s.", entityname)
		return nil, err
	}

	log.User(uuid, "mongodb.getdata", "### Flattening documents to be able to translate them. \n")

	flattenData, err := flattenizeData(data)
	if err != nil {
		log.Error(uuid, "mongodb.getdata", err, "Normalizing data from mongo to fit into fiddler.")
		return nil, err
	}

	return flattenData, nil
}
Ejemplo n.º 11
0
func main() {
	const context = "main"

	// Create a task value for execution.
	t := Task{
		Name: "test task",
	}

	// Start the job running with a specified duration.
	if err := runner.Run(context, time.Second, &t); err != nil {
		switch err {
		case runner.ErrTimeout:

			// The task did not finish within the specified duration.
			log.Error(context, "main", err, "Task timeout")

		case runner.ErrSignaled:

			// The user hit <control> c and we shutdown early.
			log.Error(context, "main", err, "Shutdown early")

		default:

			// An error occurred in the processing of the task.
			log.Error(context, "main", err, "Processing error")
		}

		os.Exit(1)
	}

	log.User(context, "main", "Completed")
}
Ejemplo n.º 12
0
func init() {

	// This is being added to showcase configuration.
	os.Setenv("KIT_LOGGING_LEVEL", "1")
	os.Setenv("KIT_MIN_ROUTINES", "1")
	os.Setenv("KIT_MAX_ROUTINES", "10")

	// Init the configuration system.
	if err := cfg.Init(cfg.EnvProvider{Namespace: configKey}); err != nil {
		fmt.Println("Error initalizing configuration system", err)
		os.Exit(1)
	}

	// Init the log system.
	logLevel := func() int {
		ll, err := cfg.Int(cfgLoggingLevel)
		if err != nil {
			return log.USER
		}
		return ll
	}
	log.Init(os.Stderr, logLevel)

	// Log all the configuration options
	log.User("startup", "init", "\n\nConfig Settings: %s\n%s\n", configKey, cfg.Log())
}
Ejemplo n.º 13
0
// TestLogLevelDEV tests the basic functioning of the logger in DEV mode.
func TestLogLevelDEV(t *testing.T) {
	t.Log("Given the need to log DEV and USER messages.")
	{
		t.Log("\tWhen we set the logging level to DEV.")
		{
			log.Init(&logdest, func() int { return log.DEV })
			resetLog()
			defer displayLog()

			dt := time.Now().Format("2006/01/02 15:04:05")

			log1 := fmt.Sprintf("%s log_test.go:81: DEV : context : FuncName : Message 1 no format\n", dt)
			log2 := fmt.Sprintf("%s log_test.go:82: USER : context : FuncName : Message 2 with format: A, B\n", dt)
			log3 := fmt.Sprintf("%s log_test.go:83: ERROR : context : FuncName : An error : Message 3 with format: C, D\n", dt)

			log.Dev("context", "FuncName", "Message 1 no format")
			log.User("context", "FuncName", "Message 2 with format: %s, %s", "A", "B")
			log.Error("context", "FuncName", errors.New("An error"), "Message 3 with format: %s, %s", "C", "D")

			if logdest.String() == log1+log2+log3 {
				t.Logf("\t\t%v : Should log the expected trace line.", succeed)
			} else {
				t.Log("***>", logdest.String())
				t.Log("***>", log1+log2+log3)
				t.Errorf("\t\t%v : Should log the expected trace line.", failed)
			}
		}
	}
}
Ejemplo n.º 14
0
// Respond sends JSON to the client.
// If code is StatusNoContent, v is expected to be nil.
func (c *Context) Respond(data interface{}, code int) {
	log.User(c.SessionID, "api : Respond", "Started : Code[%d]", code)

	c.Status = code

	if code == http.StatusNoContent {
		c.WriteHeader(http.StatusNoContent)
		return
	}

	// Set application default header values.
	c.Header().Set("Content-Type", "application/json")

	// Load any user defined header values.
	if app.userHeaders != nil {
		for key, value := range app.userHeaders {
			log.User("startup", "Init", "Setting user headers : %s:%s", key, value)
			c.Header().Set(key, value)
		}
	}

	c.WriteHeader(code)

	// Marshal the data into a JSON string.
	jsonData, err := json.Marshal(data)
	if err != nil {
		jsonData = []byte("{}")
	}

	// Look for a JSONP marker
	if cb := c.Request.URL.Query().Get("callback"); cb != "" {

		// We need to wrap the result in a function call.
		// callback_value({"data_1": "hello world", "data_2": ["the","sun","is","shining"]});
		fmt.Fprintf(c, "%s(%s)", cb, string(jsonData))

		log.User(c.SessionID, "api : Respond", "Completed")
		return
	}

	// We can send the result straight through.
	fmt.Fprintf(c, string(jsonData))

	log.User(c.SessionID, "api : Respond", "Completed")
}
Ejemplo n.º 15
0
// Midware handles the request logging.
func Midware(next web.Handler) web.Handler {

	// Create the handler that will be attached in the middleware chain.
	h := func(c *web.Context) error {

		log.User(c.SessionID, "log : Midware", "Started : Method[%s] URL[%s] RADDR[%s]", c.Request.Method, c.Request.URL.Path, c.Request.RemoteAddr)

		if err := next(c); err != nil {
			return err
		}

		log.User(c.SessionID, "log : Midware", "Completed : Status[%d] Duration[%s]", c.Status, time.Since(c.Now))

		return nil
	}

	return h
}
Ejemplo n.º 16
0
// Respond sends JSON to the client.
// If code is StatusNoContent, v is expected to be nil.
func (c *Context) Respond(data interface{}, code int) {
	log.User(c.SessionID, "api : Respond", "Started : Code[%d]", code)

	c.Status = code

	if code == http.StatusNoContent {
		c.WriteHeader(http.StatusNoContent)
		return
	}

	// Set application default header values.
	c.Header().Set("Content-Type", "application/json")

	// Load any user defined header values.
	if app.userHeaders != nil {
		for key, value := range app.userHeaders {
			log.User("startup", "Init", "Setting user headers : %s:%s", key, value)
			c.Header().Set(key, value)
		}
	}

	c.WriteHeader(code)

	// Look for a JSONP marker
	if cb := c.Request.URL.Query().Get("callback"); cb != "" {

		// We need to wrap the result in a function call.
		// callback_value({"data_1": "hello world", "data_2": ["the","sun","is","shining"]});
		b := bytes.NewBufferString(cb + "(")
		json.NewEncoder(b).Encode(data)
		b.WriteString(")")
		fmt.Fprintf(c, b.String())

	} else {

		// We can send the result straight through.
		json.NewEncoder(c).Encode(data)

	}

	log.User(c.SessionID, "api : Respond", "Completed")
}
Ejemplo n.º 17
0
// ExampleDev shows how to use the log package.
func ExampleDev(t *testing.T) {
	// Init the log package for stdout. Hardcode the logging level
	// function to use USER level logging.
	log.Init(os.Stdout, func() int { return log.USER })

	// Write a simple log line with no formatting.
	log.User("context", "ExampleDev", "This is a simple line with no formatting")

	// Write a simple log line with formatting.
	log.User("context", "ExampleDev", "This is a simple line with no formatting %d", 10)

	// Write a message error for the user.
	log.Error("context", "ExampleDev", errors.New("A user error"), "testing error")

	// Write a message error for the user with formatting.
	log.Error("context", "ExampleDev", errors.New("A user error"), "testing error %s", "value")

	// Write a message error for the developer only.
	log.Dev("context", "ExampleDev", "Formatting %v", 42)
}
Ejemplo n.º 18
0
func importFromAPI(collections []string) {

	pageAfter := ""
	log.User(uuid, "sponge.importFromAPI", "### Reading data from API. \n")

	api, ok := dbsource.(source.API)
	if !ok {
		err := fmt.Errorf("Error asserting sourcer into source.API.")
		log.Error(uuid, "sponge.importFromAPI", err, "Asserting type for source.API")
	}

	var err error
	var data []map[string]interface{}
	var nextPageAfter string
	var pollingInterval time.Duration

	if pollEnvVar, err := strconv.Atoi(os.Getenv("POLLING_INTERVAL")); err != nil || pollEnvVar == 0 {
		pollingInterval = time.Duration(options.TimeWaiting) * time.Second
	} else {
		pollingInterval = time.Duration(pollEnvVar) * time.Second
	}

	for true {
		data, nextPageAfter, err = api.GetFireHoseData(pageAfter)
		if err != nil {
			log.Error(uuid, "sponge.importFromAPI", err, "Getting data from API")
			return
		}

		if data != nil {
			processAPI(collections, data)
			pageAfter = nextPageAfter
		}

		if data == nil {
			log.User(uuid, "sponge.importFromAPI", "Waiting %s seconds for more data.", pollingInterval)
			time.Sleep(pollingInterval) // sleep timeWaiting seconds
		}
	}

}
Ejemplo n.º 19
0
// CreateIndex will read the strategy file and create index that are mentioned there for each collection
func CreateIndex(collection string) {

	log.User(uuid, "sponge.createindex", "###  Create Index.")

	//create index for everybody
	if collection == "" {
		// get data from strategy file
		tables := fiddler.GetCollections()

		// for each table
		for t := range tables {
			log.User(uuid, "sponge.createindex", "### Create index for collection %s.", tables[t])
			coral.CreateIndex(tables[t])
		}
		return
	}

	log.User(uuid, "sponge.createindex", "### Create index for collection %s.", collection)
	//create index only for collection
	coral.CreateIndex(collection)
}
Ejemplo n.º 20
0
func main() {
	log.User("startup", "Init", "Revision     : %q", GitRevision)
	log.User("startup", "Init", "Version      : %q", GitVersion)
	log.User("startup", "Init", "Build Date   : %q", BuildDate)
	log.User("startup", "Init", "Int Version  : %q", IntVersion)
	log.User("startup", "Init", "Go Version   : %q", runtime.Version())
	log.User("startup", "Init", "Go Compiler  : %q", runtime.Compiler)
	log.User("startup", "Init", "Go ARCH      : %q", runtime.GOARCH)
	log.User("startup", "Init", "Go OS        : %q", runtime.GOOS)

	handlers.Version.GitRevision = GitRevision
	handlers.Version.GitVersion = GitVersion
	handlers.Version.BuildDate = BuildDate
	handlers.Version.IntVersion = IntVersion

	// These are the absolute read and write timeouts.
	const (

		// ReadTimeout covers the time from when the connection is accepted to when the
		// request body is fully read.
		readTimeout = 10 * time.Second

		// WriteTimeout normally covers the time from the end of the request header read
		// to the end of the response write.
		writeTimeout = 30 * time.Second
	)

	host := cfg.MustString(cfgHost)

	log.User("startup", "Init", "Binding web service to %s", host)

	if err := web.Run(host, routes.API(), readTimeout, writeTimeout); err != nil {
		log.Error("shutdown", "Init", err, "App Shutdown")
		os.Exit(1)
	}

	log.User("shutdown", "Init", "App Shutdown")
}
Ejemplo n.º 21
0
// CheckShutdown checks the shutdown flag to determine
// if we have been asked to interrupt processing.
func CheckShutdown(context interface{}) bool {
	select {
	case <-runner.shutdown:

		// We have been asked to shutdown.
		log.User(context, "CheckShutdown", "Shutdown Early")
		return true

	default:

		// We have not been asked to shutdown.
		return false
	}
}
Ejemplo n.º 22
0
// API returns a handler for a set of routes.
func API() http.Handler {
	mongoURI := cfg.MustURL(cfgMongoURI)

	// The web framework middleware for Mongo is using the name of the
	// database as the name of the master session by convention. So use
	// cfg.DB as the second argument when creating the master session.
	if err := db.RegMasterSession("startup", mongoURI.Path, mongoURI.String(), 25*time.Second); err != nil {
		log.Error("startup", "Init", err, "Initializing MongoDB")
		os.Exit(1)
	}

	w := web.New(logm.Midware, errorm.Midware)

	publicKey, err := cfg.String(cfgAuthPublicKey)
	if err != nil || publicKey == "" {
		log.User("startup", "Init", "%s is missing, internal authentication is disabled", cfgAuthPublicKey)
	}

	// If the public key is provided then add the auth middleware or fail using
	// the provided public key.
	if publicKey != "" {
		log.Dev("startup", "Init", "Initializing Auth")

		authm, err := authm.Midware(publicKey, authm.MidwareOpts{})
		if err != nil {
			log.Error("startup", "Init", err, "Initializing Auth")
			os.Exit(1)
		}

		// Apply the authentication middleware on top of the application as the
		// first middleware.
		w.Use(authm)
	}

	// Add the Mongo and Cayley middlewares possibly after the auth middleware.
	w.Use(mongo.Midware(mongoURI), cayley.Midware(mongoURI))

	if cors, err := cfg.Bool(cfgEnableCORS); err == nil && cors {
		log.Dev("startup", "Init", "Initializing CORS : CORS Enabled")
		w.Use(w.CORS())
	} else {
		log.Dev("startup", "Init", "CORS Disabled")
	}

	log.Dev("startup", "Init", "Initalizing routes")
	routes(w)

	return w
}
Ejemplo n.º 23
0
// processor provides the main program logic for the program.
func processor(context interface{}, job Jobber) {
	log.User(context, "processor", "Started")

	// Variable to store any error that occurs.
	var err error

	// Defer the send on the channel so it happens
	// regardless of how this function terminates.
	defer func() {

		// Capture any potential panic.
		if r := recover(); r != nil {
			log.User(context, "processor", "Panic : %v", r)
		}

		// Signal the goroutine we have shutdown.
		runner.complete <- err
	}()

	// Run the job.
	err = job.Job(context)

	log.User(context, "processor", "Completed")
}
Ejemplo n.º 24
0
// Process is used to handle the processing of the message. This method
// is called on a routine from a pool of routines.
func (tcpReqHandler) Process(context interface{}, r *tcp.Request) {
	log.Dev(context, "Process", "Started : IP[%s] Length[%d] ReadAt[%v]", r.TCPAddr.String(), r.Length, r.ReadAt)

	log.User(context, "Process", "Data : %s", string(r.Data))

	resp := tcp.Response{
		TCPAddr: r.TCPAddr,
		Data:    []byte("GOT IT\n"),
		Length:  7,
		Complete: func(rsp *tcp.Response) {
			log.Dev(context, "Process", "*****************> %v", rsp)
		},
	}

	r.TCP.Do(context, &resp)

	log.Dev(context, "Process", "Completed")
}
Ejemplo n.º 25
0
// ImportType gets ony data related to table, transform it and send it to pillar
func importType(coralEntity string) { //dbsource source.Sourcer, limit int, offset int, orderby string, query string, modelName string, reportOnFailedRecords bool) {

	foreignEntity := source.GetForeignEntity(coralEntity)
	// Get the data
	log.User(uuid, "sponge.importTable", "### Reading data from table '%s'.", foreignEntity)

	data, err := dbsource.GetData(foreignEntity, &options) //options.offset, options.limit, options.orderby, options.query)
	if err != nil {
		log.Error(uuid, "sponge.importAll", err, "Get external data for table %s.", foreignEntity)
		//RECORD to report about failing modelName
		if options.ReportOnFailedRecords {
			report.Record(foreignEntity, "", "Failing to get data", err)
		}
		return
	}

	// Transform and send to pillar
	process(coralEntity, data)

}
Ejemplo n.º 26
0
// Import gets ALL data, transform it and send it to pillar
func importAll() {
	//dbsource source.Sourcer, limit int, offset int, orderby string, reportOnFailedRecords bool) {

	log.User(uuid, "sponge.importAll", "### Reading tables to import from strategy file.")

	//Get all the collections's names that we have in the strategy json file
	collections, err := source.GetEntities()

	if err != nil {
		log.Error(uuid, "sponge.importAll", err, "Get collections's names.")
		return
	}

	if dbsource.IsWebService() {
		importFromAPI(collections)
		return
	}

	importFromDB(collections)

}
Ejemplo n.º 27
0
func main() {
	const context = "main"
	const totalWork = 100

	wg.Add(totalWork)

	// Create the configuration.
	cfg := pool.Config{
		MinRoutines: func() int { return cfg.MustInt(cfgMinRoutines) },
		MaxRoutines: func() int { return cfg.MustInt(cfgMaxRoutines) },
	}

	// Create a pool.
	p, err := pool.New(context, "test", cfg)
	if err != nil {
		log.Error(context, "main", err, "Creating pool")
		return
	}

	// Look at stats for the work.
	go func() {
		for {
			time.Sleep(250 * time.Millisecond)
			log.User(context, "Stats", "%#v", p.Stats())
		}
	}()

	// Perform some work.
	for i := 0; i < totalWork; i++ {
		p.Do(context, &Task{Name: strconv.Itoa(i)})
	}

	// Wait until all the work is complete.
	wg.Wait()

	// Shutdown the pool.
	p.Shutdown(context)
}
Ejemplo n.º 28
0
func routes(w *web.Web) {

	// Create a new app group which will be for internal functions that may have
	// an optional layer of auth added to it.
	internal := w.Group()

	// Now we will load in the public key from the config. If found, we'll add a
	// middleware to all internal endpoints that will ensure that we validate the
	// requests coming in.

	publicKey, err := cfg.String(cfgAuthPublicKey)
	if err != nil || publicKey == "" {
		log.User("startup", "Init", "%s is missing, internal authentication is disabled", cfgAuthPublicKey)
	}

	// If the public key is provided then add the auth middleware or fail using
	// the provided public key.
	if publicKey != "" {
		log.Dev("startup", "Init", "Initializing Auth")

		// We are allowing the query string to act as the access token provider
		// because this service has endpoints that are accessed directly currently
		// and we need someway to authenticate to these endpoints.
		authmOpts := auth.MidwareOpts{
			AllowQueryString: true,
		}

		authm, err := auth.Midware(publicKey, authmOpts)
		if err != nil {
			log.Error("startup", "Init", err, "Initializing Auth")
			os.Exit(1)
		}

		// Apply the authentication middleware on top of the application as the
		// first middleware.
		internal.Use(authm)
	}

	// global
	internal.Handle("GET", "/v1/version", handlers.Version.List)

	// forms
	internal.Handle("POST", "/v1/form", handlers.Form.Upsert)
	internal.Handle("GET", "/v1/form", handlers.Form.List)
	internal.Handle("PUT", "/v1/form/:id", handlers.Form.Upsert)
	internal.Handle("PUT", "/v1/form/:id/status/:status", handlers.Form.UpdateStatus)
	internal.Handle("GET", "/v1/form/:id", handlers.Form.Retrieve)
	internal.Handle("DELETE", "/v1/form/:id", handlers.Form.Delete)

	// form aggregations
	internal.Handle("GET", "/v1/form/:form_id/digest", handlers.Aggregation.Digest)
	internal.Handle("GET", "/v1/form/:form_id/aggregate", handlers.Aggregation.Aggregate)
	internal.Handle("GET", "/v1/form/:form_id/aggregate/:group_id", handlers.Aggregation.AggregateGroup)
	internal.Handle("GET", "/v1/form/:form_id/aggregate/:group_id/submission", handlers.Aggregation.SubmissionGroup)

	// form submissions
	internal.Handle("GET", "/v1/form/:form_id/submission", handlers.FormSubmission.Search)
	internal.Handle("GET", "/v1/form/:form_id/submission/:id", handlers.FormSubmission.Retrieve)
	internal.Handle("PUT", "/v1/form/:form_id/submission/:id/status/:status", handlers.FormSubmission.UpdateStatus)
	internal.Handle("POST", "/v1/form/:form_id/submission/:id/flag/:flag", handlers.FormSubmission.AddFlag)
	internal.Handle("DELETE", "/v1/form/:form_id/submission/:id/flag/:flag", handlers.FormSubmission.RemoveFlag)
	internal.Handle("PUT", "/v1/form/:form_id/submission/:id/answer/:answer_id", handlers.FormSubmission.UpdateAnswer)
	internal.Handle("DELETE", "/v1/form/:form_id/submission/:id", handlers.FormSubmission.Delete)

	// temporal route to get CSV file - TO DO : move into a different service
	internal.Handle("GET", "/v1/form/:form_id/submission/export", handlers.FormSubmission.Download)

	// form form galleries
	internal.Handle("GET", "/v1/form/:form_id/gallery", handlers.FormGallery.RetrieveForForm)

	// form galleries
	internal.Handle("GET", "/v1/form_gallery/:id", handlers.FormGallery.Retrieve)
	internal.Handle("PUT", "/v1/form_gallery/:id", handlers.FormGallery.Update)
	internal.Handle("POST", "/v1/form_gallery/:id/submission/:submission_id/:answer_id", handlers.FormGallery.AddAnswer)
	internal.Handle("DELETE", "/v1/form_gallery/:id/submission/:submission_id/:answer_id", handlers.FormGallery.RemoveAnswer)

	// Create a new app group which will be for external functions that will need
	// to be publically exposed.
	external := w.Group()

	external.Handle("POST", "/v1/form/:form_id/submission", handlers.FormSubmission.Create)
}
Ejemplo n.º 29
0
// GetFireHoseData use the firehose to constantly GET data from the web service
func (a API) GetFireHoseData(pageAfter string) ([]map[string]interface{}, string, error) {
	var (
		flattenData   []map[string]interface{}
		nextPageAfter string
		err           error
	)

	// Get the credentials to connect to the API
	cred, err := strategy.GetCredential("service", "foreign")
	if err != nil {
		log.Error(uuid, "api.getFirehoseData", err, "Getting credentials with API")
	}

	// Assert Type into a credential API struct
	credA, ok := cred.(str.CredentialService)
	if !ok {
		log.Error(uuid, "api.getFirehoseData", err, "Asserting type.")
	}

	// TO DO: THIS IS VERY WAPO API HARCODED!
	url := connectionAPI(pageAfter)

	log.User(uuid, "api.getFirehoseData", "Querying URL %s", url)
	// Build the request
	req, err := http.NewRequest("GET", url.String(), nil)
	if err != nil {
		log.Error(uuid, "api.getFirehoseData", err, "New request.")
		return nil, pageAfter, err
	}
	req.Header.Add("User-Agent", credA.GetUserAgent())

	// For control over HTTP client headers,
	// redirect policy, and other settings,
	// create a Client
	// A Client is an HTTP client
	client := &http.Client{}

	// Send the request via a client
	// Do sends an HTTP request and
	// returns an HTTP response
	resp, err := client.Do(req)
	if err != nil {
		log.Error(uuid, "api.getFirehoseData", err, "Doing a call to API.")
		return nil, pageAfter, err
	}
	if resp.StatusCode != 200 {
		err := fmt.Errorf("Bad Request %v", resp.Status)
		log.Error(uuid, "api.getFirehoseData", err, "Doing a call to API.")
		return nil, pageAfter, err
	}

	// Callers should close resp.Body
	// when done reading from it
	// Defer the closing of the body
	defer resp.Body.Close()

	var d map[string]interface{}

	// Use json.Decode for reading streams of JSON data
	if err = json.NewDecoder(resp.Body).Decode(&d); err != nil {
		log.Error(uuid, "api.getFirehoseData", err, "Decoding data from API.")
		return nil, pageAfter, err
	}

	recordsField := credA.GetRecordsFieldName()

	// Emtpy records means there are no more data to send
	if d[recordsField] == nil {
		return nil, pageAfter, err
	}

	records, ok := d[recordsField].([]interface{}) //this are all the entries
	if !ok {
		log.Error(uuid, "api.getFirehoseData", err, "Asserting type.")
		return nil, pageAfter, err
	}

	var r []map[string]interface{}
	for _, i := range records { // all the entries in the type we need
		r = append(r, i.(map[string]interface{}))
	}

	flattenData, err = flattenizeData(r)
	if err != nil {
		log.Error(uuid, "api.getfirehosedata", err, "Normalizing data from api to fit into fiddler.")
		return nil, nextPageAfter, err
	}

	nextPageField := credA.GetNextPageField()

	if d[nextPageField] != nil {
		switch pf := d[nextPageField].(type) {
		case float64:
			nextPageAfter = strconv.FormatFloat(pf, 'f', 6, 64)
		case string:
			nextPageAfter = pf
		default:
			log.User(uuid, "api.getfirehosedata", "Do not know what is the type asserting for the Next Pagination value.")
		}
	}
	return flattenData, nextPageAfter, err
}
Ejemplo n.º 30
0
// API returns a handler for a set of routes.
func API() http.Handler {
	w := web.New(logm.Midware, errorm.Midware)

	publicKey, err := cfg.String(cfgAuthPublicKey)
	if err != nil || publicKey == "" {
		log.User("startup", "Init", "%s is missing, internal authentication is disabled", cfgAuthPublicKey)
	}

	// If the public key is provided then add the auth middleware or fail using
	// the provided public key.
	if publicKey != "" {
		log.Dev("startup", "Init", "Initializing Auth")

		authm, err := authm.Midware(publicKey, authm.MidwareOpts{})
		if err != nil {
			log.Error("startup", "Init", err, "Initializing Auth")
			os.Exit(1)
		}

		// Apply the authentication middleware on top of the application as the
		// first middleware.
		w.Use(authm)
	}

	platformPrivateKey, err := cfg.String(cfgPlatformPrivateKey)
	if err != nil || platformPrivateKey == "" {
		log.User("startup", "Init", "%s is missing, downstream platform authentication is disabled", cfgPlatformPrivateKey)
	}

	// If the platformPrivateKey is provided, then we should generate the token
	// signing function to be used when composing requests down to the platform.
	if platformPrivateKey != "" {
		log.Dev("startup", "Init", "Initializing Downstream Platform Auth")

		signer, err := auth.NewSigner(platformPrivateKey)
		if err != nil {
			log.Error("startup", "Init", err, "Initializing Downstream Platform Auth")
			os.Exit(1)
		}

		// Requests can now be signed with the given signer function which we will
		// save on the application wide context. In the event that a function
		// requires a call down to a downstream platform, we will include a signed
		// header using the signer function here.
		w.Ctx["signer"] = signer
	}

	if cors, err := cfg.Bool(cfgEnableCORS); err == nil && cors {
		log.Dev("startup", "Init", "Initializing CORS : CORS Enabled")
		w.Use(w.CORS())
	} else {
		log.Dev("startup", "Init", "CORS Disabled")
	}

	// We need the URL for the services Sponged and Xeniad that needs to be running.
	spongedURL, err = cfg.String(cfgSpongedURL)
	if err != nil || spongedURL == "" {
		log.Error("startup", "Init", err, "Service Sponged needs to be setup.")
		os.Exit(1)
	}
	w.Ctx["spongedURL"] = cfg.MustURL(cfgSpongedURL).String()

	xeniadURL, err = cfg.String(cfgXeniadURL)
	if err != nil || xeniadURL == "" {
		log.Error("startup", "Init", err, "Service Xeniad needs to be setup.")
		os.Exit(1)
	}
	w.Ctx["xeniadURL "] = cfg.MustURL(cfgXeniadURL).String()

	log.Dev("startup", "Init", "Initalizing routes")
	routes(w)

	return w
}