Exemple #1
0
func (s *DbSink) init(ctx context.Context) error {
	log, ok := autoctx.GetLogEntry(ctx)
	if !ok {
		log = logrus.WithField("Component", "CsvSink")
		log.Error("No logger found in context")
	}

	s.log = log.WithFields(logrus.Fields{
		"Component": "DbSink",
		"Type":      fmt.Sprintf("%T", s),
	})

	if s.input == nil {
		return fmt.Errorf("Input attribute not set")
	}

	if s.sql == "" {
		return fmt.Errorf("Sql attribute is required")
	}

	if s.prepFn == nil {
		return fmt.Errorf("Prepare function is missing")
	}

	s.log.Info("Component initialized")

	return nil

}
Exemple #2
0
func (c *CsvSink) init(ctx context.Context) error {
	//extract log entry
	log, ok := autoctx.GetLogEntry(ctx)
	if !ok {
		log = logrus.WithField("Component", "CsvSink")
		log.Error("No logger found in context")
	}

	c.log = log.WithFields(logrus.Fields{
		"Component": "CsvSink",
		"Type":      fmt.Sprintf("%T", c),
	})

	if c.input == nil {
		return fmt.Errorf("Input attribute not set")
	}

	// establish defaults
	if c.delimChar == 0 {
		c.delimChar = ','
	}

	var writer io.Writer
	if c.snkWriter != nil {
		writer = c.snkWriter
		c.log.Debug("Using IO Writer sink")
	} else {
		file, err := os.Create(c.filepath)
		if err != nil {
			return fmt.Errorf("Failed to create file %s: %v ", c.filepath, err)
		}
		writer = file
		c.file = file
		c.log.Debug("Using sink file", file.Name())
	}
	c.csvWriter = csv.NewWriter(writer)
	c.csvWriter.Comma = c.delimChar

	// write headers
	if len(c.headers) > 0 {
		if err := c.csvWriter.Write(c.headers); err != nil {
			return err
		}
		c.log.Debug("Wrote headers [", c.headers, "]")
	}

	c.log.Info("Component initialized")

	return nil
}
Exemple #3
0
func NewBinaryOp(ctx context.Context) *BinaryOp {
	// extract logger
	log, ok := autoctx.GetLogEntry(ctx)
	if !ok {
		log = logrus.WithField("Component", "BinaryOperator")
		log.Error("Logger not found in context")
	}

	o := new(BinaryOp)
	o.ctx = ctx
	o.log = log.WithFields(logrus.Fields{
		"Component": "BinaryOperator",
		"Type":      fmt.Sprintf("%T", o),
	})

	o.concurrency = 1
	o.output = make(chan interface{}, 1024)

	o.log.Infof("Component initialized")
	return o
}
Exemple #4
0
func (c *CsvSource) init(ctx context.Context) error {
	// extract logger
	log, ok := autoctx.GetLogEntry(ctx)
	if !ok {
		log = logrus.WithField("Component", "CsvSource")
		log.Error("No logger found incontext")
	}
	c.log = log.WithFields(logrus.Fields{
		"Component": "CsvSource",
		"Type":      fmt.Sprintf("%T", c),
	})

	// establish defaults
	if c.delimChar == 0 {
		c.delimChar = ','
	}

	if c.commentChar == 0 {
		c.commentChar = '#'
	}

	var reader io.Reader

	if c.srcReader != nil {
		reader = c.srcReader
		c.log.Info("Using IO Reader")
	} else {
		// open file
		file, err := os.Open(c.filepath)
		if err != nil {
			return fmt.Errorf("Failed to open file: %s ", err)
		}
		reader = file
		c.file = file
		c.log.Info("Using source file ", file.Name())
	}

	c.csvReader = csv.NewReader(reader)
	c.csvReader.Comment = c.commentChar
	c.csvReader.Comma = c.delimChar
	c.csvReader.TrimLeadingSpace = true
	c.csvReader.LazyQuotes = true

	// resolve header and field count
	if c.hasHeaders {
		if headers, err := c.csvReader.Read(); err == nil {
			c.fieldCount = len(headers)
			c.headers = headers
		} else {
			return fmt.Errorf("Unable to read header row: %s", err)
		}
	} else {
		if c.headers != nil {
			c.fieldCount = len(c.headers)
		}
	}
	log.Debug("hasHeaders ", c.hasHeaders, " headers [", c.headers, "]")
	c.log.Info("Source initializede")

	return nil
}