Ejemplo n.º 1
0
func assembleConfigFiles(configFile string) (map[string]interface{}, error) {
	logs.Info("Reading config file : %s", configFile)
	doc, err := yaml.ReadFile(configFile)
	if err != nil {
		return nil, err
	}

	mapping := YamlUnmarshal(doc.Root).(map[string]interface{})

	confD_path := path.Dir(configFile) + "/conf.d/*.yaml"
	logs.Info("Reading config folder : %s", confD_path)
	entries, err := filepath.Glob(confD_path)
	if err != nil {
		logs.Warn("Can't read relevant conf.d: %s", err)
	} else {
		for _, path := range entries {
			logs.Info("Reading config file : %s", path)
			doc, err := yaml.ReadFile(path)
			if err != nil {
				logs.Warn("Can't read relevant conf.d: %s", err)
			} else {
				inner := YamlUnmarshal(doc.Root).(map[string]interface{})
				RecursiveMergeNoConflict(mapping, inner, "")
			}
		}
	}
	return mapping, nil
}
Ejemplo n.º 2
0
func (*EsbulkapiEncoder) Encode(out map[string]Column, writer io.Writer) {
	var header string

	stripped := make(map[string]interface{})

	for k, v := range out {
		stripped[k] = v.Value
	}

	header = "\{ \"index\" : \{ \"_index\" : \"test\", \"_type\" : \"" + stripped["_group"] + "\} \}"

	bytes := []byte(header)
	bytes = append(bytes, '\n')

	body, err := json.Marshal(stripped)
	if err != nil {
		panic(err)
	}

	bytes = bytes + body
	bytes = append(bytes, '\n')

	logs.Info("Writing bytes to amqp: " + string(bytes[:]))

	writer.Write(bytes)
}
Ejemplo n.º 3
0
func NewAMQPReadWriter(u *url.URL, options []string) (io.ReadWriteCloser, error) {
	var exchange, rkey string

	exchange = options[1]
	rkey = options[2]

	logs.Info("AMQP Exchange: " + exchange + " and routing key: " + rkey)
	logs.Info("AMQP url: " + u.String())

	conn, err := amqp.Dial(u.String())
	if err != nil {
		return nil, err
	}

	channel, err := conn.Channel()
	if err != nil {
		return nil, err
	}

	return &rwStruct{EmptyReader, &AMQPWriter{channel, exchange, rkey}, conn}, nil
}
Ejemplo n.º 4
0
func (tail *Tail) StartWatching() {
	go func() {
		fi, err := tail.Stat()
		if err != nil {
			logs.Error("Can't stat file: %s", err)
			return
		}
		logs.Info("Start Watching: %s ", tail.Path)
		c := tail.Watcher.ChangeEvents(&tomb.Tomb{}, fi)
		go tail.pollChannel(c.Modified)
		go tail.pollChannel(c.Truncated)
		go tail.pollChannel(c.Deleted)
	}()
}
Ejemplo n.º 5
0
func (dests *Destinations) Consume(ch chan Record, finished chan bool) {
	if len(*dests) == 0 {
		logs.Warn("No destinations specified")
	}
	for {
		rec := <-ch

		if rec == nil {
			break
		} else {
			for _, dest := range *dests {
				dest.Encoder.Encode(rec, dest.RW)
			}
		}
	}
	logs.Info("Finished consuming log records")
	finished <- true
}
Ejemplo n.º 6
0
func (er *AMQPWriter) Write(p []byte) (int, error) {
	logs.Info("Publishing to amqp: " + string(p[:]))
	err := er.channel.Publish(
		er.exchange, // publish to an exchange
		er.rkey,     // routing to 0 or more queues
		false,       // mandatory
		false,       // immediate
		amqp.Publishing{
			Headers:         amqp.Table{},
			ContentType:     "text/plain",
			ContentEncoding: "",
			Body:            p[:],
			DeliveryMode:    amqp.Transient, // 1=non-persistent, 2=persistent
			Priority:        0,              // 0-9
			// a bunch of application/implementation-specific fields
		})
	if err != nil {
		panic(err)
	}

	return 0, nil
}
Ejemplo n.º 7
0
func NewRedisReadWriter(u *url.URL, options []string) (io.ReadWriteCloser, error) {
	var method, redisdest string

	switch options[1] {
	case "list":
		method = "LIST"
	case "publish":
		method = "PUBLISH"
	default:
		panic("Unknown redis method...")
	}

	redisdest = options[2]

	logs.Info("Redis method: " + method + " and destination: " + redisdest)

	conn, err := net.Dial("tcp", u.Host)

	if err != nil {
		return nil, err
	}

	return &rwStruct{EmptyReader, &RedisWriter{conn, method, redisdest}, conn}, nil
}
Ejemplo n.º 8
0
func configFromMapping(mapping map[string]interface{}, hostname string) (*Config, error) {
	b, _ := json.Marshal(mapping)
	logs.Debug("mapping: %s", string(b))
	var err error = nil
	config := new(Config)
	config.Sources = make([]SourceConfig, 0)
	config.Destinations = make([]DestinationConfig, 0)

	global, err := getMap(mapping, "global")
	if err != nil {
		return nil, fmt.Errorf("no global section in the config file")
	}

	config.OffsetDir, err = getString(global, "offset_dir")
	if err != nil {
		logs.Warn("no offset_dir specified")
		config.MaxBackfillBytes = -1
	}

	config.MaxBackfillBytes, err = getInt64(global, "max_backfill_bytes")
	if err != nil {
		logs.Warn("no max_backfill_bytes, continuing with unlimited")
		config.MaxBackfillBytes = -1
	}

	config.MaxLineSizeBytes, err = getInt64(global, "max_linesize_bytes")
	if err != nil {
		logs.Warn("no max_linesize_bytes, continuing with 32768")
		config.MaxLineSizeBytes = 32768
	}

	sources, err := getMap(mapping, "sources")
	if err != nil {
		return nil, fmt.Errorf("no sources section in the config file")
	}

	for name, _ := range sources {
		src, err := getMap(sources, name)
		if err != nil {
			logs.Warn("Invalid source: %s, continuing...", name)
			continue
		}

		var source SourceConfig
		source.Hostname = hostname
		source.Fields = make([]FieldConfig, 0)
		source.OffsetDir = config.OffsetDir
		source.MaxBackfillBytes = config.MaxBackfillBytes
		source.MaxLineSizeBytes = config.MaxLineSizeBytes
		source.Name = name
		source.Glob, err = getString(src, "glob")
		if err != nil {
			return nil, err
		}
		source.Pattern, err = getString(src, "pattern")
		if err != nil {
			source.Pattern = DefaultPattern
		}

		_, err = regexp.Compile(source.Pattern)
		if err != nil {
			logs.Warn("%s is not a valid regexp, continuing... (%s)", source.Pattern, err)
			continue
		}

		fields, err := getMap(src, "fields")
		for name, _ := range fields {
			fld, err := getMap(fields, name)
			if err != nil {
				logs.Warn("%s is not a map, continuing... (%s)", name, err)
				continue
			}

			var field FieldConfig
			field.Alias = name

			field.Name, err = getString(fld, "name")
			if err != nil {
				field.Name = field.Alias
			}

			field.Group, err = getInt(fld, "group")

			s, err := getString(fld, "type")
			if err != nil {
				field.Type = String
			} else {
				field.Type, err = parseFieldType(s)
				if err != nil {
					logs.Warn("Invalid field type: %s, continuing... (error was %s)", s, err)
					continue
				}
			}
			logs.Info("found type %s", field.Type)

			s, err = getString(fld, "treatment")
			if err != nil {
				field.Treatment = String
			} else {
				field.Treatment, err = parseFieldTreatment(s)
				if err != nil {
					logs.Warn("Invalid field treatment: %s, continuing... (error was %s)", s, err)
					continue
				}
			}
			logs.Info("found treatment %s", field.Treatment)

			field.Salt, err = getString(fld, "salt")

			field.Format, err = getString(fld, "format")

			s, err = getString(fld, "pattern")
			field.Pattern, err = regexp.Compile(s)
			if err != nil {
				logs.Warn("Invalid regex: %s, continuing... (error was %s)", s, err)
				continue
			}
			source.Fields = append(source.Fields, field)
		}
		config.Sources = append(config.Sources, source)
	}

	destinations, err := getMap(mapping, "destinations")
	if err != nil {
		return nil, fmt.Errorf("no destinations section in the config file")
	}

	for name, _ := range destinations {
		var dest DestinationConfig
		urlString, err := getString(destinations, name)
		u, err := url.Parse(urlString)
		if err != nil {
			logs.Warn("Invalid URL: %s, continuing... (error was %s)", urlString, err)
			continue
		}
		logs.Info("Found destination: %s", urlString)
		dest.Name = name
		dest.Url = u
		config.Destinations = append(config.Destinations, dest)
	}

	return config, nil
}
Ejemplo n.º 9
0
func main() {
	flag.Parse()
	runtime.GOMAXPROCS(*cpus)

	// set the logger path
	handle, err := os.OpenFile(*logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		logs.Warn("Unable to open log file %s, using stderr: %s", *logFile, err)
	} else {
		logs.Logger = log.New(handle, "", log.LstdFlags|log.Lshortfile)
	}

	// Check whether we're in debug mode
	if *debug {
		logs.SetLevel(logs.DEBUG)
		logs.Debug("logging at DEBUG")
	} else {
		logs.SetLevel(logs.INFO)
	}

	if *name == "unknown" {
		*name, err = os.Hostname()
		if err != nil {
			logs.Warn("Unable to determine hostname: %s", err)
		}
	}

	// Read the config files
	config, err := dendrite.NewConfig(*configFile, *name)
	if err != nil {
		logs.Fatal("Can't read configuration: %s", err)
	}

	// Link up all of the objects
	ch := make(chan dendrite.Record, 100)
	logs.Debug("original %s", ch)
	dests := config.CreateDestinations()
	groups := config.CreateAllTailGroups(ch)

	// If any of our destinations talk back, log it.
	go func() {
		reader := bufio.NewReader(dests.Reader())
		for {
			str, err := reader.ReadString('\n')
			if err == io.EOF {
				logs.Debug("eof")
				time.Sleep(1 * time.Second)
			} else if err != nil {
				logs.Error("error reading: %s", err)
			} else {
				logs.Info("received: %s", str)
			}
		}
	}()

	// Do the event loop
	finished := make(chan bool, 0)
	go dests.Consume(ch, finished)
	if *quitAfter >= 0 {
		start := time.Now()
		logs.Debug("starting the poll")
		i := 0
		for {
			i++
			if i%10 == 0 {
				groups.Refresh()
			}
			groups.Poll()
			if time.Now().Sub(start) >= time.Duration((*quitAfter)*float64(time.Second)) {
				break
			}
		}
	} else {
		logs.Debug("starting the loop")
		groups.Loop()
	}
	logs.Info("Closing...")
	close(ch)
	<-finished
	logs.Info("Goodbye!")
}