func LoadGeoIPData(config Geoip) *libgeo.GeoIP { geoip_paths := []string{ "/usr/share/GeoIP/GeoIP.dat", "/usr/local/var/GeoIP/GeoIP.dat", } if config.Paths != nil { geoip_paths = *config.Paths } if len(geoip_paths) == 0 { // disabled return nil } // look for the first existing path var geoip_path string for _, path := range geoip_paths { fi, err := os.Lstat(path) if err != nil { continue } if fi.Mode()&os.ModeSymlink == os.ModeSymlink { // follow symlink geoip_path, err = filepath.EvalSymlinks(path) if err != nil { logp.Warn("Could not load GeoIP data: %s", err.Error()) return nil } } else { geoip_path = path } break } if len(geoip_path) == 0 { logp.Warn("Couldn't load GeoIP database") return nil } geoLite, err := libgeo.Load(geoip_path) if err != nil { logp.Warn("Could not load GeoIP data: %s", err.Error()) } logp.Info("Loaded GeoIP data from: %s", geoip_path) return geoLite }
func (out *KafkaOutput) Reconnect() { for { err := out.Connect() if err != nil { logp.Warn("Error connecting to Kafka (%s). Retrying in %s", err, out.ReconnectInterval) time.Sleep(out.ReconnectInterval) } else { break } } }
// Create an HTTP request and send it to Elasticsearch. The request is retransmitted max_retries // before returning an error. func (es *Elasticsearch) Request(method string, path string, params map[string]string, body interface{}) ([]byte, error) { var errors []error for attempt := 0; attempt < es.MaxRetries; attempt++ { conn := es.connectionPool.GetConnection() logp.Debug("elasticsearch", "Use connection %s", conn.Url) url := conn.Url + path if len(params) > 0 { url = url + "?" + UrlEncode(params) } logp.Debug("elasticsearch", "%s %s %s", method, url, body) var obj []byte var err error if body != nil { obj, err = json.Marshal(body) if err != nil { return nil, fmt.Errorf("Fail to JSON encode the body: %s", err) } } else { obj = nil } req, err := http.NewRequest(method, url, bytes.NewReader(obj)) if err != nil { return nil, fmt.Errorf("NewRequest fails: %s", err) } resp, retry, err := es.PerformRequest(conn, req) if retry == true { // retry if err != nil { errors = append(errors, err) } continue } if err != nil { return nil, err } return resp, nil } logp.Warn("Request fails to be send after %d retries", es.MaxRetries) return nil, fmt.Errorf("Request fails after %d retries. Errors: %v", es.MaxRetries, errors) }
// Create a HTTP request containing a bunch of operations and send them to Elasticsearch. // The request is retransmitted up to max_retries before returning an error. func (es *Elasticsearch) BulkRequest(method string, path string, params map[string]string, body chan interface{}) ([]byte, error) { var buf bytes.Buffer enc := json.NewEncoder(&buf) for obj := range body { enc.Encode(obj) } if buf.Len() == 0 { logp.Debug("elasticsearch", "Empty channel. Wait for more data.") return nil, nil } var errors []error for attempt := 0; attempt < es.MaxRetries; attempt++ { conn := es.connectionPool.GetConnection() logp.Debug("elasticsearch", "Use connection %s", conn.Url) url := conn.Url + path if len(params) > 0 { url = url + "?" + UrlEncode(params) } logp.Debug("elasticsearch", "Sending bulk request to %s", url) req, err := http.NewRequest(method, url, &buf) if err != nil { return nil, fmt.Errorf("NewRequest fails: %s", err) } resp, retry, err := es.PerformRequest(conn, req) if retry == true { // retry if err != nil { errors = append(errors, err) } continue } if err != nil { return nil, fmt.Errorf("PerformRequest fails: %s", err) } return resp, nil } logp.Warn("Request fails to be send after %d retries", es.MaxRetries) return nil, fmt.Errorf("Request fails after %d retries. Errors: %v", es.MaxRetries, errors) }
func (publisher *PublisherType) Init(outputs map[string]outputs.MothershipConfig, shipper ShipperConfig) error { var err error publisher.IgnoreOutgoing = shipper.Ignore_outgoing publisher.disabled = *publishDisabled if publisher.disabled { logp.Info("Dry run mode. All output types except the file based one are disabled.") } publisher.GeoLite = common.LoadGeoIPData(shipper.Geoip) for outputId, plugin := range EnabledOutputPlugins { outputName := outputId.String() output, exists := outputs[outputName] if exists && output.Enabled && !publisher.disabled { err := plugin.Init(output, shipper.Topology_expire) if err != nil { logp.Err("Fail to initialize %s plugin as output: %s", outputName, err) return err } publisher.Output = append(publisher.Output, plugin) if output.Save_topology { if publisher.TopologyOutput != nil { logp.Err("Multiple outputs defined to store topology. Please add save_topology = true option only for one output.") return errors.New("Multiple outputs defined to store topology") } publisher.TopologyOutput = plugin logp.Info("Using %s to store the topology", outputName) } } } if !publisher.disabled { if len(publisher.Output) == 0 { logp.Info("No outputs are defined. Please define one under the shipper->output section.") return errors.New("No outputs are defined. Please define one under the shipper->output section.") } if publisher.TopologyOutput == nil { logp.Warn("No output is defined to store the topology. The server fields might not be filled.") } } publisher.name = shipper.Name if len(publisher.name) == 0 { // use the hostname publisher.name, err = os.Hostname() if err != nil { return err } logp.Info("No shipper name configured, using hostname '%s'", publisher.name) } publisher.tags = shipper.Tags if !publisher.disabled && publisher.TopologyOutput != nil { RefreshTopologyFreq := 10 * time.Second if shipper.Refresh_topology_freq != 0 { RefreshTopologyFreq = time.Duration(shipper.Refresh_topology_freq) * time.Second } publisher.RefreshTopologyTimer = time.Tick(RefreshTopologyFreq) logp.Info("Topology map refreshed every %s", RefreshTopologyFreq) // register shipper and its public IP addresses err = publisher.PublishTopology() if err != nil { logp.Err("Failed to publish topology: %s", err) return err } // update topology periodically go publisher.UpdateTopologyPeriodically() } publisher.Queue = make(chan common.MapStr, 10000) go publisher.publishFromQueue() return nil }