// New creates new instance of MetricSet.
func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
// Additional configuration options
config := struct {
ServerStatusPath string `config:"server_status_path"`
Username string `config:"username"`
Password string `config:"password"`
}{
ServerStatusPath: defaultPath,
Username: "",
Password: "",
}
if err := base.Module().UnpackConfig(&config); err != nil {
return nil, err
}
metricSet := &MetricSet{
BaseMetricSet: base,
URLs: make(map[string]string, len(base.Module().Config().Hosts)),
}
// Parse the config, create URLs, and check for errors.
var errs multierror.Errors
for _, host := range base.Module().Config().Hosts {
u, err := getURL(config.Username, config.Password, config.ServerStatusPath, host)
if err != nil {
errs = append(errs, err)
continue
}
metricSet.URLs[host] = u.String()
debugf("apache-status URL=%s", redactPassword(*u))
}
return metricSet, errs.Err()
}
// Validate validates the Settings data and returns an error describing
// all problems or nil if there are none.
func (s Settings) Validate() error {
// TODO: winlogbeat should not try to validate top-level beats config
validKeys := []string{
"fields", "fields_under_root", "tags",
"name", "refresh_topology_freq", "ignore_outgoing", "topology_expire", "geoip",
"queue_size", "bulk_queue_size", "max_procs",
"processors", "logging", "output", "path", "winlogbeat",
}
sort.Strings(validKeys)
// Check for invalid top-level keys.
var errs multierror.Errors
for k := range s.Raw {
k = strings.ToLower(k)
i := sort.SearchStrings(validKeys, k)
if i >= len(validKeys) || validKeys[i] != k {
errs = append(errs, fmt.Errorf("Invalid top-level key '%s' "+
"found. Valid keys are %s", k, strings.Join(validKeys, ", ")))
}
}
err := s.Winlogbeat.Validate()
if err != nil {
errs = append(errs, err)
}
return errs.Err()
}
// uninstallLog unregisters the event logger from the registry and closes the
// log's handle if it is open.
func uninstallLog(provider, source string, log *elog.Log) error {
var errs multierror.Errors
if log != nil {
err := clearEventLog(Handle(log.Handle), "")
if err != nil {
errs = append(errs, err)
}
err = log.Close()
if err != nil {
errs = append(errs, err)
}
}
err := elog.RemoveSource(providerName, sourceName)
if err != nil {
errs = append(errs, err)
}
err = elog.RemoveProvider(providerName)
if err != nil {
errs = append(errs, err)
}
return errs.Err()
}
// initLog initializes an event logger. It registers the source name with
// the registry if it does not already exist.
func initLog(provider, source, msgFile string) (*elog.Log, error) {
// Install entry to registry:
_, err := elog.Install(providerName, sourceName, msgFile, true, allLevels)
if err != nil {
return nil, err
}
// Open a new logger for writing events:
log, err := elog.Open(sourceName)
if err != nil {
var errs multierror.Errors
errs = append(errs, err)
err := elog.RemoveSource(providerName, sourceName)
if err != nil {
errs = append(errs, err)
}
err = elog.RemoveProvider(providerName)
if err != nil {
errs = append(errs, err)
}
return nil, errs.Err()
}
return log, nil
}
// Setup the URLs to the mod_status endpoints.
func (m *MetricSeter) Setup(ms *helper.MetricSet) error {
// Additional configuration options
config := struct {
ServerStatusPath string `config:"server_status_path"`
Username string `config:"username"`
Password string `config:"password"`
}{
ServerStatusPath: defaultPath,
Username: "",
Password: "",
}
if err := ms.Module.ProcessConfig(&config); err != nil {
return err
}
m.URLs = make(map[string]string, len(ms.Config.Hosts))
// Parse the config, create URLs, and check for errors.
var errs multierror.Errors
for _, host := range ms.Config.Hosts {
u, err := getURL(config.Username, config.Password, config.ServerStatusPath, host)
if err != nil {
errs = append(errs, err)
continue
}
m.URLs[host] = u.String()
debugf("apache-status URL=%s", redactPassword(*u))
}
return errs.Err()
}
// Validate validates the winEventLogConfig data and returns an error describing
// any problems or nil.
func (c *winEventLogConfig) Validate() error {
var errs multierror.Errors
if c.Name == "" {
errs = append(errs, fmt.Errorf("event log is missing a 'name'"))
}
return errs.Err()
}
// UpdateAll remote repositories
func UpdateAll() error {
var Errors multierror.Errors
for _, repository := range *allRepositories {
err := Update(repository.Name)
if err != nil {
Errors = append(Errors, err)
}
}
return Errors.Err()
}
// newModuleWrappers creates new Modules and their associated MetricSets based
// on the given configuration. It constructs the supporting filters and
// publisher client and stores it all in a moduleWrapper.
func newModuleWrappers(
modulesConfig []*common.Config,
r *mb.Register,
publisher *publisher.Publisher,
) ([]*moduleWrapper, error) {
modules, err := mb.NewModules(modulesConfig, r)
if err != nil {
return nil, err
}
// Wrap the Modules and MetricSet's.
var wrappers []*moduleWrapper
var errs multierror.Errors
for k, v := range modules {
debugf("initializing Module type %s, %T=%+v", k.Name(), k, k)
f, err := filter.New(k.Config().Filters)
if err != nil {
errs = append(errs, errors.Wrapf(err, "module %s", k.Name()))
continue
}
mw := &moduleWrapper{
Module: k,
filters: f,
pubClient: publisher.Connect(),
}
wrappers = append(wrappers, mw)
msws := make([]*metricSetWrapper, 0, len(v))
for _, ms := range v {
debugf("initializing MetricSet type %s/%s, %T=%+v",
ms.Module().Name(), ms.Name(), ms, ms)
msw := &metricSetWrapper{
MetricSet: ms,
module: mw,
stats: new(expvar.Map).Init(),
}
msws = append(msws, msw)
// Initialize expvar stats for this MetricSet.
fetches.Set(fmt.Sprintf("%s-%s", mw.Name(), msw.Name()), msw.stats)
msw.stats.Add(successesKey, 0)
msw.stats.Add(failuresKey, 0)
msw.stats.Add(eventsKey, 0)
}
mw.metricSets = msws
}
return wrappers, errs.Err()
}
// Validate validates the WinlogbeatConfig data and returns an error describing
// all problems or nil if there are none.
func (ebc WinlogbeatConfig) Validate() error {
var errs multierror.Errors
if len(ebc.EventLogs) == 0 {
errs = append(errs, fmt.Errorf("At least one event log must be "+
"configured as part of event_logs"))
}
if err := ebc.Metrics.Validate(); err != nil {
errs = append(errs, err)
}
return errs.Err()
}
// LoadTLSConfig will load a certificate from config with all TLS based keys
// defined. If Certificate and CertificateKey are configured, client authentication
// will be configured. If no CAs are configured, the host CA will be used by go
// built-in TLS support.
func LoadTLSConfig(config *TLSConfig) (*transport.TLSConfig, error) {
if !config.IsEnabled() {
return nil, nil
}
fail := multierror.Errors{}
logFail := func(es ...error) {
for _, e := range es {
if e != nil {
fail = append(fail, e)
}
}
}
var cipherSuites []uint16
for _, suite := range config.CipherSuites {
cipherSuites = append(cipherSuites, uint16(suite))
}
var curves []tls.CurveID
for _, id := range config.CurveTypes {
curves = append(curves, tls.CurveID(id))
}
cert, err := loadCertificate(&config.Certificate)
logFail(err)
cas, errs := loadCertificateAuthorities(config.CAs)
logFail(errs...)
// fail, if any error occurred when loading certificate files
if err = fail.Err(); err != nil {
return nil, err
}
var certs []tls.Certificate
if cert != nil {
certs = []tls.Certificate{*cert}
}
// return config if no error occurred
return &transport.TLSConfig{
Versions: config.Versions,
Verification: config.VerificationMode,
Certificates: certs,
RootCAs: cas,
CipherSuites: cipherSuites,
CurvePreferences: curves,
}, nil
}
// newBaseModulesFromConfig creates new BaseModules from a list of configs
// each containing ModuleConfig data.
func newBaseModulesFromConfig(config []*common.Config) ([]BaseModule, error) {
var errs multierror.Errors
baseModules := make([]BaseModule, 0, len(config))
for _, rawConfig := range config {
bm, err := newBaseModuleFromConfig(rawConfig)
if err != nil {
errs = append(errs, err)
continue
}
if bm.config.Enabled {
baseModules = append(baseModules, bm)
}
}
return baseModules, errs.Err()
}
// NewModuleWrappers creates new Modules and their associated MetricSets based
// on the given configuration. It constructs the supporting filters and stores
// them all in a ModuleWrapper.
func NewModuleWrappers(modulesConfig []*common.Config, r *mb.Register) ([]*ModuleWrapper, error) {
modules, err := mb.NewModules(modulesConfig, r)
if err != nil {
return nil, err
}
// Wrap the Modules and MetricSet's.
var wrappers []*ModuleWrapper
var errs multierror.Errors
for k, v := range modules {
debugf("Initializing Module type '%s': %T=%+v", k.Name(), k, k)
f, err := filter.New(k.Config().Filters)
if err != nil {
errs = append(errs, errors.Wrapf(err, "module %s", k.Name()))
continue
}
mw := &ModuleWrapper{
Module: k,
filters: f,
}
wrappers = append(wrappers, mw)
msws := make([]*metricSetWrapper, 0, len(v))
for _, ms := range v {
debugf("Initializing MetricSet type '%s/%s' for host '%s': %T=%+v",
ms.Module().Name(), ms.Name(), ms.Host(), ms, ms)
expMap, err := getMetricSetExpvarMap(mw.Name(), ms.Name())
if err != nil {
return nil, err
}
msw := &metricSetWrapper{
MetricSet: ms,
module: mw,
stats: expMap,
}
msws = append(msws, msw)
}
mw.metricSets = msws
}
return wrappers, errs.Err()
}
// Validate validates the eventLoggingConfig data and returns an error
// describing any problems or nil.
func (c *eventLoggingConfig) Validate() error {
var errs multierror.Errors
if c.Name == "" {
errs = append(errs, fmt.Errorf("event log is missing a 'name'"))
}
if c.ReadBufferSize > win.MaxEventBufferSize {
errs = append(errs, fmt.Errorf("'read_buffer_size' must be less than "+
"%d bytes", win.MaxEventBufferSize))
}
if c.FormatBufferSize > win.MaxFormatMessageBufferSize {
errs = append(errs, fmt.Errorf("'format_buffer_size' must be less than "+
"%d bytes", win.MaxFormatMessageBufferSize))
}
return errs.Err()
}
func Process(spec interface{}) error {
var el multierror.Errors
e := func(err error) {
if err != nil {
el = append(el, err)
}
}
if conf.Settings.UseDB {
e(db.Process(spec))
}
if conf.Settings.UseFile {
e(file.Process(spec))
}
if conf.Settings.UseEnv {
e(env.Process(conf.Settings.AppName, spec))
}
return el.Err()
}
// Process takes a spec and a list of files. The list of files is in
// **reverse** order of precedence, meaning that a value in the first file can
// be overridden by any subsequent file.
//
// Example:
// Process(mySpec, "/etc/myapp.json", "/usr/local/etc/myapp.json", "/home/me/.myapp.json")
// values in /home/me/.myapp.json will override any values set in /etc/myapp.json
func Process(spec interface{}) error {
var el multierror.Errors
res := map[string]interface{}{}
for _, fname := range conf.Settings.FileLocations {
contents, err := ioutil.ReadFile(fname)
if err != nil {
el = append(el, err)
continue
}
json.Unmarshal(contents, &res)
}
if err := merger.MapAndStruct(res, spec); err != nil {
el = append(el, err)
}
return el.Err()
}
func (d *DockerDaemon) KillAllContainers() error {
containers, err := d.Client.ListContainers(dockerapi.ListContainersOptions{})
if err != nil {
return err
}
var errs multierror.Errors
for _, container := range containers {
err = d.Client.KillContainer(dockerapi.KillContainerOptions{
ID: container.ID,
})
if err != nil {
errs = append(errs, err)
}
}
return errs.Err()
}
// Validate validates the Settings data and returns an error describing
// all problems or nil if there are none.
func (s Settings) Validate() error {
validKeys := []string{"filters", "logging", "output", "shipper", "winlogbeat"}
sort.Strings(validKeys)
// Check for invalid top-level keys.
var errs multierror.Errors
for k := range s.Raw {
k = strings.ToLower(k)
i := sort.SearchStrings(validKeys, k)
if i >= len(validKeys) || validKeys[i] != k {
errs = append(errs, fmt.Errorf("Invalid top-level key '%s' "+
"found. Valid keys are %s", k, strings.Join(validKeys, ", ")))
}
}
err := s.Winlogbeat.Validate()
if err != nil {
errs = append(errs, err)
}
return errs.Err()
}
func createModules(r *Register, baseModules []BaseModule) ([]Module, error) {
modules := make([]Module, 0, len(baseModules))
var errs multierror.Errors
for _, bm := range baseModules {
f := r.moduleFactory(bm.Name())
if f == nil {
f = DefaultModuleFactory
}
module, err := f(bm)
if err != nil {
errs = append(errs, err)
continue
}
modules = append(modules, module)
}
err := errs.Err()
if err != nil {
return nil, err
}
return modules, nil
}
// Build builds a query from the given parameters. The query is returned as a
// XML string and can be used with Subscribe function.
func (q Query) Build() (string, error) {
var errs multierror.Errors
if q.Log == "" {
errs = append(errs, fmt.Errorf("empty log name"))
}
qp := &queryParams{Path: q.Log}
builders := []func(Query) error{
qp.ignoreOlderSelect,
qp.eventIDSelect,
qp.levelSelect,
qp.providerSelect,
}
for _, build := range builders {
if err := build(q); err != nil {
errs = append(errs, err)
}
}
if len(errs) > 0 {
return "", errs.Err()
}
return executeTemplate(queryTemplate, qp)
}
func initMetricSets(r *Register, modules []Module) (map[Module][]MetricSet, error) {
active := map[Module][]MetricSet{}
var errs multierror.Errors
for _, bms := range newBaseMetricSets(modules) {
f, err := r.metricSetFactory(bms.Module().Name(), bms.Name())
if err != nil {
errs = append(errs, err)
continue
}
metricSet, err := f(bms)
if err != nil {
errs = append(errs, err)
continue
}
err = mustImplementFetcher(metricSet)
if err != nil {
errs = append(errs, err)
continue
}
err = mustHaveModule(metricSet, bms)
if err != nil {
errs = append(errs, err)
continue
}
module := metricSet.Module()
active[module] = append(active[module], metricSet)
}
err := errs.Err()
if err != nil {
return nil, err
}
return active, nil
}
func readConfig(
data map[string]interface{},
config interface{},
validKeys []string,
) error {
c, err := common.NewConfigFrom(data)
if err != nil {
return fmt.Errorf("Failed reading config. %v", err)
}
if err := c.Unpack(config); err != nil {
return fmt.Errorf("Failed unpacking config. %v", err)
}
var errs multierror.Errors
if len(validKeys) > 0 {
sort.Strings(validKeys)
// Check for invalid keys.
for k := range data {
k = strings.ToLower(k)
i := sort.SearchStrings(validKeys, k)
if i >= len(validKeys) || validKeys[i] != k {
errs = append(errs, fmt.Errorf("Invalid event log key '%s' "+
"found. Valid keys are %s", k, strings.Join(validKeys, ", ")))
}
}
}
if v, ok := config.(validator); ok {
if err := v.Validate(); err != nil {
errs = append(errs, err)
}
}
return errs.Err()
}
// Validate validates the EventLogConfig data and returns an error describing
// any problems or nil.
func (elc EventLogConfig) Validate() error {
var errs multierror.Errors
if elc.Name == "" {
err := fmt.Errorf("event log is missing a 'name'")
errs = append(errs, err)
}
if _, err := IgnoreOlderDuration(elc.IgnoreOlder); err != nil {
err := fmt.Errorf("Invalid ignore_older value ('%s') for event_log "+
"'%s' (%v)", elc.IgnoreOlder, elc.Name, err)
errs = append(errs, err)
}
switch strings.ToLower(elc.API) {
case "", "eventlogging", "wineventlog":
break
default:
err := fmt.Errorf("Invalid api value ('%s') for event_log '%s'",
elc.API, elc.Name)
errs = append(errs, err)
}
return errs.Err()
}
// Validates the WinlogbeatConfig data and returns an error describing all
// problems or nil if there are none.
func (ebc WinlogbeatConfig) Validate() error {
var errs multierror.Errors
if _, err := IgnoreOlderDuration(ebc.IgnoreOlder); err != nil {
errs = append(errs, fmt.Errorf("Invalid top level ignore_older value "+
"'%s' (%v)", ebc.IgnoreOlder, err))
}
if len(ebc.EventLogs) == 0 {
errs = append(errs, fmt.Errorf("At least one event log must be "+
"configured as part of event_logs"))
}
for _, eventLog := range ebc.EventLogs {
if err := eventLog.Validate(); err != nil {
errs = append(errs, err)
}
}
if err := ebc.Metrics.Validate(); err != nil {
errs = append(errs, err)
}
return errs.Err()
}
// NewModules builds new Modules and their associated MetricSets based on the
// provided configuration data. config is a list module config data (the data
// will be unpacked into ModuleConfig structs). r is the Register where the
// ModuleFactory's and MetricSetFactory's will be obtained from. This method
// returns a mapping of Modules to MetricSets or an error.
func NewModules(config []*common.Config, r *Register) (map[Module][]MetricSet, error) {
if config == nil || len(config) == 0 {
return nil, ErrEmptyConfig
}
baseModules, err := newBaseModulesFromConfig(config)
if err != nil {
return nil, err
}
// Create new Modules using the registered ModuleFactory's
modules := make([]Module, 0, len(baseModules))
var errs multierror.Errors
for _, bm := range baseModules {
f := r.moduleFactory(bm.Name())
if f == nil {
f = DefaultModuleFactory
}
module, err := f(bm)
if err != nil {
errs = append(errs, err)
continue
}
modules = append(modules, module)
}
if errs.Err() != nil {
return nil, errs.Err()
}
// Create new MetricSets for each Module using the registered MetricSetFactory's
modToMetricSets := make(map[Module][]MetricSet, len(modules))
for _, bms := range newBaseMetricSets(modules) {
f, err := r.metricSetFactory(bms.Module().Name(), bms.Name())
if err != nil {
errs = append(errs, err)
continue
}
metricSet, err := f(bms)
if err != nil {
errs = append(errs, err)
continue
}
err = mustImplementFetcher(metricSet)
if err != nil {
errs = append(errs, err)
continue
}
err = mustHaveModule(metricSet, bms)
if err != nil {
errs = append(errs, err)
continue
}
if list, ok := modToMetricSets[metricSet.Module()]; ok {
list = append(list, metricSet)
modToMetricSets[metricSet.Module()] = list
continue
}
modToMetricSets[metricSet.Module()] = []MetricSet{metricSet}
}
if errs.Err() != nil {
return nil, errs.Err()
}
if len(modToMetricSets) == 0 {
return nil, ErrAllModulesDisabled
}
debugf("mb.NewModules() is returning %s", modToMetricSets)
return modToMetricSets, nil
}