Exemple #1
0
func New() (*Importer, error) {
	importer := Importer{}

	/* define the command line arguments */
	cl, err := DefineCommandLine()
	if err != nil {
		cl.flagSet.Usage()
		return nil, err
	}
	/* parse command line arguments */
	err = cl.ParseCommandLine()
	if err != nil {
		return nil, err
	}
	importer.cl = cl

	/* prepare the Elasticsearch index pattern */
	fmtstr, err := fmtstr.CompileEvent(cl.opt.Index)
	if err != nil {
		return nil, fmt.Errorf("fail to build the Elasticsearch index pattern: %s", err)
	}
	indexSel := outil.MakeSelector(outil.FmtSelectorExpr(fmtstr, ""))

	/* connect to Elasticsearch */
	client, err := elasticsearch.NewClient(
		elasticsearch.ClientSettings{
			URL:      cl.opt.ES,
			Index:    indexSel,
			Username: cl.opt.User,
			Password: cl.opt.Pass,
			Timeout:  60 * time.Second,
		},
		nil,
	)
	if err != nil {
		return nil, fmt.Errorf("fail to connect to Elasticsearch: %s", err)
	}
	importer.client = client

	return &importer, nil

}
Exemple #2
0
func New() (*Importer, error) {
	importer := Importer{}

	/* define the command line arguments */
	cl, err := DefineCommandLine()
	if err != nil {
		cl.flagSet.Usage()
		return nil, err
	}
	/* parse command line arguments */
	err = cl.ParseCommandLine()
	if err != nil {
		return nil, err
	}
	importer.cl = cl

	/* prepare the Elasticsearch index pattern */
	fmtstr, err := fmtstr.CompileEvent(cl.opt.Index)
	if err != nil {
		return nil, fmt.Errorf("fail to build the Elasticsearch index pattern: %s", err)
	}
	indexSel := outil.MakeSelector(outil.FmtSelectorExpr(fmtstr, ""))

	var tlsConfig outputs.TLSConfig
	var tls *transport.TLSConfig

	if cl.opt.Insecure {
		tlsConfig.VerificationMode = transport.VerifyNone
	}

	if len(cl.opt.Certificate) > 0 && len(cl.opt.CertificateKey) > 0 {
		tlsConfig.Certificate = outputs.CertificateConfig{
			Certificate: cl.opt.Certificate,
			Key:         cl.opt.CertificateKey,
		}
	}

	if len(cl.opt.CertificateAuthority) > 0 {
		tlsConfig.CAs = []string{cl.opt.CertificateAuthority}
	}

	tls, err = outputs.LoadTLSConfig(&tlsConfig)
	if err != nil {
		return nil, fmt.Errorf("fail to load the SSL certificate: %s", err)
	}

	/* connect to Elasticsearch */
	client, err := elasticsearch.NewClient(
		elasticsearch.ClientSettings{
			URL:      cl.opt.ES,
			Index:    indexSel,
			TLS:      tls,
			Username: cl.opt.User,
			Password: cl.opt.Pass,
			Timeout:  60 * time.Second,
		},
		nil,
	)
	if err != nil {
		return nil, fmt.Errorf("fail to connect to Elasticsearch: %s", err)
	}
	importer.client = client

	return &importer, nil

}
Exemple #3
0
func BuildSelectorFromConfig(
	cfg *common.Config,
	settings Settings,
) (Selector, error) {
	var sel []SelectorExpr

	key := settings.Key
	multiKey := settings.MultiKey
	found := false

	if cfg.HasField(multiKey) {
		found = true
		sub, err := cfg.Child(multiKey, -1)
		if err != nil {
			return Selector{}, err
		}

		var table []*common.Config
		if err := sub.Unpack(&table); err != nil {
			return Selector{}, err
		}

		for _, config := range table {
			action, err := buildSingle(config, key)
			if err != nil {
				return Selector{}, err
			}

			if action != nilSelector {
				sel = append(sel, action)
			}
		}
	}

	if settings.EnableSingleOnly && cfg.HasField(key) {
		found = true

		// expect event-format-string
		str, err := cfg.String(key, -1)
		if err != nil {
			return Selector{}, err
		}

		fmtstr, err := fmtstr.CompileEvent(str)
		if err != nil {
			return Selector{}, fmt.Errorf("%v in %v", err, cfg.PathOf(key))
		}

		if fmtstr.IsConst() {
			str, err := fmtstr.Run(common.MapStr{})
			if err != nil {
				return Selector{}, err
			}

			if str != "" {
				sel = append(sel, ConstSelectorExpr(str))
			}
		} else {
			sel = append(sel, FmtSelectorExpr(fmtstr, ""))
		}
	}

	if settings.FailEmpty && !found {
		if settings.EnableSingleOnly {
			return Selector{}, fmt.Errorf("missing required '%v' or '%v' in %v",
				key, multiKey, cfg.Path())
		}

		return Selector{}, fmt.Errorf("missing required '%v' in %v",
			multiKey, cfg.Path())
	}

	return MakeSelector(sel...), nil
}
Exemple #4
0
func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) {
	// TODO: check for unknown fields

	// 1. extract required key-word handler
	if !cfg.HasField(key) {
		return nil, fmt.Errorf("missing %v", cfg.PathOf(key))
	}

	str, err := cfg.String(key, -1)
	if err != nil {
		return nil, err
	}

	evtfmt, err := fmtstr.CompileEvent(str)
	if err != nil {
		return nil, fmt.Errorf("%v in %v", err, cfg.PathOf(key))
	}

	// 2. extract optional `default` value
	var otherwise string
	if cfg.HasField("default") {
		tmp, err := cfg.String("default", -1)
		if err != nil {
			return nil, err
		}
		otherwise = tmp
	}

	// 3. extract optional `mapping`
	mapping := struct {
		Table map[string]string `config:"mappings"`
	}{nil}
	if cfg.HasField("mappings") {
		if err := cfg.Unpack(&mapping); err != nil {
			return nil, err
		}
	}

	// 4. extract conditional
	var cond *processors.Condition
	if cfg.HasField("when") {
		sub, err := cfg.Child("when", -1)
		if err != nil {
			return nil, err
		}

		condConfig := processors.ConditionConfig{}
		if err := sub.Unpack(&condConfig); err != nil {
			return nil, err
		}

		tmp, err := processors.NewCondition(&condConfig)
		if err != nil {
			return nil, err
		}

		cond = tmp
	}

	// 5. build selector from available fields
	var sel SelectorExpr
	if len(mapping.Table) > 0 {
		if evtfmt.IsConst() {
			str, err := evtfmt.Run(common.MapStr{})
			if err != nil {
				return nil, err
			}

			str = mapping.Table[str]
			if str == "" {
				str = otherwise
			}

			if str == "" {
				sel = nilSelector
			} else {
				sel = ConstSelectorExpr(str)
			}
		} else {
			sel = &mapSelector{
				from:      FmtSelectorExpr(evtfmt, ""),
				to:        mapping.Table,
				otherwise: otherwise,
			}
		}
	} else {
		if evtfmt.IsConst() {
			str, err := evtfmt.Run(common.MapStr{})
			if err != nil {
				return nil, err
			}

			if str == "" {
				sel = nilSelector
			} else {
				sel = ConstSelectorExpr(str)
			}
		} else {
			sel = FmtSelectorExpr(evtfmt, otherwise)
		}
	}
	if cond != nil && sel != nilSelector {
		sel = ConditionalSelectorExpr(sel, cond)
	}

	return sel, nil
}