func newTestKafkaClient(t *testing.T, topic string) *client {
hosts := []string{getTestKafkaHost()}
t.Logf("host: %v", hosts)
sel := outil.MakeSelector(outil.ConstSelectorExpr(topic))
client, err := newKafkaClient(hosts, sel, nil)
if err != nil {
t.Fatal(err)
}
return client
}
func newTestClientAuth(url, user, pass string) *Client {
client, err := NewClient(ClientSettings{
URL: url,
Index: outil.MakeSelector(),
Username: user,
Password: pass,
Timeout: 60 * time.Second,
CompressionLevel: 3,
}, nil)
if err != nil {
panic(err)
}
return client
}
func TestGetIndexStandard(t *testing.T) {
time := time.Now().UTC()
extension := fmt.Sprintf("%d.%02d.%02d", time.Year(), time.Month(), time.Day())
event := common.MapStr{
"@timestamp": common.Time(time),
"field": 1,
}
pattern := "beatname-%{+yyyy.MM.dd}"
fmtstr := fmtstr.MustCompileEvent(pattern)
indexSel := outil.MakeSelector(outil.FmtSelectorExpr(fmtstr, ""))
index := getIndex(event, indexSel)
assert.Equal(t, index, "beatname-"+extension)
}
func New() (*Importer, error) {
importer := Importer{}
/* define the command line arguments */
cl, err := DefineCommandLine()
if err != nil {
cl.flagSet.Usage()
return nil, err
}
/* parse command line arguments */
err = cl.ParseCommandLine()
if err != nil {
return nil, err
}
importer.cl = cl
/* prepare the Elasticsearch index pattern */
fmtstr, err := fmtstr.CompileEvent(cl.opt.Index)
if err != nil {
return nil, fmt.Errorf("fail to build the Elasticsearch index pattern: %s", err)
}
indexSel := outil.MakeSelector(outil.FmtSelectorExpr(fmtstr, ""))
/* connect to Elasticsearch */
client, err := elasticsearch.NewClient(
elasticsearch.ClientSettings{
URL: cl.opt.ES,
Index: indexSel,
Username: cl.opt.User,
Password: cl.opt.Pass,
Timeout: 60 * time.Second,
},
nil,
)
if err != nil {
return nil, fmt.Errorf("fail to connect to Elasticsearch: %s", err)
}
importer.client = client
return &importer, nil
}
func esConnect(t *testing.T, index string) *esConnection {
ts := time.Now().UTC()
host := getElasticsearchHost()
indexFmt := fmtstr.MustCompileEvent(fmt.Sprintf("%s-%%{+yyyy.MM.dd}", index))
indexSel := outil.MakeSelector(outil.FmtSelectorExpr(indexFmt, ""))
index, _ = indexSel.Select(common.MapStr{
"@timestamp": common.Time(ts),
})
username := os.Getenv("ES_USER")
password := os.Getenv("ES_PASS")
client, err := elasticsearch.NewClient(elasticsearch.ClientSettings{
URL: host,
Index: indexSel,
Username: username,
Password: password,
Timeout: 60 * time.Second,
}, nil)
if err != nil {
t.Fatal(err)
}
// try to drop old index if left over from failed test
_, _, _ = client.Delete(index, "", "", nil) // ignore error
_, _, err = client.CreateIndex(index, common.MapStr{
"settings": common.MapStr{
"number_of_shards": 1,
"number_of_replicas": 0,
},
})
if err != nil {
t.Fatalf("failed to create test index: %s", err)
}
es := &esConnection{}
es.t = t
es.Client = client
es.index = index
return es
}
func New() (*Importer, error) {
importer := Importer{}
/* define the command line arguments */
cl, err := DefineCommandLine()
if err != nil {
cl.flagSet.Usage()
return nil, err
}
/* parse command line arguments */
err = cl.ParseCommandLine()
if err != nil {
return nil, err
}
importer.cl = cl
/* prepare the Elasticsearch index pattern */
fmtstr, err := fmtstr.CompileEvent(cl.opt.Index)
if err != nil {
return nil, fmt.Errorf("fail to build the Elasticsearch index pattern: %s", err)
}
indexSel := outil.MakeSelector(outil.FmtSelectorExpr(fmtstr, ""))
var tlsConfig outputs.TLSConfig
var tls *transport.TLSConfig
if cl.opt.Insecure {
tlsConfig.VerificationMode = transport.VerifyNone
}
if len(cl.opt.Certificate) > 0 && len(cl.opt.CertificateKey) > 0 {
tlsConfig.Certificate = outputs.CertificateConfig{
Certificate: cl.opt.Certificate,
Key: cl.opt.CertificateKey,
}
}
if len(cl.opt.CertificateAuthority) > 0 {
tlsConfig.CAs = []string{cl.opt.CertificateAuthority}
}
tls, err = outputs.LoadTLSConfig(&tlsConfig)
if err != nil {
return nil, fmt.Errorf("fail to load the SSL certificate: %s", err)
}
/* connect to Elasticsearch */
client, err := elasticsearch.NewClient(
elasticsearch.ClientSettings{
URL: cl.opt.ES,
Index: indexSel,
TLS: tls,
Username: cl.opt.User,
Password: cl.opt.Pass,
Timeout: 60 * time.Second,
},
nil,
)
if err != nil {
return nil, fmt.Errorf("fail to connect to Elasticsearch: %s", err)
}
importer.client = client
return &importer, nil
}