forked from shinji62/firehose-to-fluentd
/
main.go
131 lines (110 loc) · 5.7 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
package main
import (
"fmt"
"github.com/boltdb/bolt"
"github.com/cloudfoundry-community/go-cfclient"
"github.com/pkg/profile"
"github.com/shinji62/firehose-to-fluentd/caching"
"github.com/shinji62/firehose-to-fluentd/events"
"github.com/shinji62/firehose-to-fluentd/extrafields"
"github.com/shinji62/firehose-to-fluentd/firehose"
"github.com/shinji62/firehose-to-fluentd/logging"
"gopkg.in/alecthomas/kingpin.v2"
"log"
"os"
"time"
)
var (
debug = kingpin.Flag("debug", "Enable debug mode. This disables forwarding to syslog").Default("false").OverrideDefaultFromEnvar("DEBUG").Bool()
apiEndpoint = kingpin.Flag("api-endpoint", "Api endpoint address. For bosh-lite installation of CF: https://api.10.244.0.34.xip.io").OverrideDefaultFromEnvar("API_ENDPOINT").Required().String()
dopplerEndpoint = kingpin.Flag("doppler-endpoint", "Overwrite default doppler endpoint return by /v2/info").OverrideDefaultFromEnvar("DOPPLER_ENDPOINT").String()
fluentdServer = kingpin.Flag("fluentd-server", "fluentd server.").OverrideDefaultFromEnvar("FLUENTD_ENDPOINT").String()
subscriptionId = kingpin.Flag("subscription-id", "Id for the subscription.").Default("firehose").OverrideDefaultFromEnvar("FIREHOSE_SUBSCRIPTION_ID").String()
user = kingpin.Flag("user", "Admin user.").Default("admin").OverrideDefaultFromEnvar("FIREHOSE_USER").String()
password = kingpin.Flag("password", "Admin password.").Default("admin").OverrideDefaultFromEnvar("FIREHOSE_PASSWORD").String()
skipSSLValidation = kingpin.Flag("skip-ssl-validation", "Please don't").Default("false").OverrideDefaultFromEnvar("SKIP_SSL_VALIDATION").Bool()
logEventTotals = kingpin.Flag("log-event-totals", "Logs the counters for all selected events since nozzle was last started.").Default("false").OverrideDefaultFromEnvar("LOG_EVENT_TOTALS").Bool()
logEventTotalsTime = kingpin.Flag("log-event-totals-time", "How frequently the event totals are calculated (in sec).").Default("30s").OverrideDefaultFromEnvar("LOG_EVENT_TOTALS_TIME").Duration()
wantedEvents = kingpin.Flag("events", fmt.Sprintf("Comma separated list of events you would like. Valid options are %s", events.GetListAuthorizedEventEvents())).Default("LogMessage").OverrideDefaultFromEnvar("EVENTS").String()
boltDatabasePath = kingpin.Flag("boltdb-path", "Bolt Database path ").Default("my.db").OverrideDefaultFromEnvar("BOLTDB_PATH").String()
tickerTime = kingpin.Flag("cc-pull-time", "CloudController Polling time in sec").Default("60s").OverrideDefaultFromEnvar("CF_PULL_TIME").Duration()
extraFields = kingpin.Flag("extra-fields", "Extra fields you want to annotate your events with, example: '--extra-fields=env:dev,something:other ").Default("").OverrideDefaultFromEnvar("EXTRA_FIELDS").String()
modeProf = kingpin.Flag("mode-prof", "Enable profiling mode, one of [cpu, mem, block]").Default("").OverrideDefaultFromEnvar("MODE_PROF").String()
pathProf = kingpin.Flag("path-prof", "Set the Path to write profiling file").Default("").OverrideDefaultFromEnvar("PATH_PROF").String()
)
const (
version = "0.0.1"
)
func main() {
kingpin.Version(version)
kingpin.Parse()
logging.LogStd(fmt.Sprintf("Starting firehose-to-fluentd %s ", version), true)
logging.SetupLogging(*fluentdServer, *debug)
c := cfclient.Config{
ApiAddress: *apiEndpoint,
Username: *user,
Password: *password,
SkipSslValidation: *skipSSLValidation,
}
cfClient := cfclient.NewClient(&c)
if len(*dopplerEndpoint) > 0 {
cfClient.Endpoint.DopplerEndpoint = *dopplerEndpoint
}
logging.LogStd(fmt.Sprintf("Using %s as doppler endpoint", cfClient.Endpoint.DopplerEndpoint), true)
logging.LogStd("Setting up event routing!", true)
err := events.SetupEventRouting(*wantedEvents)
if err != nil {
log.Fatal("Error setting up event routing: ", err)
os.Exit(1)
}
//Use bolt for in-memory - file caching
db, err := bolt.Open(*boltDatabasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
if err != nil {
log.Fatal("Error opening bolt db: ", err)
os.Exit(1)
}
defer db.Close()
if *modeProf != "" {
switch *modeProf {
case "cpu":
defer profile.Start(profile.CPUProfile, profile.ProfilePath(*pathProf)).Stop()
case "mem":
defer profile.Start(profile.MemProfile, profile.ProfilePath(*pathProf)).Stop()
case "block":
defer profile.Start(profile.BlockProfile, profile.ProfilePath(*pathProf)).Stop()
default:
// do nothing
}
}
caching.SetCfClient(cfClient)
caching.SetAppDb(db)
caching.CreateBucket()
//Let's Update the database the first time
logging.LogStd("Start filling app/space/org cache.", true)
apps := caching.GetAllApp()
logging.LogStd(fmt.Sprintf("Done filling cache! Found [%d] Apps", len(apps)), true)
//Let's start the goRoutine
caching.PerformPoollingCaching(*tickerTime)
caching.PerformStat()
// Parse extra fields from cmd call
extraFields, err := extrafields.ParseExtraFields(*extraFields)
if err != nil {
log.Fatal("Error parsing extra fields: ", err)
os.Exit(1)
}
if *logEventTotals == true {
events.LogEventTotals(*logEventTotalsTime, *dopplerEndpoint)
}
if logging.Connect() || *debug {
logging.LogStd("Connected to Fluentd Server! Connecting to Firehose...", true)
firehose := firehose.CreateFirehoseChan(cfClient.Endpoint.DopplerEndpoint, cfClient.GetToken(), *subscriptionId, *skipSSLValidation)
if firehose != nil {
logging.LogStd("Firehose Subscription Succesfull! Routing events...", true)
events.RouteEvents(firehose, extraFields)
} else {
logging.LogError("Failed connecting to Firehose...Please check settings and try again!", "")
}
} else {
logging.LogError("Failed connecting to the Fluentd Server...Please check settings and try again!", "")
}
}