func processPing(c <-chan Ping) error {
var err error
var carbonReceiver *carbon.Carbon
var ic influxdbclient.Client
switch receiverTypeFlag {
case "carbon":
carbonReceiver, err = carbon.NewCarbon(receiverHostFlag, receiverPortFlag, receiverNoopFlag, verboseFlag)
case "influxdb":
if receiverDatabaseFlag == "" {
log.Fatalln("An InfluxDB database was not specified on the command line.")
}
if len(receiverUsernameFlag) > 0 {
ic, err = influxdbclient.NewHTTPClient(
influxdbclient.HTTPConfig{
Addr: fmt.Sprintf("http://%v:%v", receiverHostFlag, receiverPortFlag),
Username: receiverUsernameFlag,
Password: receiverPasswordFlag,
})
} else {
ic, err = influxdbclient.NewHTTPClient(
influxdbclient.HTTPConfig{
Addr: fmt.Sprintf("http://%v:%v", receiverHostFlag, receiverPortFlag),
})
}
}
if err != nil {
log.Println("error in creating a connection, but ignoring")
// return err
}
for {
pingResult := <-c
if !isReceiverFullyDefined() {
log.Printf("Receiver is not fully defined: host: %v Port %v\n.",
receiverHostFlag, receiverPortFlag)
continue
}
switch receiverTypeFlag {
case "carbon":
err := carbonReceiver.SendMetrics(createCarbonMetrics(pingResult))
if err != nil {
log.Printf("Error sending metrics to Carbon: %v.\n", err)
}
case "influxdb":
ret, err := createInfluxDBMetrics(pingResult)
if err != nil {
log.Fatalln(err)
}
err = ic.Write(ret)
if err != nil {
log.Printf("Error writing metrics to Influxdb: %v.\n", err)
}
}
if verboseFlag {
log.Printf("Successfully published %v metrics to %v.\n", receiverTypeFlag, receiverHostFlag)
}
}
return nil
}
// SetResultsClient is the utility for reseting the address of the ResultsClient
func (sf *StoreFront) SetResultsClient(conf influx.HTTPConfig) {
clnt, err := influx.NewHTTPClient(conf)
if err != nil {
log.Fatalf("Error resetting results clien\n error: %v\n", err)
}
sf.ResultsClient = clnt
}
func init() {
if conf.StatisticEnable {
var err error
influxClient, err = influx.NewHTTPClient(
influx.HTTPConfig{
Addr: conf.InfluxURL,
Username: conf.InfluxUser,
Password: conf.InfluxPassword,
})
if err != nil {
log.Println("Failed to init influx client: ", err.Error())
os.Exit(1)
}
influxBatchPoints, err = influx.NewBatchPoints(influx.BatchPointsConfig{
Database: conf.InfluxDB,
Precision: "s",
})
if err != nil {
log.Println("Failed to init influx batch points: ", err.Error())
os.Exit(1)
}
go logStatisticTask()
}
}
func NewInfluxDBClient(conf InfluxDBConf, ifChan chan Message, commandChan chan string) (*InfluxDBClient, error) {
host := fmt.Sprintf("http://%s:%d", conf.Hostname, conf.Port)
log.Infof("influxdb host: %s", host)
ifConf := client.HTTPConfig{
Addr: host,
Username: conf.UserName,
Password: conf.Password,
}
con, err := client.NewHTTPClient(ifConf)
if err != nil {
return nil, err
}
log.Infof("influxdb connected.")
tick := conf.Tick
if tick == 0 {
tick = DefaultTick
}
ifc := InfluxDBClient{
Client: con,
Tick: tick,
Status: StatusStopped,
Config: conf,
// prepare 2times by MaxBufferSize for Buffer itself
Buffer: lane.NewCappedDeque(MaxBufferSize * 2),
ifChan: ifChan,
commandChan: commandChan,
}
return &ifc, nil
}
func main() {
influxURL := flag.String("url", "http://localhost:8086", "The influxdb url and port")
replication := flag.String("replication", "3", "The number of nodes in the cluster")
user := flag.String("user", "", "The influxdb username used to create DBs")
password := flag.String("password", "", "The influxdb password used to create DBs")
flag.Parse()
fmt.Printf("creating datbases for influxUrl: %v with a replication of %v using user %s\n", *influxURL, *replication, *user)
client, err := influx.NewHTTPClient(influx.HTTPConfig{
Addr: *influxURL,
Username: *user,
Password: *password,
})
if err != nil {
fmt.Printf("Error creating influx client: %v\n", err)
os.Exit(1)
}
_, _, err = client.Ping(10)
if err != nil {
fmt.Printf("Error creating influx client: %v\n", err)
os.Exit(1)
}
createCacheStats(client, replication)
createDailyStats(client, replication)
createDeliveryServiceStats(client, replication)
}
// NewStressTest creates the backend for the stress test
func NewStressTest() *StressTest {
packageCh := make(chan Package, 0)
directiveCh := make(chan Directive, 0)
responseCh := make(chan Response, 0)
clnt, _ := influx.NewHTTPClient(influx.HTTPConfig{
Addr: fmt.Sprintf("http://%v/", "localhost:8086"),
})
s := &StressTest{
TestDB: "_stressTest",
Precision: "s",
StartDate: "2016-01-02",
BatchSize: 5000,
packageChan: packageCh,
directiveChan: directiveCh,
ResultsClient: clnt,
ResultsChan: responseCh,
communes: make(map[string]*commune),
TestID: randStr(10),
}
// Start the client service
startStressClient(packageCh, directiveCh, responseCh, s.TestID)
// Listen for Results coming in
s.resultsListen()
return s
}
func Newinfluxclient(influxserver string, username string, password string, dbname string) (*InfluxdbStorage, error) {
c, err := influxpackage.NewHTTPClient(influxpackage.HTTPConfig{
Addr: influxserver,
Username: username,
Password: password,
})
if err != nil {
return nil, err
}
_, err = queryDB(c, fmt.Sprintf("CREATE DATABASE %s", dbname), dbname)
if err != nil {
return nil, err
}
hostname, err := os.Hostname()
if err != nil {
return nil, err
}
influxc := &InfluxdbStorage{
Influxclient: c,
MachineName: hostname,
Database: dbname,
LastWrite: time.Now(),
}
return influxc, nil
}
func influxConnect(config StartupConfig, runningConfig RunningConfig) (influx.Client, error) {
var hosts []*InfluxDBProps
for _, InfluxHost := range runningConfig.InfluxDBs {
if InfluxHost.InfluxClient == nil {
if config.InfluxProtocol == "udp" {
conf := influx.UDPConfig{
Addr: fmt.Sprintf("%s:%d", InfluxHost.Fqdn, InfluxHost.Port),
}
con, err := influx.NewUDPClient(conf)
if err != nil {
errHndlr(err, ERROR)
continue
}
InfluxHost.InfluxClient = con
} else {
conf := influx.HTTPConfig{
Addr: fmt.Sprintf("%s://%s:%d", config.InfluxProtocol, InfluxHost.Fqdn, InfluxHost.Port),
Username: config.InfluxUser,
Password: config.InfluxPassword,
}
con, err := influx.NewHTTPClient(conf)
if err != nil {
errHndlr(err, ERROR)
continue
}
InfluxHost.InfluxClient = con
}
}
hosts = append(hosts, InfluxHost)
}
for len(hosts) > 0 {
n := rand.Intn(len(hosts))
host := hosts[n]
hosts = append(hosts[:n], hosts[n+1:]...)
con := host.InfluxClient
//client currently does not support udp queries
if config.InfluxProtocol != "udp" {
_, _, err := con.Ping(10)
if err != nil {
errHndlr(err, ERROR)
host.InfluxClient = nil
continue
}
}
return con, nil
}
err := errors.New("Could not connect to any of the InfluxDb servers that are ONLINE in traffic ops.")
return nil, err
}
// Returns an influxdb client
func (hook *InfluxDBHook) newInfluxDBClient(config *Config) (influxdb.Client, error) {
protocol := "http"
if config.UseHTTPS {
protocol = "https"
}
return influxdb.NewHTTPClient(influxdb.HTTPConfig{
Addr: fmt.Sprintf("%s://%s:%d", protocol, config.Host, config.Port),
Username: config.Username,
Password: config.Password,
Timeout: 100 * time.Millisecond,
})
}
func (self *Server) InfluxWriter(options InfluxOptions) error {
influxClient, err := influxdb.NewHTTPClient(options.httpConfig())
if err != nil {
return err
}
// start goroutine
self.influxChan = make(chan Stat)
go self.influxWriter(influxClient, options)
return nil
}
// createClient is an internal method for creating a HTTP Connection to the database.
func createClient() (client.Client, error) {
config := client.HTTPConfig{
Addr: databaseAddress,
Username: databaseUsername,
Password: databasePassword,
}
influxClient, err := client.NewHTTPClient(config)
if err != nil {
log.Println("Error creating an HTTP Client for Influx.")
return nil, err
}
return influxClient, nil
}
// Create the client connection
// TODO(daneroo): We need to close this thing!
func (w *Writer) connect() error {
url := fmt.Sprintf("http://%s:%d", w.Host, w.Port)
var err error
w.con, err = client.NewHTTPClient(client.HTTPConfig{
Addr: url,
// Username: os.Getenv("INFLUX_USER"),
// Password: os.Getenv("INFLUX_PWD"),
})
Checkerr(err)
dur, ver, err := w.con.Ping(time.Minute)
Checkerr(err)
log.Printf("Connected to %s, InfluxDB:%s, ping:%v", url, ver, dur)
return nil
}
func queryInflux(q string) (*influx.Response, error) {
client, err := influx.NewHTTPClient(
influx.HTTPConfig{
Addr: conf.InfluxURL,
Username: conf.InfluxUser,
Password: conf.InfluxPassword,
})
if err != nil {
return nil, err
}
return client.Query(
influx.Query{
Command: q,
Database: conf.InfluxDB,
Precision: "s",
})
}
func influxConnect(config StartupConfig) (influx.Client, error) {
hosts := config.InfluxDBs
for len(hosts) > 0 {
n := rand.Intn(len(hosts))
host := hosts[n]
hosts = append(hosts[:n], hosts[n+1:]...)
parsedURL, _ := url.Parse(host.URL)
if parsedURL.Scheme == "udp" {
conf := influx.UDPConfig{
Addr: parsedURL.Host,
}
con, err := influx.NewUDPClient(conf)
if err != nil {
errHndlr(fmt.Errorf("An error occurred creating udp client. %v\n", err), ERROR)
continue
}
return con, nil
}
//if not udp assume HTTP client
conf := influx.HTTPConfig{
Addr: parsedURL.String(),
Username: config.InfluxUser,
Password: config.InfluxPassword,
}
con, err := influx.NewHTTPClient(conf)
if err != nil {
errHndlr(fmt.Errorf("An error occurred creating HTTP client. %v\n", err), ERROR)
continue
}
//Close old connections explicitly
if host.InfluxClient != nil {
host.InfluxClient.Close()
}
host.InfluxClient = con
_, _, err = con.Ping(10)
if err != nil {
errHndlr(err, WARN)
continue
}
return con, nil
}
err := errors.New("Could not connect to any of the InfluxDb servers defined in the influxUrls config.")
return nil, err
}
func (s *Stats) initClient() (err error) {
log.WithField("Addr", s.influxConfig.Addr).Debug("connecting to influxdb")
s.client, err = influx.NewHTTPClient(s.influxConfig)
if err != nil {
return err
}
defer s.client.Close()
ping, _, err := s.client.Ping(time.Second)
if err != nil {
return err
}
log.WithField("ping", ping).Debug("influx ping")
s.bp, err = influx.NewBatchPoints(influx.BatchPointsConfig{
Database: "dicod",
Precision: "s",
})
return err
}
func main() {
initFlags()
if err := checkFlags(); err != nil {
fmt.Printf("Error: %s", err.Error())
}
throttle = make(chan int, crawlers)
loadEnvironmentVariables()
redisClient := redis.NewClient(&redis.Options{
Addr: redisHost + ":" + redisPort,
Password: "",
DB: 0,
})
if pong, err := redisClient.Ping().Result(); err != nil {
log.Fatalf("%s. Cannot connect to redis client: %s", pong, err.Error())
}
influxClient, err := influx.NewHTTPClient(influx.HTTPConfig{
Addr: fmt.Sprintf("http://%s:%s", influxHost, influxPort),
Username: "******",
Password: "******",
})
if err != nil {
log.Fatalf("Error occured when connecting to InfluxDB", err.Error())
}
if search != "" {
validURL = regexp.MustCompile(search)
}
fetcher := URLFetcher{}
Crawl(seedUrl, depth, fetcher, redisClient, influxClient)
}
// NewSender returns a function that will accept datapoints to send to influxdb
func NewSender(
config interface{},
batch client.BatchPointsConfig,
batchSize int,
queueSize int,
flush int,
errFunc func(error),
) (Sender, error) {
if batchSize <= 0 {
batchSize = DefaultBatchSize
}
if queueSize <= 0 {
queueSize = DefaultQueueSize
}
if flush <= 0 {
flush = DefaultFlush
}
var conn client.Client
var err error
switch conf := config.(type) {
case client.HTTPConfig:
conn, err = client.NewHTTPClient(conf)
if err != nil {
return nil, errors.Wrap(err, "error creating HTTPClient")
}
_, _, err = conn.Ping(conf.Timeout)
if err != nil {
return nil, fmt.Errorf("cannot ping influxdb server: %s", conf.Addr)
}
if err := dbCheck(conn, batch.Database); err != nil {
return nil, errors.Wrapf(err, "check for database %s failed", batch.Database)
}
case client.UDPConfig:
conn, err = client.NewUDPClient(conf)
if err != nil {
return nil, errors.Wrap(err, "error creating UDPClient")
}
}
pts := make(chan *client.Point, queueSize)
bp, err := client.NewBatchPoints(batch)
if err != nil {
return nil, errors.Wrap(err, "batchpoints error")
}
go func() {
delay := time.Duration(flush) * time.Second
tick := time.Tick(delay)
count := 0
for {
select {
case p := <-pts:
bp.AddPoint(p)
count++
if count < batchSize {
continue
}
case <-tick:
if len(bp.Points()) == 0 {
continue
}
}
for {
if err := conn.Write(bp); err != nil {
if errFunc != nil {
errFunc(err)
}
time.Sleep(retry)
continue
}
bp, _ = client.NewBatchPoints(batch)
count = 0
break
}
}
}()
return func(key string, tags map[string]string, fields map[string]interface{}, ts time.Time) error {
pt, err := client.NewPoint(key, tags, fields, ts)
if err != nil {
return err
}
pts <- pt
return nil
}, nil
}
func main() {
sourceURL := flag.String("sourceUrl", "http://server1.kabletown.net:8086", "The influxdb url and port")
targetURL := flag.String("targetUrl", "http://server2.kabletown.net:8086", "The influxdb url and port")
database := flag.String("database", "all", "Sync a specific database")
days := flag.Int("days", 0, "Number of days in the past to sync (today - x days), 0 is all")
sourceUser := flag.String("sourceUser", "", "The source influxdb username")
sourcePass := flag.String("sourcePass", "", "The source influxdb password")
targetUser := flag.String("targetUser", "", "The target influxdb username")
targetPass := flag.String("targetPass", "", "The target influxdb password")
flag.Parse()
fmt.Printf("syncing %v to %v for %v database(s) for the past %v day(s)\n", *sourceURL, *targetURL, *database, *days)
sourceClient, err := influx.NewHTTPClient(influx.HTTPConfig{
Addr: *sourceURL,
Username: *sourceUser,
Password: *sourcePass,
})
if err != nil {
errorMessage = fmt.Sprintf("Error creating influx sourceClient: %v\n", err)
fmt.Println(errorMessage)
os.Exit(1)
}
_, _, err = sourceClient.Ping(10)
if err != nil {
errorMessage = fmt.Sprintf("Error creating influx sourceClient: %v\n", err)
fmt.Println(errorMessage)
os.Exit(1)
}
targetClient, err := influx.NewHTTPClient(influx.HTTPConfig{
Addr: *targetURL,
Username: *targetUser,
Password: *targetPass,
})
if err != nil {
errorMessage = fmt.Sprintf("Error creating influx targetClient: %v\n", err)
fmt.Println(errorMessage)
os.Exit(1)
}
_, _, err = targetClient.Ping(10)
if err != nil {
errorMessage = fmt.Sprintf("Error creating influx targetClient: %v\n", err)
fmt.Println(errorMessage)
os.Exit(1)
}
chSize := 1
if *database == "all" {
chSize = 3
}
ch := make(chan string)
switch *database {
case "all":
go syncDailyDb(ch, sourceClient, targetClient, *days)
go syncCsDb(ch, sourceClient, targetClient, *days)
go syncDsDb(ch, sourceClient, targetClient, *days)
case "cache_stats":
go syncCsDb(ch, sourceClient, targetClient, *days)
case "deliveryservice_stats":
go syncDsDb(ch, sourceClient, targetClient, *days)
case "daily_stats":
go syncDailyDb(ch, sourceClient, targetClient, *days)
}
for i := 1; i <= chSize; i++ {
fmt.Println(<-ch)
}
if errorMessage != "" {
fmt.Println(errorMessage)
return
}
fmt.Println("Traffic Stats has been synced!")
}
