Ejemplo n.º 1
0
func processPing(c <-chan Ping) error {
	var err error
	var carbonReceiver *carbon.Carbon
	var ic influxdbclient.Client

	switch receiverTypeFlag {
	case "carbon":
		carbonReceiver, err = carbon.NewCarbon(receiverHostFlag, receiverPortFlag, receiverNoopFlag, verboseFlag)
	case "influxdb":
		if receiverDatabaseFlag == "" {
			log.Fatalln("An InfluxDB database was not specified on the command line.")
		}
		if len(receiverUsernameFlag) > 0 {
			ic, err = influxdbclient.NewHTTPClient(
				influxdbclient.HTTPConfig{
					Addr:     fmt.Sprintf("http://%v:%v", receiverHostFlag, receiverPortFlag),
					Username: receiverUsernameFlag,
					Password: receiverPasswordFlag,
				})
		} else {
			ic, err = influxdbclient.NewHTTPClient(
				influxdbclient.HTTPConfig{
					Addr: fmt.Sprintf("http://%v:%v", receiverHostFlag, receiverPortFlag),
				})
		}
	}

	if err != nil {
		log.Println("error in creating a connection, but ignoring")
		//		return err
	}
	for {
		pingResult := <-c
		if !isReceiverFullyDefined() {
			log.Printf("Receiver is not fully defined: host: %v Port %v\n.",
				receiverHostFlag, receiverPortFlag)
			continue
		}
		switch receiverTypeFlag {
		case "carbon":
			err := carbonReceiver.SendMetrics(createCarbonMetrics(pingResult))
			if err != nil {
				log.Printf("Error sending metrics to Carbon: %v.\n", err)
			}
		case "influxdb":
			ret, err := createInfluxDBMetrics(pingResult)
			if err != nil {
				log.Fatalln(err)
			}
			err = ic.Write(ret)
			if err != nil {
				log.Printf("Error writing metrics to Influxdb: %v.\n", err)
			}
		}
		if verboseFlag {
			log.Printf("Successfully published %v metrics to %v.\n", receiverTypeFlag, receiverHostFlag)
		}
	}
	return nil
}
Ejemplo n.º 2
0
// SetResultsClient is the utility for reseting the address of the ResultsClient
func (sf *StoreFront) SetResultsClient(conf influx.HTTPConfig) {
	clnt, err := influx.NewHTTPClient(conf)
	if err != nil {
		log.Fatalf("Error resetting results clien\n  error: %v\n", err)
	}
	sf.ResultsClient = clnt
}
Ejemplo n.º 3
0
func init() {
	if conf.StatisticEnable {
		var err error
		influxClient, err = influx.NewHTTPClient(
			influx.HTTPConfig{
				Addr:     conf.InfluxURL,
				Username: conf.InfluxUser,
				Password: conf.InfluxPassword,
			})
		if err != nil {
			log.Println("Failed to init influx client: ", err.Error())
			os.Exit(1)
		}

		influxBatchPoints, err = influx.NewBatchPoints(influx.BatchPointsConfig{
			Database:  conf.InfluxDB,
			Precision: "s",
		})
		if err != nil {
			log.Println("Failed to init influx batch points: ", err.Error())
			os.Exit(1)
		}
		go logStatisticTask()
	}
}
Ejemplo n.º 4
0
func NewInfluxDBClient(conf InfluxDBConf, ifChan chan Message, commandChan chan string) (*InfluxDBClient, error) {
	host := fmt.Sprintf("http://%s:%d", conf.Hostname, conf.Port)
	log.Infof("influxdb host: %s", host)

	ifConf := client.HTTPConfig{
		Addr:     host,
		Username: conf.UserName,
		Password: conf.Password,
	}
	con, err := client.NewHTTPClient(ifConf)
	if err != nil {
		return nil, err
	}

	log.Infof("influxdb connected.")

	tick := conf.Tick
	if tick == 0 {
		tick = DefaultTick
	}

	ifc := InfluxDBClient{
		Client: con,
		Tick:   tick,
		Status: StatusStopped,
		Config: conf,
		// prepare 2times by MaxBufferSize for Buffer itself
		Buffer:      lane.NewCappedDeque(MaxBufferSize * 2),
		ifChan:      ifChan,
		commandChan: commandChan,
	}

	return &ifc, nil
}
func main() {

	influxURL := flag.String("url", "http://localhost:8086", "The influxdb url and port")
	replication := flag.String("replication", "3", "The number of nodes in the cluster")
	user := flag.String("user", "", "The influxdb username used to create DBs")
	password := flag.String("password", "", "The influxdb password used to create DBs")
	flag.Parse()
	fmt.Printf("creating datbases for influxUrl: %v with a replication of %v using user %s\n", *influxURL, *replication, *user)
	client, err := influx.NewHTTPClient(influx.HTTPConfig{
		Addr:     *influxURL,
		Username: *user,
		Password: *password,
	})
	if err != nil {
		fmt.Printf("Error creating influx client: %v\n", err)
		os.Exit(1)
	}
	_, _, err = client.Ping(10)
	if err != nil {
		fmt.Printf("Error creating influx client: %v\n", err)
		os.Exit(1)
	}

	createCacheStats(client, replication)
	createDailyStats(client, replication)
	createDeliveryServiceStats(client, replication)
}
Ejemplo n.º 6
0
// NewStressTest creates the backend for the stress test
func NewStressTest() *StressTest {

	packageCh := make(chan Package, 0)
	directiveCh := make(chan Directive, 0)
	responseCh := make(chan Response, 0)

	clnt, _ := influx.NewHTTPClient(influx.HTTPConfig{
		Addr: fmt.Sprintf("http://%v/", "localhost:8086"),
	})

	s := &StressTest{
		TestDB:    "_stressTest",
		Precision: "s",
		StartDate: "2016-01-02",
		BatchSize: 5000,

		packageChan:   packageCh,
		directiveChan: directiveCh,

		ResultsClient: clnt,
		ResultsChan:   responseCh,
		communes:      make(map[string]*commune),
		TestID:        randStr(10),
	}

	// Start the client service
	startStressClient(packageCh, directiveCh, responseCh, s.TestID)

	// Listen for Results coming in
	s.resultsListen()

	return s
}
Ejemplo n.º 7
0
func Newinfluxclient(influxserver string, username string, password string, dbname string) (*InfluxdbStorage, error) {
	c, err := influxpackage.NewHTTPClient(influxpackage.HTTPConfig{
		Addr:     influxserver,
		Username: username,
		Password: password,
	})

	if err != nil {
		return nil, err
	}

	_, err = queryDB(c, fmt.Sprintf("CREATE DATABASE %s", dbname), dbname)
	if err != nil {
		return nil, err
	}

	hostname, err := os.Hostname()
	if err != nil {
		return nil, err
	}

	influxc := &InfluxdbStorage{
		Influxclient: c,
		MachineName:  hostname,
		Database:     dbname,
		LastWrite:    time.Now(),
	}

	return influxc, nil
}
Ejemplo n.º 8
0
func influxConnect(config StartupConfig, runningConfig RunningConfig) (influx.Client, error) {
	var hosts []*InfluxDBProps
	for _, InfluxHost := range runningConfig.InfluxDBs {
		if InfluxHost.InfluxClient == nil {
			if config.InfluxProtocol == "udp" {
				conf := influx.UDPConfig{
					Addr: fmt.Sprintf("%s:%d", InfluxHost.Fqdn, InfluxHost.Port),
				}
				con, err := influx.NewUDPClient(conf)
				if err != nil {
					errHndlr(err, ERROR)
					continue
				}
				InfluxHost.InfluxClient = con

			} else {
				conf := influx.HTTPConfig{
					Addr:     fmt.Sprintf("%s://%s:%d", config.InfluxProtocol, InfluxHost.Fqdn, InfluxHost.Port),
					Username: config.InfluxUser,
					Password: config.InfluxPassword,
				}
				con, err := influx.NewHTTPClient(conf)
				if err != nil {
					errHndlr(err, ERROR)
					continue
				}
				InfluxHost.InfluxClient = con
			}
		}

		hosts = append(hosts, InfluxHost)
	}

	for len(hosts) > 0 {
		n := rand.Intn(len(hosts))
		host := hosts[n]
		hosts = append(hosts[:n], hosts[n+1:]...)
		con := host.InfluxClient
		//client currently does not support udp queries
		if config.InfluxProtocol != "udp" {
			_, _, err := con.Ping(10)
			if err != nil {
				errHndlr(err, ERROR)
				host.InfluxClient = nil
				continue
			}
		}
		return con, nil
	}

	err := errors.New("Could not connect to any of the InfluxDb servers that are ONLINE in traffic ops.")
	return nil, err
}
Ejemplo n.º 9
0
// Returns an influxdb client
func (hook *InfluxDBHook) newInfluxDBClient(config *Config) (influxdb.Client, error) {
	protocol := "http"
	if config.UseHTTPS {
		protocol = "https"
	}
	return influxdb.NewHTTPClient(influxdb.HTTPConfig{
		Addr:     fmt.Sprintf("%s://%s:%d", protocol, config.Host, config.Port),
		Username: config.Username,
		Password: config.Password,
		Timeout:  100 * time.Millisecond,
	})
}
Ejemplo n.º 10
0
func (self *Server) InfluxWriter(options InfluxOptions) error {
	influxClient, err := influxdb.NewHTTPClient(options.httpConfig())
	if err != nil {
		return err
	}

	// start goroutine
	self.influxChan = make(chan Stat)

	go self.influxWriter(influxClient, options)

	return nil
}
Ejemplo n.º 11
0
// createClient is an internal method for creating a HTTP Connection to the database.
func createClient() (client.Client, error) {
	config := client.HTTPConfig{
		Addr:     databaseAddress,
		Username: databaseUsername,
		Password: databasePassword,
	}

	influxClient, err := client.NewHTTPClient(config)

	if err != nil {
		log.Println("Error creating an HTTP Client for Influx.")
		return nil, err
	}

	return influxClient, nil
}
Ejemplo n.º 12
0
// Create the client connection
// TODO(daneroo): We need to close this thing!
func (w *Writer) connect() error {

	url := fmt.Sprintf("http://%s:%d", w.Host, w.Port)
	var err error
	w.con, err = client.NewHTTPClient(client.HTTPConfig{
		Addr: url,
		// Username: os.Getenv("INFLUX_USER"),
		// Password: os.Getenv("INFLUX_PWD"),
	})
	Checkerr(err)

	dur, ver, err := w.con.Ping(time.Minute)
	Checkerr(err)
	log.Printf("Connected to %s, InfluxDB:%s, ping:%v", url, ver, dur)

	return nil
}
Ejemplo n.º 13
0
func queryInflux(q string) (*influx.Response, error) {
	client, err := influx.NewHTTPClient(
		influx.HTTPConfig{
			Addr:     conf.InfluxURL,
			Username: conf.InfluxUser,
			Password: conf.InfluxPassword,
		})
	if err != nil {
		return nil, err
	}

	return client.Query(
		influx.Query{
			Command:   q,
			Database:  conf.InfluxDB,
			Precision: "s",
		})
}
Ejemplo n.º 14
0
func influxConnect(config StartupConfig) (influx.Client, error) {
	hosts := config.InfluxDBs
	for len(hosts) > 0 {
		n := rand.Intn(len(hosts))
		host := hosts[n]
		hosts = append(hosts[:n], hosts[n+1:]...)
		parsedURL, _ := url.Parse(host.URL)
		if parsedURL.Scheme == "udp" {
			conf := influx.UDPConfig{
				Addr: parsedURL.Host,
			}
			con, err := influx.NewUDPClient(conf)
			if err != nil {
				errHndlr(fmt.Errorf("An error occurred creating udp client. %v\n", err), ERROR)
				continue
			}
			return con, nil
		}
		//if not udp assume HTTP client
		conf := influx.HTTPConfig{
			Addr:     parsedURL.String(),
			Username: config.InfluxUser,
			Password: config.InfluxPassword,
		}
		con, err := influx.NewHTTPClient(conf)
		if err != nil {
			errHndlr(fmt.Errorf("An error occurred creating HTTP client.  %v\n", err), ERROR)
			continue
		}
		//Close old connections explicitly
		if host.InfluxClient != nil {
			host.InfluxClient.Close()
		}
		host.InfluxClient = con
		_, _, err = con.Ping(10)
		if err != nil {
			errHndlr(err, WARN)
			continue
		}
		return con, nil
	}
	err := errors.New("Could not connect to any of the InfluxDb servers defined in the influxUrls config.")
	return nil, err
}
Ejemplo n.º 15
0
func (s *Stats) initClient() (err error) {
	log.WithField("Addr", s.influxConfig.Addr).Debug("connecting to influxdb")
	s.client, err = influx.NewHTTPClient(s.influxConfig)
	if err != nil {
		return err
	}
	defer s.client.Close()
	ping, _, err := s.client.Ping(time.Second)
	if err != nil {
		return err
	}
	log.WithField("ping", ping).Debug("influx ping")

	s.bp, err = influx.NewBatchPoints(influx.BatchPointsConfig{
		Database:  "dicod",
		Precision: "s",
	})
	return err
}
Ejemplo n.º 16
0
func main() {
	initFlags()
	if err := checkFlags(); err != nil {
		fmt.Printf("Error: %s", err.Error())
	}

	throttle = make(chan int, crawlers)

	loadEnvironmentVariables()

	redisClient := redis.NewClient(&redis.Options{
		Addr:     redisHost + ":" + redisPort,
		Password: "",
		DB:       0,
	})

	if pong, err := redisClient.Ping().Result(); err != nil {
		log.Fatalf("%s. Cannot connect to redis client: %s", pong, err.Error())
	}

	influxClient, err := influx.NewHTTPClient(influx.HTTPConfig{
		Addr:     fmt.Sprintf("http://%s:%s", influxHost, influxPort),
		Username: "******",
		Password: "******",
	})

	if err != nil {
		log.Fatalf("Error occured when connecting to InfluxDB", err.Error())
	}

	if search != "" {
		validURL = regexp.MustCompile(search)
	}

	fetcher := URLFetcher{}
	Crawl(seedUrl, depth, fetcher, redisClient, influxClient)
}
Ejemplo n.º 17
0
// NewSender returns a function that will accept datapoints to send to influxdb
func NewSender(
	config interface{},
	batch client.BatchPointsConfig,
	batchSize int,
	queueSize int,
	flush int,
	errFunc func(error),
) (Sender, error) {
	if batchSize <= 0 {
		batchSize = DefaultBatchSize
	}
	if queueSize <= 0 {
		queueSize = DefaultQueueSize
	}
	if flush <= 0 {
		flush = DefaultFlush
	}

	var conn client.Client
	var err error

	switch conf := config.(type) {
	case client.HTTPConfig:
		conn, err = client.NewHTTPClient(conf)
		if err != nil {
			return nil, errors.Wrap(err, "error creating HTTPClient")
		}

		_, _, err = conn.Ping(conf.Timeout)
		if err != nil {
			return nil, fmt.Errorf("cannot ping influxdb server: %s", conf.Addr)
		}

		if err := dbCheck(conn, batch.Database); err != nil {
			return nil, errors.Wrapf(err, "check for database %s failed", batch.Database)
		}
	case client.UDPConfig:
		conn, err = client.NewUDPClient(conf)
		if err != nil {
			return nil, errors.Wrap(err, "error creating UDPClient")
		}
	}

	pts := make(chan *client.Point, queueSize)

	bp, err := client.NewBatchPoints(batch)
	if err != nil {
		return nil, errors.Wrap(err, "batchpoints error")
	}

	go func() {
		delay := time.Duration(flush) * time.Second
		tick := time.Tick(delay)
		count := 0
		for {
			select {
			case p := <-pts:
				bp.AddPoint(p)
				count++
				if count < batchSize {
					continue
				}
			case <-tick:
				if len(bp.Points()) == 0 {
					continue
				}
			}
			for {
				if err := conn.Write(bp); err != nil {
					if errFunc != nil {
						errFunc(err)
					}
					time.Sleep(retry)
					continue
				}
				bp, _ = client.NewBatchPoints(batch)
				count = 0
				break
			}
		}
	}()

	return func(key string, tags map[string]string, fields map[string]interface{}, ts time.Time) error {
		pt, err := client.NewPoint(key, tags, fields, ts)
		if err != nil {
			return err
		}
		pts <- pt
		return nil
	}, nil
}
func main() {

	sourceURL := flag.String("sourceUrl", "http://server1.kabletown.net:8086", "The influxdb url and port")
	targetURL := flag.String("targetUrl", "http://server2.kabletown.net:8086", "The influxdb url and port")
	database := flag.String("database", "all", "Sync a specific database")
	days := flag.Int("days", 0, "Number of days in the past to sync (today - x days), 0 is all")
	sourceUser := flag.String("sourceUser", "", "The source influxdb username")
	sourcePass := flag.String("sourcePass", "", "The source influxdb password")
	targetUser := flag.String("targetUser", "", "The target influxdb username")
	targetPass := flag.String("targetPass", "", "The target influxdb password")
	flag.Parse()
	fmt.Printf("syncing %v to %v for %v database(s) for the past %v day(s)\n", *sourceURL, *targetURL, *database, *days)
	sourceClient, err := influx.NewHTTPClient(influx.HTTPConfig{
		Addr:     *sourceURL,
		Username: *sourceUser,
		Password: *sourcePass,
	})
	if err != nil {
		errorMessage = fmt.Sprintf("Error creating influx sourceClient: %v\n", err)
		fmt.Println(errorMessage)
		os.Exit(1)
	}
	_, _, err = sourceClient.Ping(10)
	if err != nil {
		errorMessage = fmt.Sprintf("Error creating influx sourceClient: %v\n", err)
		fmt.Println(errorMessage)
		os.Exit(1)
	}
	targetClient, err := influx.NewHTTPClient(influx.HTTPConfig{
		Addr:     *targetURL,
		Username: *targetUser,
		Password: *targetPass,
	})
	if err != nil {
		errorMessage = fmt.Sprintf("Error creating influx targetClient: %v\n", err)
		fmt.Println(errorMessage)
		os.Exit(1)
	}
	_, _, err = targetClient.Ping(10)
	if err != nil {
		errorMessage = fmt.Sprintf("Error creating influx targetClient: %v\n", err)
		fmt.Println(errorMessage)
		os.Exit(1)
	}
	chSize := 1
	if *database == "all" {
		chSize = 3
	}

	ch := make(chan string)

	switch *database {
	case "all":
		go syncDailyDb(ch, sourceClient, targetClient, *days)
		go syncCsDb(ch, sourceClient, targetClient, *days)
		go syncDsDb(ch, sourceClient, targetClient, *days)
	case "cache_stats":
		go syncCsDb(ch, sourceClient, targetClient, *days)
	case "deliveryservice_stats":
		go syncDsDb(ch, sourceClient, targetClient, *days)
	case "daily_stats":
		go syncDailyDb(ch, sourceClient, targetClient, *days)
	}

	for i := 1; i <= chSize; i++ {
		fmt.Println(<-ch)
	}

	if errorMessage != "" {
		fmt.Println(errorMessage)
		return
	}

	fmt.Println("Traffic Stats has been synced!")
}