Exemple #1
0
func (d *FileDrain) Start(config *DrainConfig) {
	defer d.Done()

	overwrite, err := config.GetParamBool("overwrite", false)
	if err != nil {
		d.Kill(err)
		go d.finishedStarting(false)
		return
	}

	mode := os.O_WRONLY | os.O_CREATE
	if overwrite {
		mode |= os.O_TRUNC
	} else {
		mode |= os.O_APPEND
	}
	log.Infof("[drain:%s] Attempting to open %s (overwrite=%v) ...",
		d.name, config.Path, overwrite)
	f, err := os.OpenFile(config.Path, mode, 0600)
	if err != nil {
		d.Kill(err)
		go d.finishedStarting(false)
		return
	}
	log.Infof("[drain:%s] Successfully opened %s.", d.name, config.Path)
	defer f.Close()

	sub := logyard.Broker.Subscribe(config.Filters...)
	defer sub.Stop()

	go d.finishedStarting(true)

	for {
		select {
		case msg := <-sub.Ch:
			data, err := config.FormatJSON(msg)
			if err != nil {
				d.Kill(err)
				return
			}
			_, err = f.Write(data)
			if err != nil {
				d.Kill(err)
				return
			}
		case <-d.Dying():
			return
		}
	}
}
Exemple #2
0
func (d *RedisDrain) connect(addr string, database int64) error {
	log.Infof("[drain:%s] Attempting to connect to redis %s[#%d] ...",
		d.name, addr, database)

	if client, err := server.NewRedisClient(
		addr, "", database); err != nil {
		return err
	} else {
		d.client = client
		log.Infof("[drain:%s] Successfully connected to redis %s[#%d].",
			d.name, addr, database)
		return nil
	}
	panic("unreachable")
}
Exemple #3
0
func main() {
	var useUDP bool
	var port int
	flag.BoolVar(&useUDP, "u", false, "use UDP instead of TCP")
	flag.IntVar(&port, "p", 9090, "port number to bind to")
	flag.Parse()

	var srv *lineserver.LineServer
	var err error
	addr := fmt.Sprintf(":%d", port)
	proto := "tcp"
	if useUDP {
		proto = "udp"
	}

	if useUDP {
		srv, err = lineserver.NewLineServerUDP(addr)
	} else {
		srv, err = lineserver.NewLineServerTCP(addr)
	}

	if err != nil {
		log.Fatal(err)
	}

	log.Infof("Server running as %s://%s", proto, addr)
	go srv.Start()

	for line := range srv.Ch {
		fmt.Println(line)
	}
}
Exemple #4
0
func (manager *DrainManager) Run() {
	iteration := 0
	drains := logyard.GetConfig().Drains
	log.Infof("Found %d drains to start\n", len(drains))
	for name, uri := range drains {
		manager.StartDrain(name, uri, NewRetryerForDrain(name))
	}

	// Watch for config changes in redis.
	for {
		iteration += 1
		prefix := fmt.Sprintf("CONFIG.%d", iteration)

		select {
		case err := <-logyard.GetConfigChanges():
			if err != nil {
				log.Fatalf("Error re-loading config: %v", err)
			}
			log.Infof(
				"[%s] checking drains after a config change...",
				prefix)
			newDrains := logyard.GetConfig().Drains
			for _, c := range mapdiff.MapDiff(drains, newDrains) {
				if c.Deleted {
					log.Infof("[%s] Drain %s was deleted.", prefix, c.Key)
					manager.StopDrain(c.Key, true)
					delete(drains, c.Key)
				} else {
					log.Infof("[%s] Drain %s was added.", prefix, c.Key)
					manager.StopDrain(c.Key, false)
					manager.StartDrain(
						c.Key,
						c.NewValue,
						NewRetryerForDrain(c.Key))
					drains[c.Key] = c.NewValue
				}
			}
			log.Infof("[%s] Done checking drains.", prefix)
		case <-manager.stopCh:
			break
		}
	}
}
Exemple #5
0
func init() {
	Broker.PubAddr = "ipc:///var/stackato/run/logyardpub.sock"
	Broker.SubAddr = "ipc:///var/stackato/run/logyardsub.sock"
	Broker.BufferSize = 100

	// ZeroMQ will silently fail if .sock files can't be created
	if _, err := os.Stat("/var/stackato/run"); err != nil {
		log.Errorf("Cannot access /var/stackato/run: %v", err)
	}

	log.Infof("Loygard broker config: %+v\n", Broker)
}
Exemple #6
0
// NewRetryerForDrain chooses
func NewRetryerForDrain(name string) retry.Retryer {
	var retryLimit time.Duration
	var err error
	for prefix, duration := range logyard.GetConfig().RetryLimits {
		if strings.HasPrefix(name, prefix) {
			if retryLimit, err = time.ParseDuration(duration); err != nil {
				log.Error("[drain:%s] Invalid duration (%s) for drain prefix %s "+
					"-- %s -- using default value (infinite)",
					name, duration, prefix, err)
				retryLimit = time.Duration(0)
			}
			if retryLimit <= retry.RESET_AFTER {
				log.Error("[drain:%s] Invalid retry limit (%v) -- must be >%v -- "+
					"using default value (infinite)",
					name, retryLimit, retry.RESET_AFTER)
				retryLimit = time.Duration(0)
			}
			break
		}
	}
	log.Infof("[drain:%s] Choosing retry limit %v", name, retryLimit)
	return retry.NewProgressiveRetryer(retryLimit)
}
Exemple #7
0
func main() {
	major, minor, patch := gozmq.Version()
	log.Infof("Starting logyard (Go %s; ZeroMQ %d.%d.%d)",
		runtime.Version(), major, minor, patch)

	m := drain.NewDrainManager()
	log.Info("Starting drain manager")
	go m.Run()
	// SIGTERM handle for stopping running drains.
	go func() {
		sigchan := make(chan os.Signal)
		signal.Notify(sigchan, syscall.SIGTERM)
		<-sigchan
		log.Info("Stopping all drains before exiting")
		m.Stop()
		log.Info("Exiting now.")
		os.Exit(0)
	}()

	server.MarkRunning("logyard")

	log.Info("Running pubsub broker")
	log.Fatal(logyard.Broker.Run())
}
Exemple #8
0
func (cmd *stream) Run(args []string) (string, error) {
	ipaddr, err := server.LocalIP()
	if err != nil {
		return "", err
	}

	rand.Seed(time.Now().UnixNano())
	port := 7000 + rand.Intn(1000)
	addr := fmt.Sprintf("%s:%d", ipaddr, port)

	srv, err := lineserver.NewLineServer(CLI_STREAM_PROTO, addr)
	if err != nil {
		return "", err
	}
	go srv.Start()

	// Debug mode allows one to debug just the logyard related logs,
	// without any magical stripping.
	debugMode := false
	if len(args) == 1 && args[0] == "debug" {
		debugMode = true
		args = []string{
			"systail.logyard",
			"systail.apptail",
			"systail.logyard_sieve",
			"systail.systail",
		}
	}

	name := fmt.Sprintf("tmp.logyard-cli.%s-%d", ipaddr, port)

	uri, err := drain.ConstructDrainURI(
		name,
		fmt.Sprintf("%s://%s", CLI_STREAM_PROTO, addr),
		args,
		map[string]string{"format": "raw"})
	if err != nil {
		return "", err
	}
	if err = logyard.AddDrain(name, uri); err != nil {
		return "", err
	}
	log.Infof("Added drain %s", uri)

	deleteDrain := func() {
		if err := logyard.DeleteDrain(name); err != nil {
			log.Fatal(err)
		}
		fmt.Println("")
		log.Infof("Deleted drain %s", name)
	}
	defer deleteDrain()

	handleKeyboardInterrupt(func() {
		deleteDrain()
		os.Exit(1)
	})

	cli_stream.Stream(srv.Ch, cli_stream.MessagePrinterOptions{
		cmd.raw, cmd.raw || debugMode, cmd.time, cmd.nocolor, cmd.nodeid, cmd.json})

	return "", nil
}
Exemple #9
0
func (d *IPConnDrain) Start(config *DrainConfig) {
	defer d.Done()

	if !(config.Scheme == "udp" || config.Scheme == "tcp") {
		d.Killf("Invalid scheme: %s", config.Scheme)
		go d.finishedStarting(false)
		return
	}

	log.Infof("[drain:%s] Attempting to connect to %s://%s ...",
		d.name, config.Scheme, config.Host)

	var conn net.Conn
	dialer := NewNetDialer(config.Scheme, config.Host, 10*time.Second)

	select {
	case conn = <-dialer.Ch:
		if dialer.Error != nil {
			d.Kill(dialer.Error)
			go d.finishedStarting(false)
			return
		}
	case <-d.Dying():
		// Close the connection returned in future by the dialer.
		log.Infof("[drain:%s] Stop request; deferring close of connection",
			d.name)
		go dialer.WaitAndClose()

		// [bug 105165].
		// did not attain running state. No kill however as
		// getting aborted by the user is not really an
		// error. The state machine will treat it as a regular
		// non-error exit.
		go d.finishedStarting(false)
		return
	}
	defer conn.Close()

	log.Infof("[drain:%s] Successfully connected to %s://%s.",
		d.name, config.Scheme, config.Host)

	sub := logyard.Broker.Subscribe(config.Filters...)
	defer sub.Stop()

	go d.finishedStarting(true)

	for {
		select {
		case msg := <-sub.Ch:
			data, err := config.FormatJSON(msg)
			if err != nil {
				d.Kill(err)
				return
			}
			_, err = conn.Write(data)
			if err != nil {
				d.Kill(err)
				return
			}
		case <-d.Dying():
			return
		}
	}
}