func (d *FileDrain) Start(config *DrainConfig) {
defer d.Done()
overwrite, err := config.GetParamBool("overwrite", false)
if err != nil {
d.Kill(err)
go d.finishedStarting(false)
return
}
mode := os.O_WRONLY | os.O_CREATE
if overwrite {
mode |= os.O_TRUNC
} else {
mode |= os.O_APPEND
}
log.Infof("[drain:%s] Attempting to open %s (overwrite=%v) ...",
d.name, config.Path, overwrite)
f, err := os.OpenFile(config.Path, mode, 0600)
if err != nil {
d.Kill(err)
go d.finishedStarting(false)
return
}
log.Infof("[drain:%s] Successfully opened %s.", d.name, config.Path)
defer f.Close()
sub := logyard.Broker.Subscribe(config.Filters...)
defer sub.Stop()
go d.finishedStarting(true)
for {
select {
case msg := <-sub.Ch:
data, err := config.FormatJSON(msg)
if err != nil {
d.Kill(err)
return
}
_, err = f.Write(data)
if err != nil {
d.Kill(err)
return
}
case <-d.Dying():
return
}
}
}
func (d *RedisDrain) connect(addr string, database int64) error {
log.Infof("[drain:%s] Attempting to connect to redis %s[#%d] ...",
d.name, addr, database)
if client, err := server.NewRedisClient(
addr, "", database); err != nil {
return err
} else {
d.client = client
log.Infof("[drain:%s] Successfully connected to redis %s[#%d].",
d.name, addr, database)
return nil
}
panic("unreachable")
}
func main() {
var useUDP bool
var port int
flag.BoolVar(&useUDP, "u", false, "use UDP instead of TCP")
flag.IntVar(&port, "p", 9090, "port number to bind to")
flag.Parse()
var srv *lineserver.LineServer
var err error
addr := fmt.Sprintf(":%d", port)
proto := "tcp"
if useUDP {
proto = "udp"
}
if useUDP {
srv, err = lineserver.NewLineServerUDP(addr)
} else {
srv, err = lineserver.NewLineServerTCP(addr)
}
if err != nil {
log.Fatal(err)
}
log.Infof("Server running as %s://%s", proto, addr)
go srv.Start()
for line := range srv.Ch {
fmt.Println(line)
}
}
func (manager *DrainManager) Run() {
iteration := 0
drains := logyard.GetConfig().Drains
log.Infof("Found %d drains to start\n", len(drains))
for name, uri := range drains {
manager.StartDrain(name, uri, NewRetryerForDrain(name))
}
// Watch for config changes in redis.
for {
iteration += 1
prefix := fmt.Sprintf("CONFIG.%d", iteration)
select {
case err := <-logyard.GetConfigChanges():
if err != nil {
log.Fatalf("Error re-loading config: %v", err)
}
log.Infof(
"[%s] checking drains after a config change...",
prefix)
newDrains := logyard.GetConfig().Drains
for _, c := range mapdiff.MapDiff(drains, newDrains) {
if c.Deleted {
log.Infof("[%s] Drain %s was deleted.", prefix, c.Key)
manager.StopDrain(c.Key, true)
delete(drains, c.Key)
} else {
log.Infof("[%s] Drain %s was added.", prefix, c.Key)
manager.StopDrain(c.Key, false)
manager.StartDrain(
c.Key,
c.NewValue,
NewRetryerForDrain(c.Key))
drains[c.Key] = c.NewValue
}
}
log.Infof("[%s] Done checking drains.", prefix)
case <-manager.stopCh:
break
}
}
}
func init() {
Broker.PubAddr = "ipc:///var/stackato/run/logyardpub.sock"
Broker.SubAddr = "ipc:///var/stackato/run/logyardsub.sock"
Broker.BufferSize = 100
// ZeroMQ will silently fail if .sock files can't be created
if _, err := os.Stat("/var/stackato/run"); err != nil {
log.Errorf("Cannot access /var/stackato/run: %v", err)
}
log.Infof("Loygard broker config: %+v\n", Broker)
}
// NewRetryerForDrain chooses
func NewRetryerForDrain(name string) retry.Retryer {
var retryLimit time.Duration
var err error
for prefix, duration := range logyard.GetConfig().RetryLimits {
if strings.HasPrefix(name, prefix) {
if retryLimit, err = time.ParseDuration(duration); err != nil {
log.Error("[drain:%s] Invalid duration (%s) for drain prefix %s "+
"-- %s -- using default value (infinite)",
name, duration, prefix, err)
retryLimit = time.Duration(0)
}
if retryLimit <= retry.RESET_AFTER {
log.Error("[drain:%s] Invalid retry limit (%v) -- must be >%v -- "+
"using default value (infinite)",
name, retryLimit, retry.RESET_AFTER)
retryLimit = time.Duration(0)
}
break
}
}
log.Infof("[drain:%s] Choosing retry limit %v", name, retryLimit)
return retry.NewProgressiveRetryer(retryLimit)
}
func main() {
major, minor, patch := gozmq.Version()
log.Infof("Starting logyard (Go %s; ZeroMQ %d.%d.%d)",
runtime.Version(), major, minor, patch)
m := drain.NewDrainManager()
log.Info("Starting drain manager")
go m.Run()
// SIGTERM handle for stopping running drains.
go func() {
sigchan := make(chan os.Signal)
signal.Notify(sigchan, syscall.SIGTERM)
<-sigchan
log.Info("Stopping all drains before exiting")
m.Stop()
log.Info("Exiting now.")
os.Exit(0)
}()
server.MarkRunning("logyard")
log.Info("Running pubsub broker")
log.Fatal(logyard.Broker.Run())
}
func (cmd *stream) Run(args []string) (string, error) {
ipaddr, err := server.LocalIP()
if err != nil {
return "", err
}
rand.Seed(time.Now().UnixNano())
port := 7000 + rand.Intn(1000)
addr := fmt.Sprintf("%s:%d", ipaddr, port)
srv, err := lineserver.NewLineServer(CLI_STREAM_PROTO, addr)
if err != nil {
return "", err
}
go srv.Start()
// Debug mode allows one to debug just the logyard related logs,
// without any magical stripping.
debugMode := false
if len(args) == 1 && args[0] == "debug" {
debugMode = true
args = []string{
"systail.logyard",
"systail.apptail",
"systail.logyard_sieve",
"systail.systail",
}
}
name := fmt.Sprintf("tmp.logyard-cli.%s-%d", ipaddr, port)
uri, err := drain.ConstructDrainURI(
name,
fmt.Sprintf("%s://%s", CLI_STREAM_PROTO, addr),
args,
map[string]string{"format": "raw"})
if err != nil {
return "", err
}
if err = logyard.AddDrain(name, uri); err != nil {
return "", err
}
log.Infof("Added drain %s", uri)
deleteDrain := func() {
if err := logyard.DeleteDrain(name); err != nil {
log.Fatal(err)
}
fmt.Println("")
log.Infof("Deleted drain %s", name)
}
defer deleteDrain()
handleKeyboardInterrupt(func() {
deleteDrain()
os.Exit(1)
})
cli_stream.Stream(srv.Ch, cli_stream.MessagePrinterOptions{
cmd.raw, cmd.raw || debugMode, cmd.time, cmd.nocolor, cmd.nodeid, cmd.json})
return "", nil
}
func (d *IPConnDrain) Start(config *DrainConfig) {
defer d.Done()
if !(config.Scheme == "udp" || config.Scheme == "tcp") {
d.Killf("Invalid scheme: %s", config.Scheme)
go d.finishedStarting(false)
return
}
log.Infof("[drain:%s] Attempting to connect to %s://%s ...",
d.name, config.Scheme, config.Host)
var conn net.Conn
dialer := NewNetDialer(config.Scheme, config.Host, 10*time.Second)
select {
case conn = <-dialer.Ch:
if dialer.Error != nil {
d.Kill(dialer.Error)
go d.finishedStarting(false)
return
}
case <-d.Dying():
// Close the connection returned in future by the dialer.
log.Infof("[drain:%s] Stop request; deferring close of connection",
d.name)
go dialer.WaitAndClose()
// [bug 105165].
// did not attain running state. No kill however as
// getting aborted by the user is not really an
// error. The state machine will treat it as a regular
// non-error exit.
go d.finishedStarting(false)
return
}
defer conn.Close()
log.Infof("[drain:%s] Successfully connected to %s://%s.",
d.name, config.Scheme, config.Host)
sub := logyard.Broker.Subscribe(config.Filters...)
defer sub.Stop()
go d.finishedStarting(true)
for {
select {
case msg := <-sub.Ch:
data, err := config.FormatJSON(msg)
if err != nil {
d.Kill(err)
return
}
_, err = conn.Write(data)
if err != nil {
d.Kill(err)
return
}
case <-d.Dying():
return
}
}
}