func main() { configPath := flag.String("config", filepath.FromSlash("/etc/hekad.toml"), "Config file or directory. If directory is specified then all files "+ "in the directory will be loaded.") version := flag.Bool("version", false, "Output version and exit") flag.Parse() config := &HekadConfig{} var err error var cpuProfName string var memProfName string if *version { fmt.Println(VERSION) os.Exit(0) } config, err = LoadHekadConfig(*configPath) if err != nil { pipeline.LogError.Fatal("Error reading config: ", err) } if config.SampleDenominator <= 0 { pipeline.LogError.Fatalln("'sample_denominator' value must be greater than 0.") } globals, cpuProfName, memProfName := setGlobalConfigs(config) if err = os.MkdirAll(globals.BaseDir, 0755); err != nil { pipeline.LogError.Fatalf("Error creating 'base_dir' %s: %s", config.BaseDir, err) } if config.MaxMessageSize > 1024 { message.SetMaxMessageSize(config.MaxMessageSize) } else if config.MaxMessageSize > 0 { pipeline.LogError.Fatalln("Error: 'max_message_size' setting must be greater than 1024.") } if config.PidFile != "" { contents, err := ioutil.ReadFile(config.PidFile) if err == nil { pid, err := strconv.Atoi(strings.TrimSpace(string(contents))) if err != nil { pipeline.LogError.Fatalf("Error reading proccess id from pidfile '%s': %s", config.PidFile, err) } process, err := os.FindProcess(pid) // on Windows, err != nil if the process cannot be found if runtime.GOOS == "windows" { if err == nil { pipeline.LogError.Fatalf("Process %d is already running.", pid) } } else if process != nil { // err is always nil on POSIX, so we have to send the process // a signal to check whether it exists if err = process.Signal(syscall.Signal(0)); err == nil { pipeline.LogError.Fatalf("Process %d is already running.", pid) } } } if err = ioutil.WriteFile(config.PidFile, []byte(strconv.Itoa(os.Getpid())), 0644); err != nil { pipeline.LogError.Fatalf("Unable to write pidfile '%s': %s", config.PidFile, err) } pipeline.LogInfo.Printf("Wrote pid to pidfile '%s'", config.PidFile) defer func() { if err = os.Remove(config.PidFile); err != nil { pipeline.LogError.Printf("Unable to remove pidfile '%s': %s", config.PidFile, err) } }() } if cpuProfName != "" { profFile, err := os.Create(cpuProfName) if err != nil { pipeline.LogError.Fatalln(err) } pprof.StartCPUProfile(profFile) defer func() { pprof.StopCPUProfile() profFile.Close() }() } if memProfName != "" { defer func() { profFile, err := os.Create(memProfName) if err != nil { pipeline.LogError.Fatalln(err) } pprof.WriteHeapProfile(profFile) profFile.Close() }() } // Set up and load the pipeline configuration and start the daemon. pipeconf := pipeline.NewPipelineConfig(globals) if err = loadFullConfig(pipeconf, configPath); err != nil { pipeline.LogError.Fatal("Error reading config: ", err) } pipeline.Run(pipeconf) }
func main() { flagMatch := flag.String("match", "TRUE", "message_matcher filter expression") flagFormat := flag.String("format", "txt", "output format [txt|json|heka|count]") flagOutput := flag.String("output", "", "output filename, defaults to stdout") flagTail := flag.Bool("tail", false, "don't exit on EOF") flagOffset := flag.Int64("offset", 0, "starting offset for the input file in bytes") flagMaxMessageSize := flag.Uint64("max-message-size", 4*1024*1024, "maximum message size in bytes") flag.Parse() if flag.NArg() != 1 { flag.PrintDefaults() os.Exit(1) } if *flagMaxMessageSize < math.MaxUint32 { maxSize := uint32(*flagMaxMessageSize) message.SetMaxMessageSize(maxSize) } else { fmt.Printf("Message size is too large: %d\n", flagMaxMessageSize) os.Exit(8) } var err error var match *message.MatcherSpecification if match, err = message.CreateMatcherSpecification(*flagMatch); err != nil { fmt.Printf("Match specification - %s\n", err) os.Exit(2) } var file *os.File if file, err = os.Open(flag.Arg(0)); err != nil { fmt.Printf("%s\n", err) os.Exit(3) } defer file.Close() var out *os.File if "" == *flagOutput { out = os.Stdout } else { if out, err = os.OpenFile(*flagOutput, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil { fmt.Printf("%s\n", err) os.Exit(4) } defer out.Close() } var offset int64 if offset, err = file.Seek(*flagOffset, 0); err != nil { fmt.Printf("%s\n", err) os.Exit(5) } sRunner, err := makeSplitterRunner() if err != nil { fmt.Println(err) os.Exit(7) } msg := new(message.Message) var processed, matched int64 fmt.Printf("Input:%s Offset:%d Match:%s Format:%s Tail:%t Output:%s\n", flag.Arg(0), *flagOffset, *flagMatch, *flagFormat, *flagTail, *flagOutput) for true { n, record, err := sRunner.GetRecordFromStream(file) if n > 0 && n != len(record) { fmt.Printf("Corruption detected at offset: %d bytes: %d\n", offset, n-len(record)) } if err != nil { if err == io.EOF { if !*flagTail || "count" == *flagFormat { break } time.Sleep(time.Duration(500) * time.Millisecond) } else { break } } else { if len(record) > 0 { processed += 1 headerLen := int(record[1]) + message.HEADER_FRAMING_SIZE if err = proto.Unmarshal(record[headerLen:], msg); err != nil { fmt.Printf("Error unmarshalling message at offset: %d error: %s\n", offset, err) continue } if !match.Match(msg) { continue } matched += 1 switch *flagFormat { case "count": // no op case "json": contents, _ := json.Marshal(msg) fmt.Fprintf(out, "%s\n", contents) case "heka": fmt.Fprintf(out, "%s", record) default: fmt.Fprintf(out, "Timestamp: %s\n"+ "Type: %s\n"+ "Hostname: %s\n"+ "Pid: %d\n"+ "UUID: %s\n"+ "Logger: %s\n"+ "Payload: %s\n"+ "EnvVersion: %s\n"+ "Severity: %d\n"+ "Fields: %+v\n\n", time.Unix(0, msg.GetTimestamp()), msg.GetType(), msg.GetHostname(), msg.GetPid(), msg.GetUuidString(), msg.GetLogger(), msg.GetPayload(), msg.GetEnvVersion(), msg.GetSeverity(), msg.Fields) } } } offset += int64(n) } fmt.Printf("Processed: %d, matched: %d messages\n", processed, matched) if err != nil { fmt.Printf("%s\n", err) os.Exit(6) } }
func main() { configFile := flag.String("config", "flood.toml", "Heka Flood configuration file") configTest := flag.String("test", "default", "Test section to load") flag.Parse() if flag.NFlag() == 0 { flag.PrintDefaults() os.Exit(0) } var config FloodConfig if _, err := toml.DecodeFile(*configFile, &config); err != nil { client.LogError.Printf("Error decoding config file: %s", err) return } var test FloodTest var ok bool if test, ok = config[*configTest]; !ok { client.LogError.Printf("Configuration test: '%s' was not found", *configTest) return } if test.MsgInterval != "" { var err error if test.msgInterval, err = time.ParseDuration(test.MsgInterval); err != nil { client.LogError.Printf("Invalid message_interval duration %s: %s", test.MsgInterval, err.Error()) return } } if test.PprofFile != "" { profFile, err := os.Create(test.PprofFile) if err != nil { client.LogError.Fatalln(err) } pprof.StartCPUProfile(profFile) defer pprof.StopCPUProfile() } if test.MaxMessageSize > 0 { message.SetMaxMessageSize(test.MaxMessageSize) } var sender *client.NetworkSender var err error if test.UseTls { var goTlsConfig *tls.Config goTlsConfig, err = tcp.CreateGoTlsConfig(&test.Tls) if err != nil { client.LogError.Fatalf("Error creating TLS config: %s\n", err) } sender, err = client.NewTlsSender(test.Sender, test.IpAddress, goTlsConfig) } else { sender, err = client.NewNetworkSender(test.Sender, test.IpAddress) } if err != nil { client.LogError.Fatalf("Error creating sender: %s\n", err) } unsignedEncoder := client.NewProtobufEncoder(nil) signedEncoder := client.NewProtobufEncoder(&test.Signer) oversizedEncoder := &OversizedEncoder{} var numTestMessages = 1 var unsignedMessages [][]byte var signedMessages [][]byte var oversizedMessages [][]byte rdm := &randomDataMaker{ src: rand.NewSource(time.Now().UnixNano()), asciiOnly: test.AsciiOnly, } if test.VariableSizeMessages { numTestMessages = 64 unsignedMessages = makeVariableMessage(unsignedEncoder, numTestMessages, rdm, false) signedMessages = makeVariableMessage(signedEncoder, numTestMessages, rdm, false) oversizedMessages = makeVariableMessage(oversizedEncoder, 1, rdm, true) } else { if test.StaticMessageSize == 0 { test.StaticMessageSize = 1000 } unsignedMessages = makeFixedMessage(unsignedEncoder, test.StaticMessageSize, rdm) signedMessages = makeFixedMessage(signedEncoder, test.StaticMessageSize, rdm) } // wait for sigint sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT) var msgsSent, bytesSent, msgsDelivered uint64 var corruptPercentage, lastCorruptPercentage, signedPercentage, lastSignedPercentage, oversizedPercentage, lastOversizedPercentage float64 var corrupt bool // set up counter loop ticker := time.NewTicker(time.Duration(time.Second)) go timerLoop(&msgsSent, &bytesSent, ticker) test.CorruptPercentage /= 100.0 test.SignedPercentage /= 100.0 test.OversizedPercentage /= 100.0 var buf []byte for gotsigint := false; !gotsigint; { runtime.Gosched() select { case <-sigChan: gotsigint = true continue default: } msgId := rand.Int() % numTestMessages corruptPercentage = math.Floor(float64(msgsSent) * test.CorruptPercentage) if corruptPercentage != lastCorruptPercentage { lastCorruptPercentage = corruptPercentage corrupt = true } else { corrupt = false } signedPercentage = math.Floor(float64(msgsSent) * test.SignedPercentage) if signedPercentage != lastSignedPercentage { lastSignedPercentage = signedPercentage buf = signedMessages[msgId] } else { oversizedPercentage = math.Floor(float64(msgsSent) * test.OversizedPercentage) if oversizedPercentage != lastOversizedPercentage { lastOversizedPercentage = oversizedPercentage buf = oversizedMessages[0] } else { buf = unsignedMessages[msgId] } } bytesSent += uint64(len(buf)) if err = sendMessage(sender, buf, corrupt); err != nil { client.LogError.Printf("Error sending message: %s\n", err.Error()) } else { msgsDelivered++ } msgsSent++ if test.NumMessages != 0 && msgsSent >= test.NumMessages { break } if test.msgInterval != 0 { time.Sleep(test.msgInterval) } } sender.Close() client.LogInfo.Println("Clean shutdown: ", msgsSent, " messages sent; ", msgsDelivered, " messages delivered.") }
func main() { flagMatch := flag.String("match", "TRUE", "message_matcher filter expression") flagFormat := flag.String("format", "txt", "output format [txt|json|heka|count]") flagOutput := flag.String("output", "", "output filename, defaults to stdout") flagStdin := flag.Bool("stdin", false, "read list of s3 key names from stdin") flagBucket := flag.String("bucket", "default-bucket", "S3 Bucket name") flagAWSKey := flag.String("aws-key", "", "AWS Key") flagAWSSecretKey := flag.String("aws-secret-key", "", "AWS Secret Key") flagAWSRegion := flag.String("aws-region", "us-west-2", "AWS Region") flagMaxMessageSize := flag.Uint64("max-message-size", 4*1024*1024, "maximum message size in bytes") flagWorkers := flag.Uint64("workers", 16, "number of parallel workers") flagConnectTimeout := flag.Uint64("connect_timeout", 60, "Max seconds to wait for an S3 connection") flagReadTimeout := flag.Uint64("read_timeout", 300, "Max seconds to wait for an S3 file read to complete") flag.Parse() if !*flagStdin && flag.NArg() < 1 { flag.PrintDefaults() os.Exit(1) } if *flagMaxMessageSize < math.MaxUint32 { maxSize := uint32(*flagMaxMessageSize) message.SetMaxMessageSize(maxSize) } else { fmt.Fprintf(os.Stderr, "Message size is too large: %d\n", flagMaxMessageSize) os.Exit(8) } workers := 1 if *flagWorkers == 0 { fmt.Fprintf(os.Stderr, "Cannot run with zero workers. Using 1.\n") } else if *flagWorkers < 2000 { workers = int(*flagWorkers) } else { fmt.Fprintf(os.Stderr, "Too many workers: %d. Use a reasonable value (up to a few hundred).\n", flagWorkers) os.Exit(8) } var connectTimeout uint32 if *flagConnectTimeout < math.MaxUint32 { connectTimeout = uint32(*flagConnectTimeout) } else { fmt.Fprintf(os.Stderr, "Connection Timeout is too large:%d.\n", flagConnectTimeout) os.Exit(8) } var readTimeout uint32 if *flagReadTimeout < math.MaxUint32 { readTimeout = uint32(*flagReadTimeout) } else { fmt.Fprintf(os.Stderr, "Read Timeout is too large:%d.\n", flagReadTimeout) os.Exit(8) } var err error var match *message.MatcherSpecification if match, err = message.CreateMatcherSpecification(*flagMatch); err != nil { fmt.Fprintf(os.Stderr, "Match specification - %s\n", err) os.Exit(2) } var out *os.File if "" == *flagOutput { out = os.Stdout } else { if out, err = os.OpenFile(*flagOutput, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil { fmt.Fprintf(os.Stderr, "%s\n", err) os.Exit(3) } defer out.Close() } auth, err := aws.GetAuth(*flagAWSKey, *flagAWSSecretKey, "", time.Now()) if err != nil { fmt.Fprintf(os.Stderr, "Authentication error: %s\n", err) os.Exit(4) } region, ok := aws.Regions[*flagAWSRegion] if !ok { fmt.Fprintf(os.Stderr, "Parameter 'aws-region' must be a valid AWS Region\n") os.Exit(5) } s := s3.New(auth, region) if connectTimeout > 0 { s.ConnectTimeout = time.Duration(connectTimeout) * time.Second } if readTimeout > 0 { s.ReadTimeout = time.Duration(readTimeout) * time.Second } bucket := s.Bucket(*flagBucket) filenameChannel := make(chan string, 1000) recordChannel := make(chan s3splitfile.S3Record, 1000) doneChannel := make(chan string, 1000) allDone := make(chan int) for i := 1; i <= workers; i++ { go cat(bucket, filenameChannel, recordChannel, doneChannel) } go save(recordChannel, match, *flagFormat, out, allDone) startTime := time.Now().UTC() totalFiles := 0 pendingFiles := 0 if *flagStdin { scanner := bufio.NewScanner(os.Stdin) for scanner.Scan() { filename := scanner.Text() totalFiles++ pendingFiles++ filenameChannel <- filename if pendingFiles >= 1000 { waitFor(doneChannel, 1) pendingFiles-- } } close(filenameChannel) } else { for _, filename := range flag.Args() { totalFiles++ pendingFiles++ filenameChannel <- filename if pendingFiles >= 1000 { waitFor(doneChannel, 1) pendingFiles-- } } close(filenameChannel) } fmt.Fprintf(os.Stderr, "Waiting for last %d files\n", pendingFiles) waitFor(doneChannel, pendingFiles) close(recordChannel) bytesRead := <-allDone // All done! Win! duration := time.Now().UTC().Sub(startTime).Seconds() mb := float64(bytesRead) / 1024.0 / 1024.0 if duration == 0.0 { duration = 1.0 } fmt.Fprintf(os.Stderr, "All done processing %d files, %.2fMB in %.2f seconds (%.2fMB/s)\n", totalFiles, mb, duration, (mb / duration)) }