func main() {
configPath := flag.String("config", filepath.FromSlash("/etc/hekad.toml"),
"Config file or directory. If directory is specified then all files "+
"in the directory will be loaded.")
version := flag.Bool("version", false, "Output version and exit")
flag.Parse()
config := &HekadConfig{}
var err error
var cpuProfName string
var memProfName string
if *version {
fmt.Println(VERSION)
os.Exit(0)
}
config, err = LoadHekadConfig(*configPath)
if err != nil {
pipeline.LogError.Fatal("Error reading config: ", err)
}
if config.SampleDenominator <= 0 {
pipeline.LogError.Fatalln("'sample_denominator' value must be greater than 0.")
}
globals, cpuProfName, memProfName := setGlobalConfigs(config)
if err = os.MkdirAll(globals.BaseDir, 0755); err != nil {
pipeline.LogError.Fatalf("Error creating 'base_dir' %s: %s", config.BaseDir, err)
}
if config.MaxMessageSize > 1024 {
message.SetMaxMessageSize(config.MaxMessageSize)
} else if config.MaxMessageSize > 0 {
pipeline.LogError.Fatalln("Error: 'max_message_size' setting must be greater than 1024.")
}
if config.PidFile != "" {
contents, err := ioutil.ReadFile(config.PidFile)
if err == nil {
pid, err := strconv.Atoi(strings.TrimSpace(string(contents)))
if err != nil {
pipeline.LogError.Fatalf("Error reading proccess id from pidfile '%s': %s",
config.PidFile, err)
}
process, err := os.FindProcess(pid)
// on Windows, err != nil if the process cannot be found
if runtime.GOOS == "windows" {
if err == nil {
pipeline.LogError.Fatalf("Process %d is already running.", pid)
}
} else if process != nil {
// err is always nil on POSIX, so we have to send the process
// a signal to check whether it exists
if err = process.Signal(syscall.Signal(0)); err == nil {
pipeline.LogError.Fatalf("Process %d is already running.", pid)
}
}
}
if err = ioutil.WriteFile(config.PidFile, []byte(strconv.Itoa(os.Getpid())),
0644); err != nil {
pipeline.LogError.Fatalf("Unable to write pidfile '%s': %s", config.PidFile, err)
}
pipeline.LogInfo.Printf("Wrote pid to pidfile '%s'", config.PidFile)
defer func() {
if err = os.Remove(config.PidFile); err != nil {
pipeline.LogError.Printf("Unable to remove pidfile '%s': %s", config.PidFile, err)
}
}()
}
if cpuProfName != "" {
profFile, err := os.Create(cpuProfName)
if err != nil {
pipeline.LogError.Fatalln(err)
}
pprof.StartCPUProfile(profFile)
defer func() {
pprof.StopCPUProfile()
profFile.Close()
}()
}
if memProfName != "" {
defer func() {
profFile, err := os.Create(memProfName)
if err != nil {
pipeline.LogError.Fatalln(err)
}
pprof.WriteHeapProfile(profFile)
profFile.Close()
}()
}
// Set up and load the pipeline configuration and start the daemon.
pipeconf := pipeline.NewPipelineConfig(globals)
if err = loadFullConfig(pipeconf, configPath); err != nil {
pipeline.LogError.Fatal("Error reading config: ", err)
}
pipeline.Run(pipeconf)
}
func main() {
flagMatch := flag.String("match", "TRUE", "message_matcher filter expression")
flagFormat := flag.String("format", "txt", "output format [txt|json|heka|count]")
flagOutput := flag.String("output", "", "output filename, defaults to stdout")
flagTail := flag.Bool("tail", false, "don't exit on EOF")
flagOffset := flag.Int64("offset", 0, "starting offset for the input file in bytes")
flagMaxMessageSize := flag.Uint64("max-message-size", 4*1024*1024, "maximum message size in bytes")
flag.Parse()
if flag.NArg() != 1 {
flag.PrintDefaults()
os.Exit(1)
}
if *flagMaxMessageSize < math.MaxUint32 {
maxSize := uint32(*flagMaxMessageSize)
message.SetMaxMessageSize(maxSize)
} else {
fmt.Printf("Message size is too large: %d\n", flagMaxMessageSize)
os.Exit(8)
}
var err error
var match *message.MatcherSpecification
if match, err = message.CreateMatcherSpecification(*flagMatch); err != nil {
fmt.Printf("Match specification - %s\n", err)
os.Exit(2)
}
var file *os.File
if file, err = os.Open(flag.Arg(0)); err != nil {
fmt.Printf("%s\n", err)
os.Exit(3)
}
defer file.Close()
var out *os.File
if "" == *flagOutput {
out = os.Stdout
} else {
if out, err = os.OpenFile(*flagOutput, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil {
fmt.Printf("%s\n", err)
os.Exit(4)
}
defer out.Close()
}
var offset int64
if offset, err = file.Seek(*flagOffset, 0); err != nil {
fmt.Printf("%s\n", err)
os.Exit(5)
}
sRunner, err := makeSplitterRunner()
if err != nil {
fmt.Println(err)
os.Exit(7)
}
msg := new(message.Message)
var processed, matched int64
fmt.Printf("Input:%s Offset:%d Match:%s Format:%s Tail:%t Output:%s\n",
flag.Arg(0), *flagOffset, *flagMatch, *flagFormat, *flagTail, *flagOutput)
for true {
n, record, err := sRunner.GetRecordFromStream(file)
if n > 0 && n != len(record) {
fmt.Printf("Corruption detected at offset: %d bytes: %d\n", offset, n-len(record))
}
if err != nil {
if err == io.EOF {
if !*flagTail || "count" == *flagFormat {
break
}
time.Sleep(time.Duration(500) * time.Millisecond)
} else {
break
}
} else {
if len(record) > 0 {
processed += 1
headerLen := int(record[1]) + message.HEADER_FRAMING_SIZE
if err = proto.Unmarshal(record[headerLen:], msg); err != nil {
fmt.Printf("Error unmarshalling message at offset: %d error: %s\n", offset, err)
continue
}
if !match.Match(msg) {
continue
}
matched += 1
switch *flagFormat {
case "count":
// no op
case "json":
contents, _ := json.Marshal(msg)
fmt.Fprintf(out, "%s\n", contents)
case "heka":
fmt.Fprintf(out, "%s", record)
default:
fmt.Fprintf(out, "Timestamp: %s\n"+
"Type: %s\n"+
"Hostname: %s\n"+
"Pid: %d\n"+
"UUID: %s\n"+
"Logger: %s\n"+
"Payload: %s\n"+
"EnvVersion: %s\n"+
"Severity: %d\n"+
"Fields: %+v\n\n",
time.Unix(0, msg.GetTimestamp()), msg.GetType(),
msg.GetHostname(), msg.GetPid(), msg.GetUuidString(),
msg.GetLogger(), msg.GetPayload(), msg.GetEnvVersion(),
msg.GetSeverity(), msg.Fields)
}
}
}
offset += int64(n)
}
fmt.Printf("Processed: %d, matched: %d messages\n", processed, matched)
if err != nil {
fmt.Printf("%s\n", err)
os.Exit(6)
}
}
func main() {
configFile := flag.String("config", "flood.toml", "Heka Flood configuration file")
configTest := flag.String("test", "default", "Test section to load")
flag.Parse()
if flag.NFlag() == 0 {
flag.PrintDefaults()
os.Exit(0)
}
var config FloodConfig
if _, err := toml.DecodeFile(*configFile, &config); err != nil {
client.LogError.Printf("Error decoding config file: %s", err)
return
}
var test FloodTest
var ok bool
if test, ok = config[*configTest]; !ok {
client.LogError.Printf("Configuration test: '%s' was not found", *configTest)
return
}
if test.MsgInterval != "" {
var err error
if test.msgInterval, err = time.ParseDuration(test.MsgInterval); err != nil {
client.LogError.Printf("Invalid message_interval duration %s: %s", test.MsgInterval,
err.Error())
return
}
}
if test.PprofFile != "" {
profFile, err := os.Create(test.PprofFile)
if err != nil {
client.LogError.Fatalln(err)
}
pprof.StartCPUProfile(profFile)
defer pprof.StopCPUProfile()
}
if test.MaxMessageSize > 0 {
message.SetMaxMessageSize(test.MaxMessageSize)
}
var sender *client.NetworkSender
var err error
if test.UseTls {
var goTlsConfig *tls.Config
goTlsConfig, err = tcp.CreateGoTlsConfig(&test.Tls)
if err != nil {
client.LogError.Fatalf("Error creating TLS config: %s\n", err)
}
sender, err = client.NewTlsSender(test.Sender, test.IpAddress, goTlsConfig)
} else {
sender, err = client.NewNetworkSender(test.Sender, test.IpAddress)
}
if err != nil {
client.LogError.Fatalf("Error creating sender: %s\n", err)
}
unsignedEncoder := client.NewProtobufEncoder(nil)
signedEncoder := client.NewProtobufEncoder(&test.Signer)
oversizedEncoder := &OversizedEncoder{}
var numTestMessages = 1
var unsignedMessages [][]byte
var signedMessages [][]byte
var oversizedMessages [][]byte
rdm := &randomDataMaker{
src: rand.NewSource(time.Now().UnixNano()),
asciiOnly: test.AsciiOnly,
}
if test.VariableSizeMessages {
numTestMessages = 64
unsignedMessages = makeVariableMessage(unsignedEncoder, numTestMessages, rdm, false)
signedMessages = makeVariableMessage(signedEncoder, numTestMessages, rdm, false)
oversizedMessages = makeVariableMessage(oversizedEncoder, 1, rdm, true)
} else {
if test.StaticMessageSize == 0 {
test.StaticMessageSize = 1000
}
unsignedMessages = makeFixedMessage(unsignedEncoder, test.StaticMessageSize,
rdm)
signedMessages = makeFixedMessage(signedEncoder, test.StaticMessageSize,
rdm)
}
// wait for sigint
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT)
var msgsSent, bytesSent, msgsDelivered uint64
var corruptPercentage, lastCorruptPercentage, signedPercentage, lastSignedPercentage, oversizedPercentage, lastOversizedPercentage float64
var corrupt bool
// set up counter loop
ticker := time.NewTicker(time.Duration(time.Second))
go timerLoop(&msgsSent, &bytesSent, ticker)
test.CorruptPercentage /= 100.0
test.SignedPercentage /= 100.0
test.OversizedPercentage /= 100.0
var buf []byte
for gotsigint := false; !gotsigint; {
runtime.Gosched()
select {
case <-sigChan:
gotsigint = true
continue
default:
}
msgId := rand.Int() % numTestMessages
corruptPercentage = math.Floor(float64(msgsSent) * test.CorruptPercentage)
if corruptPercentage != lastCorruptPercentage {
lastCorruptPercentage = corruptPercentage
corrupt = true
} else {
corrupt = false
}
signedPercentage = math.Floor(float64(msgsSent) * test.SignedPercentage)
if signedPercentage != lastSignedPercentage {
lastSignedPercentage = signedPercentage
buf = signedMessages[msgId]
} else {
oversizedPercentage = math.Floor(float64(msgsSent) * test.OversizedPercentage)
if oversizedPercentage != lastOversizedPercentage {
lastOversizedPercentage = oversizedPercentage
buf = oversizedMessages[0]
} else {
buf = unsignedMessages[msgId]
}
}
bytesSent += uint64(len(buf))
if err = sendMessage(sender, buf, corrupt); err != nil {
client.LogError.Printf("Error sending message: %s\n", err.Error())
} else {
msgsDelivered++
}
msgsSent++
if test.NumMessages != 0 && msgsSent >= test.NumMessages {
break
}
if test.msgInterval != 0 {
time.Sleep(test.msgInterval)
}
}
sender.Close()
client.LogInfo.Println("Clean shutdown: ", msgsSent, " messages sent; ",
msgsDelivered, " messages delivered.")
}
func main() {
flagMatch := flag.String("match", "TRUE", "message_matcher filter expression")
flagFormat := flag.String("format", "txt", "output format [txt|json|heka|count]")
flagOutput := flag.String("output", "", "output filename, defaults to stdout")
flagStdin := flag.Bool("stdin", false, "read list of s3 key names from stdin")
flagBucket := flag.String("bucket", "default-bucket", "S3 Bucket name")
flagAWSKey := flag.String("aws-key", "", "AWS Key")
flagAWSSecretKey := flag.String("aws-secret-key", "", "AWS Secret Key")
flagAWSRegion := flag.String("aws-region", "us-west-2", "AWS Region")
flagMaxMessageSize := flag.Uint64("max-message-size", 4*1024*1024, "maximum message size in bytes")
flagWorkers := flag.Uint64("workers", 16, "number of parallel workers")
flagConnectTimeout := flag.Uint64("connect_timeout", 60, "Max seconds to wait for an S3 connection")
flagReadTimeout := flag.Uint64("read_timeout", 300, "Max seconds to wait for an S3 file read to complete")
flag.Parse()
if !*flagStdin && flag.NArg() < 1 {
flag.PrintDefaults()
os.Exit(1)
}
if *flagMaxMessageSize < math.MaxUint32 {
maxSize := uint32(*flagMaxMessageSize)
message.SetMaxMessageSize(maxSize)
} else {
fmt.Fprintf(os.Stderr, "Message size is too large: %d\n", flagMaxMessageSize)
os.Exit(8)
}
workers := 1
if *flagWorkers == 0 {
fmt.Fprintf(os.Stderr, "Cannot run with zero workers. Using 1.\n")
} else if *flagWorkers < 2000 {
workers = int(*flagWorkers)
} else {
fmt.Fprintf(os.Stderr, "Too many workers: %d. Use a reasonable value (up to a few hundred).\n", flagWorkers)
os.Exit(8)
}
var connectTimeout uint32
if *flagConnectTimeout < math.MaxUint32 {
connectTimeout = uint32(*flagConnectTimeout)
} else {
fmt.Fprintf(os.Stderr, "Connection Timeout is too large:%d.\n", flagConnectTimeout)
os.Exit(8)
}
var readTimeout uint32
if *flagReadTimeout < math.MaxUint32 {
readTimeout = uint32(*flagReadTimeout)
} else {
fmt.Fprintf(os.Stderr, "Read Timeout is too large:%d.\n", flagReadTimeout)
os.Exit(8)
}
var err error
var match *message.MatcherSpecification
if match, err = message.CreateMatcherSpecification(*flagMatch); err != nil {
fmt.Fprintf(os.Stderr, "Match specification - %s\n", err)
os.Exit(2)
}
var out *os.File
if "" == *flagOutput {
out = os.Stdout
} else {
if out, err = os.OpenFile(*flagOutput, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
os.Exit(3)
}
defer out.Close()
}
auth, err := aws.GetAuth(*flagAWSKey, *flagAWSSecretKey, "", time.Now())
if err != nil {
fmt.Fprintf(os.Stderr, "Authentication error: %s\n", err)
os.Exit(4)
}
region, ok := aws.Regions[*flagAWSRegion]
if !ok {
fmt.Fprintf(os.Stderr, "Parameter 'aws-region' must be a valid AWS Region\n")
os.Exit(5)
}
s := s3.New(auth, region)
if connectTimeout > 0 {
s.ConnectTimeout = time.Duration(connectTimeout) * time.Second
}
if readTimeout > 0 {
s.ReadTimeout = time.Duration(readTimeout) * time.Second
}
bucket := s.Bucket(*flagBucket)
filenameChannel := make(chan string, 1000)
recordChannel := make(chan s3splitfile.S3Record, 1000)
doneChannel := make(chan string, 1000)
allDone := make(chan int)
for i := 1; i <= workers; i++ {
go cat(bucket, filenameChannel, recordChannel, doneChannel)
}
go save(recordChannel, match, *flagFormat, out, allDone)
startTime := time.Now().UTC()
totalFiles := 0
pendingFiles := 0
if *flagStdin {
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
filename := scanner.Text()
totalFiles++
pendingFiles++
filenameChannel <- filename
if pendingFiles >= 1000 {
waitFor(doneChannel, 1)
pendingFiles--
}
}
close(filenameChannel)
} else {
for _, filename := range flag.Args() {
totalFiles++
pendingFiles++
filenameChannel <- filename
if pendingFiles >= 1000 {
waitFor(doneChannel, 1)
pendingFiles--
}
}
close(filenameChannel)
}
fmt.Fprintf(os.Stderr, "Waiting for last %d files\n", pendingFiles)
waitFor(doneChannel, pendingFiles)
close(recordChannel)
bytesRead := <-allDone
// All done! Win!
duration := time.Now().UTC().Sub(startTime).Seconds()
mb := float64(bytesRead) / 1024.0 / 1024.0
if duration == 0.0 {
duration = 1.0
}
fmt.Fprintf(os.Stderr, "All done processing %d files, %.2fMB in %.2f seconds (%.2fMB/s)\n", totalFiles, mb, duration, (mb / duration))
}