func getField(msg *message.Message, name string) interface{} {
switch name {
case "Uuid":
if msg.Uuid == nil {
return nil
}
return msg.GetUuidString()
case "Timestamp":
return msg.Timestamp
case "Type":
return msg.Type
case "Logger":
return msg.Logger
case "Severity":
return msg.Severity
case "Payload":
return msg.Payload
case "EnvVersion":
return msg.EnvVersion
case "Pid":
return msg.Pid
case "Hostname":
return msg.Hostname
case "_hekaTimestampMicro":
if msg.Timestamp != nil {
return *msg.Timestamp / 1000 // nano -> micro
}
return nil
default:
val, _ := msg.GetFieldValue(name)
return val
}
}
// convertMessageToValue reads a Heka Message and returns a slice of field values
func (po *PostgresOutput) convertMessageToValues(m *message.Message, insertFields []string) (fieldValues []interface{}, err error) {
fieldValues = []interface{}{}
missingFields := []string{}
for _, field := range insertFields {
// Special case: get "Timestamp" from Heka message
if field == "Timestamp" {
// Convert Heka time (Unix timestamp in nanoseconds) to Golang time
v := time.Unix(0, m.GetTimestamp())
fieldValues = append(fieldValues, v)
} else {
v, ok := m.GetFieldValue(field)
if !ok {
// If configured to do so, write NULL when a FieldValue isn't found in the Heka message
if po.allowMissingMessageFields {
v = nil
} else {
missingFields = append(missingFields, field)
continue
}
}
fieldValues = append(fieldValues, v)
}
}
if len(missingFields) > 0 {
return []interface{}{}, fmt.Errorf("message is missing expected fields: %s", strings.Join(missingFields, ", "))
}
return fieldValues, nil
}
// Parses a Heka message and extracts the information necessary to start a new
// SandboxFilter
func (this *SandboxManagerFilter) loadSandbox(fr pipeline.FilterRunner,
h pipeline.PluginHelper, dir string, msg *message.Message) (err error) {
fv, _ := msg.GetFieldValue("config")
if config, ok := fv.(string); ok {
var configFile pipeline.ConfigFile
if _, err = toml.Decode(config, &configFile); err != nil {
return fmt.Errorf("loadSandbox failed: %s\n", err)
} else {
for name, conf := range configFile {
name = getSandboxName(fr.Name(), name)
if _, ok := h.Filter(name); ok {
// todo support reload
return fmt.Errorf("loadSandbox failed: %s is already running", name)
}
fr.LogMessage(fmt.Sprintf("Loading: %s", name))
confFile := filepath.Join(dir, fmt.Sprintf("%s.toml", name))
err = ioutil.WriteFile(confFile, []byte(config), 0600)
if err != nil {
return
}
var sbc SandboxConfig
if err = toml.PrimitiveDecode(conf, &sbc); err != nil {
return fmt.Errorf("loadSandbox failed: %s\n", err)
}
scriptFile := filepath.Join(dir, fmt.Sprintf("%s.%s", name, sbc.ScriptType))
err = ioutil.WriteFile(scriptFile, []byte(msg.GetPayload()), 0600)
if err != nil {
removeAll(dir, fmt.Sprintf("%s.*", name))
return
}
// check/clear the old state preservation file
// this avoids issues with changes to the data model since the last load
// and prevents holes in the graph from looking like anomalies
os.Remove(filepath.Join(pipeline.PrependBaseDir(DATA_DIR), name+DATA_EXT))
var runner pipeline.FilterRunner
runner, err = this.createRunner(dir, name, conf)
if err != nil {
removeAll(dir, fmt.Sprintf("%s.*", name))
return
}
err = h.PipelineConfig().AddFilterRunner(runner)
if err == nil {
this.currentFilters++
}
break // only interested in the first item
}
}
}
return
}
func (self *DashboardOutput) Run(or OutputRunner, h PluginHelper) (err error) {
inChan := or.InChan()
ticker := or.Ticker()
var (
ok = true
plc *PipelineCapture
pack *PipelinePack
msg *message.Message
)
for ok {
select {
case plc, ok = <-inChan:
if !ok {
break
}
pack = plc.Pack
msg = pack.Message
switch msg.GetType() {
case "heka.all-report":
fn := path.Join(self.workingDirectory, "heka_report.json")
overwriteFile(fn, msg.GetPayload())
case "heka.sandbox-output":
tmp, ok := msg.GetFieldValue("payload_type")
if ok {
if pt, ok := tmp.(string); ok && pt == "cbuf" {
html := path.Join(self.workingDirectory, msg.GetLogger()+".html")
_, err := os.Stat(html)
if err != nil {
overwriteFile(html, fmt.Sprintf(getCbufTemplate(), msg.GetLogger(), msg.GetLogger()))
}
fn := path.Join(self.workingDirectory, msg.GetLogger()+"."+pt)
overwriteFile(fn, msg.GetPayload())
}
}
case "heka.sandbox-terminated":
fn := path.Join(self.workingDirectory, self.terminationFile)
if file, err := os.OpenFile(fn, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644); err == nil {
line := fmt.Sprintf("%d\t%s\t%v\n", msg.GetTimestamp()/1e9, msg.GetLogger(), msg.GetPayload())
file.WriteString(line)
file.Close()
}
}
plc.Pack.Recycle()
case <-ticker:
go h.PipelineConfig().allReportsMsg()
}
}
return
}
// Parses a Heka message and extracts the information necessary to start a new
// SandboxFilter
func (this *SandboxManagerFilter) loadSandbox(fr pipeline.FilterRunner,
h pipeline.PluginHelper, dir string, msg *message.Message) (err error) {
fv, _ := msg.GetFieldValue("config")
if config, ok := fv.(string); ok {
var configFile pipeline.ConfigFile
if _, err = toml.Decode(config, &configFile); err != nil {
return fmt.Errorf("loadSandbox failed: %s\n", err)
}
for name, conf := range configFile {
name = getSandboxName(fr.Name(), name)
if _, ok := h.Filter(name); ok {
// todo support reload
return fmt.Errorf("loadSandbox failed: %s is already running", name)
}
fr.LogMessage(fmt.Sprintf("Loading: %s", name))
confFile := filepath.Join(dir, fmt.Sprintf("%s.toml", name))
err = ioutil.WriteFile(confFile, []byte(config), 0600)
if err != nil {
return
}
var sbc SandboxConfig
// Default, will get overwritten if necessary
sbc.ScriptType = "lua"
if err = toml.PrimitiveDecode(conf, &sbc); err != nil {
return fmt.Errorf("loadSandbox failed: %s\n", err)
}
scriptFile := filepath.Join(dir, fmt.Sprintf("%s.%s", name, sbc.ScriptType))
err = ioutil.WriteFile(scriptFile, []byte(msg.GetPayload()), 0600)
if err != nil {
removeAll(dir, fmt.Sprintf("%s.*", name))
return
}
var runner pipeline.FilterRunner
runner, err = this.createRunner(dir, name, conf)
if err != nil {
removeAll(dir, fmt.Sprintf("%s.*", name))
return
}
err = this.pConfig.AddFilterRunner(runner)
if err == nil {
atomic.AddInt32(&this.currentFilters, 1)
}
break // only interested in the first item
}
}
return
}
// Replaces a date pattern (ex: %{2012.09.19} in the index name
func interpolateFlag(e *ElasticSearchCoordinates, m *message.Message, name string) (
interpolatedValue string, err error) {
iSlice := strings.Split(name, "%{")
for i, element := range iSlice {
elEnd := strings.Index(element, "}")
if elEnd > -1 {
elVal := element[:elEnd]
switch elVal {
case "Type":
iSlice[i] = strings.Replace(iSlice[i], element[:elEnd+1], m.GetType(), -1)
case "Hostname":
iSlice[i] = strings.Replace(iSlice[i], element[:elEnd+1], m.GetHostname(), -1)
case "Pid":
iSlice[i] = strings.Replace(iSlice[i], element[:elEnd+1],
strconv.Itoa(int(m.GetPid())), -1)
case "UUID":
iSlice[i] = strings.Replace(iSlice[i], element[:elEnd+1], m.GetUuidString(), -1)
case "Logger":
iSlice[i] = strings.Replace(iSlice[i], element[:elEnd+1], m.GetLogger(), -1)
case "EnvVersion":
iSlice[i] = strings.Replace(iSlice[i], element[:elEnd+1], m.GetEnvVersion(), -1)
case "Severity":
iSlice[i] = strings.Replace(iSlice[i], element[:elEnd+1],
strconv.Itoa(int(m.GetSeverity())), -1)
default:
if fname, ok := m.GetFieldValue(elVal); ok {
iSlice[i] = strings.Replace(iSlice[i], element[:elEnd+1], fname.(string), -1)
} else {
var t time.Time
if e.ESIndexFromTimestamp && m.Timestamp != nil {
t = time.Unix(0, *m.Timestamp).UTC()
} else {
t = time.Now().UTC()
}
iSlice[i] = strings.Replace(iSlice[i], element[:elEnd+1], t.Format(elVal), -1)
}
}
if iSlice[i] == elVal {
err = fmt.Errorf("Could not interpolate field from config: %s", name)
}
}
}
interpolatedValue = strings.Join(iSlice, "")
return
}
func (this *SandboxManagerFilter) loadSandbox(fr FilterRunner,
h PluginHelper, dir string, msg *message.Message) (err error) {
fv, _ := msg.GetFieldValue("config")
if config, ok := fv.(string); ok {
var configFile ConfigFile
if _, err = toml.Decode(config, &configFile); err != nil {
return fmt.Errorf("loadSandbox failed: %s\n", err)
} else {
for name, conf := range configFile {
name = getSandboxName(fr.Name(), name)
if _, ok := h.Filter(name); ok {
// todo support reload
return fmt.Errorf("loadSandbox failed: %s is already running", name)
}
fr.LogMessage(fmt.Sprintf("Loading: %s", name))
confFile := path.Join(dir, fmt.Sprintf("%s.toml", name))
err = ioutil.WriteFile(confFile, []byte(config), 0600)
if err != nil {
return
}
var sbfc SandboxFilterConfig
if err = toml.PrimitiveDecode(conf, &sbfc); err != nil {
return fmt.Errorf("loadSandbox failed: %s\n", err)
}
scriptFile := path.Join(dir, fmt.Sprintf("%s.%s", name, sbfc.Sbc.ScriptType))
err = ioutil.WriteFile(scriptFile, []byte(msg.GetPayload()), 0600)
if err != nil {
removeAll(dir, fmt.Sprintf("%s.*", name))
return
}
var runner FilterRunner
runner, err = createRunner(dir, name, conf)
if err != nil {
removeAll(dir, fmt.Sprintf("%s.*", name))
return
}
err = h.PipelineConfig().AddFilterRunner(runner)
if err == nil {
this.currentFilters++
}
break // only interested in the first item
}
}
}
return
}
func (self *DashboardOutput) Run(or OutputRunner, h PluginHelper) (err error) {
inChan := or.InChan()
ticker := or.Ticker()
go self.starterFunc(self)
var (
ok = true
pack *PipelinePack
msg *message.Message
)
// Maps sandbox names to plugin list items used to generate the
// sandboxes.json file.
sandboxes := make(map[string]*DashPluginListItem)
sbxsLock := new(sync.Mutex)
reNotWord, _ := regexp.Compile("\\W")
for ok {
select {
case pack, ok = <-inChan:
if !ok {
break
}
msg = pack.Message
switch msg.GetType() {
case "heka.all-report":
fn := filepath.Join(self.dataDirectory, "heka_report.json")
overwriteFile(fn, msg.GetPayload())
sbxsLock.Lock()
if err := overwritePluginListFile(self.dataDirectory, sandboxes); err != nil {
or.LogError(fmt.Errorf("Can't write plugin list file to '%s': %s",
self.dataDirectory, err))
}
sbxsLock.Unlock()
case "heka.sandbox-output":
tmp, _ := msg.GetFieldValue("payload_type")
if payloadType, ok := tmp.(string); ok {
var payloadName, nameExt string
tmp, _ := msg.GetFieldValue("payload_name")
if payloadName, ok = tmp.(string); ok {
nameExt = reNotWord.ReplaceAllString(payloadName, "")
}
if len(nameExt) > 64 {
nameExt = nameExt[:64]
}
nameExt = "." + nameExt
payloadType = reNotWord.ReplaceAllString(payloadType, "")
filterName := msg.GetLogger()
fn := filterName + nameExt + "." + payloadType
ofn := filepath.Join(self.dataDirectory, fn)
relPath := path.Join(self.relDataPath, fn) // Used for generating HTTP URLs.
overwriteFile(ofn, msg.GetPayload())
sbxsLock.Lock()
if listItem, ok := sandboxes[filterName]; !ok {
// First time we've seen this sandbox, add it to the set.
output := &DashPluginOutput{
Name: payloadName,
Filename: relPath,
}
sandboxes[filterName] = &DashPluginListItem{
Name: filterName,
Outputs: []*DashPluginOutput{output},
}
} else {
// We've seen the sandbox, see if we already have this output.
found := false
for _, output := range listItem.Outputs {
if output.Name == payloadName {
found = true
break
}
}
if !found {
output := &DashPluginOutput{
Name: payloadName,
Filename: relPath,
}
listItem.Outputs = append(listItem.Outputs, output)
}
}
sbxsLock.Unlock()
}
case "heka.sandbox-terminated":
fn := filepath.Join(self.dataDirectory, "heka_sandbox_termination.tsv")
filterName := msg.GetLogger()
if file, err := os.OpenFile(fn, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644); err == nil {
var line string
if _, ok := msg.GetFieldValue("ProcessMessageCount"); !ok {
line = fmt.Sprintf("%d\t%s\t%v\n", msg.GetTimestamp()/1e9,
msg.GetLogger(), msg.GetPayload())
} else {
pmc, _ := msg.GetFieldValue("ProcessMessageCount")
pms, _ := msg.GetFieldValue("ProcessMessageSamples")
pmd, _ := msg.GetFieldValue("ProcessMessageAvgDuration")
mad, _ := msg.GetFieldValue("MatchAvgDuration")
fcl, _ := msg.GetFieldValue("FilterChanLength")
mcl, _ := msg.GetFieldValue("MatchChanLength")
rcl, _ := msg.GetFieldValue("RouterChanLength")
line = fmt.Sprintf("%d\t%s\t%v"+
" ProcessMessageCount:%v"+
" ProcessMessageSamples:%v"+
" ProcessMessageAvgDuration:%v"+
" MatchAvgDuration:%v"+
" FilterChanLength:%v"+
" MatchChanLength:%v"+
" RouterChanLength:%v\n",
msg.GetTimestamp()/1e9,
filterName, msg.GetPayload(), pmc, pms, pmd,
mad, fcl, mcl, rcl)
}
file.WriteString(line)
file.Close()
}
sbxsLock.Lock()
delete(sandboxes, filterName)
sbxsLock.Unlock()
}
pack.Recycle()
case <-ticker:
go h.PipelineConfig().AllReportsMsg()
}
}
return
}
func (self *DashboardOutput) Run(or OutputRunner, h PluginHelper) (err error) {
inChan := or.InChan()
ticker := or.Ticker()
var (
ok = true
pack *PipelinePack
msg *message.Message
)
reNotWord, _ := regexp.Compile("\\W")
for ok {
select {
case pack, ok = <-inChan:
if !ok {
break
}
msg = pack.Message
switch msg.GetType() {
case "heka.all-report":
fn := path.Join(self.workingDirectory, "heka_report.json")
createPluginPages(self.workingDirectory, msg.GetPayload())
overwriteFile(fn, msg.GetPayload())
case "heka.sandbox-output":
tmp, _ := msg.GetFieldValue("payload_type")
if payloadType, ok := tmp.(string); ok {
var payloadName, nameExt string
tmp, _ := msg.GetFieldValue("payload_name")
if payloadName, ok = tmp.(string); ok {
nameExt = reNotWord.ReplaceAllString(payloadName, "")
}
if len(nameExt) > 64 {
nameExt = nameExt[:64]
}
nameExt = "." + nameExt
payloadType = reNotWord.ReplaceAllString(payloadType, "")
fn := msg.GetLogger() + nameExt + "." + payloadType
ofn := path.Join(self.workingDirectory, fn)
if payloadType == "cbuf" {
html := msg.GetLogger() + nameExt + ".html"
ohtml := path.Join(self.workingDirectory, html)
_, err := os.Stat(ohtml)
if err != nil {
overwriteFile(ohtml, fmt.Sprintf(getCbufTemplate(),
msg.GetLogger(),
payloadName,
fn))
}
overwriteFile(ofn, msg.GetPayload())
updatePluginMetadata(self.workingDirectory, msg.GetLogger(), html, payloadName)
} else {
overwriteFile(ofn, msg.GetPayload())
updatePluginMetadata(self.workingDirectory, msg.GetLogger(), fn, payloadName)
}
}
case "heka.sandbox-terminated":
fn := path.Join(self.workingDirectory, "heka_sandbox_termination.tsv")
if file, err := os.OpenFile(fn, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644); err == nil {
var line string
if _, ok := msg.GetFieldValue("ProcessMessageCount"); !ok {
line = fmt.Sprintf("%d\t%s\t%v\n", msg.GetTimestamp()/1e9, msg.GetLogger(), msg.GetPayload())
} else {
pmc, _ := msg.GetFieldValue("ProcessMessageCount")
pms, _ := msg.GetFieldValue("ProcessMessageSamples")
pmd, _ := msg.GetFieldValue("ProcessMessageAvgDuration")
ms, _ := msg.GetFieldValue("MatchSamples")
mad, _ := msg.GetFieldValue("MatchAvgDuration")
fcl, _ := msg.GetFieldValue("FilterChanLength")
mcl, _ := msg.GetFieldValue("MatchChanLength")
rcl, _ := msg.GetFieldValue("RouterChanLength")
line = fmt.Sprintf("%d\t%s\t%v"+
" ProcessMessageCount:%v"+
" ProcessMessageSamples:%v"+
" ProcessMessageAvgDuration:%v"+
" MatchSamples:%v"+
" MatchAvgDuration:%v"+
" FilterChanLength:%v"+
" MatchChanLength:%v"+
" RouterChanLength:%v\n",
msg.GetTimestamp()/1e9,
msg.GetLogger(), msg.GetPayload(), pmc, pms, pmd,
ms, mad, fcl, mcl, rcl)
}
file.WriteString(line)
file.Close()
}
}
pack.Recycle()
case <-ticker:
go h.PipelineConfig().allReportsMsg()
}
}
return
}
// Save matching client records locally to the given output file in the given
// format.
func save(recordChannel <-chan s3splitfile.S3Record, match *message.MatcherSpecification, format string, out *os.File, done chan<- int) {
processed := 0
matched := 0
bytes := 0
msg := new(message.Message)
ok := true
for ok {
r, ok := <-recordChannel
if !ok {
// Channel is closed
done <- bytes
break
}
bytes += len(r.Record)
processed += 1
headerLen := int(r.Record[1]) + message.HEADER_FRAMING_SIZE
messageBytes := r.Record[headerLen:]
unsnappy, decodeErr := snappy.Decode(nil, messageBytes)
if decodeErr == nil {
messageBytes = unsnappy
}
if err := proto.Unmarshal(messageBytes, msg); err != nil {
fmt.Fprintf(os.Stderr, "Error unmarshalling message %d in %s, error: %s\n", processed, r.Key, err)
continue
}
if !match.Match(msg) {
continue
}
matched += 1
switch format {
case "count":
// no op
case "json":
contents, _ := json.Marshal(msg)
fmt.Fprintf(out, "%s\n", contents)
case "heka":
fmt.Fprintf(out, "%s", r.Record)
case "offsets":
// Use offsets mode for indexing the S3 files by clientId
clientId, ok := msg.GetFieldValue("clientId")
recordLength := len(r.Record) - headerLen
if ok {
fmt.Fprintf(out, "%s\t%s\t%d\t%d\n", r.Key, clientId, (r.Offset + uint64(headerLen)), recordLength)
} else {
fmt.Fprintf(os.Stderr, "Missing client id in %s @ %d+%d\n", r.Key, r.Offset, recordLength)
}
default:
fmt.Fprintf(out, "Timestamp: %s\n"+
"Type: %s\n"+
"Hostname: %s\n"+
"Pid: %d\n"+
"UUID: %s\n"+
"Logger: %s\n"+
"Payload: %s\n"+
"EnvVersion: %s\n"+
"Severity: %d\n"+
"Fields: %+v\n\n",
time.Unix(0, msg.GetTimestamp()), msg.GetType(),
msg.GetHostname(), msg.GetPid(), msg.GetUuidString(),
msg.GetLogger(), msg.GetPayload(), msg.GetEnvVersion(),
msg.GetSeverity(), msg.Fields)
}
}
fmt.Fprintf(os.Stderr, "Processed: %d, matched: %d messages (%.2f MB)\n", processed, matched, (float64(bytes) / 1024.0 / 1024.0))
}
