func logFilenameToTime(filename string, filenameToTimeRegex *regexp.Regexp) (time.Time, error) {
if filenameToTimeRegex == nil {
return time.Now(), fmt.Errorf("filename_to_time_regex is not set in indexer configuration")
}
n1 := filenameToTimeRegex.SubexpNames()
r2s := filenameToTimeRegex.FindAllStringSubmatch(filename, -1)
if len(r2s) == 0 {
return time.Now(), fmt.Errorf("filename_to_time_regex did not match %q", filename)
}
r2 := r2s[0]
md := map[string]string{}
for i, n := range r2 {
md[n1[i]] = n
}
getOrZero := func(key string) int {
val, exists := md[key]
if !exists {
return 0
}
num, err := strconv.Atoi(val)
if err != nil {
return 0
}
return num
}
year := getOrZero("year")
month := time.Month(getOrZero("month"))
day := getOrZero("day")
hour := getOrZero("hour")
minute := getOrZero("minute")
return time.Date(year, month, day, hour, minute, 0, 0, time.UTC), nil
}
func compilePattern(pattern string) (*regexp.Regexp, error) {
var (
re *regexp.Regexp
err error
)
// Compile regexp pattern
if re, err = regexp.Compile(pattern); err != nil {
return nil, err
}
// Validate pattern keywords
groups := make(map[string]bool)
for _, key := range re.SubexpNames() {
if key == "" {
continue
} else if key == "source" || key == "metric" {
groups[key] = true
} else {
return nil, fmt.Errorf("invalid pattern keyword `%s'", key)
}
}
if !groups["source"] {
return nil, fmt.Errorf("missing pattern keyword `source'")
} else if !groups["metric"] {
return nil, fmt.Errorf("missing pattern keyword `metric'")
}
return re, nil
}
// add route
func (mvc *MvcFilter) Route(name string, pattern string, method string, controller interface{}) error {
if pattern == "" {
return NewHttpError("gomvc invalid pattern " + pattern)
}
if controller == nil {
return NewHttpError("gomvc invalid controller ")
}
var r *regexp.Regexp
var err error
if r, err = regexp.Compile(pattern); err != nil {
return err
}
route := Route{Name: name, Pattern: pattern, Method: method}
route.reg = r
route.names = r.SubexpNames()[1:]
if fv, ok := controller.(reflect.Value); ok {
route.Controller = fv
} else {
route.Controller = reflect.ValueOf(controller)
}
mvc.Routes = append(mvc.Routes, route)
return nil
}
func RegexpFilter(filterReg *regexp.Regexp, data []byte) (outdata []byte) {
if nil == filterReg {
return data
}
matches := filterReg.FindAllSubmatch(data, -1)
if nil == matches {
log.Printf("[ERROR] failed to match filter regex, pattern %s did not match", filterReg.String())
if *gDebug {
log.Println("======= debug: target data =======")
log.Println(string(data))
log.Println("==============")
}
return nil
}
for _, match := range matches {
for patInd, patName := range filterReg.SubexpNames() {
switch patName {
case PATTERN_FILTER:
outdata = append(outdata, match[patInd]...)
}
}
}
if *gDebug {
log.Println("======= debug: filter regex ========")
log.Println(filterReg.String())
log.Println("======= debug: filtered data =======")
log.Println(string(outdata))
log.Println("==============")
}
return
}
func ParseDatabaseDSN(urlRegexp *regexp.Regexp, url string) DatabaseDSN {
defer func() {
_ = recover()
}()
dsn := DatabaseDSN{}
match := urlRegexp.FindStringSubmatch(url)
for i, name := range urlRegexp.SubexpNames() {
if i != 0 {
if name == "host" {
dsn.Host = match[i]
}
if name == "port" {
dsn.Port = match[i]
}
if name == "user" {
dsn.User = match[i]
}
if name == "password" {
dsn.Password = match[i]
}
if name == "name" {
dsn.Name = match[i]
}
}
}
return dsn
}
// ServeHTTP parses the path parameters, calls SetPathParameters of
// the corresponding hander, then directs traffic to it.
func (r *RouteHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
glog.Infof("%s: '%s'", req.Method, req.URL.String())
pathstr := req.URL.Path[len(r.path):]
var match []string
var pattern *regexp.Regexp
var handler Handler
for e := r.patternHandlers.Front(); e != nil; e = e.Next() {
h := e.Value.(*patternHandler)
pattern = h.Regexp
handler = h.Handler
match = pattern.FindStringSubmatch(pathstr)
if match != nil {
break
}
}
if match == nil {
glog.Warningf("Cannot find a matching pattern for Path `%s'",
pathstr)
http.NotFound(w, req)
return
}
kvpairs := make(map[string]string)
for i, name := range pattern.SubexpNames() {
// ignore full match and unnamed submatch
if i == 0 || name == "" {
continue
}
kvpairs[name] = match[i]
}
glog.V(1).Infof("Parsed path parameters: %s", kvpairs)
handler.ServeHTTP(w, req, kvpairs)
}
// match regexp with string, and return a named group map
// Example:
// regexp: "(?P<name>[A-Za-z]+)-(?P<age>\\d+)"
// string: "CGC-30"
// return: map[string][]string{ "name":["CGC"], "age":["30"] }
func NamedUrlValuesRegexpGroup(str string, reg *regexp.Regexp) (ng url.Values, matched bool) {
rst := reg.FindStringSubmatch(str)
if len(rst) < 1 {
return
}
//for i,s :=range rst{
// fmt.Printf("%d => %s\n",i,s)
//}
ng = url.Values{}
lenRst := len(rst)
sn := reg.SubexpNames()
for k, v := range sn {
// SubexpNames contain the none named group,
// so must filter v == ""
//fmt.Printf("%s => %s\n",k,v)
if k == 0 || v == "" {
continue
}
if k+1 > lenRst {
break
}
ng.Add(v, rst[k])
}
matched = true
return
}
func FromString(dur string) (*Duration, error) {
var (
match []string
re *regexp.Regexp
)
if week.MatchString(dur) {
match = week.FindStringSubmatch(dur)
re = week
} else if full.MatchString(dur) {
match = full.FindStringSubmatch(dur)
re = full
} else {
return nil, ErrBadFormat
}
d := &Duration{}
for i, name := range re.SubexpNames() {
part := match[i]
if i == 0 || name == "" || part == "" {
continue
}
val, err := strconv.Atoi(part)
if err != nil {
return nil, err
}
// Check that no component is explicitly 0,
// e.g. PT60M0S
if val == 0 {
return nil, fmt.Errorf("%s cannot be 0", name)
}
switch name {
case "year":
d.Years = val
case "month":
return nil, ErrNoMonth
case "week":
d.Weeks = val
case "day":
d.Days = val
case "hour":
d.Hours = val
case "minute":
d.Minutes = val
case "second":
d.Seconds = val
default:
return nil, errors.New(fmt.Sprintf("unknown field %s", name))
}
}
return d, nil
}
func URESearch(pattern *regexp.Regexp, content string) map[string]string {
matches := pattern.FindStringSubmatch(content)
result := make(map[string]string)
for i, name := range pattern.SubexpNames() {
result[name] = matches[i]
}
return result
}
func ParsePubDate(formatReg *regexp.Regexp, dateStr string) (time.Time, error) {
if nil == formatReg {
log.Printf("[ERROR] error parsing pubdate, date format regexp is nil")
return time.Time{}, errors.New("date format regexp is nil")
}
pubdateStr := strings.TrimSpace(dateStr)
if "" == pubdateStr {
log.Printf("[ERROR] error parsing pubdate, pubdate string is empty")
return time.Time{}, errors.New("pubdate string is empty")
}
now := time.Now()
year := now.Year()
month := int(now.Month())
day := now.Day()
hour := now.Hour()
minute := now.Minute()
second := now.Second()
match := formatReg.FindSubmatch([]byte(pubdateStr))
if nil == match {
log.Printf("[ERROR] error parsing pubdate %s, pattern %s match failed", pubdateStr, formatReg.String())
return time.Time{}, errors.New("failed to match pubdate pattern")
}
for patInd, patName := range formatReg.SubexpNames() {
var err error
patVal := string(match[patInd])
switch patName {
case PATTERN_YEAR:
year, err = strconv.Atoi(patVal)
case PATTERN_MONTH:
month, err = ParseDateMonth(patVal)
case PATTERN_DAY:
day, err = strconv.Atoi(patVal)
case PATTERN_HOUR:
hour, err = strconv.Atoi(patVal)
case PATTERN_MINUTE:
minute, err = strconv.Atoi(patVal)
case PATTERN_SECOND:
second, err = strconv.Atoi(patVal)
}
if nil != err {
log.Printf("[ERROR] error parsing pubdate: %s, time value %s cannot be parsed: %s",
pubdateStr,
match[patInd],
err,
)
return time.Time{}, err
}
}
return time.Date(year, time.Month(month), day, hour, minute, second, 0, GOFEED_DEFAULT_TIMEZONE), nil
}
// Use the given regex to decompose a line of the thread dump into the matched fields
//regex groups start OMIT
func decomposeTreadDumpLineRe(threadDumpLine string, r *regexp.Regexp) (groups map[string]string, err error) {
matches := r.FindStringSubmatch(threadDumpLine)
names := r.SubexpNames()
groups = make(map[string]string)
for i, name := range names {
groups[name] = matches[i]
}
return
}
func mapParams(regex *regexp.Regexp, matches []string) (p Params) {
for i, n := range regex.SubexpNames() {
if len(n) > 0 {
v := Param{}
v.Key = n
v.Value = matches[i]
p = append(p, v)
}
}
return
}
// Transform our matches in a readable hash map.
//
// The resulting hash map will be something like { "Heap1": 123 }
func getMatchMap(re *regexp.Regexp, matches []string) map[string]string {
matchingNames := re.SubexpNames()[1:]
matchMap := map[string]string{}
for i, value := range matches[1:] {
if matchingNames[i] == "" {
continue
}
matchMap[matchingNames[i]] = value
}
return matchMap
}
// getKeywordArguments is called by the base handler to extract keyword arguments from a URL pattern.
func getKeywordArguments(path string, pattern *regexp.Regexp) (args URLArgs) {
args = make(URLArgs)
matches := pattern.FindAllStringSubmatch(path, -1)
for i, key := range pattern.SubexpNames() {
if i == 0 || key == "" {
continue
}
args[key] = matches[0][i]
}
return args
}
// SubexpNames returns a mapping of the sub-expression names to values if the Regexp
// successfully matches the string, otherwise, it returns false.
func SubexpNames(r *regexp.Regexp, s string) (bool, map[string]string) {
if matches := r.FindStringSubmatch(strings.TrimSpace(s)); matches != nil {
names := r.SubexpNames()
result := make(map[string]string)
for i, match := range matches {
result[names[i]] = strings.TrimSpace(match)
}
return true, result
}
return false, nil
}
// Checks if a given path matches a regex route
func MatchesRoutePath(path string, re *regexp.Regexp) (bool, map[string]string) {
matches := re.FindAllStringSubmatch(path, -1)
attrs := make(map[string]string)
if len(matches) > 0 {
subExp := re.SubexpNames()
for idx, exp := range subExp {
attrs[exp] = matches[0][idx]
}
return true, attrs
}
return false, attrs
}
func namedMatch(r *regexp.Regexp, s string) (bool, map[string]string) {
match := r.FindStringSubmatch(s)
if len(match) == 0 {
return false, nil
}
result := make(map[string]string)
for i, name := range r.SubexpNames() {
result[name] = match[i]
}
return true, result
}
func fileReadlog(file *os.File) {
rd := bufio.NewReader(file)
formatAdbStd, err := regexp.Compile(REGEXP_ADB_STD)
if err != nil {
log.Fatal("Regexp compile error", err)
}
formatAdbThreadtime, err := regexp.Compile(REGEXP_ADB_THREADTIME)
if err != nil {
log.Fatal("Regexp compile error", err)
}
formatPusslogStd, err := regexp.Compile(REGEXP_PUSSLOG_STD)
if err != nil {
log.Fatal("Regexp compile error", err)
}
for {
str, err := rd.ReadString('\n')
if err != nil {
if err != io.EOF {
log.Fatal("Read Error: ", err)
}
return
}
var format *regexp.Regexp
if formatAdbStd.MatchString(str) {
format = formatAdbStd
} else if formatAdbThreadtime.MatchString(str) {
format = formatAdbThreadtime
} else if formatPusslogStd.MatchString(str) {
format = formatPusslogStd
} else {
log.Print("Does not match any known logformat: " + str)
continue
}
var tag, prio, msg string
matches := format.FindStringSubmatch(str)
names := format.SubexpNames()
for i, n := range names {
if n == "tag" {
tag = matches[i]
} else if n == "prio" {
prio = matches[i]
} else if n == "msg" {
msg = matches[i]
}
}
logmessage("", "", 0, 0, prio, tag, msg)
}
}
// Break down incoming protocol messages.
func (c *Client) parseMessage(msg string, rx *regexp.Regexp) (map[string]string, error) {
parts := make(map[string]string)
matches := rx.FindStringSubmatch(msg)
if len(matches) == 0 {
return parts, errors.New("Could not parse protocol message: " + msg)
}
for i, name := range rx.SubexpNames() {
parts[name] = matches[i]
}
return parts, nil
}
func capture(r *regexp.Regexp, match []string) map[string]string {
captures := make(map[string]string)
for i, name := range r.SubexpNames() {
if i == 0 || name == "" || match[i] == "" {
continue
}
captures[name] = match[i]
}
return captures
}
func Components(re *regexp.Regexp, path string) map[string]string {
names := re.SubexpNames()
match := re.FindStringSubmatch(path)
if match == nil {
return nil
}
m := map[string]string{}
for i, s := range match {
m[names[i]] = s
}
return m
}
func makeHandler(fn HandlerFunc, reg *regexp.Regexp, routes *[]*route) http.HandlerFunc {
params := InitParam()
params.SetFlag("next", len(*routes)+1)
params.routes = routes
return func(w http.ResponseWriter, r *http.Request) {
m := reg.FindStringSubmatch(r.URL.Path)
n := reg.SubexpNames()
params.SetParam(m, n)
fn(w, r, params)
}
}
func (ts *ThreatSpec) matchLine(line string, re *regexp.Regexp) map[string]string {
match := re.FindStringSubmatch(line)
if match == nil {
return nil
}
result := make(map[string]string)
for i, name := range re.SubexpNames() {
result[name] = match[i]
}
return result
}
// NamedCaptureGroups does a regexp match wih the supplied pattern on the str
// parameter and extracts the namend capture groups as map and returns it
//
// when the pattern does not match (nil, false) is returned
func NamedCaptureGroups(pattern *regexp.Regexp, str string) (map[string]string, bool) {
match := pattern.FindStringSubmatch(str)
if match == nil {
return nil, false
}
result := make(map[string]string)
for i, name := range pattern.SubexpNames() {
result[name] = match[i]
}
return result, true
}
// NamedFindStringSubmatch ...
func NamedFindStringSubmatch(rexp *regexp.Regexp, text string) (map[string]string, error) {
match := rexp.FindStringSubmatch(text)
if match == nil {
return map[string]string{}, errors.New("No match found")
}
result := map[string]string{}
for i, name := range rexp.SubexpNames() {
if i != 0 {
result[name] = match[i]
}
}
return result, nil
}
// SubmatchMap returns a map of submatch names to their captures, if any.
// If no matches are found, it returns an empty dict.
// Submatch names are specified using (?P<name>[matchme])
func SubmatchMap(re *regexp.Regexp, str string) (dict map[string]string) {
dict = make(map[string]string)
names := re.SubexpNames()
matches := re.FindStringSubmatch(str)
if len(names) > 0 && len(matches) > 0 {
names = names[1:]
matches = matches[1:]
for i, name := range names {
dict[name] = matches[i] // offset from names[1:]
}
}
return dict
}
func matchLine(re *regexp.Regexp, line string) (map[string]string, bool) {
matches := re.FindStringSubmatch(line)
if len(matches) < 1 {
return map[string]string{}, false
}
kvmap := make(map[string]string)
for n, k := range re.SubexpNames() {
kvmap[k] = matches[n]
}
return kvmap, true
}
// NamedFindStringSubmatch ...
func NamedFindStringSubmatch(rexp *regexp.Regexp, text string) (map[string]string, bool) {
match := rexp.FindStringSubmatch(text)
if match == nil {
return nil, false
}
result := map[string]string{}
for i, name := range rexp.SubexpNames() {
if i != 0 {
result[name] = match[i]
}
}
return result, true
}
func ParseString(dur string) (*Duration, error) {
var (
match []string
re *regexp.Regexp
)
if week.MatchString(dur) {
match = week.FindStringSubmatch(dur)
re = week
} else if full.MatchString(dur) {
match = full.FindStringSubmatch(dur)
re = full
} else {
return nil, ErrBadFormat
}
d := time.Duration(0)
for i, name := range re.SubexpNames() {
part := match[i]
if i == 0 || name == "" || part == "" {
continue
}
val, err := strconv.Atoi(part)
if err != nil {
return nil, err
}
switch name {
case "year":
d += time.Duration(val) * Year
case "month":
d += time.Duration(val) * Month
case "week":
d += time.Duration(val) * Week
case "day":
d += time.Duration(val) * Day
case "hour":
d += time.Duration(val) * time.Hour
case "minute":
d += time.Duration(val) * time.Minute
case "second":
d += time.Duration(val) * time.Second
default:
return nil, errors.New(fmt.Sprintf("unknown field %s", name))
}
}
return &Duration{d}, nil
}
// Provided the fileMatch regexp and translation map, populate all the Logfile
// matchparts for use in sorting.
func (l Logfiles) PopulateMatchParts(fileMatch *regexp.Regexp, translation SubmatchTranslationMap) error {
errorlist := NewMultipleError()
subexpNames := fileMatch.SubexpNames()
for _, logfile := range l {
matches := fileMatch.FindStringSubmatch(logfile.FileName)
if err := logfile.PopulateMatchParts(subexpNames, matches, translation); err != nil {
errorlist.AddMessage(err.Error())
}
}
if errorlist.IsError() {
return errorlist
}
return nil
}
