func TestSearchRequestQueryString(t *testing.T) {
out, err := core.SearchUri("github", "", "actor:a*", "", 0)
//log.Println(out)
assert.T(t, &out != nil && err == nil, "Should get docs")
assert.T(t, out.Hits.Total == 616, fmt.Sprintf("Should have 616 hits but was %v", out.Hits.Total))
}
// ExportLogs exports logs from ElasticSearch.
// serviceIds: list of services to select (includes their children). Empty slice means no filter
// from: yyyy.mm.dd (inclusive), "" means unbounded
// to: yyyy.mm.dd (inclusive), "" means unbounded
// outfile: the exported logs will tgz'd and written here. "" means "./serviced-log-export.tgz".
//
// TODO: This code is racy - creating then erasing the output file does not
// guarantee that it will be safe to write to at the end of the function
func (a *api) ExportLogs(config ExportLogsConfig) (err error) {
var e error
files := []*os.File{}
fileIndex := make(map[string]map[string]int) // host => filename => index
// make sure we can write to outfile
if config.Outfile == "" {
pwd, e := os.Getwd()
if e != nil {
return fmt.Errorf("could not determine current directory: %s", e)
}
now := time.Now().UTC()
// time.RFC3339 = "2006-01-02T15:04:05Z07:00"
nowString := strings.Replace(now.Format(time.RFC3339), ":", "", -1)
config.Outfile = filepath.Join(pwd, fmt.Sprintf("serviced-log-export-%s.tgz", nowString))
}
fp, e := filepath.Abs(config.Outfile)
if e != nil {
return fmt.Errorf("could not convert '%s' to an absolute path: %v", config.Outfile, e)
}
config.Outfile = filepath.Clean(fp)
tgzfile, e := os.Create(config.Outfile)
if e != nil {
return fmt.Errorf("could not create %s: %s", config.Outfile, e)
}
tgzfile.Close()
if e = os.Remove(config.Outfile); e != nil {
return fmt.Errorf("could not remove %s: %s", config.Outfile, e)
}
// Validate and normalize the date range filter attributes "from" and "to"
if config.FromDate == "" && config.ToDate == "" {
config.ToDate = time.Now().UTC().Format("2006.01.02")
config.FromDate = time.Now().UTC().AddDate(0, 0, -1).Format("2006.01.02")
}
if config.FromDate != "" {
if config.FromDate, e = NormalizeYYYYMMDD(config.FromDate); e != nil {
return e
}
}
if config.ToDate != "" {
if config.ToDate, e = NormalizeYYYYMMDD(config.ToDate); e != nil {
return e
}
}
parts := strings.Split(options.LogstashES, ":")
if len(parts) != 2 {
return fmt.Errorf("invalid logstash-es host:port %s", options.LogstashES)
}
elastigo.Domain = parts[0]
elastigo.Port = parts[1]
query := "*"
if len(config.ServiceIDs) > 0 {
services, e := a.GetServices()
if e != nil {
return e
}
serviceMap := make(map[string]service.Service)
for _, service := range services {
serviceMap[service.ID] = service
}
serviceIDMap := make(map[string]bool) //includes serviceIds, and their children as well
for _, serviceID := range config.ServiceIDs {
serviceIDMap[serviceID] = true
}
for _, service := range services {
srvc := service
for {
found := false
for _, serviceID := range config.ServiceIDs {
if srvc.ID == serviceID {
serviceIDMap[service.ID] = true
found = true
break
}
}
if found || srvc.ParentServiceID == "" {
break
}
srvc = serviceMap[srvc.ParentServiceID]
}
}
re := regexp.MustCompile("\\A[\\w\\-]+\\z") //only letters, numbers, underscores, and dashes
queryParts := []string{}
for serviceID := range serviceIDMap {
if re.FindStringIndex(serviceID) == nil {
return fmt.Errorf("invalid service ID format: %s", serviceID)
}
queryParts = append(queryParts, fmt.Sprintf("\"%s\"", strings.Replace(serviceID, "-", "\\-", -1)))
}
query = fmt.Sprintf("service:(%s)", strings.Join(queryParts, " OR "))
}
// Get a temporary directory
tempdir, e := ioutil.TempDir("", "serviced-log-export-")
if e != nil {
return fmt.Errorf("could not create temp directory: %s", e)
}
defer os.RemoveAll(tempdir)
days, e := LogstashDays()
if e != nil {
return e
}
// create a file to hold parse warnings
parseWarningsFilename := filepath.Join(tempdir, "warnings.log")
parseWarningsFile, e := os.Create(parseWarningsFilename)
if e != nil {
return fmt.Errorf("failed to create file %s: %s", parseWarningsFilename, e)
}
defer func() {
if e := parseWarningsFile.Close(); e != nil && err == nil {
err = fmt.Errorf("failed to close file '%s' cleanly: %s", parseWarningsFilename, e)
}
}()
glog.Infof("Starting part 1 of 3: process logstash elasticsearch results using temporary dir: %s", tempdir)
numWarnings := 0
foundIndexedDay := false
for _, yyyymmdd := range days {
// Skip the indexes that are filtered out by the date range
if (config.FromDate != "" && yyyymmdd < config.FromDate) || (config.ToDate != "" && yyyymmdd > config.ToDate) {
continue
} else {
foundIndexedDay = true
}
logstashIndex := fmt.Sprintf("logstash-%s", yyyymmdd)
result, e := core.SearchUri(logstashIndex, "", query, "1m", 1000)
if e != nil {
return fmt.Errorf("failed to search elasticsearch: %s", e)
}
//TODO: Submit a patch to elastigo to support the "clear scroll" api. Add a "defer" here.
remaining := result.Hits.Total > 0
for remaining {
result, e = core.Scroll(false, result.ScrollId, "1m")
hits := result.Hits.Hits
total := len(hits)
for i := 0; i < total; i++ {
host, logfile, compactLines, warningMessage, e := parseLogSource(hits[i].Source)
if e != nil {
return e
}
if _, found := fileIndex[host]; !found {
fileIndex[host] = make(map[string]int)
}
if _, found := fileIndex[host][logfile]; !found {
index := len(files)
filename := filepath.Join(tempdir, fmt.Sprintf("%03d.log", index))
file, e := os.Create(filename)
if e != nil {
return fmt.Errorf("failed to create file %s: %s", filename, e)
}
defer func() {
if e := file.Close(); e != nil && err == nil {
err = fmt.Errorf("failed to close file '%s' cleanly: %s", filename, e)
}
}()
fileIndex[host][logfile] = index
files = append(files, file)
}
index := fileIndex[host][logfile]
file := files[index]
filename := filepath.Join(tempdir, fmt.Sprintf("%03d.log", index))
for _, line := range compactLines {
formatted := fmt.Sprintf("%016x\t%016x\t%s\n", line.Timestamp, line.Offset, line.Message)
if _, e := file.WriteString(formatted); e != nil {
return fmt.Errorf("failed writing to file %s: %s", filename, e)
}
}
if len(warningMessage) > 0 {
if _, e := parseWarningsFile.WriteString(warningMessage); e != nil {
return fmt.Errorf("failed writing to file %s: %s", parseWarningsFilename, e)
}
numWarnings++
}
}
remaining = len(hits) > 0
}
}
if !foundIndexedDay {
return fmt.Errorf("no logstash indexes exist for the given date range %s - %s", config.FromDate, config.ToDate)
}
glog.Infof("Starting part 2 of 3: sort output files")
indexData := []string{}
for host, logfileIndex := range fileIndex {
for logfile, i := range logfileIndex {
filename := filepath.Join(tempdir, fmt.Sprintf("%03d.log", i))
tmpfilename := filepath.Join(tempdir, fmt.Sprintf("%03d.log.tmp", i))
cmd := exec.Command("sort", filename, "-uo", tmpfilename)
if output, e := cmd.CombinedOutput(); e != nil {
return fmt.Errorf("failed sorting %s, error: %v, output: %s", filename, e, output)
}
if numWarnings == 0 {
cmd = exec.Command("mv", tmpfilename, filename)
if output, e := cmd.CombinedOutput(); e != nil {
return fmt.Errorf("failed moving %s %s, error: %v, output: %s", tmpfilename, filename, e, output)
}
} else {
cmd = exec.Command("cp", tmpfilename, filename)
if output, e := cmd.CombinedOutput(); e != nil {
return fmt.Errorf("failed moving %s %s, error: %v, output: %s", tmpfilename, filename, e, output)
}
}
cmd = exec.Command("sed", "s/^[0-9a-f]*\\t[0-9a-f]*\\t//", "-i", filename)
if output, e := cmd.CombinedOutput(); e != nil {
return fmt.Errorf("failed stripping sort prefixes config.FromDate %s, error: %v, output: %s", filename, e, output)
}
indexData = append(indexData, fmt.Sprintf("%03d.log\t%s\t%s", i, strconv.Quote(host), strconv.Quote(logfile)))
}
}
sort.Strings(indexData)
indexData = append([]string{"INDEX OF LOG FILES", "File\tHost\tOriginal Filename"}, indexData...)
indexData = append(indexData, "")
indexFile := filepath.Join(tempdir, "index.txt")
e = ioutil.WriteFile(indexFile, []byte(strings.Join(indexData, "\n")), 0644)
if e != nil {
return fmt.Errorf("failed writing to %s: %s", indexFile, e)
}
glog.Infof("Starting part 3 of 3: generate tar file: %s", config.Outfile)
cmd := exec.Command("tar", "-czf", config.Outfile, "-C", filepath.Dir(tempdir), filepath.Base(tempdir))
if output, e := cmd.CombinedOutput(); e != nil {
return fmt.Errorf("failed to write tgz cmd:%+v, error:%v, output:%s", cmd, e, string(output))
}
if numWarnings != 0 {
glog.Warningf("warnings for log parse are included in the tar file as: %s", filepath.Join(filepath.Base(tempdir), filepath.Base(parseWarningsFilename)))
}
return nil
}