Пример #1
0
func TestSearchRequestQueryString(t *testing.T) {
	out, err := core.SearchUri("github", "", "actor:a*", "", 0)
	//log.Println(out)
	assert.T(t, &out != nil && err == nil, "Should get docs")
	assert.T(t, out.Hits.Total == 616, fmt.Sprintf("Should have 616 hits but was %v", out.Hits.Total))
}
Пример #2
0
// ExportLogs exports logs from ElasticSearch.
// serviceIds: list of services to select (includes their children). Empty slice means no filter
// from: yyyy.mm.dd (inclusive), "" means unbounded
// to: yyyy.mm.dd (inclusive), "" means unbounded
// outfile: the exported logs will tgz'd and written here. "" means "./serviced-log-export.tgz".
//
// TODO: This code is racy - creating then erasing the output file does not
// guarantee that it will be safe to write to at the end of the function
func (a *api) ExportLogs(config ExportLogsConfig) (err error) {
	var e error
	files := []*os.File{}
	fileIndex := make(map[string]map[string]int) // host => filename => index

	// make sure we can write to outfile
	if config.Outfile == "" {
		pwd, e := os.Getwd()
		if e != nil {
			return fmt.Errorf("could not determine current directory: %s", e)
		}
		now := time.Now().UTC()
		// time.RFC3339 = "2006-01-02T15:04:05Z07:00"
		nowString := strings.Replace(now.Format(time.RFC3339), ":", "", -1)
		config.Outfile = filepath.Join(pwd, fmt.Sprintf("serviced-log-export-%s.tgz", nowString))
	}
	fp, e := filepath.Abs(config.Outfile)
	if e != nil {
		return fmt.Errorf("could not convert '%s' to an absolute path: %v", config.Outfile, e)
	}
	config.Outfile = filepath.Clean(fp)
	tgzfile, e := os.Create(config.Outfile)
	if e != nil {
		return fmt.Errorf("could not create %s: %s", config.Outfile, e)
	}
	tgzfile.Close()
	if e = os.Remove(config.Outfile); e != nil {
		return fmt.Errorf("could not remove %s: %s", config.Outfile, e)
	}

	// Validate and normalize the date range filter attributes "from" and "to"
	if config.FromDate == "" && config.ToDate == "" {
		config.ToDate = time.Now().UTC().Format("2006.01.02")
		config.FromDate = time.Now().UTC().AddDate(0, 0, -1).Format("2006.01.02")
	}
	if config.FromDate != "" {
		if config.FromDate, e = NormalizeYYYYMMDD(config.FromDate); e != nil {
			return e
		}
	}
	if config.ToDate != "" {
		if config.ToDate, e = NormalizeYYYYMMDD(config.ToDate); e != nil {
			return e
		}
	}

	parts := strings.Split(options.LogstashES, ":")
	if len(parts) != 2 {
		return fmt.Errorf("invalid logstash-es host:port %s", options.LogstashES)
	}
	elastigo.Domain = parts[0]
	elastigo.Port = parts[1]

	query := "*"
	if len(config.ServiceIDs) > 0 {
		services, e := a.GetServices()
		if e != nil {
			return e
		}
		serviceMap := make(map[string]service.Service)
		for _, service := range services {
			serviceMap[service.ID] = service
		}
		serviceIDMap := make(map[string]bool) //includes serviceIds, and their children as well
		for _, serviceID := range config.ServiceIDs {
			serviceIDMap[serviceID] = true
		}
		for _, service := range services {
			srvc := service
			for {
				found := false
				for _, serviceID := range config.ServiceIDs {
					if srvc.ID == serviceID {
						serviceIDMap[service.ID] = true
						found = true
						break
					}
				}
				if found || srvc.ParentServiceID == "" {
					break
				}
				srvc = serviceMap[srvc.ParentServiceID]
			}
		}
		re := regexp.MustCompile("\\A[\\w\\-]+\\z") //only letters, numbers, underscores, and dashes
		queryParts := []string{}
		for serviceID := range serviceIDMap {
			if re.FindStringIndex(serviceID) == nil {
				return fmt.Errorf("invalid service ID format: %s", serviceID)
			}
			queryParts = append(queryParts, fmt.Sprintf("\"%s\"", strings.Replace(serviceID, "-", "\\-", -1)))
		}
		query = fmt.Sprintf("service:(%s)", strings.Join(queryParts, " OR "))
	}

	// Get a temporary directory
	tempdir, e := ioutil.TempDir("", "serviced-log-export-")
	if e != nil {
		return fmt.Errorf("could not create temp directory: %s", e)
	}
	defer os.RemoveAll(tempdir)

	days, e := LogstashDays()
	if e != nil {
		return e
	}

	// create a file to hold parse warnings
	parseWarningsFilename := filepath.Join(tempdir, "warnings.log")
	parseWarningsFile, e := os.Create(parseWarningsFilename)
	if e != nil {
		return fmt.Errorf("failed to create file %s: %s", parseWarningsFilename, e)
	}
	defer func() {
		if e := parseWarningsFile.Close(); e != nil && err == nil {
			err = fmt.Errorf("failed to close file '%s' cleanly: %s", parseWarningsFilename, e)
		}
	}()

	glog.Infof("Starting part 1 of 3: process logstash elasticsearch results using temporary dir: %s", tempdir)
	numWarnings := 0
	foundIndexedDay := false
	for _, yyyymmdd := range days {
		// Skip the indexes that are filtered out by the date range
		if (config.FromDate != "" && yyyymmdd < config.FromDate) || (config.ToDate != "" && yyyymmdd > config.ToDate) {
			continue
		} else {
			foundIndexedDay = true
		}

		logstashIndex := fmt.Sprintf("logstash-%s", yyyymmdd)
		result, e := core.SearchUri(logstashIndex, "", query, "1m", 1000)
		if e != nil {
			return fmt.Errorf("failed to search elasticsearch: %s", e)
		}
		//TODO: Submit a patch to elastigo to support the "clear scroll" api. Add a "defer" here.
		remaining := result.Hits.Total > 0
		for remaining {
			result, e = core.Scroll(false, result.ScrollId, "1m")
			hits := result.Hits.Hits
			total := len(hits)
			for i := 0; i < total; i++ {
				host, logfile, compactLines, warningMessage, e := parseLogSource(hits[i].Source)
				if e != nil {
					return e
				}
				if _, found := fileIndex[host]; !found {
					fileIndex[host] = make(map[string]int)
				}
				if _, found := fileIndex[host][logfile]; !found {
					index := len(files)
					filename := filepath.Join(tempdir, fmt.Sprintf("%03d.log", index))
					file, e := os.Create(filename)
					if e != nil {
						return fmt.Errorf("failed to create file %s: %s", filename, e)
					}
					defer func() {
						if e := file.Close(); e != nil && err == nil {
							err = fmt.Errorf("failed to close file '%s' cleanly: %s", filename, e)
						}
					}()
					fileIndex[host][logfile] = index
					files = append(files, file)
				}
				index := fileIndex[host][logfile]
				file := files[index]
				filename := filepath.Join(tempdir, fmt.Sprintf("%03d.log", index))
				for _, line := range compactLines {
					formatted := fmt.Sprintf("%016x\t%016x\t%s\n", line.Timestamp, line.Offset, line.Message)
					if _, e := file.WriteString(formatted); e != nil {
						return fmt.Errorf("failed writing to file %s: %s", filename, e)
					}
				}
				if len(warningMessage) > 0 {
					if _, e := parseWarningsFile.WriteString(warningMessage); e != nil {
						return fmt.Errorf("failed writing to file %s: %s", parseWarningsFilename, e)
					}
					numWarnings++
				}
			}
			remaining = len(hits) > 0
		}
	}
	if !foundIndexedDay {
		return fmt.Errorf("no logstash indexes exist for the given date range %s - %s", config.FromDate, config.ToDate)
	}

	glog.Infof("Starting part 2 of 3: sort output files")

	indexData := []string{}
	for host, logfileIndex := range fileIndex {
		for logfile, i := range logfileIndex {
			filename := filepath.Join(tempdir, fmt.Sprintf("%03d.log", i))
			tmpfilename := filepath.Join(tempdir, fmt.Sprintf("%03d.log.tmp", i))
			cmd := exec.Command("sort", filename, "-uo", tmpfilename)
			if output, e := cmd.CombinedOutput(); e != nil {
				return fmt.Errorf("failed sorting %s, error: %v, output: %s", filename, e, output)
			}
			if numWarnings == 0 {
				cmd = exec.Command("mv", tmpfilename, filename)
				if output, e := cmd.CombinedOutput(); e != nil {
					return fmt.Errorf("failed moving %s %s, error: %v, output: %s", tmpfilename, filename, e, output)
				}
			} else {
				cmd = exec.Command("cp", tmpfilename, filename)
				if output, e := cmd.CombinedOutput(); e != nil {
					return fmt.Errorf("failed moving %s %s, error: %v, output: %s", tmpfilename, filename, e, output)
				}
			}
			cmd = exec.Command("sed", "s/^[0-9a-f]*\\t[0-9a-f]*\\t//", "-i", filename)
			if output, e := cmd.CombinedOutput(); e != nil {
				return fmt.Errorf("failed stripping sort prefixes config.FromDate %s, error: %v, output: %s", filename, e, output)
			}
			indexData = append(indexData, fmt.Sprintf("%03d.log\t%s\t%s", i, strconv.Quote(host), strconv.Quote(logfile)))
		}
	}
	sort.Strings(indexData)
	indexData = append([]string{"INDEX OF LOG FILES", "File\tHost\tOriginal Filename"}, indexData...)
	indexData = append(indexData, "")
	indexFile := filepath.Join(tempdir, "index.txt")
	e = ioutil.WriteFile(indexFile, []byte(strings.Join(indexData, "\n")), 0644)
	if e != nil {
		return fmt.Errorf("failed writing to %s: %s", indexFile, e)
	}

	glog.Infof("Starting part 3 of 3: generate tar file: %s", config.Outfile)

	cmd := exec.Command("tar", "-czf", config.Outfile, "-C", filepath.Dir(tempdir), filepath.Base(tempdir))
	if output, e := cmd.CombinedOutput(); e != nil {
		return fmt.Errorf("failed to write tgz cmd:%+v, error:%v, output:%s", cmd, e, string(output))
	}

	if numWarnings != 0 {
		glog.Warningf("warnings for log parse are included in the tar file as: %s", filepath.Join(filepath.Base(tempdir), filepath.Base(parseWarningsFilename)))
	}

	return nil
}