Example #1
0
// drainChannel reads non-empty log lines from a channel (until it is closed),
// extracts logging metadata from them (severity, timestamp, etc) via given
// parser and pushes log entries to the push buffer. Doesn't return errors: use
// buf.Stop() to check for any errors during sending.
func drainChannel(src chan string, parser LogParser, buf PushBuffer, logger logging.Logger) {
	for line := range src {
		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}
		entry := parser.ParseLogLine(line)
		if entry == nil {
			if logger != nil {
				logger.Errorf("skipping line, unrecognized format: %s", line)
			}
			continue
		}
		if entry.InsertID == "" {
			insertID, err := computeInsertID(entry)
			if err != nil {
				if logger != nil {
					logger.Errorf("skipping line, can't compute insertId: %s", err)
				}
				continue
			}
			entry.InsertID = insertID
		}
		buf.Add(*entry)
	}
}
Example #2
0
// zipInputFiles deterministically builds a zip archive out of input files and
// writes it to the writer. Files are written in the order given.
func zipInputFiles(files []File, w io.Writer, log logging.Logger) error {
	writer := zip.NewWriter(w)
	defer writer.Close()

	// Reports zipping progress to the log each second.
	lastReport := time.Time{}
	progress := func(count int) {
		if time.Since(lastReport) > time.Second {
			lastReport = time.Now()
			log.Infof("Zipping files: %d files left", len(files)-count)
		}
	}

	for i, in := range files {
		progress(i)

		// Intentionally do not add timestamp or file mode to make zip archive
		// deterministic. See also zip.FileInfoHeader() implementation.
		fh := zip.FileHeader{
			Name:   in.Name(),
			Method: zip.Deflate,
		}

		mode := os.FileMode(0600)
		if in.Executable() {
			mode |= 0100
		}
		if in.Symlink() {
			mode |= os.ModeSymlink
		}
		fh.SetMode(mode)

		dst, err := writer.CreateHeader(&fh)
		if err != nil {
			return err
		}
		if in.Symlink() {
			err = zipSymlinkFile(dst, in)
		} else {
			err = zipRegularFile(dst, in)
		}
		if err != nil {
			return err
		}
	}

	return nil
}
Example #3
0
// scanPackageDir finds a set of regular files (and symlinks) in a package
// instance directory and returns them as FileInfo structs (with slash-separated
// paths relative to dir directory). Skips package service directories (.cipdpkg
// and .cipd) since they contain package deployer gut files, not something that
// needs to be deployed.
func scanPackageDir(dir string, l logging.Logger) ([]FileInfo, error) {
	out := []FileInfo{}
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		rel, err := filepath.Rel(dir, path)
		if err != nil {
			return err
		}
		if rel == packageServiceDir || rel == siteServiceDir {
			return filepath.SkipDir
		}
		if info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
			symlink := ""
			ok := true
			if info.Mode()&os.ModeSymlink != 0 {
				symlink, err = os.Readlink(path)
				if err != nil {
					l.Warningf("Can't readlink %q, skipping: %s", path, err)
					ok = false
				}
			}
			if ok {
				out = append(out, FileInfo{
					Name:       filepath.ToSlash(rel),
					Size:       uint64(info.Size()),
					Executable: (info.Mode().Perm() & 0111) != 0,
					Symlink:    symlink,
				})
			}
		}
		return nil
	})
	return out, err
}