// drainChannel reads non-empty log lines from a channel (until it is closed),
// extracts logging metadata from them (severity, timestamp, etc) via given
// parser and pushes log entries to the push buffer. Doesn't return errors: use
// buf.Stop() to check for any errors during sending.
func drainChannel(src chan string, parser LogParser, buf PushBuffer, logger logging.Logger) {
for line := range src {
line = strings.TrimSpace(line)
if line == "" {
continue
}
entry := parser.ParseLogLine(line)
if entry == nil {
if logger != nil {
logger.Errorf("skipping line, unrecognized format: %s", line)
}
continue
}
if entry.InsertID == "" {
insertID, err := computeInsertID(entry)
if err != nil {
if logger != nil {
logger.Errorf("skipping line, can't compute insertId: %s", err)
}
continue
}
entry.InsertID = insertID
}
buf.Add(*entry)
}
}
// zipInputFiles deterministically builds a zip archive out of input files and
// writes it to the writer. Files are written in the order given.
func zipInputFiles(files []File, w io.Writer, log logging.Logger) error {
writer := zip.NewWriter(w)
defer writer.Close()
// Reports zipping progress to the log each second.
lastReport := time.Time{}
progress := func(count int) {
if time.Since(lastReport) > time.Second {
lastReport = time.Now()
log.Infof("Zipping files: %d files left", len(files)-count)
}
}
for i, in := range files {
progress(i)
// Intentionally do not add timestamp or file mode to make zip archive
// deterministic. See also zip.FileInfoHeader() implementation.
fh := zip.FileHeader{
Name: in.Name(),
Method: zip.Deflate,
}
mode := os.FileMode(0600)
if in.Executable() {
mode |= 0100
}
if in.Symlink() {
mode |= os.ModeSymlink
}
fh.SetMode(mode)
dst, err := writer.CreateHeader(&fh)
if err != nil {
return err
}
if in.Symlink() {
err = zipSymlinkFile(dst, in)
} else {
err = zipRegularFile(dst, in)
}
if err != nil {
return err
}
}
return nil
}
// scanPackageDir finds a set of regular files (and symlinks) in a package
// instance directory and returns them as FileInfo structs (with slash-separated
// paths relative to dir directory). Skips package service directories (.cipdpkg
// and .cipd) since they contain package deployer gut files, not something that
// needs to be deployed.
func scanPackageDir(dir string, l logging.Logger) ([]FileInfo, error) {
out := []FileInfo{}
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
rel, err := filepath.Rel(dir, path)
if err != nil {
return err
}
if rel == packageServiceDir || rel == siteServiceDir {
return filepath.SkipDir
}
if info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
symlink := ""
ok := true
if info.Mode()&os.ModeSymlink != 0 {
symlink, err = os.Readlink(path)
if err != nil {
l.Warningf("Can't readlink %q, skipping: %s", path, err)
ok = false
}
}
if ok {
out = append(out, FileInfo{
Name: filepath.ToSlash(rel),
Size: uint64(info.Size()),
Executable: (info.Mode().Perm() & 0111) != 0,
Symlink: symlink,
})
}
}
return nil
})
return out, err
}