Beispiel #1
0
func (b *diskBackend) Restore() {
	if b.tail {
		log.Info("restoring with tail ", b.directory+"/unc/"+b.filename)
		b.RestoreTail(b.directory + "/unc/" + b.filename)
		return
	}
	log.Info("restoring gzipped file ", b.directory+"/"+b.filename)
	f, err := os.Open(b.directory + "/" + b.filename)
	if err != nil {
		return
	}
	gzrd, _ := gzip.NewReader(f)
	rd := bufio.NewReader(gzrd)
	defer gzrd.Close()
	defer f.Close()
	t0 := time.Now()
	i := 1
	for {
		input, err := rd.ReadBytes('\n')
		if err != nil {
			f.Close()
			log.Info("disk: restored ", i, " removing ", b.filename)
			os.Remove(b.directory + "/" + b.filename)
			return
		}
		input2 := input[:len(input)-1]
		b.ctx.parsedLines <- &input2
		if time.Since(t0).Seconds() > 5 {
			log.Info("disk: restoring ...")
			t0 = time.Now()
		}
		i++
	}
}
func OpenDecompress(fn string) (r io.ReadCloser, err error) {
	f, err := os.Open(fn)
	if err != nil {
		return nil, err
	}

	ext := filepath.Ext(fn)

	switch ext {
	case ".log", ".txt":
		return f, err
	case ".gz":
		gzr, err := gzip.NewReader(f)
		return &WrappedDecompressor{
			ReadCloser: gzr,
			wrapped:    f,
		}, err
	case ".bz2":
		bzr := bzip2.NewReader(f)
		return &WrappedDecompressor{
			ReadCloser: ioutil.NopCloser(bzr),
			wrapped:    f,
		}, nil
	default:
		return f, err
	}
}
Beispiel #3
0
func NewDecoder(r io.Reader) (*MetadataDecoder, error) {
	m := MetadataDecoder{}

	// read header
	var h Header
	d := xdr.NewDecoder(r)
	_, err := d.Decode(&h)
	if err != nil {
		return nil, err
	}

	if h.Version != Version {
		return nil, ErrVersion
	}

	switch {
	case bytes.Compare(h.Compression[:], CompNone[:]) == 0:
		m.d = d
	case bytes.Compare(h.Compression[:], CompGZIP[:]) == 0:
		br, err := pgzip.NewReader(r)
		if err != nil {
			return nil, err
		}
		m.d = xdr.NewDecoder(br)
	default:
		return nil, ErrCompression
	}

	return &m, nil
}
Beispiel #4
0
func (c *Compressor) BenchmarkPGZIPReader(b *testing.B) {
	cr, _ := pgzip.NewReader(c.w)
	b.ResetTimer()

	_, err := io.Copy(ioutil.Discard, cr)
	if err != nil {
		b.Fatal(err)
	}
}
Beispiel #5
0
// NewGzLine reads one password per line until
// \0xa (newline) is encountered.
// The input is assumed to be gzip compressed.
// Input is streamed.
func NewGzLine(r io.Reader) (*LineReader, error) {
	l := &LineReader{in: r}
	var err error
	l.gr, err = gzip.NewReader(r)
	if err != nil {
		return nil, err
	}
	l.br = bufio.NewReader(l.gr)
	return l, nil
}
Beispiel #6
0
func NaClDecrypt(body []byte, key *[KeySize]byte) (*Header, []byte, error) {

	// obtain nonce
	var nonce [NonceSize]byte
	copy(nonce[:], body[:NonceSize])

	// decrypt payload
	payload, ok := secretbox.Open(nil, body[NonceSize:], &nonce, key)
	if !ok {
		return nil, nil, fmt.Errorf("could not decrypt body")
	}

	// deal with actual payload
	r := bytes.NewReader(payload)

	// decode header
	d := xdr.NewDecoder(r)
	var mh Header
	_, err := d.Decode(&mh)
	if err != nil {
		return nil, nil, err
	}

	// deal with compression
	var rd io.Reader
	switch mh.Compression {
	case CompNone:
		// reuse reader
		rd = r
	case CompGZIP:
		// always use parallel decompression
		rd, err = pgzip.NewReader(r)
		if err != nil {
			return nil, nil, err
		}
	default:
		return nil, nil, fmt.Errorf("invalid compression: %v",
			mh.Compression)
	}

	var cleartext bytes.Buffer
	f := bufio.NewWriter(&cleartext)

	// read left over from the xdr reader
	_, err = io.Copy(f, rd)
	if err != nil {
		return nil, nil, err
	}

	f.Flush()

	return &mh, cleartext.Bytes(), nil
}
Beispiel #7
0
// Write gunzipped data to a Writer. Returns bytes written and an error.
func gunzipWrite(w io.Writer, data []byte) (int, error) {
	// Write gzipped data to the client
	gr, err := pgzip.NewReader(bytes.NewBuffer(data))
	defer gr.Close()
	data, err = ioutil.ReadAll(gr)
	if err != nil {
		return 0, err
	}
	bytesWritten, err := w.Write(data)
	if err != nil {
		return 0, err
	}
	return bytesWritten, nil
}
Beispiel #8
0
// recurseDirectory recurses into a directory and sends all files
// fulfilling the selected options into global.filesChan
func recurseDirectory(dirname string) {
	dir, err := os.Open(dirname)
	if err != nil {
		errorLogger.Printf("cannot open directory '%s': %s\n", dirname, err)
		return
	}
	defer dir.Close()
	for {
		entries, err := dir.Readdir(256)
		if err == io.EOF {
			return
		}
		if err != nil {
			errorLogger.Printf("cannot read directory '%s': %s\n", dirname, err)
			return
		}

	nextEntry:
		for _, fi := range entries {
			fullpath := filepath.Join(dirname, fi.Name())

			// check directory include/exclude options
			if fi.IsDir() {
				if !options.Recursive {
					continue nextEntry
				}
				for _, dirPattern := range options.ExcludeDirs {
					matched, err := filepath.Match(dirPattern, fi.Name())
					if err != nil {
						errorLogger.Fatalf("cannot match malformed pattern '%s' against directory name: %s\n", dirPattern, err)
					}
					if matched {
						continue nextEntry
					}
				}
				if len(options.IncludeDirs) > 0 {
					for _, dirPattern := range options.IncludeDirs {
						matched, err := filepath.Match(dirPattern, fi.Name())
						if err != nil {
							errorLogger.Fatalf("cannot match malformed pattern '%s' against directory name: %s\n", dirPattern, err)
						}
						if matched {
							goto includeDirMatchFound
						}
					}
					continue nextEntry
				includeDirMatchFound:
				}
				recurseDirectory(fullpath)
				continue nextEntry
			}

			// check whether this is a regular file
			if fi.Mode()&os.ModeType != 0 {
				continue nextEntry
			}

			// check file path options
			if global.excludeFilepathRegex != nil {
				if global.excludeFilepathRegex.MatchString(fullpath) {
					continue nextEntry
				}
			}
			if global.includeFilepathRegex != nil {
				if !global.includeFilepathRegex.MatchString(fullpath) {
					continue nextEntry
				}
			}

			// check file extension options
			if len(options.ExcludeExtensions) > 0 {
				for _, e := range strings.Split(options.ExcludeExtensions, ",") {
					if filepath.Ext(fi.Name()) == "."+e {
						continue nextEntry
					}
				}
			}
			if len(options.IncludeExtensions) > 0 {
				for _, e := range strings.Split(options.IncludeExtensions, ",") {
					if filepath.Ext(fi.Name()) == "."+e {
						goto includeExtensionFound
					}
				}
				continue nextEntry
			includeExtensionFound:
			}

			// check file include/exclude options
			for _, filePattern := range options.ExcludeFiles {
				matched, err := filepath.Match(filePattern, fi.Name())
				if err != nil {
					errorLogger.Fatalf("cannot match malformed pattern '%s' against file name: %s\n", filePattern, err)
				}
				if matched {
					continue nextEntry
				}
			}
			if len(options.IncludeFiles) > 0 {
				for _, filePattern := range options.IncludeFiles {
					matched, err := filepath.Match(filePattern, fi.Name())
					if err != nil {
						errorLogger.Fatalf("cannot match malformed pattern '%s' against file name: %s\n", filePattern, err)
					}
					if matched {
						goto includeFileMatchFound
					}
				}
				continue nextEntry
			includeFileMatchFound:
			}

			// check file type options
			if len(options.ExcludeTypes) > 0 {
				for _, t := range strings.Split(options.ExcludeTypes, ",") {
					for _, filePattern := range global.fileTypesMap[t].Patterns {
						if matched, _ := filepath.Match(filePattern, fi.Name()); matched {
							continue nextEntry
						}
					}
					sr := global.fileTypesMap[t].ShebangRegex
					if sr != nil {
						if m, err := checkShebang(global.fileTypesMap[t].ShebangRegex, fullpath); m && err == nil {
							continue nextEntry
						}
					}
				}
			}
			if len(options.IncludeTypes) > 0 {
				for _, t := range strings.Split(options.IncludeTypes, ",") {
					for _, filePattern := range global.fileTypesMap[t].Patterns {
						if matched, _ := filepath.Match(filePattern, fi.Name()); matched {
							goto includeTypeFound
						}
					}
					sr := global.fileTypesMap[t].ShebangRegex
					if sr != nil {
						if m, err := checkShebang(global.fileTypesMap[t].ShebangRegex, fullpath); err != nil || m {
							goto includeTypeFound
						}
					}
				}
				continue nextEntry
			includeTypeFound:
			}

			global.filesChan <- fullpath
		}
	}
}

// checkShebang checks whether the first line of file matches the given regex
func checkShebang(regex *regexp.Regexp, filepath string) (bool, error) {
	f, err := os.Open(filepath)
	defer f.Close()
	if err != nil {
		return false, err
	}
	b, err := bufio.NewReader(f).ReadBytes('\n')
	return regex.Match(b), nil
}

// processFileTargets reads filesChan, builds an io.Reader for the target and calls processReader
func processFileTargets() {
	defer global.targetsWaitGroup.Done()
	dataBuffer := make([]byte, InputBlockSize)
	testBuffer := make([]byte, InputBlockSize)
	matchRegexes := make([]*regexp.Regexp, len(global.matchPatterns))
	for i := range global.matchPatterns {
		matchRegexes[i] = regexp.MustCompile(global.matchPatterns[i])
	}

	for filepath := range global.filesChan {
		var err error
		var infile *os.File
		var reader io.Reader

		if filepath == "-" {
			infile = os.Stdin
		} else {
			infile, err = os.Open(filepath)
			if err != nil {
				errorLogger.Printf("cannot open file '%s': %s\n", filepath, err)
				continue
			}
		}

		if options.Zip && strings.HasSuffix(filepath, ".gz") {
			rawReader := bufio.NewReader(infile)
			reader, err = gzip.NewReader(rawReader)
			if err != nil {
				errorLogger.Printf("error decompressing file '%s', opening as normal file\n", infile.Name())
				infile.Seek(0, 0)
				reader = bufio.NewReader(infile)
			}
		} else if infile == os.Stdin && options.Multiline {
			reader = nbreader.NewNBReader(bufio.NewReader(infile), InputBlockSize,
				nbreader.ChunkTimeout(MultilinePipeChunkTimeout), nbreader.Timeout(MultilinePipeTimeout))
		} else {
			reader = bufio.NewReader(infile)
		}

		if options.InvertMatch {
			err = processReaderInvertMatch(reader, matchRegexes, filepath)
		} else {
			err = processReader(reader, matchRegexes, dataBuffer, testBuffer, filepath)
		}
		if err != nil {
			if err == errLineTooLong {
				if !options.ErrSkipLineLength {
					errmsg := fmt.Sprintf("file contains very long lines (no newline in %d bytes)", InputBlockSize)
					errorLogger.Printf("cannot process data from file '%s': %s\n", filepath, errmsg)
				}
			} else {
				errorLogger.Printf("cannot process data from file '%s': %s\n", filepath, err)
			}
		}
		infile.Close()
	}
}

// processNetworkTarget starts a listening TCP socket and calls processReader
func processNetworkTarget(target string) {
	matchRegexes := make([]*regexp.Regexp, len(global.matchPatterns))
	for i := range global.matchPatterns {
		matchRegexes[i] = regexp.MustCompile(global.matchPatterns[i])
	}
	defer global.targetsWaitGroup.Done()

	var reader io.Reader
	netParams := global.netTcpRegex.FindStringSubmatch(target)
	proto := netParams[1]
	addr := netParams[2]

	listener, err := net.Listen(proto, addr)
	if err != nil {
		errorLogger.Fatalf("could not listen on '%s'\n", target)
	}

	conn, err := listener.Accept()
	if err != nil {
		errorLogger.Fatalf("could not accept connections on '%s'\n", target)
	}

	if options.Multiline {
		reader = nbreader.NewNBReader(conn, InputBlockSize, nbreader.ChunkTimeout(MultilinePipeChunkTimeout),
			nbreader.Timeout(MultilinePipeTimeout))
	} else {
		reader = conn
	}

	dataBuffer := make([]byte, InputBlockSize)
	testBuffer := make([]byte, InputBlockSize)
	err = processReader(reader, matchRegexes, dataBuffer, testBuffer, target)
	if err != nil {
		errorLogger.Printf("error processing data from '%s'\n", target)
		return
	}
}

func executeSearch(targets []string) (ret int, err error) {
	defer func() {
		if r := recover(); r != nil {
			ret = 2
			err = errors.New(r.(string))
		}
	}()
	tstart := time.Now()
	global.filesChan = make(chan string, 256)
	global.resultsChan = make(chan *Result, 128)
	global.resultsDoneChan = make(chan struct{})
	global.totalTargetCount = 0
	global.totalMatchCount = 0
	global.totalResultCount = 0
	go resultHandler()

	for i := 0; i < options.Cores; i++ {
		global.targetsWaitGroup.Add(1)
		go processFileTargets()
	}

	for _, target := range targets {
		switch {
		case target == "-":
			global.filesChan <- "-"
		case global.netTcpRegex.MatchString(target):
			global.targetsWaitGroup.Add(1)
			go processNetworkTarget(target)
		default:
			fileinfo, err := os.Stat(target)
			if err != nil {
				if os.IsNotExist(err) {
					errorLogger.Fatalf("no such file or directory: %s\n", target)
				} else {
					errorLogger.Fatalf("cannot open file or directory: %s\n", target)
				}
			}
			if fileinfo.IsDir() {
				recurseDirectory(target)
			} else {
				global.filesChan <- target
			}
		}
	}

	close(global.filesChan)
	global.targetsWaitGroup.Wait()

	close(global.resultsChan)
	<-global.resultsDoneChan

	var retVal int
	if global.totalResultCount > 0 {
		retVal = 0
	} else {
		retVal = 1
	}

	if options.Stats {
		tend := time.Now()
		fmt.Fprintln(os.Stderr, global.totalTargetCount, "files processed")
		fmt.Fprintln(os.Stderr, global.totalResultCount, "files match")
		fmt.Fprintln(os.Stderr, global.totalMatchCount, "matches found")
		fmt.Fprintf(os.Stderr, "in %v\n", tend.Sub(tstart))
	}

	return retVal, nil
}

func main() {
	var targets []string

	options.LoadDefaults()

	parser := flags.NewNamedParser("sift", flags.HelpFlag|flags.PassDoubleDash)
	parser.AddGroup("Options", "Options", &options)
	parser.Name = "sift"
	parser.Usage = "[OPTIONS] PATTERN [FILE|PATH|tcp://HOST:PORT...]\n" +
		"  sift [OPTIONS] [-e PATTERN | -f FILE] [FILE|PATH|tcp://HOST:PORT...]"
	args, err := parser.Parse()
	if err != nil {
		if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp {
			fmt.Println(e.Error())
			os.Exit(0)
		} else {
			errorLogger.Println(err)
			os.Exit(2)
		}
	}

	for _, pattern := range options.Patterns {
		global.matchPatterns = append(global.matchPatterns, pattern)
	}

	if options.PatternFile != "" {
		f, err := os.Open(options.PatternFile)
		if err != nil {
			errorLogger.Fatalln("Cannot open pattern file:\n", err)
		}
		scanner := bufio.NewScanner(f)
		for scanner.Scan() {
			pattern := scanner.Text()
			global.matchPatterns = append(global.matchPatterns, pattern)

		}
	}
	if len(global.matchPatterns) == 0 {
		if len(args) == 0 && !(options.PrintConfig || options.WriteConfig) {
			errorLogger.Fatalln("No pattern given. Try 'sift --help' for more information.")
		}
		if len(args) > 0 {
			global.matchPatterns = append(global.matchPatterns, args[0])
			args = args[1:len(args)]
		}
	}

	if len(args) == 0 {
		// check whether there is input on STDIN
		if !terminal.IsTerminal(int(os.Stdin.Fd())) {
			targets = []string{"-"}
		} else {
			targets = []string{"."}
		}
	} else {
		targets = args[0:len(args)]
	}

	// expand arguments containing patterns on Windows
	if runtime.GOOS == "windows" {
		targetsExpanded := []string{}
		for _, t := range targets {
			if t == "-" {
				targetsExpanded = append(targetsExpanded, t)
				continue
			}
			expanded, err := filepath.Glob(t)
			if err == filepath.ErrBadPattern {
				errorLogger.Fatalf("cannot parse argument '%s': %s\n", t, err)
			}
			if expanded != nil {
				for _, e := range expanded {
					targetsExpanded = append(targetsExpanded, e)
				}
			}
		}
		targets = targetsExpanded
	}

	if err := options.Apply(global.matchPatterns, targets); err != nil {
		errorLogger.Fatalf("cannot process options: %s\n", err)
	}

	global.matchRegexes = make([]*regexp.Regexp, len(global.matchPatterns))
	for i := range global.matchPatterns {
		global.matchRegexes[i], err = regexp.Compile(global.matchPatterns[i])
		if err != nil {
			errorLogger.Fatalf("cannot parse pattern: %s\n", err)
		}
	}

	retVal, err := executeSearch(targets)
	if err != nil {
		errorLogger.Println(err)
	}
	os.Exit(retVal)
}
Beispiel #9
0
func downloadForgeModule(name string, version string) {
	//url := "https://forgeapi.puppetlabs.com/v3/files/puppetlabs-apt-2.1.1.tar.gz"
	fileName := name + "-" + version + ".tar.gz"
	if _, err := os.Stat(config.ForgeCacheDir + name + "-" + version); os.IsNotExist(err) {
		url := "https://forgeapi.puppetlabs.com/v3/files/" + fileName
		req, err := http.NewRequest("GET", url, nil)
		req.Header.Set("User-Agent", "https://github.com/xorpaul/g10k/")
		req.Header.Set("Connection", "close")
		proxyUrl, err := http.ProxyFromEnvironment(req)
		if err != nil {
			log.Fatal("downloadForgeModule(): Error while getting http proxy with golang http.ProxyFromEnvironment()", err)
			os.Exit(1)
		}
		client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}}
		before := time.Now()
		resp, err := client.Do(req)
		duration := time.Since(before).Seconds()
		Verbosef("GETing " + url + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s")
		mutex.Lock()
		syncForgeTime += duration
		mutex.Unlock()
		if err != nil {
			log.Print("downloadForgeModule(): Error while GETing Forge module ", name, " from ", url, ": ", err)
			os.Exit(1)
		}
		defer resp.Body.Close()

		if resp.Status == "200 OK" {
			Debugf("downloadForgeModule(): Trying to create " + config.ForgeCacheDir + fileName)
			out, err := os.Create(config.ForgeCacheDir + fileName)
			if err != nil {
				log.Print("downloadForgeModule(): Error while creating file for Forge module "+config.ForgeCacheDir+fileName, err)
				os.Exit(1)
			}
			defer out.Close()
			io.Copy(out, resp.Body)
			file, err := os.Open(config.ForgeCacheDir + fileName)

			if err != nil {
				fmt.Println("downloadForgeModule(): Error while opening file", file, err)
				os.Exit(1)
			}

			defer file.Close()

			var fileReader io.ReadCloser = resp.Body
			if strings.HasSuffix(fileName, ".gz") {
				if fileReader, err = pgzip.NewReader(file); err != nil {

					fmt.Println("downloadForgeModule(): pgzip reader error for module ", fileName, " error:", err)
					os.Exit(1)
				}
				defer fileReader.Close()
			}

			tarBallReader := tar.NewReader(fileReader)
			if err = os.Chdir(config.ForgeCacheDir); err != nil {

				fmt.Println("downloadForgeModule(): error while chdir to", config.ForgeCacheDir, err)
				os.Exit(1)
			}
			for {
				header, err := tarBallReader.Next()
				if err != nil {
					if err == io.EOF {
						break
					}
					fmt.Println("downloadForgeModule(): error while tar reader.Next() for ", fileName, err)
					os.Exit(1)
				}

				// get the individual filename and extract to the current directory
				filename := header.Name
				//Debugf("downloadForgeModule(): Trying to extract file" + filename)

				switch header.Typeflag {
				case tar.TypeDir:
					// handle directory
					//fmt.Println("Creating directory :", filename)
					//err = os.MkdirAll(filename, os.FileMode(header.Mode)) // or use 0755 if you prefer
					err = os.MkdirAll(filename, os.FileMode(0755)) // or use 0755 if you prefer

					if err != nil {
						fmt.Println("downloadForgeModule(): error while MkdirAll()", filename, err)
						os.Exit(1)
					}

				case tar.TypeReg:
					// handle normal file
					//fmt.Println("Untarring :", filename)
					writer, err := os.Create(filename)

					if err != nil {
						fmt.Println("downloadForgeModule(): error while Create()", filename, err)
						os.Exit(1)
					}

					io.Copy(writer, tarBallReader)

					err = os.Chmod(filename, os.FileMode(0644))

					if err != nil {
						fmt.Println("downloadForgeModule(): error while Chmod()", filename, err)
						os.Exit(1)
					}

					writer.Close()
				default:
					fmt.Printf("Unable to untar type : %c in file %s", header.Typeflag, filename)
				}
			}

		} else {
			log.Print("downloadForgeModule(): Unexpected response code while GETing " + url + resp.Status)
			os.Exit(1)
		}
	} else {
		Debugf("downloadForgeModule(): Using cache for Forge module " + name + " version: " + version)
	}
}
Beispiel #10
0
// downloadTables retrieves files from the HTTP server. Files to download is MySQL engine specific.
func downloadTable(clientConfig clientConfigStruct, downloadInfo downloadInfoStruct) {
	downloadInfo.displayInfo.w = os.Stdout
	downloadInfo.displayInfo.fqTable = downloadInfo.schema + "." + downloadInfo.table
	downloadInfo.displayInfo.status = "Downloading"
	downloadInfo.displayChan <- downloadInfo.displayInfo

	// Use encoded schema and table if present
	var schemaFilename string
	var tableFilename string
	if downloadInfo.encodedSchema != "" {
		schemaFilename = downloadInfo.encodedSchema
	} else {
		schemaFilename = downloadInfo.schema
	}

	if downloadInfo.encodedTable != "" {
		tableFilename = downloadInfo.encodedTable
	} else {
		tableFilename = downloadInfo.table
	}

	// Ensure backup exists and check the engine type
	// Assume InnoDB first
	resp, err := http.Head(downloadInfo.backurl + path.Join(schemaFilename, tableFilename+".ibd"))
	checkErr(err)

	var engine string
	var extensions []string
	if resp.StatusCode == 200 {
		engine = "InnoDB"

		// 5.1 & 5.5 use .exp - 5.6 uses .cfg but it is ignored. Metadata checks appeared too brittle in testing.
		if strings.HasPrefix(downloadInfo.version, "5.1") || strings.HasPrefix(downloadInfo.version, "5.5") {
			extensions = append(extensions, ".exp")
		}

		extensions = append(extensions, ".ibd")
	} else {
		// Check for MyISAM
		resp, err := http.Head(downloadInfo.backurl + path.Join(schemaFilename, tableFilename+".MYD"))
		checkErr(err)

		if resp.StatusCode == 200 {
			engine = "MyISAM"
			extensions = append(extensions, ".MYI")
			extensions = append(extensions, ".MYD")
			extensions = append(extensions, ".frm")
		} else {
			errDownloadUnsupported = fmt.Errorf("Table %s.%s is using an unsupported engine", downloadInfo.schema, downloadInfo.table)
			handleDownloadError(clientConfig, &downloadInfo, errDownloadUnsupported)

			return
		}
	}

	// Update downloadInfo struct with engine type and extensions array
	downloadInfo.engine = engine
	downloadInfo.extensions = extensions

	// Loop through and download all files from extensions array
	var triteFiles []string
	for _, extension := range extensions {
		triteFile := filepath.Join(downloadInfo.mysqldir, schemaFilename, tableFilename+extension+".trite")

		// Ensure the .exp exists if we expect it
		// Checking this due to a bug encountered where XtraBackup did not create a tables .exp file
		if extension == ".exp" {
			resp, err := http.Head(downloadInfo.backurl + path.Join(schemaFilename, tableFilename+".exp"))
			checkHTTP(resp, downloadInfo.backurl+path.Join(schemaFilename, tableFilename+".exp"))
			checkErr(err)

			if resp.StatusCode != 200 {
				errDownloadExp = fmt.Errorf("The .exp file is missing for table %s.%s", downloadInfo.schema, downloadInfo.table)
				handleDownloadError(clientConfig, &downloadInfo, errDownloadExp)

				return
			}
		}

		// Request and write file
		fo, err := os.Create(triteFile)
		checkErr(err)
		defer fo.Close()

		if runtime.GOOS != "windows" {
			// Chown to mysql user
			os.Chown(triteFile, downloadInfo.uid, downloadInfo.gid)
			os.Chmod(triteFile, mysqlPerms)
		}

		// Get the size of the file from the trite server here because the file may be compressed during download in which case the content length is -1
		headfile := downloadInfo.backurl + path.Join(schemaFilename, tableFilename+extension)
		head, err := http.Head(headfile)
		checkHTTP(head, headfile)
		checkErr(err)
		sizeServer := head.ContentLength

		var urlfile string
		if clientConfig.gz == true {
			urlfile = downloadInfo.gzurl + path.Join(schemaFilename, tableFilename+extension)
		} else {
			urlfile = downloadInfo.backurl + path.Join(schemaFilename, tableFilename+extension)
		}

		// Download files from trite server
		w := bufio.NewWriter(fo)
		resp, err := http.Get(urlfile)
		checkHTTP(resp, urlfile)
		defer resp.Body.Close()
		checkErr(err)

		var r io.Reader
		if clientConfig.gz == true {
			r, _ = pgzip.NewReader(resp.Body)
		} else {
			r = bufio.NewReader(resp.Body)
		}

		var sizeDown int64
		if extension != ".exp" && sizeServer > clientConfig.minDownloadProgressSize*1073741824 {
			progressReader := &reader{
				reader:     r,
				size:       sizeServer,
				drawFunc:   drawTerminalf(downloadInfo.displayInfo.w, drawTextFormatPercent),
				drawPrefix: "Downloading: " + downloadInfo.schema + "." + downloadInfo.table,
			}
			sizeDown, err = w.ReadFrom(progressReader)

		} else {
			sizeDown, err = w.ReadFrom(r)

		}

		checkErr(err)
		w.Flush()

		// Check if size of file downloaded matches size on server -- Add retry ability
		if sizeDown != sizeServer {
			// Remove partial file download
			os.Remove(triteFile)

			errDownloadSize = fmt.Errorf("The %s file did not download properly for %s.%s", extension, downloadInfo.schema, downloadInfo.table)
			handleDownloadError(clientConfig, &downloadInfo, errDownloadSize)
		}

		triteFiles = append(triteFiles, triteFile)
	}

	downloadInfo.triteFiles = triteFiles

	// Call applyTables
	go applyTables(clientConfig, &downloadInfo)
}
Beispiel #11
0
func extractForgeModule(wgForgeModule *sync.WaitGroup, file *io.PipeReader, fileName string) {
	defer wgForgeModule.Done()
	funcName := funcName()

	before := time.Now()
	fileReader, err := pgzip.NewReader(file)
	if err != nil {
		Fatalf(funcName + "(): pgzip reader error for module " + fileName + " error:" + err.Error())
	}
	defer fileReader.Close()

	tarBallReader := tar.NewReader(fileReader)
	for {
		header, err := tarBallReader.Next()
		if err != nil {
			if err == io.EOF {
				break
			}
			Fatalf(funcName + "(): error while tar reader.Next() for " + fileName + err.Error())
		}

		// get the individual filename and extract to the current directory
		filename := header.Name
		targetFilename := config.ForgeCacheDir + "/" + filename
		//Debugf("Trying to extract file" + filename)

		switch header.Typeflag {
		case tar.TypeDir:
			// handle directory
			//fmt.Println("Creating directory :", filename)
			//err = os.MkdirAll(targetFilename, os.FileMode(header.Mode)) // or use 0755 if you prefer
			err = os.MkdirAll(targetFilename, os.FileMode(0755)) // or use 0755 if you prefer

			if err != nil {
				Fatalf(funcName + "(): error while MkdirAll() " + filename + err.Error())
			}

		case tar.TypeReg:
			// handle normal file
			//fmt.Println("Untarring :", filename)
			writer, err := os.Create(targetFilename)

			if err != nil {
				Fatalf(funcName + "(): error while Create() " + filename + err.Error())
			}

			io.Copy(writer, tarBallReader)

			err = os.Chmod(targetFilename, os.FileMode(0644))

			if err != nil {
				Fatalf(funcName + "(): error while Chmod() " + filename + err.Error())
			}

			writer.Close()
		default:
			Fatalf(funcName + "(): Unable to untar type: " + string(header.Typeflag) + " in file " + filename)
		}
	}

	duration := time.Since(before).Seconds()
	Verbosef("Extracting " + config.ForgeCacheDir + fileName + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s")
	mutex.Lock()
	ioForgeTime += duration
	mutex.Unlock()
}
Beispiel #12
0
func ReadSynthesisFile(repo Repository, file string, out chan<- Package) error {
	f, err := os.Open(file)
	if err != nil {
		return err
	}
	defer f.Close()

	gz, err := gzip.NewReader(f)
	if err != nil {
		return err
	}
	defer gz.Close()

	r := bufio.NewReader(gz)

	cur := Package{}
	for err == nil {
		line, err := r.ReadString('\n')
		if err == io.EOF {
			break
		}

		if err != nil {
			return fmt.Errorf("can't read synthesis file: %v", err)
		}

		line = strings.Trim(line, "\n\r")

		// @[email protected]@libvo-amrwbenc0[== 0.1.2-1:2014.1]
		// @[email protected]@libc.so.6(GLIBC_2.0)@libc.so.6(GLIBC_2.1.3)
		// @summary@VisualOn AMR-WB encoder library
		// @filesize@68793
		// @[email protected]@0@143156@System/Libraries@[email protected]

		if strings.HasPrefix(line, "@info") {
			var size string

			err = splitLine(line,
				&cur.FileName,
				nil,
				&size,
				&cur.Group,
				&cur.Disttag,
				&cur.Distepoch)
			if err != nil {
				return err
			}

			cur.Repository = repo.Name

			cur.Size, err = strconv.Atoi(size)
			if err != nil {
				return fmt.Errorf("Can't read synthesis file: incorrect filesize '%v': %v", size, err)
			}

			items := strings.Split(cur.FileName, "-")
			if len(items) < 4 {
				return fmt.Errorf("Can't parse package filename %s", cur.FileName)
			}

			s := items[len(items)-1]
			n := strings.LastIndex(s, ".")
			cur.Arch = s[n+1:]

			cur.Version = items[len(items)-3] + "-" + items[len(items)-2]
			cur.Name = strings.Join(items[:len(items)-3], "-")

			out <- cur
			cur = Package{}
			continue
		}

		if strings.HasPrefix(line, "@summary@") {
			cur.Summary = line[9:]
			continue
		}

		if strings.HasPrefix(line, "@filesize@") {
			items := strings.Split(line, "@")
			if len(items) < 3 {
				return fmt.Errorf("can't read synthesis file: package filesize not found %v", line)
			}

			cur.RPMSize, err = strconv.Atoi(items[2])
			if err != nil {
				return fmt.Errorf("Can't read synthesis file: incorrect RPM size %v: %v", items[2], err)
			}
			continue
		}
	}

	return nil
}