Exemple #1
0
// Global
var (
	// ConfigFile is the config file data structure
	ConfigFile *goconfig.ConfigFile
	// HomeDir is the home directory of the user
	HomeDir = configHome()
	// ConfigPath points to the config file
	ConfigPath = path.Join(HomeDir, configFileName)
	// Config is the global config
	Config = &ConfigInfo{}
	// Flags
	verbose        = pflag.BoolP("verbose", "v", false, "Print lots more stuff")
	quiet          = pflag.BoolP("quiet", "q", false, "Print as little stuff as possible")
	modifyWindow   = pflag.DurationP("modify-window", "", time.Nanosecond, "Max time diff to be considered the same")
	checkers       = pflag.IntP("checkers", "", 8, "Number of checkers to run in parallel.")
	transfers      = pflag.IntP("transfers", "", 4, "Number of file transfers to run in parallel.")
	configFile     = pflag.StringP("config", "", ConfigPath, "Config file.")
	checkSum       = pflag.BoolP("checksum", "c", false, "Skip based on checksum & size, not mod-time & size")
	sizeOnly       = pflag.BoolP("size-only", "", false, "Skip based on size only, not mod-time or checksum")
	ignoreExisting = pflag.BoolP("ignore-existing", "", false, "Skip all files that exist on destination")
	dryRun         = pflag.BoolP("dry-run", "n", false, "Do a trial run with no permanent changes")
	connectTimeout = pflag.DurationP("contimeout", "", 60*time.Second, "Connect timeout")
	timeout        = pflag.DurationP("timeout", "", 5*60*time.Second, "IO idle timeout")
	dumpHeaders    = pflag.BoolP("dump-headers", "", false, "Dump HTTP headers - may contain sensitive info")
	dumpBodies     = pflag.BoolP("dump-bodies", "", false, "Dump HTTP headers and bodies - may contain sensitive info")
	skipVerify     = pflag.BoolP("no-check-certificate", "", false, "Do not verify the server SSL certificate. Insecure.")
	deleteBefore   = pflag.BoolP("delete-before", "", false, "When synchronizing, delete files on destination before transfering")
	deleteDuring   = pflag.BoolP("delete-during", "", false, "When synchronizing, delete files during transfer (default)")
	deleteAfter    = pflag.BoolP("delete-after", "", false, "When synchronizing, delete files on destination after transfering")
	bwLimit        SizeSuffix
Exemple #2
0
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	"github.com/ncw/rclone/fs"
)

// Globals
var (
	// Flags
	cpuProfile    = pflag.StringP("cpuprofile", "", "", "Write cpu profile to file")
	memProfile    = pflag.String("memprofile", "", "Write memory profile to file")
	statsInterval = pflag.DurationP("stats", "", time.Minute*1, "Interval to print stats (0 to disable)")
	version       bool
	logFile       = pflag.StringP("log-file", "", "", "Log everything to this file")
	retries       = pflag.IntP("retries", "", 3, "Retry operations this many times if they fail")
)

// Root is the main rclone command
var Root = &cobra.Command{
	Use:   "rclone",
	Short: "Sync files and directories to and from local and remote object stores - " + fs.Version,
	Long: `
Rclone is a command line program to sync files and directories to and
from various cloud storage systems, such as:

  * Google Drive
  * Amazon S3
  * Openstack Swift / Rackspace cloud files / Memset Memstore
  * Dropbox
  * Google Cloud Storage
Exemple #3
0
package cli

import (
	"fmt"
	"os"

	"github.com/pilebones/backdoorGolang/core/common"
	"github.com/pilebones/backdoorGolang/core/socket"
	"github.com/spf13/pflag"
)

var (
	host          = pflag.StringP("host", "h", "localhost", "Set hostname to use")
	port          = pflag.IntP("port", "p", 9876, "Set port number to use")
	isListenMode  = pflag.BoolP("listen", "l", false, "Enable listen mode (server socket mode)")
	isVerboseMode = pflag.BoolP("verbose", "v", false, "Enable mode verbose")
	isDebugMode   = pflag.BoolP("debug", "d", false, "Enable mode debug")
	isVersionMode = pflag.BoolP("version", "V", false, "Display version number")
)

/** Print data when debug mode is enabled */
func DisplayAsDebug(message string) {
	if UseDebugMode() {
		fmt.Println(message)
	}
}

/** Return true if mode use from parameter */
func UseMode(mode *bool) bool {
	if *mode {
		return true
Exemple #4
0
// Global
var (
	// ConfigFile is the config file data structure
	ConfigFile *goconfig.ConfigFile
	// HomeDir is the home directory of the user
	HomeDir = configHome()
	// ConfigPath points to the config file
	ConfigPath = path.Join(HomeDir, configFileName)
	// Config is the global config
	Config = &ConfigInfo{}
	// Flags
	verbose         = pflag.BoolP("verbose", "v", false, "Print lots more stuff")
	quiet           = pflag.BoolP("quiet", "q", false, "Print as little stuff as possible")
	modifyWindow    = pflag.DurationP("modify-window", "", time.Nanosecond, "Max time diff to be considered the same")
	checkers        = pflag.IntP("checkers", "", 8, "Number of checkers to run in parallel.")
	transfers       = pflag.IntP("transfers", "", 4, "Number of file transfers to run in parallel.")
	configFile      = pflag.StringP("config", "", ConfigPath, "Config file.")
	checkSum        = pflag.BoolP("checksum", "c", false, "Skip based on checksum & size, not mod-time & size")
	sizeOnly        = pflag.BoolP("size-only", "", false, "Skip based on size only, not mod-time or checksum")
	ignoreTimes     = pflag.BoolP("ignore-times", "I", false, "Don't skip files that match size and time - transfer all files")
	ignoreExisting  = pflag.BoolP("ignore-existing", "", false, "Skip all files that exist on destination")
	dryRun          = pflag.BoolP("dry-run", "n", false, "Do a trial run with no permanent changes")
	connectTimeout  = pflag.DurationP("contimeout", "", 60*time.Second, "Connect timeout")
	timeout         = pflag.DurationP("timeout", "", 5*60*time.Second, "IO idle timeout")
	dumpHeaders     = pflag.BoolP("dump-headers", "", false, "Dump HTTP headers - may contain sensitive info")
	dumpBodies      = pflag.BoolP("dump-bodies", "", false, "Dump HTTP headers and bodies - may contain sensitive info")
	skipVerify      = pflag.BoolP("no-check-certificate", "", false, "Do not verify the server SSL certificate. Insecure.")
	AskPassword     = pflag.BoolP("ask-password", "", true, "Allow prompt for password for encrypted configuration.")
	deleteBefore    = pflag.BoolP("delete-before", "", false, "When synchronizing, delete files on destination before transfering")
	deleteDuring    = pflag.BoolP("delete-during", "", false, "When synchronizing, delete files during transfer (default)")
Exemple #5
0
	"encoding/json"
	"fmt"
	flag "github.com/spf13/pflag"
	"io"
	"net"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	dateFormat = "0102061504"
)

var portP = flag.IntP("port", "p", 9000, "Default port to use")
var addressP = flag.StringP("address", "a", "127.0.0.1", "Default listen address to use")

type Header struct {
	Data [16]byte
}

type Record struct {
	Date        [6]byte
	_           byte
	Time        [4]byte
	_           byte
	SecDur      [5]byte
	_           byte
	CodCode     [1]byte
	_           byte
Exemple #6
0
// IntP defines a flag which can be overridden by an environment variable
//
// It is a thin wrapper around pflag.IntP
func IntP(name, shorthand string, value int, usage string) (out *int) {
	out = pflag.IntP(name, shorthand, value, usage)
	setDefaultFromEnv(name)
	return out
}
Exemple #7
0
func main() {

	conf = viper.New()

	conf.SetEnvPrefix("FIFO2KINESIS")

	conf.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
	conf.AutomaticEnv()

	viper.SetConfigName("fifo2kinesis")

	pflag.IntP("buffer-queue-limit", "l", 500, "The maximum number of items in the buffer before it is flushed")
	conf.BindPFlag("buffer-queue-limit", pflag.Lookup("buffer-queue-limit"))
	conf.SetDefault("buffer-queue-limit", 500)

	pflag.BoolP("debug", "d", false, "Show debug level log messages")
	conf.BindPFlag("debug", pflag.Lookup("debug"))
	conf.SetDefault("debug", "")

	pflag.StringP("failed-attempts-dir", "D", "", "The path to the directory containing failed attempts")
	conf.BindPFlag("failed-attempts-dir", pflag.Lookup("failed-attempts-dir"))
	conf.SetDefault("failed-attempts-dir", "")

	pflag.StringP("fifo-name", "f", "", "The absolute path of the named pipe, e.g. /var/test.pipe")
	conf.BindPFlag("fifo-name", pflag.Lookup("fifo-name"))
	conf.SetDefault("fifo-name", "")

	pflag.StringP("flush-handler", "h", "kinesis", "Defaults to \"kinesis\", use \"logger\" for debugging")
	conf.BindPFlag("flush-handler", pflag.Lookup("flush-handler"))
	conf.SetDefault("flush-handler", "kinesis")

	pflag.IntP("flush-interval", "i", 5, "The number of seconds before the buffer is flushed and written to Kinesis")
	conf.BindPFlag("flush-interval", pflag.Lookup("flush-interval"))
	conf.SetDefault("flush-interval", 5)

	pflag.StringP("partition-key", "p", "", "The partition key, defaults to a 12 character random string if omitted")
	conf.BindPFlag("partition-key", pflag.Lookup("partition-key"))
	conf.SetDefault("partition-key", "")

	pflag.StringP("region", "R", "", "The AWS region that the Kinesis stream is provisioned in")
	conf.BindPFlag("region", pflag.Lookup("region"))
	conf.SetDefault("region", "")

	pflag.StringP("role-arn", "r", "", "The ARN of the AWS role being assumed.")
	conf.BindPFlag("role-arn", pflag.Lookup("role-arn"))
	conf.SetDefault("role-arn", "")

	pflag.StringP("role-session-name", "S", "", "The session name used when assuming a role.")
	conf.BindPFlag("role-session-name", pflag.Lookup("role-session-name"))
	conf.SetDefault("role-session-name", "")

	pflag.StringP("stream-name", "s", "", "The name of the Kinesis stream")
	conf.BindPFlag("stream-name", pflag.Lookup("stream-name"))
	conf.SetDefault("stream-name", "")

	pflag.Parse()

	if conf.GetBool("debug") {
		logger = NewLogger(LOG_DEBUG)
	} else {
		logger = NewLogger(LOG_INFO)
	}

	logger.Debug("configuration parsed")

	h := conf.GetString("flush-handler")
	if h != "kinesis" && h != "logger" {
		logger.Fatalf("flush handler not valid: %s", h)
	}

	fn := conf.GetString("fifo-name")
	if fn == "" {
		logger.Fatal("missing required option: fifo-name")
	}

	sn := conf.GetString("stream-name")
	if h == "kinesis" && sn == "" {
		logger.Fatal("missing required option: stream-name")
	}

	ql := conf.GetInt("buffer-queue-limit")
	if ql < 1 {
		logger.Fatal("buffer queue limit must be greater than 0")
	} else if h == "kinesis" && ql > 500 {
		logger.Fatal("buffer queue cannot exceed 500 items when using the kinesis handler")
	}

	fifo := &Fifo{fn}

	bw := &MemoryBufferWriter{
		Fifo:          fifo,
		FlushInterval: conf.GetInt("flush-interval"),
		QueueLimit:    ql,
	}

	var bf BufferFlusher
	if h == "kinesis" {
		pk := conf.GetString("partition-key")
		bf = NewKinesisBufferFlusher(sn, pk)
	} else {
		bf = &LoggerBufferFlusher{}
	}

	var fh FailedAttemptHandler
	dir := conf.GetString("failed-attempts-dir")
	if dir == "" {
		fh = &NullFailedAttemptHandler{}
	} else {
		stat, err := os.Stat(dir)
		if os.IsNotExist(err) {
			logger.Fatal("failed attempts directory does not exist")
		} else if !stat.IsDir() {
			logger.Fatal("failed attempts directory is not a directory")
		} else if unix.Access(dir, unix.R_OK) != nil {
			logger.Fatal("failed attempts directory is not readable")
		} else {
			fh = &FileFailedAttemptHandler{dir, fifo}
		}
	}

	shutdown := EventListener()
	RunPipeline(fifo, &Buffer{bw, bf, fh}, shutdown)
}