// Global
var (
// ConfigFile is the config file data structure
ConfigFile *goconfig.ConfigFile
// HomeDir is the home directory of the user
HomeDir = configHome()
// ConfigPath points to the config file
ConfigPath = path.Join(HomeDir, configFileName)
// Config is the global config
Config = &ConfigInfo{}
// Flags
verbose = pflag.BoolP("verbose", "v", false, "Print lots more stuff")
quiet = pflag.BoolP("quiet", "q", false, "Print as little stuff as possible")
modifyWindow = pflag.DurationP("modify-window", "", time.Nanosecond, "Max time diff to be considered the same")
checkers = pflag.IntP("checkers", "", 8, "Number of checkers to run in parallel.")
transfers = pflag.IntP("transfers", "", 4, "Number of file transfers to run in parallel.")
configFile = pflag.StringP("config", "", ConfigPath, "Config file.")
checkSum = pflag.BoolP("checksum", "c", false, "Skip based on checksum & size, not mod-time & size")
sizeOnly = pflag.BoolP("size-only", "", false, "Skip based on size only, not mod-time or checksum")
ignoreExisting = pflag.BoolP("ignore-existing", "", false, "Skip all files that exist on destination")
dryRun = pflag.BoolP("dry-run", "n", false, "Do a trial run with no permanent changes")
connectTimeout = pflag.DurationP("contimeout", "", 60*time.Second, "Connect timeout")
timeout = pflag.DurationP("timeout", "", 5*60*time.Second, "IO idle timeout")
dumpHeaders = pflag.BoolP("dump-headers", "", false, "Dump HTTP headers - may contain sensitive info")
dumpBodies = pflag.BoolP("dump-bodies", "", false, "Dump HTTP headers and bodies - may contain sensitive info")
skipVerify = pflag.BoolP("no-check-certificate", "", false, "Do not verify the server SSL certificate. Insecure.")
deleteBefore = pflag.BoolP("delete-before", "", false, "When synchronizing, delete files on destination before transfering")
deleteDuring = pflag.BoolP("delete-during", "", false, "When synchronizing, delete files during transfer (default)")
deleteAfter = pflag.BoolP("delete-after", "", false, "When synchronizing, delete files on destination after transfering")
bwLimit SizeSuffix
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/ncw/rclone/fs"
)
// Globals
var (
// Flags
cpuProfile = pflag.StringP("cpuprofile", "", "", "Write cpu profile to file")
memProfile = pflag.String("memprofile", "", "Write memory profile to file")
statsInterval = pflag.DurationP("stats", "", time.Minute*1, "Interval to print stats (0 to disable)")
version bool
logFile = pflag.StringP("log-file", "", "", "Log everything to this file")
retries = pflag.IntP("retries", "", 3, "Retry operations this many times if they fail")
)
// Root is the main rclone command
var Root = &cobra.Command{
Use: "rclone",
Short: "Sync files and directories to and from local and remote object stores - " + fs.Version,
Long: `
Rclone is a command line program to sync files and directories to and
from various cloud storage systems, such as:
* Google Drive
* Amazon S3
* Openstack Swift / Rackspace cloud files / Memset Memstore
* Dropbox
* Google Cloud Storage
package cli
import (
"fmt"
"os"
"github.com/pilebones/backdoorGolang/core/common"
"github.com/pilebones/backdoorGolang/core/socket"
"github.com/spf13/pflag"
)
var (
host = pflag.StringP("host", "h", "localhost", "Set hostname to use")
port = pflag.IntP("port", "p", 9876, "Set port number to use")
isListenMode = pflag.BoolP("listen", "l", false, "Enable listen mode (server socket mode)")
isVerboseMode = pflag.BoolP("verbose", "v", false, "Enable mode verbose")
isDebugMode = pflag.BoolP("debug", "d", false, "Enable mode debug")
isVersionMode = pflag.BoolP("version", "V", false, "Display version number")
)
/** Print data when debug mode is enabled */
func DisplayAsDebug(message string) {
if UseDebugMode() {
fmt.Println(message)
}
}
/** Return true if mode use from parameter */
func UseMode(mode *bool) bool {
if *mode {
return true
// Global
var (
// ConfigFile is the config file data structure
ConfigFile *goconfig.ConfigFile
// HomeDir is the home directory of the user
HomeDir = configHome()
// ConfigPath points to the config file
ConfigPath = path.Join(HomeDir, configFileName)
// Config is the global config
Config = &ConfigInfo{}
// Flags
verbose = pflag.BoolP("verbose", "v", false, "Print lots more stuff")
quiet = pflag.BoolP("quiet", "q", false, "Print as little stuff as possible")
modifyWindow = pflag.DurationP("modify-window", "", time.Nanosecond, "Max time diff to be considered the same")
checkers = pflag.IntP("checkers", "", 8, "Number of checkers to run in parallel.")
transfers = pflag.IntP("transfers", "", 4, "Number of file transfers to run in parallel.")
configFile = pflag.StringP("config", "", ConfigPath, "Config file.")
checkSum = pflag.BoolP("checksum", "c", false, "Skip based on checksum & size, not mod-time & size")
sizeOnly = pflag.BoolP("size-only", "", false, "Skip based on size only, not mod-time or checksum")
ignoreTimes = pflag.BoolP("ignore-times", "I", false, "Don't skip files that match size and time - transfer all files")
ignoreExisting = pflag.BoolP("ignore-existing", "", false, "Skip all files that exist on destination")
dryRun = pflag.BoolP("dry-run", "n", false, "Do a trial run with no permanent changes")
connectTimeout = pflag.DurationP("contimeout", "", 60*time.Second, "Connect timeout")
timeout = pflag.DurationP("timeout", "", 5*60*time.Second, "IO idle timeout")
dumpHeaders = pflag.BoolP("dump-headers", "", false, "Dump HTTP headers - may contain sensitive info")
dumpBodies = pflag.BoolP("dump-bodies", "", false, "Dump HTTP headers and bodies - may contain sensitive info")
skipVerify = pflag.BoolP("no-check-certificate", "", false, "Do not verify the server SSL certificate. Insecure.")
AskPassword = pflag.BoolP("ask-password", "", true, "Allow prompt for password for encrypted configuration.")
deleteBefore = pflag.BoolP("delete-before", "", false, "When synchronizing, delete files on destination before transfering")
deleteDuring = pflag.BoolP("delete-during", "", false, "When synchronizing, delete files during transfer (default)")
"encoding/json"
"fmt"
flag "github.com/spf13/pflag"
"io"
"net"
"os"
"strconv"
"strings"
"time"
)
const (
dateFormat = "0102061504"
)
var portP = flag.IntP("port", "p", 9000, "Default port to use")
var addressP = flag.StringP("address", "a", "127.0.0.1", "Default listen address to use")
type Header struct {
Data [16]byte
}
type Record struct {
Date [6]byte
_ byte
Time [4]byte
_ byte
SecDur [5]byte
_ byte
CodCode [1]byte
_ byte
// IntP defines a flag which can be overridden by an environment variable
//
// It is a thin wrapper around pflag.IntP
func IntP(name, shorthand string, value int, usage string) (out *int) {
out = pflag.IntP(name, shorthand, value, usage)
setDefaultFromEnv(name)
return out
}
func main() {
conf = viper.New()
conf.SetEnvPrefix("FIFO2KINESIS")
conf.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
conf.AutomaticEnv()
viper.SetConfigName("fifo2kinesis")
pflag.IntP("buffer-queue-limit", "l", 500, "The maximum number of items in the buffer before it is flushed")
conf.BindPFlag("buffer-queue-limit", pflag.Lookup("buffer-queue-limit"))
conf.SetDefault("buffer-queue-limit", 500)
pflag.BoolP("debug", "d", false, "Show debug level log messages")
conf.BindPFlag("debug", pflag.Lookup("debug"))
conf.SetDefault("debug", "")
pflag.StringP("failed-attempts-dir", "D", "", "The path to the directory containing failed attempts")
conf.BindPFlag("failed-attempts-dir", pflag.Lookup("failed-attempts-dir"))
conf.SetDefault("failed-attempts-dir", "")
pflag.StringP("fifo-name", "f", "", "The absolute path of the named pipe, e.g. /var/test.pipe")
conf.BindPFlag("fifo-name", pflag.Lookup("fifo-name"))
conf.SetDefault("fifo-name", "")
pflag.StringP("flush-handler", "h", "kinesis", "Defaults to \"kinesis\", use \"logger\" for debugging")
conf.BindPFlag("flush-handler", pflag.Lookup("flush-handler"))
conf.SetDefault("flush-handler", "kinesis")
pflag.IntP("flush-interval", "i", 5, "The number of seconds before the buffer is flushed and written to Kinesis")
conf.BindPFlag("flush-interval", pflag.Lookup("flush-interval"))
conf.SetDefault("flush-interval", 5)
pflag.StringP("partition-key", "p", "", "The partition key, defaults to a 12 character random string if omitted")
conf.BindPFlag("partition-key", pflag.Lookup("partition-key"))
conf.SetDefault("partition-key", "")
pflag.StringP("region", "R", "", "The AWS region that the Kinesis stream is provisioned in")
conf.BindPFlag("region", pflag.Lookup("region"))
conf.SetDefault("region", "")
pflag.StringP("role-arn", "r", "", "The ARN of the AWS role being assumed.")
conf.BindPFlag("role-arn", pflag.Lookup("role-arn"))
conf.SetDefault("role-arn", "")
pflag.StringP("role-session-name", "S", "", "The session name used when assuming a role.")
conf.BindPFlag("role-session-name", pflag.Lookup("role-session-name"))
conf.SetDefault("role-session-name", "")
pflag.StringP("stream-name", "s", "", "The name of the Kinesis stream")
conf.BindPFlag("stream-name", pflag.Lookup("stream-name"))
conf.SetDefault("stream-name", "")
pflag.Parse()
if conf.GetBool("debug") {
logger = NewLogger(LOG_DEBUG)
} else {
logger = NewLogger(LOG_INFO)
}
logger.Debug("configuration parsed")
h := conf.GetString("flush-handler")
if h != "kinesis" && h != "logger" {
logger.Fatalf("flush handler not valid: %s", h)
}
fn := conf.GetString("fifo-name")
if fn == "" {
logger.Fatal("missing required option: fifo-name")
}
sn := conf.GetString("stream-name")
if h == "kinesis" && sn == "" {
logger.Fatal("missing required option: stream-name")
}
ql := conf.GetInt("buffer-queue-limit")
if ql < 1 {
logger.Fatal("buffer queue limit must be greater than 0")
} else if h == "kinesis" && ql > 500 {
logger.Fatal("buffer queue cannot exceed 500 items when using the kinesis handler")
}
fifo := &Fifo{fn}
bw := &MemoryBufferWriter{
Fifo: fifo,
FlushInterval: conf.GetInt("flush-interval"),
QueueLimit: ql,
}
var bf BufferFlusher
if h == "kinesis" {
pk := conf.GetString("partition-key")
bf = NewKinesisBufferFlusher(sn, pk)
} else {
bf = &LoggerBufferFlusher{}
}
var fh FailedAttemptHandler
dir := conf.GetString("failed-attempts-dir")
if dir == "" {
fh = &NullFailedAttemptHandler{}
} else {
stat, err := os.Stat(dir)
if os.IsNotExist(err) {
logger.Fatal("failed attempts directory does not exist")
} else if !stat.IsDir() {
logger.Fatal("failed attempts directory is not a directory")
} else if unix.Access(dir, unix.R_OK) != nil {
logger.Fatal("failed attempts directory is not readable")
} else {
fh = &FileFailedAttemptHandler{dir, fifo}
}
}
shutdown := EventListener()
RunPipeline(fifo, &Buffer{bw, bf, fh}, shutdown)
}