示例#1
0
func main() {
	flag.Parse()

	httputil.InitHTTPTracker(false)

	if flag.NArg() == 0 {
		log.Fatalf("Seriesly URL required")
	}

	s, err := serieslyclient.New(flag.Arg(0))
	maybeFatal(err, "Parsing %v: %v", flag.Arg(0), err)

	wg := &sync.WaitGroup{}
	ch := make(chan string)

	for i := 0; i < *concurrency; i++ {
		wg.Add(1)
		go dump(wg, s, ch)
	}

	if *dbName == "" {
		dbs, err := s.List()
		maybeFatal(err, "Error listing: %v", err)
		for _, db := range dbs {
			ch <- db
		}
	} else {
		ch <- *dbName
	}
	close(ch)

	wg.Wait()
}
示例#2
0
func main() {
	flag.Parse()

	if flag.NArg() < 1 {
		log.Fatalf("Usage:  gzip -dc backup.gz | %v http://seriesly:3133/dbname",
			os.Args[0])
	}

	httputil.InitHTTPTracker(false)

	u := flag.Arg(0)
	setupDb(u)

	minTime := parseMinTime()

	t := time.Tick(5 * time.Second)
	i := 0
	var latestKey string

	d := json.NewDecoder(os.Stdin)
	for {
		kv := map[string]*json.RawMessage{}

		err := d.Decode(&kv)
		if err == io.EOF {
			log.Printf("Done!  Processed %v items. Last was %v",
				i, latestKey)
			break
		}
		maybeFatal(err)

		for k, v := range kv {
			if !minTime.IsZero() {
				thist, err := timelib.ParseTime(k)
				if err == nil && minTime.After(thist) {
					continue
				}
			}
			body := []byte(*v)
			sendOne(u, k, body)
			latestKey = k
		}

		i++
		select {
		case <-t:
			log.Printf("Processed %v items, latest was %v", i, latestKey)
		default:
		}
	}
}
示例#3
0
文件: download.go 项目: 40a/cbfs
func downloadCommand(u string, args []string) {
	src := dlFlags.Arg(0)
	destbase := dlFlags.Arg(1)

	if destbase == "" {
		destbase = filepath.Base(src)
	}

	for len(src) > 0 && src[0] == '/' {
		src = src[1:]
	}

	httputil.InitHTTPTracker(false)

	client, err := cbfsclient.New(u)
	cbfstool.MaybeFatal(err, "Can't build a client: %v", err)

	things, err := client.ListDepth(src, 4096)
	cbfstool.MaybeFatal(err, "Can't list things: %v", err)

	start := time.Now()
	oids := []string{}
	dests := map[string][]string{}
	for fn, inf := range things.Files {
		fn = fn[len(src):]
		dests[inf.OID] = append(dests[inf.OID],
			filepath.Join(destbase, fn))
		oids = append(oids, inf.OID)
	}

	err = client.Blobs(*totalConcurrency, *nodeConcurrency,
		func(oid string, r io.Reader) error {
			return saveDownload(dests[oid], oid, r)
		}, oids...)

	cbfstool.MaybeFatal(err, "Error getting blobs: %v", err)

	b := atomic.AddInt64(&totalBytes, 0)
	d := time.Since(start)
	cbfstool.Verbose(*dlverbose, "Moved %s in %v (%s/s)", humanize.Bytes(uint64(b)),
		d, humanize.Bytes(uint64(float64(b)/d.Seconds())))
}
示例#4
0
文件: find.go 项目: 40a/cbfs
func findCommand(u string, args []string) {
	if *findDashName != "" && *findDashIName != "" {
		log.Fatalf("Can't specify both -name and -iname")
	}
	src := findFlags.Arg(0)
	for src[len(src)-1] == '/' {
		src = src[:len(src)-1]
	}

	tmpl := cbfstool.GetTemplate(*findTemplate, *findTemplateFile,
		defaultFindTemplate)

	httputil.InitHTTPTracker(false)

	client, err := cbfsclient.New(u)
	cbfstool.MaybeFatal(err, "Can't build a client: %v", err)

	things, err := client.ListDepth(src, *findDashDepth)
	cbfstool.MaybeFatal(err, "Can't list things: %v", err)

	metaMatcher := findGetRefTimeMatch(time.Now())
	matcher := newDirAndFileMatcher()
	for fn, inf := range things.Files {
		if !metaMatcher(inf.Modified) {
			continue
		}
		if len(fn) > len(src)+1 {
			fn = fn[len(src)+1:]
		}
		for _, match := range matcher.matches(fn) {
			if err := tmpl.Execute(os.Stdout, struct {
				Name  string
				IsDir bool
				Meta  cbfsclient.FileMeta
			}{match.path, match.isDir, inf}); err != nil {
				log.Fatalf("Error executing template: %v", err)
			}
		}
	}
}
示例#5
0
文件: main.go 项目: zhgwenming/cbfs
func main() {
	flag.Parse()

	rand.Seed(time.Now().UnixNano())

	initLogger(*useSyslog)
	initNodeListKeys()

	http.DefaultTransport = TimeoutTransport(*internodeTimeout)
	expvar.Publish("httpclients", httputil.InitHTTPTracker(false))

	if getHash() == nil {
		fmt.Fprintf(os.Stderr,
			"Unsupported hash specified: %v.  Supported hashes:\n",
			globalConfig.Hash)
		for h := range hashBuilders {
			fmt.Fprintf(os.Stderr, " * %v\n", h)
		}
		os.Exit(1)
	}

	err := initServerId()
	if err != nil {
		log.Fatalf("Error initializing server ID: %v", err)
	}

	if *maxStorageString != "" {
		ms, err := humanize.ParseBytes(*maxStorageString)
		if err != nil {
			log.Fatalf("Error parsing max storage parameter: %v",
				err)
		}
		maxStorage = int64(ms)
	}

	couchbase, err = dbConnect()
	if err != nil {
		log.Fatalf("Can't connect to couchbase: %v", err)
	}

	if err = os.MkdirAll(*root, 0777); err != nil {
		log.Fatalf("Couldn't create storage dir: %v", err)
	}

	err = updateConfig()
	if err != nil && !gomemcached.IsNotFound(err) {
		log.Printf("Error updating initial config, using default: %v",
			err)
	}
	if *verbose {
		log.Printf("Server config:")
		globalConfig.Dump(os.Stdout)
	}

	go reloadConfig()

	go dnsServices()

	internodeTaskQueue = make(chan internodeTask, *taskWorkers*1024)
	initTaskQueueWorkers()

	go heartbeat()
	go startTasks()

	time.AfterFunc(time.Second*time.Duration(rand.Intn(30)+5), grabSomeData)

	go serveFrame()

	s := &http.Server{
		Addr:        *bindAddr,
		Handler:     http.HandlerFunc(httpHandler),
		ReadTimeout: *readTimeout,
	}
	log.Printf("Listening to web requests on %s as server %s",
		*bindAddr, serverId)

	l, err := rateListen("tcp", *bindAddr)
	if err != nil {
		log.Fatalf("Error listening: %v", err)
	}
	log.Fatal(s.Serve(l))
}
示例#6
0
文件: upload.go 项目: 40a/cbfs
func uploadCommand(u string, args []string) {
	initCrypto()
	httputil.InitHTTPTracker(false)

	uploadFlags.Visit(func(f *flag.Flag) {
		if f.Name == "revs" {
			uploadRevsSet = true
		}
	})

	if *uploadIgnore != "" {
		err := loadIgnorePatternsFromFile(*uploadIgnore)
		cbfstool.MaybeFatal(err, "Error loading ignores: %v", err)
	}

	client, err := cbfsclient.New(u)
	cbfstool.MaybeFatal(err, "Error setting up client: %v", err)

	srcFn := uploadFlags.Arg(0)
	dest := uploadFlags.Arg(1)

	// Special case stdin.
	if srcFn == "-" {
		err := uploadStream(client, os.Stdin, "", dest, "")
		cbfstool.MaybeFatal(err, "Error uploading stdin: %v", err)
		return
	}

	// Special case http as well
	if strings.HasPrefix(srcFn, "http:") || strings.HasPrefix(srcFn, "https:") {
		req, err := http.NewRequest("GET", srcFn, nil)
		// Disable compression so we have a chance at getting a res length
		req.Header.Set("Accept-Encoding", "")
		cbfstool.MaybeFatal(err, "Error creating request: %v", err)
		res, err := http.DefaultClient.Do(req)
		cbfstool.MaybeFatal(err, "Error making http request to %v: %v",
			srcFn, err)
		defer res.Body.Close()
		if res.StatusCode != 200 {
			log.Printf("HTTP error fetching %v: %v", srcFn, err)
			io.Copy(os.Stderr, res.Body)
			os.Exit(1)
		}
		var r io.ReadCloser = res.Body
		if res.ContentLength > 0 {
			r = newProgressReader(res.Body, res.ContentLength)
			defer r.Close()
		}
		err = uploadStream(client, r, srcFn, dest, "")
		cbfstool.MaybeFatal(err, "Error uploading from URL: %v", err)
		return
	}

	fi, err := os.Stat(srcFn)
	cbfstool.MaybeFatal(err, "Error statting %v: %v", srcFn, err)

	if fi.IsDir() {
		ch := make(chan uploadReq, 1000)
		ech := make(chan error, *uploadWorkers)

		for i := 0; i < *uploadWorkers; i++ {
			uploadWg.Add(1)
			go uploadWorker(client, ch, ech)
		}

		start := time.Now()
		syncUp(client, srcFn, dest, ch)

		close(ch)
		cbfstool.Verbose(*uploadVerbose, "Finished traversal in %v",
			time.Since(start))
		waitch := make(chan bool)

		go func() {
			uploadWg.Wait()
			close(waitch)
		}()

		rc := 0
	collect:
		for {
			select {
			case <-waitch:
				close(ech)
				waitch = nil
			case err, ok := <-ech:
				if !ok {
					break collect
				}
				log.Printf("Permanent upload error: %v", err)
				rc = 1
			}
		}

		cbfstool.Verbose(*uploadVerbose, "Finished sync in %v",
			time.Since(start))
		os.Exit(rc)
	} else {
		err = uploadFile(client, srcFn, dest, localHash(srcFn))
		cbfstool.MaybeFatal(err, "Error uploading file: %v", err)
	}
}