Example #1
0
func main() {
	buckets := flag.String("buckets", "", "buckets file (file format: new-line separated list of bucket names)")
	config_file := flag.String("config", "", "Transport config file")
	flag.Parse()

	if *config_file == "" {
		log.Fatal("You must specify config file")
	}

	for _, h := range proxy_handlers {
		h.Estimator = estimator.NewEstimator()
	}
	estimator_scan_handlers = proxy_handlers

	var err error

	conf := &config.ProxyConfig{}
	err = conf.Load(*config_file)
	if err != nil {
		log.Fatalf("Could not load config %s: %q", *config_file, err)
	}

	if *buckets == "" && len(conf.Elliptics.BucketList) == 0 {
		log.Fatalf("There is no buckets file and there is no 'bucket-list' option in elliptics config.")
	}

	if len(conf.Proxy.Address) == 0 {
		log.Fatalf("'address' must be specified in proxy config '%s'\n", *config_file)
	}

	if conf.Proxy.RedirectPort == 0 || conf.Proxy.RedirectPort >= 65536 {
		log.Printf("redirect is not allowed because of invalid redirect port %d",
			conf.Proxy.RedirectPort)
	}

	proxy.last_errors = make([]ErrorInfo, last_errors_length, last_errors_length)

	proxy.ell, err = etransport.NewEllipticsTransport(conf)
	if err != nil {
		log.Fatalf("Could not create Elliptics transport: %v", err)
	}

	rand.Seed(time.Now().Unix())

	proxy.bctl, err = bucket.NewBucketCtl(proxy.ell, *buckets, *config_file)
	if err != nil {
		log.Fatalf("Could not create new bucket controller: %v", err)
	}

	go func() {
		debug.SetGCPercent(10000)
		var stats debug.GCStats

		for {
			time.Sleep(5 * time.Second)

			runtime.GC()
			debug.ReadGCStats(&stats)

			log.Printf("gc: start: %s, duration: %s\n", stats.LastGC.String(), stats.Pause[0].String())
		}
	}()

	if len(conf.Proxy.HTTPSAddress) != 0 {
		if len(conf.Proxy.CertFile) == 0 {
			log.Fatalf("If you have specified HTTPS address there MUST be certificate file option")
		}

		if len(conf.Proxy.KeyFile) == 0 {
			log.Fatalf("If you have specified HTTPS address there MUST be key file option")
		}

		// this is needed to allow both HTTPS and HTTP handlers
		go func() {
			server := proxy.getTimeoutServer(proxy.bctl.Conf.Proxy.HTTPSAddress, http.HandlerFunc(generic_handler))
			log.Fatal(server.ListenAndServeTLS(conf.Proxy.CertFile, conf.Proxy.KeyFile))
		}()
	}

	if len(conf.Proxy.Address) != 0 {
		server := proxy.getTimeoutServer(proxy.bctl.Conf.Proxy.Address, http.HandlerFunc(generic_handler))
		log.Fatal(server.ListenAndServe())
	}
}
Example #2
0
func (bt *BackrunnerTest) StartEllipticsClientProxy(proxy_path string) {
	bt.conf.Elliptics.LogFile = bt.proxy_log
	bt.conf.Elliptics.LogPrefix = "backrunner: "

	file := fmt.Sprintf("%s/elliptics_transport.conf", bt.base)
	err := bt.conf.Save(file)
	if err != nil {
		log.Fatalf("Could not save client transport config: %v", err)
	}

	bt.ell, err = etransport.NewEllipticsTransport(&bt.conf.Elliptics)
	if err != nil {
		log.Fatal("Could not connect to elliptics server %v: %v", bt.elliptics_address, err)
	}

	bt.bucket_file = fmt.Sprintf("%s/buckets", bt.base)
	fd, err := os.Create(bt.bucket_file)
	if err != nil {
		log.Fatalf("Could not create bucket file %s: %v\n", bt.bucket_file, err)
	}
	defer fd.Close()

	_, err = fmt.Fprintf(fd, "%s\n", bt.acl_bucket)
	if err != nil {
		log.Fatalf("Could not write acl bucket into bucket file: %v\n", err)
	}
	for _, g := range bt.groups {
		b := fmt.Sprintf("b%d", g)
		_, err = fmt.Fprintf(fd, "%s\n", b)
		if err != nil {
			log.Fatalf("Could not write bucket into bucket file: %v\n", err)
		}

		meta := bucket.BucketMsgpack{
			Version: 1,
			Name:    b,
			Groups:  []uint32{g},
			Acl:     make(map[string]bucket.BucketACL),
		}

		acl := bucket.BucketACL{
			Version: 1,
			User:    bt.all_allowed_user,
			Token:   bt.all_allowed_token,
			Flags:   bucket.BucketAuthWrite | bucket.BucketAuthNoToken,
		}
		meta.Acl[acl.User] = acl

		_, err := bucket.WriteBucket(bt.ell, &meta)
		if err != nil {
			log.Fatal("Could not upload bucket %s into storage: %v", b, err)
		}

		bt.io_buckets = append(bt.io_buckets, b)
	}

	bt.ACLInit()

	file = fmt.Sprintf("%s/proxy.conf", bt.base)
	err = bt.conf.Save(file)
	if err != nil {
		log.Fatalf("Could not save proxy config: %v", err)
	}

	cmd := exec.Command(proxy_path, "-config", file, "-buckets", bt.bucket_file)
	cmd.Stdout = &bt.proxy_stdout
	cmd.Stderr = &bt.proxy_stderr

	err = cmd.Start()
	if err != nil {
		log.Fatalf("Could not start proxy process: %v\n", err)
	}

	bt.proxy_cmd = cmd

	// wait 1 second for proxy to start
	time.Sleep(1 * time.Second)
}
Example #3
0
func main() {
	bname := flag.String("bucket", "", "bucket name to read")
	config_file := flag.String("config", "", "transport config file")
	backrunner_config := flag.String("upload-backrunner-config", "",
		"backrunner config file to upload into elliptics using 'backrunner-config-key' config parameter")
	bucket_list_upload := flag.String("upload-bucket-list", "",
		"bucket list to upload into elliptics using 'bucket-list-key' config parameter")
	upload := flag.String("upload", "", "bucket json file to upload/rewrite")
	flag.Parse()

	if *bname == "" && *upload == "" && *backrunner_config == "" && *bucket_list_upload == "" {
		log.Fatal("You must specify one (or more) of the following options:\n" +
			"* bucket name to read\n" +
			"* file with buckets metadata to upload\n" +
			"* list of buckets to upload\n" +
			"* backrunner config to upload\n")
	}

	if *config_file == "" {
		log.Fatal("You must specify Elliptics config file")
	}

	conf := &config.ProxyConfig{}
	err := conf.Load(*config_file)
	if err != nil {
		log.Fatalf("Could not load config file '%s': %v", *config_file, err)
	}

	ell, err := etransport.NewEllipticsTransport(conf)
	if err != nil {
		log.Fatalf("Could not create Elliptics transport: %v", err)
	}

	var b *bucket.Bucket

	if *upload != "" {
		err = bmeta_read_upload(ell, *upload)
		if err != nil {
			log.Fatalf("Could not write some buckets: %v", err)
		}
	}

	if *bname != "" {
		b, err = bucket.ReadBucket(ell, *bname)
		if err != nil {
			log.Fatalf("Could not read bucket %s: %v", *bname, err)
		}

		log.Printf("%s\n", b.Meta.String())
		fmt.Printf("%s\n", b.Meta.String())
	}

	if *backrunner_config != "" {
		if len(conf.Elliptics.BackrunnerConfig) == 0 {
			log.Fatalf("Requested uploading %s into elliptics as backrunner config, "+
				"but there is no 'backrunner-config-key' option",
				*backrunner_config)
		}

		err = bmeta_read_upload_file(ell, *backrunner_config, conf.Elliptics.BackrunnerConfig)
		if err != nil {
			log.Fatalf("upload: %v", err)
		}
	}

	if *bucket_list_upload != "" {
		if len(conf.Elliptics.BucketList) == 0 {
			log.Fatalf("Requested uploading %s into elliptics as list of buckets, "+
				"but there is no 'bucket-list-key' option",
				*bucket_list_upload)
		}

		err = bmeta_read_upload_file(ell, *bucket_list_upload, conf.Elliptics.BucketList)
		if err != nil {
			log.Fatalf("upload: %v", err)
		}
	}
}