コード例 #1
0
ファイル: datapath.go プロジェクト: jchris/indexing
func main() {
	clusters := argParse()

	// setup cbauth
	up := strings.Split(options.auth, ":")
	_, err := cbauth.InternalRetryDefaultInit(clusters[0], up[0], up[1])
	if err != nil {
		log.Fatalf("Failed to initialize cbauth: %s", err)
	}

	maxvbs := c.SystemConfig["maxVbuckets"].Int()
	dconf := c.SystemConfig.SectionConfig("indexer.dataport.", true)

	// start dataport servers.
	for _, endpoint := range options.endpoints {
		go dataport.Application(
			endpoint, options.stat, options.timeout, maxvbs, dconf,
			func(addr string, msg interface{}) bool { return true })
	}
	go dataport.Application(options.coordEndpoint, 0, 0, maxvbs, dconf, nil)

	for _, cluster := range clusters {
		adminport := getProjectorAdminport(cluster, "default")
		if options.projector {
			config := c.SystemConfig.Clone()
			config.SetValue("projector.clusterAddr", cluster)
			config.SetValue("projector.adminport.listenAddr", adminport)
			econf := c.SystemConfig.SectionConfig("projector.dataport.", true)
			epfactory := NewEndpointFactory(cluster, maxvbs, econf)
			config.SetValue("projector.routerEndpointFactory", epfactory)
			projector.NewProjector(maxvbs, config) // start projector daemon
		}

		// projector-client
		cconfig := c.SystemConfig.SectionConfig("indexer.projectorclient.", true)
		projectors[cluster] = projc.NewClient(adminport, maxvbs, cconfig)
	}

	// index instances for specified buckets.
	instances := protobuf.ExampleIndexInstances(
		options.buckets, options.endpoints, options.coordEndpoint)

	// start backfill stream on each projector
	for _, client := range projectors {
		// start backfill stream on each projector
		_, err := client.InitialTopicRequest(
			"backfill" /*topic*/, "default", /*pooln*/
			"dataport" /*endpointType*/, instances)
		if err != nil {
			log.Fatal(err)
		}
	}

	time.Sleep(1000 * time.Second)
	//<-make(chan bool) // wait for ever
}
コード例 #2
0
ファイル: streamwait.go プロジェクト: prataprc/indexing
func main() {
	cluster := argParse()[0]

	// setup cbauth
	if options.auth != "" {
		up := strings.Split(options.auth, ":")
		if _, err := cbauth.InternalRetryDefaultInit(cluster, up[0], up[1]); err != nil {
			logging.Fatalf("Failed to initialize cbauth: %s", err)
		}
	}
	go startFeed(cluster, "streamwait-feed1")
	go startFeed(cluster, "streamwait-feed2")
	time.Sleep(1000 * time.Second)
}
コード例 #3
0
ファイル: cbauth-example.go プロジェクト: aartamonau/cbauth
func maybeReinitCBAuth() {
	if authFlag == "" {
		return
	}
	up := strings.Split(authFlag, ":")
	authU, authP := up[0], up[1]
	u, err := url.Parse(mgmtURLFlag)
	if err != nil {
		log.Fatal("Failed to parse mgmtURLFlag: ", err)
	}
	_, err = cbauth.InternalRetryDefaultInit(u.Host, authU, authP)
	if err != nil {
		log.Fatal("Failed to initialize cbauth: ", err)
	}
}
コード例 #4
0
ファイル: main.go プロジェクト: jchris/indexing
func main() {
	platform.HideConsole(true)
	defer platform.HideConsole(false)
	common.SeedProcess()

	logging.Infof("Indexer started with command line: %v\n", os.Args)
	flag.Parse()

	logging.SetLogLevel(logging.Level(*logLevel))
	forestdb.Log = &logging.SystemLogger

	// setup cbauth
	if *auth != "" {
		up := strings.Split(*auth, ":")
		logging.Tracef("Initializing cbauth with user %v for cluster %v\n", up[0], *cluster)
		if _, err := cbauth.InternalRetryDefaultInit(*cluster, up[0], up[1]); err != nil {
			logging.Fatalf("Failed to initialize cbauth: %s", err)
		}
	}

	go platform.DumpOnSignal()
	go common.ExitOnStdinClose()

	config := common.SystemConfig
	config.SetValue("indexer.clusterAddr", *cluster)
	config.SetValue("indexer.numVbuckets", *numVbuckets)
	config.SetValue("indexer.enableManager", *enableManager)
	config.SetValue("indexer.adminPort", *adminPort)
	config.SetValue("indexer.scanPort", *scanPort)
	config.SetValue("indexer.httpPort", *httpPort)
	config.SetValue("indexer.streamInitPort", *streamInitPort)
	config.SetValue("indexer.streamCatchupPort", *streamCatchupPort)
	config.SetValue("indexer.streamMaintPort", *streamMaintPort)
	config.SetValue("indexer.storage_dir", *storageDir)

	storage_dir := config["indexer.storage_dir"].String()
	if err := os.MkdirAll(storage_dir, 0755); err != nil {
		common.CrashOnError(err)
	}

	_, msg := indexer.NewIndexer(config)

	if msg.GetMsgType() != indexer.MSG_SUCCESS {
		logging.Warnf("Indexer Failure to Init %v", msg)
	}

	logging.Infof("Indexer exiting normally\n")
}
コード例 #5
0
ファイル: upr.go プロジェクト: prataprc/indexing
func main() {
	cluster := argParse()

	// setup cbauth
	if options.auth != "" {
		up := strings.Split(options.auth, ":")
		if _, err := cbauth.InternalRetryDefaultInit(cluster, up[0], up[1]); err != nil {
			logging.Fatalf("Failed to initialize cbauth: %s", err)
		}
	}

	for _, bucket := range options.buckets {
		go startBucket(cluster, bucket)
	}
	receive()
}
コード例 #6
0
ファイル: common_test.go プロジェクト: jchris/indexing
func init() {
	log.Printf("In init()")
	logging.SetLogLevel(logging.Warn)

	var configpath string
	flag.StringVar(&configpath, "cbconfig", "../config/clusterrun_conf.json", "Path of the configuration file with data about Couchbase Cluster")
	flag.Parse()
	clusterconfig = tc.GetClusterConfFromFile(configpath)
	kvaddress = clusterconfig.KVAddress
	indexManagementAddress = clusterconfig.KVAddress
	indexScanAddress = clusterconfig.KVAddress
	seed = 1
	proddir, bagdir = tc.FetchMonsterToolPath()

	// setup cbauth
	if _, err := cbauth.InternalRetryDefaultInit(kvaddress, clusterconfig.Username, clusterconfig.Password); err != nil {
		log.Fatalf("Failed to initialize cbauth: %s", err)
	}
	secondaryindex.CheckCollation = true
	e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
	tc.HandleError(e, "Error in DropAllSecondaryIndexes")

	time.Sleep(5 * time.Second)
	// Working with Users10k and Users_mut dataset.
	u, _ := user.Current()
	dataFilePath = filepath.Join(u.HomeDir, "testdata/Users10k.txt.gz")
	mutationFilePath = filepath.Join(u.HomeDir, "testdata/Users_mut.txt.gz")
	tc.DownloadDataFile(tc.IndexTypesStaticJSONDataS3, dataFilePath, true)
	tc.DownloadDataFile(tc.IndexTypesMutationJSONDataS3, mutationFilePath, true)
	docs = datautility.LoadJSONFromCompressedFile(dataFilePath, "docid")
	mut_docs = datautility.LoadJSONFromCompressedFile(mutationFilePath, "docid")
	log.Printf("Emptying the default bucket")
	kvutility.EnableBucketFlush("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	kvutility.FlushBucket("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	time.Sleep(5 * time.Second)

	log.Printf("Create Index On the empty default Bucket()")
	var indexName = "index_eyeColor"
	var bucketName = "default"

	err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{"eyeColor"}, false, nil, true, defaultIndexActiveTimeout, nil)
	tc.HandleError(err, "Error in creating the index")

	// Populate the bucket now
	log.Printf("Populating the default bucket")
	kvutility.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)
}
コード例 #7
0
ファイル: ts.go プロジェクト: jchris/indexing
func main() {
	argParse()

	// setup cbauth
	if options.auth != "" {
		up := strings.Split(options.auth, ":")
		if _, err := cbauth.InternalRetryDefaultInit(options.cluster, up[0], up[1]); err != nil {
			log.Fatalf("Failed to initialize cbauth: %s", err)
		}
	}

	numVb := numVbuckets(options.cluster, options.bucket)
	log.Printf("bucket %q has %v vbuckets\n", options.bucket, numVb)

	bucketTs(options.cluster, options.bucket, numVb)
	//getCurrentKVTs_old(options.cluster, options.bucket, numVb)
	getCurrentKVTs(options.cluster, options.bucket, numVb)
}
コード例 #8
0
ファイル: common_test.go プロジェクト: jchris/indexing
func init() {
	log.Printf("In init()")

	logging.SetLogLevel(logging.Warn)
	var configpath string
	seed = 1
	flag.StringVar(&configpath, "cbconfig", "../config/clusterrun_conf.json", "Path of the configuration file with data about Couchbase Cluster")
	flag.Parse()
	clusterconfig = tc.GetClusterConfFromFile(configpath)
	kvaddress = clusterconfig.KVAddress
	indexManagementAddress = clusterconfig.KVAddress
	indexScanAddress = clusterconfig.KVAddress

	// setup cbauth
	if _, err := cbauth.InternalRetryDefaultInit(kvaddress, clusterconfig.Username, clusterconfig.Password); err != nil {
		log.Fatalf("Failed to initialize cbauth: %s", err)
	}

	proddir, bagdir = tc.FetchMonsterToolPath()
}
コード例 #9
0
ファイル: main.go プロジェクト: jchris/indexing
func main() {
	platform.HideConsole(true)
	defer platform.HideConsole(false)
	c.SeedProcess()

	logging.Infof("Projector started with command line: %v\n", os.Args)

	cluster := argParse() // eg. "localhost:9000"

	config := c.SystemConfig.Clone()
	logging.SetLogLevel(logging.Level(options.loglevel))

	config.SetValue("maxVbuckets", options.numVbuckets)
	if f := getlogFile(); f != nil {
		fmt.Printf("Projector logging to %q\n", f.Name())
		logging.SetLogWriter(f)
		config.SetValue("log.file", f.Name())
	}
	config.SetValue("projector.clusterAddr", cluster)
	config.SetValue("projector.adminport.listenAddr", options.adminport)

	// setup cbauth
	if options.auth != "" {
		up := strings.Split(options.auth, ":")
		if _, err := cbauth.InternalRetryDefaultInit(cluster, up[0], up[1]); err != nil {
			logging.Fatalf("Failed to initialize cbauth: %s", err)
		}
	}

	epfactory := NewEndpointFactory(cluster, options.numVbuckets)
	config.SetValue("projector.routerEndpointFactory", epfactory)

	logging.Infof("%v\n", c.LogOs())
	logging.Infof("%v\n", c.LogRuntime())

	go c.ExitOnStdinClose()
	projector.NewProjector(options.numVbuckets, config)

	<-done
}
コード例 #10
0
ファイル: loadgen.go プロジェクト: jchris/indexing
func main() {
	cluster := argParse()
	if !strings.HasPrefix(cluster, "http://") {
		cluster = "http://" + cluster
	}

	// setup cbauth
	if options.auth != "" {
		up := strings.Split(options.auth, ":")
		if _, err := cbauth.InternalRetryDefaultInit(cluster, up[0], up[1]); err != nil {
			logging.Fatalf("Failed to initialize cbauth: %s", err)
		}
	}

	n := 0
	for i, bucket := range options.buckets {
		prodfile := options.prods[i]
		n += loadBucket(cluster, bucket, prodfile, options.count)
	}
	for n > 0 {
		<-done
		n--
	}
}
コード例 #11
0
ファイル: main.go プロジェクト: jchris/indexing
func main() {
	help := flag.Bool("help", false, "Help")
	config := flag.String("configfile", "config.json", "Scan load config file")
	outfile := flag.String("resultfile", "results.json", "Result report file")
	cpus := flag.Int("cpus", runtime.NumCPU(), "Number of CPUs")
	cluster := flag.String("cluster", "127.0.0.1:9000", "Cluster server address")
	auth := flag.String("auth", "Administrator:asdasd", "Auth")
	statsfile := flag.String("statsfile", "", "Periodic statistics report file")
	cpuprofile := flag.String("cpuprofile", "", "write cpu profile to file")
	memprofile := flag.String("memprofile", "", "write mem profile to file")
	logLevel := flag.String("logLevel", "error", "Log Level")

	flag.Parse()

	logging.SetLogLevel(logging.Level(*logLevel))
	fmt.Println("Log Level =", *logLevel)

	if *help {
		flag.PrintDefaults()
		os.Exit(0)
	}

	if *cpuprofile != "" {
		fd, err := os.Create(*cpuprofile)
		if err != nil {
			fmt.Println("Failed create cpu profile file")
			os.Exit(1)
		}
		pprof.StartCPUProfile(fd)
		defer pprof.StopCPUProfile()
	}
	if *memprofile != "" {
		fd, err := os.Create(*memprofile)
		if err != nil {
			fmt.Println("Failed create mem profile file")
			os.Exit(1)
		}
		defer pprof.WriteHeapProfile(fd)
	}

	runtime.GOMAXPROCS(*cpus)
	up := strings.Split(*auth, ":")
	_, err := cbauth.InternalRetryDefaultInit(*cluster, up[0], up[1])
	if err != nil {
		fmt.Println("Failed to initialize cbauth: %s\n", err)
		os.Exit(1)
	}

	cfg, err := parseConfig(*config)
	handleError(err)

	var statsW io.Writer
	if *statsfile != "" {
		if f, err := os.Create(*statsfile); err != nil {
			handleError(err)
		} else {
			statsW = f
			defer f.Close()
		}
	}

	t0 := time.Now()
	res, err := RunCommands(*cluster, cfg, statsW)
	handleError(err)
	dur := time.Now().Sub(t0)

	totalRows := uint64(0)
	for _, result := range res.ScanResults {
		totalRows += result.Rows
	}
	res.Rows = totalRows
	res.Duration = dur.Seconds() - res.WarmupDuration

	rate := int(float64(totalRows) / res.Duration)

	fmt.Printf("Throughput = %d rows/sec\n", rate)

	os.Remove(*outfile)
	err = writeResults(res, *outfile)
	handleError(err)

}
コード例 #12
0
ファイル: docmd.go プロジェクト: jchris/indexing
// ParseArgs into Command object, return the list of arguments,
// flagset used for parseing and error if any.
func ParseArgs(arguments []string) (*Command, []string, *flag.FlagSet, error) {
	var fields, bindexes string
	var inclusion uint
	var equal, low, high string
	var useSessionCons bool

	cmdOptions := &Command{Consistency: c.AnyConsistency}
	fset := flag.NewFlagSet("cmd", flag.ExitOnError)

	// basic options
	fset.StringVar(&cmdOptions.Server, "server", "", "Cluster server address")
	fset.StringVar(&cmdOptions.Auth, "auth", "", "Auth user and password")
	fset.StringVar(&cmdOptions.Bucket, "bucket", "", "Bucket name")
	fset.StringVar(&cmdOptions.OpType, "type", "", "Command: scan|stats|scanAll|count|nodes|create|build|drop|list|config")
	fset.StringVar(&cmdOptions.IndexName, "index", "", "Index name")
	// options for create-index
	fset.StringVar(&cmdOptions.WhereStr, "where", "", "where clause for create index")
	fset.StringVar(&fields, "fields", "", "Comma separated on-index fields") // secStrs
	fset.BoolVar(&cmdOptions.IsPrimary, "primary", false, "Is primary index")
	fset.StringVar(&cmdOptions.With, "with", "", "index specific properties")
	// options for build-indexes, drop-indexes
	fset.StringVar(&bindexes, "indexes", "", "csv list of bucket.index to build")
	// options for Range, Statistics, Count
	fset.StringVar(&low, "low", "[]", "Span.Range: [low]")
	fset.StringVar(&high, "high", "[]", "Span.Range: [high]")
	fset.StringVar(&equal, "equal", "", "Span.Lookup: [key]")
	fset.UintVar(&inclusion, "incl", 0, "Range: 0|1|2|3")
	fset.Int64Var(&cmdOptions.Limit, "limit", 10, "Row limit")
	fset.BoolVar(&cmdOptions.Help, "h", false, "print help")
	fset.BoolVar(&useSessionCons, "consistency", false, "Use session consistency")
	// options for setting configuration
	fset.StringVar(&cmdOptions.ConfigKey, "ckey", "", "Config key")
	fset.StringVar(&cmdOptions.ConfigVal, "cval", "", "Config value")
	fset.StringVar(&cmdOptions.Using, "using", c.ForestDB, "storate type to use")

	// not useful to expose in sherlock
	cmdOptions.ExprType = "N1QL"
	cmdOptions.PartnStr = "partn"

	if err := fset.Parse(arguments); err != nil {
		return nil, nil, fset, err
	}

	if useSessionCons {
		cmdOptions.Consistency = c.SessionConsistency
	}

	// if server is not specified, try guessing
	if cmdOptions.Server == "" {
		if guess := guessServer(); guess != "" {
			cmdOptions.Server = guess
			fset.Set("server", guess)
		}
	}

	// if server is not specified, try guessing
	if cmdOptions.Auth == "" {
		if guess := guessAuth(cmdOptions.Server); guess != "" {
			cmdOptions.Auth = guess
			fset.Set("auth", guess)
		}
	}

	// validate combinations
	err := validate(cmdOptions, fset)
	if err != nil {
		return nil, nil, fset, err
	}
	// bindexes
	if len(bindexes) > 0 {
		cmdOptions.Bindexes = strings.Split(bindexes, ",")
	}

	// inclusion, secStrs, equal, low, high
	cmdOptions.Inclusion = qclient.Inclusion(inclusion)
	cmdOptions.SecStrs = make([]string, 0)
	if fields != "" {
		for _, field := range strings.Split(fields, ",") {
			expr, err := n1ql.ParseExpression(field)
			if err != nil {
				msgf := "Error occured: Invalid field (%v) %v\n"
				return nil, nil, fset, fmt.Errorf(msgf, field, err)
			}
			secStr := expression.NewStringer().Visit(expr)
			cmdOptions.SecStrs = append(cmdOptions.SecStrs, secStr)
		}
	}
	if equal != "" {
		cmdOptions.Equal = c.SecondaryKey(Arg2Key([]byte(equal)))
	}
	cmdOptions.Low = c.SecondaryKey(Arg2Key([]byte(low)))
	cmdOptions.High = c.SecondaryKey(Arg2Key([]byte(high)))

	// with
	if len(cmdOptions.With) > 0 {
		err := json.Unmarshal([]byte(cmdOptions.With), &cmdOptions.WithPlan)
		if err != nil {
			logging.Fatalf("%v\n", err)
			os.Exit(1)
		}
	}

	// setup cbauth
	if cmdOptions.Auth != "" {
		up := strings.Split(cmdOptions.Auth, ":")
		_, err := cbauth.InternalRetryDefaultInit(cmdOptions.Server, up[0], up[1])
		if err != nil {
			logging.Fatalf("Failed to initialize cbauth: %s\n", err)
			os.Exit(1)
		}
	}

	return cmdOptions, fset.Args(), fset, err
}
コード例 #13
0
ファイル: couchbase.go プロジェクト: jmptrader/query
// NewSite creates a new Couchbase site for the given url.
func NewDatastore(u string) (s datastore.Datastore, e errors.Error) {

	var client cb.Client
	var cbAuthInit bool

	// try and initialize cbauth

	c, err := initCbAuth(u)
	if err != nil {
		logging.Errorf(" Unable to initialize cbauth. Error %v", err)
		url, err := url.Parse(u)
		if err != nil {
			return nil, errors.NewCbUrlParseError(err, "url "+u)
		}

		if url.User != nil {
			password, _ := url.User.Password()
			if password == "" {
				logging.Errorf("No password found in url %s", u)
			}

			// intialize cb_auth variables manually
			logging.Infof(" Trying to init cbauth with credentials %s %s", url.Host, url.User.Username())
			set, err := cbauth.InternalRetryDefaultInit(url.Host, url.User.Username(), password)
			if set == false || err != nil {
				logging.Errorf(" Unable to initialize cbauth variables. Error %v", err)
			} else {
				c, err = initCbAuth("http://" + url.Host)
				if err != nil {
					logging.Errorf("Unable to initliaze cbauth.  Error %v", err)
				} else {
					client = *c
					cbAuthInit = true
				}
			}
		}
	} else {
		client = *c
		cbAuthInit = true
	}

	if cbAuthInit == false {
		// connect without auth
		logging.Warnf("Unable to intialize cbAuth, access to couchbase buckets may be restricted")
		cb.HTTPClient = &http.Client{}
		client, err = cb.Connect(u)
		if err != nil {
			return nil, errors.NewCbConnectionError(err, "url "+u)
		}
	}

	site := &site{
		client:         client,
		namespaceCache: make(map[string]*namespace),
		CbAuthInit:     cbAuthInit,
	}

	// initialize the default pool.
	// TODO can couchbase server contain more than one pool ?

	defaultPool, Err := loadNamespace(site, "default")
	if Err != nil {
		logging.Errorf("Cannot connect to default pool")
		return nil, Err
	}

	site.namespaceCache["default"] = defaultPool
	logging.Infof("New site created with url %s", u)

	return site, nil
}