func main() {
clusters := argParse()
// setup cbauth
up := strings.Split(options.auth, ":")
_, err := cbauth.InternalRetryDefaultInit(clusters[0], up[0], up[1])
if err != nil {
log.Fatalf("Failed to initialize cbauth: %s", err)
}
maxvbs := c.SystemConfig["maxVbuckets"].Int()
dconf := c.SystemConfig.SectionConfig("indexer.dataport.", true)
// start dataport servers.
for _, endpoint := range options.endpoints {
go dataport.Application(
endpoint, options.stat, options.timeout, maxvbs, dconf,
func(addr string, msg interface{}) bool { return true })
}
go dataport.Application(options.coordEndpoint, 0, 0, maxvbs, dconf, nil)
for _, cluster := range clusters {
adminport := getProjectorAdminport(cluster, "default")
if options.projector {
config := c.SystemConfig.Clone()
config.SetValue("projector.clusterAddr", cluster)
config.SetValue("projector.adminport.listenAddr", adminport)
econf := c.SystemConfig.SectionConfig("projector.dataport.", true)
epfactory := NewEndpointFactory(cluster, maxvbs, econf)
config.SetValue("projector.routerEndpointFactory", epfactory)
projector.NewProjector(maxvbs, config) // start projector daemon
}
// projector-client
cconfig := c.SystemConfig.SectionConfig("indexer.projectorclient.", true)
projectors[cluster] = projc.NewClient(adminport, maxvbs, cconfig)
}
// index instances for specified buckets.
instances := protobuf.ExampleIndexInstances(
options.buckets, options.endpoints, options.coordEndpoint)
// start backfill stream on each projector
for _, client := range projectors {
// start backfill stream on each projector
_, err := client.InitialTopicRequest(
"backfill" /*topic*/, "default", /*pooln*/
"dataport" /*endpointType*/, instances)
if err != nil {
log.Fatal(err)
}
}
time.Sleep(1000 * time.Second)
//<-make(chan bool) // wait for ever
}
func main() {
cluster := argParse()[0]
// setup cbauth
if options.auth != "" {
up := strings.Split(options.auth, ":")
if _, err := cbauth.InternalRetryDefaultInit(cluster, up[0], up[1]); err != nil {
logging.Fatalf("Failed to initialize cbauth: %s", err)
}
}
go startFeed(cluster, "streamwait-feed1")
go startFeed(cluster, "streamwait-feed2")
time.Sleep(1000 * time.Second)
}
func maybeReinitCBAuth() {
if authFlag == "" {
return
}
up := strings.Split(authFlag, ":")
authU, authP := up[0], up[1]
u, err := url.Parse(mgmtURLFlag)
if err != nil {
log.Fatal("Failed to parse mgmtURLFlag: ", err)
}
_, err = cbauth.InternalRetryDefaultInit(u.Host, authU, authP)
if err != nil {
log.Fatal("Failed to initialize cbauth: ", err)
}
}
func main() {
platform.HideConsole(true)
defer platform.HideConsole(false)
common.SeedProcess()
logging.Infof("Indexer started with command line: %v\n", os.Args)
flag.Parse()
logging.SetLogLevel(logging.Level(*logLevel))
forestdb.Log = &logging.SystemLogger
// setup cbauth
if *auth != "" {
up := strings.Split(*auth, ":")
logging.Tracef("Initializing cbauth with user %v for cluster %v\n", up[0], *cluster)
if _, err := cbauth.InternalRetryDefaultInit(*cluster, up[0], up[1]); err != nil {
logging.Fatalf("Failed to initialize cbauth: %s", err)
}
}
go platform.DumpOnSignal()
go common.ExitOnStdinClose()
config := common.SystemConfig
config.SetValue("indexer.clusterAddr", *cluster)
config.SetValue("indexer.numVbuckets", *numVbuckets)
config.SetValue("indexer.enableManager", *enableManager)
config.SetValue("indexer.adminPort", *adminPort)
config.SetValue("indexer.scanPort", *scanPort)
config.SetValue("indexer.httpPort", *httpPort)
config.SetValue("indexer.streamInitPort", *streamInitPort)
config.SetValue("indexer.streamCatchupPort", *streamCatchupPort)
config.SetValue("indexer.streamMaintPort", *streamMaintPort)
config.SetValue("indexer.storage_dir", *storageDir)
storage_dir := config["indexer.storage_dir"].String()
if err := os.MkdirAll(storage_dir, 0755); err != nil {
common.CrashOnError(err)
}
_, msg := indexer.NewIndexer(config)
if msg.GetMsgType() != indexer.MSG_SUCCESS {
logging.Warnf("Indexer Failure to Init %v", msg)
}
logging.Infof("Indexer exiting normally\n")
}
func main() {
cluster := argParse()
// setup cbauth
if options.auth != "" {
up := strings.Split(options.auth, ":")
if _, err := cbauth.InternalRetryDefaultInit(cluster, up[0], up[1]); err != nil {
logging.Fatalf("Failed to initialize cbauth: %s", err)
}
}
for _, bucket := range options.buckets {
go startBucket(cluster, bucket)
}
receive()
}
func init() {
log.Printf("In init()")
logging.SetLogLevel(logging.Warn)
var configpath string
flag.StringVar(&configpath, "cbconfig", "../config/clusterrun_conf.json", "Path of the configuration file with data about Couchbase Cluster")
flag.Parse()
clusterconfig = tc.GetClusterConfFromFile(configpath)
kvaddress = clusterconfig.KVAddress
indexManagementAddress = clusterconfig.KVAddress
indexScanAddress = clusterconfig.KVAddress
seed = 1
proddir, bagdir = tc.FetchMonsterToolPath()
// setup cbauth
if _, err := cbauth.InternalRetryDefaultInit(kvaddress, clusterconfig.Username, clusterconfig.Password); err != nil {
log.Fatalf("Failed to initialize cbauth: %s", err)
}
secondaryindex.CheckCollation = true
e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
tc.HandleError(e, "Error in DropAllSecondaryIndexes")
time.Sleep(5 * time.Second)
// Working with Users10k and Users_mut dataset.
u, _ := user.Current()
dataFilePath = filepath.Join(u.HomeDir, "testdata/Users10k.txt.gz")
mutationFilePath = filepath.Join(u.HomeDir, "testdata/Users_mut.txt.gz")
tc.DownloadDataFile(tc.IndexTypesStaticJSONDataS3, dataFilePath, true)
tc.DownloadDataFile(tc.IndexTypesMutationJSONDataS3, mutationFilePath, true)
docs = datautility.LoadJSONFromCompressedFile(dataFilePath, "docid")
mut_docs = datautility.LoadJSONFromCompressedFile(mutationFilePath, "docid")
log.Printf("Emptying the default bucket")
kvutility.EnableBucketFlush("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
kvutility.FlushBucket("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
time.Sleep(5 * time.Second)
log.Printf("Create Index On the empty default Bucket()")
var indexName = "index_eyeColor"
var bucketName = "default"
err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{"eyeColor"}, false, nil, true, defaultIndexActiveTimeout, nil)
tc.HandleError(err, "Error in creating the index")
// Populate the bucket now
log.Printf("Populating the default bucket")
kvutility.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)
}
func main() {
argParse()
// setup cbauth
if options.auth != "" {
up := strings.Split(options.auth, ":")
if _, err := cbauth.InternalRetryDefaultInit(options.cluster, up[0], up[1]); err != nil {
log.Fatalf("Failed to initialize cbauth: %s", err)
}
}
numVb := numVbuckets(options.cluster, options.bucket)
log.Printf("bucket %q has %v vbuckets\n", options.bucket, numVb)
bucketTs(options.cluster, options.bucket, numVb)
//getCurrentKVTs_old(options.cluster, options.bucket, numVb)
getCurrentKVTs(options.cluster, options.bucket, numVb)
}
func init() {
log.Printf("In init()")
logging.SetLogLevel(logging.Warn)
var configpath string
seed = 1
flag.StringVar(&configpath, "cbconfig", "../config/clusterrun_conf.json", "Path of the configuration file with data about Couchbase Cluster")
flag.Parse()
clusterconfig = tc.GetClusterConfFromFile(configpath)
kvaddress = clusterconfig.KVAddress
indexManagementAddress = clusterconfig.KVAddress
indexScanAddress = clusterconfig.KVAddress
// setup cbauth
if _, err := cbauth.InternalRetryDefaultInit(kvaddress, clusterconfig.Username, clusterconfig.Password); err != nil {
log.Fatalf("Failed to initialize cbauth: %s", err)
}
proddir, bagdir = tc.FetchMonsterToolPath()
}
func main() {
platform.HideConsole(true)
defer platform.HideConsole(false)
c.SeedProcess()
logging.Infof("Projector started with command line: %v\n", os.Args)
cluster := argParse() // eg. "localhost:9000"
config := c.SystemConfig.Clone()
logging.SetLogLevel(logging.Level(options.loglevel))
config.SetValue("maxVbuckets", options.numVbuckets)
if f := getlogFile(); f != nil {
fmt.Printf("Projector logging to %q\n", f.Name())
logging.SetLogWriter(f)
config.SetValue("log.file", f.Name())
}
config.SetValue("projector.clusterAddr", cluster)
config.SetValue("projector.adminport.listenAddr", options.adminport)
// setup cbauth
if options.auth != "" {
up := strings.Split(options.auth, ":")
if _, err := cbauth.InternalRetryDefaultInit(cluster, up[0], up[1]); err != nil {
logging.Fatalf("Failed to initialize cbauth: %s", err)
}
}
epfactory := NewEndpointFactory(cluster, options.numVbuckets)
config.SetValue("projector.routerEndpointFactory", epfactory)
logging.Infof("%v\n", c.LogOs())
logging.Infof("%v\n", c.LogRuntime())
go c.ExitOnStdinClose()
projector.NewProjector(options.numVbuckets, config)
<-done
}
func main() {
cluster := argParse()
if !strings.HasPrefix(cluster, "http://") {
cluster = "http://" + cluster
}
// setup cbauth
if options.auth != "" {
up := strings.Split(options.auth, ":")
if _, err := cbauth.InternalRetryDefaultInit(cluster, up[0], up[1]); err != nil {
logging.Fatalf("Failed to initialize cbauth: %s", err)
}
}
n := 0
for i, bucket := range options.buckets {
prodfile := options.prods[i]
n += loadBucket(cluster, bucket, prodfile, options.count)
}
for n > 0 {
<-done
n--
}
}
func main() {
help := flag.Bool("help", false, "Help")
config := flag.String("configfile", "config.json", "Scan load config file")
outfile := flag.String("resultfile", "results.json", "Result report file")
cpus := flag.Int("cpus", runtime.NumCPU(), "Number of CPUs")
cluster := flag.String("cluster", "127.0.0.1:9000", "Cluster server address")
auth := flag.String("auth", "Administrator:asdasd", "Auth")
statsfile := flag.String("statsfile", "", "Periodic statistics report file")
cpuprofile := flag.String("cpuprofile", "", "write cpu profile to file")
memprofile := flag.String("memprofile", "", "write mem profile to file")
logLevel := flag.String("logLevel", "error", "Log Level")
flag.Parse()
logging.SetLogLevel(logging.Level(*logLevel))
fmt.Println("Log Level =", *logLevel)
if *help {
flag.PrintDefaults()
os.Exit(0)
}
if *cpuprofile != "" {
fd, err := os.Create(*cpuprofile)
if err != nil {
fmt.Println("Failed create cpu profile file")
os.Exit(1)
}
pprof.StartCPUProfile(fd)
defer pprof.StopCPUProfile()
}
if *memprofile != "" {
fd, err := os.Create(*memprofile)
if err != nil {
fmt.Println("Failed create mem profile file")
os.Exit(1)
}
defer pprof.WriteHeapProfile(fd)
}
runtime.GOMAXPROCS(*cpus)
up := strings.Split(*auth, ":")
_, err := cbauth.InternalRetryDefaultInit(*cluster, up[0], up[1])
if err != nil {
fmt.Println("Failed to initialize cbauth: %s\n", err)
os.Exit(1)
}
cfg, err := parseConfig(*config)
handleError(err)
var statsW io.Writer
if *statsfile != "" {
if f, err := os.Create(*statsfile); err != nil {
handleError(err)
} else {
statsW = f
defer f.Close()
}
}
t0 := time.Now()
res, err := RunCommands(*cluster, cfg, statsW)
handleError(err)
dur := time.Now().Sub(t0)
totalRows := uint64(0)
for _, result := range res.ScanResults {
totalRows += result.Rows
}
res.Rows = totalRows
res.Duration = dur.Seconds() - res.WarmupDuration
rate := int(float64(totalRows) / res.Duration)
fmt.Printf("Throughput = %d rows/sec\n", rate)
os.Remove(*outfile)
err = writeResults(res, *outfile)
handleError(err)
}
// ParseArgs into Command object, return the list of arguments,
// flagset used for parseing and error if any.
func ParseArgs(arguments []string) (*Command, []string, *flag.FlagSet, error) {
var fields, bindexes string
var inclusion uint
var equal, low, high string
var useSessionCons bool
cmdOptions := &Command{Consistency: c.AnyConsistency}
fset := flag.NewFlagSet("cmd", flag.ExitOnError)
// basic options
fset.StringVar(&cmdOptions.Server, "server", "", "Cluster server address")
fset.StringVar(&cmdOptions.Auth, "auth", "", "Auth user and password")
fset.StringVar(&cmdOptions.Bucket, "bucket", "", "Bucket name")
fset.StringVar(&cmdOptions.OpType, "type", "", "Command: scan|stats|scanAll|count|nodes|create|build|drop|list|config")
fset.StringVar(&cmdOptions.IndexName, "index", "", "Index name")
// options for create-index
fset.StringVar(&cmdOptions.WhereStr, "where", "", "where clause for create index")
fset.StringVar(&fields, "fields", "", "Comma separated on-index fields") // secStrs
fset.BoolVar(&cmdOptions.IsPrimary, "primary", false, "Is primary index")
fset.StringVar(&cmdOptions.With, "with", "", "index specific properties")
// options for build-indexes, drop-indexes
fset.StringVar(&bindexes, "indexes", "", "csv list of bucket.index to build")
// options for Range, Statistics, Count
fset.StringVar(&low, "low", "[]", "Span.Range: [low]")
fset.StringVar(&high, "high", "[]", "Span.Range: [high]")
fset.StringVar(&equal, "equal", "", "Span.Lookup: [key]")
fset.UintVar(&inclusion, "incl", 0, "Range: 0|1|2|3")
fset.Int64Var(&cmdOptions.Limit, "limit", 10, "Row limit")
fset.BoolVar(&cmdOptions.Help, "h", false, "print help")
fset.BoolVar(&useSessionCons, "consistency", false, "Use session consistency")
// options for setting configuration
fset.StringVar(&cmdOptions.ConfigKey, "ckey", "", "Config key")
fset.StringVar(&cmdOptions.ConfigVal, "cval", "", "Config value")
fset.StringVar(&cmdOptions.Using, "using", c.ForestDB, "storate type to use")
// not useful to expose in sherlock
cmdOptions.ExprType = "N1QL"
cmdOptions.PartnStr = "partn"
if err := fset.Parse(arguments); err != nil {
return nil, nil, fset, err
}
if useSessionCons {
cmdOptions.Consistency = c.SessionConsistency
}
// if server is not specified, try guessing
if cmdOptions.Server == "" {
if guess := guessServer(); guess != "" {
cmdOptions.Server = guess
fset.Set("server", guess)
}
}
// if server is not specified, try guessing
if cmdOptions.Auth == "" {
if guess := guessAuth(cmdOptions.Server); guess != "" {
cmdOptions.Auth = guess
fset.Set("auth", guess)
}
}
// validate combinations
err := validate(cmdOptions, fset)
if err != nil {
return nil, nil, fset, err
}
// bindexes
if len(bindexes) > 0 {
cmdOptions.Bindexes = strings.Split(bindexes, ",")
}
// inclusion, secStrs, equal, low, high
cmdOptions.Inclusion = qclient.Inclusion(inclusion)
cmdOptions.SecStrs = make([]string, 0)
if fields != "" {
for _, field := range strings.Split(fields, ",") {
expr, err := n1ql.ParseExpression(field)
if err != nil {
msgf := "Error occured: Invalid field (%v) %v\n"
return nil, nil, fset, fmt.Errorf(msgf, field, err)
}
secStr := expression.NewStringer().Visit(expr)
cmdOptions.SecStrs = append(cmdOptions.SecStrs, secStr)
}
}
if equal != "" {
cmdOptions.Equal = c.SecondaryKey(Arg2Key([]byte(equal)))
}
cmdOptions.Low = c.SecondaryKey(Arg2Key([]byte(low)))
cmdOptions.High = c.SecondaryKey(Arg2Key([]byte(high)))
// with
if len(cmdOptions.With) > 0 {
err := json.Unmarshal([]byte(cmdOptions.With), &cmdOptions.WithPlan)
if err != nil {
logging.Fatalf("%v\n", err)
os.Exit(1)
}
}
// setup cbauth
if cmdOptions.Auth != "" {
up := strings.Split(cmdOptions.Auth, ":")
_, err := cbauth.InternalRetryDefaultInit(cmdOptions.Server, up[0], up[1])
if err != nil {
logging.Fatalf("Failed to initialize cbauth: %s\n", err)
os.Exit(1)
}
}
return cmdOptions, fset.Args(), fset, err
}
// NewSite creates a new Couchbase site for the given url.
func NewDatastore(u string) (s datastore.Datastore, e errors.Error) {
var client cb.Client
var cbAuthInit bool
// try and initialize cbauth
c, err := initCbAuth(u)
if err != nil {
logging.Errorf(" Unable to initialize cbauth. Error %v", err)
url, err := url.Parse(u)
if err != nil {
return nil, errors.NewCbUrlParseError(err, "url "+u)
}
if url.User != nil {
password, _ := url.User.Password()
if password == "" {
logging.Errorf("No password found in url %s", u)
}
// intialize cb_auth variables manually
logging.Infof(" Trying to init cbauth with credentials %s %s", url.Host, url.User.Username())
set, err := cbauth.InternalRetryDefaultInit(url.Host, url.User.Username(), password)
if set == false || err != nil {
logging.Errorf(" Unable to initialize cbauth variables. Error %v", err)
} else {
c, err = initCbAuth("http://" + url.Host)
if err != nil {
logging.Errorf("Unable to initliaze cbauth. Error %v", err)
} else {
client = *c
cbAuthInit = true
}
}
}
} else {
client = *c
cbAuthInit = true
}
if cbAuthInit == false {
// connect without auth
logging.Warnf("Unable to intialize cbAuth, access to couchbase buckets may be restricted")
cb.HTTPClient = &http.Client{}
client, err = cb.Connect(u)
if err != nil {
return nil, errors.NewCbConnectionError(err, "url "+u)
}
}
site := &site{
client: client,
namespaceCache: make(map[string]*namespace),
CbAuthInit: cbAuthInit,
}
// initialize the default pool.
// TODO can couchbase server contain more than one pool ?
defaultPool, Err := loadNamespace(site, "default")
if Err != nil {
logging.Errorf("Cannot connect to default pool")
return nil, Err
}
site.namespaceCache["default"] = defaultPool
logging.Infof("New site created with url %s", u)
return site, nil
}