func main() { logging.SetLogLevel(logging.Warn) runtime.GOMAXPROCS(runtime.NumCPU()) cmdOptions, _, fset, err := querycmd.ParseArgs(os.Args[1:]) if err != nil { logging.Fatalf("%v\n", err) os.Exit(1) } else if cmdOptions.Help || len(cmdOptions.OpType) < 1 { usage(fset) os.Exit(0) } config := c.SystemConfig.SectionConfig("queryport.client.", true) client, err := qclient.NewGsiClient(cmdOptions.Server, config) if err != nil { logging.Fatalf("%v\n", err) os.Exit(1) } if err = querycmd.HandleCommand(client, cmdOptions, false, os.Stdout); err != nil { fmt.Fprintf(os.Stderr, "Error occured %v\n", err) } client.Close() }
//panicHandler handles the panic from underlying stream library func (r *mutationStreamReader) panicHandler() { //panic recovery if rc := recover(); rc != nil { logging.Fatalf("MutationStreamReader::panicHandler Received Panic for Stream %v", r.streamId) var err error switch x := rc.(type) { case string: err = errors.New(x) case error: err = x default: err = errors.New("Unknown panic") } logging.Fatalf("StreamReader Panic Err %v", err) logging.Fatalf("%s", logging.StackTrace()) //panic from stream library, propagate to supervisor msg := &MsgStreamError{streamId: r.streamId, err: Error{code: ERROR_STREAM_READER_PANIC, severity: FATAL, category: STREAM_READER, cause: err}} r.supvRespch <- msg } }
func ValidateBucket(cluster, bucket string, uuids []string) bool { var cinfo *common.ClusterInfoCache url, err := common.ClusterAuthUrl(cluster) if err == nil { cinfo, err = common.NewClusterInfoCache(url, DEFAULT_POOL) } if err != nil { logging.Fatalf("Indexer::Fail to init ClusterInfoCache : %v", err) common.CrashOnError(err) } cinfo.Lock() defer cinfo.Unlock() if err := cinfo.Fetch(); err != nil { logging.Errorf("Indexer::Fail to init ClusterInfoCache : %v", err) common.CrashOnError(err) } if nids, err := cinfo.GetNodesByBucket(bucket); err == nil && len(nids) != 0 { // verify UUID currentUUID := cinfo.GetBucketUUID(bucket) for _, uuid := range uuids { if uuid != currentUUID { return false } } return true } else { logging.Fatalf("Indexer::Error Fetching Bucket Info: %v Nids: %v", err, nids) return false } }
//panicHandler handles the panic from index manager func (c *clustMgrAgent) panicHandler() { //panic recovery if rc := recover(); rc != nil { var err error switch x := rc.(type) { case string: err = errors.New(x) case error: err = x default: err = errors.New("Unknown panic") } logging.Fatalf("ClusterMgrAgent Panic Err %v", err) logging.Fatalf("%s", logging.StackTrace()) //panic, propagate to supervisor msg := &MsgError{ err: Error{code: ERROR_INDEX_MANAGER_PANIC, severity: FATAL, category: CLUSTER_MGR, cause: err}} c.supvRespch <- msg } }
func main() { logging.SetLogLevel(logging.Error) runtime.GOMAXPROCS(runtime.NumCPU()) cmdOptions, args, fset, err := querycmd.ParseArgs(os.Args[1:]) if err != nil { logging.Fatalf("%v", err) os.Exit(0) } else if cmdOptions.Help { usage(fset) os.Exit(0) } else if len(args) < 1 { logging.Fatalf("%v", "specify a command") } b, err := c.ConnectBucket(cmdOptions.Server, "default", "default") if err != nil { log.Fatal(err) } defer b.Close() maxvb, err := c.MaxVbuckets(b) if err != nil { log.Fatal(err) } config := c.SystemConfig.SectionConfig("queryport.client.", true) client, err := qclient.NewGsiClient(cmdOptions.Server, config) if err != nil { log.Fatal(err) } switch args[0] { case "sanity": err = doSanityTests(cmdOptions.Server, client) if err != nil { fmt.Fprintf(os.Stderr, "Error occured %v\n", err) } case "mb14786": err = doMB14786(cmdOptions.Server, client) if err != nil { fmt.Fprintf(os.Stderr, "Error occured %v\n", err) } case "mb13339": err = doMB13339(cmdOptions.Server, client) if err != nil { fmt.Fprintf(os.Stderr, "Error occured %v\n", err) } case "benchmark": doBenchmark(cmdOptions.Server, "localhost:8101") case "consistency": doConsistency(cmdOptions.Server, maxvb, client) } client.Close() }
func processMutations( vbs []*protobuf.VbKeyVersions, bucketWise map[string]map[byte]int, keys map[uint64]map[string]int) int { var secvalues []interface{} mutations := 0 for _, vb := range vbs { bucket, kvs := vb.GetBucketname(), vb.GetKvs() commandWise, ok := bucketWise[bucket] if !ok { commandWise = make(map[byte]int) } for _, kv := range kvs { mutations++ uuids, seckeys := kv.GetUuids(), kv.GetKeys() for i, command := range kv.GetCommands() { cmd, uuid, key := byte(command), uuids[i], string(seckeys[i]) if _, ok := commandWise[cmd]; !ok { commandWise[cmd] = 0 } commandWise[cmd]++ if cmd == 0 || cmd == c.Snapshot || uuid == 0 || key == "" { continue } m, ok := keys[uuid] if !ok { m = make(map[string]int) } if err := json.Unmarshal([]byte(key), &secvalues); err != nil { logging.Fatalf("Error in unmarshalling - %v", err) } else if len(secvalues) > 0 { secJSON, err := json.Marshal(secvalues[:len(secvalues)-1]) if err != nil { logging.Fatalf("Error in marshaling - %v", err) } key = string(secJSON) if _, ok := m[key]; !ok { m[key] = 0 } m[key]++ keys[uuid] = m } } } bucketWise[bucket] = commandWise } return mutations }
func (meta *metaNotifier) OnIndexBuild(indexDefnList []common.IndexDefnId, buckets []string) map[common.IndexInstId]error { logging.Infof("clustMgrAgent::OnIndexBuild Notification "+ "Received for Build Index %v", indexDefnList) respCh := make(MsgChannel) var indexInstList []common.IndexInstId for _, defnId := range indexDefnList { indexInstList = append(indexInstList, common.IndexInstId(defnId)) } meta.adminCh <- &MsgBuildIndex{indexInstList: indexInstList, respCh: respCh, bucketList: buckets} //wait for response if res, ok := <-respCh; ok { switch res.GetMsgType() { case CLUST_MGR_BUILD_INDEX_DDL_RESPONSE: errMap := res.(*MsgBuildIndexResponse).GetErrorMap() logging.Infof("clustMgrAgent::OnIndexBuild returns "+ "for Build Index %v", indexDefnList) return errMap case MSG_ERROR: logging.Errorf("clustMgrAgent::OnIndexBuild Error "+ "for Build Index %v. Error %v.", indexDefnList, res) err := res.(*MsgError).GetError() errMap := make(map[common.IndexInstId]error) for _, instId := range indexDefnList { errMap[common.IndexInstId(instId)] = errors.New(err.String()) } return errMap default: logging.Fatalf("clustMgrAgent::OnIndexBuild Unknown Response "+ "Received for Build Index %v. Response %v", indexDefnList, res) common.CrashOnError(errors.New("Unknown Response")) } } else { logging.Fatalf("clustMgrAgent::OnIndexBuild Unexpected Channel Close "+ "for Create Index %v", indexDefnList) common.CrashOnError(errors.New("Unknown Response")) } return nil }
func (meta *metaNotifier) OnIndexCreate(indexDefn *common.IndexDefn) error { logging.Infof("clustMgrAgent::OnIndexCreate Notification "+ "Received for Create Index %v", indexDefn) pc := meta.makeDefaultPartitionContainer() idxInst := common.IndexInst{InstId: common.IndexInstId(indexDefn.DefnId), Defn: *indexDefn, State: common.INDEX_STATE_CREATED, Pc: pc, } respCh := make(MsgChannel) meta.adminCh <- &MsgCreateIndex{mType: CLUST_MGR_CREATE_INDEX_DDL, indexInst: idxInst, respCh: respCh} //wait for response if res, ok := <-respCh; ok { switch res.GetMsgType() { case MSG_SUCCESS: logging.Infof("clustMgrAgent::OnIndexCreate Success "+ "for Create Index %v", indexDefn) return nil case MSG_ERROR: logging.Errorf("clustMgrAgent::OnIndexCreate Error "+ "for Create Index %v. Error %v.", indexDefn, res) err := res.(*MsgError).GetError() return err.cause default: logging.Fatalf("clustMgrAgent::OnIndexCreate Unknown Response "+ "Received for Create Index %v. Response %v", indexDefn, res) common.CrashOnError(errors.New("Unknown Response")) } } else { logging.Fatalf("clustMgrAgent::OnIndexCreate Unexpected Channel Close "+ "for Create Index %v", indexDefn) common.CrashOnError(errors.New("Unknown Response")) } return nil }
//send the actual MutationStreamRequest on adminport func (k *kvSender) sendMutationTopicRequest(ap *projClient.Client, topic string, reqTimestamps *protobuf.TsVbuuid, instances []*protobuf.Instance) (*protobuf.TopicResponse, error) { logging.Infof("KVSender::sendMutationTopicRequest Projector %v Topic %v %v \n\tInstances %v", ap, topic, reqTimestamps.GetBucket(), instances) logging.LazyVerbosef("KVSender::sendMutationTopicRequest RequestTS %v", reqTimestamps.Repr) endpointType := "dataport" if res, err := ap.MutationTopicRequest(topic, endpointType, []*protobuf.TsVbuuid{reqTimestamps}, instances); err != nil { logging.Fatalf("KVSender::sendMutationTopicRequest Projector %v Topic %v %v \n\tUnexpected Error %v", ap, topic, reqTimestamps.GetBucket(), err) return res, err } else { logging.Infof("KVSender::sendMutationTopicRequest Success Projector %v Topic %v %v InstanceIds %v", ap, topic, reqTimestamps.GetBucket(), res.GetInstanceIds()) if logging.IsEnabled(logging.Verbose) { logging.Verbosef("KVSender::sendMutationTopicRequest ActiveTs %v \n\tRollbackTs %v", debugPrintTs(res.GetActiveTimestamps(), reqTimestamps.GetBucket()), debugPrintTs(res.GetRollbackTimestamps(), reqTimestamps.GetBucket())) } return res, nil } }
//run starts the mutation manager loop which listens to messages //from its workers(stream_reader and flusher) and //supervisor(indexer) func (m *mutationMgr) run() { defer m.panicHandler() go m.handleWorkerMsgs() go m.listenWorkerMsgs() //main Mutation Manager loop loop: for { select { case cmd, ok := <-m.supvCmdch: if ok { if cmd.GetMsgType() == MUT_MGR_SHUTDOWN { //shutdown and exit the mutation manager loop msg := m.shutdown() m.supvCmdch <- msg break loop } else { m.handleSupervisorCommands(cmd) } } else { logging.Fatalf("Supervisor Channel Closed Unexpectedly." + "Mutation Manager Shutting Itself Down.") m.shutdown() break loop } } } }
//send the actual AddInstances request on adminport func sendAddInstancesRequest(ap *projClient.Client, topic string, instances []*protobuf.Instance) (*protobuf.TimestampResponse, error) { logging.Infof("KVSender::sendAddInstancesRequest Projector %v Topic %v \nInstances %v", ap, topic, instances) if res, err := ap.AddInstances(topic, instances); err != nil { logging.Fatalf("KVSender::sendAddInstancesRequest Unexpected Error During "+ "Add Instances Request Projector %v Topic %v IndexInst %v. Err %v", ap, topic, instances, err) return res, err } else { logging.Infof("KVSender::sendAddInstancesRequest Success Projector %v Topic %v", ap, topic) logging.LazyDebug(func() string { return fmt.Sprintf( "KVSender::sendAddInstancesRequest \n\tActiveTs %v ", debugPrintTs(res.GetCurrentTimestamps(), "")) }) return res, nil } }
func getlogFile() *os.File { switch options.logFile { case "": return nil case "tempfile": f, err := ioutil.TempFile("", "projector") if err != nil { logging.Fatalf("%v", err) } return f } f, err := os.Create(options.logFile) if err != nil { logging.Fatalf("%v", err) } return f }
// Arg2Key convert JSON string to golang-native. func Arg2Key(arg []byte) []interface{} { var key []interface{} if err := json.Unmarshal(arg, &key); err != nil { logging.Fatalf("%v\n", err) os.Exit(1) } return key }
func (meta *metaNotifier) OnIndexDelete(defnId common.IndexDefnId, bucket string) error { logging.Infof("clustMgrAgent::OnIndexDelete Notification "+ "Received for Drop IndexId %v", defnId) respCh := make(MsgChannel) //Treat DefnId as InstId for now meta.adminCh <- &MsgDropIndex{mType: CLUST_MGR_DROP_INDEX_DDL, indexInstId: common.IndexInstId(defnId), respCh: respCh, bucket: bucket} //wait for response if res, ok := <-respCh; ok { switch res.GetMsgType() { case MSG_SUCCESS: logging.Infof("clustMgrAgent::OnIndexDelete Success "+ "for Drop IndexId %v", defnId) return nil case MSG_ERROR: logging.Errorf("clustMgrAgent::OnIndexDelete Error "+ "for Drop IndexId %v. Error %v", defnId, res) err := res.(*MsgError).GetError() return err.cause default: logging.Fatalf("clustMgrAgent::OnIndexDelete Unknown Response "+ "Received for Drop IndexId %v. Response %v", defnId, res) common.CrashOnError(errors.New("Unknown Response")) } } else { logging.Fatalf("clustMgrAgent::OnIndexDelete Unexpected Channel Close "+ "for Drop IndexId %v", defnId) common.CrashOnError(errors.New("Unknown Response")) } return nil }
func main() { flag.Parse() ls, e := net.Listen("tcp", fmt.Sprintf(":%d", *port)) if e != nil { logging.Fatalf("Got an error: %s", e) } waitForConnections(ls) }
//sendMsgToStreamReader sends the provided message to the stream reader //and sends the response back. In case the stream reader panics during the //communication, error is captured and returned back. func (m *mutationMgr) sendMsgToStreamReader(streamId common.StreamId, msg Message) Message { //use select to send message to stream reader, //in case stream reader has exited, a message on streamReaderExitCh //will unblock this call select { case m.streamReaderCmdChMap[streamId] <- msg: case <-m.streamReaderExitChMap[streamId]: logging.Fatalf("MutationMgr::sendMsgToStreamReader Unexpected Stream Reader Exit") return &MsgError{ err: Error{code: ERROR_STREAM_READER_PANIC, severity: FATAL, category: MUTATION_MANAGER}} } //use select to wait for stream reader response. //incase stream reader has exited, a message on streamReaderExitCh //will unblock this call select { case respMsg, ok := <-m.streamReaderCmdChMap[streamId]: if ok { return respMsg } else { logging.Fatalf("MutationMgr:sendMsgToStreamReader Internal Error. Unexpected" + "Close for Stream Reader Command Channel") return &MsgError{ err: Error{code: ERROR_MUT_MGR_INTERNAL_ERROR, severity: FATAL, category: MUTATION_MANAGER}} } case <-m.streamReaderExitChMap[streamId]: logging.Fatalf("MutationMgr::sendMsgToStreamReader Unexpected Stream Reader Exit") return &MsgError{ err: Error{code: ERROR_STREAM_READER_PANIC, severity: FATAL, category: MUTATION_MANAGER}} } }
// NewEndpointFactory to create endpoint instances based on config. func NewEndpointFactory(cluster string, nvbs int) c.RouterEndpointFactory { return func(topic, endpointType, addr string, config c.Config) (c.RouterEndpoint, error) { switch endpointType { case "dataport": return dataport.NewRouterEndpoint(cluster, topic, addr, nvbs, config) default: logging.Fatalf("Unknown endpoint type\n") } return nil, nil } }
func (k *kvSender) sendRestartVbuckets(ap *projClient.Client, topic string, connErrVbs []Vbucket, restartTs *protobuf.TsVbuuid) (*protobuf.TopicResponse, error) { logging.Infof("KVSender::sendRestartVbuckets Projector %v Topic %v %v", ap, topic, restartTs.GetBucket()) logging.LazyVerbosef("KVSender::sendRestartVbuckets RestartTs %v", restartTs.Repr) //Shutdown the vbucket before restart if there was a ConnErr. If the vbucket is already //running, projector will ignore the request otherwise if len(connErrVbs) != 0 { logging.Infof("KVSender::sendRestartVbuckets ShutdownVbuckets %v Topic %v %v ConnErrVbs %v", ap, topic, restartTs.GetBucket(), connErrVbs) // Only shutting down the Vb that receieve connection error. It is probably not harmful // to shutdown every VB in the repairTS, including those that only receive StreamEnd. // But due to network / projecctor latency, a VB StreamBegin may be coming on the way // for those VB (especially when RepairStream has already retried a couple of times). // So shutting all VB in restartTs may unnecessarily causing race condition and // make the protocol longer to converge. ShutdownVbuckets should have no effect on // projector that does not own the Vb. shutdownTs := k.computeShutdownTs(restartTs, connErrVbs) logging.Infof("KVSender::sendRestartVbuckets ShutdownVbuckets Projector %v Topic %v %v \n\tShutdownTs %v", ap, topic, restartTs.GetBucket(), shutdownTs.Repr()) if err := ap.ShutdownVbuckets(topic, []*protobuf.TsVbuuid{shutdownTs}); err != nil { logging.Errorf("KVSender::sendRestartVbuckets Unexpected Error During "+ "ShutdownVbuckets Request for Projector %v Topic %v. Err %v.", ap, topic, err) //all shutdownVbuckets errors are treated as success as it is a best-effort call. //RestartVbuckets errors will be acted upon. } } if res, err := ap.RestartVbuckets(topic, []*protobuf.TsVbuuid{restartTs}); err != nil { logging.Fatalf("KVSender::sendRestartVbuckets Unexpected Error During "+ "Restart Vbuckets Request for Projector %v Topic %v %v . Err %v.", ap, topic, restartTs.GetBucket(), err) return res, err } else { logging.Infof("KVSender::sendRestartVbuckets Success Projector %v Topic %v %v", ap, topic, restartTs.GetBucket()) if logging.IsEnabled(logging.Verbose) { logging.Verbosef("KVSender::sendRestartVbuckets \nActiveTs %v \nRollbackTs %v", debugPrintTs(res.GetActiveTimestamps(), restartTs.GetBucket()), debugPrintTs(res.GetRollbackTimestamps(), restartTs.GetBucket())) } return res, nil } }
//handleSupervisorCommands handles the messages from Supervisor //Each operation acquires the mutex to make the itself atomic. func (m *mutationMgr) handleSupervisorCommands(cmd Message) { switch cmd.GetMsgType() { case OPEN_STREAM: m.handleOpenStream(cmd) case ADD_INDEX_LIST_TO_STREAM: m.handleAddIndexListToStream(cmd) case REMOVE_INDEX_LIST_FROM_STREAM: m.handleRemoveIndexListFromStream(cmd) case REMOVE_BUCKET_FROM_STREAM: m.handleRemoveBucketFromStream(cmd) case CLOSE_STREAM: m.handleCloseStream(cmd) case CLEANUP_STREAM: m.handleCleanupStream(cmd) case MUT_MGR_PERSIST_MUTATION_QUEUE: m.handlePersistMutationQueue(cmd) case MUT_MGR_DRAIN_MUTATION_QUEUE: m.handleDrainMutationQueue(cmd) case MUT_MGR_GET_MUTATION_QUEUE_HWT: m.handleGetMutationQueueHWT(cmd) case MUT_MGR_GET_MUTATION_QUEUE_LWT: m.handleGetMutationQueueLWT(cmd) case UPDATE_INDEX_INSTANCE_MAP: m.handleUpdateIndexInstMap(cmd) case UPDATE_INDEX_PARTITION_MAP: m.handleUpdateIndexPartnMap(cmd) case MUT_MGR_ABORT_PERSIST: m.handleAbortPersist(cmd) case CONFIG_SETTINGS_UPDATE: m.handleConfigUpdate(cmd) default: logging.Fatalf("MutationMgr::handleSupervisorCommands Received Unknown Command %v", cmd) common.CrashOnError(errors.New("Unknown Command On Supervisor Channel")) } }
func main() { cluster := argParse()[0] // setup cbauth if options.auth != "" { up := strings.Split(options.auth, ":") if _, err := cbauth.InternalRetryDefaultInit(cluster, up[0], up[1]); err != nil { logging.Fatalf("Failed to initialize cbauth: %s", err) } } go startFeed(cluster, "streamwait-feed1") go startFeed(cluster, "streamwait-feed2") time.Sleep(1000 * time.Second) }
//panicHandler handles the panic from underlying stream library func (r *mutationMgr) panicHandler() { //panic recovery if rc := recover(); rc != nil { var err error switch x := rc.(type) { case string: err = errors.New(x) case error: err = x default: err = errors.New("Unknown panic") } logging.Fatalf("MutationManager Panic Err %v", err) logging.Fatalf("%s", logging.StackTrace()) //shutdown the mutation manager select { case <-r.shutdownCh: //if the shutdown channel is closed, this means shutdown was in progress //when panic happened, skip calling shutdown again default: r.shutdown() } //panic, propagate to supervisor msg := &MsgError{ err: Error{code: ERROR_MUT_MGR_PANIC, severity: FATAL, category: MUTATION_MANAGER, cause: err}} r.supvRespch <- msg } }
//send the actual ShutdownStreamRequest on adminport func sendShutdownTopic(ap *projClient.Client, topic string) error { logging.Infof("KVSender::sendShutdownTopic Projector %v Topic %v", ap, topic) if err := ap.ShutdownTopic(topic); err != nil { logging.Fatalf("KVSender::sendShutdownTopic Unexpected Error During "+ "Shutdown Projector %v Topic %v. Err %v", ap, topic, err) return err } else { logging.Infof("KVSender::sendShutdownTopic Success Projector %v Topic %v", ap, topic) return nil } }
func (k *kvSender) makeRestartTsFromKV(bucket string, vbnos []uint32) (*protobuf.TsVbuuid, error) { flogs, err := k.getFailoverLogs(bucket, vbnos) if err != nil { logging.Fatalf("KVSender::makeRestartTS Unexpected Error During Failover "+ "Log Request for Bucket %v. Err %v", bucket, err) return nil, err } ts := protobuf.NewTsVbuuid(DEFAULT_POOL, bucket, len(vbnos)) ts = ts.ComputeRestartTs(flogs.ToFailoverLog(c.Vbno32to16(vbnos))) return ts, nil }
func (feed *DcpFeed) connectToNodes( kvaddrs []string, opaque uint16, config map[string]interface{}) error { prefix := feed.logPrefix kvcache := make(map[string]bool) m, err := feed.bucket.GetVBmap(kvaddrs) if err != nil { fmsg := "%v ##%x GetVBmap(%v) failed: %v\n" logging.Fatalf(fmsg, prefix, opaque, kvaddrs, err) return memcached.ErrorInvalidFeed } for kvaddr := range m { kvcache[kvaddr] = true } for _, serverConn := range feed.bucket.getConnPools() { if _, ok := kvcache[serverConn.host]; !ok { continue } nodeFeed := feed.nodeFeeds[serverConn.host] if nodeFeed != nil { nodeFeed.dcpFeed.Close() // and continue to spawn a new one ... } var name DcpFeedName if feed.name == "" { name = NewDcpFeedName("DefaultDcpClient") } else { name = feed.name } singleFeed, err := serverConn.StartDcpFeed( name, feed.sequence, feed.output, opaque, config) if err != nil { for _, nodeFeed := range feed.nodeFeeds { nodeFeed.dcpFeed.Close() } return memcached.ErrorInvalidFeed } // add the node to the connection map feedInfo := &FeedInfo{ dcpFeed: singleFeed, host: serverConn.host, } feed.nodeFeeds[serverConn.host] = feedInfo } return nil }
func main() { platform.HideConsole(true) defer platform.HideConsole(false) common.SeedProcess() logging.Infof("Indexer started with command line: %v\n", os.Args) flag.Parse() logging.SetLogLevel(logging.Level(*logLevel)) forestdb.Log = &logging.SystemLogger // setup cbauth if *auth != "" { up := strings.Split(*auth, ":") logging.Tracef("Initializing cbauth with user %v for cluster %v\n", up[0], *cluster) if _, err := cbauth.InternalRetryDefaultInit(*cluster, up[0], up[1]); err != nil { logging.Fatalf("Failed to initialize cbauth: %s", err) } } go platform.DumpOnSignal() go common.ExitOnStdinClose() config := common.SystemConfig config.SetValue("indexer.clusterAddr", *cluster) config.SetValue("indexer.numVbuckets", *numVbuckets) config.SetValue("indexer.enableManager", *enableManager) config.SetValue("indexer.adminPort", *adminPort) config.SetValue("indexer.scanPort", *scanPort) config.SetValue("indexer.httpPort", *httpPort) config.SetValue("indexer.streamInitPort", *streamInitPort) config.SetValue("indexer.streamCatchupPort", *streamCatchupPort) config.SetValue("indexer.streamMaintPort", *streamMaintPort) config.SetValue("indexer.storage_dir", *storageDir) storage_dir := config["indexer.storage_dir"].String() if err := os.MkdirAll(storage_dir, 0755); err != nil { common.CrashOnError(err) } _, msg := indexer.NewIndexer(config) if msg.GetMsgType() != indexer.MSG_SUCCESS { logging.Warnf("Indexer Failure to Init %v", msg) } logging.Infof("Indexer exiting normally\n") }
func main() { cluster := argParse() // setup cbauth if options.auth != "" { up := strings.Split(options.auth, ":") if _, err := cbauth.InternalRetryDefaultInit(cluster, up[0], up[1]); err != nil { logging.Fatalf("Failed to initialize cbauth: %s", err) } } for _, bucket := range options.buckets { go startBucket(cluster, bucket) } receive() }
// Application is example application logic that uses query-port server func Application(config c.Config) { killch := make(chan bool) s, err := NewServer( "localhost:9990", func(req interface{}, conn net.Conn, quitch <-chan interface{}) { requestHandler(req, conn, quitch, killch) }, config) if err != nil { logging.Fatalf("Listen failed - %v", err) } <-killch s.Close() }
// GetBulk gets keys in bulk func (c *Client) GetBulk(vb uint16, keys []string) (map[string]*transport.MCResponse, error) { rv := map[string]*transport.MCResponse{} going := true defer func() { going = false }() errch := make(chan error, 2) go func() { defer func() { errch <- nil }() for going { res, err := c.Receive() if err != nil { errch <- err return } switch res.Opcode { case transport.GET: going = false case transport.GETQ: default: logging.Fatalf("Unexpected opcode in GETQ response: %+v", res) } rv[keys[res.Opaque]] = res } }() for i, k := range keys { op := transport.GETQ if i == len(keys)-1 { op = transport.GET } err := c.Transmit(&transport.MCRequest{ Opcode: op, VBucket: vb, Key: []byte(k), Opaque: uint32(i), }) if err != nil { return rv, err } } return rv, <-errch }
// GetIndex for bucket/indexName. func GetIndex( client *qclient.GsiClient, bucket, indexName string) (*mclient.IndexMetadata, bool) { indexes, err := client.Refresh() if err != nil { logging.Fatalf("%v\n", err) os.Exit(1) } for _, index := range indexes { defn := index.Definition if defn.Bucket == bucket && defn.Name == indexName { return index, true //return uint64(index.Definition.DefnId), true } } return nil, false }
//send the actual DelBuckets request on adminport func sendDelBucketsRequest(ap *projClient.Client, topic string, buckets []string) error { logging.Infof("KVSender::sendDelBucketsRequest Projector %v Topic %v Buckets %v", ap, topic, buckets) if err := ap.DelBuckets(topic, buckets); err != nil { logging.Fatalf("KVSender::sendDelBucketsRequest Unexpected Error During "+ "Del Buckets Request Projector %v Topic %v Buckets %v. Err %v", ap, topic, buckets, err) return err } else { logging.Infof("KVSender::sendDelBucketsRequest Success Projector %v Topic %v Buckets %v", ap, topic, buckets) return nil } }