// Index partitionsfor testing func SeedPartitionMap(bucket base.Bucket, numPartitions uint16) error { maxVbNo := uint16(1024) //maxVbNo := uint16(64) partitionDefs := make(base.PartitionStorageSet, numPartitions) vbPerPartition := maxVbNo / numPartitions for partition := uint16(0); partition < numPartitions; partition++ { storage := base.PartitionStorage{ Index: partition, VbNos: make([]uint16, vbPerPartition), } for index := uint16(0); index < vbPerPartition; index++ { vb := partition*vbPerPartition + index storage.VbNos[index] = vb } partitionDefs[partition] = storage } // Persist to bucket value, err := json.Marshal(partitionDefs) if err != nil { return err } bucket.SetRaw(base.KIndexPartitionKey, 0, value) return nil }
// Creates a new Shadower. func NewShadower(context *DatabaseContext, bucket base.Bucket, docIDPattern *regexp.Regexp) (*Shadower, error) { tapFeed, err := bucket.StartTapFeed(sgbucket.TapArguments{Backfill: 0}) if err != nil { return nil, err } s := &Shadower{context: context, bucket: bucket, tapFeed: tapFeed, docIDPattern: docIDPattern} go s.readTapFeed() return s, nil }
// Creates a new Shadower. func NewShadower(context *DatabaseContext, bucket base.Bucket, docIDPattern *regexp.Regexp) (*Shadower, error) { tapFeed, err := bucket.StartTapFeed(sgbucket.TapArguments{Backfill: 0, Notify: func(bucket string, err error) { context.TakeDbOffline("Lost shadower TAP Feed") }}) if err != nil { return nil, err } s := &Shadower{context: context, bucket: bucket, tapFeed: tapFeed, docIDPattern: docIDPattern} go s.readTapFeed() return s, nil }
func (context *DatabaseContext) GetCBGTIndexNameForBucket(bucket base.Bucket) (indexName string) { // Real Couchbase buckets use an index name that includes UUID. cbBucket, ok := bucket.(base.CouchbaseBucket) if ok { indexName = cbBucket.GetCBGTIndexName() } else { indexName = bucket.GetName() } return indexName }
func appendWrite(bucket base.Bucket, key string, vb int, sequence uint64) error { sequenceBytes := getSequenceAsBytes(sequence) err := bucket.Append(key, sequenceBytes) // TODO: assumes err means it hasn't been created yet if err != nil { added, err := bucket.AddRaw(key, 0, sequenceBytes) if err != nil || added == false { log.Printf("AddRaw also failed?! %s:%v", key, err) } } return nil }
// Starts a changeListener on a given Bucket. func (listener *changeListener) Start(bucket base.Bucket, trackDocs bool, notify sgbucket.BucketNotifyFn) error { listener.bucket = bucket listener.TapArgs = sgbucket.TapArguments{ Backfill: sgbucket.TapNoBackfill, Notify: notify, } tapFeed, err := bucket.StartTapFeed(listener.TapArgs) if err != nil { return err } listener.tapFeed = tapFeed listener.counter = 1 listener.terminateCheckCounter = 0 listener.keyCounts = map[string]uint64{} listener.tapNotifier = sync.NewCond(&sync.Mutex{}) if trackDocs { listener.DocChannel = make(chan sgbucket.TapEvent, 100) } // Start a goroutine to broadcast to the tapNotifier whenever a channel or user/role changes: go func() { defer func() { listener.notifyStopping() if listener.DocChannel != nil { close(listener.DocChannel) } }() for event := range tapFeed.Events() { if event.Opcode == sgbucket.TapMutation || event.Opcode == sgbucket.TapDeletion { key := string(event.Key) if strings.HasPrefix(key, auth.UserKeyPrefix) || strings.HasPrefix(key, auth.RoleKeyPrefix) { if listener.OnDocChanged != nil { listener.OnDocChanged(key, event.Value, event.Sequence, event.VbNo) } listener.Notify(base.SetOf(key)) } else if trackDocs && !strings.HasPrefix(key, KSyncKeyPrefix) && !strings.HasPrefix(key, kIndexPrefix) { if listener.OnDocChanged != nil { listener.OnDocChanged(key, event.Value, event.Sequence, event.VbNo) } listener.DocChannel <- event } } } }() return nil }
func verifyVBMapping(bucket base.Bucket, channelName string) error { channelVbNo := uint32(0) for i := 0; i < 1024; i++ { docId := fmt.Sprintf("_index::%s::%d::%s", channelName, 1, vbSuffixMap[i]) vbNo := bucket.VBHash(docId) if channelVbNo == 0 { channelVbNo = vbNo } if vbNo != channelVbNo { return errors.New("vb numbers don't match") } } return nil }
func installViews(bucket base.Bucket) error { // View for finding every Couchbase doc (used when deleting a database) // Key is docid; value is null allbits_map := `function (doc, meta) { emit(meta.id, null); }` // View for _all_docs // Key is docid; value is [revid, sequence] alldocs_map := `function (doc, meta) { var sync = doc._sync; if (sync === undefined || meta.id.substring(0,6) == "_sync:") return; if ((sync.flags & 1) || sync.deleted) return; var channels = sync.channels; var channelNames = []; for (ch in channels) { if (channels[ch] == null) channelNames.push(ch); } emit(meta.id, {r:sync.rev, s:sync.sequence, c:channelNames}); }` // View for importing unknown docs // Key is [existing?, docid] where 'existing?' is false for unknown docs import_map := `function (doc, meta) { if(meta.id.substring(0,6) != "_sync:") { var exists = (doc["_sync"] !== undefined); emit([exists, meta.id], null); } }` // View for compaction -- finds all revision docs // Key and value are ignored. oldrevs_map := `function (doc, meta) { var sync = doc._sync; if (meta.id.substring(0,10) == "_sync:rev:") emit("",null); }` // Sessions view - used for session delete // Key is username; value is docid sessions_map := `function (doc, meta) { var prefix = meta.id.substring(0,%d); if (prefix == %q) emit(doc.username, meta.id);}` sessions_map = fmt.Sprintf(sessions_map, len(auth.SessionKeyPrefix), auth.SessionKeyPrefix) // All-principals view // Key is name; value is true for user, false for role principals_map := `function (doc, meta) { var prefix = meta.id.substring(0,11); var isUser = (prefix == %q); if (isUser || prefix == %q) emit(meta.id.substring(%d), isUser); }` principals_map = fmt.Sprintf(principals_map, auth.UserKeyPrefix, auth.RoleKeyPrefix, len(auth.UserKeyPrefix)) // By-channels view. // Key is [channelname, sequence]; value is [docid, revid, flag?] // where flag is true for doc deletion, false for removed from channel, missing otherwise channels_map := `function (doc, meta) { var sync = doc._sync; if (sync === undefined || meta.id.substring(0,6) == "_sync:") return; var sequence = sync.sequence; if (sequence === undefined) return; var value = {rev:sync.rev}; if (sync.flags) { value.flags = sync.flags } else if (sync.deleted) { value.flags = %d // channels.Deleted } if (%v) // EnableStarChannelLog emit(["*", sequence], value); var channels = sync.channels; if (channels) { for (var name in channels) { removed = channels[name]; if (!removed) emit([name, sequence], value); else { var flags = removed.del ? %d : %d; // channels.Removed/Deleted emit([name, removed.seq], {rev:removed.rev, flags: flags}); } } } }` channels_map = fmt.Sprintf(channels_map, channels.Deleted, EnableStarChannelLog, channels.Removed|channels.Deleted, channels.Removed) // Channel access view, used by ComputeChannelsForPrincipal() // Key is username; value is dictionary channelName->firstSequence (compatible with TimedSet) access_map := `function (doc, meta) { var sync = doc._sync; if (sync === undefined || meta.id.substring(0,6) == "_sync:") return; var access = sync.access; if (access) { for (var name in access) { emit(name, access[name]); } } }` // Role access view, used by ComputeRolesForUser() // Key is username; value is array of role names roleAccess_map := `function (doc, meta) { var sync = doc._sync; if (sync === undefined || meta.id.substring(0,6) == "_sync:") return; var access = sync.role_access; if (access) { for (var name in access) { emit(name, access[name]); } } }` designDocMap := map[string]walrus.DesignDoc{} designDocMap[DesignDocSyncGateway] = walrus.DesignDoc{ Views: walrus.ViewMap{ ViewPrincipals: walrus.ViewDef{Map: principals_map}, ViewChannels: walrus.ViewDef{Map: channels_map}, ViewAccess: walrus.ViewDef{Map: access_map}, ViewRoleAccess: walrus.ViewDef{Map: roleAccess_map}, }, } designDocMap[DesignDocSyncHousekeeping] = walrus.DesignDoc{ Views: walrus.ViewMap{ ViewAllBits: walrus.ViewDef{Map: allbits_map}, ViewAllDocs: walrus.ViewDef{Map: alldocs_map, Reduce: "_count"}, ViewImport: walrus.ViewDef{Map: import_map, Reduce: "_count"}, ViewOldRevs: walrus.ViewDef{Map: oldrevs_map, Reduce: "_count"}, ViewSessions: walrus.ViewDef{Map: sessions_map}, }, } // add all design docs from map into bucket for designDocName, designDoc := range designDocMap { if err := bucket.PutDDoc(designDocName, designDoc); err != nil { base.Warn("Error installing Couchbase design doc: %v", err) return err } } return nil }
func installViews(bucket base.Bucket) error { // View for finding every Couchbase doc (used when deleting a database) // Key is docid; value is null allbits_map := `function (doc, meta) { emit(meta.id, null); }` // View for _all_docs // Key is docid; value is [revid, sequence] alldocs_map := `function (doc, meta) { var sync = doc._sync; if (sync === undefined || meta.id.substring(0,6) == "_sync:") return; if ((sync.flags & 1) || sync.deleted) return; var channels = sync.channels; var channelNames = []; for (ch in channels) { if (channels[ch] == null) channelNames.push(ch); } emit(meta.id, {r:sync.rev, s:sync.sequence, c:channelNames}); }` // View for importing unknown docs // Key is [existing?, docid] where 'existing?' is false for unknown docs import_map := `function (doc, meta) { if(meta.id.substring(0,6) != "_sync:") { var exists = (doc["_sync"] !== undefined); emit([exists, meta.id], null); } }` // View for compaction -- finds all revision docs // Key and value are ignored. oldrevs_map := `function (doc, meta) { var sync = doc._sync; if (meta.id.substring(0,10) == "_sync:rev:") emit("",null); }` // Sessions view - used for session delete // Key is username; value is docid sessions_map := `function (doc, meta) { var prefix = meta.id.substring(0,%d); if (prefix == %q) emit(doc.username, meta.id);}` sessions_map = fmt.Sprintf(sessions_map, len(auth.SessionKeyPrefix), auth.SessionKeyPrefix) // All-principals view // Key is name; value is true for user, false for role principals_map := `function (doc, meta) { var prefix = meta.id.substring(0,11); var isUser = (prefix == %q); if (isUser || prefix == %q) emit(meta.id.substring(%d), isUser); }` principals_map = fmt.Sprintf(principals_map, auth.UserKeyPrefix, auth.RoleKeyPrefix, len(auth.UserKeyPrefix)) // By-channels view. // Key is [channelname, sequence]; value is [docid, revid, flag?] // where flag is true for doc deletion, false for removed from channel, missing otherwise channels_map := `function (doc, meta) { var sync = doc._sync; if (sync === undefined || meta.id.substring(0,6) == "_sync:") return; var sequence = sync.sequence; if (sequence === undefined) return; var value = {rev:sync.rev}; if (sync.flags) { value.flags = sync.flags } else if (sync.deleted) { value.flags = %d // channels.Deleted } if (%v) // EnableStarChannelLog emit(["*", sequence], value); var channels = sync.channels; if (channels) { for (var name in channels) { removed = channels[name]; if (!removed) emit([name, sequence], value); else { var flags = removed.del ? %d : %d; // channels.Removed/Deleted emit([name, removed.seq], {rev:removed.rev, flags: flags}); } } } }` channels_map = fmt.Sprintf(channels_map, channels.Deleted, EnableStarChannelLog, channels.Removed|channels.Deleted, channels.Removed) // Channel access view, used by ComputeChannelsForPrincipal() // Key is username; value is dictionary channelName->firstSequence (compatible with TimedSet) access_map := `function (doc, meta) { var sync = doc._sync; if (sync === undefined || meta.id.substring(0,6) == "_sync:") return; var access = sync.access; if (access) { for (var name in access) { emit(name, access[name]); } } }` // Vbucket sequence version of channel access view, used by ComputeChannelsForPrincipal() // Key is username; value is dictionary channelName->firstSequence (compatible with TimedSet) access_vbSeq_map := `function (doc, meta) { var sync = doc._sync; if (sync === undefined || meta.id.substring(0,6) == "_sync:") return; var access = sync.access; if (access) { for (var name in access) { // Build a timed set based on vb and vbseq of this revision var value = {}; for (var channel in access[name]) { var timedSetWithVbucket = {}; timedSetWithVbucket["vb"] = parseInt(meta.vb, 10); timedSetWithVbucket["seq"] = parseInt(meta.seq, 10); value[channel] = timedSetWithVbucket; } emit(name, value) } } }` // Role access view, used by ComputeRolesForUser() // Key is username; value is array of role names roleAccess_map := `function (doc, meta) { var sync = doc._sync; if (sync === undefined || meta.id.substring(0,6) == "_sync:") return; var access = sync.role_access; if (access) { for (var name in access) { emit(name, access[name]); } } }` designDocMap := map[string]sgbucket.DesignDoc{} designDocMap[DesignDocSyncGateway] = sgbucket.DesignDoc{ Views: sgbucket.ViewMap{ ViewPrincipals: sgbucket.ViewDef{Map: principals_map}, ViewChannels: sgbucket.ViewDef{Map: channels_map}, ViewAccess: sgbucket.ViewDef{Map: access_map}, ViewAccessVbSeq: sgbucket.ViewDef{Map: access_vbSeq_map}, ViewRoleAccess: sgbucket.ViewDef{Map: roleAccess_map}, }, } designDocMap[DesignDocSyncHousekeeping] = sgbucket.DesignDoc{ Views: sgbucket.ViewMap{ ViewAllBits: sgbucket.ViewDef{Map: allbits_map}, ViewAllDocs: sgbucket.ViewDef{Map: alldocs_map, Reduce: "_count"}, ViewImport: sgbucket.ViewDef{Map: import_map, Reduce: "_count"}, ViewOldRevs: sgbucket.ViewDef{Map: oldrevs_map, Reduce: "_count"}, ViewSessions: sgbucket.ViewDef{Map: sessions_map}, }, } sleeper := base.CreateDoublingSleeperFunc( 11, //MaxNumRetries approx 10 seconds total retry duration 5, //InitialRetrySleepTimeMS ) // add all design docs from map into bucket for designDocName, designDoc := range designDocMap { //start a retry loop to put design document backing off double the delay each time worker := func() (shouldRetry bool, err error, value interface{}) { err = bucket.PutDDoc(designDocName, designDoc) if err != nil { base.Warn("Error installing Couchbase design doc: %v", err) } return err != nil, err, nil } description := fmt.Sprintf("Attempt to install Couchbase design doc bucket : %v", designDocName) err, _ := base.RetryLoop(description, worker, sleeper) if err != nil { return err } } return nil }
// Creates a new DatabaseContext on a bucket. The bucket will be closed when this context closes. func NewDatabaseContext(dbName string, bucket base.Bucket, autoImport bool, options DatabaseContextOptions) (*DatabaseContext, error) { if err := ValidateDatabaseName(dbName); err != nil { return nil, err } context := &DatabaseContext{ Name: dbName, Bucket: bucket, StartTime: time.Now(), RevsLimit: DefaultRevsLimit, autoImport: autoImport, Options: options, } context.revisionCache = NewRevisionCache(int(options.RevisionCacheCapacity), context.revCacheLoader) context.EventMgr = NewEventManager() var err error context.sequences, err = newSequenceAllocator(bucket) if err != nil { return nil, err } lastSeq, err := context.sequences.lastSequence() if err != nil { return nil, err } if options.IndexOptions == nil { // In-memory channel cache context.SequenceType = IntSequenceType context.changeCache = &changeCache{} } else { // KV channel index context.SequenceType = ClockSequenceType context.changeCache = &kvChangeIndex{} context.SequenceHasher, err = NewSequenceHasher(options.SequenceHashOptions) if err != nil { return nil, err } } context.changeCache.Init(context, SequenceID{Seq: lastSeq}, func(changedChannels base.Set) { context.tapListener.Notify(changedChannels) }, options.CacheOptions, options.IndexOptions) context.SetOnChangeCallback(context.changeCache.DocChanged) // Initialize the tap Listener for notify handling context.tapListener.Init(bucket.GetName()) // If not using channel index, start the tap feed if options.IndexOptions == nil { if err = context.tapListener.Start(bucket, options.TrackDocs, func(bucket string, err error) { context.TakeDbOffline("Lost TAP Feed") }); err != nil { return nil, err } } // Load providers into provider map. Does basic validation on the provider definition, and identifies the default provider. if options.OIDCOptions != nil { context.OIDCProviders = make(auth.OIDCProviderMap) for name, provider := range options.OIDCOptions.Providers { if provider.Issuer == "" || provider.ClientID == nil { base.Warn("Issuer and ClientID required for OIDC Provider - skipping provider %q", name) continue } if provider.ValidationKey == nil { base.Warn("Validation Key not defined in config for provider %q - auth code flow will not be supported for this provider", name) } if strings.Contains(name, "_") { return nil, fmt.Errorf("OpenID Connect provider names cannot contain underscore:%s", name) } provider.Name = name if _, ok := context.OIDCProviders[provider.Issuer]; ok { base.Warn("Multiple OIDC providers defined for issuer %v", provider.Issuer) return nil, fmt.Errorf("Multiple OIDC providers defined for issuer %v", provider.Issuer) } // If this is the default provider, or there's only one provider defined, set IsDefault if (options.OIDCOptions.DefaultProvider != nil && name == *options.OIDCOptions.DefaultProvider) || len(options.OIDCOptions.Providers) == 1 { provider.IsDefault = true } // If this isn't the default provider, add the provider to the callback URL (needed to identify provider to _oidc_callback) if !provider.IsDefault && provider.CallbackURL != nil { var updatedCallback string if strings.Contains(*provider.CallbackURL, "?") { updatedCallback = fmt.Sprintf("%s&provider=%s", *provider.CallbackURL, name) } else { updatedCallback = fmt.Sprintf("%s?provider=%s", *provider.CallbackURL, name) } provider.CallbackURL = &updatedCallback } context.OIDCProviders[name] = provider } if len(context.OIDCProviders) == 0 { return nil, errors.New("OpenID Connect defined in config, but no valid OpenID Connect providers specified.") } } go context.watchDocChanges() return context, nil }
func (d *DenseBlock) loadBlock(bucket base.Bucket) (err error) { d.value, d.cas, err = bucket.GetRaw(d.Key) d.clock = nil return err }