Beispiel #1
0
func newKeyspace(p *namespace, name string) (datastore.Keyspace, errors.Error) {

	cbNamespace := p.getPool()
	cbbucket, err := cbNamespace.GetBucket(name)

	if err != nil {
		logging.Infof(" keyspace %s not found %v", name, err)
		// go-couchbase caches the buckets
		// to be sure no such bucket exists right now
		// we trigger a refresh
		p.refresh(true)
		cbNamespace = p.getPool()

		// and then check one more time
		logging.Infof(" Retrying bucket %s", name)
		cbbucket, err = cbNamespace.GetBucket(name)
		if err != nil {
			// really no such bucket exists
			return nil, errors.NewCbKeyspaceNotFoundError(err, "keyspace "+name)
		}
	}

	if strings.EqualFold(cbbucket.Type, "memcached") {
		return nil, errors.NewCbBucketTypeNotSupportedError(nil, cbbucket.Type)
	}

	rv := &keyspace{
		namespace: p,
		name:      name,
		cbbucket:  cbbucket,
	}

	// Initialize index providers
	rv.viewIndexer = newViewIndexer(rv)

	logging.Infof("Created New Bucket %s", name)

	//discover existing indexes
	if ierr := rv.loadIndexes(); ierr != nil {
		logging.Warnf("Error loading indexes for keyspace %s, Error %v", name, ierr)
	}

	var qerr errors.Error
	rv.gsiIndexer, qerr = gsi.NewGSIIndexer(p.site.URL(), p.Name(), name)
	if qerr != nil {
		logging.Warnf("Error loading GSI indexes for keyspace %s. Error %v", name, qerr)
	}

	// Create a bucket updater that will keep the couchbase bucket fresh.
	cbbucket.RunBucketUpdater(p.KeyspaceDeleteCallback)

	return rv, nil
}
Beispiel #2
0
func newViewIndex(name string, on datastore.IndexKey, where expression.Expression, view *viewIndexer) (*viewIndex, error) {

	doc, err := newDesignDoc(name, view.keyspace.Name(), on, where)
	if err != nil {
		return nil, err
	}

	inst := viewIndex{
		name:     name,
		using:    datastore.VIEW,
		on:       on,
		where:    where,
		ddoc:     doc,
		view:     view,
		keyspace: view.keyspace,
	}

	logging.Infof("Created index %s on %s with key %v on where %v", name, view.keyspace.Name(), on, where)

	err = inst.putDesignDoc()
	if err != nil {
		return nil, err
	}

	err = inst.WaitForIndex()
	if err != nil {
		return nil, err
	}

	return &inst, nil
}
Beispiel #3
0
func (b *keyspace) refreshIndexer(url string, poolName string) {
	var err error
	b.gsiIndexer, err = gsi.NewGSIIndexer(url, poolName, b.Name())
	if err == nil {
		logging.Infof(" GSI Indexer loaded ")
	}
}
Beispiel #4
0
func TestStub(t *testing.T) {
	logger := NewLogger(os.Stdout, logging.DEBUG, false)
	logging.SetLogger(logger)

	logger.Infof("This is a message from %s", "test")
	logging.Infof("This is a message from %s", "test")
	logger.Infop("This is a message from ", logging.Pair{"name", "test"}, logging.Pair{"Queue Size", 10}, logging.Pair{"Debug Mode", false})
	logging.Infop("This is a message from ", logging.Pair{"name", "test"})

	logger.Infom("This is a message from ", logging.Map{"name": "test", "Queue Size": 10, "Debug Mode": false})
	logging.Infom("This is a message from ", logging.Map{"name": "test"})

	logger.Requestf(logging.WARN, "This is a Request from %s", "test")
	logging.Requestf(logging.INFO, "This is a Request from %s", "test")
	logger.Requestp(logging.DEBUG, "This is a Request from ", logging.Pair{"name", "test"})
	logging.Requestp(logging.ERROR, "This is a Request from ", logging.Pair{"name", "test"})

	logger.SetLevel(logging.WARN)
	fmt.Printf("Log level is %s\n", logger.Level())

	logger.Requestf(logging.WARN, "This is a Request from %s", "test")
	logging.Requestf(logging.INFO, "This is a Request from %s", "test")
	logger.Requestp(logging.DEBUG, "This is a Request from ", logging.Pair{"name", "test"})
	logging.Requestp(logging.ERROR, "This is a Request from ", logging.Pair{"name", "test"})

	logger.Warnf("This is a message from %s", "test")
	logging.Infof("This is a message from %s", "test")
	logger.Debugp("This is a message from ", logging.Pair{"name", "test"})
	logging.Errorp("This is a message from ", logging.Pair{"name", "test"})

	fmt.Printf("Changing to json formatter\n")
	logger.entryFormatter = &jsonFormatter{}
	logger.SetLevel(logging.DEBUG)

	logger.Infof("This is a message from %s", "test")
	logging.Infof("This is a message from %s", "test")
	logger.Infop("This is a message from ", logging.Pair{"name", "test"}, logging.Pair{"Queue Size", 10}, logging.Pair{"Debug Mode", false})
	logging.Infop("This is a message from ", logging.Pair{"name", "test"})

	logger.Infom("This is a message from ", logging.Map{"name": "test", "Queue Size": 10, "Debug Mode": false})
	logging.Infom("This is a message from ", logging.Map{"name": "test"})

	logger.Requestf(logging.WARN, "This is a Request from %s", "test")
	logging.Requestf(logging.INFO, "This is a Request from %s", "test")
	logger.Requestp(logging.DEBUG, "This is a Request from ", logging.Pair{"name", "test"})
	logging.Requestp(logging.ERROR, "This is a Request from ", logging.Pair{"name", "test"})
}
Beispiel #5
0
func initCbAuth(url string) (*cb.Client, error) {

	transport := cbauth.WrapHTTPTransport(cb.HTTPTransport, nil)
	cb.HTTPClient.Transport = transport

	client, err := cb.ConnectWithAuth(url, cbauth.NewAuthHandler(nil))
	if err != nil {
		return nil, err
	}

	logging.Infof(" Initialization of cbauth succeeded ")

	return &client, nil
}
Beispiel #6
0
// Called by go-couchbase if a configured keyspace is deleted
func (p *namespace) KeyspaceDeleteCallback(name string, err error) {

	p.lock.Lock()
	defer p.lock.Unlock()

	ks, ok := p.keyspaceCache[name]
	if ok {
		logging.Infof("Keyspace %v being deleted", name)
		ks.(*keyspace).deleted = true
		delete(p.keyspaceCache, name)

	} else {
		logging.Warnf("Keyspace %v not configured on this server", name)
	}
}
Beispiel #7
0
func (vi *viewIndex) Drop(requestId string) errors.Error {

	err := vi.DropViewIndex()
	if err != nil {
		return errors.NewCbViewsDropIndexError(err, vi.Name())
	}
	// TODO need mutex

	vi.view.Lock()
	vi.view.Unlock()

	delete(vi.view.indexes, vi.name)
	if vi.IsPrimary() == true {
		logging.Infof(" Primary index being dropped ")
		delete(vi.view.primary, vi.name)
	}
	return nil
}
Beispiel #8
0
func (view *viewIndexer) loadViewIndexes() errors.Error {

	// recreate indexes from ddocs
	indexes := make(map[string]datastore.Index)
	primary := make(map[string]datastore.PrimaryIndex)

	defer func() {

		// only if the indexes have changed then update
		if view.indexesUpdated(view.indexes, indexes) {
			logging.Infof("Indexes updated ")

			view.Lock()
			view.indexes = indexes
			view.primary = primary
			view.Unlock()
		}
	}()

	indexList, err := loadViewIndexes(view)
	if err != nil {
		return errors.NewCbLoadIndexesError(err, "Keyspace "+view.KeyspaceId())
	}

	if len(indexList) == 0 {
		return nil
	}

	for _, index := range indexList {
		name := (*index).Name()
		indexes[name] = *index
		switch (*index).(type) {
		case *primaryIndex:
			primary[name] = (*index).(datastore.PrimaryIndex)
		}
	}

	return nil
}
Beispiel #9
0
func (b *keyspace) Delete(deletes []string) ([]string, errors.Error) {

	failedDeletes := make([]string, 0)
	actualDeletes := make([]string, 0)
	var err error
	for _, key := range deletes {
		if err = b.cbbucket.Delete(key); err != nil {
			if !isNotFoundError(err) {
				logging.Infof("Failed to delete key %s Error %s", key, err)
				failedDeletes = append(failedDeletes, key)
			}
		} else {
			actualDeletes = append(actualDeletes, key)
		}
	}

	if len(failedDeletes) > 0 {
		return actualDeletes, errors.NewCbDeleteFailedError(err, "Some keys were not deleted "+fmt.Sprintf("%v", failedDeletes))
	}

	return actualDeletes, nil
}
Beispiel #10
0
func newViewIndexFromExistingMap(name, ddName string, on expression.Expressions, view *viewIndexer) (*viewIndex, error) {

	var doc designdoc

	mapfn, err := getMapFunction(view, ddName)
	if err != nil {
		return nil, err
	}

	doc.name = "ddl_" + name
	doc.viewname = name
	doc.mapfn = mapfn

	inst := viewIndex{
		name:     name,
		using:    datastore.VIEW,
		on:       on,
		ddoc:     &doc,
		view:     view,
		keyspace: view.keyspace,
	}

	logging.Infof("Created index %s on %s with key %v", name, view.keyspace.Name(), on)

	err = inst.putDesignDoc()
	if err != nil {
		return nil, err
	}

	err = inst.WaitForIndex()
	if err != nil {
		return nil, err
	}

	return &inst, nil
}
func (b *indexKeyspace) fetchOne(key string) ([]datastore.AnnotatedPair, errors.Error) {
	rv := make([]datastore.AnnotatedPair, 0, 2)
	ids := strings.SplitN(key, "/", 3)

	actualStore := b.namespace.store.actualStore
	namespace, err := actualStore.NamespaceById(ids[0])
	if err != nil {
		return nil, err
	}

	keyspace, err := namespace.KeyspaceById(ids[1])
	if err != nil {
		return nil, err
	}

	indexers, err := keyspace.Indexers()
	if err != nil {
		logging.Infof("Indexer returned error %v", err)
		return nil, err
	}

	for _, indexer := range indexers {
		index, err := indexer.IndexById(ids[2])
		if err != nil {
			continue
		}

		state, msg, err := index.State()
		if err != nil {
			return nil, err
		}

		doc := value.NewAnnotatedValue(map[string]interface{}{
			"id":           index.Id(),
			"name":         index.Name(),
			"keyspace_id":  keyspace.Id(),
			"namespace_id": namespace.Id(),
			"datastore_id": actualStore.Id(),
			"index_key":    datastoreObjectToJSONSafe(indexKeyToIndexKeyStringArray(index.RangeKey())),
			"using":        datastoreObjectToJSONSafe(index.Type()),
			"state":        string(state),
		})

		doc.SetAttachment("meta", map[string]interface{}{
			"id": key,
		})

		if msg != "" {
			doc.SetField("message", msg)
		}

		cond := index.Condition()
		if cond != nil {
			doc.SetField("condition", cond.String())
		}

		if index.IsPrimary() {
			doc.SetField("is_primary", true)
		}

		rv = append(rv, datastore.AnnotatedPair{key, doc})
	}

	return rv, nil
}
Beispiel #12
0
// NewSite creates a new Couchbase site for the given url.
func NewDatastore(u string) (s datastore.Datastore, e errors.Error) {

	var client cb.Client
	var cbAuthInit bool

	// try and initialize cbauth

	c, err := initCbAuth(u)
	if err != nil {
		logging.Errorf(" Unable to initialize cbauth. Error %v", err)
		url, err := url.Parse(u)
		if err != nil {
			return nil, errors.NewCbUrlParseError(err, "url "+u)
		}

		if url.User != nil {
			password, _ := url.User.Password()
			if password == "" {
				logging.Errorf("No password found in url %s", u)
			}

			// intialize cb_auth variables manually
			logging.Infof(" Trying to init cbauth with credentials %s %s", url.Host, url.User.Username())
			set, err := cbauth.InternalRetryDefaultInit(url.Host, url.User.Username(), password)
			if set == false || err != nil {
				logging.Errorf(" Unable to initialize cbauth variables. Error %v", err)
			} else {
				c, err = initCbAuth("http://" + url.Host)
				if err != nil {
					logging.Errorf("Unable to initliaze cbauth.  Error %v", err)
				} else {
					client = *c
					cbAuthInit = true
				}
			}
		}
	} else {
		client = *c
		cbAuthInit = true
	}

	if cbAuthInit == false {
		// connect without auth
		logging.Warnf("Unable to intialize cbAuth, access to couchbase buckets may be restricted")
		cb.HTTPClient = &http.Client{}
		client, err = cb.Connect(u)
		if err != nil {
			return nil, errors.NewCbConnectionError(err, "url "+u)
		}
	}

	site := &site{
		client:         client,
		namespaceCache: make(map[string]*namespace),
		CbAuthInit:     cbAuthInit,
	}

	// initialize the default pool.
	// TODO can couchbase server contain more than one pool ?

	defaultPool, Err := loadNamespace(site, "default")
	if Err != nil {
		logging.Errorf("Cannot connect to default pool")
		return nil, Err
	}

	site.namespaceCache["default"] = defaultPool
	logging.Infof("New site created with url %s", u)

	return site, nil
}
Beispiel #13
0
func (s *site) Authorize(privileges datastore.Privileges, credentials datastore.Credentials) errors.Error {

	var authResult bool
	var err error

	if s.CbAuthInit == false {
		// cbauth is not initialized. Access to SASL protected buckets will be
		// denied by the couchbase server
		logging.Warnf("CbAuth not intialized")
		return nil
	}

	// if the authentication fails for any of the requested privileges return an error
	for keyspace, privilege := range privileges {

		if strings.Contains(keyspace, ":") {
			q := strings.Split(keyspace, ":")
			pool := q[0]
			keyspace = q[1]

			if strings.EqualFold(pool, "#system") {
				// trying auth on system keyspace
				return nil
			}
		}

		logging.Debugf("Authenticating for keyspace %s", keyspace)

		if len(credentials) == 0 {
			authResult, err = doAuth(keyspace, "", keyspace, privilege)
			if authResult == false || err != nil {
				logging.Infof("Auth failed for keyspace %s", keyspace)
				return errors.NewDatastoreAuthorizationError(err, "Keyspace "+keyspace)
			}
		} else {
			//look for either the bucket name or the admin credentials
			for username, password := range credentials {

				var un string
				userCreds := strings.Split(username, ":")
				if len(userCreds) == 1 {
					un = userCreds[0]
				} else {
					un = userCreds[1]
				}

				logging.Debugf(" Credentials %v %v", un, userCreds)

				if strings.EqualFold(un, "Administrator") || strings.EqualFold(userCreds[0], "admin") {
					authResult, err = doAuth(un, password, keyspace, privilege)
				} else if un != "" && password != "" {
					authResult, err = doAuth(un, password, keyspace, privilege)
				} else {
					//try with empty password
					authResult, err = doAuth(keyspace, "", keyspace, privilege)
				}

				if err != nil {
					return errors.NewDatastoreAuthorizationError(err, "Keyspace "+keyspace)

				}

				// Auth succeeded
				if authResult == true {
					break
				}
				continue
			}
		}

	}

	if authResult == false {
		return errors.NewDatastoreAuthorizationError(err, "")
	}
	return nil
}