Exemplo n.º 1
0
func (this *JSConverter) Visit(expr expression.Expression) string {
	var buf bytes.Buffer
	s, err := expr.Accept(this)
	if err != nil {
		logging.Errorf("Unexpected error in JSConverter: %v", err)
		return ""
	}

	switch s := s.(type) {
	case string:
		buf.WriteString(s)
		for this.stack.Size() != 0 {
			funcExpr := this.stack.Pop().(*funcExpr)
			buf.WriteString(funcExpr.name)
			if funcExpr.operands.Front() != nil {
				buf.WriteString(writeOperands(funcExpr.operands))
			}
		}

	case []byte:
		buf.WriteString(string(s))
		for this.stack.Size() != 0 {
			funcExpr := this.stack.Pop().(*funcExpr)
			buf.WriteString(funcExpr.name)
			if funcExpr.operands.Front() != nil {
				buf.WriteString(writeOperands(funcExpr.operands))
			}
		}

	default:
		buf.WriteString(s.(string))
	}

	return buf.String()
}
Exemplo n.º 2
0
func doNotFound(endpoint *HttpEndpoint, w http.ResponseWriter, req *http.Request) (interface{}, errors.Error) {
	acctStore := endpoint.server.AccountingStore()
	reg := acctStore.MetricRegistry()
	if reg == nil {
		logging.Errorf("http.NotFoundHandler - nil metric registry")
	} else {
		reg.Counter(accounting.INVALID_REQUESTS).Inc(1)
	}
	return nil, nil
}
Exemplo n.º 3
0
func CpuTimes() (int64, int64) {
	ru := syscall.Rusage{}
	if err := syscall.Getrusage(syscall.RUSAGE_SELF, &ru); err != nil {
		logging.Errorf(err.Error())
		return int64(0), int64(0)
	}

	newUtime := int64(ru.Utime.Nano())
	newStime := int64(ru.Stime.Nano())
	return newUtime, newStime
}
Exemplo n.º 4
0
func (pi *indexIndex) ScanEntries(requestId string, limit int64, cons datastore.ScanConsistency,
	vector timestamp.Vector, conn *datastore.IndexConnection) {
	defer close(conn.EntryChannel())

	// eliminate duplicate keys
	keys := make(map[string]string, 64)

	actualStore := pi.keyspace.namespace.store.actualStore
	namespaceIds, err := actualStore.NamespaceIds()
	if err == nil {
		for _, namespaceId := range namespaceIds {
			namespace, err := actualStore.NamespaceById(namespaceId)
			if err == nil {
				keyspaceIds, err := namespace.KeyspaceIds()
				if err == nil {
					for _, keyspaceId := range keyspaceIds {
						keyspace, err := namespace.KeyspaceById(keyspaceId)
						if err == nil {
							indexers, err := keyspace.Indexers()
							if err == nil {
								for _, indexer := range indexers {
									err = indexer.Refresh()
									if err != nil {
										logging.Errorf("Refreshing indexes failed %v", err)
										conn.Error(errors.NewSystemDatastoreError(err, ""))
										// don't return here but continue processing, because other keyspaces may still be responsive. MB-15834
										continue
									}

									indexIds, err := indexer.IndexIds()
									if err == nil {
										for _, indexId := range indexIds {
											key := fmt.Sprintf("%s/%s/%s", namespaceId, keyspaceId, indexId)
											keys[key] = key
										}
									}
								}
							}
						}
					}
				}
			}
		}
	}

	for k, _ := range keys {
		entry := datastore.IndexEntry{PrimaryKey: k}
		conn.EntryChannel() <- &entry
	}
}
Exemplo n.º 5
0
func doParse(lex *lexer) {
	defer func() {
		r := recover()
		if r != nil {
			lex.Error(fmt.Sprintf("Error while parsing: %v", r))

			// Log this error
			buf := make([]byte, 2048)
			n := runtime.Stack(buf, false)
			logging.Errorf("Error while parsing: %v\n%s", r, string(buf[0:n]))
		}
	}()

	yyParse(lex)
}
Exemplo n.º 6
0
func (view *viewIndexer) indexesUpdated(a, b map[string]datastore.Index) bool {

	if len(a) != len(b) {
		return true
	}

	view.RLock()
	defer view.RUnlock()

	defer func() {
		if err := recover(); err != nil {
			logging.Errorf("Panic in compare", err)
		}
	}()

	// if the checksum of each index is the same
	for name, idx_a := range a {
		idx_b, ok := b[name]
		if !ok {
			return true
		}

		switch idx_a.(type) {
		case *primaryIndex:
			if idx_a.(*primaryIndex).signature() != idx_b.(*primaryIndex).signature() {
				return true
			}
		default:
			if idx_a.(*viewIndex).signature() != idx_b.(*viewIndex).signature() {
				return true
			}
		}
	}

	return false
}
Exemplo n.º 7
0
func WalkViewInBatches(result chan cb.ViewRow, errs chan errors.Error, stop chan bool, bucket *cb.Bucket,
	ddoc string, view string, isPrimary bool, options map[string]interface{}, batchSize int64, limit int64) {

	if limit != 0 && limit < batchSize {
		batchSize = limit
	}

	defer close(result)
	defer close(errs)

	defer func() {
		r := recover()
		if r != nil {
			logging.Errorf("View Walking Panic: %v\n%s", r, debug.Stack())
			errs <- errors.NewCbViewsAccessError(nil, "Panic In walking view "+view)
		}
	}()

	options["limit"] = batchSize + 1

	numRead := int64(0)
	numSent := int64(0)
	keysSent := map[string]bool{}
	ok := true
	for ok {

		logURL, err := bucket.ViewURL(ddoc, view, options)
		if err == nil {
			logging.Debugf("Request View: %v", logURL)
		}
		vres, err := bucket.View(ddoc, view, options)
		if err != nil {
			errs <- errors.NewCbViewsAccessError(err, "View name "+view)
			return
		}

		for i, row := range vres.Rows {
			// dont process the last row, its just used to see if we
			// need to continue processing
			if int64(i) < batchSize {
				// Send the row if its primary key has not been sent
				if isPrimary || !keysSent[row.ID] {
					select {
					case result <- row:
						numSent += 1
					case <-stop:
						ok = false
						break
					}
				}
				// For non primary views, mark the row's primary key as sent
				if !isPrimary {
					keysSent[row.ID] = true
				}
				numRead += 1
			}
		}

		if (int64(len(vres.Rows)) > batchSize) && (limit == 0 || (limit != 0 && numRead < limit)) {
			// prepare for next run
			skey := vres.Rows[batchSize].Key
			skeydocid := vres.Rows[batchSize].ID
			options["startkey"] = skey
			options["startkey_docid"] = cb.DocID(skeydocid)
		} else {
			// stop
			ok = false
		}
	}
	logging.Debugf("WalkViewInBatches %s: %d rows fetched, %d rows sent", view, numRead, numSent)
}
Exemplo n.º 8
0
func loadViewIndexes(v *viewIndexer) ([]*datastore.Index, error) {

	b := v.keyspace
	rows, err := b.cbbucket.GetDDocsWithRetry()
	if err != nil {
		return nil, err
	}

	inames := make([]string, 0, len(rows.Rows))
	nonUsableIndexes := make([]string, 0)

	for _, row := range rows.Rows {
		cdoc := row.DDoc
		id := cdoc.Meta["id"].(string)
		if strings.HasPrefix(id, "_design/ddl_") {
			iname := strings.TrimPrefix(id, "_design/ddl_")
			inames = append(inames, iname)
		} else if strings.HasPrefix(id, "_design/dev_") {
			// append this to the list of non-usuable indexes
			iname := strings.TrimPrefix(id, "_design/dev_")
			for _, name := range v.nonUsableIndexes {
				if iname == name {
					continue
				}
			}
			nonUsableIndexes = append(nonUsableIndexes, iname)

		} else if strings.HasPrefix(id, "_design/") {
			iname := strings.TrimPrefix(id, "_design/")
			for _, name := range v.nonUsableIndexes {
				if iname == name {
					continue
				}
			}
			nonUsableIndexes = append(nonUsableIndexes, iname)
		}

	}

	indexes := make([]*datastore.Index, 0, len(inames))
	for _, iname := range inames {
		ddname := "ddl_" + iname
		jdoc, err := getDesignDoc(b, ddname)
		if err != nil {
			return nil, err
		}
		jview, ok := jdoc.Views[iname]
		if !ok {
			nonUsableIndexes = append(nonUsableIndexes, iname)
			logging.Errorf("Missing view for index %v ", iname)
			continue
		}

		exprlist := make([]expression.Expression, 0, len(jdoc.IndexOn))

		for _, ser := range jdoc.IndexOn {
			if jdoc.PrimaryIndex == true {
				doc := expression.NewIdentifier(b.Name())
				meta := expression.NewMeta(doc)
				mdid := expression.NewField(meta, expression.NewFieldName("id", false))
				exprlist = append(exprlist, mdid)
			} else {
				expr, err := parser.Parse(ser)
				if err != nil {
					nonUsableIndexes = append(nonUsableIndexes, iname)
					logging.Errorf("Cannot unmarshal expression for index  %v", iname)
					continue
				}
				exprlist = append(exprlist, expr)
			}
		}
		if len(exprlist) != len(jdoc.IndexOn) {
			continue
		}

		var conditionExpr expression.Expression
		if jdoc.Condition != "" {
			conditionExpr, err = parser.Parse(jdoc.Condition)
			if err != nil {
				logging.Errorf("Unable to parse condition expression. Err %v", err)
				continue
			}
		}

		ddoc := designdoc{
			name:     ddname,
			viewname: iname,
			mapfn:    jview.Map,
			reducefn: jview.Reduce,
		}
		if ddoc.checksum() != jdoc.IndexChecksum {
			nonUsableIndexes = append(nonUsableIndexes, iname)
			logging.Errorf("Warning - checksum failed on index  %v", iname)
			continue
		}

		var index datastore.Index

		if jdoc.PrimaryIndex == true {
			index = &primaryIndex{
				viewIndex{
					name:      iname,
					keyspace:  b,
					view:      v,
					using:     datastore.VIEW,
					ddoc:      &ddoc,
					on:        exprlist,
					where:     conditionExpr,
					isPrimary: jdoc.PrimaryIndex,
				},
			}

		} else {
			index = &viewIndex{
				name:      iname,
				keyspace:  b,
				view:      v,
				using:     datastore.VIEW,
				ddoc:      &ddoc,
				on:        exprlist,
				where:     conditionExpr,
				isPrimary: jdoc.PrimaryIndex,
			}
		}

		indexes = append(indexes, &index)

	}
	v.nonUsableIndexes = nonUsableIndexes

	if len(indexes) == 0 {
		return nil, nil
	}

	return indexes, nil
}
Exemplo n.º 9
0
func (vi *viewIndex) Scan(requestId string, span *datastore.Span, distinct bool, limit int64,
	cons datastore.ScanConsistency, vector timestamp.Vector, conn *datastore.IndexConnection) {
	defer close(conn.EntryChannel())

	// For primary indexes, bounds must always be strings, so we
	// can just enforce that directly

	viewOptions := map[string]interface{}{}
	viewOptions = generateViewOptions(cons, span) /*span.Range.Low, span.Range.High, span.Range.Inclusion) */
	viewRowChannel := make(chan cb.ViewRow)
	viewErrChannel := make(chan errors.Error)
	doneChannel := make(chan bool)
	defer close(doneChannel)

	go WalkViewInBatches(viewRowChannel, viewErrChannel, doneChannel, vi.keyspace.cbbucket,
		vi.DDocName(), vi.ViewName(), vi.IsPrimary(), viewOptions, _BATCH_SIZE, limit)

	var viewRow cb.ViewRow
	var err errors.Error
	sentRows := false
	ok := true
	numRows := 0
	errs := make([]error, 0, 10)
	for ok {
		select {
		case viewRow, ok = <-viewRowChannel:
			if ok {
				entry := datastore.IndexEntry{PrimaryKey: viewRow.ID}

				// try to add the view row key as the entry key (unless this is _all_docs)
				if vi.DDocName() != "" /* FIXME && vi.IsPrimary() == false */ {
					lookupValue, err := convertCouchbaseViewKeyToLookupValue(viewRow.Key)
					if err == nil {
						entry.EntryKey = lookupValue
					} else {
						errs = append(errs, fmt.Errorf("unable to convert index key to lookup value err:%v key %v", err, entry))
					}
				}
				select {
				case conn.EntryChannel() <- &entry:
					sentRows = true
					numRows++
				case <-conn.StopChannel():
					logging.Debugf(" Asked to stop after sending %v rows", numRows)
					ok = false
				}
			}
		case err, ok = <-viewErrChannel:
			if err != nil {
				logging.Errorf("%v", err)
				// check to possibly detect a bucket that was already deleted
				if !sentRows {
					logging.Debugf("Checking bucket URI: %v", vi.keyspace.cbbucket.URI)
					_, err := http.Get(vi.keyspace.cbbucket.URI)
					if err != nil {
						logging.Errorf("%v", err)

						// remove this specific bucket from the pool cache
						vi.keyspace.namespace.lock.Lock()
						delete(vi.keyspace.namespace.keyspaceCache, vi.keyspace.Name())
						vi.keyspace.namespace.lock.Unlock()
						// close this bucket
						vi.keyspace.Release()
						// ask the pool to refresh
						vi.keyspace.namespace.refresh(true)
						// bucket doesnt exist any more
						conn.Error(errors.NewCbViewsAccessError(nil, "keyspace "+vi.keyspace.Name()+" or view index missing"))
						return
					}

				}

				conn.Error(err)
				return
			}
		}
	}

	if errs != nil {
		logging.Debugf("Errors with converting lookup value to entry key. num errrs %v", len(errs))
	}

	logging.Debugf("Number of entries fetched from the index %d", numRows)

}
Exemplo n.º 10
0
func (b *keyspace) performOp(op int, inserts []datastore.Pair) ([]datastore.Pair, errors.Error) {

	if len(inserts) == 0 {
		return nil, nil
	}

	insertedKeys := make([]datastore.Pair, 0, len(inserts))
	var err error

	for _, kv := range inserts {
		key := kv.Key
		val := kv.Value.Actual()

		//mv := kv.Value.GetAttachment("meta")

		// TODO Need to also set meta
		switch op {

		case INSERT:
			var added bool
			// add the key to the backend
			added, err = b.cbbucket.Add(key, 0, val)
			if added == false {
				// false & err == nil => given key aready exists in the bucket
				if err != nil {
					err = errors.NewError(err, "Key "+key)
				} else {
					err = errors.NewError(nil, "Duplicate Key "+key)
				}
			}
		case UPDATE:
			// check if the key exists and if so then use the cas value
			// to update the key
			var meta map[string]interface{}
			var cas uint64
			var flags uint32

			an := kv.Value.(value.AnnotatedValue)
			meta = an.GetAttachment("meta").(map[string]interface{})

			cas, flags, err = getMeta(key, meta)
			if err != nil {
				// Don't perform the update if the meta values are not found
				logging.Errorf("Failed to get meta values for key %v, error %v", key, err)
			} else {

				logging.Debugf("CAS Value (Update) for key %v is %v flags %v value %v", key, uint64(cas), flags, val)
				_, err = b.cbbucket.CasWithMeta(key, int(flags), 0, uint64(cas), val)
			}

		case UPSERT:
			err = b.cbbucket.Set(key, 0, val)
		}

		if err != nil {
			if isEExistError(err) {
				logging.Errorf("Failed to perform update on key %s. CAS mismatch due to concurrent modifications", key)
			} else {
				logging.Errorf("Failed to perform %s on key %s for Keyspace %s Error %v", opToString(op), key, b.Name(), err)
			}
		} else {
			insertedKeys = append(insertedKeys, kv)
		}
	}

	if len(insertedKeys) == 0 {
		return nil, errors.NewCbDMLError(err, "Failed to perform "+opToString(op))
	}

	return insertedKeys, nil

}
Exemplo n.º 11
0
func (p *namespace) refresh(changed bool) {
	// trigger refresh of this pool
	logging.Debugf("Refreshing pool %s", p.name)

	newpool, err := p.site.client.GetPool(p.name)
	if err != nil {

		var client cb.Client

		logging.Errorf("Error updating pool name %s: Error %v", p.name, err)
		url := p.site.URL()

		/*
			transport := cbauth.WrapHTTPTransport(cb.HTTPTransport, nil)
			cb.HTTPClient.Transport = transport
		*/

		if p.site.CbAuthInit == true {
			client, err = cb.ConnectWithAuth(url, cbauth.NewAuthHandler(nil))
		} else {
			client, err = cb.Connect(url)
		}
		if err != nil {
			logging.Errorf("Error connecting to URL %s", url)
			return
		}
		// check if the default pool exists
		newpool, err = client.GetPool(p.name)
		if err != nil {
			logging.Errorf("Retry Failed Error updating pool name %s: Error %v", p.name, err)
			return
		}
		p.site.client = client

	}

	p.lock.Lock()
	defer p.lock.Unlock()
	for name, ks := range p.keyspaceCache {
		logging.Debugf(" Checking keyspace %s", name)
		newbucket, err := newpool.GetBucket(name)
		if err != nil {
			changed = true
			ks.(*keyspace).deleted = true
			logging.Errorf(" Error retrieving bucket %s", name)
			delete(p.keyspaceCache, name)

		} else if ks.(*keyspace).cbbucket.UUID != newbucket.UUID {

			logging.Debugf(" UUid of keyspace %v uuid now %v", ks.(*keyspace).cbbucket.UUID, newbucket.UUID)
			// UUID has changed. Update the keyspace struct with the newbucket
			ks.(*keyspace).cbbucket = newbucket
		}
		// Not deleted. Check if GSI indexer is available
		if ks.(*keyspace).gsiIndexer == nil {
			ks.(*keyspace).refreshIndexer(p.site.URL(), p.Name())
		}
	}

	if changed == true {
		p.setPool(newpool)
	}
}
Exemplo n.º 12
0
// NewSite creates a new Couchbase site for the given url.
func NewDatastore(u string) (s datastore.Datastore, e errors.Error) {

	var client cb.Client
	var cbAuthInit bool

	// try and initialize cbauth

	c, err := initCbAuth(u)
	if err != nil {
		logging.Errorf(" Unable to initialize cbauth. Error %v", err)
		url, err := url.Parse(u)
		if err != nil {
			return nil, errors.NewCbUrlParseError(err, "url "+u)
		}

		if url.User != nil {
			password, _ := url.User.Password()
			if password == "" {
				logging.Errorf("No password found in url %s", u)
			}

			// intialize cb_auth variables manually
			logging.Infof(" Trying to init cbauth with credentials %s %s", url.Host, url.User.Username())
			set, err := cbauth.InternalRetryDefaultInit(url.Host, url.User.Username(), password)
			if set == false || err != nil {
				logging.Errorf(" Unable to initialize cbauth variables. Error %v", err)
			} else {
				c, err = initCbAuth("http://" + url.Host)
				if err != nil {
					logging.Errorf("Unable to initliaze cbauth.  Error %v", err)
				} else {
					client = *c
					cbAuthInit = true
				}
			}
		}
	} else {
		client = *c
		cbAuthInit = true
	}

	if cbAuthInit == false {
		// connect without auth
		logging.Warnf("Unable to intialize cbAuth, access to couchbase buckets may be restricted")
		cb.HTTPClient = &http.Client{}
		client, err = cb.Connect(u)
		if err != nil {
			return nil, errors.NewCbConnectionError(err, "url "+u)
		}
	}

	site := &site{
		client:         client,
		namespaceCache: make(map[string]*namespace),
		CbAuthInit:     cbAuthInit,
	}

	// initialize the default pool.
	// TODO can couchbase server contain more than one pool ?

	defaultPool, Err := loadNamespace(site, "default")
	if Err != nil {
		logging.Errorf("Cannot connect to default pool")
		return nil, Err
	}

	site.namespaceCache["default"] = defaultPool
	logging.Infof("New site created with url %s", u)

	return site, nil
}