func ViewTotalRows(bucket *cb.Bucket, ddoc string, view string, options map[string]interface{}) (int64, errors.Error) { options["limit"] = 0 logURL, err := bucket.ViewURL(ddoc, view, options) if err == nil { logging.Debugf("Request View: %v", logURL) } vres, err := bucket.View(ddoc, view, options) if err != nil { return 0, errors.NewCbViewsAccessError(err, "View Name"+view) } return int64(vres.TotalRows), nil }
func WalkViewInBatches(result chan cb.ViewRow, errs chan errors.Error, stop chan bool, bucket *cb.Bucket, ddoc string, view string, isPrimary bool, options map[string]interface{}, batchSize int64, limit int64) { if limit != 0 && limit < batchSize { batchSize = limit } defer close(result) defer close(errs) defer func() { r := recover() if r != nil { logging.Errorf("View Walking Panic: %v\n%s", r, debug.Stack()) errs <- errors.NewCbViewsAccessError(nil, "Panic In walking view "+view) } }() options["limit"] = batchSize + 1 numRead := int64(0) numSent := int64(0) keysSent := map[string]bool{} ok := true for ok { logURL, err := bucket.ViewURL(ddoc, view, options) if err == nil { logging.Debugf("Request View: %v", logURL) } vres, err := bucket.View(ddoc, view, options) if err != nil { errs <- errors.NewCbViewsAccessError(err, "View name "+view) return } for i, row := range vres.Rows { // dont process the last row, its just used to see if we // need to continue processing if int64(i) < batchSize { // Send the row if its primary key has not been sent if isPrimary || !keysSent[row.ID] { select { case result <- row: numSent += 1 case <-stop: ok = false break } } // For non primary views, mark the row's primary key as sent if !isPrimary { keysSent[row.ID] = true } numRead += 1 } } if (int64(len(vres.Rows)) > batchSize) && (limit == 0 || (limit != 0 && numRead < limit)) { // prepare for next run skey := vres.Rows[batchSize].Key skeydocid := vres.Rows[batchSize].ID options["startkey"] = skey options["startkey_docid"] = cb.DocID(skeydocid) } else { // stop ok = false } } logging.Debugf("WalkViewInBatches %s: %d rows fetched, %d rows sent", view, numRead, numSent) }
func (vi *viewIndex) Scan(requestId string, span *datastore.Span, distinct bool, limit int64, cons datastore.ScanConsistency, vector timestamp.Vector, conn *datastore.IndexConnection) { defer close(conn.EntryChannel()) // For primary indexes, bounds must always be strings, so we // can just enforce that directly viewOptions := map[string]interface{}{} viewOptions = generateViewOptions(cons, span) /*span.Range.Low, span.Range.High, span.Range.Inclusion) */ viewRowChannel := make(chan cb.ViewRow) viewErrChannel := make(chan errors.Error) doneChannel := make(chan bool) defer close(doneChannel) go WalkViewInBatches(viewRowChannel, viewErrChannel, doneChannel, vi.keyspace.cbbucket, vi.DDocName(), vi.ViewName(), vi.IsPrimary(), viewOptions, _BATCH_SIZE, limit) var viewRow cb.ViewRow var err errors.Error sentRows := false ok := true numRows := 0 errs := make([]error, 0, 10) for ok { select { case viewRow, ok = <-viewRowChannel: if ok { entry := datastore.IndexEntry{PrimaryKey: viewRow.ID} // try to add the view row key as the entry key (unless this is _all_docs) if vi.DDocName() != "" /* FIXME && vi.IsPrimary() == false */ { lookupValue, err := convertCouchbaseViewKeyToLookupValue(viewRow.Key) if err == nil { entry.EntryKey = lookupValue } else { errs = append(errs, fmt.Errorf("unable to convert index key to lookup value err:%v key %v", err, entry)) } } select { case conn.EntryChannel() <- &entry: sentRows = true numRows++ case <-conn.StopChannel(): logging.Debugf(" Asked to stop after sending %v rows", numRows) ok = false } } case err, ok = <-viewErrChannel: if err != nil { logging.Errorf("%v", err) // check to possibly detect a bucket that was already deleted if !sentRows { logging.Debugf("Checking bucket URI: %v", vi.keyspace.cbbucket.URI) _, err := http.Get(vi.keyspace.cbbucket.URI) if err != nil { logging.Errorf("%v", err) // remove this specific bucket from the pool cache vi.keyspace.namespace.lock.Lock() delete(vi.keyspace.namespace.keyspaceCache, vi.keyspace.Name()) vi.keyspace.namespace.lock.Unlock() // close this bucket vi.keyspace.Release() // ask the pool to refresh vi.keyspace.namespace.refresh(true) // bucket doesnt exist any more conn.Error(errors.NewCbViewsAccessError(nil, "keyspace "+vi.keyspace.Name()+" or view index missing")) return } } conn.Error(err) return } } } if errs != nil { logging.Debugf("Errors with converting lookup value to entry key. num errrs %v", len(errs)) } logging.Debugf("Number of entries fetched from the index %d", numRows) }