func (udc *UpsideDownCouch) Update(doc *document.Document) (err error) { // do analysis before acquiring write lock analysisStart := time.Now() resultChan := make(chan *AnalysisResult) aw := AnalysisWork{ udc: udc, d: doc, rc: resultChan, } // put the work on the queue go func() { udc.analysisQueue.Queue(&aw) }() // wait for the result result := <-resultChan close(resultChan) atomic.AddUint64(&udc.stats.analysisTime, uint64(time.Since(analysisStart))) // start a writer for this update indexStart := time.Now() var kvwriter store.KVWriter kvwriter, err = udc.store.Writer() if err != nil { return } defer func() { if cerr := kvwriter.Close(); err == nil && cerr != nil { err = cerr } }() // first we lookup the backindex row for the doc id if it exists // lookup the back index row var backIndexRow *BackIndexRow backIndexRow, err = udc.backIndexRowForDoc(kvwriter, doc.ID) if err != nil { atomic.AddUint64(&udc.stats.errors, 1) return } // prepare a list of rows addRows := make([]UpsideDownCouchRow, 0) updateRows := make([]UpsideDownCouchRow, 0) deleteRows := make([]UpsideDownCouchRow, 0) addRows, updateRows, deleteRows = udc.mergeOldAndNew(backIndexRow, result.rows, addRows, updateRows, deleteRows) err = udc.batchRows(kvwriter, addRows, updateRows, deleteRows) if err == nil && backIndexRow == nil { udc.docCount++ } atomic.AddUint64(&udc.stats.indexTime, uint64(time.Since(indexStart))) if err == nil { atomic.AddUint64(&udc.stats.updates, 1) } else { atomic.AddUint64(&udc.stats.errors, 1) } return }
func (udc *UpsideDownCouch) DeleteInternal(key []byte) (err error) { internalRow := NewInternalRow(key, nil) var writer store.KVWriter writer, err = udc.store.Writer() if err != nil { return } defer func() { if cerr := writer.Close(); err == nil && cerr != nil { err = cerr } }() return writer.Delete(internalRow.Key()) }
func (udc *UpsideDownCouch) Open() (err error) { // install the merge operator udc.store.SetMergeOperator(&mergeOperator) // now open the kv store err = udc.store.Open() if err != nil { return } // start a writer for the open process var kvwriter store.KVWriter kvwriter, err = udc.store.Writer() if err != nil { return } defer func() { if cerr := kvwriter.Close(); err == nil && cerr != nil { err = cerr } }() var value []byte value, err = kvwriter.Get(VersionKey) if err != nil { return } // init new index OR load schema if value == nil { err = udc.init(kvwriter) if err != nil { return } } else { err = udc.loadSchema(kvwriter) if err != nil { return } } // set doc count udc.docCount, err = udc.countDocs(kvwriter) return }
func (udc *UpsideDownCouch) Delete(id string) (err error) { indexStart := time.Now() // start a writer for this delete var kvwriter store.KVWriter kvwriter, err = udc.store.Writer() if err != nil { return } defer func() { if cerr := kvwriter.Close(); err == nil && cerr != nil { err = cerr } }() // lookup the back index row var backIndexRow *BackIndexRow backIndexRow, err = udc.backIndexRowForDoc(kvwriter, id) if err != nil { atomic.AddUint64(&udc.stats.errors, 1) return } if backIndexRow == nil { atomic.AddUint64(&udc.stats.deletes, 1) return } deleteRows := make([]UpsideDownCouchRow, 0) deleteRows = udc.deleteSingle(id, backIndexRow, deleteRows) err = udc.batchRows(kvwriter, nil, nil, deleteRows) if err == nil { udc.docCount-- } atomic.AddUint64(&udc.stats.indexTime, uint64(time.Since(indexStart))) if err == nil { atomic.AddUint64(&udc.stats.deletes, 1) } else { atomic.AddUint64(&udc.stats.errors, 1) } return }
func (udc *UpsideDownCouch) batchRows(writer store.KVWriter, addRows []UpsideDownCouchRow, updateRows []UpsideDownCouchRow, deleteRows []UpsideDownCouchRow) (err error) { // prepare batch wb := writer.NewBatch() // add for _, row := range addRows { tfr, ok := row.(*TermFrequencyRow) if ok { // need to increment counter dictionaryKey := tfr.DictionaryRowKey() wb.Merge(dictionaryKey, dictionaryTermIncr) } wb.Set(row.Key(), row.Value()) } // update for _, row := range updateRows { wb.Set(row.Key(), row.Value()) } // delete for _, row := range deleteRows { tfr, ok := row.(*TermFrequencyRow) if ok { // need to decrement counter dictionaryKey := tfr.DictionaryRowKey() wb.Merge(dictionaryKey, dictionaryTermDecr) } wb.Delete(row.Key()) } // write out the batch err = wb.Execute() if err != nil { return } return }
func (udc *UpsideDownCouch) Batch(batch *index.Batch) (err error) { analysisStart := time.Now() resultChan := make(chan *AnalysisResult) var numUpdates uint64 for _, doc := range batch.IndexOps { if doc != nil { numUpdates++ } } go func() { for _, doc := range batch.IndexOps { if doc != nil { aw := AnalysisWork{ udc: udc, d: doc, rc: resultChan, } // put the work on the queue udc.analysisQueue.Queue(&aw) } } }() newRowsMap := make(map[string][]UpsideDownCouchRow) // wait for the result var itemsDeQueued uint64 for itemsDeQueued < numUpdates { result := <-resultChan newRowsMap[result.docID] = result.rows itemsDeQueued++ } close(resultChan) atomic.AddUint64(&udc.stats.analysisTime, uint64(time.Since(analysisStart))) indexStart := time.Now() // start a writer for this batch var kvwriter store.KVWriter kvwriter, err = udc.store.Writer() if err != nil { return } defer func() { if cerr := kvwriter.Close(); err == nil && cerr != nil { err = cerr } }() // first lookup all the back index rows var backIndexRows map[string]*BackIndexRow backIndexRows, err = udc.backIndexRowsForBatch(kvwriter, batch) if err != nil { return } // prepare a list of rows addRows := make([]UpsideDownCouchRow, 0) updateRows := make([]UpsideDownCouchRow, 0) deleteRows := make([]UpsideDownCouchRow, 0) docsAdded := uint64(0) docsDeleted := uint64(0) for docID, doc := range batch.IndexOps { backIndexRow := backIndexRows[docID] if doc == nil && backIndexRow != nil { // delete deleteRows = udc.deleteSingle(docID, backIndexRow, deleteRows) docsDeleted++ } else if doc != nil { addRows, updateRows, deleteRows = udc.mergeOldAndNew(backIndexRow, newRowsMap[docID], addRows, updateRows, deleteRows) if backIndexRow == nil { docsAdded++ } } } // add the internal ops for internalKey, internalValue := range batch.InternalOps { if internalValue == nil { // delete deleteInternalRow := NewInternalRow([]byte(internalKey), nil) deleteRows = append(deleteRows, deleteInternalRow) } else { updateInternalRow := NewInternalRow([]byte(internalKey), internalValue) updateRows = append(updateRows, updateInternalRow) } } err = udc.batchRows(kvwriter, addRows, updateRows, deleteRows) atomic.AddUint64(&udc.stats.indexTime, uint64(time.Since(indexStart))) if err == nil { udc.docCount += docsAdded udc.docCount -= docsDeleted atomic.AddUint64(&udc.stats.updates, numUpdates) atomic.AddUint64(&udc.stats.deletes, docsDeleted) atomic.AddUint64(&udc.stats.batches, 1) } else { atomic.AddUint64(&udc.stats.errors, 1) } return }