func (udc *UpsideDownCouch) Update(doc *document.Document) (err error) {
// do analysis before acquiring write lock
analysisStart := time.Now()
resultChan := make(chan *AnalysisResult)
aw := AnalysisWork{
udc: udc,
d: doc,
rc: resultChan,
}
// put the work on the queue
go func() {
udc.analysisQueue.Queue(&aw)
}()
// wait for the result
result := <-resultChan
close(resultChan)
atomic.AddUint64(&udc.stats.analysisTime, uint64(time.Since(analysisStart)))
// start a writer for this update
indexStart := time.Now()
var kvwriter store.KVWriter
kvwriter, err = udc.store.Writer()
if err != nil {
return
}
defer func() {
if cerr := kvwriter.Close(); err == nil && cerr != nil {
err = cerr
}
}()
// first we lookup the backindex row for the doc id if it exists
// lookup the back index row
var backIndexRow *BackIndexRow
backIndexRow, err = udc.backIndexRowForDoc(kvwriter, doc.ID)
if err != nil {
atomic.AddUint64(&udc.stats.errors, 1)
return
}
// prepare a list of rows
addRows := make([]UpsideDownCouchRow, 0)
updateRows := make([]UpsideDownCouchRow, 0)
deleteRows := make([]UpsideDownCouchRow, 0)
addRows, updateRows, deleteRows = udc.mergeOldAndNew(backIndexRow, result.rows, addRows, updateRows, deleteRows)
err = udc.batchRows(kvwriter, addRows, updateRows, deleteRows)
if err == nil && backIndexRow == nil {
udc.docCount++
}
atomic.AddUint64(&udc.stats.indexTime, uint64(time.Since(indexStart)))
if err == nil {
atomic.AddUint64(&udc.stats.updates, 1)
} else {
atomic.AddUint64(&udc.stats.errors, 1)
}
return
}
func (udc *UpsideDownCouch) DeleteInternal(key []byte) (err error) {
internalRow := NewInternalRow(key, nil)
var writer store.KVWriter
writer, err = udc.store.Writer()
if err != nil {
return
}
defer func() {
if cerr := writer.Close(); err == nil && cerr != nil {
err = cerr
}
}()
return writer.Delete(internalRow.Key())
}
func (udc *UpsideDownCouch) Open() (err error) {
// install the merge operator
udc.store.SetMergeOperator(&mergeOperator)
// now open the kv store
err = udc.store.Open()
if err != nil {
return
}
// start a writer for the open process
var kvwriter store.KVWriter
kvwriter, err = udc.store.Writer()
if err != nil {
return
}
defer func() {
if cerr := kvwriter.Close(); err == nil && cerr != nil {
err = cerr
}
}()
var value []byte
value, err = kvwriter.Get(VersionKey)
if err != nil {
return
}
// init new index OR load schema
if value == nil {
err = udc.init(kvwriter)
if err != nil {
return
}
} else {
err = udc.loadSchema(kvwriter)
if err != nil {
return
}
}
// set doc count
udc.docCount, err = udc.countDocs(kvwriter)
return
}
func (udc *UpsideDownCouch) Delete(id string) (err error) {
indexStart := time.Now()
// start a writer for this delete
var kvwriter store.KVWriter
kvwriter, err = udc.store.Writer()
if err != nil {
return
}
defer func() {
if cerr := kvwriter.Close(); err == nil && cerr != nil {
err = cerr
}
}()
// lookup the back index row
var backIndexRow *BackIndexRow
backIndexRow, err = udc.backIndexRowForDoc(kvwriter, id)
if err != nil {
atomic.AddUint64(&udc.stats.errors, 1)
return
}
if backIndexRow == nil {
atomic.AddUint64(&udc.stats.deletes, 1)
return
}
deleteRows := make([]UpsideDownCouchRow, 0)
deleteRows = udc.deleteSingle(id, backIndexRow, deleteRows)
err = udc.batchRows(kvwriter, nil, nil, deleteRows)
if err == nil {
udc.docCount--
}
atomic.AddUint64(&udc.stats.indexTime, uint64(time.Since(indexStart)))
if err == nil {
atomic.AddUint64(&udc.stats.deletes, 1)
} else {
atomic.AddUint64(&udc.stats.errors, 1)
}
return
}
func (udc *UpsideDownCouch) batchRows(writer store.KVWriter, addRows []UpsideDownCouchRow, updateRows []UpsideDownCouchRow, deleteRows []UpsideDownCouchRow) (err error) {
// prepare batch
wb := writer.NewBatch()
// add
for _, row := range addRows {
tfr, ok := row.(*TermFrequencyRow)
if ok {
// need to increment counter
dictionaryKey := tfr.DictionaryRowKey()
wb.Merge(dictionaryKey, dictionaryTermIncr)
}
wb.Set(row.Key(), row.Value())
}
// update
for _, row := range updateRows {
wb.Set(row.Key(), row.Value())
}
// delete
for _, row := range deleteRows {
tfr, ok := row.(*TermFrequencyRow)
if ok {
// need to decrement counter
dictionaryKey := tfr.DictionaryRowKey()
wb.Merge(dictionaryKey, dictionaryTermDecr)
}
wb.Delete(row.Key())
}
// write out the batch
err = wb.Execute()
if err != nil {
return
}
return
}
func (udc *UpsideDownCouch) Batch(batch *index.Batch) (err error) {
analysisStart := time.Now()
resultChan := make(chan *AnalysisResult)
var numUpdates uint64
for _, doc := range batch.IndexOps {
if doc != nil {
numUpdates++
}
}
go func() {
for _, doc := range batch.IndexOps {
if doc != nil {
aw := AnalysisWork{
udc: udc,
d: doc,
rc: resultChan,
}
// put the work on the queue
udc.analysisQueue.Queue(&aw)
}
}
}()
newRowsMap := make(map[string][]UpsideDownCouchRow)
// wait for the result
var itemsDeQueued uint64
for itemsDeQueued < numUpdates {
result := <-resultChan
newRowsMap[result.docID] = result.rows
itemsDeQueued++
}
close(resultChan)
atomic.AddUint64(&udc.stats.analysisTime, uint64(time.Since(analysisStart)))
indexStart := time.Now()
// start a writer for this batch
var kvwriter store.KVWriter
kvwriter, err = udc.store.Writer()
if err != nil {
return
}
defer func() {
if cerr := kvwriter.Close(); err == nil && cerr != nil {
err = cerr
}
}()
// first lookup all the back index rows
var backIndexRows map[string]*BackIndexRow
backIndexRows, err = udc.backIndexRowsForBatch(kvwriter, batch)
if err != nil {
return
}
// prepare a list of rows
addRows := make([]UpsideDownCouchRow, 0)
updateRows := make([]UpsideDownCouchRow, 0)
deleteRows := make([]UpsideDownCouchRow, 0)
docsAdded := uint64(0)
docsDeleted := uint64(0)
for docID, doc := range batch.IndexOps {
backIndexRow := backIndexRows[docID]
if doc == nil && backIndexRow != nil {
// delete
deleteRows = udc.deleteSingle(docID, backIndexRow, deleteRows)
docsDeleted++
} else if doc != nil {
addRows, updateRows, deleteRows = udc.mergeOldAndNew(backIndexRow, newRowsMap[docID], addRows, updateRows, deleteRows)
if backIndexRow == nil {
docsAdded++
}
}
}
// add the internal ops
for internalKey, internalValue := range batch.InternalOps {
if internalValue == nil {
// delete
deleteInternalRow := NewInternalRow([]byte(internalKey), nil)
deleteRows = append(deleteRows, deleteInternalRow)
} else {
updateInternalRow := NewInternalRow([]byte(internalKey), internalValue)
updateRows = append(updateRows, updateInternalRow)
}
}
err = udc.batchRows(kvwriter, addRows, updateRows, deleteRows)
atomic.AddUint64(&udc.stats.indexTime, uint64(time.Since(indexStart)))
if err == nil {
udc.docCount += docsAdded
udc.docCount -= docsDeleted
atomic.AddUint64(&udc.stats.updates, numUpdates)
atomic.AddUint64(&udc.stats.deletes, docsDeleted)
atomic.AddUint64(&udc.stats.batches, 1)
} else {
atomic.AddUint64(&udc.stats.errors, 1)
}
return
}
