Beispiel #1
0
func (im *IndexMapping) mapDocument(doc *document.Document, data interface{}) error {
	// see if the top level object is a byte array, and possibly run through a converter
	byteArrayData, ok := data.([]byte)
	if ok {
		byteArrayConverterConstructor := registry.ByteArrayConverterByName(im.ByteArrayConverter)
		if byteArrayConverterConstructor != nil {
			byteArrayConverter, err := byteArrayConverterConstructor(nil, nil)
			if err == nil {
				convertedData, err := byteArrayConverter.Convert(byteArrayData)
				if err != nil {
					return err
				}
				data = convertedData
			} else {
				logger.Printf("error creating byte array converter: %v", err)
			}
		} else {
			logger.Printf("no byte array converter named: %s", im.ByteArrayConverter)
		}
	}

	docType := im.determineType(data)
	docMapping := im.mappingForType(docType)
	walkContext := im.newWalkContext(doc, docMapping)
	docMapping.walkDocument(data, []string{}, []uint64{}, walkContext)

	// see if the _all field was disabled
	allMapping := docMapping.documentMappingForPath("_all")
	if allMapping == nil || (allMapping.Enabled != false) {
		field := document.NewCompositeFieldWithIndexingOptions("_all", true, []string{}, walkContext.excludedFromAll, document.IndexField|document.IncludeTermVectors)
		doc.AddField(field)
	}

	return nil
}
Beispiel #2
0
func (f *Firestorm) Update(doc *document.Document) (err error) {

	// assign this document a number
	doc.Number = atomic.AddUint64(&f.highDocNumber, 1)

	// do analysis before acquiring write lock
	analysisStart := time.Now()
	resultChan := make(chan *index.AnalysisResult)
	aw := index.NewAnalysisWork(f, doc, resultChan)

	// put the work on the queue
	f.analysisQueue.Queue(aw)

	// wait for the result
	result := <-resultChan
	close(resultChan)
	atomic.AddUint64(&f.stats.analysisTime, uint64(time.Since(analysisStart)))

	// start a writer for this update
	indexStart := time.Now()
	var kvwriter store.KVWriter
	kvwriter, err = f.store.Writer()
	if err != nil {
		return
	}
	defer func() {
		if cerr := kvwriter.Close(); err == nil && cerr != nil {
			err = cerr
		}
	}()

	var dictionaryDeltas map[string]int64
	dictionaryDeltas, err = f.batchRows(kvwriter, [][]index.IndexRow{result.Rows}, nil)
	if err != nil {
		_ = kvwriter.Close()
		atomic.AddUint64(&f.stats.errors, 1)
		return
	}

	f.compensator.Mutate([]byte(doc.ID), doc.Number)
	f.lookuper.NotifyBatch([]*InFlightItem{&InFlightItem{[]byte(doc.ID), doc.Number}})
	f.dictUpdater.NotifyBatch(dictionaryDeltas)

	atomic.AddUint64(&f.stats.indexTime, uint64(time.Since(indexStart)))
	return
}