func (self *Shard) executeQueryForSeries(querySpec *parser.QuerySpec, seriesName string, columns []string, processor engine.Processor) error { fields, err := self.getFieldsForSeries(querySpec.Database(), seriesName, columns) if err != nil { log.Error("Error looking up fields for %s: %s", seriesName, err) return err } if querySpec.IsSinglePointQuery() { log.Debug("Running single query for series %s, fields %v", seriesName, fields) return self.executeSinglePointQuery(querySpec, seriesName, fields, processor) } startTime := querySpec.GetStartTime() endTime := querySpec.GetEndTime() query := querySpec.SelectQuery() aliases := query.GetTableAliases(seriesName) fieldNames, iterators := self.getIterators(fields, startTime, endTime, query.Ascending) seriesOutgoing := &protocol.Series{Name: protocol.String(seriesName), Fields: fieldNames, Points: make([]*protocol.Point, 0, self.pointBatchSize)} pi := NewPointIterator(iterators, fields, querySpec.GetStartTime(), querySpec.GetEndTime(), query.Ascending) defer pi.Close() for pi.Valid() { p := pi.Point() seriesOutgoing.Points = append(seriesOutgoing.Points, p) if len(seriesOutgoing.Points) >= self.pointBatchSize { ok, err := yieldToProcessor(seriesOutgoing, processor, aliases) if !ok || err != nil { log.Debug("Stopping processing.") if err != nil { log.Error("Error while processing data: %v", err) return err } } seriesOutgoing = &protocol.Series{Name: protocol.String(seriesName), Fields: fieldNames, Points: make([]*protocol.Point, 0, self.pointBatchSize)} } pi.Next() } if err := pi.Error(); err != nil { return err } //Yield remaining data if ok, err := yieldToProcessor(seriesOutgoing, processor, aliases); !ok || err != nil { log.Debug("Stopping processing remaining points...") if err != nil { log.Error("Error while processing data: %v", err) return err } } log.Debug("Finished running query %s", query.GetQueryString()) return nil }
func (self *Shard) getPointIteratorForSeries(querySpec *parser.QuerySpec, name string, columns []string) ([]string, *PointIterator, error) { fields, err := self.getFieldsForSeries(querySpec.Database(), name, columns) if err != nil { log.Error("Error looking up fields for %s: %s", name, err) return nil, nil, err } startTime := querySpec.GetStartTime() endTime := querySpec.GetEndTime() query := querySpec.SelectQuery() iterators := self.getIterators(fields, startTime, endTime, query.Ascending) pi := NewPointIterator(iterators, fields, querySpec.GetStartTime(), querySpec.GetEndTime(), query.Ascending) columns = make([]string, len(fields)) for i := range fields { columns[i] = fields[i].Name } return columns, pi, nil }
func (self *Shard) executeQueryForSeries(querySpec *parser.QuerySpec, seriesName string, columns []string, processor cluster.QueryProcessor) error { startTimeBytes := self.byteArrayForTime(querySpec.GetStartTime()) endTimeBytes := self.byteArrayForTime(querySpec.GetEndTime()) fields, err := self.getFieldsForSeries(querySpec.Database(), seriesName, columns) if err != nil { log.Error("Error looking up fields for %s: %s", seriesName, err) return err } fieldCount := len(fields) rawColumnValues := make([]rawColumnValue, fieldCount, fieldCount) query := querySpec.SelectQuery() aliases := query.GetTableAliases(seriesName) if querySpec.IsSinglePointQuery() { series, err := self.fetchSinglePoint(querySpec, seriesName, fields) if err != nil { log.Error("Error reading a single point: %s", err) return err } if len(series.Points) > 0 { processor.YieldPoint(series.Name, series.Fields, series.Points[0]) } return nil } fieldNames, iterators := self.getIterators(fields, startTimeBytes, endTimeBytes, query.Ascending) defer func() { for _, it := range iterators { it.Close() } }() seriesOutgoing := &protocol.Series{Name: protocol.String(seriesName), Fields: fieldNames, Points: make([]*protocol.Point, 0, self.pointBatchSize)} // TODO: clean up, this is super gnarly // optimize for the case where we're pulling back only a single column or aggregate buffer := bytes.NewBuffer(nil) valueBuffer := proto.NewBuffer(nil) for { isValid := false point := &protocol.Point{Values: make([]*protocol.FieldValue, fieldCount, fieldCount)} for i, it := range iterators { if rawColumnValues[i].value != nil || !it.Valid() { if err := it.Error(); err != nil { return err } continue } key := it.Key() if len(key) < 16 { continue } if !isPointInRange(fields[i].IdAsBytes(), startTimeBytes, endTimeBytes, key) { continue } value := it.Value() sequenceNumber := key[16:] rawTime := key[8:16] rawColumnValues[i] = rawColumnValue{time: rawTime, sequence: sequenceNumber, value: value} } var pointTimeRaw []byte var pointSequenceRaw []byte // choose the highest (or lowest in case of ascending queries) timestamp // and sequence number. that will become the timestamp and sequence of // the next point. for _, value := range rawColumnValues { if value.value == nil { continue } pointTimeRaw, pointSequenceRaw = value.updatePointTimeAndSequence(pointTimeRaw, pointSequenceRaw, query.Ascending) } for i, iterator := range iterators { // if the value is nil or doesn't match the point's timestamp and sequence number // then skip it if rawColumnValues[i].value == nil || !bytes.Equal(rawColumnValues[i].time, pointTimeRaw) || !bytes.Equal(rawColumnValues[i].sequence, pointSequenceRaw) { point.Values[i] = &protocol.FieldValue{IsNull: &TRUE} continue } // if we emitted at lease one column, then we should keep // trying to get more points isValid = true // advance the iterator to read a new value in the next iteration if query.Ascending { iterator.Next() } else { iterator.Prev() } fv := &protocol.FieldValue{} valueBuffer.SetBuf(rawColumnValues[i].value) err := valueBuffer.Unmarshal(fv) if err != nil { log.Error("Error while running query: %s", err) return err } point.Values[i] = fv rawColumnValues[i].value = nil } var sequence uint64 var t uint64 // set the point sequence number and timestamp buffer.Reset() buffer.Write(pointSequenceRaw) binary.Read(buffer, binary.BigEndian, &sequence) buffer.Reset() buffer.Write(pointTimeRaw) binary.Read(buffer, binary.BigEndian, &t) time := self.convertUintTimestampToInt64(&t) point.SetTimestampInMicroseconds(time) point.SequenceNumber = &sequence // stop the loop if we ran out of points if !isValid { break } shouldContinue := true seriesOutgoing.Points = append(seriesOutgoing.Points, point) if len(seriesOutgoing.Points) >= self.pointBatchSize { for _, alias := range aliases { series := &protocol.Series{ Name: proto.String(alias), Fields: fieldNames, Points: seriesOutgoing.Points, } if !processor.YieldSeries(series) { log.Info("Stopping processing") shouldContinue = false } } seriesOutgoing = &protocol.Series{Name: protocol.String(seriesName), Fields: fieldNames, Points: make([]*protocol.Point, 0, self.pointBatchSize)} } if !shouldContinue { break } } //Yield remaining data for _, alias := range aliases { log.Debug("Final Flush %s", alias) series := &protocol.Series{Name: protocol.String(alias), Fields: seriesOutgoing.Fields, Points: seriesOutgoing.Points} if !processor.YieldSeries(series) { log.Debug("Cancelled...") } } log.Debug("Finished running query %s", query.GetQueryString()) return nil }