func BenchmarkParsePointWithPrecisionU(b *testing.B) { line := `cpu value=1i 1000000000` defaultTime := time.Now().UTC() for i := 0; i < b.N; i++ { models.ParsePointsWithPrecision([]byte(line), defaultTime, "u") b.SetBytes(int64(len(line))) } }
// MustWritePointsString parses the line protocol (with second precision) and // inserts the resulting points into the shard. Panic on error. func (sh *Shard) MustWritePointsString(s string) { a, err := models.ParsePointsWithPrecision([]byte(strings.TrimSpace(s)), time.Time{}, "s") if err != nil { panic(err) } if err := sh.WritePoints(a); err != nil { panic(err) } }
func TestParsePointsWithPrecisionComments(t *testing.T) { tests := []struct { name string batch string exp string lenPoints int }{ { name: "comment only", batch: `# comment only`, exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345", lenPoints: 0, }, { name: "point with comment above", batch: `# a point is below cpu,host=serverA,region=us-east value=1.0 946730096789012345`, exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345", lenPoints: 1, }, { name: "point with comment below", batch: `cpu,host=serverA,region=us-east value=1.0 946730096789012345 # end of points`, exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345", lenPoints: 1, }, { name: "indented comment", batch: ` # a point is below cpu,host=serverA,region=us-east value=1.0 946730096789012345`, exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345", lenPoints: 1, }, } for _, test := range tests { pts, err := models.ParsePointsWithPrecision([]byte(test.batch), time.Now().UTC(), "") if err != nil { t.Fatalf(`%s: ParsePoints() failed. got %s`, test.name, err) } pointsLength := len(pts) if exp := test.lenPoints; pointsLength != exp { t.Errorf("%s: ParsePoint() len mismatch: got %v, exp %v", test.name, pointsLength, exp) } if pointsLength > 0 { pt := pts[0] got := pt.String() if got != test.exp { t.Errorf("%s: ParsePoint() to string mismatch:\n got %v\n exp %v", test.name, got, test.exp) } } } }
// MustWriteToShardString parses the line protocol (with second precision) and // inserts the resulting points into a shard. Panic on error. func (s *Store) MustWriteToShardString(shardID int, data ...string) { var points []models.Point for i := range data { a, err := models.ParsePointsWithPrecision([]byte(strings.TrimSpace(data[i])), time.Time{}, "s") if err != nil { panic(err) } points = append(points, a...) } if err := s.WriteToShard(uint64(shardID), points); err != nil { panic(err) } }
// serveWriteLine receives incoming series data in line protocol format and writes it to the database. func (h *Handler) serveWriteLine(w http.ResponseWriter, r *http.Request, body []byte, user auth.User) { precision := r.FormValue("precision") if precision == "" { precision = "n" } points, err := models.ParsePointsWithPrecision(body, time.Now().UTC(), precision) if err != nil { if err.Error() == "EOF" { w.WriteHeader(http.StatusOK) return } h.writeError(w, influxql.Result{Err: err}, http.StatusBadRequest) return } database := r.FormValue("db") if database == "" { h.writeError(w, influxql.Result{Err: fmt.Errorf("database is required")}, http.StatusBadRequest) return } action := auth.Action{ Resource: auth.DatabaseResource(database), Privilege: auth.WritePrivilege, } if err := user.AuthorizeAction(action); err != nil { h.writeError(w, influxql.Result{Err: fmt.Errorf("%q user is not authorized to write to database %q", user.Name(), database)}, http.StatusUnauthorized) return } // Write points. if err := h.PointsWriter.WritePoints( database, r.FormValue("rp"), models.ConsistencyLevelAll, points, ); influxdb.IsClientError(err) { h.statMap.Add(statPointsWrittenFail, int64(len(points))) h.writeError(w, influxql.Result{Err: err}, http.StatusBadRequest) return } else if err != nil { h.statMap.Add(statPointsWrittenFail, int64(len(points))) h.writeError(w, influxql.Result{Err: err}, http.StatusInternalServerError) return } h.statMap.Add(statPointsWrittenOK, int64(len(points))) w.WriteHeader(http.StatusNoContent) }
func test(t *testing.T, line string, point TestPoint) { pts, err := models.ParsePointsWithPrecision([]byte(line), time.Unix(0, 0), "n") if err != nil { t.Fatalf(`ParsePoints("%s") mismatch. got %v, exp nil`, line, err) } if exp := 1; len(pts) != exp { t.Fatalf(`ParsePoints("%s") len mismatch. got %d, exp %d`, line, len(pts), exp) } if exp := point.Key(); !bytes.Equal(pts[0].Key(), exp) { t.Errorf("ParsePoints(\"%s\") key mismatch.\ngot %v\nexp %v", line, string(pts[0].Key()), string(exp)) } if exp := len(point.Tags()); len(pts[0].Tags()) != exp { t.Errorf(`ParsePoints("%s") tags mismatch. got %v, exp %v`, line, pts[0].Tags(), exp) } for tag, value := range pts[0].Tags() { if value != point.RawTags[tag] { t.Errorf(`ParsePoints("%s") tags mismatch. got %v, exp %v`, line, value, point.RawTags[tag]) } } for name, value := range point.RawFields { val := pts[0].Fields()[name] expfval, ok := val.(float64) if ok && math.IsNaN(expfval) { gotfval, ok := value.(float64) if ok && !math.IsNaN(gotfval) { t.Errorf(`ParsePoints("%s") field '%s' mismatch. exp NaN`, line, name) } } else if !reflect.DeepEqual(pts[0].Fields()[name], value) { t.Errorf(`ParsePoints("%s") field '%s' mismatch. got %[3]v (%[3]T), exp %[4]v (%[4]T)`, line, name, pts[0].Fields()[name], value) } } if !pts[0].Time().Equal(point.Time()) { t.Errorf(`ParsePoints("%s") time mismatch. got %v, exp %v`, line, pts[0].Time(), point.Time()) } if !strings.HasPrefix(pts[0].String(), line) { t.Errorf("ParsePoints string mismatch.\ngot: %v\nexp: %v", pts[0].String(), line) } }
func (c *commune) point(precision string) models.Point { pt := []byte(<-c.ch) p, err := models.ParsePointsWithPrecision(pt, time.Now().UTC(), precision) if err != nil { log.Fatalf("Error parsing point for commune\n point: %v\n error: %v\n", pt, err) } if len(p) == 0 { return c.storedPoint } c.storedPoint = p[0] return p[0] }
func (p *InfluxParser) ParseWithDefaultTime(buf []byte, t time.Time) ([]telegraf.Metric, error) { // parse even if the buffer begins with a newline buf = bytes.TrimPrefix(buf, []byte("\n")) points, err := models.ParsePointsWithPrecision(buf, t, "n") metrics := make([]telegraf.Metric, len(points)) for i, point := range points { for k, v := range p.DefaultTags { // only set the default tag if it doesn't already exist: if tmp := point.Tags().GetString(k); tmp == "" { point.AddTag(k, v) } } // Ignore error here because it's impossible that a model.Point // wouldn't parse into client.Point properly metrics[i] = telegraf.NewMetricFromPoint(point) } return metrics, err }
func (s *Service) parser() { defer s.wg.Done() for { select { case <-s.done: return case buf := <-s.parserChan: points, err := models.ParsePointsWithPrecision(buf, time.Now().UTC(), s.config.Precision) if err != nil { s.statMap.Add(statPointsParseFail, 1) s.Logger.Printf("Failed to parse points: %s", err) continue } for _, point := range points { s.batcher.In() <- point } s.statMap.Add(statPointsReceived, int64(len(points))) } } }
func readPointsFromIO(data io.ReadCloser, points chan<- models.Point, precision string) error { defer data.Close() defer close(points) now := time.Time{} in := bufio.NewScanner(data) for in.Scan() { db := in.Text() if !in.Scan() { return fmt.Errorf("invalid replay file format, expected another line") } rp := in.Text() if !in.Scan() { return fmt.Errorf("invalid replay file format, expected another line") } mps, err := dbmodels.ParsePointsWithPrecision( in.Bytes(), now, precision, ) if err != nil { return err } mp := mps[0] p := models.Point{ Database: db, RetentionPolicy: rp, Name: mp.Name(), Group: models.NilGroup, Tags: models.Tags(mp.Tags().Map()), Fields: models.Fields(mp.Fields()), Time: mp.Time().UTC(), } points <- p } return nil }
// serveWrite receives incoming series data in line protocol format and writes it to the database. func (h *Handler) serveWrite(w http.ResponseWriter, r *http.Request, user *meta.UserInfo) { atomic.AddInt64(&h.stats.WriteRequests, 1) atomic.AddInt64(&h.stats.ActiveWriteRequests, 1) defer func(start time.Time) { atomic.AddInt64(&h.stats.ActiveWriteRequests, -1) atomic.AddInt64(&h.stats.WriteRequestDuration, time.Since(start).Nanoseconds()) }(time.Now()) database := r.URL.Query().Get("db") if database == "" { h.resultError(w, influxql.Result{Err: fmt.Errorf("database is required")}, http.StatusBadRequest) return } if di := h.MetaClient.Database(database); di == nil { h.resultError(w, influxql.Result{Err: fmt.Errorf("database not found: %q", database)}, http.StatusNotFound) return } if h.Config.AuthEnabled && user == nil { h.resultError(w, influxql.Result{Err: fmt.Errorf("user is required to write to database %q", database)}, http.StatusUnauthorized) return } if h.Config.AuthEnabled { if err := h.WriteAuthorizer.AuthorizeWrite(user.Name, database); err != nil { h.resultError(w, influxql.Result{Err: fmt.Errorf("%q user is not authorized to write to database %q", user.Name, database)}, http.StatusUnauthorized) return } } // Handle gzip decoding of the body body := r.Body if r.Header.Get("Content-Encoding") == "gzip" { b, err := gzip.NewReader(r.Body) if err != nil { h.resultError(w, influxql.Result{Err: err}, http.StatusBadRequest) return } defer b.Close() body = b } var bs []byte if clStr := r.Header.Get("Content-Length"); clStr != "" { if length, err := strconv.Atoi(clStr); err == nil { // This will just be an initial hint for the gzip reader, as the // bytes.Buffer will grow as needed when ReadFrom is called bs = make([]byte, 0, length) } } buf := bytes.NewBuffer(bs) _, err := buf.ReadFrom(body) if err != nil { if h.Config.WriteTracing { h.Logger.Print("Write handler unable to read bytes from request body") } h.resultError(w, influxql.Result{Err: err}, http.StatusBadRequest) return } atomic.AddInt64(&h.stats.WriteRequestBytesReceived, int64(buf.Len())) if h.Config.WriteTracing { h.Logger.Printf("Write body received by handler: %s", buf.Bytes()) } points, parseError := models.ParsePointsWithPrecision(buf.Bytes(), time.Now().UTC(), r.URL.Query().Get("precision")) // Not points parsed correctly so return the error now if parseError != nil && len(points) == 0 { if parseError.Error() == "EOF" { h.writeHeader(w, http.StatusOK) return } h.resultError(w, influxql.Result{Err: parseError}, http.StatusBadRequest) return } // Determine required consistency level. level := r.URL.Query().Get("consistency") consistency := models.ConsistencyLevelOne if level != "" { var err error consistency, err = models.ParseConsistencyLevel(level) if err != nil { h.resultError(w, influxql.Result{Err: err}, http.StatusBadRequest) return } } // Write points. if err := h.PointsWriter.WritePoints(database, r.URL.Query().Get("rp"), consistency, points); influxdb.IsClientError(err) { atomic.AddInt64(&h.stats.PointsWrittenFail, int64(len(points))) h.resultError(w, influxql.Result{Err: err}, http.StatusBadRequest) return } else if err != nil { atomic.AddInt64(&h.stats.PointsWrittenFail, int64(len(points))) h.resultError(w, influxql.Result{Err: err}, http.StatusInternalServerError) return } else if parseError != nil { // We wrote some of the points atomic.AddInt64(&h.stats.PointsWrittenOK, int64(len(points))) // The other points failed to parse which means the client sent invalid line protocol. We return a 400 // response code as well as the lines that failed to parse. h.resultError(w, influxql.Result{Err: fmt.Errorf("partial write:\n%v", parseError)}, http.StatusBadRequest) return } atomic.AddInt64(&h.stats.PointsWrittenOK, int64(len(points))) h.writeHeader(w, http.StatusNoContent) }
func TestParsePointsWithPrecisionNoTime(t *testing.T) { line := `cpu,host=serverA,region=us-east value=1.0` tm, _ := time.Parse(time.RFC3339Nano, "2000-01-01T12:34:56.789012345Z") tests := []struct { name string precision string exp string }{ { name: "no precision", precision: "", exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345", }, { name: "nanosecond precision", precision: "n", exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345", }, { name: "microsecond precision", precision: "u", exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012000", }, { name: "millisecond precision", precision: "ms", exp: "cpu,host=serverA,region=us-east value=1.0 946730096789000000", }, { name: "second precision", precision: "s", exp: "cpu,host=serverA,region=us-east value=1.0 946730096000000000", }, { name: "minute precision", precision: "m", exp: "cpu,host=serverA,region=us-east value=1.0 946730040000000000", }, { name: "hour precision", precision: "h", exp: "cpu,host=serverA,region=us-east value=1.0 946728000000000000", }, } for _, test := range tests { pts, err := models.ParsePointsWithPrecision([]byte(line), tm, test.precision) if err != nil { t.Fatalf(`%s: ParsePoints() failed. got %s`, test.name, err) } if exp := 1; len(pts) != exp { t.Errorf("%s: ParsePoint() len mismatch: got %v, exp %v", test.name, len(pts), exp) } pt := pts[0] got := pt.String() if got != test.exp { t.Errorf("%s: ParsePoint() to string mismatch:\n got %v\n exp %v", test.name, got, test.exp) } } }
func TestParsePointsWithPrecision(t *testing.T) { tests := []struct { name string line string precision string exp string }{ { name: "nanosecond by default", line: `cpu,host=serverA,region=us-east value=1.0 946730096789012345`, precision: "", exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345", }, { name: "nanosecond", line: `cpu,host=serverA,region=us-east value=1.0 946730096789012345`, precision: "n", exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345", }, { name: "microsecond", line: `cpu,host=serverA,region=us-east value=1.0 946730096789012`, precision: "u", exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012000", }, { name: "millisecond", line: `cpu,host=serverA,region=us-east value=1.0 946730096789`, precision: "ms", exp: "cpu,host=serverA,region=us-east value=1.0 946730096789000000", }, { name: "second", line: `cpu,host=serverA,region=us-east value=1.0 946730096`, precision: "s", exp: "cpu,host=serverA,region=us-east value=1.0 946730096000000000", }, { name: "minute", line: `cpu,host=serverA,region=us-east value=1.0 15778834`, precision: "m", exp: "cpu,host=serverA,region=us-east value=1.0 946730040000000000", }, { name: "hour", line: `cpu,host=serverA,region=us-east value=1.0 262980`, precision: "h", exp: "cpu,host=serverA,region=us-east value=1.0 946728000000000000", }, } for _, test := range tests { pts, err := models.ParsePointsWithPrecision([]byte(test.line), time.Now().UTC(), test.precision) if err != nil { t.Fatalf(`%s: ParsePoints() failed. got %s`, test.name, err) } if exp := 1; len(pts) != exp { t.Errorf("%s: ParsePoint() len mismatch: got %v, exp %v", test.name, len(pts), exp) } pt := pts[0] got := pt.String() if got != test.exp { t.Errorf("%s: ParsePoint() to string mismatch:\n got %v\n exp %v", test.name, got, test.exp) } } }
// serveWriteLine receives incoming series data in line protocol format and writes it to the database. func (h *Handler) serveWriteLine(w http.ResponseWriter, r *http.Request, body []byte, user *meta.UserInfo) { // Some clients may not set the content-type header appropriately and send JSON with a non-json // content-type. If the body looks JSON, try to handle it as as JSON instead if len(body) > 0 { var i int for { // JSON requests must start w/ an opening bracket if body[i] == '{' { h.serveWriteJSON(w, r, body, user) return } // check that the byte is in the standard ascii code range if body[i] > 32 || i >= len(body)-1 { break } i++ } } precision := r.FormValue("precision") if precision == "" { precision = "n" } points, parseError := models.ParsePointsWithPrecision(body, time.Now().UTC(), precision) // Not points parsed correctly so return the error now if parseError != nil && len(points) == 0 { if parseError.Error() == "EOF" { w.WriteHeader(http.StatusOK) return } resultError(w, influxql.Result{Err: parseError}, http.StatusBadRequest) return } database := r.FormValue("db") if database == "" { resultError(w, influxql.Result{Err: fmt.Errorf("database is required")}, http.StatusBadRequest) return } if di, err := h.MetaClient.Database(database); err != nil { resultError(w, influxql.Result{Err: fmt.Errorf("metastore database error: %s", err)}, http.StatusInternalServerError) return } else if di == nil { resultError(w, influxql.Result{Err: fmt.Errorf("database not found: %q", database)}, http.StatusNotFound) return } if h.requireAuthentication && user == nil { resultError(w, influxql.Result{Err: fmt.Errorf("user is required to write to database %q", database)}, http.StatusUnauthorized) return } if h.requireAuthentication && !user.Authorize(influxql.WritePrivilege, database) { resultError(w, influxql.Result{Err: fmt.Errorf("%q user is not authorized to write to database %q", user.Name, database)}, http.StatusUnauthorized) return } // Write points. if err := h.PointsWriter.WritePoints(database, r.FormValue("rp"), models.ConsistencyLevelAny, points); influxdb.IsClientError(err) { h.statMap.Add(statPointsWrittenFail, int64(len(points))) resultError(w, influxql.Result{Err: err}, http.StatusBadRequest) return } else if err != nil { h.statMap.Add(statPointsWrittenFail, int64(len(points))) resultError(w, influxql.Result{Err: err}, http.StatusInternalServerError) return } else if parseError != nil { // We wrote some of the points h.statMap.Add(statPointsWrittenOK, int64(len(points))) // The other points failed to parse which means the client sent invalid line protocol. We return a 400 // response code as well as the lines that failed to parse. resultError(w, influxql.Result{Err: fmt.Errorf("partial write:\n%v", parseError)}, http.StatusBadRequest) return } h.statMap.Add(statPointsWrittenOK, int64(len(points))) w.WriteHeader(http.StatusNoContent) }