func BenchmarkParsePointWithPrecisionU(b *testing.B) {
line := `cpu value=1i 1000000000`
defaultTime := time.Now().UTC()
for i := 0; i < b.N; i++ {
models.ParsePointsWithPrecision([]byte(line), defaultTime, "u")
b.SetBytes(int64(len(line)))
}
}
// MustWritePointsString parses the line protocol (with second precision) and
// inserts the resulting points into the shard. Panic on error.
func (sh *Shard) MustWritePointsString(s string) {
a, err := models.ParsePointsWithPrecision([]byte(strings.TrimSpace(s)), time.Time{}, "s")
if err != nil {
panic(err)
}
if err := sh.WritePoints(a); err != nil {
panic(err)
}
}
func TestParsePointsWithPrecisionComments(t *testing.T) {
tests := []struct {
name string
batch string
exp string
lenPoints int
}{
{
name: "comment only",
batch: `# comment only`,
exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345",
lenPoints: 0,
},
{
name: "point with comment above",
batch: `# a point is below
cpu,host=serverA,region=us-east value=1.0 946730096789012345`,
exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345",
lenPoints: 1,
},
{
name: "point with comment below",
batch: `cpu,host=serverA,region=us-east value=1.0 946730096789012345
# end of points`,
exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345",
lenPoints: 1,
},
{
name: "indented comment",
batch: ` # a point is below
cpu,host=serverA,region=us-east value=1.0 946730096789012345`,
exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345",
lenPoints: 1,
},
}
for _, test := range tests {
pts, err := models.ParsePointsWithPrecision([]byte(test.batch), time.Now().UTC(), "")
if err != nil {
t.Fatalf(`%s: ParsePoints() failed. got %s`, test.name, err)
}
pointsLength := len(pts)
if exp := test.lenPoints; pointsLength != exp {
t.Errorf("%s: ParsePoint() len mismatch: got %v, exp %v", test.name, pointsLength, exp)
}
if pointsLength > 0 {
pt := pts[0]
got := pt.String()
if got != test.exp {
t.Errorf("%s: ParsePoint() to string mismatch:\n got %v\n exp %v", test.name, got, test.exp)
}
}
}
}
// MustWriteToShardString parses the line protocol (with second precision) and
// inserts the resulting points into a shard. Panic on error.
func (s *Store) MustWriteToShardString(shardID int, data ...string) {
var points []models.Point
for i := range data {
a, err := models.ParsePointsWithPrecision([]byte(strings.TrimSpace(data[i])), time.Time{}, "s")
if err != nil {
panic(err)
}
points = append(points, a...)
}
if err := s.WriteToShard(uint64(shardID), points); err != nil {
panic(err)
}
}
// serveWriteLine receives incoming series data in line protocol format and writes it to the database.
func (h *Handler) serveWriteLine(w http.ResponseWriter, r *http.Request, body []byte, user auth.User) {
precision := r.FormValue("precision")
if precision == "" {
precision = "n"
}
points, err := models.ParsePointsWithPrecision(body, time.Now().UTC(), precision)
if err != nil {
if err.Error() == "EOF" {
w.WriteHeader(http.StatusOK)
return
}
h.writeError(w, influxql.Result{Err: err}, http.StatusBadRequest)
return
}
database := r.FormValue("db")
if database == "" {
h.writeError(w, influxql.Result{Err: fmt.Errorf("database is required")}, http.StatusBadRequest)
return
}
action := auth.Action{
Resource: auth.DatabaseResource(database),
Privilege: auth.WritePrivilege,
}
if err := user.AuthorizeAction(action); err != nil {
h.writeError(w, influxql.Result{Err: fmt.Errorf("%q user is not authorized to write to database %q", user.Name(), database)}, http.StatusUnauthorized)
return
}
// Write points.
if err := h.PointsWriter.WritePoints(
database,
r.FormValue("rp"),
models.ConsistencyLevelAll,
points,
); influxdb.IsClientError(err) {
h.statMap.Add(statPointsWrittenFail, int64(len(points)))
h.writeError(w, influxql.Result{Err: err}, http.StatusBadRequest)
return
} else if err != nil {
h.statMap.Add(statPointsWrittenFail, int64(len(points)))
h.writeError(w, influxql.Result{Err: err}, http.StatusInternalServerError)
return
}
h.statMap.Add(statPointsWrittenOK, int64(len(points)))
w.WriteHeader(http.StatusNoContent)
}
func test(t *testing.T, line string, point TestPoint) {
pts, err := models.ParsePointsWithPrecision([]byte(line), time.Unix(0, 0), "n")
if err != nil {
t.Fatalf(`ParsePoints("%s") mismatch. got %v, exp nil`, line, err)
}
if exp := 1; len(pts) != exp {
t.Fatalf(`ParsePoints("%s") len mismatch. got %d, exp %d`, line, len(pts), exp)
}
if exp := point.Key(); !bytes.Equal(pts[0].Key(), exp) {
t.Errorf("ParsePoints(\"%s\") key mismatch.\ngot %v\nexp %v", line, string(pts[0].Key()), string(exp))
}
if exp := len(point.Tags()); len(pts[0].Tags()) != exp {
t.Errorf(`ParsePoints("%s") tags mismatch. got %v, exp %v`, line, pts[0].Tags(), exp)
}
for tag, value := range pts[0].Tags() {
if value != point.RawTags[tag] {
t.Errorf(`ParsePoints("%s") tags mismatch. got %v, exp %v`, line, value, point.RawTags[tag])
}
}
for name, value := range point.RawFields {
val := pts[0].Fields()[name]
expfval, ok := val.(float64)
if ok && math.IsNaN(expfval) {
gotfval, ok := value.(float64)
if ok && !math.IsNaN(gotfval) {
t.Errorf(`ParsePoints("%s") field '%s' mismatch. exp NaN`, line, name)
}
} else if !reflect.DeepEqual(pts[0].Fields()[name], value) {
t.Errorf(`ParsePoints("%s") field '%s' mismatch. got %[3]v (%[3]T), exp %[4]v (%[4]T)`, line, name, pts[0].Fields()[name], value)
}
}
if !pts[0].Time().Equal(point.Time()) {
t.Errorf(`ParsePoints("%s") time mismatch. got %v, exp %v`, line, pts[0].Time(), point.Time())
}
if !strings.HasPrefix(pts[0].String(), line) {
t.Errorf("ParsePoints string mismatch.\ngot: %v\nexp: %v", pts[0].String(), line)
}
}
func (c *commune) point(precision string) models.Point {
pt := []byte(<-c.ch)
p, err := models.ParsePointsWithPrecision(pt, time.Now().UTC(), precision)
if err != nil {
log.Fatalf("Error parsing point for commune\n point: %v\n error: %v\n", pt, err)
}
if len(p) == 0 {
return c.storedPoint
}
c.storedPoint = p[0]
return p[0]
}
func (p *InfluxParser) ParseWithDefaultTime(buf []byte, t time.Time) ([]telegraf.Metric, error) {
// parse even if the buffer begins with a newline
buf = bytes.TrimPrefix(buf, []byte("\n"))
points, err := models.ParsePointsWithPrecision(buf, t, "n")
metrics := make([]telegraf.Metric, len(points))
for i, point := range points {
for k, v := range p.DefaultTags {
// only set the default tag if it doesn't already exist:
if tmp := point.Tags().GetString(k); tmp == "" {
point.AddTag(k, v)
}
}
// Ignore error here because it's impossible that a model.Point
// wouldn't parse into client.Point properly
metrics[i] = telegraf.NewMetricFromPoint(point)
}
return metrics, err
}
func (s *Service) parser() {
defer s.wg.Done()
for {
select {
case <-s.done:
return
case buf := <-s.parserChan:
points, err := models.ParsePointsWithPrecision(buf, time.Now().UTC(), s.config.Precision)
if err != nil {
s.statMap.Add(statPointsParseFail, 1)
s.Logger.Printf("Failed to parse points: %s", err)
continue
}
for _, point := range points {
s.batcher.In() <- point
}
s.statMap.Add(statPointsReceived, int64(len(points)))
}
}
}
func readPointsFromIO(data io.ReadCloser, points chan<- models.Point, precision string) error {
defer data.Close()
defer close(points)
now := time.Time{}
in := bufio.NewScanner(data)
for in.Scan() {
db := in.Text()
if !in.Scan() {
return fmt.Errorf("invalid replay file format, expected another line")
}
rp := in.Text()
if !in.Scan() {
return fmt.Errorf("invalid replay file format, expected another line")
}
mps, err := dbmodels.ParsePointsWithPrecision(
in.Bytes(),
now,
precision,
)
if err != nil {
return err
}
mp := mps[0]
p := models.Point{
Database: db,
RetentionPolicy: rp,
Name: mp.Name(),
Group: models.NilGroup,
Tags: models.Tags(mp.Tags().Map()),
Fields: models.Fields(mp.Fields()),
Time: mp.Time().UTC(),
}
points <- p
}
return nil
}
// serveWrite receives incoming series data in line protocol format and writes it to the database.
func (h *Handler) serveWrite(w http.ResponseWriter, r *http.Request, user *meta.UserInfo) {
atomic.AddInt64(&h.stats.WriteRequests, 1)
atomic.AddInt64(&h.stats.ActiveWriteRequests, 1)
defer func(start time.Time) {
atomic.AddInt64(&h.stats.ActiveWriteRequests, -1)
atomic.AddInt64(&h.stats.WriteRequestDuration, time.Since(start).Nanoseconds())
}(time.Now())
database := r.URL.Query().Get("db")
if database == "" {
h.resultError(w, influxql.Result{Err: fmt.Errorf("database is required")}, http.StatusBadRequest)
return
}
if di := h.MetaClient.Database(database); di == nil {
h.resultError(w, influxql.Result{Err: fmt.Errorf("database not found: %q", database)}, http.StatusNotFound)
return
}
if h.Config.AuthEnabled && user == nil {
h.resultError(w, influxql.Result{Err: fmt.Errorf("user is required to write to database %q", database)}, http.StatusUnauthorized)
return
}
if h.Config.AuthEnabled {
if err := h.WriteAuthorizer.AuthorizeWrite(user.Name, database); err != nil {
h.resultError(w, influxql.Result{Err: fmt.Errorf("%q user is not authorized to write to database %q", user.Name, database)}, http.StatusUnauthorized)
return
}
}
// Handle gzip decoding of the body
body := r.Body
if r.Header.Get("Content-Encoding") == "gzip" {
b, err := gzip.NewReader(r.Body)
if err != nil {
h.resultError(w, influxql.Result{Err: err}, http.StatusBadRequest)
return
}
defer b.Close()
body = b
}
var bs []byte
if clStr := r.Header.Get("Content-Length"); clStr != "" {
if length, err := strconv.Atoi(clStr); err == nil {
// This will just be an initial hint for the gzip reader, as the
// bytes.Buffer will grow as needed when ReadFrom is called
bs = make([]byte, 0, length)
}
}
buf := bytes.NewBuffer(bs)
_, err := buf.ReadFrom(body)
if err != nil {
if h.Config.WriteTracing {
h.Logger.Print("Write handler unable to read bytes from request body")
}
h.resultError(w, influxql.Result{Err: err}, http.StatusBadRequest)
return
}
atomic.AddInt64(&h.stats.WriteRequestBytesReceived, int64(buf.Len()))
if h.Config.WriteTracing {
h.Logger.Printf("Write body received by handler: %s", buf.Bytes())
}
points, parseError := models.ParsePointsWithPrecision(buf.Bytes(), time.Now().UTC(), r.URL.Query().Get("precision"))
// Not points parsed correctly so return the error now
if parseError != nil && len(points) == 0 {
if parseError.Error() == "EOF" {
h.writeHeader(w, http.StatusOK)
return
}
h.resultError(w, influxql.Result{Err: parseError}, http.StatusBadRequest)
return
}
// Determine required consistency level.
level := r.URL.Query().Get("consistency")
consistency := models.ConsistencyLevelOne
if level != "" {
var err error
consistency, err = models.ParseConsistencyLevel(level)
if err != nil {
h.resultError(w, influxql.Result{Err: err}, http.StatusBadRequest)
return
}
}
// Write points.
if err := h.PointsWriter.WritePoints(database, r.URL.Query().Get("rp"), consistency, points); influxdb.IsClientError(err) {
atomic.AddInt64(&h.stats.PointsWrittenFail, int64(len(points)))
h.resultError(w, influxql.Result{Err: err}, http.StatusBadRequest)
return
} else if err != nil {
atomic.AddInt64(&h.stats.PointsWrittenFail, int64(len(points)))
h.resultError(w, influxql.Result{Err: err}, http.StatusInternalServerError)
return
} else if parseError != nil {
// We wrote some of the points
atomic.AddInt64(&h.stats.PointsWrittenOK, int64(len(points)))
// The other points failed to parse which means the client sent invalid line protocol. We return a 400
// response code as well as the lines that failed to parse.
h.resultError(w, influxql.Result{Err: fmt.Errorf("partial write:\n%v", parseError)}, http.StatusBadRequest)
return
}
atomic.AddInt64(&h.stats.PointsWrittenOK, int64(len(points)))
h.writeHeader(w, http.StatusNoContent)
}
func TestParsePointsWithPrecisionNoTime(t *testing.T) {
line := `cpu,host=serverA,region=us-east value=1.0`
tm, _ := time.Parse(time.RFC3339Nano, "2000-01-01T12:34:56.789012345Z")
tests := []struct {
name string
precision string
exp string
}{
{
name: "no precision",
precision: "",
exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345",
},
{
name: "nanosecond precision",
precision: "n",
exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345",
},
{
name: "microsecond precision",
precision: "u",
exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012000",
},
{
name: "millisecond precision",
precision: "ms",
exp: "cpu,host=serverA,region=us-east value=1.0 946730096789000000",
},
{
name: "second precision",
precision: "s",
exp: "cpu,host=serverA,region=us-east value=1.0 946730096000000000",
},
{
name: "minute precision",
precision: "m",
exp: "cpu,host=serverA,region=us-east value=1.0 946730040000000000",
},
{
name: "hour precision",
precision: "h",
exp: "cpu,host=serverA,region=us-east value=1.0 946728000000000000",
},
}
for _, test := range tests {
pts, err := models.ParsePointsWithPrecision([]byte(line), tm, test.precision)
if err != nil {
t.Fatalf(`%s: ParsePoints() failed. got %s`, test.name, err)
}
if exp := 1; len(pts) != exp {
t.Errorf("%s: ParsePoint() len mismatch: got %v, exp %v", test.name, len(pts), exp)
}
pt := pts[0]
got := pt.String()
if got != test.exp {
t.Errorf("%s: ParsePoint() to string mismatch:\n got %v\n exp %v", test.name, got, test.exp)
}
}
}
func TestParsePointsWithPrecision(t *testing.T) {
tests := []struct {
name string
line string
precision string
exp string
}{
{
name: "nanosecond by default",
line: `cpu,host=serverA,region=us-east value=1.0 946730096789012345`,
precision: "",
exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345",
},
{
name: "nanosecond",
line: `cpu,host=serverA,region=us-east value=1.0 946730096789012345`,
precision: "n",
exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345",
},
{
name: "microsecond",
line: `cpu,host=serverA,region=us-east value=1.0 946730096789012`,
precision: "u",
exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012000",
},
{
name: "millisecond",
line: `cpu,host=serverA,region=us-east value=1.0 946730096789`,
precision: "ms",
exp: "cpu,host=serverA,region=us-east value=1.0 946730096789000000",
},
{
name: "second",
line: `cpu,host=serverA,region=us-east value=1.0 946730096`,
precision: "s",
exp: "cpu,host=serverA,region=us-east value=1.0 946730096000000000",
},
{
name: "minute",
line: `cpu,host=serverA,region=us-east value=1.0 15778834`,
precision: "m",
exp: "cpu,host=serverA,region=us-east value=1.0 946730040000000000",
},
{
name: "hour",
line: `cpu,host=serverA,region=us-east value=1.0 262980`,
precision: "h",
exp: "cpu,host=serverA,region=us-east value=1.0 946728000000000000",
},
}
for _, test := range tests {
pts, err := models.ParsePointsWithPrecision([]byte(test.line), time.Now().UTC(), test.precision)
if err != nil {
t.Fatalf(`%s: ParsePoints() failed. got %s`, test.name, err)
}
if exp := 1; len(pts) != exp {
t.Errorf("%s: ParsePoint() len mismatch: got %v, exp %v", test.name, len(pts), exp)
}
pt := pts[0]
got := pt.String()
if got != test.exp {
t.Errorf("%s: ParsePoint() to string mismatch:\n got %v\n exp %v", test.name, got, test.exp)
}
}
}
// serveWriteLine receives incoming series data in line protocol format and writes it to the database.
func (h *Handler) serveWriteLine(w http.ResponseWriter, r *http.Request, body []byte, user *meta.UserInfo) {
// Some clients may not set the content-type header appropriately and send JSON with a non-json
// content-type. If the body looks JSON, try to handle it as as JSON instead
if len(body) > 0 {
var i int
for {
// JSON requests must start w/ an opening bracket
if body[i] == '{' {
h.serveWriteJSON(w, r, body, user)
return
}
// check that the byte is in the standard ascii code range
if body[i] > 32 || i >= len(body)-1 {
break
}
i++
}
}
precision := r.FormValue("precision")
if precision == "" {
precision = "n"
}
points, parseError := models.ParsePointsWithPrecision(body, time.Now().UTC(), precision)
// Not points parsed correctly so return the error now
if parseError != nil && len(points) == 0 {
if parseError.Error() == "EOF" {
w.WriteHeader(http.StatusOK)
return
}
resultError(w, influxql.Result{Err: parseError}, http.StatusBadRequest)
return
}
database := r.FormValue("db")
if database == "" {
resultError(w, influxql.Result{Err: fmt.Errorf("database is required")}, http.StatusBadRequest)
return
}
if di, err := h.MetaClient.Database(database); err != nil {
resultError(w, influxql.Result{Err: fmt.Errorf("metastore database error: %s", err)}, http.StatusInternalServerError)
return
} else if di == nil {
resultError(w, influxql.Result{Err: fmt.Errorf("database not found: %q", database)}, http.StatusNotFound)
return
}
if h.requireAuthentication && user == nil {
resultError(w, influxql.Result{Err: fmt.Errorf("user is required to write to database %q", database)}, http.StatusUnauthorized)
return
}
if h.requireAuthentication && !user.Authorize(influxql.WritePrivilege, database) {
resultError(w, influxql.Result{Err: fmt.Errorf("%q user is not authorized to write to database %q", user.Name, database)}, http.StatusUnauthorized)
return
}
// Write points.
if err := h.PointsWriter.WritePoints(database, r.FormValue("rp"), models.ConsistencyLevelAny, points); influxdb.IsClientError(err) {
h.statMap.Add(statPointsWrittenFail, int64(len(points)))
resultError(w, influxql.Result{Err: err}, http.StatusBadRequest)
return
} else if err != nil {
h.statMap.Add(statPointsWrittenFail, int64(len(points)))
resultError(w, influxql.Result{Err: err}, http.StatusInternalServerError)
return
} else if parseError != nil {
// We wrote some of the points
h.statMap.Add(statPointsWrittenOK, int64(len(points)))
// The other points failed to parse which means the client sent invalid line protocol. We return a 400
// response code as well as the lines that failed to parse.
resultError(w, influxql.Result{Err: fmt.Errorf("partial write:\n%v", parseError)}, http.StatusBadRequest)
return
}
h.statMap.Add(statPointsWrittenOK, int64(len(points)))
w.WriteHeader(http.StatusNoContent)
}