func (s *Server) HandleSocket(socket *net.UDPConn) { // From https://collectd.org/wiki/index.php/Binary_protocol // 1024 bytes (payload only, not including UDP / IP headers) // In versions 4.0 through 4.7, the receive buffer has a fixed size // of 1024 bytes. When longer packets are received, the trailing data // is simply ignored. Since version 4.8, the buffer size can be // configured. Version 5.0 will increase the default buffer size to // 1452 bytes (the maximum payload size when using UDP/IPv6 over // Ethernet). buffer := make([]byte, 1452) for { n, _, err := socket.ReadFromUDP(buffer) if err != nil || n == 0 { log.Error("Collectd ReadFromUDP error: %s", err) continue } packets, err := collectd.Packets(buffer[:n], s.typesdb) if err != nil { log.Error("Collectd parse error: %s", err) continue } for _, packet := range *packets { series := packetToSeries(&packet) err = s.coordinator.WriteSeriesData(s.user, s.database, series) if err != nil { log.Error("Collectd cannot write data: %s", err) continue } } } }
func (cas *CollectdApiSuite) TestPacketToSeriesWithMultiDataSet(c *C) { // Raw data from a Wireshark capture buf, err := hex.DecodeString("0000001a7377697463682d3137322e31362e3233312e323530000008000c15160fd86fdb575500020009736e6d70000004000e69665f6f637465747300000500184769676162697445746865726e6574305f3138000006001800020202000000000fae0fb7000000000e7d8ac3") c.Assert(err, IsNil) // Get a collectd types DB typesDB, err := gollectd.TypesDB([]byte(typesDBText)) c.Assert(err, IsNil) // Use gollectd to parse raw packet data into *[]gollectd.Packet packets, err := gollectd.Packets(buf, typesDB) c.Assert(err, IsNil) c.Assert(len(*packets), Equals, 1) // Test InfluxDB collectd API's packetToSeries function packet := &(*packets)[0] series := packetToSeries(packet) timestamp0 := *series[0].Points[0].Timestamp values0 := series[0].Points[0].Values dsname0 := values0[5].GetStringValue() dsval0 := values0[7].GetDoubleValue() c.Assert(timestamp0, Equals, int64(1415069537000000)) c.Assert(dsname0, Equals, "rx") c.Assert(dsval0, Equals, float64(263065527)) timestamp1 := *series[1].Points[0].Timestamp values1 := series[1].Points[0].Values dsname1 := values1[5].GetStringValue() dsval1 := values1[7].GetDoubleValue() c.Assert(timestamp1, Equals, int64(1415069537000000)) c.Assert(dsname1, Equals, "tx") c.Assert(dsval1, Equals, float64(243108547)) }
func (cas *CollectdApiSuite) TestPacketToSeriesWithHiResTimestamp(c *C) { // Raw data from a Wireshark capture buf, err := hex.DecodeString("0000000e6c6f63616c686f7374000008000c1512b2e40f5da16f0009000c00000002800000000002000e70726f636573736573000004000d70735f7374617465000005000c72756e6e696e67000006000f000101000000000000f03f0008000c1512b2e40f5db90f0005000d736c656570696e67000006000f0001010000000000c06f400008000c1512b2e40f5dc4a40005000c7a6f6d62696573000006000f00010100000000000000000008000c1512b2e40f5de10b0005000c73746f70706564000006000f00010100000000000000000008000c1512b2e40f5deac20005000b706167696e67000006000f00010100000000000000000008000c1512b2e40f5df59b0005000c626c6f636b6564000006000f00010100000000000000000008000c1512b2e40f7ee0610004000e666f726b5f726174650000050005000006000f000102000000000004572f0008000c1512b2e68e0635e6000200086370750000030006300000040008637075000005000975736572000006000f0001020000000000204f9c0008000c1512b2e68e0665d6000500096e696365000006000f000102000000000000caa30008000c1512b2e68e06789c0005000b73797374656d000006000f00010200000000000607050008000c1512b2e68e06818e0005000969646c65000006000f0001020000000003b090ae0008000c1512b2e68e068bcf0005000977616974000006000f000102000000000000f6810008000c1512b2e68e069c7d0005000e696e74657272757074000006000f000102000000000000001d0008000c1512b2e68e069fec0005000c736f6674697271000006000f0001020000000000000a2a0008000c1512b2e68e06a2b20005000a737465616c000006000f00010200000000000000000008000c1512b2e68e0708d60003000631000005000975736572000006000f00010200000000001d48c60008000c1512b2e68e070c16000500096e696365000006000f0001020000000000007fe60008000c1512b2e68e0710790005000b73797374656d000006000f00010200000000000667890008000c1512b2e68e0713bb0005000969646c65000006000f00010200000000025d0e470008000c1512b2e68e0717790005000977616974000006000f000102000000000002500e0008000c1512b2e68e071bc00005000e696e74657272757074000006000f00010200000000000000000008000c1512b2e68e071f800005000c736f6674697271000006000f00010200000000000006050008000c1512b2e68e07221e0005000a737465616c000006000f00010200000000000000000008000c1512b2e68e0726eb0003000632000005000975736572000006000f00010200000000001ff3e40008000c1512b2e68e0728cb000500096e696365000006000f000102000000000000ca210008000c1512b2e68e072ae70005000b73797374656d000006000f000102000000000006eabe0008000c1512b2e68e072f2f0005000977616974000006000f000102000000000000c1300008000c1512b2e68e072ccb0005000969646c65000006000f00010200000000025b5abb0008000c1512b2e68e07312c0005000e696e74657272757074000006000f00010200000000000000070008000c1512b2e68e0733520005000c736f6674697271000006000f00010200000000000007260008000c1512b2e68e0735b60005000a737465616c000006000f00010200000000000000000008000c1512b2e68e07828d0003000633000005000975736572000006000f000102000000000020f50a0008000c1512b2e68e0787ac000500096e696365000006000f0001020000000000008368") c.Assert(err, IsNil) // Get a collectd types DB typesDB, err := gollectd.TypesDB([]byte(typesDBText)) c.Assert(err, IsNil) // Use gollectd to parse raw packet data into *[]gollectd.Packet packets, err := gollectd.Packets(buf, typesDB) c.Assert(err, IsNil) c.Assert(len(*packets), Equals, 33) // Test InfluxDB collectd API's packetToSeries function packet := &(*packets)[0] series := packetToSeries(packet) timestamp := *series[0].Points[0].Timestamp values := series[0].Points[0].Values dsname := values[5].GetStringValue() dsval := values[7].GetDoubleValue() c.Assert(timestamp, Equals, int64(1414187920000000)) c.Assert(dsname, Equals, "value") c.Assert(dsval, Equals, float64(1)) }
func (cas *CollectdApiSuite) TestPacketToSeriesWithUnixTimestamp(c *C) { // Raw data from a Wireshark capture buf, err := hex.DecodeString("000000167066312d36322d3231302d39342d313733000001000c00000000544928ff0007000c00000000000000050002000c656e74726f7079000004000c656e74726f7079000006000f0001010000000000007240000200086370750000030006310000040008637075000005000969646c65000006000f0001000000000000a674620005000977616974000006000f0001000000000000000000000200076466000003000500000400076466000005000d6c6976652d636f7700000600180002010100000000a090b641000000a0cb6a2742000200086370750000030006310000040008637075000005000e696e74657272757074000006000f00010000000000000000fe0005000c736f6674697271000006000f000100000000000000000000020007646600000300050000040007646600000500096c6976650000060018000201010000000000000000000000e0ec972742000200086370750000030006310000040008637075000005000a737465616c000006000f00010000000000000000000003000632000005000975736572000006000f0001000000000000005f36000500096e696365000006000f0001000000000000000ad80002000e696e746572666163650000030005000004000e69665f6f6374657473000005000b64756d6d79300000060018000200000000000000000000000000000000041a000200076466000004000764660000050008746d70000006001800020101000000000000f240000000a0ea972742000200086370750000030006320000040008637075000005000b73797374656d000006000f00010000000000000045d30002000e696e746572666163650000030005000004000f69665f7061636b657473000005000b64756d6d79300000060018000200000000000000000000000000000000000f000200086370750000030006320000040008637075000005000969646c65000006000f0001000000000000a66480000200076466000003000500000400076466000005000d72756e2d6c6f636b000006001800020101000000000000000000000000000054410002000e696e74657266616365000004000e69665f6572726f7273000005000b64756d6d793000000600180002000000000000000000000000000000000000000200086370750000030006320000040008637075000005000977616974000006000f00010000000000000000000005000e696e74657272757074000006000f0001000000000000000132") c.Assert(err, IsNil) // Get a collectd types DB typesDB, err := gollectd.TypesDB([]byte(typesDBText)) c.Assert(err, IsNil) // Use gollectd to parse raw packet data into *[]gollectd.Packet packets, err := gollectd.Packets(buf, typesDB) c.Assert(err, IsNil) c.Assert(len(*packets), Equals, 19) // Test InfluxDB collectd API's packetToSeries function packet := &(*packets)[0] series := packetToSeries(packet) timestamp := *series[0].Points[0].Timestamp values := series[0].Points[0].Values dsname := values[5].GetStringValue() dsval := values[7].GetDoubleValue() c.Assert(timestamp, Equals, int64(1414080767000000)) c.Assert(dsname, Equals, "value") c.Assert(dsval, Equals, float64(288)) }
func (s *Service) handleMessage(buffer []byte) { packets, err := gollectd.Packets(buffer, s.typesdb) if err != nil { s.Logger.Printf("Collectd parse error: %s", err) return } for _, packet := range *packets { points := Unmarshal(&packet) for _, p := range points { s.batcher.In() <- p } } }
func (s *Service) handleMessage(buffer []byte) { packets, err := gollectd.Packets(buffer, s.typesdb) if err != nil { s.statMap.Add(statPointsParseFail, 1) s.Logger.Printf("Collectd parse error: %s", err) return } for _, packet := range *packets { points := Unmarshal(&packet) for _, p := range points { s.batcher.In() <- p } s.statMap.Add(statPointsReceived, int64(len(points))) } }
func (s *Server) handleMessage(buffer []byte) { packets, err := gollectd.Packets(buffer, s.typesdb) if err != nil { log.Printf("Collectd parse error: %s", err) return } for _, packet := range *packets { points := Unmarshal(&packet) for _, p := range points { _, err := s.writer.WriteSeries(s.Database, "", []influxdb.Point{p}) if err != nil { log.Printf("Collectd cannot write data: %s", err) continue } } } }
// Listen takes collectd network packets and breaks them into individual samples. func Listen(config ListenConfig, c chan collectd.Packet) { // Initialise the error counts errorCounts.Add("fetch.receive", 0) laddr, err := net.ResolveUDPAddr("udp", config.Bind) if err != nil { log.Fatalln("[fatal] Listen: failed to resolve address", err) } conn, err := net.ListenUDP("udp", laddr) if err != nil { log.Fatalln("[fatal] Listen: failed to listen", err) } types, err := collectd.TypesDBFile(config.Typesdb) if err != nil { log.Fatalln("[fatal] Listen: failed to parse types.db", err) } for { // 1452 is collectd 5's default buffer size. See: // https://collectd.org/wiki/index.php/Binary_protocol buf := make([]byte, 1452) n, err := conn.Read(buf[:]) if err != nil { log.Println("[error] Listen: Failed to receive packet", err) errorCounts.Add("fetch.receive", 1) continue } listenCounts.Add("raw", 1) packets, err := collectd.Packets(buf[0:n], types) for _, p := range *packets { listenCounts.Add("decoded", 1) c <- p } } }
// Listen for collectd network packets, parse , and send them over a channel func Listen(addr string, c chan collectd.Packet, typesdb string) { laddr, err := net.ResolveUDPAddr("udp", addr) if err != nil { log.Fatalln("fatal: failed to resolve address", err) } conn, err := net.ListenUDP("udp", laddr) if err != nil { log.Fatalln("fatal: failed to listen", err) } types, err := collectd.TypesDBFile(typesdb) if err != nil { log.Fatalln("fatal: failed to parse types.db", err) } for { // 1452 is collectd 5's default buffer size. See: // https://collectd.org/wiki/index.php/Binary_protocol buf := make([]byte, 1452) n, err := conn.Read(buf[:]) if err != nil { log.Println("error: Failed to receive packet", err) continue } packets, err := collectd.Packets(buf[0:n], types) if err != nil { log.Println("error: Failed to receive packet", err) continue } for _, p := range *packets { c <- p } } }