/
wcld.go
141 lines (128 loc) · 2.9 KB
/
wcld.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
package main
import (
"bufio"
"database/sql"
"flag"
"github.com/bmizerany/pq"
"log"
"net"
"os"
"regexp"
"strings"
)
var checkpoint = flag.Int("checkpoint", 1, "1 for max durability, 1000 for max throughput")
var pg *sql.DB
var LineRe = regexp.MustCompile(`\d+ \<\d+\>1 \d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\+00:00 d\.[a-z0-9-]+ ([a-z0-9\-\_\.]+) ([a-z0-9\-\_\.]+) \- \- (.*)`)
var AttrsRe = regexp.MustCompile(`( *)([a-zA-Z0-9\_\-\.]+)=?(([a-zA-Z0-9\.\-\_\.]+)|("([^\"]+)"))?`)
func main() {
flag.Parse()
cs, err := pq.ParseURL(os.Getenv("DATABASE_URL"))
if err != nil {
log.Println("unable to parse database url")
os.Exit(1)
}
pg, err = sql.Open("postgres", cs)
if err != nil {
log.Println("unable to connect to database")
os.Exit(1)
}
log.Println("bind tcp", os.Getenv("PORT"))
server, err := net.Listen("tcp", "0.0.0.0:"+os.Getenv("PORT"))
if err != nil {
log.Println("unable to bind tcp")
os.Exit(1)
}
conns := clientConns(server)
for {
go readData(<-conns)
}
}
func clientConns(listener net.Listener) (ch chan net.Conn) {
ch = make(chan net.Conn)
go func() {
for {
client, err := listener.Accept()
if err != nil {
log.Printf("error=true action=tcp_accept message=%v\n", err)
}
log.Printf("action=tcp_accept remote= %v\n", client.RemoteAddr())
ch <- client
}
}()
return ch
}
func readData(client net.Conn) {
b := bufio.NewReadWriter(bufio.NewReader(client), bufio.NewWriter(client))
i := 0
var err error
var tx *sql.Tx
for {
if i == 0 {
tx, err = pg.Begin()
if err != nil {
log.Printf("error=true action=begin message=%v", err)
}
i += 1
} else if i == (*checkpoint + 1) {
//checkpoint is set by flag
// we inc checkpoint for the case when it is set to 1
err = tx.Commit()
if err != nil {
log.Printf("error=true action=commit message=%v", err)
}
log.Printf("action=commit")
i = 0
} else {
line, err := b.ReadString('\n')
if err != nil {
break
}
handleInput(*tx, line)
i += 1
}
}
}
func handleInput(tx sql.Tx, logLine string) {
data := hstore(parse(logLine))
if len(data) > 0 {
_, err := tx.Exec("INSERT INTO log_data (data, time) VALUES ($1::hstore, now())", data)
if err != nil {
log.Printf("error=true action=insert \n message=%v \n data=%v\n", err, data)
}
}
return
}
func hstore(m map[string]string) (s string) {
i := 0
max := len(m)
for k, v := range m {
s += k + `=>` + v
i += 1
if i != max {
s += ", "
}
}
return
}
func parse(logLine string) map[string]string {
logLine = strings.Trim(logLine, "\n")
kvs := make(map[string]string)
data := LineRe.FindStringSubmatch(logLine)
if len(data) > 0 {
d := data[3]
words := AttrsRe.FindAllStringSubmatch(d, -1)
for _, match := range words {
k := match[2]
v1 := match[3]
v2 := match[5]
if len(v1) != 0 {
kvs[k] = v1
} else if len(v2) != 0 {
kvs[k] = v2
} else {
kvs[k] = "true"
}
}
}
return kvs
}