/
fetcher.go
161 lines (134 loc) · 3.78 KB
/
fetcher.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
package main
import (
"log"
"github.com/bndr/gopencils"
"database/sql"
_ "github.com/go-sql-driver/mysql"
"time"
"strconv"
)
func fetchNewData(configuration Configuration) {
db, err := sql.Open("mysql", configuration.Datasource)
if err != nil {
panic(err.Error())
}
defer db.Close()
err = db.Ping()
if err != nil {
panic(err.Error())
}
logsOut, err := db.Prepare("SELECT `time`, `index` FROM `logs` ORDER BY `time` DESC, `index` DESC LIMIT 1")
if err != nil {
log.Println(err.Error())
}
defer logsOut.Close()
logsIns, err := db.Prepare("INSERT INTO logs VALUES( ?, ?, ?, ?, ?, ? )")
if err != nil {
log.Println(err.Error())
}
defer logsIns.Close()
logAttributesIns, err := db.Prepare("INSERT INTO log_attributes VALUES( ?, ?, ?, ? )")
if err != nil {
log.Println(err.Error())
}
defer logAttributesIns.Close()
var (
start time.Time
elapsed time.Duration
logEntryTime time.Time
index, preCount int64
extraLogs int64 = 10
serverLogLimit int64 = 10000
api = gopencils.Api(configuration.BaseUrl)
resp = new(RespStruct)
firstLog, logCount, refid, participantid float64
finalCount string
querystring map[string]string
entries int
)
for {
start = time.Now()
err = logsOut.QueryRow().Scan(&logEntryTime, &index)
if err != nil {
log.Println(err.Error())
}
log.Printf("Last log entry was from: %v", logEntryTime)
log.Printf("Last log entry index was: %d", index)
_, err = api.Res("log/overview", resp).Get()
firstLog = resp.Response["first"].(float64)
logCount = resp.Response["count"].(float64) + firstLog
if (logCount == 0) {
log.Println("Logcount was 0")
// Fresh restarted server, not much to do here
elapsed := time.Since(start)
log.Println("Server just restarted, nothing to fetch")
log.Printf("Fetching data took %s", elapsed)
log.Println("Fetcher idling for 15 minutes")
time.Sleep(15 * time.Minute)
continue
}
preCount = (int64(logCount)-index) + extraLogs
if (preCount < 0) {
log.Println("Precount was less than 0!")
preCount = 1
}
if (preCount > serverLogLimit+extraLogs ) {
log.Println("precount was larger than server log limit + extra logs")
preCount = serverLogLimit+extraLogs
}
finalCount = strconv.Itoa(int(preCount))
log.Printf("Fetching %s log entries from server", finalCount)
querystring = map[string]string{"offset": "-" + finalCount, "count": finalCount}
_, err = api.Res("log/range", resp).Get(querystring)
entries = 0
if err != nil {
log.Println(err)
} else {
for _, event := range resp.Response["events"].([]interface{}) {
event, _ := event.(map[string]interface{})
if int64(event["index"].(float64)) <= index && preCount != 1 {
continue
}
entries++
switch event["refid"].(type) {
case float64:
refid = event["refid"].(float64)
default:
refid = 0
}
switch event["participantid"].(type) {
case float64:
participantid = event["participantid"].(float64)
default:
participantid = 0
}
result, err := logsIns.Exec(
nil,
event["index"].(float64),
time.Unix(int64(event["time"].(float64)), 0),
event["name"].(string),
refid,
participantid,
)
if err != nil {
log.Println(err.Error())
}
lastInsertedId, err := result.LastInsertId()
if err != nil {
log.Println(err.Error())
}
for key, value := range event["attributes"].(map[string]interface{}) {
_, err = logAttributesIns.Exec(nil, lastInsertedId, key, value)
if err != nil {
log.Println(err.Error())
}
}
}
}
log.Printf("Added %d new log entries out of %s fetched ones", entries, finalCount)
elapsed = time.Since(start)
log.Printf("Fetching data took %s", elapsed)
log.Println("Fetcher idling for 15 minutes")
time.Sleep(15 * time.Minute)
}
}