/
report-2-1.go
199 lines (176 loc) · 4.49 KB
/
report-2-1.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
package main
import (
"encoding/csv"
"encoding/json"
"fmt"
"gopkg.in/olivere/elastic.v3"
"os"
"reflect"
"strings"
"sync"
)
//Sample output struct
type Tweet struct {
User string
Post_date string
Message string
}
//csvWriter interface
type Output interface {
CsvWriter()
}
//Database Interface gets client according to database
type GetClient interface {
GetClient()
}
//filter struct
type Filter struct {
filter []Fpair
}
type Fpair struct {
Qkey string
Qvalue string
}
//types of database structs
type DatabaseType int
//Constants of type DatabaseType
const (
Elasticsearch DatabaseType = 1 + iota //Provide int values to each of constants
Dynamo
Mysql
)
//Enums should be able to print as strings, so we declare slice of strings
var database = [...]string{
"Elasticsearch",
"Dynamo",
"Mysql",
}
//To control the default format for custom database type
func (db DatabaseType) String() string {
return database[db-1]
}
func (typedb DatabaseType) GetClient() (interface{}, error) {
if typedb == Elasticsearch {
client, err := elastic.NewClient()
return client, err
}
fmt.Println("No such Database", typedb)
return nil, nil
}
func GetWriter() (*csv.Writer, error) {
file, err := os.Create("result.csv")
writer := csv.NewWriter(file)
return writer, err
}
func (c Tweet) CsvWriter(writer *csv.Writer, m chan Tweet) {
var mutex = &sync.Mutex{}
for i := range m {
c = i
//fmt.Println(c)
data := []string{c.User, c.Post_date, c.Message}
//Introduced locks for write to csv file
mutex.Lock()
writer.Write(data)
writer.Flush()
mutex.Unlock()
//lock closed
}
}
//Now Generic lookup is possible thanks to this function
func GetField(v *Tweet, field string) string {
r := reflect.ValueOf(v)
f := reflect.Indirect(r).FieldByName(field)
// fmt.Println(string(f.String()))
return string(f.String())
}
//gets the searchresult and filters them for writing to csv
func Filtering(search chan *elastic.SearchResult) {
var t Tweet
var data chan Tweet = make(chan Tweet)
writer, err := GetWriter()
CheckError(err)
go t.CsvWriter(writer, data) // spawning the csvwriter routine
for i := range search {
searchResult := i
for _, hit := range searchResult.Hits.Hits {
err := json.Unmarshal(*hit.Source, &t)
CheckError(err)
//Filtering data
q.filter[0].Qkey = strings.Replace(q.filter[0].Qkey, q.filter[0].Qkey[:1], strings.ToUpper(q.filter[0].Qkey[:1]), 1)
if GetField(&t, q.filter[0].Qkey) == q.filter[0].Qvalue {
fmt.Println(t)
data <- t
}
}
}
close(data) // closing the channel
}
//Scrolls elasticsearch like cursors in SQL
func GetReportEL(client *elastic.Client) {
result := make(chan *elastic.SearchResult)
// spawinng the Filtering routine
go Filtering(result)
// the termquery uses all lower but for matching to filter exactly we have to convert the first letter to upper
boolq := elastic.NewBoolQuery()
termQuery := boolq.Filter(elastic.NewTermQuery(q.filter[0].Qkey, q.filter[0].Qvalue))
count, err := client.Count().
Query(termQuery).
Do()
CheckError(err)
//Gives count of total records found
fmt.Println("Count", count)
scrollService := elastic.NewScrollService(client)
searchResult, err := scrollService.Scroll("5m").Size(1).Do()
CheckError(err)
pages := 0
scroll_indexId := searchResult.ScrollId
for {
searchResult, err := scrollService.Query(termQuery).Scroll("5m").
Size(1).
ScrollId(scroll_indexId).
Do()
if err != nil {
break
}
result <- searchResult // sending data into channel received by Filtering function
pages += 1
scroll_indexId = searchResult.ScrollId
if scroll_indexId == "" {
fmt.Println(scroll_indexId)
}
}
if pages <= 0 {
fmt.Println(pages, "Records found")
}
close(result) //closing the channel
}
func CheckError(err error) {
if err != nil {
fmt.Println(err)
panic(err)
}
}
var q Filter // Global because it has to be used at different routines
func main() {
var k Fpair
var str DatabaseType
fmt.Println("Select Database :->\n 1. For Elasticsearch\n", "2. For Dynamo\n", "3. For Mysql\n", "Enter choice")
fmt.Scan(&str)
// fmt.Println(reflect.TypeOf(str))
fmt.Println("Enter the search Field")
fmt.Scan(&k.Qkey)
fmt.Println("Enter the search string")
fmt.Scan(&k.Qvalue)
q = Filter{filter: []Fpair{k}}
fmt.Println(q.filter[0])
client, err := str.GetClient()
CheckError(err)
//type assertion for getclient
switch v := client.(type) {
case *elastic.Client:
fmt.Println("Calling With Elasticsearch Client")
GetReportEL(v)
default:
fmt.Println("No such Client available", v)
}
}