// Message handling --------------------------------------------------------------
func throwMsg(msg string, external bool, fmtSpec string) {
customer_text := "An error occurred while retrieving the requested information"
out_level := os.Stderr
status := "error"
if msg == "Query returned no results" {
out_level = os.Stdout
status = "ok"
}
// If called interactively, show full error message to user (otherwise show
// customer friendly generic message)
if !(msg != "Query returned no results" && external) {
customer_text = msg
}
if fmtSpec == "json" {
// If called non-interactively, write full error message to message log
if external {
if goDB.InitDBLog() != nil {
return
}
goDB.SysLog.Err(msg)
}
message := map[string]string{
"status": status,
"statusMessage": customer_text,
}
json_out, _ := json.Marshal(message)
fmt.Fprintf(out_level, "%s", json_out)
} else {
fmt.Fprintf(out_level, "%s\n", customer_text)
}
}
func main() {
// specify number of threads that can be used
numCpu := runtime.NumCPU()
runtime.GOMAXPROCS(numCpu)
// parse command line arguments
var config Config
parseCommandLineArgs(&config)
// sanity check the input
if config.FilePath == "" || config.SavePath == "" {
fmt.Println("Empty path specified. Usage: ./goConvert -i <interface> -f <file path> -s <save path>")
return
}
if config.Iface == "" {
fmt.Println("No interface specified. Usage: ./goConvert -i <interface> -f <file path> -s <save path>")
return
}
// get number of lines to read in the specified file
cmd := exec.Command("wc", "-l", config.FilePath)
out, cmderr := cmd.Output()
if cmderr != nil {
fmt.Println("Could not execute line count on file", config.FilePath)
return
}
nlString := strings.Split(string(out), " ")
nl_in_file, _ := strconv.ParseInt(nlString[0], 10, 32)
if int(nl_in_file) < config.NumLines && nl_in_file > 0 {
config.NumLines = int(nl_in_file)
}
fmt.Printf("Converting %d rows in file %s\n", config.NumLines, config.FilePath)
// create channel to pass to the storage writer
dataChan := make(chan goDB.DBData)
doneChan := make(chan bool)
// init goprobe log
goDB.InitDBLog()
// open file
var (
file *os.File
err error
br, bs, pr, ps []byte
dip, sip []byte
dport, l7proto, proto []byte
)
if file, err = os.Open(config.FilePath); err != nil {
fmt.Println("File open error: " + err.Error())
}
// spawn database writer
writer := goDB.NewDBStorageWrite(config.SavePath)
writer.WriteFlowsToDatabase(int64(0), dataChan, doneChan)
fmt.Print("Progress: 0% |")
go func() {
// scan file line by line
scanner := bufio.NewScanner(file)
var lines_read int
var active_block_stamp int64
var perc_done, prev_perc int
var prev_block_stamp int64
for scanner.Scan() {
if lines_read == config.NumLines {
break
}
perc_done = int(float64(lines_read) / float64(config.NumLines) * 100)
if perc_done != prev_perc {
if perc_done%50 == 0 {
fmt.Print(" 50% ")
runtime.GC()
debug.FreeOSMemory()
} else if perc_done%10 == 0 {
fmt.Printf("|")
runtime.GC()
debug.FreeOSMemory()
} else if perc_done%2 == 0 {
fmt.Printf("-")
runtime.GC()
debug.FreeOSMemory()
}
}
prev_perc = perc_done
fields := strings.Split(scanner.Text(), ",")
// handle timestamp to find out when to ship the DBData to the channel
time, _ := strconv.ParseInt(fields[9], 10, 64)
// ignore all those lines which do not abide by the temporal ordering
if time < prev_block_stamp {
prev_block_stamp = time
continue
}
cur_block_stamp := time - (time % DB_WRITE_INTERVAL)
// if the timestamp is in another interval, create a new DBData block
if cur_block_stamp != active_block_stamp {
if active_block_stamp != 0 {
var tstampArr = []byte{uint8(active_block_stamp >> 56),
uint8(active_block_stamp >> 48),
uint8(active_block_stamp >> 40),
uint8(active_block_stamp >> 32),
uint8(active_block_stamp >> 24),
uint8(active_block_stamp >> 16),
uint8(active_block_stamp >> 8),
uint8(active_block_stamp & 0xff)}
// place the timestamp at the end of the arrays
br = append(br, tstampArr...)
bs = append(bs, tstampArr...)
pr = append(pr, tstampArr...)
ps = append(ps, tstampArr...)
dip = append(dip, tstampArr...)
sip = append(sip, tstampArr...)
dport = append(dport, tstampArr...)
l7proto = append(l7proto, tstampArr...)
proto = append(proto, tstampArr...)
// if the block was switched write out the current arrays
dataChan <- goDB.NewDBData(br, bs, pr, ps, dip, sip, dport, l7proto, proto, active_block_stamp, config.Iface)
// reset the arrays
br, bs, pr, ps = []byte{}, []byte{}, []byte{}, []byte{}
dip, sip = []byte{}, []byte{}
dport, l7proto, proto = []byte{}, []byte{}, []byte{}
// the new timestamp becomes the active timestamp
active_block_stamp = cur_block_stamp
var tstampArrNew = []byte{uint8(active_block_stamp >> 56),
uint8(active_block_stamp >> 48),
uint8(active_block_stamp >> 40),
uint8(active_block_stamp >> 32),
uint8(active_block_stamp >> 24),
uint8(active_block_stamp >> 16),
uint8(active_block_stamp >> 8),
uint8(active_block_stamp & 0xff)}
// place the new timestamp at the beginning of the arrays
br = append(br, tstampArrNew...)
bs = append(bs, tstampArrNew...)
pr = append(pr, tstampArrNew...)
ps = append(ps, tstampArrNew...)
dip = append(dip, tstampArrNew...)
sip = append(sip, tstampArrNew...)
dport = append(dport, tstampArrNew...)
l7proto = append(l7proto, tstampArrNew...)
proto = append(proto, tstampArrNew...)
} else {
active_block_stamp = cur_block_stamp
var tstampArr = []byte{uint8(active_block_stamp >> 56),
uint8(active_block_stamp >> 48),
uint8(active_block_stamp >> 40),
uint8(active_block_stamp >> 32),
uint8(active_block_stamp >> 24),
uint8(active_block_stamp >> 16),
uint8(active_block_stamp >> 8),
uint8(active_block_stamp & 0xff)}
// place the timestamp at the beginning of the arrays
br = append(br, tstampArr...)
bs = append(bs, tstampArr...)
pr = append(pr, tstampArr...)
ps = append(ps, tstampArr...)
dip = append(dip, tstampArr...)
sip = append(sip, tstampArr...)
dport = append(dport, tstampArr...)
l7proto = append(l7proto, tstampArr...)
proto = append(proto, tstampArr...)
}
}
// handle counters
br_int, _ := strconv.ParseUint(fields[0], 10, 64)
br = append(br,
uint8(br_int>>56), uint8(br_int>>48),
uint8(br_int>>40), uint8(br_int>>32),
uint8(br_int>>24), uint8(br_int>>16),
uint8(br_int>>8), uint8(br_int&0xff))
bs_int, _ := strconv.ParseUint(fields[1], 10, 64)
bs = append(bs,
uint8(bs_int>>56), uint8(bs_int>>48),
uint8(bs_int>>40), uint8(bs_int>>32),
uint8(bs_int>>24), uint8(bs_int>>16),
uint8(bs_int>>8), uint8(bs_int&0xff))
pr_int, _ := strconv.ParseUint(fields[5], 10, 64)
pr = append(pr,
uint8(pr_int>>56), uint8(pr_int>>48),
uint8(pr_int>>40), uint8(pr_int>>32),
uint8(pr_int>>24), uint8(pr_int>>16),
uint8(pr_int>>8), uint8(pr_int&0xff))
ps_int, _ := strconv.ParseUint(fields[6], 10, 64)
ps = append(ps,
uint8(ps_int>>56), uint8(ps_int>>48),
uint8(ps_int>>40), uint8(ps_int>>32),
uint8(ps_int>>24), uint8(ps_int>>16),
uint8(ps_int>>8), uint8(ps_int&0xff))
// handle ips
dip_str := strings.Split(fields[2], ".")
for i := 0; i < 16; i++ {
if i < 4 {
octet, _ := strconv.Atoi(dip_str[i])
dip = append(dip, uint8(octet))
} else {
dip = append(dip, 0x00)
}
}
sip_str := strings.Split(fields[8], ".")
for i := 0; i < 16; i++ {
if i < 4 {
octet, _ := strconv.Atoi(sip_str[i])
sip = append(sip, uint8(octet))
} else {
sip = append(sip, 0x00)
}
}
// handle port and protos
prot_num, _ := strconv.Atoi(fields[7])
proto = append(proto, uint8(prot_num))
dport_num, _ := strconv.Atoi(fields[3])
dport = append(dport, uint8(dport_num>>8), uint8(dport_num&0xff))
l7p_num, _ := strconv.Atoi(fields[4])
l7proto = append(l7proto, uint8(l7p_num>>8), uint8(l7p_num&0xff))
lines_read++
prev_block_stamp = cur_block_stamp
}
// push empty DBData onto channel to signal that we are done
dataChan <- goDB.DBData{}
fmt.Print("| 100%")
}()
// return if the data write failed or exited
if <-doneChan {
fmt.Println("\nExiting")
return
}
return
}