func startAnalysis(clients db.Clients) {
go func() {
tick := time.NewTicker(10 * time.Second)
lastID, err := clients.DB.GetLastProcessedSampleID()
if err != nil {
lg.Warningln(err)
}
lg.Infof("starting analysis from sample ID %d", lastID)
lastPersistedID := lastID
for range tick.C {
results, err := clients.DB.GetSamples(uint64(lastID), "")
if err != nil {
lg.Fatal(err)
}
n := 0
start := time.Now()
loop:
for s := range results {
n++
if s.ID > lastID {
lastID = s.ID
}
if s.Type == "NewClientToken" {
if !shared.AcceptedHost(s.Host) {
lg.Warningln("not accepted host id:", s.ID, s.Host)
continue loop
}
err := clients.DB.PublishHost(s)
if err != nil {
lg.Warning(err)
}
}
}
if n != 0 && lg.V(15) {
lg.Infof("processed %d samples in %s", n, time.Since(start).String())
}
if lastID != lastPersistedID {
err = clients.DB.SetLastProcessedSampleID(lastID)
if err != nil {
lg.Errorln(err)
} else {
lastPersistedID = lastID
}
}
}
}()
}
func samplesAnalyzer() {
loop:
for samples := range sampleAnalysisC {
var (
newTokenSample db.Sample
clientSamples = make(map[string]db.Sample, 0)
centralSamples = make(map[string]db.Sample, 0)
)
// organize input data
for _, s := range samples {
switch s.Type {
case "NewClientToken":
if newTokenSample.Token != "" {
lg.Error("got more than one newTokenSample, aborting")
continue loop
}
newTokenSample = s
case "HTTPHeader", "DNSQuery":
switch s.Origin {
case "Central":
centralSamples[s.Type] = s
case "Client":
clientSamples[s.Type] = s
}
default:
lg.Errorf("dont know how to handle %d %s, skipping", s.ID, s.Type)
}
}
// validate that wanted data types are available
if newTokenSample.Token == "" {
lg.Errorln("No newTokenSample, aborting")
continue loop
}
if !shared.AcceptedHost(newTokenSample.Host) {
lg.Warningln("not accepted host id:", newTokenSample.ID, newTokenSample.Host)
continue loop
}
for _, smap := range []map[string]db.Sample{clientSamples, centralSamples} {
for _, stype := range []string{"HTTPHeader"} {
if _, ok := smap[stype]; !ok {
lg.Errorf("missing %s, cannot analyse", stype)
continue loop
}
}
}
// parse data
clientSample := clientSamples["HTTPHeader"]
var clientHeader measure.HTTPHeaderResult
centralSample := centralSamples["HTTPHeader"]
var centralHeader measure.HTTPHeaderResult
if err := json.Unmarshal(clientSample.Data, &clientHeader); err != nil {
lg.Errorln(err)
continue loop
}
if err := json.Unmarshal(centralSample.Data, ¢ralHeader); err != nil {
lg.Error(err)
continue loop
}
// score data
score := scoreHTTPHeaders(clientHeader, centralHeader)
lg.V(10).Infof("session:%s %s", newTokenSample.Token, score)
if score.Score() >= 0.5 {
lg.Infoln("publishing session", newTokenSample.Token)
hostPublishC <- newTokenSample
} else {
lg.Infoln("not publishing session", newTokenSample.Token)
}
}
}