Exemple #1
0
func startAnalysis(clients db.Clients) {
	go func() {
		tick := time.NewTicker(10 * time.Second)

		lastID, err := clients.DB.GetLastProcessedSampleID()
		if err != nil {
			lg.Warningln(err)
		}
		lg.Infof("starting analysis from sample ID %d", lastID)

		lastPersistedID := lastID

		for range tick.C {
			results, err := clients.DB.GetSamples(uint64(lastID), "")
			if err != nil {
				lg.Fatal(err)
			}
			n := 0
			start := time.Now()
		loop:
			for s := range results {
				n++
				if s.ID > lastID {
					lastID = s.ID
				}
				if s.Type == "NewClientToken" {
					if !shared.AcceptedHost(s.Host) {
						lg.Warningln("not accepted host id:", s.ID, s.Host)
						continue loop
					}
					err := clients.DB.PublishHost(s)
					if err != nil {
						lg.Warning(err)
					}
				}
			}
			if n != 0 && lg.V(15) {
				lg.Infof("processed %d samples in %s", n, time.Since(start).String())
			}
			if lastID != lastPersistedID {
				err = clients.DB.SetLastProcessedSampleID(lastID)
				if err != nil {
					lg.Errorln(err)
				} else {
					lastPersistedID = lastID
				}
			}
		}
	}()
}
Exemple #2
0
func samplesAnalyzer() {
loop:
	for samples := range sampleAnalysisC {
		var (
			newTokenSample db.Sample
			clientSamples  = make(map[string]db.Sample, 0)
			centralSamples = make(map[string]db.Sample, 0)
		)

		// organize input data
		for _, s := range samples {
			switch s.Type {
			case "NewClientToken":
				if newTokenSample.Token != "" {
					lg.Error("got more than one newTokenSample, aborting")
					continue loop
				}
				newTokenSample = s
			case "HTTPHeader", "DNSQuery":
				switch s.Origin {
				case "Central":
					centralSamples[s.Type] = s
				case "Client":
					clientSamples[s.Type] = s
				}
			default:
				lg.Errorf("dont know how to handle %d %s, skipping", s.ID, s.Type)
			}
		}

		// validate that wanted data types are available
		if newTokenSample.Token == "" {
			lg.Errorln("No newTokenSample, aborting")
			continue loop
		}
		if !shared.AcceptedHost(newTokenSample.Host) {
			lg.Warningln("not accepted host id:", newTokenSample.ID, newTokenSample.Host)
			continue loop
		}
		for _, smap := range []map[string]db.Sample{clientSamples, centralSamples} {
			for _, stype := range []string{"HTTPHeader"} {
				if _, ok := smap[stype]; !ok {
					lg.Errorf("missing %s, cannot analyse", stype)
					continue loop
				}
			}
		}

		// parse data
		clientSample := clientSamples["HTTPHeader"]
		var clientHeader measure.HTTPHeaderResult
		centralSample := centralSamples["HTTPHeader"]
		var centralHeader measure.HTTPHeaderResult

		if err := json.Unmarshal(clientSample.Data, &clientHeader); err != nil {
			lg.Errorln(err)
			continue loop
		}
		if err := json.Unmarshal(centralSample.Data, &centralHeader); err != nil {
			lg.Error(err)
			continue loop
		}

		// score data
		score := scoreHTTPHeaders(clientHeader, centralHeader)
		lg.V(10).Infof("session:%s %s", newTokenSample.Token, score)

		if score.Score() >= 0.5 {
			lg.Infoln("publishing session", newTokenSample.Token)
			hostPublishC <- newTokenSample
		} else {
			lg.Infoln("not publishing session", newTokenSample.Token)
		}
	}
}