示例#1
0
func startMeasurer(dbclients db.Clients) {

	for n := 0; n < 10; n++ {

		go func() {
			var ps PreparedSample
			err := ps.Update(dbclients)
			if err != nil {
				lg.Error("could not resolve public ip address", err)
			}

			lg.V(5).Infoln("starting measurer")
			for r := range requestMeasurements {
				lg.V(50).Infoln("got measurement", r)
				if ps.lastUpdated.Before(time.Now().Add(-time.Hour * 5)) {
					lg.V(15).Info("updating prepared sample", ps)
					err := ps.Update(dbclients)
					if err != nil {
						lg.Warning(err)
					}

				}
			measurerLoop:
				for _, v := range r.measurers {
					measurement, err := v.Measure()
					if err != nil {
						lg.Errorf("could not measure:%v error:%s", v, err.Error())
						continue measurerLoop
					}
					switch measurement.Type() {
					case sampletypes.DNSQuery, sampletypes.HTTPHeader:

						data, err := measurement.Marshal()
						if err != nil {
							lg.Errorf("could not decode %v error:%s", measurement, err.Error())
							continue measurerLoop
						}
						err = dbclients.DB.InsertSample(db.Sample{
							Host:        measurement.Host(),
							CountryCode: ps.s.CountryCode,
							Token:       r.token,
							ASN:         ps.s.ASN,
							Type:        measurement.Type().String(),
							Origin:      sampleorigins.Central.String(),
							Data:        data,
						})
						if err != nil {
							lg.Errorln(err.Error())
							continue measurerLoop
						}
					default:
						lg.Errorf("could not measure:%v error:%s", v, err.Error())
						continue measurerLoop
					}
				}
			}
		}()
	}
}
示例#2
0
func exit() {
	lg.Infoln("alkasir is shutting down")
	atexitMu.Lock() // this lock should be kept, one shutdown should be enough for everyone.
	lg.Flush()
	if err := clientconfig.Write(); err != nil {
		lg.Errorf("could not save config file: %s", err.Error())
	}
	lg.V(9).Infoln("running atexit funcs")
	for _, f := range atexitFuncs {
		funcName := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
		lg.V(10).Infoln("Running at exit func", funcName)
		f()
		lg.V(10).Infoln("Finished at exit func", funcName)
	}
	atexitFuncs = atexitFuncs[:0]
	lg.V(9).Infoln("atexit funcs done")
	lg.V(9).Infoln("stopping connectionmanager")
	service.StopConnectionManager()
	lg.V(9).Infoln("stopping services")
	service.StopAll()
	lg.V(9).Infoln("services stopped")
	lg.V(9).Infoln("waiting for UI shutdown to finish")
	uiRunning.Wait()
	lg.V(9).Infoln("ui shut down")

	lg.Flush()
	lg.Infoln("alkasir shutdown complete")
	lg.Flush()
	time.Sleep(time.Millisecond * 1)
	os.Exit(0)
}
示例#3
0
func debugImportDebug(files []string) error {
	if len(files) == 0 {
		fmt.Println("need argument: files...")
		return errNoValue
	}

files:
	for _, filename := range files {
		data, err := ioutil.ReadFile(filename)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}

		var debuginfo debugexport.DebugResponse
		err = json.Unmarshal(data, &debuginfo)
		if err != nil {
			fmt.Printf("could not decode %s: %s \n", filename, err.Error())
			os.Exit(1)
		}
		if debuginfo.Encrypted != "" {
			err := debuginfo.Decrypt(debugPublicKey, debugSecretKey)
			if err != nil {
				lg.Errorf("%s could not be decrypted, skipping: %v", filename, err)
				continue files
			}
		}
		debuginfo.WriteToDisk()
	}
	return nil
}
示例#4
0
func StartAnalysis(clients db.Clients) {

	tick := time.NewTicker(5 * time.Second)
	lastID, err := clients.DB.GetLastProcessedSampleID()
	if err != nil {
		lg.Warningln(err)
	}

	for n := 0; n < 4; n++ {
		go sessionFetcher(clients)
		go samplesAnalyzer()
		go hostPublisher(clients)
	}

	lg.Infof("starting analysis from sample ID %d", lastID)

	lastPersistedID := lastID

	for range tick.C {
		results, err := clients.DB.GetSamples(uint64(lastID), "")
		if err != nil {
			lg.Errorf("database err (skipping): %v", err)
			continue
		}
		n := 0
		start := time.Now()
		for s := range results {
			n++
			if s.ID > lastID {
				lastID = s.ID
			}

			if s.Origin == "Central" && s.Type == "HTTPHeader" {
				sessionFetchC <- s.Token
			}
		}
		if n != 0 && lg.V(15) {
			lg.Infof("processed %d samples in %s", n, time.Since(start).String())
		}
		if lastID != lastPersistedID {
			err = clients.DB.SetLastProcessedSampleID(lastID)
			if err != nil {
				lg.Errorln(err)
			} else {
				lastPersistedID = lastID
			}
		}
	}

}
示例#5
0
// StartBlocklistUpgrader react to certain conitions for when the list of
// blocked urls should be updated.
//
// This function runs in it's own goroutine.
func StartBlocklistUpgrader() {
	connectionEventListener := make(chan service.ConnectionHistory)
	uChecker, _ := NewUpdateChecker("blocklist")
	service.AddListener(connectionEventListener)
	currentCountry := clientconfig.Get().Settings.Local.CountryCode
	checkCountrySettingC := time.NewTicker(2 * time.Second)
	defer checkCountrySettingC.Stop()
loop:
	for {
		select {
		// Update when the transport connection comes up
		case event := <-connectionEventListener:
			if event.IsUp() {
				uChecker.Activate()
				uChecker.UpdateNow()
			}

		// Tell updatechecker to request update when user changes country settings
		case <-checkCountrySettingC.C:
			conf := clientconfig.Get()
			if currentCountry != conf.Settings.Local.CountryCode {
				currentCountry = conf.Settings.Local.CountryCode
				uChecker.UpdateNow()
			}

		// Update by request of the update checker
		case request := <-uChecker.RequestC:
			conf := clientconfig.Get()
			if conf.Settings.Local.CountryCode == "__" {
				lg.V(9).Infoln("Country is __, skipping blocklist updates")
				continue loop
			}
			currentCountry = conf.Settings.Local.CountryCode
			n, err := upgradeBlockList()
			if err != nil {
				lg.Errorf("blocklist update cc:%s err:%v", currentCountry, err)
				ui.Notify("blocklist_update_error_message")
				request.ResponseC <- UpdateError
			} else {
				lg.V(5).Infof("blocklist update success cc:%s, got %d entries", currentCountry, n)
				ui.Notify("blocklist_update_success_message")
				request.ResponseC <- UpdateSuccess
			}
		}
	}
}
示例#6
0
// String converts a ValidationErrorType into its corresponding error message.
func (t ValidationErrorType) String() string {
	switch t {
	case ValidationErrorTypeNotFound:
		return "not found"
	case ValidationErrorTypeRequired:
		return "required value"
	case ValidationErrorTypeDuplicate:
		return "duplicate value"
	case ValidationErrorTypeInvalid:
		return "invalid value"
	case ValidationErrorTypeNotSupported:
		return "unsupported value"
	case ValidationErrorTypeForbidden:
		return "forbidden"
	case ValidationErrorTypeTooLong:
		return "too long"
	default:
		glog.Errorf("unrecognized validation type: %#v", t)
		return ""
	}
}
示例#7
0
func samplesAnalyzer() {
loop:
	for samples := range sampleAnalysisC {
		var (
			newTokenSample db.Sample
			clientSamples  = make(map[string]db.Sample, 0)
			centralSamples = make(map[string]db.Sample, 0)
		)

		// organize input data
		for _, s := range samples {
			switch s.Type {
			case "NewClientToken":
				if newTokenSample.Token != "" {
					lg.Error("got more than one newTokenSample, aborting")
					continue loop
				}
				newTokenSample = s
			case "HTTPHeader", "DNSQuery":
				switch s.Origin {
				case "Central":
					centralSamples[s.Type] = s
				case "Client":
					clientSamples[s.Type] = s
				}
			default:
				lg.Errorf("dont know how to handle %d %s, skipping", s.ID, s.Type)
			}
		}

		// validate that wanted data types are available
		if newTokenSample.Token == "" {
			lg.Errorln("No newTokenSample, aborting")
			continue loop
		}
		if !shared.AcceptedHost(newTokenSample.Host) {
			lg.Warningln("not accepted host id:", newTokenSample.ID, newTokenSample.Host)
			continue loop
		}
		for _, smap := range []map[string]db.Sample{clientSamples, centralSamples} {
			for _, stype := range []string{"HTTPHeader"} {
				if _, ok := smap[stype]; !ok {
					lg.Errorf("missing %s, cannot analyse", stype)
					continue loop
				}
			}
		}

		// parse data
		clientSample := clientSamples["HTTPHeader"]
		var clientHeader measure.HTTPHeaderResult
		centralSample := centralSamples["HTTPHeader"]
		var centralHeader measure.HTTPHeaderResult

		if err := json.Unmarshal(clientSample.Data, &clientHeader); err != nil {
			lg.Errorln(err)
			continue loop
		}
		if err := json.Unmarshal(centralSample.Data, &centralHeader); err != nil {
			lg.Error(err)
			continue loop
		}

		// score data
		score := scoreHTTPHeaders(clientHeader, centralHeader)
		lg.V(10).Infof("session:%s %s", newTokenSample.Token, score)

		if score.Score() >= 0.5 {
			lg.Infoln("publishing session", newTokenSample.Token)
			hostPublishC <- newTokenSample
		} else {
			lg.Infoln("not publishing session", newTokenSample.Token)
		}
	}
}
示例#8
0
func createUpgradeAuto(args []string) error {
	var (
		privPemFlag string
		pubPemFlag  string
	)
	fs := flag.NewFlagSet("upgrade create", flag.ExitOnError)
	fs.StringVar(&privPemFlag, "privpem", "upgrades-private-key.pem", "path to load private key file from")
	fs.StringVar(&pubPemFlag, "pubpem", "upgrades-public-key.pem", "path to load public key file from")
	fs.Parse(args)
	args = fs.Args()

	privPem, err := ioutil.ReadFile(privPemFlag)
	if err != nil {
		if os.IsNotExist(err) {
			lg.Errorf("%s does not exist", privPemFlag)
			return nil
		}
		return err
	}

	pubPem, err := ioutil.ReadFile(pubPemFlag)
	if err != nil {
		if os.IsNotExist(err) {
			lg.Errorf("%s does not exist", pubPemFlag)
			return nil
		}
		return err
	}

	results, err := makepatch.RunPatchesCreate(
		jobQs, string(privPem), string(pubPem), nWorkersFlag)
	if err != nil {
		panic(err)
	}
	if len(results) < 1 {
		lg.Fatalln("no patch results returned")
	}

	var allFiles []*tar.Header
	err = filepath.Walk("diffs", func(path string, f os.FileInfo, err error) error {
		if f.IsDir() {
			return nil
		}

		allFiles = append(allFiles, &tar.Header{
			Name: path,
			Mode: 0600,
			Size: f.Size(),
		})
		return nil
	})
	if err != nil {
		lg.Fatal(err)
	}

	latestVersion := results[0].NewVersion
	filename := fmt.Sprintf("alkasir-binpatches-for-%s.tar", latestVersion)
	tarfile, err := os.Create(filename)
	if err != nil {
		panic(err)
	}

	tw := tar.NewWriter(tarfile)

	for _, hdr := range allFiles {
		if err := tw.WriteHeader(hdr); err != nil {
			log.Fatalln(err)
		}
		s, err := os.Open(hdr.Name)
		if err != nil {
			return err
		}
		_, err = io.Copy(tw, s)
		if err != nil {
			lg.Fatal(err)
		}

		err = s.Close()
		if err != nil {
			lg.Fatal(err)
		}

	}

	if err := tw.Close(); err != nil {
		log.Fatalln(err)
	}

	lg.Infoln("done")
	return nil
}
示例#9
0
文件: api.go 项目: thomasf/alkasir
// CreateSuggestion .
func CreateSuggestion(w rest.ResponseWriter, r *rest.Request) {

	form := shared.BrowserSuggestionTokenRequest{}
	err := r.DecodeJsonPayload(&form)
	if err != nil {
		// apiError(w, err.Error(), http.StatusInternalServerError)
		apiutils.WriteRestError(w, err)
		return
	}

	var invalids fielderrors.ValidationErrorList

	// parse and validate url.
	URL := strings.TrimSpace(form.URL)
	if URL == "" {
		invalids = append(invalids, fielderrors.NewFieldRequired("URL"))
		// apiError(w, "no or empty URL", http.StatusBadRequest)

	}
	u, err := url.Parse(URL)
	if err != nil {
		invalids = append(invalids, fielderrors.NewFieldInvalid("URL", URL, err.Error()))
		// apiError(w, fmt.Sprintf("%s is not a valid URL", URL), http.StatusBadRequest)

	}

	host := u.Host
	if strings.Contains(host, ":") {
		host, _, err = net.SplitHostPort(u.Host)
		if err != nil {
			invalids = append(invalids, fielderrors.NewFieldInvalid("URL", URL, err.Error()))
		}
	}

	if !shared.AcceptedURL(u) {
		invalids = append(invalids, fielderrors.NewFieldValueNotSupported("URL", URL, nil))
	}

	if len(invalids) > 0 {
		apiutils.WriteRestError(w, apierrors.NewInvalid("create-suggestion", "URL", invalids))
		return
	}

	s := client.NewSuggestion(u.String())
	defer s.DoneAddingSamples()
	measurers, err := measure.DefaultMeasurements(form.URL)
	if err != nil {
		apiutils.WriteRestError(w, apierrors.NewInternalError(err))
		return
	}

	for _, v := range measurers {
		m, err := v.Measure()
		if err != nil {
			lg.Errorf("could not measure: %s", err.Error())
		} else {
			switch m.Type() {
			case sampletypes.DNSQuery, sampletypes.HTTPHeader:
				err = s.AddMeasurement(m)
				if err != nil {
					lg.Errorln(err.Error())
					return
				}
			default:
				lg.Warningf("unsupported sample type: %s", m.Type().String())
			}
		}
	}
}
示例#10
0
文件: api.go 项目: thomasf/alkasir
// Update list of blocked hosts for an IP address.
func GetHosts(dbclients db.Clients) func(w rest.ResponseWriter, r *rest.Request) {
	relh := relatedHosts{
		dbclients: dbclients,
	}
	relh.update()
	go func() {
		for range time.NewTicker(10 * time.Minute).C {
			relh.update()
		}
	}()
	return func(w rest.ResponseWriter, r *rest.Request) {

		// HANDLE USERIP BEGIN
		req := shared.UpdateHostlistRequest{}
		err := r.DecodeJsonPayload(&req)
		if err != nil {
			apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError)
			return
		}

		// parse/validate client ip address.
		IP := req.ClientAddr
		if IP == nil {
			apiError(w, "bad ClientAddr", http.StatusBadRequest)
			return
		}

		// resolve ip to asn.
		var ASN int
		ASNres, err := dbclients.Internet.IP2ASN(IP)
		if err != nil {
			lg.Errorln(shared.SafeClean(err.Error()))
			apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError)
			return
		}
		if ASNres != nil {
			ASN = ASNres.ASN
		} else {
			lg.Warningf("no ASN lookup result for IP: %s ", shared.SafeClean(IP.String()))
		}

		// resolve ip to country code.
		countryCode := dbclients.Maxmind.IP2CountryCode(IP)

		req.ClientAddr = net.IPv4zero
		IP = net.IPv4zero
		// HANDLE USERIP END

		hosts, err := dbclients.DB.GetBlockedHosts(countryCode, ASN)
		if err != nil {
			apiError(w, err.Error(), http.StatusInternalServerError)
			return
		}

		simpleData := struct {
			ClientVersion string `json:"version"` // client version idientifer
		}{
			req.ClientVersion,
		}
		data, err := json.Marshal(simpleData)
		if err != nil {
			lg.Errorln(err)
		}

		ss := db.SimpleSample{
			CountryCode: countryCode,
			ASN:         ASN,
			Type:        "ClientBlocklistUpdate",
			OriginID:    req.UpdateID,
			Data:        data,
		}
		err = dbclients.DB.InsertSimpleSample(ss)
		if err != nil {
			lg.Errorf("error persisting simplesample %v", ss)
		}

		err = w.WriteJson(shared.UpdateHostlistResponse{
			Hosts: relh.fill(hosts),
		})
		if err != nil {
			lg.Error(err)
			return
		}

	}
}
示例#11
0
// initDone means handing off the service process output to it's own goroutine.
func (s *Service) initDone() error {

	lg.V(5).Infof("s.request: %+v", s.Request)
	lg.V(5).Infof("s.response: %+v", s.Response)
	s.waiter.Add(1)
	s.running = true
	go func() {
		defer func() {
			s.running = false
		}()

		go func() {
			scanner := bufio.NewScanner(s.stdout)
			for scanner.Scan() {
				if scanner.Err() != nil {
					lg.Warningln("service stdout", scanner.Err())
					break
				}
				line := scanner.Text()
				lg.Infoln(s.ID, "stdout:", line)
			}
			lg.V(20).Infof("service outch closed %s", s.ID)
		}()

		go func() {
			scanner := bufio.NewScanner(s.stderr)
			for scanner.Scan() {
				if scanner.Err() != nil {
					lg.Warningln("service stderr", scanner.Err())
					break
				}
				line := scanner.Text()
				lg.Infoln(s.ID, "stderr:", line)
			}
			lg.V(20).Infof("service stderr closed %s", s.ID)
		}()

		s.quit = make(chan bool)
		defer close(s.quit)

		select {
		case <-s.quit:
			lg.V(6).Infof("stopping service %s", s.ID)

			if err := s.stdin.Close(); err != nil {
				lg.Errorf("could not close stdin for %s: %s", s.ID, err.Error())
			}

			if err := s.stdout.Close(); err != nil {
				lg.Errorf("could not close stdout for %s: %s", s.ID, err.Error())
			}

			if err := s.stderr.Close(); err != nil {
				lg.Errorf("could not close stderr for %s: %s", s.ID, err.Error())
			}

			lg.V(10).Infof("Killing process service %s", s.ID)

		}

		lg.V(10).Infof("stopped service %s", s.ID)
		if s.removeOnStop {
			ManagedServices.remove(s)
		}
		s.waiter.Done()
	}()
	return nil
}