Exemplo n.º 1
0
func startMeasurer(dbclients db.Clients) {

	for n := 0; n < 10; n++ {

		go func() {
			var ps PreparedSample
			err := ps.Update(dbclients)
			if err != nil {
				lg.Error("could not resolve public ip address", err)
			}

			lg.V(5).Infoln("starting measurer")
			for r := range requestMeasurements {
				lg.V(50).Infoln("got measurement", r)
				if ps.lastUpdated.Before(time.Now().Add(-time.Hour * 5)) {
					lg.V(15).Info("updating prepared sample", ps)
					err := ps.Update(dbclients)
					if err != nil {
						lg.Warning(err)
					}

				}
			measurerLoop:
				for _, v := range r.measurers {
					measurement, err := v.Measure()
					if err != nil {
						lg.Errorf("could not measure:%v error:%s", v, err.Error())
						continue measurerLoop
					}
					switch measurement.Type() {
					case sampletypes.DNSQuery, sampletypes.HTTPHeader:

						data, err := measurement.Marshal()
						if err != nil {
							lg.Errorf("could not decode %v error:%s", measurement, err.Error())
							continue measurerLoop
						}
						err = dbclients.DB.InsertSample(db.Sample{
							Host:        measurement.Host(),
							CountryCode: ps.s.CountryCode,
							Token:       r.token,
							ASN:         ps.s.ASN,
							Type:        measurement.Type().String(),
							Origin:      sampleorigins.Central.String(),
							Data:        data,
						})
						if err != nil {
							lg.Errorln(err.Error())
							continue measurerLoop
						}
					default:
						lg.Errorf("could not measure:%v error:%s", v, err.Error())
						continue measurerLoop
					}
				}
			}
		}()
	}
}
Exemplo n.º 2
0
// IP2CountryCode resolves an IP address to ISO country code using an geoip
// database.
func (m *Maxmind) IP2CountryCode(IP net.IP) string {
	var record onlyCountry // Or any appropriate struct
	err := m.mmCountryDB.Lookup(IP, &record)
	if err != nil {
		lg.Warning(err)
	}
	return record.Country.IsoCode
}
Exemplo n.º 3
0
func (m *Maxmind) IP2CityGeoNameID(IP net.IP) uint {

	var record onlyCity // Or any appropriate struct
	if m.mmCityDB != nil {
		err := m.mmCityDB.Lookup(IP, &record)
		if err != nil {
			lg.Warning(err)
		}
		return record.City.GeoNameID
	}
	return 0
}
Exemplo n.º 4
0
// PostLogRequest
func PostLog(w rest.ResponseWriter, r *rest.Request) {
	form := PostLogRequest{}
	err := r.DecodeJsonPayload(&form)
	if err != nil {
		lg.Warning(err)
		apiutils.ErrToAPIStatus(err)
		// apiutils.WriteRestError(w, apierrors.NewBadRequest("can't decode json"))
		// apiutils.WriteRestError(w, apierrors.NewBadRequest("can't decode json"))
		return
	}
	sender := r.PathParam("sender")
	lg.V(form.Level).Infof("{%s} %s: %s", sender, form.Context, form.Message)
	w.WriteJson(true)
}
Exemplo n.º 5
0
func startAnalysis(clients db.Clients) {
	go func() {
		tick := time.NewTicker(10 * time.Second)

		lastID, err := clients.DB.GetLastProcessedSampleID()
		if err != nil {
			lg.Warningln(err)
		}
		lg.Infof("starting analysis from sample ID %d", lastID)

		lastPersistedID := lastID

		for range tick.C {
			results, err := clients.DB.GetSamples(uint64(lastID), "")
			if err != nil {
				lg.Fatal(err)
			}
			n := 0
			start := time.Now()
		loop:
			for s := range results {
				n++
				if s.ID > lastID {
					lastID = s.ID
				}
				if s.Type == "NewClientToken" {
					if !shared.AcceptedHost(s.Host) {
						lg.Warningln("not accepted host id:", s.ID, s.Host)
						continue loop
					}
					err := clients.DB.PublishHost(s)
					if err != nil {
						lg.Warning(err)
					}
				}
			}
			if n != 0 && lg.V(15) {
				lg.Infof("processed %d samples in %s", n, time.Since(start).String())
			}
			if lastID != lastPersistedID {
				err = clients.DB.SetLastProcessedSampleID(lastID)
				if err != nil {
					lg.Errorln(err)
				} else {
					lastPersistedID = lastID
				}
			}
		}
	}()
}
Exemplo n.º 6
0
func (d *DB) GetExportSamples(req shared.ExportSampleRequest) ([]shared.ExportSampleEntry, string, error) {
	var results []shared.ExportSampleEntry

	psql := squirrel.StatementBuilder.PlaceholderFormat(squirrel.Dollar)

	i := psql.
		Select("id", "host", "country_code", "asn", "created_at", "origin", "type", "token", "data", "extra_data").
		From("samples").
		OrderBy("id desc").
		Limit(PageLength + 1)
	if req.IDMax != 0 {
		i = i.Where("id < ?", req.IDMax)
	}
	rows, err := i.RunWith(d.cache).Query()
	if err != nil {
		logSQLErr(err, &i)
		return nil, "", err
	}
	defer rows.Close()
	next := ""
	count := 0
	for rows.Next() {
		var i shared.ExportSampleEntry
		err := rows.Scan(
			&i.ID,
			&i.Host,
			&i.CountryCode,
			&i.ASN,
			&i.CreatedAt,
			&i.Origin,
			&i.Type,
			&i.Token,
			&i.Data,
			&i.ExtraData,
		)
		count++
		if err != nil {
			lg.Warning(err)
			continue
		}
		if count > PageLength {
			next = i.ID
		} else {
			results = append(results, i)
		}
	}
	return results, next, nil
}
Exemplo n.º 7
0
func (d *DB) GetExportBlockedHosts(req shared.BlockedContentRequest) ([]shared.HostsPublishLog, string, error) {
	var results []shared.HostsPublishLog

	psql := squirrel.StatementBuilder.PlaceholderFormat(squirrel.Dollar)

	i := psql.
		Select("id", "host", "country_code", "asn", "created_at", "sticky", "action").
		From("hosts_publish_log").
		OrderBy("id desc").
		Limit(PageLength + 1)
	if req.IDMax != 0 {
		i = i.Where("id < ?", req.IDMax)
	}
	rows, err := i.RunWith(d.cache).Query()
	if err != nil {
		logSQLErr(err, &i)
		return nil, "", err
	}
	defer rows.Close()
	next := ""
	count := 0
	for rows.Next() {
		var item shared.HostsPublishLog
		err := rows.Scan(
			&item.ID,
			&item.Host,
			&item.CountryCode,
			&item.ASN,
			&item.CreatedAt,
			&item.Sticky,
			&item.Action,
		)
		count++
		if err != nil {
			lg.Warning(err)
			continue
		}
		if count > PageLength {
			next = item.ID
		} else {
			results = append(results, item)
		}
	}
	return results, next, nil
}
Exemplo n.º 8
0
func (d *DB) GetSessionSamples(token shared.SuggestionToken) ([]Sample, error) {
	psql := squirrel.StatementBuilder.PlaceholderFormat(squirrel.Dollar)

	//	fromID := 0
	i := psql.
		Select("id", "host", "country_code", "asn", "created_at", "origin", "type", "token", "data", "extra_data").
		From("samples").
		Where(squirrel.Eq{"token": string(token)})

	rows, err := i.RunWith(d.cache).Query()

	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var results []Sample

	for rows.Next() {
		var sample Sample
		var token string
		err := rows.Scan(
			&sample.ID,
			&sample.Host,
			&sample.CountryCode,
			&sample.ASN,
			&sample.CreatedAt,
			&sample.Origin,
			&sample.Type,
			&token,
			&sample.Data,
			&sample.ExtraData,
		)
		if err != nil {
			lg.Warning(err)
			continue
		}
		sample.Token = shared.SuggestionToken(token)
		results = append(results, sample)
	}

	return results, nil
}
Exemplo n.º 9
0
func WriteRestError(w rest.ResponseWriter, err error) {
	var status shared.Status
	if _, ok := err.(*apierrors.StatusError); !ok {
		err = apierrors.NewInternalError(err)
	}
	switch err := err.(type) {
	case *apierrors.StatusError:
		status = err.Status()
	default:
		panic(err)
	}
	if status.Code > 299 && lg.V(19) {
		lg.WarningDepth(1, fmt.Sprintf("apierror %d: %+v", status.Code, status))
	}
	w.WriteHeader(status.Code)
	err = w.WriteJson(&status)
	if err != nil {
		lg.Warning(err)
	}
}
Exemplo n.º 10
0
func (d *DB) GetSamples(fromID uint64, sampleType string) (chan Sample, error) {
	psql := squirrel.StatementBuilder.PlaceholderFormat(squirrel.Dollar)

	i := psql.
		Select("id", "host", "country_code", "asn", "created_at", "origin", "type", "token", "data", "extra_data").
		From("samples").
		Where("id > ?", fromID)
	rows, err := i.RunWith(d.cache).Query()
	if err != nil {
		return nil, err
	}

	results := make(chan Sample, 0)
	go func(rows *sql.Rows) {
		defer rows.Close()
		defer close(results)
		for rows.Next() {
			var token string
			var sample Sample
			err := rows.Scan(
				&sample.ID,
				&sample.Host,
				&sample.CountryCode,
				&sample.ASN,
				&sample.CreatedAt,
				&sample.Origin,
				&sample.Type,
				&token,
				&sample.Data,
				&sample.ExtraData,
			)
			if err != nil {
				lg.Warning(err)
				continue
			}
			sample.Token = shared.SuggestionToken(token)
			results <- sample
		}
	}(rows)
	return results, nil
}
Exemplo n.º 11
0
func (d *DB) GetLastProcessedSampleID() (uint64, error) {
	psql := squirrel.StatementBuilder.PlaceholderFormat(squirrel.Dollar)
	s := psql.Select("value").From("central_state").Where(
		squirrel.Eq{"name": "last_processed_sample_id"},
	).Limit(1)

	row := s.RunWith(d.cache).QueryRow()
	var valuestr string
	err := row.Scan(&valuestr)
	if err != nil {
		if err == sql.ErrNoRows {
			lg.Warning("last processed sample id not set, returning 0 (default value)")
			return 0, nil
		}
		return 0, err
	}
	v, err := strconv.ParseUint(valuestr, 10, 64)
	if err != nil {
		return 0, err
	}
	return v, nil
}
Exemplo n.º 12
0
// Init does precondition check if the application can/should be started.
// Init will return an error message with reason for exit printed.
func Run() {
	if debugEnabled {
		log.Println("ALKASIR_DEBUG ENABLED!")
		err := os.Setenv("ALKASIR_DEBUG", "1")
		if err != nil {
			log.Fatal(err)
		}
	}
	if hotEnabled {
		log.Println("ALKASIR_HOT ENABLED!")
		err := os.Setenv("ALKASIR_HOT", "1")
		if err != nil {
			log.Fatal(err)
		}
	}

	// the darwin systray does not exit the main loop
	if runtime.GOOS != "darwin" {
		uiRunning.Add(1)
	}

	err := ui.Run(func() {
		Atexit(ui.Done)

		// start the getpublic ip updater.
		go func() {
			_ = shared.GetPublicIPAddr()
		}()

		if debugEnabled {
			go func() {
				err := http.ListenAndServe(
					fmt.Sprintf("localhost:%d", debugPort), nil)
				if err != nil {
					panic(err)
				}
			}()
		}

		// wipe user data
		if wipeData {
			settingsdir := clientconfig.ConfigPath()
			if settingsdir == "" {
				log.Println("[wipe] Configdir not set")
				os.Exit(1)
			}
			settingsfile := clientconfig.ConfigPath("settings.json")
			if _, err := os.Stat(settingsfile); os.IsNotExist(err) {
				log.Println("[wipe] No settings.json in configdir, will NOT wipe data")
				os.Exit(1)
			}
			log.Println("Wiping all user data")
			if err := os.RemoveAll(settingsdir); err != nil {
				log.Println(err)
			}
		}

		// Prepare logging
		logdir := clientconfig.ConfigPath("log")
		err := os.MkdirAll(logdir, 0775)
		if err != nil {
			log.Println("Could not create logging directory")
			os.Exit(1)
		}
		err = flag.Set("log_dir", logdir)
		if err != nil {
			panic(err)
		}
		lg.SetSrcHighlight("alkasir/cmd", "alkasir/pkg")
		lg.CopyStandardLogTo("INFO")

		// Start init
		if VERSION != "" {
			lg.Infoln("Alkasir v" + VERSION)
		} else {
			lg.Warningln("Alkasir dev version (VERSION not set)")
		}

		lg.V(1).Info("Log v-level:", lg.Verbosity())
		_, err = clientconfig.Read()
		if err != nil {
			lg.Infoln("Could not read config")
			exit()
		}
		lg.V(30).Infoln("settings", clientconfig.Get().Settings)

		if saveChromeExt {
			err := saveChromeExtension()
			if err != nil {
				lg.Fatal(err)
			}
		}

		{
			configChanged, err := clientconfig.UpgradeConfig()
			if err != nil {
				lg.Fatalln("Could not upgrade config", err)

			}
			clientconfig.Update(func(conf *clientconfig.Config) error {

				if clientAuthKeyFlag != "" {
					lg.Warningln("Overriding generated authKey with", clientAuthKeyFlag)
					conf.Settings.Local.ClientAuthKey = clientAuthKeyFlag
					configChanged = true
				}

				if bindAddrFlag != "" {
					lg.Warningln("Overriding configured bindAddr with", bindAddrFlag)
					conf.Settings.Local.ClientBindAddr = bindAddrFlag
					configChanged = true
				}

				if centralAddrFlag != "" {
					lg.Warningln("Overriding central server addr with", centralAddrFlag)
					conf.Settings.Local.CentralAddr = centralAddrFlag
					configChanged = true
				}

				return nil
			})

			if configChanged {
				if err := clientconfig.Write(); err != nil {
					lg.Warning(err)
				}
			}

		}
		conf := clientconfig.Get()
		loadTranslations(LanguageOptions...)
		if err := ui.Language(conf.Settings.Local.Language); err != nil {
			lg.Warningln(err)
		}

		go func() {
			select {
			case <-sigIntC:
				exit()
			case <-ui.Actions.Quit:
				exit()
			}
		}()

		for _, e := range []error{
			mime.AddExtensionType(".json", "application/json"),
			mime.AddExtensionType(".js", "application/javascript"),
			mime.AddExtensionType(".css", "text/css"),
			mime.AddExtensionType(".md", "text/plain"),
		} {
			if e != nil {
				lg.Warning(e)
			}
		}

		err = startInternalHTTPServer(conf.Settings.Local.ClientAuthKey)
		if err != nil {
			lg.Fatal("could not start internal http services")
		}

		// Connect the default transport
		service.UpdateConnections(conf.Settings.Connections)
		service.UpdateTransports(conf.Settings.Transports)
		go service.StartConnectionManager(conf.Settings.Local.ClientAuthKey)

		// TODO: async
		pac.UpdateDirectList(conf.DirectHosts.Hosts)
		pac.UpdateBlockedList(conf.BlockedHostsCentral.Hosts, conf.BlockedHosts.Hosts)
		lastBlocklistChange = time.Now()

		go StartBlocklistUpgrader()
		if upgradeDiffsBaseURL != "" {
			lg.V(19).Infoln("upgradeDiffsBaseURL is ", upgradeDiffsBaseURL)
			go StartBinaryUpgradeChecker(upgradeDiffsBaseURL)
		} else {
			lg.Warningln("empty upgradeDiffsBaseURL, disabling upgrade checks")
		}

		lg.V(5).Info("Alkasir has started")

	})
	// the darwin systray does not exit the main loop
	if runtime.GOOS != "darwin" {
		uiRunning.Done()
	}

	lg.Infoln("ui.Run ended")
	if err != nil {
		log.Println("client.Run error:", err)
	}
}
Exemplo n.º 13
0
// SubmitSuggestion initiates the comminication with Central for a Submission
// session.
func SubmitSuggestion(w rest.ResponseWriter, r *rest.Request) {

	// // TODO This is the response that must be forwarded from central/api and parsed by client and passed on to the browser.
	// apiutils.WriteRestError(w,
	// 	apierrors.NewInvalid("object", "suggestion",
	// 		fielderrors.ValidationErrorList{
	// 			fielderrors.NewFieldValueNotSupported("URL", "...", []string{})}))
	// return

	ID := r.PathParam("id")
	suggestion, ok := client.GetSuggestion(ID)
	if !ok {
		apiutils.WriteRestError(w, apierrors.NewNotFound("suggestion", ID))
		return
	}
	wanip := shared.GetPublicIPAddr()
	if wanip == nil {
		lg.Warning("could not resolve public ip addr")
	}

	conf := clientconfig.Get()
	restclient, err := NewRestClient()
	if err != nil {
		apiutils.WriteRestError(w, apierrors.NewInternalError(err))
		return
	}

	tokenResp, err := suggestion.RequestToken(
		restclient, wanip, conf.Settings.Local.CountryCode)
	if err != nil {
		if apiutils.IsNetError(err) {
			apiutils.WriteRestError(w, apierrors.NewServerTimeout("alkasir-central", "request-submission-token", 0))
		} else {
			apiutils.WriteRestError(w, apierrors.NewInternalError(err))
		}
		return
	}
	n, err := suggestion.SendSamples(restclient)
	if err != nil {
		lg.Warningln("error sending samples", err.Error())
	}
	lg.V(5).Infof("sent %d samples", n)

	// continue sending samples if future measuremetns are expected to come
	prepared, err := suggestion.Prepared()
	if err != nil {
		lg.Errorln(err)
	} else if !prepared {
		lg.V(5).Infof("all samples not collected, will try to send the rest when they are done")
		go func(s client.Suggestion) {
			start := time.Now()
			t := time.NewTicker(30 * time.Second)
			defer t.Stop()

		submitSamples:
			for range t.C {
				if time.Now().After(start.Add(15 * time.Minute)) {
					lg.Errorln("Stopping trying to send additional samples")
					return
				}

				prepared, err := suggestion.Prepared()
				if err != nil {
					lg.Errorln(err)
					return
				}
				if prepared {
					restclient, err := NewRestClient()
					if err != nil {
						continue submitSamples
					}
					n, err := suggestion.SendSamples(restclient)
					lg.V(5).Infof("sent %d samples", n)
					if err != nil {
						lg.Warningln("error sending samples", err.Error())
						continue submitSamples
					}
					return
				}
			}
		}(suggestion)
	}

	u, err := url.Parse(suggestion.URL)
	if err != nil {
		lg.Errorln(err)
	} else {
		err := clientconfig.Update(func(conf *clientconfig.Config) error {
			conf.BlockedHosts.Add(u.Host)
			lastBlocklistChange = time.Now()

			pac.UpdateBlockedList(conf.BlockedHostsCentral.Hosts,
				conf.BlockedHosts.Hosts)
			return nil
		})
		if err != nil {
			lg.Errorln(err)
		}
	}
	w.WriteJson(tokenResp)
}
Exemplo n.º 14
0
// SubmitSuggestion initiates the comminication with Central for a Submission
// session.
func SubmitSuggestion(w rest.ResponseWriter, r *rest.Request) {

	// // TODO This is the response that must be forwarded from central/api and parsed by client and passed on to the browser.
	// apiutils.WriteRestError(w,
	// 	apierrors.NewInvalid("object", "suggestion",
	// 		fielderrors.ValidationErrorList{
	// 			fielderrors.NewFieldValueNotSupported("URL", "...", []string{})}))
	// return

	ID := r.PathParam("id")
	suggestion, ok := client.GetSuggestion(ID)
	if !ok {
		apiutils.WriteRestError(w, apierrors.NewNotFound("suggestion", ID))
		return
	}
	wanip := shared.GetPublicIPAddr()
	if wanip == nil {
		lg.Warning("could not resolve public ip addr")
	}

	conf := clientconfig.Get()
	restclient, err := NewRestClient()
	if err != nil {
		apiutils.WriteRestError(w, apierrors.NewInternalError(err))
		return
	}

	tokenResp, err := suggestion.RequestToken(
		restclient, wanip, conf.Settings.Local.CountryCode)
	if err != nil {
		if apiutils.IsNetError(err) {
			apiutils.WriteRestError(w, apierrors.NewServerTimeout("alkasir-central", "request-submission-token", 0))
		} else {
			apiutils.WriteRestError(w, apierrors.NewInternalError(err))
		}
		return
	}
	n, err := suggestion.SendSamples(restclient)
	if err != nil {
		lg.Warningln("error sending samples", err.Error())
	}

	lg.V(5).Infoln("sent ", n)
	// FIXME PRESENTATION: just add the url locally
	u, err := url.Parse(suggestion.URL)
	if err != nil {
		lg.Errorln(err)
	} else {
		err := clientconfig.Update(func(conf *clientconfig.Config) error {
			conf.BlockedHosts.Add(u.Host)
			lastBlocklistChange = time.Now()

			pac.UpdateBlockedList(conf.BlockedHostsCentral.Hosts,
				conf.BlockedHosts.Hosts)
			return nil
		})
		if err != nil {
			lg.Errorln(err)
		}
	}
	w.WriteJson(tokenResp)
}
Exemplo n.º 15
0
func startInternalHTTPServer(authKey string) error {

	lg.V(15).Infoln("starting internal api server")
	mux, err := createServeMux()
	if err != nil {
		lg.Error("failed to create router")
		return err
	}

	auth := Auth{Key: authKey, wrapped: mux}
	conf := clientconfig.Get()
	listener, err := net.Listen("tcp", conf.Settings.Local.ClientBindAddr)
	if err != nil {
		lg.Warning(err)
		listener, err = net.Listen("tcp", "127.0.0.1:")
	}
	if err != nil {
		lg.Warning(err)
		ui.Notify("Could not bind any local port (bootstrap)")
		lg.Errorln("Could not bind any local port (bootstrap)")
		return err
	}

	go func(listenaddr string) {
		// baseURL := fmt.Sprintf()
		baseURL := fmt.Sprintf("http://%s?suk=", listenaddr)
		for {
			select {
			case <-ui.Actions.CopyBrowserCodeToClipboard:
				bc := browsercode.BrowserCode{Key: authKey}
				err := bc.SetHostport(listener.Addr().String())
				if err != nil {
					lg.Errorln(err)
					continue
				}
				err = bc.CopyToClipboard()
				if err != nil {
					lg.Errorln(err)
				}

			case <-ui.Actions.OpenInBrowser:
				browser.OpenURL(baseURL + singleUseKeys.New() + "#/")

			case <-ui.Actions.Help:
				browser.OpenURL(baseURL + singleUseKeys.New() + "#/docs/__/index")
			}
			singleUseKeys.Cleanup()
		}
	}(listener.Addr().String())

	doneC := make(chan bool, 1)
	go func() {
		defer listener.Close()
		err = http.Serve(listener, auth)
		if err != nil {
			doneC <- false
		}
	}()
	select {
	case ok := <-doneC:
		if !ok {
			return errors.New("Could not start internal http server")
		}
	case <-time.After(time.Millisecond * 200):
		return nil
	}
	return nil
}