func startMeasurer(dbclients db.Clients) {
for n := 0; n < 10; n++ {
go func() {
var ps PreparedSample
err := ps.Update(dbclients)
if err != nil {
lg.Error("could not resolve public ip address", err)
}
lg.V(5).Infoln("starting measurer")
for r := range requestMeasurements {
lg.V(50).Infoln("got measurement", r)
if ps.lastUpdated.Before(time.Now().Add(-time.Hour * 5)) {
lg.V(15).Info("updating prepared sample", ps)
err := ps.Update(dbclients)
if err != nil {
lg.Warning(err)
}
}
measurerLoop:
for _, v := range r.measurers {
measurement, err := v.Measure()
if err != nil {
lg.Errorf("could not measure:%v error:%s", v, err.Error())
continue measurerLoop
}
switch measurement.Type() {
case sampletypes.DNSQuery, sampletypes.HTTPHeader:
data, err := measurement.Marshal()
if err != nil {
lg.Errorf("could not decode %v error:%s", measurement, err.Error())
continue measurerLoop
}
err = dbclients.DB.InsertSample(db.Sample{
Host: measurement.Host(),
CountryCode: ps.s.CountryCode,
Token: r.token,
ASN: ps.s.ASN,
Type: measurement.Type().String(),
Origin: sampleorigins.Central.String(),
Data: data,
})
if err != nil {
lg.Errorln(err.Error())
continue measurerLoop
}
default:
lg.Errorf("could not measure:%v error:%s", v, err.Error())
continue measurerLoop
}
}
}
}()
}
}
// IP2CountryCode resolves an IP address to ISO country code using an geoip
// database.
func (m *Maxmind) IP2CountryCode(IP net.IP) string {
var record onlyCountry // Or any appropriate struct
err := m.mmCountryDB.Lookup(IP, &record)
if err != nil {
lg.Warning(err)
}
return record.Country.IsoCode
}
func (m *Maxmind) IP2CityGeoNameID(IP net.IP) uint {
var record onlyCity // Or any appropriate struct
if m.mmCityDB != nil {
err := m.mmCityDB.Lookup(IP, &record)
if err != nil {
lg.Warning(err)
}
return record.City.GeoNameID
}
return 0
}
// PostLogRequest
func PostLog(w rest.ResponseWriter, r *rest.Request) {
form := PostLogRequest{}
err := r.DecodeJsonPayload(&form)
if err != nil {
lg.Warning(err)
apiutils.ErrToAPIStatus(err)
// apiutils.WriteRestError(w, apierrors.NewBadRequest("can't decode json"))
// apiutils.WriteRestError(w, apierrors.NewBadRequest("can't decode json"))
return
}
sender := r.PathParam("sender")
lg.V(form.Level).Infof("{%s} %s: %s", sender, form.Context, form.Message)
w.WriteJson(true)
}
func startAnalysis(clients db.Clients) {
go func() {
tick := time.NewTicker(10 * time.Second)
lastID, err := clients.DB.GetLastProcessedSampleID()
if err != nil {
lg.Warningln(err)
}
lg.Infof("starting analysis from sample ID %d", lastID)
lastPersistedID := lastID
for range tick.C {
results, err := clients.DB.GetSamples(uint64(lastID), "")
if err != nil {
lg.Fatal(err)
}
n := 0
start := time.Now()
loop:
for s := range results {
n++
if s.ID > lastID {
lastID = s.ID
}
if s.Type == "NewClientToken" {
if !shared.AcceptedHost(s.Host) {
lg.Warningln("not accepted host id:", s.ID, s.Host)
continue loop
}
err := clients.DB.PublishHost(s)
if err != nil {
lg.Warning(err)
}
}
}
if n != 0 && lg.V(15) {
lg.Infof("processed %d samples in %s", n, time.Since(start).String())
}
if lastID != lastPersistedID {
err = clients.DB.SetLastProcessedSampleID(lastID)
if err != nil {
lg.Errorln(err)
} else {
lastPersistedID = lastID
}
}
}
}()
}
func (d *DB) GetExportSamples(req shared.ExportSampleRequest) ([]shared.ExportSampleEntry, string, error) {
var results []shared.ExportSampleEntry
psql := squirrel.StatementBuilder.PlaceholderFormat(squirrel.Dollar)
i := psql.
Select("id", "host", "country_code", "asn", "created_at", "origin", "type", "token", "data", "extra_data").
From("samples").
OrderBy("id desc").
Limit(PageLength + 1)
if req.IDMax != 0 {
i = i.Where("id < ?", req.IDMax)
}
rows, err := i.RunWith(d.cache).Query()
if err != nil {
logSQLErr(err, &i)
return nil, "", err
}
defer rows.Close()
next := ""
count := 0
for rows.Next() {
var i shared.ExportSampleEntry
err := rows.Scan(
&i.ID,
&i.Host,
&i.CountryCode,
&i.ASN,
&i.CreatedAt,
&i.Origin,
&i.Type,
&i.Token,
&i.Data,
&i.ExtraData,
)
count++
if err != nil {
lg.Warning(err)
continue
}
if count > PageLength {
next = i.ID
} else {
results = append(results, i)
}
}
return results, next, nil
}
func (d *DB) GetExportBlockedHosts(req shared.BlockedContentRequest) ([]shared.HostsPublishLog, string, error) {
var results []shared.HostsPublishLog
psql := squirrel.StatementBuilder.PlaceholderFormat(squirrel.Dollar)
i := psql.
Select("id", "host", "country_code", "asn", "created_at", "sticky", "action").
From("hosts_publish_log").
OrderBy("id desc").
Limit(PageLength + 1)
if req.IDMax != 0 {
i = i.Where("id < ?", req.IDMax)
}
rows, err := i.RunWith(d.cache).Query()
if err != nil {
logSQLErr(err, &i)
return nil, "", err
}
defer rows.Close()
next := ""
count := 0
for rows.Next() {
var item shared.HostsPublishLog
err := rows.Scan(
&item.ID,
&item.Host,
&item.CountryCode,
&item.ASN,
&item.CreatedAt,
&item.Sticky,
&item.Action,
)
count++
if err != nil {
lg.Warning(err)
continue
}
if count > PageLength {
next = item.ID
} else {
results = append(results, item)
}
}
return results, next, nil
}
func (d *DB) GetSessionSamples(token shared.SuggestionToken) ([]Sample, error) {
psql := squirrel.StatementBuilder.PlaceholderFormat(squirrel.Dollar)
// fromID := 0
i := psql.
Select("id", "host", "country_code", "asn", "created_at", "origin", "type", "token", "data", "extra_data").
From("samples").
Where(squirrel.Eq{"token": string(token)})
rows, err := i.RunWith(d.cache).Query()
if err != nil {
return nil, err
}
defer rows.Close()
var results []Sample
for rows.Next() {
var sample Sample
var token string
err := rows.Scan(
&sample.ID,
&sample.Host,
&sample.CountryCode,
&sample.ASN,
&sample.CreatedAt,
&sample.Origin,
&sample.Type,
&token,
&sample.Data,
&sample.ExtraData,
)
if err != nil {
lg.Warning(err)
continue
}
sample.Token = shared.SuggestionToken(token)
results = append(results, sample)
}
return results, nil
}
func WriteRestError(w rest.ResponseWriter, err error) {
var status shared.Status
if _, ok := err.(*apierrors.StatusError); !ok {
err = apierrors.NewInternalError(err)
}
switch err := err.(type) {
case *apierrors.StatusError:
status = err.Status()
default:
panic(err)
}
if status.Code > 299 && lg.V(19) {
lg.WarningDepth(1, fmt.Sprintf("apierror %d: %+v", status.Code, status))
}
w.WriteHeader(status.Code)
err = w.WriteJson(&status)
if err != nil {
lg.Warning(err)
}
}
func (d *DB) GetSamples(fromID uint64, sampleType string) (chan Sample, error) {
psql := squirrel.StatementBuilder.PlaceholderFormat(squirrel.Dollar)
i := psql.
Select("id", "host", "country_code", "asn", "created_at", "origin", "type", "token", "data", "extra_data").
From("samples").
Where("id > ?", fromID)
rows, err := i.RunWith(d.cache).Query()
if err != nil {
return nil, err
}
results := make(chan Sample, 0)
go func(rows *sql.Rows) {
defer rows.Close()
defer close(results)
for rows.Next() {
var token string
var sample Sample
err := rows.Scan(
&sample.ID,
&sample.Host,
&sample.CountryCode,
&sample.ASN,
&sample.CreatedAt,
&sample.Origin,
&sample.Type,
&token,
&sample.Data,
&sample.ExtraData,
)
if err != nil {
lg.Warning(err)
continue
}
sample.Token = shared.SuggestionToken(token)
results <- sample
}
}(rows)
return results, nil
}
func (d *DB) GetLastProcessedSampleID() (uint64, error) {
psql := squirrel.StatementBuilder.PlaceholderFormat(squirrel.Dollar)
s := psql.Select("value").From("central_state").Where(
squirrel.Eq{"name": "last_processed_sample_id"},
).Limit(1)
row := s.RunWith(d.cache).QueryRow()
var valuestr string
err := row.Scan(&valuestr)
if err != nil {
if err == sql.ErrNoRows {
lg.Warning("last processed sample id not set, returning 0 (default value)")
return 0, nil
}
return 0, err
}
v, err := strconv.ParseUint(valuestr, 10, 64)
if err != nil {
return 0, err
}
return v, nil
}
// Init does precondition check if the application can/should be started.
// Init will return an error message with reason for exit printed.
func Run() {
if debugEnabled {
log.Println("ALKASIR_DEBUG ENABLED!")
err := os.Setenv("ALKASIR_DEBUG", "1")
if err != nil {
log.Fatal(err)
}
}
if hotEnabled {
log.Println("ALKASIR_HOT ENABLED!")
err := os.Setenv("ALKASIR_HOT", "1")
if err != nil {
log.Fatal(err)
}
}
// the darwin systray does not exit the main loop
if runtime.GOOS != "darwin" {
uiRunning.Add(1)
}
err := ui.Run(func() {
Atexit(ui.Done)
// start the getpublic ip updater.
go func() {
_ = shared.GetPublicIPAddr()
}()
if debugEnabled {
go func() {
err := http.ListenAndServe(
fmt.Sprintf("localhost:%d", debugPort), nil)
if err != nil {
panic(err)
}
}()
}
// wipe user data
if wipeData {
settingsdir := clientconfig.ConfigPath()
if settingsdir == "" {
log.Println("[wipe] Configdir not set")
os.Exit(1)
}
settingsfile := clientconfig.ConfigPath("settings.json")
if _, err := os.Stat(settingsfile); os.IsNotExist(err) {
log.Println("[wipe] No settings.json in configdir, will NOT wipe data")
os.Exit(1)
}
log.Println("Wiping all user data")
if err := os.RemoveAll(settingsdir); err != nil {
log.Println(err)
}
}
// Prepare logging
logdir := clientconfig.ConfigPath("log")
err := os.MkdirAll(logdir, 0775)
if err != nil {
log.Println("Could not create logging directory")
os.Exit(1)
}
err = flag.Set("log_dir", logdir)
if err != nil {
panic(err)
}
lg.SetSrcHighlight("alkasir/cmd", "alkasir/pkg")
lg.CopyStandardLogTo("INFO")
// Start init
if VERSION != "" {
lg.Infoln("Alkasir v" + VERSION)
} else {
lg.Warningln("Alkasir dev version (VERSION not set)")
}
lg.V(1).Info("Log v-level:", lg.Verbosity())
_, err = clientconfig.Read()
if err != nil {
lg.Infoln("Could not read config")
exit()
}
lg.V(30).Infoln("settings", clientconfig.Get().Settings)
if saveChromeExt {
err := saveChromeExtension()
if err != nil {
lg.Fatal(err)
}
}
{
configChanged, err := clientconfig.UpgradeConfig()
if err != nil {
lg.Fatalln("Could not upgrade config", err)
}
clientconfig.Update(func(conf *clientconfig.Config) error {
if clientAuthKeyFlag != "" {
lg.Warningln("Overriding generated authKey with", clientAuthKeyFlag)
conf.Settings.Local.ClientAuthKey = clientAuthKeyFlag
configChanged = true
}
if bindAddrFlag != "" {
lg.Warningln("Overriding configured bindAddr with", bindAddrFlag)
conf.Settings.Local.ClientBindAddr = bindAddrFlag
configChanged = true
}
if centralAddrFlag != "" {
lg.Warningln("Overriding central server addr with", centralAddrFlag)
conf.Settings.Local.CentralAddr = centralAddrFlag
configChanged = true
}
return nil
})
if configChanged {
if err := clientconfig.Write(); err != nil {
lg.Warning(err)
}
}
}
conf := clientconfig.Get()
loadTranslations(LanguageOptions...)
if err := ui.Language(conf.Settings.Local.Language); err != nil {
lg.Warningln(err)
}
go func() {
select {
case <-sigIntC:
exit()
case <-ui.Actions.Quit:
exit()
}
}()
for _, e := range []error{
mime.AddExtensionType(".json", "application/json"),
mime.AddExtensionType(".js", "application/javascript"),
mime.AddExtensionType(".css", "text/css"),
mime.AddExtensionType(".md", "text/plain"),
} {
if e != nil {
lg.Warning(e)
}
}
err = startInternalHTTPServer(conf.Settings.Local.ClientAuthKey)
if err != nil {
lg.Fatal("could not start internal http services")
}
// Connect the default transport
service.UpdateConnections(conf.Settings.Connections)
service.UpdateTransports(conf.Settings.Transports)
go service.StartConnectionManager(conf.Settings.Local.ClientAuthKey)
// TODO: async
pac.UpdateDirectList(conf.DirectHosts.Hosts)
pac.UpdateBlockedList(conf.BlockedHostsCentral.Hosts, conf.BlockedHosts.Hosts)
lastBlocklistChange = time.Now()
go StartBlocklistUpgrader()
if upgradeDiffsBaseURL != "" {
lg.V(19).Infoln("upgradeDiffsBaseURL is ", upgradeDiffsBaseURL)
go StartBinaryUpgradeChecker(upgradeDiffsBaseURL)
} else {
lg.Warningln("empty upgradeDiffsBaseURL, disabling upgrade checks")
}
lg.V(5).Info("Alkasir has started")
})
// the darwin systray does not exit the main loop
if runtime.GOOS != "darwin" {
uiRunning.Done()
}
lg.Infoln("ui.Run ended")
if err != nil {
log.Println("client.Run error:", err)
}
}
// SubmitSuggestion initiates the comminication with Central for a Submission
// session.
func SubmitSuggestion(w rest.ResponseWriter, r *rest.Request) {
// // TODO This is the response that must be forwarded from central/api and parsed by client and passed on to the browser.
// apiutils.WriteRestError(w,
// apierrors.NewInvalid("object", "suggestion",
// fielderrors.ValidationErrorList{
// fielderrors.NewFieldValueNotSupported("URL", "...", []string{})}))
// return
ID := r.PathParam("id")
suggestion, ok := client.GetSuggestion(ID)
if !ok {
apiutils.WriteRestError(w, apierrors.NewNotFound("suggestion", ID))
return
}
wanip := shared.GetPublicIPAddr()
if wanip == nil {
lg.Warning("could not resolve public ip addr")
}
conf := clientconfig.Get()
restclient, err := NewRestClient()
if err != nil {
apiutils.WriteRestError(w, apierrors.NewInternalError(err))
return
}
tokenResp, err := suggestion.RequestToken(
restclient, wanip, conf.Settings.Local.CountryCode)
if err != nil {
if apiutils.IsNetError(err) {
apiutils.WriteRestError(w, apierrors.NewServerTimeout("alkasir-central", "request-submission-token", 0))
} else {
apiutils.WriteRestError(w, apierrors.NewInternalError(err))
}
return
}
n, err := suggestion.SendSamples(restclient)
if err != nil {
lg.Warningln("error sending samples", err.Error())
}
lg.V(5).Infof("sent %d samples", n)
// continue sending samples if future measuremetns are expected to come
prepared, err := suggestion.Prepared()
if err != nil {
lg.Errorln(err)
} else if !prepared {
lg.V(5).Infof("all samples not collected, will try to send the rest when they are done")
go func(s client.Suggestion) {
start := time.Now()
t := time.NewTicker(30 * time.Second)
defer t.Stop()
submitSamples:
for range t.C {
if time.Now().After(start.Add(15 * time.Minute)) {
lg.Errorln("Stopping trying to send additional samples")
return
}
prepared, err := suggestion.Prepared()
if err != nil {
lg.Errorln(err)
return
}
if prepared {
restclient, err := NewRestClient()
if err != nil {
continue submitSamples
}
n, err := suggestion.SendSamples(restclient)
lg.V(5).Infof("sent %d samples", n)
if err != nil {
lg.Warningln("error sending samples", err.Error())
continue submitSamples
}
return
}
}
}(suggestion)
}
u, err := url.Parse(suggestion.URL)
if err != nil {
lg.Errorln(err)
} else {
err := clientconfig.Update(func(conf *clientconfig.Config) error {
conf.BlockedHosts.Add(u.Host)
lastBlocklistChange = time.Now()
pac.UpdateBlockedList(conf.BlockedHostsCentral.Hosts,
conf.BlockedHosts.Hosts)
return nil
})
if err != nil {
lg.Errorln(err)
}
}
w.WriteJson(tokenResp)
}
// SubmitSuggestion initiates the comminication with Central for a Submission
// session.
func SubmitSuggestion(w rest.ResponseWriter, r *rest.Request) {
// // TODO This is the response that must be forwarded from central/api and parsed by client and passed on to the browser.
// apiutils.WriteRestError(w,
// apierrors.NewInvalid("object", "suggestion",
// fielderrors.ValidationErrorList{
// fielderrors.NewFieldValueNotSupported("URL", "...", []string{})}))
// return
ID := r.PathParam("id")
suggestion, ok := client.GetSuggestion(ID)
if !ok {
apiutils.WriteRestError(w, apierrors.NewNotFound("suggestion", ID))
return
}
wanip := shared.GetPublicIPAddr()
if wanip == nil {
lg.Warning("could not resolve public ip addr")
}
conf := clientconfig.Get()
restclient, err := NewRestClient()
if err != nil {
apiutils.WriteRestError(w, apierrors.NewInternalError(err))
return
}
tokenResp, err := suggestion.RequestToken(
restclient, wanip, conf.Settings.Local.CountryCode)
if err != nil {
if apiutils.IsNetError(err) {
apiutils.WriteRestError(w, apierrors.NewServerTimeout("alkasir-central", "request-submission-token", 0))
} else {
apiutils.WriteRestError(w, apierrors.NewInternalError(err))
}
return
}
n, err := suggestion.SendSamples(restclient)
if err != nil {
lg.Warningln("error sending samples", err.Error())
}
lg.V(5).Infoln("sent ", n)
// FIXME PRESENTATION: just add the url locally
u, err := url.Parse(suggestion.URL)
if err != nil {
lg.Errorln(err)
} else {
err := clientconfig.Update(func(conf *clientconfig.Config) error {
conf.BlockedHosts.Add(u.Host)
lastBlocklistChange = time.Now()
pac.UpdateBlockedList(conf.BlockedHostsCentral.Hosts,
conf.BlockedHosts.Hosts)
return nil
})
if err != nil {
lg.Errorln(err)
}
}
w.WriteJson(tokenResp)
}
func startInternalHTTPServer(authKey string) error {
lg.V(15).Infoln("starting internal api server")
mux, err := createServeMux()
if err != nil {
lg.Error("failed to create router")
return err
}
auth := Auth{Key: authKey, wrapped: mux}
conf := clientconfig.Get()
listener, err := net.Listen("tcp", conf.Settings.Local.ClientBindAddr)
if err != nil {
lg.Warning(err)
listener, err = net.Listen("tcp", "127.0.0.1:")
}
if err != nil {
lg.Warning(err)
ui.Notify("Could not bind any local port (bootstrap)")
lg.Errorln("Could not bind any local port (bootstrap)")
return err
}
go func(listenaddr string) {
// baseURL := fmt.Sprintf()
baseURL := fmt.Sprintf("http://%s?suk=", listenaddr)
for {
select {
case <-ui.Actions.CopyBrowserCodeToClipboard:
bc := browsercode.BrowserCode{Key: authKey}
err := bc.SetHostport(listener.Addr().String())
if err != nil {
lg.Errorln(err)
continue
}
err = bc.CopyToClipboard()
if err != nil {
lg.Errorln(err)
}
case <-ui.Actions.OpenInBrowser:
browser.OpenURL(baseURL + singleUseKeys.New() + "#/")
case <-ui.Actions.Help:
browser.OpenURL(baseURL + singleUseKeys.New() + "#/docs/__/index")
}
singleUseKeys.Cleanup()
}
}(listener.Addr().String())
doneC := make(chan bool, 1)
go func() {
defer listener.Close()
err = http.Serve(listener, auth)
if err != nil {
doneC <- false
}
}()
select {
case ok := <-doneC:
if !ok {
return errors.New("Could not start internal http server")
}
case <-time.After(time.Millisecond * 200):
return nil
}
return nil
}