// AcceptedHost return true if the supplied host:port or host is allowed to be added to alkasir. func AcceptedHost(host string) bool { if strings.TrimSpace(host) == "" { if lg.V(50) { lg.Warningf("empty url host is not allowed") } return false } if strings.Contains(host, ":") { onlyhost, _, err := net.SplitHostPort(host) if err == nil { host = onlyhost } } if _, ok := disallowedHosts[host]; ok { if lg.V(50) { lg.Warningf("url host %s is not allowed", host) } return false } IP := net.ParseIP(host) if IP != nil { return AcceptedIP(IP) } return true }
func readSettings(c *Config) error { lg.V(5).Info("Reading settings file") isRead := false _, err := mkConfigDir() if err != nil { return err } data, err := ioutil.ReadFile(ConfigPath("settings.json")) if err != nil { lg.Infof("Error loading settings.json %s", err) } else { settings, err := parseConfig(data) if err != nil { lg.Warningf("Config file error, deleting and resetting") err := os.Remove(ConfigPath("settings.json")) if err != nil { lg.Warningf("Could not delete old settingsfile (should probably panic here)") } } else { currentConfig.Settings = *settings isRead = true } } if !isRead { settings, err := parseConfig([]byte(settingsTemplate)) if err != nil { panic("invalid defaultsettings") } currentConfig.Settings = *settings } transports := make(map[string]shared.Transport, 0) if currentConfig.Settings.Transports != nil { for _, v := range currentConfig.Settings.Transports { transports[v.Name] = v } } for _, v := range []shared.Transport{ {Name: "obfs3", Bundled: true, TorPT: true}, {Name: "obfs4", Bundled: true, TorPT: true}, {Name: "shadowsocks-client", Bundled: true}, } { transports[v.Name] = v } currentConfig.Settings.Transports = transports return nil }
func (p *PreparedSample) Update(dbclients db.Clients) error { IP := shared.GetPublicIPAddr() if IP == nil { return errors.New("could not get own public ip address") } // resolve ip to asn. var ASN int ASNres, err := dbclients.Internet.IP2ASN(IP) if err != nil { lg.Errorln(err.Error()) return err } if ASNres != nil { ASN = ASNres.ASN } else { lg.Warningf("no ASN lookup result for IP: %s ", IP) return fmt.Errorf("no ASN lookup result for IP: %s ", IP) } // resolve ip to country code. countryCode := dbclients.Maxmind.IP2CountryCode(IP) s := db.Sample{ CountryCode: countryCode, ASN: ASN, } p.lastUpdated = time.Now() p.s = s return nil }
func upgradeBinaryCheck(diffsBaseURL string) error { artifactNameMu.Lock() artifact := artifactName artifactNameMu.Unlock() cl, err := NewRestClient() if err != nil { return err } res, found, err := cl.CheckBinaryUpgrade(shared.BinaryUpgradeRequest{ Artifact: artifact, FromVersion: VERSION, }) if err != nil { return err } if !found { lg.Infoln("no update found") return nil } lg.Warningf("found update %+v", res) httpclient, err := service.NewTransportHTTPClient(2 * time.Hour) if err != nil { return err } opts, err := upgradebin.NewUpdaterOptions(res, shared.UpgradeVerificationPublicKey) if err != nil { return err } u, err := url.Parse(diffsBaseURL) if err != nil { lg.Errorln(err) return err } u.Path = path.Join(u.Path, artifact, VERSION, res.Version) URL := u.String() lg.Infoln("downloading", URL) resp, err := httpclient.Get(URL) if err != nil { lg.Errorln(err) return err } defer resp.Body.Close() err = upgradebin.Apply(resp.Body, opts) if err != nil { lg.Errorln(err) // will be retried the next time the client starts return nil } return nil }
// Init initializes the server. func Init() error { lg.SetSrcHighlight("alkasir/cmd", "alkasir/pkg") lg.CopyStandardLogTo("INFO") lg.V(1).Info("Log v-level:", lg.Verbosity()) lg.V(1).Info("Active country codes:", shared.CountryCodes) lg.Flush() if *datadirFlag == "" { u, err := user.Current() if err != nil { lg.Fatal(err) } datadir = filepath.Join(u.HomeDir, ".alkasir-central") } else { datadir = *datadirFlag } validCountryCodes = make(map[string]bool, len(shared.CountryCodes)) validCountryCodesMu.Lock() for _, cc := range shared.CountryCodes { validCountryCodes[cc] = true } validCountryCodesMu.Unlock() err := InitDB() if err != nil { lg.Fatalln(err) return err } redisPool = newRedisPool(*redisServer, *redisPassword) internet.SetDataDir(filepath.Join(datadir, "internet")) countryFile := filepath.Join(datadir, "internet", "GeoLite2-Country.mmdb") if _, err := os.Stat(countryFile); os.IsNotExist(err) { // http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz lg.Fatalf("cannot enable IP2CountryCode lookups, %s is missing", countryFile) } else { var err error mmCountryDB, err = maxminddb.Open(countryFile) if err != nil { lg.Fatal(err) } } cityFile := filepath.Join(datadir, "internet", "GeoLite2-City.mmdb") if _, err := os.Stat(cityFile); os.IsNotExist(err) { // http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz lg.Warningf("cannot enable IP2CityGeoNameID lookups, %s is missing", cityFile) } else { mmCityDB, err = maxminddb.Open(cityFile) if err != nil { lg.Fatal(err) } // defer mmCityDB.Close() } return nil }
// AcceptedURL returns true if the supplied IP is allowed to be added to alkasir. func AcceptedIP(ip net.IP) bool { if ip.IsLoopback() || ip.Equal(net.IPv4zero) || ip.Equal(net.IPv6zero) { if lg.V(50) { lg.Warningf("ip address %s is not allowed because loopback or zeoro address", ip.String()) } return false } for _, v := range disallowedNets { if v.Contains(ip) { if lg.V(50) { lg.Warningf("ip %s is not allowed because network %s is not allowed", ip.String(), v.String()) } return false } } return true }
func upgradeBinaryCheck() error { if !upgradeEnabled { lg.Infoln("binary upgrades are disabled using the command line flag") return nil } artifactNameMu.Lock() artifact := artifactName artifactNameMu.Unlock() cl, err := NewRestClient() if err != nil { return err } // TODO: check for current artifact + version (need to add artifact id to cmd's) res, found, err := cl.CheckBinaryUpgrade(shared.BinaryUpgradeRequest{ Artifact: artifact, FromVersion: VERSION, }) if err != nil { return err } if !found { lg.Infoln("no update found") return nil } lg.Warningf("found update %+v", res) httpclient, err := service.NewTransportHTTPClient() if err != nil { return err } opts, err := upgradebin.NewUpdaterOptions(res, shared.UpgradeVerificationPublicKey) if err != nil { return err } URL := fmt.Sprintf("https://central.server.domain/u/%s/%s/%s", artifact, VERSION, res.Version) lg.Infoln("downloading %s", URL) resp, err := httpclient.Get(URL) if err != nil { lg.Errorln(err) return err } defer resp.Body.Close() err = upgradebin.Apply(resp.Body, opts) if err != nil { lg.Errorln(err) // will be retried the next time the client starts return nil } return nil }
// AcceptedURL returns true if the supplied url is allowed to be added to alkasir. func AcceptedURL(u *url.URL) bool { if _, ok := allowedProtocols[u.Scheme]; !ok { if lg.V(50) { lg.Warningf("url scheme %s is not allowed", u.Scheme) } return false } return AcceptedHost(u.Host) }
// Download downloads the artifact to a local directory func (a *Artifact) Download() error { shortURL := a.Info().URL shortURL = shortURL[strings.LastIndex(shortURL, "/")+1:] lg.Infof("Downloading %s", shortURL) err := os.MkdirAll(a.Dir("dl"), 0775) if err != nil { return err } err = download(a.Path(), a.Info().URL) if err != nil { lg.Warningf("Error downloading %s", shortURL) } else { lg.Infof("Downloaded %s", shortURL) } return err }
// Prefix adds a prefix to the Field of every ValidationError in the list. // Returns the list for convenience. func (list ValidationErrorList) Prefix(prefix string) ValidationErrorList { for i := range list { if err, ok := list[i].(*ValidationError); ok { if strings.HasPrefix(err.Field, "[") { err.Field = prefix + err.Field } else if len(err.Field) != 0 { err.Field = prefix + "." + err.Field } else { err.Field = prefix } list[i] = err } else { glog.Warningf("Programmer error: ValidationErrorList holds non-ValidationError: %#v", list[i]) } } return list }
// StartBinaryUpgradeChecker checks for binary upgrades when the connection is // up and on a schedule. // // This function runs in it's own goroutine. func StartBinaryUpgradeChecker(diffsBaseURL string) { if !upgradeEnabled { lg.Infoln("binary upgrades are disabled using the command line flag") return } if VERSION == "" { lg.Warningln("VERSION not set, binary upgrades are disabled") return } _, err := version.NewVersion(VERSION) if err != nil { lg.Warningf("VERSION '%s' is not a valid semver version, binary upgrades are disabled: %v", VERSION, err) return } connectionEventListener := make(chan service.ConnectionHistory) uChecker, _ := NewUpdateChecker("binary") service.AddListener(connectionEventListener) for { select { // Update when the transport connection comes up case event := <-connectionEventListener: if event.IsUp() { uChecker.Activate() uChecker.UpdateNow() } // Update by request of the update checker case request := <-uChecker.RequestC: err := upgradeBinaryCheck(diffsBaseURL) if err != nil { lg.Errorln(err) request.ResponseC <- UpdateError } else { request.ResponseC <- UpdateSuccess } } } }
func GetPublicIPAddr() net.IP { publicIP.init.Do(func() { lg.Infoln("starting public ip address updater") var gotIP sync.Once publicIP.hasIP.Add(1) go func() { timeout := time.Duration(10 * time.Second) client := http.Client{ Timeout: timeout, Transport: v4Transport, } var services []string = make([]string, 0) services = append(services, wanipservices...) shuffleStrings(services) serviceIdx := 0 refreshTicker := time.Tick(time.Minute * 10) loop: for { serviceIdx = (serviceIdx + 1) % (len(services)) if serviceIdx == 0 { gotIP.Do(func() { publicIP.hasIP.Done() }) } URL := services[serviceIdx] resp, err := client.Get(URL) if err != nil { lg.Warningf("Could not read response from %s: %v", URL, err) <-time.After(time.Second * 2) continue loop } data, err := ioutil.ReadAll(resp.Body) if err != nil { lg.Warningf("Could not read response from %s: %v", URL, err) resp.Body.Close() <-time.After(time.Second * 2) continue loop } resp.Body.Close() str := strings.TrimSpace(string(data)) ip := net.ParseIP(str) if ip == nil { lg.Warningf("error parsing ip from %s from: %.50s", URL, str) <-time.After(time.Second * 2) continue loop } publicIP.Lock() if publicIP.ip == nil || !publicIP.ip.Equal(ip) { lg.V(5).Infof("Public ip address change: %s -> %s via %s", publicIP.ip, ip, URL) } publicIP.ip = &ip publicIP.Unlock() gotIP.Do(func() { publicIP.hasIP.Done() }) select { case <-refreshTicker: lg.V(30).Infoln("refreshing IP address") } } }() }) publicIP.hasIP.Wait() publicIP.Lock() ip := publicIP.ip publicIP.Unlock() if ip != nil { return ip.To4() } return nil }
func insertUpgrades([]string) error { if err := OpenDB(); err != nil { return err } files, err := findJSONFiles("diffs/") if err != nil { return err } var upgrades []db.UpgradeMeta for _, v := range files { lg.V(5).Infoln("reading", v) data, err := ioutil.ReadFile(v) if err != nil { return err } var cpr makepatch.CreatePatchResult err = json.Unmarshal(data, &cpr) if err != nil { return err } um, ok, err := sqlDB.GetUpgrade(db.GetUpgradeQuery{ Artifact: cpr.Artifact, Version: cpr.NewVersion, AlsoUnpublished: true, }) if err != nil { return err } if ok && um.Artifact == cpr.Artifact && um.Version == cpr.NewVersion { lgheader := cpr.Artifact + " " + cpr.NewVersion if um.ED25519Signature != cpr.ED25519Signature { lg.Warningf("%s signatures does not match!", lgheader) } if um.SHA256Sum != cpr.SHA256Sum { lg.Warningf("%s shasum does not match!", lgheader) } lg.Infof("%s is already imported, skipping", lgheader) continue } upgrades = append(upgrades, db.UpgradeMeta{ Artifact: cpr.Artifact, Version: cpr.NewVersion, SHA256Sum: cpr.SHA256Sum, ED25519Signature: cpr.ED25519Signature, }) } { // NOTE: this will be removed later, a quick hack before other upgrades refactoring takes place uniqeUpgrades := make(map[string]db.UpgradeMeta, 0) for _, v := range upgrades { uniqeUpgrades[fmt.Sprintf("%s---%s", v.Artifact, v.Version)] = v } upgrades = upgrades[:0] for _, v := range uniqeUpgrades { upgrades = append(upgrades, v) } } fmt.Println(upgrades) err = sqlDB.InsertUpgrades(upgrades) if err != nil { lg.Errorln(err) return err } return nil }
// Update list of blocked hosts for an IP address. func GetHosts(dbclients db.Clients) func(w rest.ResponseWriter, r *rest.Request) { relh := relatedHosts{ dbclients: dbclients, } relh.update() go func() { for range time.NewTicker(10 * time.Minute).C { relh.update() } }() return func(w rest.ResponseWriter, r *rest.Request) { // HANDLE USERIP BEGIN req := shared.UpdateHostlistRequest{} err := r.DecodeJsonPayload(&req) if err != nil { apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError) return } // parse/validate client ip address. IP := req.ClientAddr if IP == nil { apiError(w, "bad ClientAddr", http.StatusBadRequest) return } // resolve ip to asn. var ASN int ASNres, err := dbclients.Internet.IP2ASN(IP) if err != nil { lg.Errorln(shared.SafeClean(err.Error())) apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError) return } if ASNres != nil { ASN = ASNres.ASN } else { lg.Warningf("no ASN lookup result for IP: %s ", shared.SafeClean(IP.String())) } // resolve ip to country code. countryCode := dbclients.Maxmind.IP2CountryCode(IP) req.ClientAddr = net.IPv4zero IP = net.IPv4zero // HANDLE USERIP END hosts, err := dbclients.DB.GetBlockedHosts(countryCode, ASN) if err != nil { apiError(w, err.Error(), http.StatusInternalServerError) return } simpleData := struct { ClientVersion string `json:"version"` // client version idientifer }{ req.ClientVersion, } data, err := json.Marshal(simpleData) if err != nil { lg.Errorln(err) } ss := db.SimpleSample{ CountryCode: countryCode, ASN: ASN, Type: "ClientBlocklistUpdate", OriginID: req.UpdateID, Data: data, } err = dbclients.DB.InsertSimpleSample(ss) if err != nil { lg.Errorf("error persisting simplesample %v", ss) } err = w.WriteJson(shared.UpdateHostlistResponse{ Hosts: relh.fill(hosts), }) if err != nil { lg.Error(err) return } } }
// SuggestionToken JSON API method. func SuggestionToken(dbclients db.Clients) func(w rest.ResponseWriter, r *rest.Request) { return func(w rest.ResponseWriter, r *rest.Request) { // HANDLE USERIP BEGIN req := shared.SuggestionTokenRequest{} err := r.DecodeJsonPayload(&req) if err != nil { apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError) return } // validate country code. if !validCountryCode(req.CountryCode) { apiError(w, fmt.Sprintf("invalid country code: %s", req.CountryCode), http.StatusBadRequest) return } // parse/validate client ip address. IP := req.ClientAddr if IP == nil { apiError(w, "bad ClientAddr", http.StatusBadRequest) return } // parse and validate url. URL := strings.TrimSpace(req.URL) if URL == "" { apiError(w, "no or empty URL", http.StatusBadRequest) return } u, err := url.Parse(URL) if err != nil { apiError(w, fmt.Sprintf("%s is not a valid URL", URL), http.StatusBadRequest) return } if !shared.AcceptedURL(u) { apiError(w, fmt.Sprintf("%s is not a valid URL", URL), http.StatusBadRequest) return } // resolve ip to asn. var ASN int ASNres, err := dbclients.Internet.IP2ASN(IP) if err != nil { lg.Errorln(shared.SafeClean(err.Error())) apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError) return } if ASNres != nil { ASN = ASNres.ASN } else { lg.Warningf("no ASN lookup result for IP: %s ", shared.SafeClean(IP.String())) } // reoslve ip to country code. countryCode := dbclients.Maxmind.IP2CountryCode(IP) // resolve ip to city geonameid geoCityID := dbclients.Maxmind.IP2CityGeoNameID(IP) req.ClientAddr = net.IPv4zero IP = net.IPv4zero // HANDLE USERIP END { supported, err := dbclients.DB.IsURLAllowed(u, countryCode) if err != nil { // TODO: standardize http status codes apiError(w, err.Error(), http.StatusForbidden) return } if !supported { lg.Infof("got request for unsupported URL %s") w.WriteJson(shared.SuggestionTokenResponse{ Ok: false, URL: req.URL, }) return } } // start new submission token session token := db.SessionTokens.New(URL) // create newclienttoken sample data sample := shared.NewClientTokenSample{ URL: URL, CountryCode: req.CountryCode, } sampleData, err := json.Marshal(sample) if err != nil { lg.Errorln(err) apiError(w, "error #20150424-002542-CEST", http.StatusInternalServerError) return } // create extraData extra := shared.IPExtraData{ CityGeoNameID: geoCityID, } extraData, err := json.Marshal(extra) if err != nil { lg.Errorln(err) apiError(w, "error #20150427-211052-CEST", http.StatusInternalServerError) return } // insert into db { err := dbclients.DB.InsertSample(db.Sample{ Host: u.Host, CountryCode: countryCode, ASN: ASN, Type: "NewClientToken", Origin: "Central", Token: token, Data: sampleData, ExtraData: extraData, }) if err != nil { apiError(w, err.Error(), http.StatusInternalServerError) return } } // queue central measurements measurements, err := measure.DefaultMeasurements(req.URL) if err != nil { lg.Warningf("could not create standard measurements: %s", err.Error()) } else { queueMeasurements(token, measurements...) } // write json response { err := w.WriteJson(shared.SuggestionTokenResponse{ Ok: true, URL: URL, Token: token, }) if err != nil { lg.Errorln(err.Error()) return } } } }
// StoreSample JSON API method. func StoreSample(dbclients db.Clients) func(w rest.ResponseWriter, r *rest.Request) { return func(w rest.ResponseWriter, r *rest.Request) { // HANDLE USERIP BEGIN req := shared.StoreSampleRequest{} err := r.DecodeJsonPayload(&req) if err != nil { apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError) return } // validate sample type. if _, ok := clientSampleTypes[req.SampleType]; !ok { apiError(w, "invalid sample type: "+req.SampleType, http.StatusBadRequest) return } // get/validate suggestion session token. tokenData, validToken := db.SessionTokens.Get(req.Token) if !validToken { apiError(w, "invalid token", http.StatusBadRequest) return } // parse/validate url. URL := strings.TrimSpace(req.URL) if tokenData.URL != URL { lg.V(2).Infof("invalid URL %s for token session, was expecting %s", req.URL, tokenData.URL) apiError(w, "invalid URL for token session", http.StatusBadRequest) return } u, err := url.Parse(URL) if err != nil { apiError(w, fmt.Sprintf("%s is not a valid URL", URL), http.StatusBadRequest) return } // parse/validate client ip address. IP := net.ParseIP(req.ClientAddr) if IP == nil { apiError(w, "bad ClientAddr", http.StatusBadRequest) return } // resolve ip to asn. var ASN int ASNres, err := dbclients.Internet.IP2ASN(IP) if err != nil { lg.Errorln(shared.SafeClean(err.Error())) apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError) return } if ASNres != nil { ASN = ASNres.ASN } else { lg.Warningf("no ASN lookup result for IP: %s ", shared.SafeClean(IP.String())) } countryCode := dbclients.Maxmind.IP2CountryCode(IP) req.ClientAddr = "" IP = net.IPv4zero // HANDLE USERIP END // insert into db { err := dbclients.DB.InsertSample(db.Sample{ Host: u.Host, CountryCode: countryCode, ASN: ASN, Type: req.SampleType, Origin: "Client", Token: req.Token, Data: []byte(req.Data), }) if err != nil { lg.Errorln(err.Error()) apiError(w, "error 20150424-011948-CEST", http.StatusInternalServerError) return } } w.WriteJson(shared.StoreSampleResponse{ Ok: true, }) return } }
// CreateSuggestion . func CreateSuggestion(w rest.ResponseWriter, r *rest.Request) { form := shared.BrowserSuggestionTokenRequest{} err := r.DecodeJsonPayload(&form) if err != nil { // apiError(w, err.Error(), http.StatusInternalServerError) apiutils.WriteRestError(w, err) return } var invalids fielderrors.ValidationErrorList // parse and validate url. URL := strings.TrimSpace(form.URL) if URL == "" { invalids = append(invalids, fielderrors.NewFieldRequired("URL")) // apiError(w, "no or empty URL", http.StatusBadRequest) } u, err := url.Parse(URL) if err != nil { invalids = append(invalids, fielderrors.NewFieldInvalid("URL", URL, err.Error())) // apiError(w, fmt.Sprintf("%s is not a valid URL", URL), http.StatusBadRequest) } host := u.Host if strings.Contains(host, ":") { host, _, err = net.SplitHostPort(u.Host) if err != nil { invalids = append(invalids, fielderrors.NewFieldInvalid("URL", URL, err.Error())) } } if !shared.AcceptedURL(u) { invalids = append(invalids, fielderrors.NewFieldValueNotSupported("URL", URL, nil)) } if len(invalids) > 0 { apiutils.WriteRestError(w, apierrors.NewInvalid("create-suggestion", "URL", invalids)) return } s := client.NewSuggestion(u.String()) defer s.DoneAddingSamples() measurers, err := measure.DefaultMeasurements(form.URL) if err != nil { apiutils.WriteRestError(w, apierrors.NewInternalError(err)) return } for _, v := range measurers { m, err := v.Measure() if err != nil { lg.Errorf("could not measure: %s", err.Error()) } else { switch m.Type() { case sampletypes.DNSQuery, sampletypes.HTTPHeader: err = s.AddMeasurement(m) if err != nil { lg.Errorln(err.Error()) return } default: lg.Warningf("unsupported sample type: %s", m.Type().String()) } } } }