// AcceptedHost return true if the supplied host:port or host is allowed to be added to alkasir.
func AcceptedHost(host string) bool {
if strings.TrimSpace(host) == "" {
if lg.V(50) {
lg.Warningf("empty url host is not allowed")
}
return false
}
if strings.Contains(host, ":") {
onlyhost, _, err := net.SplitHostPort(host)
if err == nil {
host = onlyhost
}
}
if _, ok := disallowedHosts[host]; ok {
if lg.V(50) {
lg.Warningf("url host %s is not allowed", host)
}
return false
}
IP := net.ParseIP(host)
if IP != nil {
return AcceptedIP(IP)
}
return true
}
// NewRestClient returns an central server client using the current default
// transport if the central server is not runing locally.
func NewRestClient() (*client.Client, error) {
conf := clientconfig.Get()
apiurl := conf.Settings.Local.CentralAddr
u, err := url.Parse(apiurl)
if err != nil {
return nil, err
}
host := u.Host
if strings.Contains(host, ":") {
host, _, err = net.SplitHostPort(host)
if err != nil {
return nil, err
}
}
if host == "localhost" || host == "127.0.0.1" {
lg.V(19).Infoln("Opening restclient to localhost central api without transport")
return client.NewClient(apiurl, nil), nil
}
httpclient, err := service.NewTransportHTTPClient()
if err != nil {
return nil, err
}
lg.V(19).Infoln("Opening restclient thru transport")
return client.NewClient(apiurl, httpclient), nil
}
// Init initializes the server.
func Init() error {
lg.SetSrcHighlight("alkasir/cmd", "alkasir/pkg")
lg.CopyStandardLogTo("INFO")
lg.V(1).Info("Log v-level:", lg.Verbosity())
lg.V(1).Info("Active country codes:", shared.CountryCodes)
lg.Flush()
if *datadirFlag == "" {
u, err := user.Current()
if err != nil {
lg.Fatal(err)
}
datadir = filepath.Join(u.HomeDir, ".alkasir-central")
} else {
datadir = *datadirFlag
}
validCountryCodes = make(map[string]bool, len(shared.CountryCodes))
validCountryCodesMu.Lock()
for _, cc := range shared.CountryCodes {
validCountryCodes[cc] = true
}
validCountryCodesMu.Unlock()
err := InitDB()
if err != nil {
lg.Fatalln(err)
return err
}
redisPool = newRedisPool(*redisServer, *redisPassword)
internet.SetDataDir(filepath.Join(datadir, "internet"))
countryFile := filepath.Join(datadir, "internet", "GeoLite2-Country.mmdb")
if _, err := os.Stat(countryFile); os.IsNotExist(err) {
// http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz
lg.Fatalf("cannot enable IP2CountryCode lookups, %s is missing", countryFile)
} else {
var err error
mmCountryDB, err = maxminddb.Open(countryFile)
if err != nil {
lg.Fatal(err)
}
}
cityFile := filepath.Join(datadir, "internet", "GeoLite2-City.mmdb")
if _, err := os.Stat(cityFile); os.IsNotExist(err) {
// http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz
lg.Warningf("cannot enable IP2CityGeoNameID lookups, %s is missing", cityFile)
} else {
mmCityDB, err = maxminddb.Open(cityFile)
if err != nil {
lg.Fatal(err)
}
// defer mmCityDB.Close()
}
return nil
}
func startMeasurer(dbclients db.Clients) {
for n := 0; n < 10; n++ {
go func() {
var ps PreparedSample
err := ps.Update(dbclients)
if err != nil {
lg.Error("could not resolve public ip address", err)
}
lg.V(5).Infoln("starting measurer")
for r := range requestMeasurements {
lg.V(50).Infoln("got measurement", r)
if ps.lastUpdated.Before(time.Now().Add(-time.Hour * 5)) {
lg.V(15).Info("updating prepared sample", ps)
err := ps.Update(dbclients)
if err != nil {
lg.Warning(err)
}
}
measurerLoop:
for _, v := range r.measurers {
measurement, err := v.Measure()
if err != nil {
lg.Errorf("could not measure:%v error:%s", v, err.Error())
continue measurerLoop
}
switch measurement.Type() {
case sampletypes.DNSQuery, sampletypes.HTTPHeader:
data, err := measurement.Marshal()
if err != nil {
lg.Errorf("could not decode %v error:%s", measurement, err.Error())
continue measurerLoop
}
err = dbclients.DB.InsertSample(db.Sample{
Host: measurement.Host(),
CountryCode: ps.s.CountryCode,
Token: r.token,
ASN: ps.s.ASN,
Type: measurement.Type().String(),
Origin: sampleorigins.Central.String(),
Data: data,
})
if err != nil {
lg.Errorln(err.Error())
continue measurerLoop
}
default:
lg.Errorf("could not measure:%v error:%s", v, err.Error())
continue measurerLoop
}
}
}
}()
}
}
func exit() {
lg.Infoln("alkasir is shutting down")
atexitMu.Lock() // this lock should be kept, one shutdown should be enough for everyone.
lg.Flush()
if err := clientconfig.Write(); err != nil {
lg.Errorf("could not save config file: %s", err.Error())
}
lg.V(9).Infoln("running atexit funcs")
for _, f := range atexitFuncs {
funcName := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
lg.V(10).Infoln("Running at exit func", funcName)
f()
lg.V(10).Infoln("Finished at exit func", funcName)
}
atexitFuncs = atexitFuncs[:0]
lg.V(9).Infoln("atexit funcs done")
lg.V(9).Infoln("stopping connectionmanager")
service.StopConnectionManager()
lg.V(9).Infoln("stopping services")
service.StopAll()
lg.V(9).Infoln("services stopped")
lg.V(9).Infoln("waiting for UI shutdown to finish")
uiRunning.Wait()
lg.V(9).Infoln("ui shut down")
lg.Flush()
lg.Infoln("alkasir shutdown complete")
lg.Flush()
time.Sleep(time.Millisecond * 1)
os.Exit(0)
}
// adds api routes to given mix router
func AddRoutes(mux *http.ServeMux) error {
api := rest.NewApi()
logger := log.New(nil, "", 0)
lg.CopyLoggerTo("INFO", logger)
loggerWarning := log.New(nil, "", 0)
lg.CopyLoggerTo("WARNING", loggerWarning)
if lg.V(100) {
api.Use(&middlewares.AccessLogApacheMiddleware{
Format: "%S\033[0m \033[36;1m%Dμs\033[0m \"%r\" \033[1;30m%u \"%{User-Agent}i\"\033[0m",
})
} else if lg.V(20) {
api.Use(&middlewares.AccessLogApacheMiddleware{
Format: "%s %Dμs %r",
})
} else if lg.V(3) {
api.Use(&middlewares.AccessLogApacheErrorMiddleware{
AccessLogApacheMiddleware: &middlewares.AccessLogApacheMiddleware{
Format: "%s %Dμs %r",
},
})
}
api.Use(
&rest.TimerMiddleware{},
&rest.RecorderMiddleware{},
&rest.PoweredByMiddleware{},
// &rest.ContentTypeCheckerMiddleware{},
)
if lg.V(5) {
api.Use(
&rest.RecoverMiddleware{
EnableResponseStackTrace: true,
Logger: logger,
},
)
} else {
api.Use(
&rest.RecoverMiddleware{
Logger: logger,
},
)
}
router, err := rest.MakeRouter(routes...)
if err != nil {
panic(err)
}
api.SetApp(router)
handler := api.MakeHandler()
mux.Handle("/api/", http.StripPrefix("/api", handler))
return err
}
// Wait blocks until the underlying process is stopped
func (s *Service) Wait() {
if s.cmd != nil {
lg.V(10).Infof("Waiting for process %s to exit", s.ID)
err := s.cmd.Wait()
if err != nil {
lg.Warningln(err)
}
lg.V(10).Infof("%s exited", s.ID)
}
s.waiter.Wait()
}
// Remove a service to list of managed servers
func (s *Services) remove(service *Service) (err error) {
lg.V(5).Infof("removing service %s", service.ID)
s.Lock()
defer s.Unlock()
id := service.ID
if s.items[id] == nil {
return errors.New("service not registered, cannot be removed")
}
delete(s.items, id)
lg.V(19).Infof("removed service %s", service.ID)
return
}
// AtexitKillCmd takes care of killing a command on application exit.
//
// TODO: currently this does not clean up references to dead processes, it just
// adds forever.
func AtexitKillCmd(cmd *exec.Cmd) {
Atexit(func() {
lg.V(10).Info("Atexit kill ", cmd.Path, cmd.Args)
err := cmd.Process.Kill()
if err != nil {
lg.V(5).Info("kill failed:", err)
}
// TODO: possible deadlock?
if err := cmd.Wait(); err != nil {
lg.Warningln(err)
}
})
}
// Wait blocks until the underlying process is stopped
func (s *Service) wait() {
if s.isCopy {
lg.Fatal("wait called on copy of service!")
}
if s.cmd != nil {
lg.V(10).Infof("Waiting for process %s to exit", s.ID)
err := s.cmd.Wait()
if err != nil {
lg.Warningln(err)
}
lg.V(10).Infof("%s exited", s.ID)
}
s.waiter.Wait()
}
func PostTransportTraffic(w rest.ResponseWriter, r *rest.Request) {
form := shared.TransportTraffic{}
err := r.DecodeJsonPayload(&form)
if err != nil {
apiutils.WriteRestError(w, apierrors.NewInternalError(err))
return
}
transportTrafficMu.Lock()
defer transportTrafficMu.Unlock()
transportTraffic = form
if lg.V(10) {
if len(transportTrafficLog) == 6 {
lg.Infof("transport traffic: %.0fkb/s %.0fkb/s %.0fkb/s %.0fkb/s %.0fkb/s %.0fkb/s",
(transportTrafficLog[0].Throughput)/1024,
(transportTrafficLog[1].Throughput)/1024,
(transportTrafficLog[2].Throughput)/1024,
(transportTrafficLog[3].Throughput)/1024,
(transportTrafficLog[4].Throughput)/1024,
(transportTrafficLog[5].Throughput)/1024,
)
transportTrafficLog = make([]shared.TransportTraffic, 0)
}
if transportTraffic.Throughput > 1024 {
transportTrafficLog = append(transportTrafficLog, form)
}
}
response := true
w.WriteJson(response)
}
func testConn(event *ConnectionEvent) error {
defaultTransportM.RLock()
defer defaultTransportM.RUnlock()
if defaultTransport == nil {
transportOkC <- false
event.newState(TestFailed)
event.newState(NotConfigured)
event.newState(Ended)
return errors.New("No active transport")
}
err := testSocks5Internet(defaultTransport.Service.Response["bindaddr"])
if err != nil {
transportOkC <- false
event.newState(TestFailed)
event.newState(Failed)
event.newState(Ended)
return errors.New("Http get test failed")
} else {
if event.State != Up && lg.V(4) {
lg.Infof("event: tested %s -> %s (%s)", event.State, Up, event.ServiceID)
}
transportOkC <- true
if event.State != Up {
event.newState(Up)
}
}
transportOkC <- true
return nil
}
// AcceptedURL returns true if the supplied IP is allowed to be added to alkasir.
func AcceptedIP(ip net.IP) bool {
if ip.IsLoopback() || ip.Equal(net.IPv4zero) || ip.Equal(net.IPv6zero) {
if lg.V(50) {
lg.Warningf("ip address %s is not allowed because loopback or zeoro address", ip.String())
}
return false
}
for _, v := range disallowedNets {
if v.Contains(ip) {
if lg.V(50) {
lg.Warningf("ip %s is not allowed because network %s is not allowed", ip.String(), v.String())
}
return false
}
}
return true
}
// StartBlocklistUpgrader react to certain conitions for when the list of
// blocked urls should be updated.
//
// This function runs in it's own goroutine.
func StartBlocklistUpgrader() {
connectionEventListener := make(chan service.ConnectionHistory)
uChecker, _ := NewUpdateChecker("blocklist")
service.AddListener(connectionEventListener)
currentCountry := clientconfig.Get().Settings.Local.CountryCode
checkCountrySettingC := time.NewTicker(2 * time.Second)
defer checkCountrySettingC.Stop()
loop:
for {
select {
// Update when the transport connection comes up
case event := <-connectionEventListener:
if event.IsUp() {
uChecker.Activate()
uChecker.UpdateNow()
}
// Tell updatechecker to request update when user changes country settings
case <-checkCountrySettingC.C:
conf := clientconfig.Get()
if currentCountry != conf.Settings.Local.CountryCode {
currentCountry = conf.Settings.Local.CountryCode
uChecker.UpdateNow()
}
// Update by request of the update checker
case request := <-uChecker.RequestC:
conf := clientconfig.Get()
if conf.Settings.Local.CountryCode == "__" {
lg.V(9).Infoln("Country is __, skipping blocklist updates")
continue loop
}
currentCountry = conf.Settings.Local.CountryCode
n, err := upgradeBlockList()
if err != nil {
lg.Errorf("blocklist update cc:%s err:%v", currentCountry, err)
ui.Notify("blocklist_update_error_message")
request.ResponseC <- UpdateError
} else {
lg.V(5).Infof("blocklist update success cc:%s, got %d entries", currentCountry, n)
ui.Notify("blocklist_update_success_message")
request.ResponseC <- UpdateSuccess
}
}
}
}
// AcceptedURL returns true if the supplied url is allowed to be added to alkasir.
func AcceptedURL(u *url.URL) bool {
if _, ok := allowedProtocols[u.Scheme]; !ok {
if lg.V(50) {
lg.Warningf("url scheme %s is not allowed", u.Scheme)
}
return false
}
return AcceptedHost(u.Host)
}
// Return list of all registered services
func (s *Services) stopAll() error {
s.RLock()
defer s.RUnlock()
for _, s := range s.items {
if s != nil {
lg.V(10).Infof("stopping service %v", s)
s.Stop()
}
}
for _, s := range s.items {
if s != nil {
lg.V(10).Infof("waiting for service to stop: %v", s)
s.wait()
}
}
return nil
}
// Write delegates write to everything that is persisted below this level.
func Write() error {
if lg.V(15) {
lg.InfoDepth(1, "called config write")
}
currentConrfigMu.RLock()
defer currentConrfigMu.RUnlock()
filename := ConfigPath("settings.json")
lg.V(5).Infoln("Saving settings file")
data, err := json.MarshalIndent(¤tConfig.Settings, "", " ")
if err != nil {
return err
}
err = ioutil.WriteFile(filename, data, 0644)
if err != nil {
return err
}
return nil
}
func readSettings(c *Config) error {
lg.V(5).Info("Reading settings file")
isRead := false
_, err := mkConfigDir()
if err != nil {
return err
}
data, err := ioutil.ReadFile(ConfigPath("settings.json"))
if err != nil {
lg.Infof("Error loading settings.json %s", err)
} else {
settings, err := parseConfig(data)
if err != nil {
lg.Warningf("Config file error, deleting and resetting")
err := os.Remove(ConfigPath("settings.json"))
if err != nil {
lg.Warningf("Could not delete old settingsfile (should probably panic here)")
}
} else {
currentConfig.Settings = *settings
isRead = true
}
}
if !isRead {
settings, err := parseConfig([]byte(settingsTemplate))
if err != nil {
panic("invalid defaultsettings")
}
currentConfig.Settings = *settings
}
transports := make(map[string]shared.Transport, 0)
if currentConfig.Settings.Transports != nil {
for _, v := range currentConfig.Settings.Transports {
transports[v.Name] = v
}
}
for _, v := range []shared.Transport{
{Name: "obfs3", Bundled: true, TorPT: true},
{Name: "obfs4", Bundled: true, TorPT: true},
{Name: "shadowsocks-client", Bundled: true},
} {
transports[v.Name] = v
}
currentConfig.Settings.Transports = transports
return nil
}
// SetConfig reads configuration from json byte stream
func parseConfig(config []byte) (*Settings, error) {
s := &Settings{}
err := json.Unmarshal(config, &s)
if err != nil {
return nil, err
}
for i, c := range s.Connections {
err := c.EnsureID()
if err != nil {
lg.Fatal(err)
}
lg.V(15).Infof("connection id: %s", c.ID)
if lg.V(50) {
v, _ := c.Encode()
lg.Infof("connection encoded: %s", v)
lg.Infof("connection full: %+v", c)
}
s.Connections[i] = c
}
return s, nil
}
func apiError(w rest.ResponseWriter, error string, code int) {
w.WriteHeader(code)
if lg.V(5) {
lg.InfoDepth(1, fmt.Sprintf("%d: %s", code, error))
}
err := w.WriteJson(map[string]string{
"Error": error,
"Ok": "false",
})
if err != nil {
panic(err)
}
}
// StopAll stops all services, blocks until everything is shut down.
func StopAll() {
as := ManagedServices.AllServices()
var waiters []func()
for _, s := range as {
lg.V(10).Infof("stopping service %v", s)
if s.Running() {
s.Stop()
waiters = append(waiters, s.Wait)
}
}
for _, v := range waiters {
v()
}
}
func logSQLErr(err error, query squirrel.Sqlizer) {
if err != nil {
if lg.V(10) {
sql, args, qerr := query.ToSql()
var msg string
if qerr != nil {
msg = fmt.Sprintf("sql error: %s", err)
} else {
msg = fmt.Sprintf("sql error: %s: %+v, %s", sql, args, err)
}
lg.ErrorDepth(1, msg)
} else {
lg.ErrorDepth(1, "sql error: %s", err.Error())
}
} else if lg.V(19) {
sql, _, qerr := query.ToSql()
if qerr != nil {
lg.ErrorDepth(1, fmt.Sprintf("sql error: %s", err))
}
lg.InfoDepth(1, fmt.Sprintf("sql query: %s", sql))
}
}
// NewUpdateChecker creates and returns an UpdateChecker instance.
// The caller should then listen on the RequestC channel for UpdateRequests.
func NewUpdateChecker(name string) (*UpdateChecker, error) {
c := &UpdateChecker{
Interval: time.Duration(1*time.Hour + (time.Minute * (time.Duration(rand.Intn(120))))),
}
c.response = make(chan UpdateResult)
c.RequestC = make(chan UpdateRequest)
c.forceRequestC = make(chan bool)
lg.Infof("Setting up update timer for %s every %f minute(s) ",
name, c.Interval.Minutes())
ticker := time.NewTicker(c.Interval)
go func() {
for {
select {
case <-c.forceRequestC:
if !c.active {
continue
}
c.RequestC <- UpdateRequest{
ResponseC: c.response,
}
case <-ticker.C:
if !c.active {
continue
}
c.RequestC <- UpdateRequest{
ResponseC: c.response,
}
case response := <-c.response:
c.LastCheck = time.Now()
switch response {
case UpdateSuccess:
lg.V(5).Infoln("UpdateSuccess")
c.LastUpdate = c.LastCheck
c.LastFailedCheck = time.Time{}
case UpdateError:
lg.Warningln("update check failed")
c.LastFailedCheck = c.LastCheck
<-time.After(3*time.Second + time.Duration(rand.Intn(5)))
go func() {
c.forceRequestC <- true
}()
}
}
}
}()
return c, nil
}
func (r *relatedHosts) update() {
lg.V(19).Infoln("updating related hosts..")
related, err := r.dbclients.DB.GetRelatedHosts()
if err != nil {
lg.Fatal(err)
}
curated := make(map[string][]string, len(related))
for k, v := range related {
curated[strings.TrimPrefix(k, "www.")] = v
}
r.Lock()
r.items = curated
r.Unlock()
}
func StartAnalysis(clients db.Clients) {
tick := time.NewTicker(5 * time.Second)
lastID, err := clients.DB.GetLastProcessedSampleID()
if err != nil {
lg.Warningln(err)
}
for n := 0; n < 4; n++ {
go sessionFetcher(clients)
go samplesAnalyzer()
go hostPublisher(clients)
}
lg.Infof("starting analysis from sample ID %d", lastID)
lastPersistedID := lastID
for range tick.C {
results, err := clients.DB.GetSamples(uint64(lastID), "")
if err != nil {
lg.Errorf("database err (skipping): %v", err)
continue
}
n := 0
start := time.Now()
for s := range results {
n++
if s.ID > lastID {
lastID = s.ID
}
if s.Origin == "Central" && s.Type == "HTTPHeader" {
sessionFetchC <- s.Token
}
}
if n != 0 && lg.V(15) {
lg.Infof("processed %d samples in %s", n, time.Since(start).String())
}
if lastID != lastPersistedID {
err = clients.DB.SetLastProcessedSampleID(lastID)
if err != nil {
lg.Errorln(err)
} else {
lastPersistedID = lastID
}
}
}
}
// PostLogRequest
func PostLog(w rest.ResponseWriter, r *rest.Request) {
form := PostLogRequest{}
err := r.DecodeJsonPayload(&form)
if err != nil {
lg.Warning(err)
apiutils.ErrToAPIStatus(err)
// apiutils.WriteRestError(w, apierrors.NewBadRequest("can't decode json"))
// apiutils.WriteRestError(w, apierrors.NewBadRequest("can't decode json"))
return
}
sender := r.PathParam("sender")
lg.V(form.Level).Infof("{%s} %s: %s", sender, form.Context, form.Message)
w.WriteJson(true)
}
func startAnalysis(clients db.Clients) {
go func() {
tick := time.NewTicker(10 * time.Second)
lastID, err := clients.DB.GetLastProcessedSampleID()
if err != nil {
lg.Warningln(err)
}
lg.Infof("starting analysis from sample ID %d", lastID)
lastPersistedID := lastID
for range tick.C {
results, err := clients.DB.GetSamples(uint64(lastID), "")
if err != nil {
lg.Fatal(err)
}
n := 0
start := time.Now()
loop:
for s := range results {
n++
if s.ID > lastID {
lastID = s.ID
}
if s.Type == "NewClientToken" {
if !shared.AcceptedHost(s.Host) {
lg.Warningln("not accepted host id:", s.ID, s.Host)
continue loop
}
err := clients.DB.PublishHost(s)
if err != nil {
lg.Warning(err)
}
}
}
if n != 0 && lg.V(15) {
lg.Infof("processed %d samples in %s", n, time.Since(start).String())
}
if lastID != lastPersistedID {
err = clients.DB.SetLastProcessedSampleID(lastID)
if err != nil {
lg.Errorln(err)
} else {
lastPersistedID = lastID
}
}
}
}()
}
func (a *Artifact) Extract() error {
extractor := extractor.NewDetectable()
dir := a.Dir("extracted")
err := os.RemoveAll(dir)
if err != nil {
return err
}
os.MkdirAll(dir, 0775)
apath, err := filepath.Abs(a.Path())
if err != nil {
return err
}
lg.V(5).Infof("extracting %s into %s", apath, dir)
return extractor.Extract(apath, dir)
}
func loadTranslations(languages ...string) {
for _, lang := range languages {
filename := fmt.Sprintf("messages/%s/messages.json", lang)
lg.V(6).Infoln("Loading translation", lang, filename)
data, err := res.Asset(filename)
if err != nil {
panic(err)
}
err = i18n.AddBundle(lang, data)
if err != nil {
panic(err)
}
}
}
func connect(connection shared.Connection, authKey string) {
defaultTransportM.RLock()
if defaultTransport != nil {
err := defaultTransport.Remove()
if err != nil {
lg.Warningln(err)
}
}
defaultTransportM.RUnlock()
event := newConnectionEventhistory(connection)
event.newState(ServiceInit)
ts, err := NewTransportService(connection)
ts.authSecret = authKey
if lg.V(6) {
ts.SetVerbose()
}
ts.SetBindaddr(DefaultProxyBindAddr)
if err != nil {
event.newState(Failed)
event.newState(Ended)
return
}
event.newState(ServiceStart)
event.ServiceID = ts.Service.ID
err = ts.Start()
if err != nil {
event.newState(Failed)
event.newState(Ended)
return
}
response := ts.Service.Response
if response["protocol"] != "socks5" {
event.newState(WrongProtocol)
event.newState(Failed)
}
event.newState(Test)
go testConn(&event)
defaultTransportM.Lock()
defaultTransport = ts
defaultTransportM.Unlock()
}