func startMeasurer(dbclients db.Clients) {
for n := 0; n < 10; n++ {
go func() {
var ps PreparedSample
err := ps.Update(dbclients)
if err != nil {
lg.Error("could not resolve public ip address", err)
}
lg.V(5).Infoln("starting measurer")
for r := range requestMeasurements {
lg.V(50).Infoln("got measurement", r)
if ps.lastUpdated.Before(time.Now().Add(-time.Hour * 5)) {
lg.V(15).Info("updating prepared sample", ps)
err := ps.Update(dbclients)
if err != nil {
lg.Warning(err)
}
}
measurerLoop:
for _, v := range r.measurers {
measurement, err := v.Measure()
if err != nil {
lg.Errorf("could not measure:%v error:%s", v, err.Error())
continue measurerLoop
}
switch measurement.Type() {
case sampletypes.DNSQuery, sampletypes.HTTPHeader:
data, err := measurement.Marshal()
if err != nil {
lg.Errorf("could not decode %v error:%s", measurement, err.Error())
continue measurerLoop
}
err = dbclients.DB.InsertSample(db.Sample{
Host: measurement.Host(),
CountryCode: ps.s.CountryCode,
Token: r.token,
ASN: ps.s.ASN,
Type: measurement.Type().String(),
Origin: sampleorigins.Central.String(),
Data: data,
})
if err != nil {
lg.Errorln(err.Error())
continue measurerLoop
}
default:
lg.Errorf("could not measure:%v error:%s", v, err.Error())
continue measurerLoop
}
}
}
}()
}
}
func exit() {
lg.Infoln("alkasir is shutting down")
atexitMu.Lock() // this lock should be kept, one shutdown should be enough for everyone.
lg.Flush()
if err := clientconfig.Write(); err != nil {
lg.Errorf("could not save config file: %s", err.Error())
}
lg.V(9).Infoln("running atexit funcs")
for _, f := range atexitFuncs {
funcName := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
lg.V(10).Infoln("Running at exit func", funcName)
f()
lg.V(10).Infoln("Finished at exit func", funcName)
}
atexitFuncs = atexitFuncs[:0]
lg.V(9).Infoln("atexit funcs done")
lg.V(9).Infoln("stopping connectionmanager")
service.StopConnectionManager()
lg.V(9).Infoln("stopping services")
service.StopAll()
lg.V(9).Infoln("services stopped")
lg.V(9).Infoln("waiting for UI shutdown to finish")
uiRunning.Wait()
lg.V(9).Infoln("ui shut down")
lg.Flush()
lg.Infoln("alkasir shutdown complete")
lg.Flush()
time.Sleep(time.Millisecond * 1)
os.Exit(0)
}
func debugImportDebug(files []string) error {
if len(files) == 0 {
fmt.Println("need argument: files...")
return errNoValue
}
files:
for _, filename := range files {
data, err := ioutil.ReadFile(filename)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
var debuginfo debugexport.DebugResponse
err = json.Unmarshal(data, &debuginfo)
if err != nil {
fmt.Printf("could not decode %s: %s \n", filename, err.Error())
os.Exit(1)
}
if debuginfo.Encrypted != "" {
err := debuginfo.Decrypt(debugPublicKey, debugSecretKey)
if err != nil {
lg.Errorf("%s could not be decrypted, skipping: %v", filename, err)
continue files
}
}
debuginfo.WriteToDisk()
}
return nil
}
func StartAnalysis(clients db.Clients) {
tick := time.NewTicker(5 * time.Second)
lastID, err := clients.DB.GetLastProcessedSampleID()
if err != nil {
lg.Warningln(err)
}
for n := 0; n < 4; n++ {
go sessionFetcher(clients)
go samplesAnalyzer()
go hostPublisher(clients)
}
lg.Infof("starting analysis from sample ID %d", lastID)
lastPersistedID := lastID
for range tick.C {
results, err := clients.DB.GetSamples(uint64(lastID), "")
if err != nil {
lg.Errorf("database err (skipping): %v", err)
continue
}
n := 0
start := time.Now()
for s := range results {
n++
if s.ID > lastID {
lastID = s.ID
}
if s.Origin == "Central" && s.Type == "HTTPHeader" {
sessionFetchC <- s.Token
}
}
if n != 0 && lg.V(15) {
lg.Infof("processed %d samples in %s", n, time.Since(start).String())
}
if lastID != lastPersistedID {
err = clients.DB.SetLastProcessedSampleID(lastID)
if err != nil {
lg.Errorln(err)
} else {
lastPersistedID = lastID
}
}
}
}
// StartBlocklistUpgrader react to certain conitions for when the list of
// blocked urls should be updated.
//
// This function runs in it's own goroutine.
func StartBlocklistUpgrader() {
connectionEventListener := make(chan service.ConnectionHistory)
uChecker, _ := NewUpdateChecker("blocklist")
service.AddListener(connectionEventListener)
currentCountry := clientconfig.Get().Settings.Local.CountryCode
checkCountrySettingC := time.NewTicker(2 * time.Second)
defer checkCountrySettingC.Stop()
loop:
for {
select {
// Update when the transport connection comes up
case event := <-connectionEventListener:
if event.IsUp() {
uChecker.Activate()
uChecker.UpdateNow()
}
// Tell updatechecker to request update when user changes country settings
case <-checkCountrySettingC.C:
conf := clientconfig.Get()
if currentCountry != conf.Settings.Local.CountryCode {
currentCountry = conf.Settings.Local.CountryCode
uChecker.UpdateNow()
}
// Update by request of the update checker
case request := <-uChecker.RequestC:
conf := clientconfig.Get()
if conf.Settings.Local.CountryCode == "__" {
lg.V(9).Infoln("Country is __, skipping blocklist updates")
continue loop
}
currentCountry = conf.Settings.Local.CountryCode
n, err := upgradeBlockList()
if err != nil {
lg.Errorf("blocklist update cc:%s err:%v", currentCountry, err)
ui.Notify("blocklist_update_error_message")
request.ResponseC <- UpdateError
} else {
lg.V(5).Infof("blocklist update success cc:%s, got %d entries", currentCountry, n)
ui.Notify("blocklist_update_success_message")
request.ResponseC <- UpdateSuccess
}
}
}
}
// String converts a ValidationErrorType into its corresponding error message.
func (t ValidationErrorType) String() string {
switch t {
case ValidationErrorTypeNotFound:
return "not found"
case ValidationErrorTypeRequired:
return "required value"
case ValidationErrorTypeDuplicate:
return "duplicate value"
case ValidationErrorTypeInvalid:
return "invalid value"
case ValidationErrorTypeNotSupported:
return "unsupported value"
case ValidationErrorTypeForbidden:
return "forbidden"
case ValidationErrorTypeTooLong:
return "too long"
default:
glog.Errorf("unrecognized validation type: %#v", t)
return ""
}
}
func samplesAnalyzer() {
loop:
for samples := range sampleAnalysisC {
var (
newTokenSample db.Sample
clientSamples = make(map[string]db.Sample, 0)
centralSamples = make(map[string]db.Sample, 0)
)
// organize input data
for _, s := range samples {
switch s.Type {
case "NewClientToken":
if newTokenSample.Token != "" {
lg.Error("got more than one newTokenSample, aborting")
continue loop
}
newTokenSample = s
case "HTTPHeader", "DNSQuery":
switch s.Origin {
case "Central":
centralSamples[s.Type] = s
case "Client":
clientSamples[s.Type] = s
}
default:
lg.Errorf("dont know how to handle %d %s, skipping", s.ID, s.Type)
}
}
// validate that wanted data types are available
if newTokenSample.Token == "" {
lg.Errorln("No newTokenSample, aborting")
continue loop
}
if !shared.AcceptedHost(newTokenSample.Host) {
lg.Warningln("not accepted host id:", newTokenSample.ID, newTokenSample.Host)
continue loop
}
for _, smap := range []map[string]db.Sample{clientSamples, centralSamples} {
for _, stype := range []string{"HTTPHeader"} {
if _, ok := smap[stype]; !ok {
lg.Errorf("missing %s, cannot analyse", stype)
continue loop
}
}
}
// parse data
clientSample := clientSamples["HTTPHeader"]
var clientHeader measure.HTTPHeaderResult
centralSample := centralSamples["HTTPHeader"]
var centralHeader measure.HTTPHeaderResult
if err := json.Unmarshal(clientSample.Data, &clientHeader); err != nil {
lg.Errorln(err)
continue loop
}
if err := json.Unmarshal(centralSample.Data, ¢ralHeader); err != nil {
lg.Error(err)
continue loop
}
// score data
score := scoreHTTPHeaders(clientHeader, centralHeader)
lg.V(10).Infof("session:%s %s", newTokenSample.Token, score)
if score.Score() >= 0.5 {
lg.Infoln("publishing session", newTokenSample.Token)
hostPublishC <- newTokenSample
} else {
lg.Infoln("not publishing session", newTokenSample.Token)
}
}
}
func createUpgradeAuto(args []string) error {
var (
privPemFlag string
pubPemFlag string
)
fs := flag.NewFlagSet("upgrade create", flag.ExitOnError)
fs.StringVar(&privPemFlag, "privpem", "upgrades-private-key.pem", "path to load private key file from")
fs.StringVar(&pubPemFlag, "pubpem", "upgrades-public-key.pem", "path to load public key file from")
fs.Parse(args)
args = fs.Args()
privPem, err := ioutil.ReadFile(privPemFlag)
if err != nil {
if os.IsNotExist(err) {
lg.Errorf("%s does not exist", privPemFlag)
return nil
}
return err
}
pubPem, err := ioutil.ReadFile(pubPemFlag)
if err != nil {
if os.IsNotExist(err) {
lg.Errorf("%s does not exist", pubPemFlag)
return nil
}
return err
}
results, err := makepatch.RunPatchesCreate(
jobQs, string(privPem), string(pubPem), nWorkersFlag)
if err != nil {
panic(err)
}
if len(results) < 1 {
lg.Fatalln("no patch results returned")
}
var allFiles []*tar.Header
err = filepath.Walk("diffs", func(path string, f os.FileInfo, err error) error {
if f.IsDir() {
return nil
}
allFiles = append(allFiles, &tar.Header{
Name: path,
Mode: 0600,
Size: f.Size(),
})
return nil
})
if err != nil {
lg.Fatal(err)
}
latestVersion := results[0].NewVersion
filename := fmt.Sprintf("alkasir-binpatches-for-%s.tar", latestVersion)
tarfile, err := os.Create(filename)
if err != nil {
panic(err)
}
tw := tar.NewWriter(tarfile)
for _, hdr := range allFiles {
if err := tw.WriteHeader(hdr); err != nil {
log.Fatalln(err)
}
s, err := os.Open(hdr.Name)
if err != nil {
return err
}
_, err = io.Copy(tw, s)
if err != nil {
lg.Fatal(err)
}
err = s.Close()
if err != nil {
lg.Fatal(err)
}
}
if err := tw.Close(); err != nil {
log.Fatalln(err)
}
lg.Infoln("done")
return nil
}
// CreateSuggestion .
func CreateSuggestion(w rest.ResponseWriter, r *rest.Request) {
form := shared.BrowserSuggestionTokenRequest{}
err := r.DecodeJsonPayload(&form)
if err != nil {
// apiError(w, err.Error(), http.StatusInternalServerError)
apiutils.WriteRestError(w, err)
return
}
var invalids fielderrors.ValidationErrorList
// parse and validate url.
URL := strings.TrimSpace(form.URL)
if URL == "" {
invalids = append(invalids, fielderrors.NewFieldRequired("URL"))
// apiError(w, "no or empty URL", http.StatusBadRequest)
}
u, err := url.Parse(URL)
if err != nil {
invalids = append(invalids, fielderrors.NewFieldInvalid("URL", URL, err.Error()))
// apiError(w, fmt.Sprintf("%s is not a valid URL", URL), http.StatusBadRequest)
}
host := u.Host
if strings.Contains(host, ":") {
host, _, err = net.SplitHostPort(u.Host)
if err != nil {
invalids = append(invalids, fielderrors.NewFieldInvalid("URL", URL, err.Error()))
}
}
if !shared.AcceptedURL(u) {
invalids = append(invalids, fielderrors.NewFieldValueNotSupported("URL", URL, nil))
}
if len(invalids) > 0 {
apiutils.WriteRestError(w, apierrors.NewInvalid("create-suggestion", "URL", invalids))
return
}
s := client.NewSuggestion(u.String())
defer s.DoneAddingSamples()
measurers, err := measure.DefaultMeasurements(form.URL)
if err != nil {
apiutils.WriteRestError(w, apierrors.NewInternalError(err))
return
}
for _, v := range measurers {
m, err := v.Measure()
if err != nil {
lg.Errorf("could not measure: %s", err.Error())
} else {
switch m.Type() {
case sampletypes.DNSQuery, sampletypes.HTTPHeader:
err = s.AddMeasurement(m)
if err != nil {
lg.Errorln(err.Error())
return
}
default:
lg.Warningf("unsupported sample type: %s", m.Type().String())
}
}
}
}
// Update list of blocked hosts for an IP address.
func GetHosts(dbclients db.Clients) func(w rest.ResponseWriter, r *rest.Request) {
relh := relatedHosts{
dbclients: dbclients,
}
relh.update()
go func() {
for range time.NewTicker(10 * time.Minute).C {
relh.update()
}
}()
return func(w rest.ResponseWriter, r *rest.Request) {
// HANDLE USERIP BEGIN
req := shared.UpdateHostlistRequest{}
err := r.DecodeJsonPayload(&req)
if err != nil {
apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError)
return
}
// parse/validate client ip address.
IP := req.ClientAddr
if IP == nil {
apiError(w, "bad ClientAddr", http.StatusBadRequest)
return
}
// resolve ip to asn.
var ASN int
ASNres, err := dbclients.Internet.IP2ASN(IP)
if err != nil {
lg.Errorln(shared.SafeClean(err.Error()))
apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError)
return
}
if ASNres != nil {
ASN = ASNres.ASN
} else {
lg.Warningf("no ASN lookup result for IP: %s ", shared.SafeClean(IP.String()))
}
// resolve ip to country code.
countryCode := dbclients.Maxmind.IP2CountryCode(IP)
req.ClientAddr = net.IPv4zero
IP = net.IPv4zero
// HANDLE USERIP END
hosts, err := dbclients.DB.GetBlockedHosts(countryCode, ASN)
if err != nil {
apiError(w, err.Error(), http.StatusInternalServerError)
return
}
simpleData := struct {
ClientVersion string `json:"version"` // client version idientifer
}{
req.ClientVersion,
}
data, err := json.Marshal(simpleData)
if err != nil {
lg.Errorln(err)
}
ss := db.SimpleSample{
CountryCode: countryCode,
ASN: ASN,
Type: "ClientBlocklistUpdate",
OriginID: req.UpdateID,
Data: data,
}
err = dbclients.DB.InsertSimpleSample(ss)
if err != nil {
lg.Errorf("error persisting simplesample %v", ss)
}
err = w.WriteJson(shared.UpdateHostlistResponse{
Hosts: relh.fill(hosts),
})
if err != nil {
lg.Error(err)
return
}
}
}
// initDone means handing off the service process output to it's own goroutine.
func (s *Service) initDone() error {
lg.V(5).Infof("s.request: %+v", s.Request)
lg.V(5).Infof("s.response: %+v", s.Response)
s.waiter.Add(1)
s.running = true
go func() {
defer func() {
s.running = false
}()
go func() {
scanner := bufio.NewScanner(s.stdout)
for scanner.Scan() {
if scanner.Err() != nil {
lg.Warningln("service stdout", scanner.Err())
break
}
line := scanner.Text()
lg.Infoln(s.ID, "stdout:", line)
}
lg.V(20).Infof("service outch closed %s", s.ID)
}()
go func() {
scanner := bufio.NewScanner(s.stderr)
for scanner.Scan() {
if scanner.Err() != nil {
lg.Warningln("service stderr", scanner.Err())
break
}
line := scanner.Text()
lg.Infoln(s.ID, "stderr:", line)
}
lg.V(20).Infof("service stderr closed %s", s.ID)
}()
s.quit = make(chan bool)
defer close(s.quit)
select {
case <-s.quit:
lg.V(6).Infof("stopping service %s", s.ID)
if err := s.stdin.Close(); err != nil {
lg.Errorf("could not close stdin for %s: %s", s.ID, err.Error())
}
if err := s.stdout.Close(); err != nil {
lg.Errorf("could not close stdout for %s: %s", s.ID, err.Error())
}
if err := s.stderr.Close(); err != nil {
lg.Errorf("could not close stderr for %s: %s", s.ID, err.Error())
}
lg.V(10).Infof("Killing process service %s", s.ID)
}
lg.V(10).Infof("stopped service %s", s.ID)
if s.removeOnStop {
ManagedServices.remove(s)
}
s.waiter.Done()
}()
return nil
}