// StopAll stops all services, blocks until everything is shut down.
func StopAll() {
err := ManagedServices.stopAll()
if err != nil {
lg.Error(err)
}
}
func (d *DebugResponse) WriteToDisk() error {
dir := d.filename()
if err := os.MkdirAll(dir, 0775); err != nil {
panic(err)
}
writeTextFile := func(data []string, basename string) error {
filename := filepath.Join(dir, basename+".txt")
f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0665)
if err != nil {
return err
}
defer f.Close()
for _, v := range data {
_, err := f.WriteString(v + "\n")
if err != nil {
return err
}
}
return nil
}
writeJsonFile := func(data interface{}, basename string) error {
bytes, err := json.MarshalIndent(&data, "", " ")
if err != nil {
panic(err)
}
filename := filepath.Join(dir, basename+".json")
err = ioutil.WriteFile(filename, bytes, 0665)
if err != nil {
return err
}
return nil
}
failed := false
for _, v := range []error{
writeJsonFile(d.Header, "header"),
writeTextFile(d.Log, "log"),
writeTextFile(d.Heap, "heap"),
writeTextFile(d.GoRoutines, "goroutines"),
writeTextFile(d.Block, "block"),
writeTextFile(d.ThreadCreate, "threadcreate"),
writeJsonFile(d.Config, "config"),
} {
if v != nil {
failed = true
lg.Error(v)
}
}
if failed {
return fmt.Errorf("errors writing out report %s", dir)
}
lg.Infof("wrote report for %s", dir)
return nil
}
func startMeasurer(dbclients db.Clients) {
for n := 0; n < 10; n++ {
go func() {
var ps PreparedSample
err := ps.Update(dbclients)
if err != nil {
lg.Error("could not resolve public ip address", err)
}
lg.V(5).Infoln("starting measurer")
for r := range requestMeasurements {
lg.V(50).Infoln("got measurement", r)
if ps.lastUpdated.Before(time.Now().Add(-time.Hour * 5)) {
lg.V(15).Info("updating prepared sample", ps)
err := ps.Update(dbclients)
if err != nil {
lg.Warning(err)
}
}
measurerLoop:
for _, v := range r.measurers {
measurement, err := v.Measure()
if err != nil {
lg.Errorf("could not measure:%v error:%s", v, err.Error())
continue measurerLoop
}
switch measurement.Type() {
case sampletypes.DNSQuery, sampletypes.HTTPHeader:
data, err := measurement.Marshal()
if err != nil {
lg.Errorf("could not decode %v error:%s", measurement, err.Error())
continue measurerLoop
}
err = dbclients.DB.InsertSample(db.Sample{
Host: measurement.Host(),
CountryCode: ps.s.CountryCode,
Token: r.token,
ASN: ps.s.ASN,
Type: measurement.Type().String(),
Origin: sampleorigins.Central.String(),
Data: data,
})
if err != nil {
lg.Errorln(err.Error())
continue measurerLoop
}
default:
lg.Errorf("could not measure:%v error:%s", v, err.Error())
continue measurerLoop
}
}
}
}()
}
}
func hostPublisher(clients db.Clients) {
for sample := range hostPublishC {
err := clients.DB.PublishHost(sample)
if err != nil {
lg.Error(err)
}
}
}
func apiError(w rest.ResponseWriter, error string, code int) {
w.WriteHeader(code)
if lg.V(5) {
lg.InfoDepth(1, fmt.Sprintf("%d: %s", code, error))
}
err := w.WriteJson(map[string]string{
"Error": error,
"Ok": "false",
})
if err != nil {
lg.Error(err)
return
}
}
func sessionFetcher(clients db.Clients) {
loop:
for token := range sessionFetchC {
if token == "" {
lg.Errorln("empty token received, skipping")
continue loop
}
samples, err := clients.DB.GetSessionSamples(token)
if err != nil {
lg.Error(err)
continue loop
}
sampleAnalysisC <- samples
}
}
func samplesAnalyzer() {
loop:
for samples := range sampleAnalysisC {
var (
newTokenSample db.Sample
clientSamples = make(map[string]db.Sample, 0)
centralSamples = make(map[string]db.Sample, 0)
)
// organize input data
for _, s := range samples {
switch s.Type {
case "NewClientToken":
if newTokenSample.Token != "" {
lg.Error("got more than one newTokenSample, aborting")
continue loop
}
newTokenSample = s
case "HTTPHeader", "DNSQuery":
switch s.Origin {
case "Central":
centralSamples[s.Type] = s
case "Client":
clientSamples[s.Type] = s
}
default:
lg.Errorf("dont know how to handle %d %s, skipping", s.ID, s.Type)
}
}
// validate that wanted data types are available
if newTokenSample.Token == "" {
lg.Errorln("No newTokenSample, aborting")
continue loop
}
if !shared.AcceptedHost(newTokenSample.Host) {
lg.Warningln("not accepted host id:", newTokenSample.ID, newTokenSample.Host)
continue loop
}
for _, smap := range []map[string]db.Sample{clientSamples, centralSamples} {
for _, stype := range []string{"HTTPHeader"} {
if _, ok := smap[stype]; !ok {
lg.Errorf("missing %s, cannot analyse", stype)
continue loop
}
}
}
// parse data
clientSample := clientSamples["HTTPHeader"]
var clientHeader measure.HTTPHeaderResult
centralSample := centralSamples["HTTPHeader"]
var centralHeader measure.HTTPHeaderResult
if err := json.Unmarshal(clientSample.Data, &clientHeader); err != nil {
lg.Errorln(err)
continue loop
}
if err := json.Unmarshal(centralSample.Data, ¢ralHeader); err != nil {
lg.Error(err)
continue loop
}
// score data
score := scoreHTTPHeaders(clientHeader, centralHeader)
lg.V(10).Infof("session:%s %s", newTokenSample.Token, score)
if score.Score() >= 0.5 {
lg.Infoln("publishing session", newTokenSample.Token)
hostPublishC <- newTokenSample
} else {
lg.Infoln("not publishing session", newTokenSample.Token)
}
}
}
func startInternalHTTPServer(authKey string) error {
lg.V(15).Infoln("starting internal api server")
mux, err := createServeMux()
if err != nil {
lg.Error("failed to create router")
return err
}
auth := Auth{Key: authKey, wrapped: mux}
conf := clientconfig.Get()
listener, err := net.Listen("tcp", conf.Settings.Local.ClientBindAddr)
if err != nil {
lg.Warning(err)
listener, err = net.Listen("tcp", "127.0.0.1:")
}
if err != nil {
lg.Warning(err)
ui.Notify("Could not bind any local port (bootstrap)")
lg.Errorln("Could not bind any local port (bootstrap)")
return err
}
go func(listenaddr string) {
// baseURL := fmt.Sprintf()
baseURL := fmt.Sprintf("http://%s?suk=", listenaddr)
for {
select {
case <-ui.Actions.CopyBrowserCodeToClipboard:
bc := browsercode.BrowserCode{Key: authKey}
err := bc.SetHostport(listener.Addr().String())
if err != nil {
lg.Errorln(err)
continue
}
err = bc.CopyToClipboard()
if err != nil {
lg.Errorln(err)
}
case <-ui.Actions.OpenInBrowser:
browser.OpenURL(baseURL + singleUseKeys.New() + "#/")
case <-ui.Actions.Help:
browser.OpenURL(baseURL + singleUseKeys.New() + "#/docs/__/index")
}
singleUseKeys.Cleanup()
}
}(listener.Addr().String())
doneC := make(chan bool, 1)
go func() {
defer listener.Close()
err = http.Serve(listener, auth)
if err != nil {
doneC <- false
}
}()
select {
case ok := <-doneC:
if !ok {
return errors.New("Could not start internal http server")
}
case <-time.After(time.Millisecond * 200):
return nil
}
return nil
}
// Update list of blocked hosts for an IP address.
func GetHosts(dbclients db.Clients) func(w rest.ResponseWriter, r *rest.Request) {
relh := relatedHosts{
dbclients: dbclients,
}
relh.update()
go func() {
for range time.NewTicker(10 * time.Minute).C {
relh.update()
}
}()
return func(w rest.ResponseWriter, r *rest.Request) {
// HANDLE USERIP BEGIN
req := shared.UpdateHostlistRequest{}
err := r.DecodeJsonPayload(&req)
if err != nil {
apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError)
return
}
// parse/validate client ip address.
IP := req.ClientAddr
if IP == nil {
apiError(w, "bad ClientAddr", http.StatusBadRequest)
return
}
// resolve ip to asn.
var ASN int
ASNres, err := dbclients.Internet.IP2ASN(IP)
if err != nil {
lg.Errorln(shared.SafeClean(err.Error()))
apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError)
return
}
if ASNres != nil {
ASN = ASNres.ASN
} else {
lg.Warningf("no ASN lookup result for IP: %s ", shared.SafeClean(IP.String()))
}
// resolve ip to country code.
countryCode := dbclients.Maxmind.IP2CountryCode(IP)
req.ClientAddr = net.IPv4zero
IP = net.IPv4zero
// HANDLE USERIP END
hosts, err := dbclients.DB.GetBlockedHosts(countryCode, ASN)
if err != nil {
apiError(w, err.Error(), http.StatusInternalServerError)
return
}
simpleData := struct {
ClientVersion string `json:"version"` // client version idientifer
}{
req.ClientVersion,
}
data, err := json.Marshal(simpleData)
if err != nil {
lg.Errorln(err)
}
ss := db.SimpleSample{
CountryCode: countryCode,
ASN: ASN,
Type: "ClientBlocklistUpdate",
OriginID: req.UpdateID,
Data: data,
}
err = dbclients.DB.InsertSimpleSample(ss)
if err != nil {
lg.Errorf("error persisting simplesample %v", ss)
}
err = w.WriteJson(shared.UpdateHostlistResponse{
Hosts: relh.fill(hosts),
})
if err != nil {
lg.Error(err)
return
}
}
}