// SuggestionToken JSON API method. func SuggestionToken(dbclients db.Clients) func(w rest.ResponseWriter, r *rest.Request) { return func(w rest.ResponseWriter, r *rest.Request) { // HANDLE USERIP BEGIN req := shared.SuggestionTokenRequest{} err := r.DecodeJsonPayload(&req) if err != nil { apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError) return } // validate country code. if !validCountryCode(req.CountryCode) { apiError(w, fmt.Sprintf("invalid country code: %s", req.CountryCode), http.StatusBadRequest) return } // parse/validate client ip address. IP := req.ClientAddr if IP == nil { apiError(w, "bad ClientAddr", http.StatusBadRequest) return } // parse and validate url. URL := strings.TrimSpace(req.URL) if URL == "" { apiError(w, "no or empty URL", http.StatusBadRequest) return } u, err := url.Parse(URL) if err != nil { apiError(w, fmt.Sprintf("%s is not a valid URL", URL), http.StatusBadRequest) return } if !shared.AcceptedURL(u) { apiError(w, fmt.Sprintf("%s is not a valid URL", URL), http.StatusBadRequest) return } // resolve ip to asn. var ASN int ASNres, err := dbclients.Internet.IP2ASN(IP) if err != nil { lg.Errorln(shared.SafeClean(err.Error())) apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError) return } if ASNres != nil { ASN = ASNres.ASN } else { lg.Warningf("no ASN lookup result for IP: %s ", shared.SafeClean(IP.String())) } // reoslve ip to country code. countryCode := dbclients.Maxmind.IP2CountryCode(IP) // resolve ip to city geonameid geoCityID := dbclients.Maxmind.IP2CityGeoNameID(IP) req.ClientAddr = net.IPv4zero IP = net.IPv4zero // HANDLE USERIP END { supported, err := dbclients.DB.IsURLAllowed(u, countryCode) if err != nil { // TODO: standardize http status codes apiError(w, err.Error(), http.StatusForbidden) return } if !supported { lg.Infof("got request for unsupported URL %s") w.WriteJson(shared.SuggestionTokenResponse{ Ok: false, URL: req.URL, }) return } } // start new submission token session token := db.SessionTokens.New(URL) // create newclienttoken sample data sample := shared.NewClientTokenSample{ URL: URL, CountryCode: req.CountryCode, } sampleData, err := json.Marshal(sample) if err != nil { lg.Errorln(err) apiError(w, "error #20150424-002542-CEST", http.StatusInternalServerError) return } // create extraData extra := shared.IPExtraData{ CityGeoNameID: geoCityID, } extraData, err := json.Marshal(extra) if err != nil { lg.Errorln(err) apiError(w, "error #20150427-211052-CEST", http.StatusInternalServerError) return } // insert into db { err := dbclients.DB.InsertSample(db.Sample{ Host: u.Host, CountryCode: countryCode, ASN: ASN, Type: "NewClientToken", Origin: "Central", Token: token, Data: sampleData, ExtraData: extraData, }) if err != nil { apiError(w, err.Error(), http.StatusInternalServerError) return } } // queue central measurements measurements, err := measure.DefaultMeasurements(req.URL) if err != nil { lg.Warningf("could not create standard measurements: %s", err.Error()) } else { queueMeasurements(token, measurements...) } // write json response { err := w.WriteJson(shared.SuggestionTokenResponse{ Ok: true, URL: URL, Token: token, }) if err != nil { lg.Errorln(err.Error()) return } } } }
// StoreSample JSON API method. func StoreSample(dbclients db.Clients) func(w rest.ResponseWriter, r *rest.Request) { return func(w rest.ResponseWriter, r *rest.Request) { // HANDLE USERIP BEGIN req := shared.StoreSampleRequest{} err := r.DecodeJsonPayload(&req) if err != nil { apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError) return } // validate sample type. if _, ok := clientSampleTypes[req.SampleType]; !ok { apiError(w, "invalid sample type: "+req.SampleType, http.StatusBadRequest) return } // get/validate suggestion session token. tokenData, validToken := db.SessionTokens.Get(req.Token) if !validToken { apiError(w, "invalid token", http.StatusBadRequest) return } // parse/validate url. URL := strings.TrimSpace(req.URL) if tokenData.URL != URL { lg.V(2).Infof("invalid URL %s for token session, was expecting %s", req.URL, tokenData.URL) apiError(w, "invalid URL for token session", http.StatusBadRequest) return } u, err := url.Parse(URL) if err != nil { apiError(w, fmt.Sprintf("%s is not a valid URL", URL), http.StatusBadRequest) return } // parse/validate client ip address. IP := net.ParseIP(req.ClientAddr) if IP == nil { apiError(w, "bad ClientAddr", http.StatusBadRequest) return } // resolve ip to asn. var ASN int ASNres, err := dbclients.Internet.IP2ASN(IP) if err != nil { lg.Errorln(shared.SafeClean(err.Error())) apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError) return } if ASNres != nil { ASN = ASNres.ASN } else { lg.Warningf("no ASN lookup result for IP: %s ", shared.SafeClean(IP.String())) } countryCode := dbclients.Maxmind.IP2CountryCode(IP) req.ClientAddr = "" IP = net.IPv4zero // HANDLE USERIP END // insert into db { err := dbclients.DB.InsertSample(db.Sample{ Host: u.Host, CountryCode: countryCode, ASN: ASN, Type: req.SampleType, Origin: "Client", Token: req.Token, Data: []byte(req.Data), }) if err != nil { lg.Errorln(err.Error()) apiError(w, "error 20150424-011948-CEST", http.StatusInternalServerError) return } } w.WriteJson(shared.StoreSampleResponse{ Ok: true, }) return } }
// Update list of blocked hosts for an IP address. func GetHosts(dbclients db.Clients) func(w rest.ResponseWriter, r *rest.Request) { relh := relatedHosts{ dbclients: dbclients, } relh.update() go func() { for range time.NewTicker(10 * time.Minute).C { relh.update() } }() return func(w rest.ResponseWriter, r *rest.Request) { // HANDLE USERIP BEGIN req := shared.UpdateHostlistRequest{} err := r.DecodeJsonPayload(&req) if err != nil { apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError) return } // parse/validate client ip address. IP := req.ClientAddr if IP == nil { apiError(w, "bad ClientAddr", http.StatusBadRequest) return } // resolve ip to asn. var ASN int ASNres, err := dbclients.Internet.IP2ASN(IP) if err != nil { lg.Errorln(shared.SafeClean(err.Error())) apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError) return } if ASNres != nil { ASN = ASNres.ASN } else { lg.Warningf("no ASN lookup result for IP: %s ", shared.SafeClean(IP.String())) } // resolve ip to country code. countryCode := dbclients.Maxmind.IP2CountryCode(IP) req.ClientAddr = net.IPv4zero IP = net.IPv4zero // HANDLE USERIP END hosts, err := dbclients.DB.GetBlockedHosts(countryCode, ASN) if err != nil { apiError(w, err.Error(), http.StatusInternalServerError) return } simpleData := struct { ClientVersion string `json:"version"` // client version idientifer }{ req.ClientVersion, } data, err := json.Marshal(simpleData) if err != nil { lg.Errorln(err) } ss := db.SimpleSample{ CountryCode: countryCode, ASN: ASN, Type: "ClientBlocklistUpdate", OriginID: req.UpdateID, Data: data, } err = dbclients.DB.InsertSimpleSample(ss) if err != nil { lg.Errorf("error persisting simplesample %v", ss) } err = w.WriteJson(shared.UpdateHostlistResponse{ Hosts: relh.fill(hosts), }) if err != nil { lg.Error(err) return } } }