// SuggestionToken JSON API method.
func SuggestionToken(dbclients db.Clients) func(w rest.ResponseWriter, r *rest.Request) {
return func(w rest.ResponseWriter, r *rest.Request) {
// HANDLE USERIP BEGIN
req := shared.SuggestionTokenRequest{}
err := r.DecodeJsonPayload(&req)
if err != nil {
apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError)
return
}
// validate country code.
if !validCountryCode(req.CountryCode) {
apiError(w,
fmt.Sprintf("invalid country code: %s", req.CountryCode),
http.StatusBadRequest)
return
}
// parse/validate client ip address.
IP := req.ClientAddr
if IP == nil {
apiError(w, "bad ClientAddr", http.StatusBadRequest)
return
}
// parse and validate url.
URL := strings.TrimSpace(req.URL)
if URL == "" {
apiError(w, "no or empty URL", http.StatusBadRequest)
return
}
u, err := url.Parse(URL)
if err != nil {
apiError(w, fmt.Sprintf("%s is not a valid URL", URL), http.StatusBadRequest)
return
}
if !shared.AcceptedURL(u) {
apiError(w, fmt.Sprintf("%s is not a valid URL", URL), http.StatusBadRequest)
return
}
// resolve ip to asn.
var ASN int
ASNres, err := dbclients.Internet.IP2ASN(IP)
if err != nil {
lg.Errorln(shared.SafeClean(err.Error()))
apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError)
return
}
if ASNres != nil {
ASN = ASNres.ASN
} else {
lg.Warningf("no ASN lookup result for IP: %s ", shared.SafeClean(IP.String()))
}
// reoslve ip to country code.
countryCode := dbclients.Maxmind.IP2CountryCode(IP)
// resolve ip to city geonameid
geoCityID := dbclients.Maxmind.IP2CityGeoNameID(IP)
req.ClientAddr = net.IPv4zero
IP = net.IPv4zero
// HANDLE USERIP END
{
supported, err := dbclients.DB.IsURLAllowed(u, countryCode)
if err != nil {
// TODO: standardize http status codes
apiError(w, err.Error(), http.StatusForbidden)
return
}
if !supported {
lg.Infof("got request for unsupported URL %s")
w.WriteJson(shared.SuggestionTokenResponse{
Ok: false,
URL: req.URL,
})
return
}
}
// start new submission token session
token := db.SessionTokens.New(URL)
// create newclienttoken sample data
sample := shared.NewClientTokenSample{
URL: URL,
CountryCode: req.CountryCode,
}
sampleData, err := json.Marshal(sample)
if err != nil {
lg.Errorln(err)
apiError(w, "error #20150424-002542-CEST", http.StatusInternalServerError)
return
}
// create extraData
extra := shared.IPExtraData{
CityGeoNameID: geoCityID,
}
extraData, err := json.Marshal(extra)
if err != nil {
lg.Errorln(err)
apiError(w, "error #20150427-211052-CEST", http.StatusInternalServerError)
return
}
// insert into db
{
err := dbclients.DB.InsertSample(db.Sample{
Host: u.Host,
CountryCode: countryCode,
ASN: ASN,
Type: "NewClientToken",
Origin: "Central",
Token: token,
Data: sampleData,
ExtraData: extraData,
})
if err != nil {
apiError(w, err.Error(), http.StatusInternalServerError)
return
}
}
// queue central measurements
measurements, err := measure.DefaultMeasurements(req.URL)
if err != nil {
lg.Warningf("could not create standard measurements: %s", err.Error())
} else {
queueMeasurements(token, measurements...)
}
// write json response
{
err := w.WriteJson(shared.SuggestionTokenResponse{
Ok: true,
URL: URL,
Token: token,
})
if err != nil {
lg.Errorln(err.Error())
return
}
}
}
}
// StoreSample JSON API method.
func StoreSample(dbclients db.Clients) func(w rest.ResponseWriter, r *rest.Request) {
return func(w rest.ResponseWriter, r *rest.Request) {
// HANDLE USERIP BEGIN
req := shared.StoreSampleRequest{}
err := r.DecodeJsonPayload(&req)
if err != nil {
apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError)
return
}
// validate sample type.
if _, ok := clientSampleTypes[req.SampleType]; !ok {
apiError(w, "invalid sample type: "+req.SampleType, http.StatusBadRequest)
return
}
// get/validate suggestion session token.
tokenData, validToken := db.SessionTokens.Get(req.Token)
if !validToken {
apiError(w, "invalid token", http.StatusBadRequest)
return
}
// parse/validate url.
URL := strings.TrimSpace(req.URL)
if tokenData.URL != URL {
lg.V(2).Infof("invalid URL %s for token session, was expecting %s", req.URL, tokenData.URL)
apiError(w, "invalid URL for token session", http.StatusBadRequest)
return
}
u, err := url.Parse(URL)
if err != nil {
apiError(w, fmt.Sprintf("%s is not a valid URL", URL), http.StatusBadRequest)
return
}
// parse/validate client ip address.
IP := net.ParseIP(req.ClientAddr)
if IP == nil {
apiError(w, "bad ClientAddr", http.StatusBadRequest)
return
}
// resolve ip to asn.
var ASN int
ASNres, err := dbclients.Internet.IP2ASN(IP)
if err != nil {
lg.Errorln(shared.SafeClean(err.Error()))
apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError)
return
}
if ASNres != nil {
ASN = ASNres.ASN
} else {
lg.Warningf("no ASN lookup result for IP: %s ", shared.SafeClean(IP.String()))
}
countryCode := dbclients.Maxmind.IP2CountryCode(IP)
req.ClientAddr = ""
IP = net.IPv4zero
// HANDLE USERIP END
// insert into db
{
err := dbclients.DB.InsertSample(db.Sample{
Host: u.Host,
CountryCode: countryCode,
ASN: ASN,
Type: req.SampleType,
Origin: "Client",
Token: req.Token,
Data: []byte(req.Data),
})
if err != nil {
lg.Errorln(err.Error())
apiError(w, "error 20150424-011948-CEST", http.StatusInternalServerError)
return
}
}
w.WriteJson(shared.StoreSampleResponse{
Ok: true,
})
return
}
}
// Update list of blocked hosts for an IP address.
func GetHosts(dbclients db.Clients) func(w rest.ResponseWriter, r *rest.Request) {
relh := relatedHosts{
dbclients: dbclients,
}
relh.update()
go func() {
for range time.NewTicker(10 * time.Minute).C {
relh.update()
}
}()
return func(w rest.ResponseWriter, r *rest.Request) {
// HANDLE USERIP BEGIN
req := shared.UpdateHostlistRequest{}
err := r.DecodeJsonPayload(&req)
if err != nil {
apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError)
return
}
// parse/validate client ip address.
IP := req.ClientAddr
if IP == nil {
apiError(w, "bad ClientAddr", http.StatusBadRequest)
return
}
// resolve ip to asn.
var ASN int
ASNres, err := dbclients.Internet.IP2ASN(IP)
if err != nil {
lg.Errorln(shared.SafeClean(err.Error()))
apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError)
return
}
if ASNres != nil {
ASN = ASNres.ASN
} else {
lg.Warningf("no ASN lookup result for IP: %s ", shared.SafeClean(IP.String()))
}
// resolve ip to country code.
countryCode := dbclients.Maxmind.IP2CountryCode(IP)
req.ClientAddr = net.IPv4zero
IP = net.IPv4zero
// HANDLE USERIP END
hosts, err := dbclients.DB.GetBlockedHosts(countryCode, ASN)
if err != nil {
apiError(w, err.Error(), http.StatusInternalServerError)
return
}
simpleData := struct {
ClientVersion string `json:"version"` // client version idientifer
}{
req.ClientVersion,
}
data, err := json.Marshal(simpleData)
if err != nil {
lg.Errorln(err)
}
ss := db.SimpleSample{
CountryCode: countryCode,
ASN: ASN,
Type: "ClientBlocklistUpdate",
OriginID: req.UpdateID,
Data: data,
}
err = dbclients.DB.InsertSimpleSample(ss)
if err != nil {
lg.Errorf("error persisting simplesample %v", ss)
}
err = w.WriteJson(shared.UpdateHostlistResponse{
Hosts: relh.fill(hosts),
})
if err != nil {
lg.Error(err)
return
}
}
}
