func DefaultMeasurements(URL string) ([]Measurer, error) {
URL = strings.TrimSpace(URL)
var result []Measurer
if URL == "" {
return nil, errors.New("emtpy url")
}
u, err := url.Parse(URL)
if err != nil {
return nil, err
}
if !shared.AcceptedURL(u) {
return nil, ErrorHostNotAllowed{URL}
}
host := u.Host
if strings.Contains(host, ":") {
host, _, err = net.SplitHostPort(u.Host)
}
for _, resolver := range []string{"", "8.8.8.8:53"} {
dnsm := DNSQuery{
Hostname: host,
Resolver: resolver,
}
result = append(result, dnsm)
}
httphm := HTTPHeader{
URL: URL,
}
result = append(result, httphm)
return result, nil
}
// CreateSuggestion .
func CreateSuggestion(w rest.ResponseWriter, r *rest.Request) {
form := shared.BrowserSuggestionTokenRequest{}
err := r.DecodeJsonPayload(&form)
if err != nil {
// apiError(w, err.Error(), http.StatusInternalServerError)
apiutils.WriteRestError(w, err)
return
}
var invalids fielderrors.ValidationErrorList
// parse and validate url.
URL := strings.TrimSpace(form.URL)
if URL == "" {
invalids = append(invalids, fielderrors.NewFieldRequired("URL"))
// apiError(w, "no or empty URL", http.StatusBadRequest)
}
u, err := url.Parse(URL)
if err != nil {
invalids = append(invalids, fielderrors.NewFieldInvalid("URL", URL, err.Error()))
// apiError(w, fmt.Sprintf("%s is not a valid URL", URL), http.StatusBadRequest)
}
host := u.Host
if strings.Contains(host, ":") {
host, _, err = net.SplitHostPort(u.Host)
if err != nil {
invalids = append(invalids, fielderrors.NewFieldInvalid("URL", URL, err.Error()))
}
}
if !shared.AcceptedURL(u) {
invalids = append(invalids, fielderrors.NewFieldValueNotSupported("URL", URL, nil))
}
if len(invalids) > 0 {
apiutils.WriteRestError(w, apierrors.NewInvalid("create-suggestion", "URL", invalids))
return
}
s := client.NewSuggestion(u.String())
defer s.DoneAddingSamples()
measurers, err := measure.DefaultMeasurements(form.URL)
if err != nil {
apiutils.WriteRestError(w, apierrors.NewInternalError(err))
return
}
for _, v := range measurers {
m, err := v.Measure()
if err != nil {
lg.Errorf("could not measure: %s", err.Error())
} else {
switch m.Type() {
case sampletypes.DNSQuery, sampletypes.HTTPHeader:
err = s.AddMeasurement(m)
if err != nil {
lg.Errorln(err.Error())
return
}
default:
lg.Warningf("unsupported sample type: %s", m.Type().String())
}
}
}
}
// SuggestionToken JSON API method.
func SuggestionToken(dbclients db.Clients) func(w rest.ResponseWriter, r *rest.Request) {
return func(w rest.ResponseWriter, r *rest.Request) {
// HANDLE USERIP BEGIN
req := shared.SuggestionTokenRequest{}
err := r.DecodeJsonPayload(&req)
if err != nil {
apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError)
return
}
// validate country code.
if !validCountryCode(req.CountryCode) {
apiError(w,
fmt.Sprintf("invalid country code: %s", req.CountryCode),
http.StatusBadRequest)
return
}
// parse/validate client ip address.
IP := req.ClientAddr
if IP == nil {
apiError(w, "bad ClientAddr", http.StatusBadRequest)
return
}
// parse and validate url.
URL := strings.TrimSpace(req.URL)
if URL == "" {
apiError(w, "no or empty URL", http.StatusBadRequest)
return
}
u, err := url.Parse(URL)
if err != nil {
apiError(w, fmt.Sprintf("%s is not a valid URL", URL), http.StatusBadRequest)
return
}
if !shared.AcceptedURL(u) {
apiError(w, fmt.Sprintf("%s is not a valid URL", URL), http.StatusBadRequest)
return
}
// resolve ip to asn.
var ASN int
ASNres, err := dbclients.Internet.IP2ASN(IP)
if err != nil {
lg.Errorln(shared.SafeClean(err.Error()))
apiError(w, shared.SafeClean(err.Error()), http.StatusInternalServerError)
return
}
if ASNres != nil {
ASN = ASNres.ASN
} else {
lg.Warningf("no ASN lookup result for IP: %s ", shared.SafeClean(IP.String()))
}
// reoslve ip to country code.
countryCode := dbclients.Maxmind.IP2CountryCode(IP)
// resolve ip to city geonameid
geoCityID := dbclients.Maxmind.IP2CityGeoNameID(IP)
req.ClientAddr = net.IPv4zero
IP = net.IPv4zero
// HANDLE USERIP END
{
supported, err := dbclients.DB.IsURLAllowed(u, countryCode)
if err != nil {
// TODO: standardize http status codes
apiError(w, err.Error(), http.StatusForbidden)
return
}
if !supported {
lg.Infof("got request for unsupported URL %s")
w.WriteJson(shared.SuggestionTokenResponse{
Ok: false,
URL: req.URL,
})
return
}
}
// start new submission token session
token := db.SessionTokens.New(URL)
// create newclienttoken sample data
sample := shared.NewClientTokenSample{
URL: URL,
CountryCode: req.CountryCode,
}
sampleData, err := json.Marshal(sample)
if err != nil {
lg.Errorln(err)
apiError(w, "error #20150424-002542-CEST", http.StatusInternalServerError)
return
}
// create extraData
extra := shared.IPExtraData{
CityGeoNameID: geoCityID,
}
extraData, err := json.Marshal(extra)
if err != nil {
lg.Errorln(err)
apiError(w, "error #20150427-211052-CEST", http.StatusInternalServerError)
return
}
// insert into db
{
err := dbclients.DB.InsertSample(db.Sample{
Host: u.Host,
CountryCode: countryCode,
ASN: ASN,
Type: "NewClientToken",
Origin: "Central",
Token: token,
Data: sampleData,
ExtraData: extraData,
})
if err != nil {
apiError(w, err.Error(), http.StatusInternalServerError)
return
}
}
// queue central measurements
measurements, err := measure.DefaultMeasurements(req.URL)
if err != nil {
lg.Warningf("could not create standard measurements: %s", err.Error())
} else {
queueMeasurements(token, measurements...)
}
// write json response
{
err := w.WriteJson(shared.SuggestionTokenResponse{
Ok: true,
URL: URL,
Token: token,
})
if err != nil {
lg.Errorln(err.Error())
return
}
}
}
}