// InsertUpgrades inserts a list of upgrade entreis into the database
func (d *DB) InsertUpgrades(u []UpgradeMeta) error {
psql := squirrel.StatementBuilder.PlaceholderFormat(squirrel.Dollar)
tx, err := d.cache.Begin()
if err != nil {
return err
}
for _, v := range u {
i := psql.Insert("upgrades").
Columns("artifact", "version", "sha256sum", "ed25519sig").
Values(v.Artifact, v.Version, v.SHA256Sum, v.ED25519Signature)
_, err := i.RunWith(d.cache).Exec()
if err != nil {
logSQLErr(err, &i)
if err := tx.Rollback(); err != nil {
lg.Errorln(err)
}
return err
}
}
err = tx.Commit()
if err != nil {
lg.Errorln(err)
}
return nil
}
// Encrypt encrypts the response using the provided hex encoded public key
func (d *DebugResponse) Encrypt(pubKey string) error {
if pubKey == "" {
return fmt.Errorf("pubKey must be supplied")
}
pk := new([32]byte)
dpk, err := hex.DecodeString(pubKey)
if err != nil {
lg.Fatalln("Could not decode debug public key")
}
copy(pk[:], dpk[:32])
data, err := json.Marshal(&d)
if err != nil {
lg.Errorln("could not marshal debug response", err)
return nil
}
encrypted, err := sodiumbox.Seal(data, pk)
if err != nil {
lg.Errorln("could not encrypt debug response", err)
return nil
}
*d = DebugResponse{
Header: d.Header,
Encrypted: hex.EncodeToString(encrypted.Box),
}
return nil
}
func upgradeBinaryCheck(diffsBaseURL string) error {
artifactNameMu.Lock()
artifact := artifactName
artifactNameMu.Unlock()
cl, err := NewRestClient()
if err != nil {
return err
}
res, found, err := cl.CheckBinaryUpgrade(shared.BinaryUpgradeRequest{
Artifact: artifact,
FromVersion: VERSION,
})
if err != nil {
return err
}
if !found {
lg.Infoln("no update found")
return nil
}
lg.Warningf("found update %+v", res)
httpclient, err := service.NewTransportHTTPClient(2 * time.Hour)
if err != nil {
return err
}
opts, err := upgradebin.NewUpdaterOptions(res, shared.UpgradeVerificationPublicKey)
if err != nil {
return err
}
u, err := url.Parse(diffsBaseURL)
if err != nil {
lg.Errorln(err)
return err
}
u.Path = path.Join(u.Path, artifact, VERSION, res.Version)
URL := u.String()
lg.Infoln("downloading", URL)
resp, err := httpclient.Get(URL)
if err != nil {
lg.Errorln(err)
return err
}
defer resp.Body.Close()
err = upgradebin.Apply(resp.Body, opts)
if err != nil {
lg.Errorln(err)
// will be retried the next time the client starts
return nil
}
return nil
}
func upgradeBinaryCheck() error {
if !upgradeEnabled {
lg.Infoln("binary upgrades are disabled using the command line flag")
return nil
}
artifactNameMu.Lock()
artifact := artifactName
artifactNameMu.Unlock()
cl, err := NewRestClient()
if err != nil {
return err
}
// TODO: check for current artifact + version (need to add artifact id to cmd's)
res, found, err := cl.CheckBinaryUpgrade(shared.BinaryUpgradeRequest{
Artifact: artifact,
FromVersion: VERSION,
})
if err != nil {
return err
}
if !found {
lg.Infoln("no update found")
return nil
}
lg.Warningf("found update %+v", res)
httpclient, err := service.NewTransportHTTPClient()
if err != nil {
return err
}
opts, err := upgradebin.NewUpdaterOptions(res, shared.UpgradeVerificationPublicKey)
if err != nil {
return err
}
URL := fmt.Sprintf("https://central.server.domain/u/%s/%s/%s", artifact, VERSION, res.Version)
lg.Infoln("downloading %s", URL)
resp, err := httpclient.Get(URL)
if err != nil {
lg.Errorln(err)
return err
}
defer resp.Body.Close()
err = upgradebin.Apply(resp.Body, opts)
if err != nil {
lg.Errorln(err)
// will be retried the next time the client starts
return nil
}
return nil
}
func PostUserSettings(w rest.ResponseWriter, r *rest.Request) {
form := UserSettings{}
err := r.DecodeJsonPayload(&form)
if err != nil {
apiutils.WriteRestError(w, apierrors.NewBadRequest(err.Error()))
return
}
changed := false // true if config really was updated
err = clientconfig.Update(func(conf *clientconfig.Config) error {
s := &conf.Settings.Local
prevLang := s.Language
if ValidLanguage(form.Language) {
s.Language = form.Language
} else {
s.Language = "en"
}
if prevLang != s.Language {
ui.Language(s.Language)
changed = true
}
if s.CountryCode != form.CountryCode {
s.CountryCode = form.CountryCode
changed = true
}
if s.ClientAutoUpdate != form.ClientAutoUpdate {
s.ClientAutoUpdate = form.ClientAutoUpdate
changed = true
}
if s.BlocklistAutoUpdate != form.BlocklistAutoUpdate {
s.BlocklistAutoUpdate = form.BlocklistAutoUpdate
changed = true
}
return nil
})
if changed {
err := clientconfig.Write()
if err != nil {
lg.Errorln(err)
}
}
if err != nil {
lg.Errorln(err)
}
}
// StartBinaryUpgradeChecker checks for binary upgrades when the connection is
// up and on a schedule.
//
// This function runs in it's own goroutine.
func StartBinaryUpgradeChecker() {
connectionEventListener := make(chan service.ConnectionHistory)
uChecker, _ := NewUpdateChecker("binary")
service.AddListener(connectionEventListener)
for {
select {
// Update when the transport connection comes up
case event := <-connectionEventListener:
if event.IsUp() {
uChecker.Activate()
uChecker.UpdateNow()
}
// Update by request of the update checker
case request := <-uChecker.RequestC:
err := upgradeBinaryCheck()
// err := fakeUpgradeBinaryCheck()
if err != nil {
lg.Errorln(err)
request.ResponseC <- UpdateError
} else {
request.ResponseC <- UpdateSuccess
}
}
}
}
func (p *PreparedSample) Update(dbclients db.Clients) error {
IP := shared.GetPublicIPAddr()
if IP == nil {
return errors.New("could not get own public ip address")
}
// resolve ip to asn.
var ASN int
ASNres, err := dbclients.Internet.IP2ASN(IP)
if err != nil {
lg.Errorln(err.Error())
return err
}
if ASNres != nil {
ASN = ASNres.ASN
} else {
lg.Warningf("no ASN lookup result for IP: %s ", IP)
return fmt.Errorf("no ASN lookup result for IP: %s ", IP)
}
// resolve ip to country code.
countryCode := dbclients.Maxmind.IP2CountryCode(IP)
s := db.Sample{
CountryCode: countryCode,
ASN: ASN,
}
p.lastUpdated = time.Now()
p.s = s
return nil
}
func startMeasurer(dbclients db.Clients) {
for n := 0; n < 10; n++ {
go func() {
var ps PreparedSample
err := ps.Update(dbclients)
if err != nil {
lg.Error("could not resolve public ip address", err)
}
lg.V(5).Infoln("starting measurer")
for r := range requestMeasurements {
lg.V(50).Infoln("got measurement", r)
if ps.lastUpdated.Before(time.Now().Add(-time.Hour * 5)) {
lg.V(15).Info("updating prepared sample", ps)
err := ps.Update(dbclients)
if err != nil {
lg.Warning(err)
}
}
measurerLoop:
for _, v := range r.measurers {
measurement, err := v.Measure()
if err != nil {
lg.Errorf("could not measure:%v error:%s", v, err.Error())
continue measurerLoop
}
switch measurement.Type() {
case sampletypes.DNSQuery, sampletypes.HTTPHeader:
data, err := measurement.Marshal()
if err != nil {
lg.Errorf("could not decode %v error:%s", measurement, err.Error())
continue measurerLoop
}
err = dbclients.DB.InsertSample(db.Sample{
Host: measurement.Host(),
CountryCode: ps.s.CountryCode,
Token: r.token,
ASN: ps.s.ASN,
Type: measurement.Type().String(),
Origin: sampleorigins.Central.String(),
Data: data,
})
if err != nil {
lg.Errorln(err.Error())
continue measurerLoop
}
default:
lg.Errorf("could not measure:%v error:%s", v, err.Error())
continue measurerLoop
}
}
}
}()
}
}
// UpgradeConfig updates the config, if needed.
func UpgradeConfig() (bool, error) {
currentConrfigMu.Lock()
defer currentConrfigMu.Unlock()
if !currentConfig.configRead {
return false, errors.New("config not read")
}
prevVer := currentConfig.Settings.Version
switch currentConfig.Settings.Version {
case 0: // when config file was created from template
currentConfig.Settings.Version = 1
fallthrough
case 1:
lg.Infoln("updating configuration to v1")
m := &modifyConnections{
Connections: currentConfig.Settings.Connections,
Add: []string{
"aieyJ0Ijoib2JmczQiLCJzIjoiXCJjZXJ0PUdzVFAxVmNwcjBJeE9aUkNnUHZ0Z1JsZVJWQzFTRmpYWVhxSDVvSEhYVFJ4M25QVnZXN2xHK2RkREhKWmw4YjBOVFZ1VGc7aWF0LW1vZGU9MFwiIiwiYSI6IjEzOS4xNjIuMjIxLjEyMzo0NDMifQ==",
"aieyJ0Ijoib2JmczQiLCJzIjoiXCJjZXJ0PTMzdXNkSUVDemFyRUpsWFpQM0w0Y2x0bi9vNXdhVUlOcHRUU0JSYk5BQVpVcVlsajhiZm5MWGYyb3BFNHE2c0NlYzY3Ync7aWF0LW1vZGU9MFwiIiwiYSI6IjQ2LjIxLjEwMi4xMDk6NDQzIn0=",
},
Remove: []string{
// NOTE: old hash function used, not a current format ID
"xFa9T1i6bJMJIvK6kxFA1xvQGfW58BY3OLkrPXbpvAY=",
},
}
currentConfig.Settings.Connections = m.Update()
currentConfig.Settings.Version = 2
fallthrough
case 2:
m := &modifyConnections{
Connections: currentConfig.Settings.Connections,
Protect: []string{
"z0VZS-Kx9tMfoBEyX6br19GaJgKDe0IK0i6JKyhKp2s",
"ipJ2oW8xr9TFDvfU92qGvDaPwZttf_GSjGZ4KW7inBI",
},
Remove: []string{
"Ul3D2G1dI3_Z4sLXQ9IUQdIFH4pSDyNjTwf_auy93Os",
},
}
currentConfig.Settings.Connections = m.Update()
currentConfig.Settings.Version = 3
fallthrough
case 3:
key := browsercode.NewKey()
currentConfig.Settings.Local.ClientAuthKey = key
currentConfig.Settings.Local.ClientBindAddr = "127.0.0.1:8899"
currentConfig.Settings.Version = 4
fallthrough
case 4:
lg.Infoln("Settings version", currentConfig.Settings.Version)
default:
lg.Errorln("Future configuration version!", currentConfig.Settings.Version)
}
return currentConfig.Settings.Version != prevVer, nil
}
func saveChromeExtension() error {
exportPath := filepath.Join(osutil.HomePath(), "AlkasirChromeExtension")
if err := os.RemoveAll(exportPath); err != nil {
lg.Errorln(err)
}
err := chrome.RestoreAssets(exportPath, "")
if err != nil {
return err
}
return nil
}
func testPatch(pr CreatePatchResult, publicKey string) error {
lg.Infof("verifying %s %s>%s", pr.Artifact, pr.OldVersion, pr.NewVersion)
tmpfile := fmt.Sprintf("/tmp/%s-%s-o", pr.Artifact, pr.OldVersion)
err := cp(tmpfile, pr.job.OldBinary)
if err != nil {
lg.Fatal(err)
}
defer func() {
err = os.Remove(tmpfile)
if err != nil {
lg.Errorln(err)
}
}()
sum, err := base64.RawURLEncoding.DecodeString(pr.SHA256Sum)
if err != nil {
return err
}
sig, err := upgradebin.DecodeSignature(pr.ED25519Signature)
if err != nil {
return err
}
pub, err := upgradebin.DecodePublicKey([]byte(publicKey))
if err != nil {
return err
}
opts := update.Options{
Patcher: update.NewBSDiffPatcher(),
Verifier: upgradebin.NewED25519Verifier(),
Hash: crypto.SHA256,
Checksum: sum,
Signature: sig[:],
PublicKey: pub,
TargetPath: tmpfile,
}
diffFile, err := os.Open(pr.DiffFile)
if err != nil {
return err
}
defer diffFile.Close()
err = update.Apply(diffFile, opts)
if err != nil {
return err
}
return nil
}
func listVersions([]string) error {
for _, v := range jobQs {
versions, err := v.GetVersions()
if err != nil {
lg.Errorln(err)
continue
}
for _, v := range versions {
spew.Dump(v)
}
}
return nil
}
func StartAnalysis(clients db.Clients) {
tick := time.NewTicker(5 * time.Second)
lastID, err := clients.DB.GetLastProcessedSampleID()
if err != nil {
lg.Warningln(err)
}
for n := 0; n < 4; n++ {
go sessionFetcher(clients)
go samplesAnalyzer()
go hostPublisher(clients)
}
lg.Infof("starting analysis from sample ID %d", lastID)
lastPersistedID := lastID
for range tick.C {
results, err := clients.DB.GetSamples(uint64(lastID), "")
if err != nil {
lg.Errorf("database err (skipping): %v", err)
continue
}
n := 0
start := time.Now()
for s := range results {
n++
if s.ID > lastID {
lastID = s.ID
}
if s.Origin == "Central" && s.Type == "HTTPHeader" {
sessionFetchC <- s.Token
}
}
if n != 0 && lg.V(15) {
lg.Infof("processed %d samples in %s", n, time.Since(start).String())
}
if lastID != lastPersistedID {
err = clients.DB.SetLastProcessedSampleID(lastID)
if err != nil {
lg.Errorln(err)
} else {
lastPersistedID = lastID
}
}
}
}
func startAnalysis(clients db.Clients) {
go func() {
tick := time.NewTicker(10 * time.Second)
lastID, err := clients.DB.GetLastProcessedSampleID()
if err != nil {
lg.Warningln(err)
}
lg.Infof("starting analysis from sample ID %d", lastID)
lastPersistedID := lastID
for range tick.C {
results, err := clients.DB.GetSamples(uint64(lastID), "")
if err != nil {
lg.Fatal(err)
}
n := 0
start := time.Now()
loop:
for s := range results {
n++
if s.ID > lastID {
lastID = s.ID
}
if s.Type == "NewClientToken" {
if !shared.AcceptedHost(s.Host) {
lg.Warningln("not accepted host id:", s.ID, s.Host)
continue loop
}
err := clients.DB.PublishHost(s)
if err != nil {
lg.Warning(err)
}
}
}
if n != 0 && lg.V(15) {
lg.Infof("processed %d samples in %s", n, time.Since(start).String())
}
if lastID != lastPersistedID {
err = clients.DB.SetLastProcessedSampleID(lastID)
if err != nil {
lg.Errorln(err)
} else {
lastPersistedID = lastID
}
}
}
}()
}
// NewDebugResponse creates a filled DebugResponse struct
func NewDebugResposne(version string, config interface{}) *DebugResponse {
ID, err := shared.SecureRandomString(12)
if err != nil {
panic("could not generate random number")
}
response := &DebugResponse{
Header: DebugHeader{
Cmd: filepath.Base(os.Args[0]),
ID: ID,
Version: version,
CreatedAt: time.Now(),
OS: runtime.GOOS,
Arch: runtime.GOARCH,
GoVersion: runtime.Version(),
},
Config: config,
}
getProfile := func(name string) []string {
buf := bytes.NewBuffer(nil)
err := pprof.Lookup(name).WriteTo(buf, 2)
if err != nil {
lg.Errorln(err)
} else {
return strings.Split(
buf.String(),
"\n")
}
return []string{}
}
response.Heap = getProfile("heap")
response.GoRoutines = getProfile("goroutine")
response.ThreadCreate = getProfile("threadcreate")
response.Block = getProfile("block")
// memlog should be last so that it can catch errors up to the point of
// collection.
response.Log = lg.Memlog()
if PublicKey != "" {
response.Encrypt(PublicKey)
} else {
lg.Warningln("PublicKey not set, exporting unencrypted debug log")
}
return response
}
func sessionFetcher(clients db.Clients) {
loop:
for token := range sessionFetchC {
if token == "" {
lg.Errorln("empty token received, skipping")
continue loop
}
samples, err := clients.DB.GetSessionSamples(token)
if err != nil {
lg.Error(err)
continue loop
}
sampleAnalysisC <- samples
}
}
func DeleteConnection(w rest.ResponseWriter, r *rest.Request) {
ID := r.PathParam("id")
err := clientconfig.Update(func(conf *clientconfig.Config) error {
all := conf.Settings.Connections
var result []shared.Connection
// validate that the id exists
found := false
nEnabled := 0
for _, v := range all {
if v.ID == ID {
found = true
if v.Protected {
apiutils.WriteRestError(w, apierrors.NewForbidden("connection", ID, nil))
return nil
}
} else {
if !v.Disabled {
nEnabled += 1
}
result = append(result, v)
}
}
if !found {
apiutils.WriteRestError(w, apierrors.NewNotFound("connection", ID))
return nil
}
if nEnabled < 1 {
apiutils.WriteRestError(w,
apierrors.NewForbidden("connection", ID, errors.New("one connection must be enabled")))
return nil
}
service.UpdateConnections(result)
conf.Settings.Connections = result
return nil
})
if err != nil {
lg.Errorln(err)
}
clientconfig.Write()
w.WriteJson(true)
}
func ToggleConnection(w rest.ResponseWriter, r *rest.Request) {
ID := r.PathParam("id")
err := clientconfig.Update(func(conf *clientconfig.Config) error {
// validate that the id exists, if supplied
all := conf.Settings.Connections
foundIdx := 0
found := false
nEnabled := 0
for k, v := range all {
if !v.Disabled {
nEnabled += 1
}
if v.ID == ID {
found = true
foundIdx = k
}
}
if !found {
apiutils.WriteRestError(w, apierrors.NewNotFound("connection", ID))
return nil
}
if nEnabled < 2 && !conf.Settings.Connections[foundIdx].Disabled {
apiutils.WriteRestError(w,
apierrors.NewForbidden("connection", ID, errors.New("one connection must be enabled")))
return nil
}
conf.Settings.Connections[foundIdx].Disabled = !conf.Settings.Connections[foundIdx].Disabled
service.UpdateConnections(conf.Settings.Connections)
return nil
})
if err != nil {
lg.Errorln(err)
}
clientconfig.Write()
w.WriteJson(true)
}
func Open() error {
ctx := usb.NewContext()
// ctx.Debug(10)
devs, err := ctx.ListDevices(func(desc *usb.Descriptor) bool {
if desc.Vendor.String() == "0590" {
return true
}
return false
})
if err != nil {
for _, v := range devs {
err := v.Close()
if err != nil {
lg.Errorln(err)
}
}
return err
}
usbdevice = devs[0]
return nil
}
// StartBinaryUpgradeChecker checks for binary upgrades when the connection is
// up and on a schedule.
//
// This function runs in it's own goroutine.
func StartBinaryUpgradeChecker(diffsBaseURL string) {
if !upgradeEnabled {
lg.Infoln("binary upgrades are disabled using the command line flag")
return
}
if VERSION == "" {
lg.Warningln("VERSION not set, binary upgrades are disabled")
return
}
_, err := version.NewVersion(VERSION)
if err != nil {
lg.Warningf("VERSION '%s' is not a valid semver version, binary upgrades are disabled: %v", VERSION, err)
return
}
connectionEventListener := make(chan service.ConnectionHistory)
uChecker, _ := NewUpdateChecker("binary")
service.AddListener(connectionEventListener)
for {
select {
// Update when the transport connection comes up
case event := <-connectionEventListener:
if event.IsUp() {
uChecker.Activate()
uChecker.UpdateNow()
}
// Update by request of the update checker
case request := <-uChecker.RequestC:
err := upgradeBinaryCheck(diffsBaseURL)
if err != nil {
lg.Errorln(err)
request.ResponseC <- UpdateError
} else {
request.ResponseC <- UpdateSuccess
}
}
}
}
func importFromDevice(db *db.DB) error {
if err := omronread.Open(); err != nil {
return err
}
defer func() {
if err := omronread.Close(); err != nil {
lg.Errorln(err)
}
}()
var all []omron.Entry
{
entries, err := omronread.Read(0)
if err != nil {
return err
}
all = append(all, entries...)
}
{
entries, err := omronread.Read(1)
if err != nil {
return err
}
all = append(all, entries...)
}
sort.Sort(omron.ByTime(all))
// data, err := json.MarshalIndent(all, "", " ")
// if err != nil {
// lg.Fatal(err)
// }
return db.SaveEntries(all)
}
func main() {
runtime.LockOSThread()
flag.Set("logtostderr", "true")
lg.CopyStandardLogTo("INFO")
lg.SetSrcHighlight("libomron")
flag.Parse()
bdb, err := bolt.Open("bpchart.db", 0600, &bolt.Options{Timeout: 1 * time.Second})
if err != nil {
lg.Fatal(err)
}
db := &db.DB{DB: bdb, BucketName: []byte(entriesBucketName)}
var wg sync.WaitGroup
wg.Add(1)
go func() {
lg.Fatal(httpServer(db))
wg.Done()
}()
wg.Add(1)
go func() {
if FAKE {
err = fakeImportFromDevice(db)
} else {
err = importFromDevice(db)
}
if err != nil {
lg.Errorln(err)
}
wg.Done()
}()
wg.Wait()
}
func exit() {
lg.Infoln("alkasir is shutting down")
atexitMu.Lock() // this lock should be kept, one shutdown should be enough for everyone.
lg.Flush()
if err := clientconfig.Write(); err != nil {
lg.Errorf("could not save config file: %s", err.Error())
}
lg.V(9).Infoln("running atexit funcs")
for _, f := range atexitFuncs {
funcName := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
lg.V(10).Infoln("Running at exit func", funcName)
f()
lg.V(10).Infoln("Finished at exit func", funcName)
}
atexitFuncs = atexitFuncs[:0]
lg.V(9).Infoln("atexit funcs done")
{
lg.V(9).Infoln("stopping connectionmanager")
done := make(chan bool)
go func() {
service.StopConnectionManager()
done <- true
}()
select {
case <-done:
case <-time.After(10 * time.Second):
lg.Errorln("services stop timed out")
}
}
{
done := make(chan bool)
go func() {
service.StopAll()
done <- true
}()
select {
case <-done:
lg.V(9).Infoln("services stopped")
case <-time.After(10 * time.Second):
lg.Errorln("services stop timed out")
}
}
{
lg.V(9).Infoln("waiting for UI shutdown to finish")
done := make(chan bool)
go func() {
uiRunning.Wait()
done <- true
}()
select {
case <-done:
case <-time.After(10 * time.Second):
lg.Errorln("timed out watning for ui shutdown")
}
}
lg.Flush()
lg.Infoln("alkasir shutdown complete")
lg.Flush()
time.Sleep(time.Millisecond * 1)
os.Exit(0)
}
// SubmitSuggestion initiates the comminication with Central for a Submission
// session.
func SubmitSuggestion(w rest.ResponseWriter, r *rest.Request) {
// // TODO This is the response that must be forwarded from central/api and parsed by client and passed on to the browser.
// apiutils.WriteRestError(w,
// apierrors.NewInvalid("object", "suggestion",
// fielderrors.ValidationErrorList{
// fielderrors.NewFieldValueNotSupported("URL", "...", []string{})}))
// return
ID := r.PathParam("id")
suggestion, ok := client.GetSuggestion(ID)
if !ok {
apiutils.WriteRestError(w, apierrors.NewNotFound("suggestion", ID))
return
}
wanip := shared.GetPublicIPAddr()
if wanip == nil {
lg.Warning("could not resolve public ip addr")
}
conf := clientconfig.Get()
restclient, err := NewRestClient()
if err != nil {
apiutils.WriteRestError(w, apierrors.NewInternalError(err))
return
}
tokenResp, err := suggestion.RequestToken(
restclient, wanip, conf.Settings.Local.CountryCode)
if err != nil {
if apiutils.IsNetError(err) {
apiutils.WriteRestError(w, apierrors.NewServerTimeout("alkasir-central", "request-submission-token", 0))
} else {
apiutils.WriteRestError(w, apierrors.NewInternalError(err))
}
return
}
n, err := suggestion.SendSamples(restclient)
if err != nil {
lg.Warningln("error sending samples", err.Error())
}
lg.V(5).Infoln("sent ", n)
// FIXME PRESENTATION: just add the url locally
u, err := url.Parse(suggestion.URL)
if err != nil {
lg.Errorln(err)
} else {
err := clientconfig.Update(func(conf *clientconfig.Config) error {
conf.BlockedHosts.Add(u.Host)
lastBlocklistChange = time.Now()
pac.UpdateBlockedList(conf.BlockedHostsCentral.Hosts,
conf.BlockedHosts.Hosts)
return nil
})
if err != nil {
lg.Errorln(err)
}
}
w.WriteJson(tokenResp)
}
func samplesAnalyzer() {
loop:
for samples := range sampleAnalysisC {
var (
newTokenSample db.Sample
clientSamples = make(map[string]db.Sample, 0)
centralSamples = make(map[string]db.Sample, 0)
)
// organize input data
for _, s := range samples {
switch s.Type {
case "NewClientToken":
if newTokenSample.Token != "" {
lg.Error("got more than one newTokenSample, aborting")
continue loop
}
newTokenSample = s
case "HTTPHeader", "DNSQuery":
switch s.Origin {
case "Central":
centralSamples[s.Type] = s
case "Client":
clientSamples[s.Type] = s
}
default:
lg.Errorf("dont know how to handle %d %s, skipping", s.ID, s.Type)
}
}
// validate that wanted data types are available
if newTokenSample.Token == "" {
lg.Errorln("No newTokenSample, aborting")
continue loop
}
if !shared.AcceptedHost(newTokenSample.Host) {
lg.Warningln("not accepted host id:", newTokenSample.ID, newTokenSample.Host)
continue loop
}
for _, smap := range []map[string]db.Sample{clientSamples, centralSamples} {
for _, stype := range []string{"HTTPHeader"} {
if _, ok := smap[stype]; !ok {
lg.Errorf("missing %s, cannot analyse", stype)
continue loop
}
}
}
// parse data
clientSample := clientSamples["HTTPHeader"]
var clientHeader measure.HTTPHeaderResult
centralSample := centralSamples["HTTPHeader"]
var centralHeader measure.HTTPHeaderResult
if err := json.Unmarshal(clientSample.Data, &clientHeader); err != nil {
lg.Errorln(err)
continue loop
}
if err := json.Unmarshal(centralSample.Data, ¢ralHeader); err != nil {
lg.Error(err)
continue loop
}
// score data
score := scoreHTTPHeaders(clientHeader, centralHeader)
lg.V(10).Infof("session:%s %s", newTokenSample.Token, score)
if score.Score() >= 0.5 {
lg.Infoln("publishing session", newTokenSample.Token)
hostPublishC <- newTokenSample
} else {
lg.Infoln("not publishing session", newTokenSample.Token)
}
}
}
func PostConnection(w rest.ResponseWriter, r *rest.Request) {
form := ConnectionSetting{}
err := r.DecodeJsonPayload(&form)
if err != nil {
apiutils.WriteRestError(w, apierrors.NewBadRequest(err.Error()))
return
}
err = clientconfig.Update(func(conf *clientconfig.Config) error {
all := conf.Settings.Connections
var connection shared.Connection // decoded connection
var invalids fielderrors.ValidationErrorList
// Verify that the encoded field is set
{
if form.Encoded == "" {
invalids = append(invalids, fielderrors.NewFieldRequired("encoded"))
}
}
// verify that the connection is decodeable
{
var err error
connection, err = shared.DecodeConnection(form.Encoded)
if err != nil {
invalids = append(invalids, fielderrors.NewFieldInvalid("encoded", form.Encoded, "invalid formatting"))
}
}
// validate that the id exists, if supplied
foundIdx := 0
{
if form.ID != "" {
found := false
for k, v := range all {
if v.ID == form.ID {
found = true
foundIdx = k
}
}
if !found {
invalids = append(invalids, fielderrors.NewFieldNotFound("id", form.ID))
}
}
}
// validate that the connection doesnt alreay exist
{
encoded, err := connection.Encode()
if err != nil {
apiutils.WriteRestError(w, err)
return nil
}
found := false
for _, v := range all {
enc2, err := v.Encode()
if err != nil {
lg.Errorln(err)
continue
}
if enc2 == encoded {
found = true
}
}
if found {
invalids = append(invalids, fielderrors.NewFieldDuplicate("encoded", form.Encoded))
}
}
// end of field validations
if len(invalids) > 0 {
apiutils.WriteRestError(w, apierrors.NewInvalid("post-connection", "", invalids))
return nil
}
// add connection to settings and save
if form.ID == "" {
connection.EnsureID()
conf.Settings.Connections = append(conf.Settings.Connections, connection)
} else {
conf.Settings.Connections[foundIdx] = connection
}
service.UpdateConnections(conf.Settings.Connections)
return nil
})
if err != nil {
lg.Errorln(err)
}
clientconfig.Write()
w.WriteJson(true)
}
func (h HTTPHeader) Measure() (Measurement, error) {
timeout := h.Timeout
if timeout == 0 {
timeout = 45 * time.Second
}
resultchan := make(chan HTTPHeaderResult, 0)
go func() {
rr := &redirectRecorder{
Transport: http.DefaultTransport.(*http.Transport),
}
client := http.Client{
Timeout: timeout,
Transport: rr,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
if len(via) >= 10 {
return errors.New("stopped after 10 redirects")
}
return nil
},
}
resp, err := client.Get(h.URL)
if err != nil {
resultchan <- HTTPHeaderResult{
URL: h.URL,
Error: err.Error(),
}
return
}
defer func() {
err := resp.Body.Close()
if err != nil {
// resp.Body should always have been closed at this point so an
// error is expected.
lg.V(20).Infoln(err)
}
}()
respHeaders := make(map[string]string, 0)
for k := range resp.Header {
respHeaders[k] = resp.Header.Get(k)
}
if err := resp.Body.Close(); err != nil {
lg.Errorln(err)
}
resultchan <- HTTPHeaderResult{
URL: h.URL,
ResponseHeader: respHeaders,
StatusCode: resp.StatusCode,
Redirects: rr.Redirects,
}
}()
select {
case res := <-resultchan:
return res, nil
case <-time.After(timeout):
return HTTPHeaderResult{
URL: h.URL,
Error: "timeout: " + timeout.String(),
}, nil
}
}
// CreateSuggestion .
func CreateSuggestion(w rest.ResponseWriter, r *rest.Request) {
form := shared.BrowserSuggestionTokenRequest{}
err := r.DecodeJsonPayload(&form)
if err != nil {
// apiError(w, err.Error(), http.StatusInternalServerError)
apiutils.WriteRestError(w, err)
return
}
var invalids fielderrors.ValidationErrorList
// parse and validate url.
URL := strings.TrimSpace(form.URL)
if URL == "" {
invalids = append(invalids, fielderrors.NewFieldRequired("URL"))
// apiError(w, "no or empty URL", http.StatusBadRequest)
}
u, err := url.Parse(URL)
if err != nil {
invalids = append(invalids, fielderrors.NewFieldInvalid("URL", URL, err.Error()))
// apiError(w, fmt.Sprintf("%s is not a valid URL", URL), http.StatusBadRequest)
}
host := u.Host
if strings.Contains(host, ":") {
host, _, err = net.SplitHostPort(u.Host)
if err != nil {
invalids = append(invalids, fielderrors.NewFieldInvalid("URL", URL, err.Error()))
}
}
if !shared.AcceptedURL(u) {
invalids = append(invalids, fielderrors.NewFieldValueNotSupported("URL", URL, nil))
}
if len(invalids) > 0 {
apiutils.WriteRestError(w, apierrors.NewInvalid("create-suggestion", "URL", invalids))
return
}
s := client.NewSuggestion(u.String())
defer s.DoneAddingSamples()
measurers, err := measure.DefaultMeasurements(form.URL)
if err != nil {
apiutils.WriteRestError(w, apierrors.NewInternalError(err))
return
}
for _, v := range measurers {
m, err := v.Measure()
if err != nil {
lg.Errorf("could not measure: %s", err.Error())
} else {
switch m.Type() {
case sampletypes.DNSQuery, sampletypes.HTTPHeader:
err = s.AddMeasurement(m)
if err != nil {
lg.Errorln(err.Error())
return
}
default:
lg.Warningf("unsupported sample type: %s", m.Type().String())
}
}
}
}
func insertUpgrades([]string) error {
if err := OpenDB(); err != nil {
return err
}
files, err := findJSONFiles("diffs/")
if err != nil {
return err
}
var upgrades []db.UpgradeMeta
for _, v := range files {
lg.V(5).Infoln("reading", v)
data, err := ioutil.ReadFile(v)
if err != nil {
return err
}
var cpr makepatch.CreatePatchResult
err = json.Unmarshal(data, &cpr)
if err != nil {
return err
}
um, ok, err := sqlDB.GetUpgrade(db.GetUpgradeQuery{
Artifact: cpr.Artifact,
Version: cpr.NewVersion,
AlsoUnpublished: true,
})
if err != nil {
return err
}
if ok && um.Artifact == cpr.Artifact && um.Version == cpr.NewVersion {
lgheader := cpr.Artifact + " " + cpr.NewVersion
if um.ED25519Signature != cpr.ED25519Signature {
lg.Warningf("%s signatures does not match!", lgheader)
}
if um.SHA256Sum != cpr.SHA256Sum {
lg.Warningf("%s shasum does not match!", lgheader)
}
lg.Infof("%s is already imported, skipping", lgheader)
continue
}
upgrades = append(upgrades, db.UpgradeMeta{
Artifact: cpr.Artifact,
Version: cpr.NewVersion,
SHA256Sum: cpr.SHA256Sum,
ED25519Signature: cpr.ED25519Signature,
})
}
{
// NOTE: this will be removed later, a quick hack before other upgrades refactoring takes place
uniqeUpgrades := make(map[string]db.UpgradeMeta, 0)
for _, v := range upgrades {
uniqeUpgrades[fmt.Sprintf("%s---%s", v.Artifact, v.Version)] = v
}
upgrades = upgrades[:0]
for _, v := range uniqeUpgrades {
upgrades = append(upgrades, v)
}
}
fmt.Println(upgrades)
err = sqlDB.InsertUpgrades(upgrades)
if err != nil {
lg.Errorln(err)
return err
}
return nil
}
// SubmitSuggestion initiates the comminication with Central for a Submission
// session.
func SubmitSuggestion(w rest.ResponseWriter, r *rest.Request) {
// // TODO This is the response that must be forwarded from central/api and parsed by client and passed on to the browser.
// apiutils.WriteRestError(w,
// apierrors.NewInvalid("object", "suggestion",
// fielderrors.ValidationErrorList{
// fielderrors.NewFieldValueNotSupported("URL", "...", []string{})}))
// return
ID := r.PathParam("id")
suggestion, ok := client.GetSuggestion(ID)
if !ok {
apiutils.WriteRestError(w, apierrors.NewNotFound("suggestion", ID))
return
}
wanip := shared.GetPublicIPAddr()
if wanip == nil {
lg.Warning("could not resolve public ip addr")
}
conf := clientconfig.Get()
restclient, err := NewRestClient()
if err != nil {
apiutils.WriteRestError(w, apierrors.NewInternalError(err))
return
}
tokenResp, err := suggestion.RequestToken(
restclient, wanip, conf.Settings.Local.CountryCode)
if err != nil {
if apiutils.IsNetError(err) {
apiutils.WriteRestError(w, apierrors.NewServerTimeout("alkasir-central", "request-submission-token", 0))
} else {
apiutils.WriteRestError(w, apierrors.NewInternalError(err))
}
return
}
n, err := suggestion.SendSamples(restclient)
if err != nil {
lg.Warningln("error sending samples", err.Error())
}
lg.V(5).Infof("sent %d samples", n)
// continue sending samples if future measuremetns are expected to come
prepared, err := suggestion.Prepared()
if err != nil {
lg.Errorln(err)
} else if !prepared {
lg.V(5).Infof("all samples not collected, will try to send the rest when they are done")
go func(s client.Suggestion) {
start := time.Now()
t := time.NewTicker(30 * time.Second)
defer t.Stop()
submitSamples:
for range t.C {
if time.Now().After(start.Add(15 * time.Minute)) {
lg.Errorln("Stopping trying to send additional samples")
return
}
prepared, err := suggestion.Prepared()
if err != nil {
lg.Errorln(err)
return
}
if prepared {
restclient, err := NewRestClient()
if err != nil {
continue submitSamples
}
n, err := suggestion.SendSamples(restclient)
lg.V(5).Infof("sent %d samples", n)
if err != nil {
lg.Warningln("error sending samples", err.Error())
continue submitSamples
}
return
}
}
}(suggestion)
}
u, err := url.Parse(suggestion.URL)
if err != nil {
lg.Errorln(err)
} else {
err := clientconfig.Update(func(conf *clientconfig.Config) error {
conf.BlockedHosts.Add(u.Host)
lastBlocklistChange = time.Now()
pac.UpdateBlockedList(conf.BlockedHostsCentral.Hosts,
conf.BlockedHosts.Hosts)
return nil
})
if err != nil {
lg.Errorln(err)
}
}
w.WriteJson(tokenResp)
}
