func (s *fileSender) send(entry *logrus.Entry) (err error) {
msg, err := entry.String()
if err != nil {
return
}
file, err := os.OpenFile(utils.GetLogPath(),
os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
if err != nil {
err = &WriteError{
errors.Wrap(err, "logger: Failed to open log file"),
}
return
}
defer file.Close()
_, err = file.WriteString(msg)
if err != nil {
err = &WriteError{
errors.Wrap(err, "logger: Failed to write to log file"),
}
return
}
return
}
// Writes an image to a file. The extension of the file is used to determine the
// encoding method.
func CreateImage(i image.Image, file string) error {
ext := filepath.Ext(file)
m := mime.TypeByExtension(ext)
switch m {
case "image/jpeg", "image/png", "image/tiff":
default:
return errors.Newf("unsupported extension/mime type: %s %s", ext, m)
}
f, err := os.Create(file)
if err != nil {
return errors.Wrap(err, "could not create file")
}
defer f.Close()
switch m {
case "image/jpeg":
err = jpeg.Encode(f, i, &jpeg.Options{Quality: 98})
case "image/png":
err = png.Encode(f, i)
case "image/tiff":
err = tiff.Encode(f, i, &tiff.Options{Compression: tiff.Deflate, Predictor: true})
default:
panic("unreachable")
}
if err != nil {
return errors.Wrap(err, "could not encode image")
}
return nil
}
func (s *Saml) Authorize(state, respEncoded string) (
data *UserData, err error) {
resp, err := saml.ParseEncodedResponse(respEncoded)
if err != nil {
err = &SamlError{
errors.Wrap(err, "saml: Failed to parse response"),
}
return
}
err = resp.Validate(&s.provider)
if err != nil {
err = &SamlError{
errors.Wrap(err, "saml: Failed to validate response"),
}
return
}
data = &UserData{
Username: resp.GetAttribute("username"),
Email: resp.GetAttribute("email"),
Org: resp.GetAttribute("org"),
Secondary: resp.GetAttribute("secondary"),
}
if data.Username == "" {
data.Username = resp.Assertion.Subject.NameID.Value
}
return
}
func (c *Oauth2Client) GetJson(url string, resp interface{}) (err error) {
httpResp, err := c.client.Get(url)
if err != nil {
err = &errortypes.UnknownError{
errors.Wrap(err, "oauth.oauth2: Unknown api error"),
}
return
}
defer httpResp.Body.Close()
body, err := ioutil.ReadAll(httpResp.Body)
if err != nil {
err = &errortypes.UnknownError{
errors.Wrap(err, "oauth.oauth2: Unknown parse error"),
}
return
}
err = json.Unmarshal(body, resp)
if err != nil {
err = &errortypes.UnknownError{
errors.Wrap(err, "oauth.oauth2: Unknown parse error"),
}
return
}
return
}
// This returns the list of local ip addresses which other hosts can connect
// to (NOTE: Loopback ip is ignored).
// Also resolves Hostname to an address and adds it to the list too, so
// IPs from /etc/hosts can work too.
func GetLocalIPs() ([]*net.IP, error) {
hostname, err := os.Hostname()
if err != nil {
return nil, errors.Wrap(err, "Failed to lookup hostname")
}
// Resolves IP Address from Hostname, this way overrides in /etc/hosts
// can work too for IP resolution.
ipInfo, err := net.ResolveIPAddr("ip4", hostname)
if err != nil {
return nil, errors.Wrap(err, "Failed to resolve ip")
}
ips := []*net.IP{&ipInfo.IP}
// TODO(zviad): Is rest of the code really necessary?
addrs, err := net.InterfaceAddrs()
if err != nil {
return nil, errors.Wrap(err, "Failed to get interface addresses.")
}
for _, addr := range addrs {
ipnet, ok := addr.(*net.IPNet)
if !ok {
continue
}
if ipnet.IP.IsLoopback() {
continue
}
ips = append(ips, &ipnet.IP)
}
return ips, nil
}
func Page(q PageQuery) (*PageResponse, error) {
if q.ID == 0 {
return nil, errors.New("a page id is required")
}
resp, err := http.Get(q.urlString())
if err != nil {
return nil, errors.Wrap(err, "could not get http response")
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return nil, errors.Wrapf(errors.New(resp.Status), "StatusCode: %d; URL: %s", resp.StatusCode, q.urlString())
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrap(err, "could not read http response body")
}
var response PageResponse
if err := json.Unmarshal(body, &response); err != nil {
return nil, errors.Wrap(err, "could not unmarshal http response")
}
return &response, nil
}
// Performs the HTTP request using our HTTP client
func (pool *SimplePool) Do(req *http.Request) (resp *http.Response, err error) {
conn, err := pool.Get()
if err != nil {
return nil, errors.Wrap(err, err.Error())
}
if pool.params.UseSSL {
req.URL.Scheme = "https"
} else {
req.URL.Scheme = "http"
}
if pool.params.HostHeader != nil {
req.URL.Host = *pool.params.HostHeader
} else {
req.URL.Host = pool.addr
}
resp, err = conn.Do(req)
if err != nil {
if urlErr, ok := err.(*url.Error); ok &&
strings.HasPrefix(urlErr.Err.Error(), dialErrorMsgPrefix) {
err = DialError{errors.Wrap(err, "SimplePool: Dial Error")}
} else {
err = errors.Wrap(err, err.Error())
}
}
return
}
func clean() (err error) {
paths := []string{
filepath.Join(pathSep, "usr", "local", "bin", "pritunl-openvpn"),
filepath.Join(pathSep, "private", "var", "db", "receipts",
"com.pritunl.pkg.Pritunl.bom"),
filepath.Join(pathSep, "private", "var", "db", "receipts",
"com.pritunl.pkg.Pritunl.plist"),
filepath.Join(pathSep, "private", "tmp", "pritunl"),
filepath.Join(pathSep, "Applications", "Pritunl.app"),
filepath.Join(pathSep, "Library", "LaunchAgents",
"com.pritunl.client.plist"),
}
homesPath := filepath.Join(pathSep, "Users")
homes, err := ioutil.ReadDir(homesPath)
if err != nil {
err = &ParseError{
errors.Wrap(err, "autoclean: Failed to read home directories"),
}
return
}
for _, home := range homes {
if !home.IsDir() {
continue
}
paths = append(paths, filepath.Join(homesPath, home.Name(),
"Library", "Application Support", "pritunl"))
paths = append(paths, filepath.Join(homesPath, home.Name(),
"Library", "Caches", "pritunl"))
paths = append(paths, filepath.Join(homesPath, home.Name(),
"Library", "Preferences", "com.electron.pritunl.plist"))
}
paths = append(paths, filepath.Join(pathSep, "usr", "local",
"bin", "pritunl-service"))
paths = append(paths, filepath.Join(pathSep, "Library", "LaunchDaemons",
"com.pritunl.service.plist"))
for _, path := range paths {
if len(path) < 20 {
panic("autoclean: Bad path " + path)
}
err = os.RemoveAll(path)
if err != nil {
err = &RemoveError{
errors.Wrap(err, "autoclean: Failed to remove file"),
}
}
}
return
}
// Checks out an HTTP connection from an instance pool, favoring less loaded instances.
func (pool *LoadBalancedPool) Get() (*http.Client, error) {
_, instance, _, err := pool.getInstance()
if err != nil {
return nil, errors.Wrap(err, "can't get HTTP connection")
}
conn, err := instance.Get()
if err != nil {
return nil, errors.Wrap(err, "couldn't Get from LoadBalancedPool")
}
return conn, err
}
func (q *StationQuery) fetchAndWriteStationData(offset int) (stationCount int, err error) {
u := fmt.Sprintf("%s/cdo-web/api/v2/stations?limit=%d&datasetid=GHCND&offset=%d", BaseURL, StationPageLimit, offset)
body, err := request(u, q.APITokens)
if err != nil {
return 0, err
}
db, err := bolt.Open(q.CachePath, 0600, nil)
if err != nil {
return 0, errors.Wrap(err, "could not open station cache")
}
defer db.Close()
// if !NorthAmerica.Contains(geo.NewPoint(r.Latitude, r.Longitude)) {
// fmt.Println("IS NOT CONTAINED IN NORTH AMERICA")
// continue
// }
var response stationAPI
if err := json.Unmarshal(body, &response); err != nil {
return 0, errors.Wrap(err, "could not unmarshal response")
}
if err := db.Update(func(tx *bolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists([]byte("stations"))
if err != nil {
return err
}
for _, r := range response.Results {
value, err := json.Marshal(r)
if err != nil {
return errors.Wrap(err, "could not marshal station value")
}
if err := bucket.Put([]byte(r.ID), value); err != nil {
return err
}
}
return nil
}); err != nil {
return 0, errors.Wrap(err, "could not update station collection")
}
return response.Meta.ResultSet.Count, nil
}
// Opens and decodes (see image.Decode) an image file.
func OpenImage(file string) (image.Image, error) {
f, err := os.Open(file)
if err != nil {
return nil, errors.Wrap(err, "could not open file")
}
defer f.Close()
i, _, err := image.Decode(f)
if err != nil {
return nil, errors.Wrap(err, "could not decode image")
}
return i, nil
}
func (s *SearchQuery) request(page int) error {
select {
case <-s.tmb.Dying():
return nil
default:
}
resp, err := http.Get(s.url(page))
if err != nil {
return errors.Wrap(err, "could not get http response")
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return errors.Wrapf(errors.New(resp.Status), "StatusCode: %d; URL: %s", resp.StatusCode, s.url(page))
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return errors.Wrap(err, "could not read http response body")
}
var response results
if err := json.Unmarshal(body, &response); err != nil {
return errors.Wrap(err, "could not unmarshal http response")
}
for i := range response.Results {
s.ch <- response.Results[i]
}
if page > 1 {
return nil
}
// If this the first page, schedule the remaining requests
totalRequests := math.Ceil(response.TotalResults / response.ItemsPerPage)
var wg sync.WaitGroup
for i := 2; i <= int(totalRequests); i++ {
func(count int) {
wg.Add(1)
go s.tmb.Go(func() error {
defer wg.Done()
return s.request(count)
})
}(i)
}
wg.Wait()
return nil
}
func PublicKeyFile(file string) (auth ssh.AuthMethod, err error) {
buffer, err := ioutil.ReadFile(file)
if err != nil {
err = errors.Wrap(err, "")
return
}
key, err := ssh.ParsePrivateKey(buffer)
if err != nil {
err = errors.Wrap(err, "")
return
}
return ssh.PublicKeys(key), nil
}
// See Client interface for documentation.
func (c *ShardedClient) Stat(statsKey string) StatResponse {
statEntries := make(map[int](map[string]string))
var err error
for shard, conn := range c.manager.GetAllShards() {
response := c.statHelper(shard, conn, statsKey)
if response.Error() != nil {
if err == nil {
err = response.Error()
} else {
err = errors.Wrap(response.Error(), err.Error())
}
}
for shardId, entries := range response.Entries() {
statEntries[shardId] = entries
}
}
if err != nil {
return NewStatErrorResponse(err, statEntries)
}
return NewStatResponse(StatusNoError, statEntries)
}
func (q *StationQuery) countCache() (count int, err error) {
db, err := bolt.Open(q.CachePath, 0600, nil)
if err != nil {
return 0, errors.Wrap(err, "could not open station cache")
}
defer db.Close()
if err := db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("stations"))
if b == nil {
return ErrStationCacheEmpty
}
c := b.Cursor()
for k, _ := c.First(); k != nil; k, _ = c.Next() {
count++
}
return nil
}); err != nil {
return 0, err
}
return count, nil
}
// Issues an HTTP request, distributing more load to relatively unloaded instances.
func (pool *LoadBalancedPool) Do(req *http.Request) (*http.Response, error) {
for i := 0; ; i++ {
idx, instance, err := pool.getInstance()
if err != nil {
return nil, errors.Wrap(err, "can't get HTTP connection")
}
resp, err := instance.Do(req)
if err != nil || resp.StatusCode == 500 {
// 500s are also treated as service being down momentarily,
// note that even if all servers get marked down LBPool continues
// to send requests in round robin manner, thus this provides extra
// protection when service may still be up but have higher rate of
// 500s for whatever reason.
pool.markInstanceDown(idx, instance, time.Now().Add(markDownDuration).Unix())
}
if err != nil {
if _, ok := err.(DialError); !ok {
return resp, err
}
if (i + 1) < connectionAttempts {
continue
}
}
return resp, err
}
}
func (c *inExpression) SerializeSql(out *bytes.Buffer) error {
if c.err != nil {
return errors.Wrap(c.err, "Invalid IN expression")
}
if c.lhs == nil {
return errors.Newf(
"lhs of in expression is nil. Generated sql: %s",
out.String())
}
// We'll serialize the lhs even if we don't need it to ensure no error
buf := &bytes.Buffer{}
err := c.lhs.SerializeSql(buf)
if err != nil {
return err
}
if c.rhs == nil {
out.WriteString("FALSE")
return nil
}
out.WriteString(buf.String())
out.WriteString(" IN ")
err = c.rhs.SerializeSql(out)
if err != nil {
return err
}
return nil
}
func (o *Oauth2) Authorize(db *database.Database, state, code string) (
acct *account.Account, tokn *Token, err error) {
coll := db.Tokens()
tokn = &Token{}
err = coll.FindOneId(state, tokn)
if err != nil {
err = database.ParseError(err)
return
}
accessTokn, err := o.conf.Exchange(oauth2.NoContext, code)
if err != nil {
err = &errortypes.UnknownError{
errors.Wrap(err, "oauth.oauth2: Unknown api error"),
}
return
}
acct = &account.Account{
Type: o.Type,
Oauth2AccTokn: accessTokn.AccessToken,
Oauth2RefTokn: accessTokn.RefreshToken,
Oauth2Exp: accessTokn.Expiry,
}
return
}
// See Client interface for documentation.
func (c *ShardedClient) Version() VersionResponse {
shardConns := c.manager.GetAllShards()
var err error
versions := make(map[int]string)
for shard, conn := range shardConns {
response := c.versionHelper(shard, conn)
if response.Error() != nil {
if err == nil {
err = response.Error()
} else {
err = errors.Wrap(response.Error(), err.Error())
}
continue
}
for shardId, versionString := range response.Versions() {
versions[shardId] = versionString
}
}
if err != nil {
return NewVersionErrorResponse(err, versions)
}
return NewVersionResponse(StatusNoError, versions)
}
func ImportProfiles() (prfls []*Profile, err error) {
path := getPath()
exists, err := utils.Exists(path)
if err != nil {
return
}
if !exists {
return
}
prfls = []*Profile{}
data, err := utils.Read(path)
if err != nil {
return
}
err = json.Unmarshal([]byte(data), &prfls)
if err != nil {
err = errortypes.ParseError{
errors.Wrap(err, "profile: Failed to parse profiles"),
}
return
}
return
}
func (o *Oauth2) Request(db *database.Database, remoteState, remoteSecret,
remoteCallback string, version int) (url string, err error) {
coll := db.Tokens()
state := utils.RandStr(64)
url = o.conf.AuthCodeURL(state, oauth2.AccessTypeOffline,
oauth2.ApprovalForce)
if err != nil {
err = &errortypes.UnknownError{
errors.Wrap(err, "oauth.oauth2: Unknown api error"),
}
return
}
tokn := &Token{
Id: state,
RemoteCallback: remoteCallback,
RemoteState: remoteState,
RemoteSecret: remoteSecret,
Type: o.Type,
Version: version,
}
err = coll.Insert(tokn)
if err != nil {
err = database.ParseError(err)
return
}
return
}
func Update() (err error) {
ifaces, err := net.Interfaces()
if err != nil {
err = &SystemError{
errors.Wrap(err, "networks: Error getting interfaces"),
}
return
}
networks := map[string]*net.IPNet{}
for _, iface := range ifaces {
if !strings.Contains(iface.Name, "tun") {
continue
}
addrs, err := iface.Addrs()
if err != nil {
continue
}
for _, addr := range addrs {
_, subnet, err := net.ParseCIDR(addr.String())
if err != nil {
continue
}
networks[subnet.String()] = subnet
}
}
Networks = networks
return
}
// Returns an escaped literal string
func Literal(v interface{}) Expression {
value, err := sqltypes.BuildValue(v)
if err != nil {
panic(errors.Wrap(err, "Invalid literal value"))
}
return &literalExpression{value: value}
}
func (s *Saml) Init() (err error) {
certPath := GetCertPath()
err = utils.Write(certPath, s.Cert)
if err != nil {
return
}
s.provider = saml.ServiceProviderSettings{
PublicCertPath: filepath.Join(
constants.SamlCertDir, constants.SamlCert),
PrivateKeyPath: filepath.Join(
constants.SamlCertDir, constants.SamlKey),
IDPSSOURL: s.SsoUrl,
IDPSSODescriptorURL: s.IssuerUrl,
IDPPublicCertPath: certPath,
SPSignRequest: true,
AssertionConsumerServiceURL: SamlCallbackUrl,
}
err = s.provider.Init()
if err != nil {
err = &constants.ReadError{
errors.Wrap(err, "saml: Failed to init provider"),
}
return
}
return
}
func (p *Profile) Parse(data string) (err error) {
lines := strings.Split(data, "\n")
jsonData := ""
conf := ""
for i, line := range lines {
if strings.HasPrefix(line, "#") {
jsonData += strings.TrimSpace(line[1:])
} else {
conf = strings.Join(lines[i:], "\n")
break
}
}
err = json.Unmarshal([]byte(jsonData), p)
if err != nil {
err = errortypes.ParseError{
errors.Wrap(err, "profile: Failed to parse json data"),
}
return
}
p.Conf = conf
p.NetworkLinks = NetworkLinks
return
}
// on a droplet assuming root user and ssh key at id_rsa.pub deployed
func RunRemoteCommand(addr string, command string) (s string, err error) {
usr, err := user.Current()
if err != nil {
err = errors.Wrap(err, "")
return
}
authMethod, err := PublicKeyFile(usr.HomeDir + "/.ssh/id_rsa")
if err != nil {
err = errors.Wrap(err, "")
return
}
sshConfig := &ssh.ClientConfig{
User: "******",
Auth: []ssh.AuthMethod{authMethod},
}
conn, err := ssh.Dial("tcp", addr+":22", sshConfig)
if err != nil {
err = errors.Wrap(err, "ssh dial failed.")
return
}
defer conn.Close()
session, err := conn.NewSession()
if err != nil {
err = errors.Wrap(err, "session failed.")
return
}
defer session.Close()
var stdoutBuf bytes.Buffer
session.Stdout = &stdoutBuf
err = session.Run(command)
if err != nil {
err = errors.Wrap(err, "cmd failed.")
return
}
return stdoutBuf.String(), nil
}
// Issues an HTTP request, distributing more load to relatively unloaded instances.
func (pool *LoadBalancedPool) DoWithTimeout(req *http.Request,
timeout time.Duration) (*http.Response, error) {
var requestErr error = nil
deadline := time.Time{}
if timeout > 0 {
deadline = time.Now().Add(timeout)
}
for i := 0; ; i++ {
idx, instance, isDown, err := pool.getInstance()
if err != nil {
return nil, errors.Wrap(err, "can't get HTTP connection")
}
if isDown && requestErr != nil {
// If current pool instance is marked down, that means all instances in the pool
// are most likely marked down, thus avoid performing any connect retries, to fail
// faster.
return nil, requestErr
}
var timer *time.Timer
if !deadline.IsZero() {
timeout = deadline.Sub(time.Now())
if timeout > 0 {
timer = time.AfterFunc(timeout, func() {
instance.transport.CancelRequest(req)
})
}
}
resp, err := instance.Do(req)
if timer != nil && err == nil {
resp.Body = &cancelTimerBody{timer, resp.Body}
}
if err != nil || resp.StatusCode == 500 {
// 500s are also treated as service being down momentarily,
// note that even if all servers get marked down LBPool continues
// to send requests in round robin manner, thus this provides extra
// protection when service may still be up but have higher rate of
// 500s for whatever reason.
pool.markInstanceDown(idx, instance, time.Now().Add(markDownDuration).Unix())
} else if isDown {
// If an instance was marked as down, but succeeded, reset the mark down timer, so
// instance is treated as healthy right away.
pool.markInstanceUp(idx, instance)
}
if err != nil {
if _, ok := err.(DialError); !ok {
return resp, err
}
if (i + 1) < connectionAttempts {
requestErr = err
continue
}
}
return resp, err
}
}
// See net.Conn for documentation
func (c *ManagedConnImpl) Write(b []byte) (n int, err error) {
if atomic.LoadInt32(&c.isActive) != 1 {
return 0, errors.New("The connection is no longer active")
}
n, err = c.conn.Write(b)
if err != nil {
err = errors.Wrap(err, "Write error")
}
return
}
func (q *Query) getAPIList(station Station) ([]*Record, error) {
// Message from the API:
// {"status":"400","developerMessage":"The date range must be less than 1 year.","userMessage":"There was an error with the request."}
// TODO: Extend the API to divide the range into multiple years.
if int(q.End.Sub(q.Start)/(24*time.Hour)) > 365 {
return nil, errors.Newf("The NOAA API requires the date range[%v, %v] to be less than 1 year.", q.Start.Format("2006-01-02"), q.End.Format("2006-01-02"))
}
var (
offset int
list []*Record
)
for {
u := fmt.Sprintf(
"%s/cdo-web/api/v2/data?datasetid=GHCND&stationid=%s&startdate=%s&enddate=%s&limit=1000&offset=%d",
BaseURL,
station.ID,
q.Start.Format("2006-01-02"),
q.End.Format("2006-01-02"),
offset,
)
body, err := request(u, q.APITokens)
if err != nil {
return nil, err
}
var response ghcndAPIResults
if err := json.Unmarshal(body, &response); err != nil {
return nil, errors.Wrap(err, "could not unmarshall api response")
}
if response.Meta.ResultSet.Count == 0 {
return nil, nil
}
for _, record := range response.Results {
list = q.addToList(list, &record, station)
}
offset = response.Meta.ResultSet.Offset + response.Meta.ResultSet.Limit
if offset > response.Meta.ResultSet.Count {
break
}
}
return list, nil
}
func GetLocalAddress() (addr string, err error) {
name, err := os.Hostname()
if err != nil {
err = &constants.UnknownError{
errors.Wrap(err, "utils: Get ip"),
}
return
}
addrs, err := net.LookupHost(name)
if err != nil {
err = &constants.UnknownError{
errors.Wrap(err, "utils: Get ip"),
}
return
}
addr = addrs[0]
return
}
