// connectedAPIRequestHandler implements the "connected" API request.
// Clients make the connected request once a tunnel connection has been
// established and at least once per day. The last_connected input value,
// which should be a connected_timestamp output from a previous connected
// response, is used to calculate unique user stats.
func connectedAPIRequestHandler(
support *SupportServices,
geoIPData GeoIPData,
params requestJSONObject) ([]byte, error) {
err := validateRequestParams(support, params, connectedRequestParams)
if err != nil {
return nil, common.ContextError(err)
}
log.LogRawFieldsWithTimestamp(
getRequestLogFields(
support,
"connected",
geoIPData,
params,
connectedRequestParams))
connectedResponse := common.ConnectedResponse{
ConnectedTimestamp: common.TruncateTimestampToHour(common.GetCurrentTimestamp()),
}
responsePayload, err := json.Marshal(connectedResponse)
if err != nil {
return nil, common.ContextError(err)
}
return responsePayload, nil
}
// Directly call DecodeServerEntry and ValidateServerEntry with invalid inputs
func TestInvalidServerEntries(t *testing.T) {
testCases := [2]string{_INVALID_WINDOWS_REGISTRY_LEGACY_SERVER_ENTRY, _INVALID_MALFORMED_IP_ADDRESS_SERVER_ENTRY}
for _, testCase := range testCases {
encodedServerEntry := hex.EncodeToString([]byte(testCase))
serverEntry, err := DecodeServerEntry(
encodedServerEntry, common.GetCurrentTimestamp(), SERVER_ENTRY_SOURCE_EMBEDDED)
if err != nil {
t.Error(err.Error())
}
err = ValidateServerEntry(serverEntry)
if err == nil {
t.Error("server entry should not validate: %s", testCase)
}
}
}
func storeServerEntries(serverList string) error {
serverEntries, err := protocol.DecodeAndValidateServerEntryList(
serverList,
common.GetCurrentTimestamp(),
protocol.SERVER_ENTRY_SOURCE_REMOTE)
if err != nil {
return common.ContextError(err)
}
// TODO: record stats for newly discovered servers
err = StoreServerEntries(serverEntries, true)
if err != nil {
return common.ContextError(err)
}
return nil
}
// RecordRemoteServerListStat records a completed common or OSL
// remote server list resource download. These stats use the same
// persist-until-reported mechanism described in RecordTunnelStats.
func RecordRemoteServerListStat(
url, etag string) error {
remoteServerListStat := struct {
ClientDownloadTimestamp string `json:"client_download_timestamp"`
URL string `json:"url"`
ETag string `json:"etag"`
}{
common.TruncateTimestampToHour(common.GetCurrentTimestamp()),
url,
etag,
}
remoteServerListStatJson, err := json.Marshal(remoteServerListStat)
if err != nil {
return common.ContextError(err)
}
return StorePersistentStat(
PERSISTENT_STAT_TYPE_REMOTE_SERVER_LIST, remoteServerListStatJson)
}
// DecodeAndValidateServerEntryList should return 2 valid decoded entries from the input list of 4
func TestDecodeAndValidateServerEntryList(t *testing.T) {
testEncodedServerEntryList := hex.EncodeToString([]byte(_VALID_NORMAL_SERVER_ENTRY)) + "\n" +
hex.EncodeToString([]byte(_VALID_BLANK_LEGACY_SERVER_ENTRY)) + "\n" +
hex.EncodeToString([]byte(_INVALID_WINDOWS_REGISTRY_LEGACY_SERVER_ENTRY)) + "\n" +
hex.EncodeToString([]byte(_INVALID_MALFORMED_IP_ADDRESS_SERVER_ENTRY))
serverEntries, err := DecodeAndValidateServerEntryList(
testEncodedServerEntryList, common.GetCurrentTimestamp(), SERVER_ENTRY_SOURCE_EMBEDDED)
if err != nil {
t.Error(err.Error())
t.FailNow()
}
if len(serverEntries) != 2 {
t.Error("unexpected number of valid server entries")
}
for _, serverEntry := range serverEntries {
if serverEntry.IpAddress != _EXPECTED_IP_ADDRESS {
t.Error("unexpected IP address in decoded server entry: %s", serverEntry.IpAddress)
}
}
}
// newTargetServerEntryIterator is a helper for initializing the TargetServerEntry case
func newTargetServerEntryIterator(config *Config) (iterator *ServerEntryIterator, err error) {
serverEntry, err := protocol.DecodeServerEntry(
config.TargetServerEntry, common.GetCurrentTimestamp(), protocol.SERVER_ENTRY_SOURCE_TARGET)
if err != nil {
return nil, err
}
if config.EgressRegion != "" && serverEntry.Region != config.EgressRegion {
return nil, errors.New("TargetServerEntry does not support EgressRegion")
}
if config.TunnelProtocol != "" {
// Note: same capability/protocol mapping as in StoreServerEntry
requiredCapability := strings.TrimSuffix(config.TunnelProtocol, "-OSSH")
if !common.Contains(serverEntry.Capabilities, requiredCapability) {
return nil, errors.New("TargetServerEntry does not support TunnelProtocol")
}
}
iterator = &ServerEntryIterator{
isTargetServerEntryIterator: true,
hasNextTargetServerEntry: true,
targetServerEntry: serverEntry,
}
NoticeInfo("using TargetServerEntry: %s", serverEntry.IpAddress)
return iterator, nil
}
func Start(
configJson, embeddedServerEntryList string,
provider PsiphonProvider,
useDeviceBinder bool) error {
controllerMutex.Lock()
defer controllerMutex.Unlock()
if controller != nil {
return fmt.Errorf("already started")
}
config, err := psiphon.LoadConfig([]byte(configJson))
if err != nil {
return fmt.Errorf("error loading configuration file: %s", err)
}
config.NetworkConnectivityChecker = provider
if useDeviceBinder {
config.DeviceBinder = provider
config.DnsServerGetter = provider
}
psiphon.SetNoticeOutput(psiphon.NewNoticeReceiver(
func(notice []byte) {
provider.Notice(string(notice))
}))
psiphon.NoticeBuildInfo()
// TODO: should following errors be Notices?
err = psiphon.InitDataStore(config)
if err != nil {
return fmt.Errorf("error initializing datastore: %s", err)
}
serverEntries, err := protocol.DecodeAndValidateServerEntryList(
embeddedServerEntryList,
common.GetCurrentTimestamp(),
protocol.SERVER_ENTRY_SOURCE_EMBEDDED)
if err != nil {
return fmt.Errorf("error decoding embedded server entry list: %s", err)
}
err = psiphon.StoreServerEntries(serverEntries, false)
if err != nil {
return fmt.Errorf("error storing embedded server entry list: %s", err)
}
controller, err = psiphon.NewController(config)
if err != nil {
return fmt.Errorf("error initializing controller: %s", err)
}
shutdownBroadcast = make(chan struct{})
controllerWaitGroup = new(sync.WaitGroup)
controllerWaitGroup.Add(1)
go func() {
defer controllerWaitGroup.Done()
controller.Run(shutdownBroadcast)
}()
return nil
}
func main() {
// Define command-line parameters
var configFilename string
flag.StringVar(&configFilename, "config", "", "configuration input file")
var embeddedServerEntryListFilename string
flag.StringVar(&embeddedServerEntryListFilename, "serverList", "", "embedded server entry list input file")
var formatNotices bool
flag.BoolVar(&formatNotices, "formatNotices", false, "emit notices in human-readable format")
var profileFilename string
flag.StringVar(&profileFilename, "profile", "", "CPU profile output file")
var interfaceName string
flag.StringVar(&interfaceName, "listenInterface", "", "Interface Name")
flag.Parse()
// Initialize default Notice output (stderr)
var noticeWriter io.Writer
noticeWriter = os.Stderr
if formatNotices {
noticeWriter = psiphon.NewNoticeConsoleRewriter(noticeWriter)
}
psiphon.SetNoticeOutput(noticeWriter)
psiphon.NoticeBuildInfo()
// Handle required config file parameter
if configFilename == "" {
psiphon.NoticeError("configuration file is required")
os.Exit(1)
}
configFileContents, err := ioutil.ReadFile(configFilename)
if err != nil {
psiphon.NoticeError("error loading configuration file: %s", err)
os.Exit(1)
}
config, err := psiphon.LoadConfig(configFileContents)
if err != nil {
psiphon.NoticeError("error processing configuration file: %s", err)
os.Exit(1)
}
// When a logfile is configured, reinitialize Notice output
if config.LogFilename != "" {
logFile, err := os.OpenFile(config.LogFilename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
psiphon.NoticeError("error opening log file: %s", err)
os.Exit(1)
}
defer logFile.Close()
var noticeWriter io.Writer
noticeWriter = logFile
if formatNotices {
noticeWriter = psiphon.NewNoticeConsoleRewriter(noticeWriter)
}
psiphon.SetNoticeOutput(noticeWriter)
}
// Handle optional profiling parameter
if profileFilename != "" {
profileFile, err := os.Create(profileFilename)
if err != nil {
psiphon.NoticeError("error opening profile file: %s", err)
os.Exit(1)
}
pprof.StartCPUProfile(profileFile)
defer pprof.StopCPUProfile()
}
// Initialize data store
err = psiphon.InitDataStore(config)
if err != nil {
psiphon.NoticeError("error initializing datastore: %s", err)
os.Exit(1)
}
// Handle optional embedded server list file parameter
// If specified, the embedded server list is loaded and stored. When there
// are no server candidates at all, we wait for this import to complete
// before starting the Psiphon controller. Otherwise, we import while
// concurrently starting the controller to minimize delay before attempting
// to connect to existing candidate servers.
// If the import fails, an error notice is emitted, but the controller is
// still started: either existing candidate servers may suffice, or the
// remote server list fetch may obtain candidate servers.
if embeddedServerEntryListFilename != "" {
embeddedServerListWaitGroup := new(sync.WaitGroup)
embeddedServerListWaitGroup.Add(1)
go func() {
defer embeddedServerListWaitGroup.Done()
serverEntryList, err := ioutil.ReadFile(embeddedServerEntryListFilename)
if err != nil {
psiphon.NoticeError("error loading embedded server entry list file: %s", err)
return
}
// TODO: stream embedded server list data? also, the cast makes an unnecessary copy of a large buffer?
serverEntries, err := psiphon.DecodeAndValidateServerEntryList(
string(serverEntryList),
common.GetCurrentTimestamp(),
common.SERVER_ENTRY_SOURCE_EMBEDDED)
if err != nil {
psiphon.NoticeError("error decoding embedded server entry list file: %s", err)
return
}
// Since embedded server list entries may become stale, they will not
// overwrite existing stored entries for the same server.
err = psiphon.StoreServerEntries(serverEntries, false)
if err != nil {
psiphon.NoticeError("error storing embedded server entry list data: %s", err)
return
}
}()
if psiphon.CountServerEntries(config.EgressRegion, config.TunnelProtocol) == 0 {
embeddedServerListWaitGroup.Wait()
} else {
defer embeddedServerListWaitGroup.Wait()
}
}
if interfaceName != "" {
config.ListenInterface = interfaceName
}
// Run Psiphon
controller, err := psiphon.NewController(config)
if err != nil {
psiphon.NoticeError("error creating controller: %s", err)
os.Exit(1)
}
controllerStopSignal := make(chan struct{}, 1)
shutdownBroadcast := make(chan struct{})
controllerWaitGroup := new(sync.WaitGroup)
controllerWaitGroup.Add(1)
go func() {
defer controllerWaitGroup.Done()
controller.Run(shutdownBroadcast)
controllerStopSignal <- *new(struct{})
}()
// Wait for an OS signal or a Run stop signal, then stop Psiphon and exit
systemStopSignal := make(chan os.Signal, 1)
signal.Notify(systemStopSignal, os.Interrupt, os.Kill)
select {
case <-systemStopSignal:
psiphon.NoticeInfo("shutdown by system")
close(shutdownBroadcast)
controllerWaitGroup.Wait()
case <-controllerStopSignal:
psiphon.NoticeInfo("shutdown by controller")
}
}
// handshakeAPIRequestHandler implements the "handshake" API request.
// Clients make the handshake immediately after establishing a tunnel
// connection; the response tells the client what homepage to open, what
// stats to record, etc.
func handshakeAPIRequestHandler(
support *SupportServices,
apiProtocol string,
geoIPData GeoIPData,
params requestJSONObject) ([]byte, error) {
// Note: ignoring "known_servers" params
err := validateRequestParams(support, params, baseRequestParams)
if err != nil {
return nil, common.ContextError(err)
}
log.LogRawFieldsWithTimestamp(
getRequestLogFields(
support,
"handshake",
geoIPData,
params,
baseRequestParams))
// Note: ignoring param format errors as params have been validated
sessionID, _ := getStringRequestParam(params, "client_session_id")
sponsorID, _ := getStringRequestParam(params, "sponsor_id")
clientVersion, _ := getStringRequestParam(params, "client_version")
clientPlatform, _ := getStringRequestParam(params, "client_platform")
isMobile := isMobileClientPlatform(clientPlatform)
normalizedPlatform := normalizeClientPlatform(clientPlatform)
// Flag the SSH client as having completed its handshake. This
// may reselect traffic rules and starts allowing port forwards.
// TODO: in the case of SSH API requests, the actual sshClient could
// be passed in and used here. The session ID lookup is only strictly
// necessary to support web API requests.
err = support.TunnelServer.SetClientHandshakeState(
sessionID,
handshakeState{
completed: true,
apiProtocol: apiProtocol,
apiParams: copyBaseRequestParams(params),
})
if err != nil {
return nil, common.ContextError(err)
}
// Note: no guarantee that PsinetDatabase won't reload between database calls
db := support.PsinetDatabase
handshakeResponse := common.HandshakeResponse{
SSHSessionID: sessionID,
Homepages: db.GetRandomHomepage(sponsorID, geoIPData.Country, isMobile),
UpgradeClientVersion: db.GetUpgradeClientVersion(clientVersion, normalizedPlatform),
PageViewRegexes: make([]map[string]string, 0),
HttpsRequestRegexes: db.GetHttpsRequestRegexes(sponsorID),
EncodedServerList: db.DiscoverServers(geoIPData.DiscoveryValue),
ClientRegion: geoIPData.Country,
ServerTimestamp: common.GetCurrentTimestamp(),
}
responsePayload, err := json.Marshal(handshakeResponse)
if err != nil {
return nil, common.ContextError(err)
}
return responsePayload, nil
}
// FetchRemoteServerList downloads a remote server list JSON record from
// config.RemoteServerListUrl; validates its digital signature using the
// public key config.RemoteServerListSignaturePublicKey; and parses the
// data field into ServerEntry records.
func FetchRemoteServerList(
config *Config,
tunnel *Tunnel,
untunneledDialConfig *DialConfig) error {
NoticeInfo("fetching remote server list")
// Select tunneled or untunneled configuration
httpClient, requestUrl, err := MakeDownloadHttpClient(
config,
tunnel,
untunneledDialConfig,
config.RemoteServerListUrl,
time.Duration(*config.FetchRemoteServerListTimeoutSeconds)*time.Second)
if err != nil {
return common.ContextError(err)
}
// Proceed with download
downloadFilename := config.RemoteServerListDownloadFilename
if downloadFilename == "" {
splitPath := strings.Split(config.RemoteServerListUrl, "/")
downloadFilename = splitPath[len(splitPath)-1]
}
lastETag, err := GetUrlETag(config.RemoteServerListUrl)
if err != nil {
return common.ContextError(err)
}
n, responseETag, err := ResumeDownload(
httpClient, requestUrl, downloadFilename, lastETag)
NoticeRemoteServerListDownloadedBytes(n)
if err != nil {
return common.ContextError(err)
}
if responseETag == lastETag {
// The remote server list is unchanged and no data was downloaded
return nil
}
NoticeRemoteServerListDownloaded(downloadFilename)
// The downloaded content is a zlib compressed authenticated
// data package containing a list of encoded server entries.
downloadContent, err := os.Open(downloadFilename)
if err != nil {
return common.ContextError(err)
}
defer downloadContent.Close()
zlibReader, err := zlib.NewReader(downloadContent)
if err != nil {
return common.ContextError(err)
}
dataPackage, err := ioutil.ReadAll(zlibReader)
zlibReader.Close()
if err != nil {
return common.ContextError(err)
}
remoteServerList, err := ReadAuthenticatedDataPackage(
dataPackage, config.RemoteServerListSignaturePublicKey)
if err != nil {
return common.ContextError(err)
}
serverEntries, err := DecodeAndValidateServerEntryList(
remoteServerList,
common.GetCurrentTimestamp(),
common.SERVER_ENTRY_SOURCE_REMOTE)
if err != nil {
return common.ContextError(err)
}
err = StoreServerEntries(serverEntries, true)
if err != nil {
return common.ContextError(err)
}
// Now that the server entries are successfully imported, store the response
// ETag so we won't re-download this same data again.
if responseETag != "" {
err := SetUrlETag(config.RemoteServerListUrl, responseETag)
if err != nil {
NoticeAlert("failed to set remote server list ETag: %s", common.ContextError(err))
// This fetch is still reported as a success, even if we can't store the etag
}
}
return nil
}