func main() { var generateTrafficRulesFilename string var generateServerEntryFilename string var generateLogFilename string var generateServerIPaddress string var generateServerNetworkInterface string var generateWebServerPort int var generateProtocolPorts stringListFlag var configFilename string flag.StringVar( &generateTrafficRulesFilename, "trafficRules", server.SERVER_TRAFFIC_RULES_FILENAME, "generate with this traffic rules `filename`") flag.StringVar( &generateServerEntryFilename, "serverEntry", server.SERVER_ENTRY_FILENAME, "generate with this server entry `filename`") flag.StringVar( &generateLogFilename, "logFilename", "", "set application log file name and path; blank for stderr") flag.StringVar( &generateServerIPaddress, "ipaddress", server.DEFAULT_SERVER_IP_ADDRESS, "generate with this server `IP address`") flag.StringVar( &generateServerNetworkInterface, "interface", "", "generate with server IP address from this `network-interface`") flag.IntVar( &generateWebServerPort, "web", 0, "generate with web server `port`; 0 for no web server") flag.Var( &generateProtocolPorts, "protocol", "generate with `protocol:port`; flag may be repeated to enable multiple protocols") flag.StringVar( &configFilename, "config", server.SERVER_CONFIG_FILENAME, "run or generate with this config `filename`") flag.Usage = func() { fmt.Fprintf(os.Stderr, "Usage:\n\n"+ "%s <flags> generate generates configuration files\n"+ "%s <flags> run runs configured services\n\n", os.Args[0], os.Args[0]) flag.PrintDefaults() } flag.Parse() args := flag.Args() if len(args) < 1 { flag.Usage() os.Exit(1) } else if args[0] == "generate" { serverIPaddress := generateServerIPaddress if generateServerNetworkInterface != "" { var err error serverIPaddress, err = common.GetInterfaceIPAddress(generateServerNetworkInterface) if err != nil { fmt.Printf("generate failed: %s\n", err) os.Exit(1) } } tunnelProtocolPorts := make(map[string]int) for _, protocolPort := range generateProtocolPorts { parts := strings.Split(protocolPort, ":") if len(parts) == 2 { port, err := strconv.Atoi(parts[1]) if err != nil { fmt.Printf("generate failed: %s\n", err) os.Exit(1) } tunnelProtocolPorts[parts[0]] = port } } configJSON, trafficRulesJSON, encodedServerEntry, err := server.GenerateConfig( &server.GenerateConfigParams{ LogFilename: generateLogFilename, ServerIPAddress: serverIPaddress, EnableSSHAPIRequests: true, WebServerPort: generateWebServerPort, TunnelProtocolPorts: tunnelProtocolPorts, TrafficRulesFilename: generateTrafficRulesFilename, }) if err != nil { fmt.Printf("generate failed: %s\n", err) os.Exit(1) } err = ioutil.WriteFile(configFilename, configJSON, 0600) if err != nil { fmt.Printf("error writing configuration file: %s\n", err) os.Exit(1) } err = ioutil.WriteFile(generateTrafficRulesFilename, trafficRulesJSON, 0600) if err != nil { fmt.Printf("error writing traffic rule configuration file: %s\n", err) os.Exit(1) } err = ioutil.WriteFile(generateServerEntryFilename, encodedServerEntry, 0600) if err != nil { fmt.Printf("error writing server entry file: %s\n", err) os.Exit(1) } } else if args[0] == "run" { configJSON, err := ioutil.ReadFile(configFilename) if err != nil { fmt.Printf("error loading configuration file: %s\n", err) os.Exit(1) } err = server.RunServices(configJSON) if err != nil { fmt.Printf("run failed: %s\n", err) os.Exit(1) } } }
// Run executes the controller. It launches components and then monitors // for a shutdown signal; after receiving the signal it shuts down the // controller. // The components include: // - the periodic remote server list fetcher // - the connected reporter // - the tunnel manager // - a local SOCKS proxy that port forwards through the pool of tunnels // - a local HTTP proxy that port forwards through the pool of tunnels func (controller *Controller) Run(shutdownBroadcast <-chan struct{}) { ReportAvailableRegions() // Start components listenIP, err := common.GetInterfaceIPAddress(controller.config.ListenInterface) if err != nil { NoticeError("error getting listener IP: %s", err) return } socksProxy, err := NewSocksProxy(controller.config, controller, listenIP) if err != nil { NoticeAlert("error initializing local SOCKS proxy: %s", err) return } defer socksProxy.Close() httpProxy, err := NewHttpProxy( controller.config, controller.untunneledDialConfig, controller, listenIP) if err != nil { NoticeAlert("error initializing local HTTP proxy: %s", err) return } defer httpProxy.Close() if !controller.config.DisableRemoteServerListFetcher { retryPeriod := time.Duration( *controller.config.FetchRemoteServerListRetryPeriodSeconds) * time.Second if controller.config.RemoteServerListUrl != "" { controller.runWaitGroup.Add(1) go controller.remoteServerListFetcher( "common", FetchCommonRemoteServerList, controller.signalFetchCommonRemoteServerList, retryPeriod, FETCH_REMOTE_SERVER_LIST_STALE_PERIOD) } if controller.config.ObfuscatedServerListRootURL != "" { controller.runWaitGroup.Add(1) go controller.remoteServerListFetcher( "obfuscated", FetchObfuscatedServerLists, controller.signalFetchObfuscatedServerLists, retryPeriod, FETCH_REMOTE_SERVER_LIST_STALE_PERIOD) } } if controller.config.UpgradeDownloadUrl != "" && controller.config.UpgradeDownloadFilename != "" { controller.runWaitGroup.Add(1) go controller.upgradeDownloader() } /// Note: the connected reporter isn't started until a tunnel is // established controller.runWaitGroup.Add(1) go controller.runTunnels() if *controller.config.EstablishTunnelTimeoutSeconds != 0 { controller.runWaitGroup.Add(1) go controller.establishTunnelWatcher() } // Wait while running select { case <-shutdownBroadcast: NoticeInfo("controller shutdown by request") case <-controller.componentFailureSignal: NoticeAlert("controller shutdown due to component failure") } close(controller.shutdownBroadcast) // Interrupts and stops establish workers blocking on // tunnel establishment network operations. controller.establishPendingConns.CloseAll() // Interrupts and stops workers blocking on untunneled // network operations. This includes fetch remote server // list and untunneled uprade download. // Note: this doesn't interrupt the final, untunneled status // requests started in operateTunnel after shutdownBroadcast. // This is by design -- we want to give these requests a short // timer period to succeed and deliver stats. These particular // requests opt out of untunneledPendingConns and use the // PSIPHON_API_SHUTDOWN_SERVER_TIMEOUT timeout (see // doUntunneledStatusRequest). controller.untunneledPendingConns.CloseAll() // Now with all workers signaled to stop and with all // blocking network operations interrupted, wait for // all workers to terminate. controller.runWaitGroup.Wait() controller.splitTunnelClassifier.Shutdown() NoticeInfo("exiting controller") NoticeExiting() }
func runServer(t *testing.T, runConfig *runServerConfig) { // create a server var err error serverIPaddress := "" for _, interfaceName := range []string{"eth0", "en0"} { serverIPaddress, err = common.GetInterfaceIPAddress(interfaceName) if err == nil { break } } if err != nil { t.Fatalf("error getting server IP address: %s", err) } serverConfigJSON, _, encodedServerEntry, err := GenerateConfig( &GenerateConfigParams{ ServerIPAddress: serverIPaddress, EnableSSHAPIRequests: runConfig.enableSSHAPIRequests, WebServerPort: 8000, TunnelProtocolPorts: map[string]int{runConfig.tunnelProtocol: 4000}, }) if err != nil { t.Fatalf("error generating server config: %s", err) } // customize server config // Pave psinet with random values to test handshake homepages. psinetFilename := filepath.Join(testDataDirName, "psinet.json") sponsorID, expectedHomepageURL := pavePsinetDatabaseFile(t, psinetFilename) // Pave traffic rules file which exercises handshake parameter filtering. Client // must handshake with specified sponsor ID in order to allow ports for tunneled // requests. trafficRulesFilename := filepath.Join(testDataDirName, "traffic_rules.json") paveTrafficRulesFile(t, trafficRulesFilename, sponsorID, runConfig.denyTrafficRules) oslConfigFilename := filepath.Join(testDataDirName, "osl_config.json") propagationChannelID := paveOSLConfigFile(t, oslConfigFilename) var serverConfig map[string]interface{} json.Unmarshal(serverConfigJSON, &serverConfig) serverConfig["GeoIPDatabaseFilename"] = "" serverConfig["PsinetDatabaseFilename"] = psinetFilename serverConfig["TrafficRulesFilename"] = trafficRulesFilename serverConfig["OSLConfigFilename"] = oslConfigFilename serverConfig["LogLevel"] = "error" serverConfigJSON, _ = json.Marshal(serverConfig) // run server serverWaitGroup := new(sync.WaitGroup) serverWaitGroup.Add(1) go func() { defer serverWaitGroup.Done() err := RunServices(serverConfigJSON) if err != nil { // TODO: wrong goroutine for t.FatalNow() t.Fatalf("error running server: %s", err) } }() defer func() { // Test: orderly server shutdown p, _ := os.FindProcess(os.Getpid()) p.Signal(os.Interrupt) shutdownTimeout := time.NewTimer(5 * time.Second) shutdownOk := make(chan struct{}, 1) go func() { serverWaitGroup.Wait() shutdownOk <- *new(struct{}) }() select { case <-shutdownOk: case <-shutdownTimeout.C: t.Fatalf("server shutdown timeout exceeded") } }() // TODO: monitor logs for more robust wait-until-loaded time.Sleep(1 * time.Second) // Test: hot reload (of psinet and traffic rules) if runConfig.doHotReload { // Pave a new psinet and traffic rules with different random values. sponsorID, expectedHomepageURL = pavePsinetDatabaseFile(t, psinetFilename) paveTrafficRulesFile(t, trafficRulesFilename, sponsorID, runConfig.denyTrafficRules) p, _ := os.FindProcess(os.Getpid()) p.Signal(syscall.SIGUSR1) // TODO: monitor logs for more robust wait-until-reloaded time.Sleep(1 * time.Second) // After reloading psinet, the new sponsorID/expectedHomepageURL // should be active, as tested in the client "Homepage" notice // handler below. } // Exercise server_load logging p, _ := os.FindProcess(os.Getpid()) p.Signal(syscall.SIGUSR2) // connect to server with client // TODO: currently, TargetServerEntry only works with one tunnel numTunnels := 1 localSOCKSProxyPort := 1081 localHTTPProxyPort := 8081 establishTunnelPausePeriodSeconds := 1 // Note: calling LoadConfig ensures all *int config fields are initialized clientConfigJSON := ` { "ClientPlatform" : "Windows", "ClientVersion" : "0", "SponsorId" : "0", "PropagationChannelId" : "0", "DisableRemoteServerListFetcher" : true }` clientConfig, _ := psiphon.LoadConfig([]byte(clientConfigJSON)) clientConfig.SponsorId = sponsorID clientConfig.PropagationChannelId = propagationChannelID clientConfig.ConnectionWorkerPoolSize = numTunnels clientConfig.TunnelPoolSize = numTunnels clientConfig.EstablishTunnelPausePeriodSeconds = &establishTunnelPausePeriodSeconds clientConfig.TargetServerEntry = string(encodedServerEntry) clientConfig.TunnelProtocol = runConfig.tunnelProtocol clientConfig.LocalSocksProxyPort = localSOCKSProxyPort clientConfig.LocalHttpProxyPort = localHTTPProxyPort clientConfig.EmitSLOKs = true if runConfig.doClientVerification { clientConfig.ClientPlatform = "Android" } clientConfig.DataStoreDirectory = testDataDirName err = psiphon.InitDataStore(clientConfig) if err != nil { t.Fatalf("error initializing client datastore: %s", err) } controller, err := psiphon.NewController(clientConfig) if err != nil { t.Fatalf("error creating client controller: %s", err) } tunnelsEstablished := make(chan struct{}, 1) homepageReceived := make(chan struct{}, 1) slokSeeded := make(chan struct{}, 1) verificationRequired := make(chan struct{}, 1) verificationCompleted := make(chan struct{}, 1) psiphon.SetNoticeOutput(psiphon.NewNoticeReceiver( func(notice []byte) { //fmt.Printf("%s\n", string(notice)) noticeType, payload, err := psiphon.GetNotice(notice) if err != nil { return } switch noticeType { case "Tunnels": // Do not set verification payload until tunnel is // established. Otherwise will silently take no action. controller.SetClientVerificationPayloadForActiveTunnels("") count := int(payload["count"].(float64)) if count >= numTunnels { sendNotificationReceived(tunnelsEstablished) } case "Homepage": homepageURL := payload["url"].(string) if homepageURL != expectedHomepageURL { // TODO: wrong goroutine for t.FatalNow() t.Fatalf("unexpected homepage: %s", homepageURL) } sendNotificationReceived(homepageReceived) case "SLOKSeeded": sendNotificationReceived(slokSeeded) case "ClientVerificationRequired": sendNotificationReceived(verificationRequired) controller.SetClientVerificationPayloadForActiveTunnels(dummyClientVerificationPayload) case "NoticeClientVerificationRequestCompleted": sendNotificationReceived(verificationCompleted) } })) controllerShutdownBroadcast := make(chan struct{}) controllerWaitGroup := new(sync.WaitGroup) controllerWaitGroup.Add(1) go func() { defer controllerWaitGroup.Done() controller.Run(controllerShutdownBroadcast) }() defer func() { close(controllerShutdownBroadcast) shutdownTimeout := time.NewTimer(20 * time.Second) shutdownOk := make(chan struct{}, 1) go func() { controllerWaitGroup.Wait() shutdownOk <- *new(struct{}) }() select { case <-shutdownOk: case <-shutdownTimeout.C: t.Fatalf("controller shutdown timeout exceeded") } }() // Test: tunnels must be established, and correct homepage // must be received, within 30 seconds timeoutSignal := make(chan struct{}) go func() { timer := time.NewTimer(30 * time.Second) <-timer.C close(timeoutSignal) }() waitOnNotification(t, tunnelsEstablished, timeoutSignal, "tunnel establish timeout exceeded") waitOnNotification(t, homepageReceived, timeoutSignal, "homepage received timeout exceeded") if runConfig.doClientVerification { waitOnNotification(t, verificationRequired, timeoutSignal, "verification required timeout exceeded") waitOnNotification(t, verificationCompleted, timeoutSignal, "verification completed timeout exceeded") } if runConfig.doTunneledWebRequest { // Test: tunneled web site fetch err = makeTunneledWebRequest(t, localHTTPProxyPort) if err == nil { if runConfig.denyTrafficRules { t.Fatalf("unexpected tunneled web request success") } } else { if !runConfig.denyTrafficRules { t.Fatalf("tunneled web request failed: %s", err) } } } if runConfig.doTunneledNTPRequest { // Test: tunneled UDP packets udpgwServerAddress := serverConfig["UDPInterceptUdpgwServerAddress"].(string) err = makeTunneledNTPRequest(t, localSOCKSProxyPort, udpgwServerAddress) if err == nil { if runConfig.denyTrafficRules { t.Fatalf("unexpected tunneled NTP request success") } } else { if !runConfig.denyTrafficRules { t.Fatalf("tunneled NTP request failed: %s", err) } } } // Test: await SLOK payload if !runConfig.denyTrafficRules { time.Sleep(1 * time.Second) waitOnNotification(t, slokSeeded, timeoutSignal, "SLOK seeded timeout exceeded") } }
func TestObfuscatedRemoteServerLists(t *testing.T) { testDataDirName, err := ioutil.TempDir("", "psiphon-remote-server-list-test") if err != nil { t.Fatalf("TempDir failed: %s", err) } defer os.RemoveAll(testDataDirName) // // create a server // serverIPaddress := "" for _, interfaceName := range []string{"eth0", "en0"} { serverIPaddress, err = common.GetInterfaceIPAddress(interfaceName) if err == nil { break } } if err != nil { t.Fatalf("error getting server IP address: %s", err) } serverConfigJSON, _, encodedServerEntry, err := server.GenerateConfig( &server.GenerateConfigParams{ ServerIPAddress: serverIPaddress, EnableSSHAPIRequests: true, WebServerPort: 8001, TunnelProtocolPorts: map[string]int{"OSSH": 4001}, }) if err != nil { t.Fatalf("error generating server config: %s", err) } // // pave OSLs // oslConfigJSONTemplate := ` { "Schemes" : [ { "Epoch" : "%s", "Regions" : [], "PropagationChannelIDs" : ["%s"], "MasterKey" : "vwab2WY3eNyMBpyFVPtsivMxF4MOpNHM/T7rHJIXctg=", "SeedSpecs" : [ { "ID" : "KuP2V6gLcROIFzb/27fUVu4SxtEfm2omUoISlrWv1mA=", "UpstreamSubnets" : ["0.0.0.0/0"], "Targets" : { "BytesRead" : 1, "BytesWritten" : 1, "PortForwardDurationNanoseconds" : 1 } } ], "SeedSpecThreshold" : 1, "SeedPeriodNanoseconds" : %d, "SeedPeriodKeySplits": [ { "Total": 1, "Threshold": 1 } ] } ] }` now := time.Now().UTC() seedPeriod := 24 * time.Hour epoch := now.Truncate(seedPeriod) epochStr := epoch.Format(time.RFC3339Nano) propagationChannelID, _ := common.MakeRandomStringHex(8) oslConfigJSON := fmt.Sprintf( oslConfigJSONTemplate, epochStr, propagationChannelID, seedPeriod) oslConfig, err := osl.LoadConfig([]byte(oslConfigJSON)) if err != nil { t.Fatalf("error loading OSL config: %s", err) } signingPublicKey, signingPrivateKey, err := common.GenerateAuthenticatedDataPackageKeys() if err != nil { t.Fatalf("error generating package keys: %s", err) } paveFiles, err := oslConfig.Pave( epoch, propagationChannelID, signingPublicKey, signingPrivateKey, []map[time.Time]string{ map[time.Time]string{ epoch: string(encodedServerEntry), }, }) if err != nil { t.Fatalf("error paving OSL files: %s", err) } // // mock seeding SLOKs // singleton = dataStore{} os.Remove(filepath.Join(testDataDirName, DATA_STORE_FILENAME)) err = InitDataStore(&Config{DataStoreDirectory: testDataDirName}) if err != nil { t.Fatalf("error initializing client datastore: %s", err) } if CountServerEntries("", "") > 0 { t.Fatalf("unexpected server entries") } seedState := oslConfig.NewClientSeedState("", propagationChannelID, nil) seedPortForward := seedState.NewClientSeedPortForward(net.ParseIP("0.0.0.0")) seedPortForward.UpdateProgress(1, 1, 1) payload := seedState.GetSeedPayload() if len(payload.SLOKs) != 1 { t.Fatalf("expected 1 SLOKs, got %d", len(payload.SLOKs)) } SetSLOK(payload.SLOKs[0].ID, payload.SLOKs[0].Key) // // run mock remote server list host // remoteServerListHostAddress := net.JoinHostPort(serverIPaddress, "8081") // The common remote server list fetches will 404 remoteServerListURL := fmt.Sprintf("http://%s/server_list_compressed", remoteServerListHostAddress) remoteServerListDownloadFilename := filepath.Join(testDataDirName, "server_list_compressed") obfuscatedServerListRootURL := fmt.Sprintf("http://%s/", remoteServerListHostAddress) obfuscatedServerListDownloadDirectory := testDataDirName go func() { startTime := time.Now() serveMux := http.NewServeMux() for _, paveFile := range paveFiles { file := paveFile serveMux.HandleFunc("/"+file.Name, func(w http.ResponseWriter, req *http.Request) { md5sum := md5.Sum(file.Contents) w.Header().Add("Content-Type", "application/octet-stream") w.Header().Add("ETag", hex.EncodeToString(md5sum[:])) http.ServeContent(w, req, file.Name, startTime, bytes.NewReader(file.Contents)) }) } httpServer := &http.Server{ Addr: remoteServerListHostAddress, Handler: serveMux, } err := httpServer.ListenAndServe() if err != nil { // TODO: wrong goroutine for t.FatalNow() t.Fatalf("error running remote server list host: %s", err) } }() // // run Psiphon server // go func() { err := server.RunServices(serverConfigJSON) if err != nil { // TODO: wrong goroutine for t.FatalNow() t.Fatalf("error running server: %s", err) } }() // // disrupt remote server list downloads // disruptorProxyAddress := "127.0.0.1:2162" disruptorProxyURL := "socks4a://" + disruptorProxyAddress go func() { listener, err := socks.ListenSocks("tcp", disruptorProxyAddress) if err != nil { fmt.Errorf("disruptor proxy listen error: %s", err) return } for { localConn, err := listener.AcceptSocks() if err != nil { fmt.Errorf("disruptor proxy accept error: %s", err) return } go func() { remoteConn, err := net.Dial("tcp", localConn.Req.Target) if err != nil { fmt.Errorf("disruptor proxy dial error: %s", err) return } err = localConn.Grant(&net.TCPAddr{IP: net.ParseIP("0.0.0.0"), Port: 0}) if err != nil { fmt.Errorf("disruptor proxy grant error: %s", err) return } waitGroup := new(sync.WaitGroup) waitGroup.Add(1) go func() { defer waitGroup.Done() io.Copy(remoteConn, localConn) }() if localConn.Req.Target == remoteServerListHostAddress { io.CopyN(localConn, remoteConn, 500) } else { io.Copy(localConn, remoteConn) } localConn.Close() remoteConn.Close() waitGroup.Wait() }() } }() // // connect to Psiphon server with Psiphon client // SetEmitDiagnosticNotices(true) // Note: calling LoadConfig ensures all *int config fields are initialized clientConfigJSONTemplate := ` { "ClientPlatform" : "", "ClientVersion" : "0", "SponsorId" : "0", "PropagationChannelId" : "0", "ConnectionPoolSize" : 1, "EstablishTunnelPausePeriodSeconds" : 1, "FetchRemoteServerListRetryPeriodSeconds" : 1, "RemoteServerListSignaturePublicKey" : "%s", "RemoteServerListUrl" : "%s", "RemoteServerListDownloadFilename" : "%s", "ObfuscatedServerListRootURL" : "%s", "ObfuscatedServerListDownloadDirectory" : "%s", "UpstreamProxyUrl" : "%s" }` clientConfigJSON := fmt.Sprintf( clientConfigJSONTemplate, signingPublicKey, remoteServerListURL, remoteServerListDownloadFilename, obfuscatedServerListRootURL, obfuscatedServerListDownloadDirectory, disruptorProxyURL) clientConfig, _ := LoadConfig([]byte(clientConfigJSON)) controller, err := NewController(clientConfig) if err != nil { t.Fatalf("error creating client controller: %s", err) } tunnelEstablished := make(chan struct{}, 1) SetNoticeOutput(NewNoticeReceiver( func(notice []byte) { noticeType, payload, err := GetNotice(notice) if err != nil { return } printNotice := false switch noticeType { case "Tunnels": printNotice = true count := int(payload["count"].(float64)) if count == 1 { tunnelEstablished <- *new(struct{}) } case "RemoteServerListResourceDownloadedBytes": // TODO: check for resumed download for each URL //url := payload["url"].(string) printNotice = true case "RemoteServerListResourceDownloaded": printNotice = true } if printNotice { fmt.Printf("%s\n", string(notice)) } })) go func() { controller.Run(make(chan struct{})) }() establishTimeout := time.NewTimer(30 * time.Second) select { case <-tunnelEstablished: case <-establishTimeout.C: t.Fatalf("tunnel establish timeout exceeded") } for _, paveFile := range paveFiles { u, _ := url.Parse(obfuscatedServerListRootURL) u.Path = path.Join(u.Path, paveFile.Name) etag, _ := GetUrlETag(u.String()) md5sum := md5.Sum(paveFile.Contents) if etag != hex.EncodeToString(md5sum[:]) { t.Fatalf("unexpected ETag for %s", u) } } }