// Run the application, start http and scp server.
func Run(appConfig Config) {
config = appConfig
// Connect to DB
db, err := lib.Connect()
if err != nil {
fmt.Printf("Failed to connect to db")
return
}
// Logging
log := logrus.New()
log.Level = logrus.DebugLevel
log.Out = os.Stdout
log.Formatter = &logrus.TextFormatter{}
// Websockets
ws := ws.NewServer()
// Shared dependencies between all controller
deps := dependencies.Dependencies{
Fs: afero.NewOsFs(),
Logger: log,
DB: db,
WS: ws,
}
ws.Dependencies = &deps
go ws.Start()
// // Start SCP
// scp := scp.Server{}
// scp.DB = deps.DB
// scp.Logger = deps.Logger
// scp.CertPath = "certs/scp.rsa"
// scp.BindAddr = config.SCPBindAddr
// go scp.ListenAndServe()
if config.Secure {
c := autocert.DirCache("certs")
m := autocert.Manager{
Cache: c,
Prompt: autocert.AcceptTOS,
HostPolicy: autocert.HostWhitelist("x.zqz.ca"),
}
s := &http.Server{
Addr: config.HTTPBindAddr,
TLSConfig: &tls.Config{GetCertificate: m.GetCertificate},
}
deps.Info("Listening for HTTP1.1 Connections", "addr", ":3001")
deps.Info("Listening for HTTP2 Connections", "addr", config.HTTPBindAddr)
go http.ListenAndServe(":3001", secureRedirect())
s.ListenAndServeTLS("", "")
} else {
deps.Info("Listening for HTTP1.1 Connections", "addr", config.HTTPBindAddr)
http.ListenAndServe(config.HTTPBindAddr, Routes(deps))
}
}
// LETSENCRYPTPROD returns a new Automatic TLS Listener using letsencrypt.org service
// receives two parameters, the first is the domain of the server
// and the second is optionally, the cache directory, if you skip it then the cache directory is "./certcache"
// if you want to disable cache directory then simple give it a value of empty string ""
//
// does NOT supports localhost domains for testing, use LETSENCRYPT instead.
//
// this is the recommended function to use when you're ready for production state
func LETSENCRYPTPROD(addr string, cacheDirOptional ...string) (net.Listener, error) {
if portIdx := strings.IndexByte(addr, ':'); portIdx == -1 {
addr += ":443"
}
ln, err := TCP4(addr)
if err != nil {
return nil, err
}
cacheDir := "./certcache"
if len(cacheDirOptional) > 0 {
cacheDir = cacheDirOptional[0]
}
m := autocert.Manager{
Prompt: autocert.AcceptTOS,
} // HostPolicy is missing, if user wants it, then she/he should manually
// configure the autocertmanager and use the `iris.Serve` to pass that listener
if cacheDir == "" {
// then the user passed empty by own will, then I guess she/he doesnt' want any cache directory
} else {
m.Cache = autocert.DirCache(cacheDir)
}
tlsConfig := &tls.Config{GetCertificate: m.GetCertificate}
tlsLn := tls.NewListener(ln, tlsConfig)
return tlsLn, nil
}
func serveHTTPS(httpServer *http.Server) error {
log.Printf("Starting TLS server on %s", *httpsAddr)
httpsServer := new(http.Server)
*httpsServer = *httpServer
httpsServer.Addr = *httpsAddr
cacheDir := autocert.DirCache("letsencrypt.cache")
var domain string
if !inProd {
if *tlsCertFile != "" && *tlsKeyFile != "" {
return httpsServer.ListenAndServeTLS(*tlsCertFile, *tlsKeyFile)
}
// Otherwise use Let's Encrypt, i.e. same use case as in prod
if strings.HasPrefix(*httpsAddr, ":") {
return errors.New("for Let's Encrypt, -https needs to start with a host name")
}
host, _, err := net.SplitHostPort(*httpsAddr)
if err != nil {
return err
}
domain = host
} else {
domain = "camlistore.org"
cacheDir = autocert.DirCache(prodLECacheDir)
}
m := autocert.Manager{
Prompt: autocert.AcceptTOS,
HostPolicy: autocert.HostWhitelist(domain),
Cache: cacheDir,
}
if *adminEmail != "" {
m.Email = *adminEmail
}
httpsServer.TLSConfig = &tls.Config{
GetCertificate: m.GetCertificate,
}
log.Printf("Listening for HTTPS on %v", *httpsAddr)
ln, err := net.Listen("tcp", *httpsAddr)
if err != nil {
return err
}
return httpsServer.Serve(tls.NewListener(tcpKeepAliveListener{ln.(*net.TCPListener)}, httpsServer.TLSConfig))
}
func main() {
flag.Parse()
log.Printf("Starting (prod=%t)", *prod)
client := &acme.Client{}
if !*prod {
client = &acme.Client{
DirectoryURL: "https://acme-staging.api.letsencrypt.org/directory",
}
}
m := autocert.Manager{
Prompt: autocert.AcceptTOS,
HostPolicy: autocert.HostWhitelist("www.homedroids.io"),
Cache: autocert.DirCache("certs"),
Client: client,
}
s := &http.Server{
Addr: *httpsAddr,
TLSConfig: &tls.Config{GetCertificate: m.GetCertificate},
}
log.Fatal(s.ListenAndServeTLS("", ""))
}
func serveProdTLS() error {
const cacheDir = "/var/cache/autocert"
if err := os.MkdirAll(cacheDir, 0700); err != nil {
return err
}
m := autocert.Manager{
Cache: autocert.DirCache(cacheDir),
Prompt: autocert.AcceptTOS,
HostPolicy: autocert.HostWhitelist("http2.golang.org"),
}
srv := &http.Server{
TLSConfig: &tls.Config{
GetCertificate: m.GetCertificate,
},
}
http2.ConfigureServer(srv, &http2.Server{})
ln, err := net.Listen("tcp", ":443")
if err != nil {
return err
}
return srv.Serve(tls.NewListener(tcpKeepAliveListener{ln.(*net.TCPListener)}, srv.TLSConfig))
}
func main() {
log.SetLevel(log.DebugLevel)
log.SetFlags(0)
// выводим информацию о версии сборки
log.WithFields(log.Fields{
"version": version,
"date": date,
"build": build,
"name": appName,
}).Info("starting service")
// разбираем параметры запуска приложения
config := flag.String("config", appName+".json", "configuration `filename`")
address := flag.String("address", host, "server address and `port`")
flag.Parse()
// загружаем конфигурацию сервиса
log.WithField("file", *config).Info("loading service config")
service, err := LoadConfig(*config)
if err != nil {
log.WithError(err).Error("loading config error")
os.Exit(1)
}
// выводим список поддерживаемых MX-серверов
mxnames := service.Auth.Names()
if len(mxnames) == 0 {
log.Error("no MX servers configured")
os.Exit(1)
}
log.WithField("names", strings.Join(mxnames, ", ")).
Info("supported MX servers")
// выводим информацию о кеше авторизации
log.WithField("lifeTime", mxlogin.CacheLifetime).
Info("authorization cache")
// удаляем устаревшие файлы раз в сутки
cleanInterval := time.Hour * 24
// выводим информацию о хранилище файлов
log.WithFields(log.Fields{
"path": service.FileStore.Store.Path(),
"lifeTime": service.FileStore.Lifetime,
"cleanInterval": cleanInterval,
}).Info("file store")
// запускаем процесс автоматического удаления старых файлов
go func() {
for {
time.Sleep(cleanInterval)
err = service.FileStore.Clean()
if err != nil {
log.WithField("service", "filestore").
WithError(err).Debug("file store clean error")
}
}
}()
// инициализируем мультиплексор HTTP-запросов
mux := &rest.ServeMux{
Headers: map[string]string{
"Server": "MXStore/2.0",
"X-API-Version": "1.1",
"X-Service-Version": version,
},
Logger: log.WithField("service", "http"),
Encoder: Encoder,
}
// обработчики файлов хранилища и аватарок
mux.Handles(rest.Paths{
"/mx/:mx-name/store": {
// переходим на основное имя MX сервера, если используется синоним
// возвращает пустой ответ, если имя сервера и так основное
"GET": nil,
// сохраняем в хранилище новый файл
"POST": service.FileStore.Save,
},
// отдаем файл по запросу
"/mx/:mx-name/store/*filename": {
"GET": service.FileStore.Get,
},
"/mx/:mx-name/avatar": {
// переходим на основное имя MX сервера, если используется синоним
// возвращаем список идентификаторов пользователей и аватарок
"GET": service.Avatars.List,
// сохраняем в хранилище новый файл
"POST": service.Avatars.Save,
},
// отдаем файл с аватаркой по запросу
"/mx/:mx-name/avatar/*filename": {
"GET": service.Avatars.Get,
},
},
// для всех запросов требуется авторизация
service.Auth.Authorize,
)
// добавляем обработчик отдачи документации
mux.Handle("GET", "/",
rest.Redirect("https://www.connector73.com/en#softphone"))
// запускаем сервис с ответом об информации о ссылке
mux.Handle("GET", "/urlinfo/*url", URLInfo)
// генератор аватарок
mux.Handle("GET", "/avatar/:id", AvatarGenerator)
// инициализируем HTTP-сервер
server := &http.Server{
Addr: *address,
Handler: mux,
ReadTimeout: time.Second * 60,
WriteTimeout: time.Second * 120,
}
// для защищенного соединения проделываем дополнительные настройки
host, port, err := net.SplitHostPort(*address)
if err != nil {
log.WithError(err).Error("bad server address")
os.Exit(2)
}
if port == "https" || port == "443" {
if host != "localhost" && host != "127.0.0.1" {
manager := autocert.Manager{
Prompt: autocert.AcceptTOS,
HostPolicy: autocert.HostWhitelist(host),
Email: "*****@*****.**",
Cache: autocert.DirCache("letsEncript.cache"),
}
server.TLSConfig = &tls.Config{
GetCertificate: manager.GetCertificate,
}
} else {
// исключительно для отладки
cert, err := tls.X509KeyPair(LocalhostCert, LocalhostKey)
if err != nil {
panic(fmt.Sprintf("local certificates error: %v", err))
}
server.TLSConfig = &tls.Config{
Certificates: []tls.Certificate{cert},
}
}
// запускаем автоматический переход для HTTP на HTTPS
go func() {
log.Info("starting http to https redirect server")
err := http.ListenAndServe(":http", http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r,
"https://"+r.Host+r.URL.String(),
http.StatusMovedPermanently)
}))
if err != nil {
log.WithError(err).Warning("http redirect server error")
}
}()
// запускаем основной сервер
go func() {
log.WithFields(log.Fields{
"address": server.Addr,
"host": host,
}).Info("starting https server")
err = server.ListenAndServeTLS("", "")
// // корректно закрываем сервисы по окончании работы
// service.Close()
log.WithError(err).Warning("https server stoped")
os.Exit(3)
}()
} else {
// не защищенный HTTP сервер
go func() {
log.WithField("address", *address).Info("starting service")
err = server.ListenAndServe()
// service.Close()
log.WithError(err).Warning("http server stoped")
os.Exit(3)
}()
}
// инициализируем поддержку системных сигналов и ждем, когда он случится
monitorSignals(os.Interrupt, os.Kill)
// service.Close()
log.Info("service stoped")
}
// If cert/key files are specified, and found, use them.
// If cert/key files are specified, not found, and the default values, generate
// them (self-signed CA used as a cert), and use them.
// If cert/key files are not specified, use Let's Encrypt.
func setupTLS(ws *webserver.Server, config *serverinit.Config, hostname string) {
cert, key := config.OptionalString("httpsCert", ""), config.OptionalString("httpsKey", "")
if !config.OptionalBool("https", true) {
return
}
if (cert != "") != (key != "") {
exitf("httpsCert and httpsKey must both be either present or absent")
}
defCert := osutil.DefaultTLSCert()
defKey := osutil.DefaultTLSKey()
const hint = "You must add this certificate's fingerprint to your client's trusted certs list to use it. Like so:\n\"trustedCerts\": [\"%s\"],"
if cert == defCert && key == defKey {
_, err1 := wkfs.Stat(cert)
_, err2 := wkfs.Stat(key)
if err1 != nil || err2 != nil {
if os.IsNotExist(err1) || os.IsNotExist(err2) {
sig, err := httputil.GenSelfTLSFiles(hostname, defCert, defKey)
if err != nil {
exitf("Could not generate self-signed TLS cert: %q", err)
}
log.Printf(hint, sig)
} else {
exitf("Could not stat cert or key: %q, %q", err1, err2)
}
}
}
if cert == "" && key == "" {
// Use Let's Encrypt if no files are specified, and we have a usable hostname.
if netutil.IsFQDN(hostname) {
m := autocert.Manager{
Prompt: autocert.AcceptTOS,
HostPolicy: autocert.HostWhitelist(hostname),
Cache: autocert.DirCache(osutil.DefaultLetsEncryptCache()),
}
log.Print("TLS enabled, with Let's Encrypt")
ws.SetTLS(webserver.TLSSetup{
CertManager: m.GetCertificate,
})
return
}
// Otherwise generate new certificates
sig, err := httputil.GenSelfTLSFiles(hostname, defCert, defKey)
if err != nil {
exitf("Could not generate self signed creds: %q", err)
}
log.Printf(hint, sig)
cert = defCert
key = defKey
}
data, err := wkfs.ReadFile(cert)
if err != nil {
exitf("Failed to read pem certificate: %s", err)
}
sig, err := httputil.CertFingerprint(data)
if err != nil {
exitf("certificate error: %v", err)
}
log.Printf("TLS enabled, with SHA-256 certificate fingerprint: %v", sig)
ws.SetTLS(webserver.TLSSetup{
CertFile: cert,
KeyFile: key,
})
}
func (r *Runner) start() error {
r.authKey = os.Getenv("AUTH_KEY")
if r.authKey == "" {
return errors.New("AUTH_KEY not set")
}
r.runEnv["TEST_RUNNER_AUTH_KEY"] = r.authKey
for _, s := range []string{"S3", "GCS", "AZURE"} {
name := fmt.Sprintf("BLOBSTORE_%s_CONFIG", s)
if c := os.Getenv(name); c != "" {
r.runEnv[name] = c
} else {
return fmt.Errorf("%s not set", name)
}
}
r.githubToken = os.Getenv("GITHUB_TOKEN")
if r.githubToken == "" {
return errors.New("GITHUB_TOKEN not set")
}
am := autocert.Manager{
Prompt: autocert.AcceptTOS,
Cache: autocert.DirCache(args.TLSDir),
HostPolicy: autocert.HostWhitelist(args.Domain),
}
awsAuth, err := aws.EnvCreds()
if err != nil {
return err
}
r.s3 = s3.New(awsAuth, "us-east-1", nil)
_, listenPort, err = net.SplitHostPort(args.ListenAddr)
if err != nil {
return err
}
bc := r.bc
bc.Network = r.allocateNet()
if r.rootFS, err = cluster.BuildFlynn(bc, args.RootFS, "origin/master", false, os.Stdout); err != nil {
return fmt.Errorf("could not build flynn: %s", err)
}
r.releaseNet(bc.Network)
shutdown.BeforeExit(func() { removeRootFS(r.rootFS) })
db, err := bolt.Open(args.DBPath, 0600, &bolt.Options{Timeout: 5 * time.Second})
if err != nil {
return fmt.Errorf("could not open db: %s", err)
}
r.db = db
shutdown.BeforeExit(func() { r.db.Close() })
if err := r.db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists(dbBucket)
return err
}); err != nil {
return fmt.Errorf("could not create builds bucket: %s", err)
}
for i := 0; i < args.ConcurrentBuilds; i++ {
r.buildCh <- struct{}{}
}
if err := r.buildPending(); err != nil {
log.Printf("could not build pending builds: %s", err)
}
go r.connectIRC()
go r.watchEvents()
router := httprouter.New()
router.RedirectTrailingSlash = true
router.Handler("GET", "/", http.RedirectHandler("/builds", 302))
router.POST("/", r.handleEvent)
router.GET("/builds/:build", r.getBuildLog)
router.GET("/builds/:build/download", r.downloadBuildLog)
router.POST("/builds/:build/restart", r.restartBuild)
router.POST("/builds/:build/explain", r.explainBuild)
router.GET("/builds", r.getBuilds)
router.ServeFiles("/assets/*filepath", http.Dir(args.AssetsDir))
router.GET("/cluster/:cluster", r.clusterAPI(r.getCluster))
router.POST("/cluster/:cluster", r.clusterAPI(r.addHost))
router.POST("/cluster/:cluster/release", r.clusterAPI(r.addReleaseHosts))
router.DELETE("/cluster/:cluster/:host", r.clusterAPI(r.removeHost))
srv := &http.Server{
Addr: args.ListenAddr,
Handler: router,
TLSConfig: tlsconfig.SecureCiphers(&tls.Config{
GetCertificate: am.GetCertificate,
}),
}
log.Println("Listening on", args.ListenAddr, "...")
if err := srv.ListenAndServeTLS("", ""); err != nil {
return fmt.Errorf("ListenAndServeTLS: %s", err)
}
return nil
}
func main() {
launchConfig.MaybeDeploy()
flag.Parse()
var kv keyValue
var httpsListenAddr string
if metadata.OnGCE() {
httpsListenAddr = ":443"
dsClient, err := datastore.NewClient(context.Background(), GCEProjectID)
if err != nil {
log.Fatalf("Error creating datastore client for records: %v", err)
}
kv = cachedStore{
dsClient: dsClient,
cache: lru.New(cacheSize),
}
} else {
httpsListenAddr = ":4430"
kv = memkv{skv: sorted.NewMemoryKeyValue()}
}
if err := kv.Set("6401800c.camlistore.net.", "159.203.246.79"); err != nil {
log.Fatalf("Error adding %v:%v record: %v", "6401800c.camlistore.net.", "159.203.246.79", err)
}
if err := kv.Set(domain, *flagServerIP); err != nil {
log.Fatalf("Error adding %v:%v record: %v", domain, *flagServerIP, err)
}
if err := kv.Set("www.camlistore.net.", *flagServerIP); err != nil {
log.Fatalf("Error adding %v:%v record: %v", "www.camlistore.net.", *flagServerIP, err)
}
ds := newDNSServer(kv)
cs := &gpgchallenge.Server{
OnSuccess: func(identity string, address string) error {
log.Printf("Adding %v.camlistore.net. as %v", identity, address)
return ds.dataSource.Set(strings.ToLower(identity+".camlistore.net."), address)
},
}
tcperr := make(chan error, 1)
udperr := make(chan error, 1)
httperr := make(chan error, 1)
log.Printf("serving DNS on %s\n", *addr)
go func() {
tcperr <- dns.ListenAndServe(*addr, "tcp", ds)
}()
go func() {
udperr <- dns.ListenAndServe(*addr, "udp", ds)
}()
if metadata.OnGCE() {
// TODO(mpl): if we want to get a cert for anything
// *.camlistore.net, it's a bit of a chicken and egg problem, since
// we need camnetdns itself to be already running and answering DNS
// queries. It's probably doable, but easier for now to just ask
// one for camnetdns.camlistore.org, since that name is not
// resolved by camnetdns.
hostname := strings.TrimSuffix(authorityNS, ".")
m := autocert.Manager{
Prompt: autocert.AcceptTOS,
HostPolicy: autocert.HostWhitelist(hostname),
Cache: autocert.DirCache(osutil.DefaultLetsEncryptCache()),
}
ln, err := tls.Listen("tcp", httpsListenAddr, &tls.Config{
Rand: rand.Reader,
Time: time.Now,
NextProtos: []string{http2.NextProtoTLS, "http/1.1"},
MinVersion: tls.VersionTLS12,
GetCertificate: m.GetCertificate,
})
if err != nil {
log.Fatalf("Error listening on %v: %v", httpsListenAddr, err)
}
go func() {
httperr <- http.Serve(ln, cs)
}()
}
select {
case err := <-tcperr:
log.Fatalf("DNS over TCP error: %v", err)
case err := <-udperr:
log.Fatalf("DNS error: %v", err)
case err := <-httperr:
log.Fatalf("HTTP server error: %v", err)
}
}
func (d roCertCacheDir) Get(ctx context.Context, key string) ([]byte, error) {
return autocert.DirCache(d).Get(ctx, key)
}