Beispiel #1
0
func main() {
	// STARTINIT OMIT
	me := "http://10.0.0.1"
	peers := groupcache.NewHTTPPool(me)

	// Whenever peers change:
	peers.Set("http://10.0.0.1", "http://10.0.0.2", "http://10.0.0.3")
	// ENDINIT OMIT

	// STARTGROUP OMIT
	var thumbNails = groupcache.NewGroup("thumbnail", 64<<20, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			fileName := key
			dest.SetBytes(generateThumbnail(fileName))
			return nil
		}))
	// ENDGROUP OMIT

	var ctx groupcache.Context
	var w http.ResponseWriter
	var r *http.Request

	// STARTUSE OMIT
	var data []byte
	err := thumbNails.Get(ctx, "big-file.jpg",
		groupcache.AllocatingByteSliceSink(&data))
	// ...
	_ = err               // OMIT
	var modTime time.Time // OMIT
	http.ServeContent(w, r, "big-file-thumb.jpg", modTime, bytes.NewReader(data))
	// ENDUSE OMIT
}
Beispiel #2
0
Datei: main.go Projekt: qmdx/mygo
func main() {
	flag.Parse()

	peers := gc.NewHTTPPool("http://localhost:" + *port)
	peers.Set("http://localhost:8001", "http://localhost:8002")

	cg = gc.NewGroup("QRCache", 1<<20, gc.GetterFunc(
		func(ctx gc.Context, key string, dest gc.Sink) error {
			fmt.Printf("asking for data of %s\n", key)
			url := fmt.Sprintf("http://chart.apis.google.com/chart?chs=300x300&cht=qr&choe=UTF-8&chl=%s", key)
			resp, err := http.Get(url)
			if err != nil {
				return nil
			}
			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return nil
			}
			value := base64.StdEncoding.EncodeToString(body)
			dest.SetBytes([]byte(value))
			return nil
		}))

	// run groupcache process in goroutine
	go http.ListenAndServe("localhost:"+*port, peers)

	http.Handle("/", http.HandlerFunc(QR))
	err := http.ListenAndServe(*addr, nil)
	if err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}
Beispiel #3
0
func main() {
	flag.Parse()

	// Setup the doozer connection.
	d, err := doozer.Dial(daddr)
	if err != nil {
		log.Fatalf("connecting to doozer: %v\n", err)
	}
	defer d.Close()

	// Setup the cache.
	pool = groupcache.NewHTTPPool("http://" + addr)
	dict = groupcache.NewGroup("dict", 64<<20, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			def, err := query(key)
			if err != nil {
				err = fmt.Errorf("querying remote dictionary: %v", err)
				log.Println(err)
				return err
			}

			log.Println("retrieved remote definition for", key)
			dest.SetString(def)
			return nil
		}))

	// Start watching for changes and signals.
	go watch(d)

	// Add the handler for definition requests and then start the
	// server.
	http.Handle("/define/", http.HandlerFunc(handler))
	log.Println(http.ListenAndServe(addr, nil))
}
Beispiel #4
0
func main() {
	me := ":" + os.Args[1]
	peers := groupcache.NewHTTPPool("http://localhost" + me)
	peers.Set("http://localhost:8081", "http://localhost:8082",
		"http://localhost:8083")
	helloworld := groupcache.NewGroup("helloworld",
		1024*1024*10, groupcache.GetterFunc(
			func(ctx groupcache.Context,
				key string,
				dest groupcache.Sink) error {
				log.Println(me)
				dest.SetString(me + "->" + key)
				return nil
			}))
	fmt.Println("GroupName: ", helloworld.Name())
	http.HandleFunc("/xbox/", func(w http.ResponseWriter, r *http.Request) {
		parts := strings.SplitN(r.URL.Path[len("/xbox/"):], "/", 1)
		for _, x := range parts {
			fmt.Println("get: ", x)
		}
		if len(parts) != 1 {
			http.Error(w, "Bad Request", http.StatusBadRequest)
			return
		}
		var data []byte
		helloworld.Get(nil, parts[0], groupcache.AllocatingByteSliceSink(&data))
		w.Write(data)
		log.Println("Gets: ", helloworld.Stats.Gets.String())
		log.Println("Load: ", helloworld.Stats.Loads.String())
		log.Println("LocalLoad: ", helloworld.Stats.LocalLoads.String())
		log.Println("PeerError: ", helloworld.Stats.PeerErrors.String())
		log.Println("PeerLoad: ", helloworld.Stats.PeerLoads.String())
	})
	http.ListenAndServe(me, nil)
}
Beispiel #5
0
func Pool(ec2conn *ec2.EC2) *EC2CachePool {
	localip := getLocalIP(ec2conn)

	peers := &EC2CachePool{
		groupcache.NewHTTPPool(fmt.Sprintf("http://%s:%s", localip, *cacheport)),
		ec2conn,
	}

	peerAddrs, err := peers.discoverPeers()
	if err != nil {
		log.Fatal("discovery:", err)
	}

	log.Println("Alerting peers:", peerAddrs)

	// Inform each peer that they should also discover peers
	for _, peer := range peerAddrs {
		client, err := rpc.DialHTTP("tcp", fmt.Sprintf("%s:%s", peer, *rpcport))
		if err != nil {
			log.Println("error dialing:", err)
			continue
		}

		var reply int
		err = client.Call("EC2CachePool.RefreshPeers", new(Args), &reply)
		if err != nil {
			log.Fatal("refresh rpc failure:", err)
		}

		log.Println("Refreshed peers on:", peer)
	}

	return peers
}
Beispiel #6
0
func main() {
	// STARTINIT OMIT
	me := "http://127.0.0.1:11211"
	peers := groupcache.NewHTTPPool(me)

	// Whenever peers change:
	peers.Set("http://127.0.0.1:11211")
	// ENDINIT OMIT

	// STARTGROUP OMIT
	var thumbNails = groupcache.NewGroup("thumbnail", 64<<20, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			dest.SetBytes(generateThumbnail(key))
			return nil
		}))
	// ENDGROUP OMIT

	var ctx groupcache.Context

	// STARTUSE OMIT
	var data []byte
	thumbNails.Get(ctx, "big-file.jpg", groupcache.AllocatingByteSliceSink(&data))

	fmt.Println(string(data))
}
Beispiel #7
0
func DebugPool() *DebugCachePool {
	peers := &DebugCachePool{
		groupcache.NewHTTPPool("http://localhost:9001"),
	}

	peers.Set("http://localhost:9001")

	return peers
}
Beispiel #8
0
func setupGroupcache(config GroupcacheConfig) error {
	if config.GB == 0 {
		return nil
	}
	var cacheBytes int64
	cacheBytes = int64(config.GB) << 30

	pool := groupcache.NewHTTPPool(config.Host)
	if pool != nil {
		dvid.Infof("Initializing groupcache with %d GB at %s...\n", config.GB, config.Host)
		manager.gcache.cache = groupcache.NewGroup("immutable", cacheBytes, groupcache.GetterFunc(
			func(c groupcache.Context, key string, dest groupcache.Sink) error {
				// Use KeyValueDB defined as context.
				gctx, ok := c.(GroupcacheCtx)
				if !ok {
					return fmt.Errorf("bad groupcache context: expected GroupcacheCtx, got %v", c)
				}

				// First four bytes of key is instance ID to isolate groupcache collisions.
				tk := TKey(key[4:])
				data, err := gctx.KeyValueDB.Get(gctx.Context, tk)
				if err != nil {
					return err
				}
				return dest.SetBytes(data)
			}))
		manager.gcache.supported = make(map[dvid.DataSpecifier]struct{})
		for _, dataspec := range config.Instances {
			name := strings.Trim(dataspec, "\"")
			parts := strings.Split(name, ":")
			switch len(parts) {
			case 2:
				dataid := dvid.GetDataSpecifier(dvid.InstanceName(parts[0]), dvid.UUID(parts[1]))
				manager.gcache.supported[dataid] = struct{}{}
			default:
				dvid.Errorf("bad data instance specification %q given for groupcache support in config file\n", dataspec)
			}
		}

		// If we have additional peers, add them and start a listener via the HTTP port.
		if len(config.Peers) > 0 {
			peers := []string{config.Host}
			peers = append(peers, config.Peers...)
			pool.Set(peers...)
			dvid.Infof("Groupcache configuration has %d peers in addition to local host.\n", len(config.Peers))
			dvid.Infof("Starting groupcache HTTP server on %s\n", config.Host)
			http.ListenAndServe(config.Host, http.HandlerFunc(pool.ServeHTTP))
		}
	}
	return nil
}
Beispiel #9
0
func main() {
	var err error
	p = groupcache.NewHTTPPool("http://127.0.0.1:50000")
	// update p's list
	p.Set("http://127.0.0.1:50001", "http://127.0.0.1:50002")

	// create a new group by name and set a function for get a value by key
	// if you g.Get KEY at first time, it will store the value use sink, run the function
	// if you g.Get KEY not at first time, actually it NOT entry the function
	// does not support update a key's value
	g = groupcache.NewGroup("dns", 64<<20, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) (err error) {
			if ctx == nil {
				ctx = "f**k"
			}
			log.Println(key, ctx)
			err = dest.SetString(ctx.(string))
			return
		}))

	// let it listen
	go http.ListenAndServe("127.0.0.1:50000", nil)

	// get a key's value into data

	time.Sleep(2 * time.Second)
	var data []byte
	var key = "key"
	err = g.Get("aa", key, groupcache.AllocatingByteSliceSink(&data))
	if err != nil {
		log.Println("get error:", err)
	}
	log.Println(string(data))

	time.Sleep(2 * time.Second)
	key = "key1"
	err = g.Get("bb", key, groupcache.AllocatingByteSliceSink(&data))
	if err != nil {
		log.Println("get error:", err)
	}
	log.Println(string(data))

	time.Sleep(2 * time.Second)
	key = "key"
	err = g.Get("cc", key, groupcache.AllocatingByteSliceSink(&data))
	if err != nil {
		log.Println("get error:", err)
	}
	log.Println(string(data))
}
Beispiel #10
0
func initGroupcacheHTTPPool() {
	self := (&url.URL{Scheme: "http", Host: flagHTTP}).String()
	var peers []string
	peers = append(peers, self)
	for _, p := range strings.Split(flagGroupcachePeers, ",") {
		if p == "" {
			continue
		}
		peer := (&url.URL{Scheme: "http", Host: p}).String()
		peers = append(peers, peer)
	}
	pool := groupcache.NewHTTPPool(self)
	pool.Context = imageserver_cache_groupcache.HTTPPoolContext
	pool.Transport = imageserver_cache_groupcache.NewHTTPPoolTransport(nil)
	pool.Set(peers...)
}
Beispiel #11
0
func start() {
	pool = groupcache.NewHTTPPool("http://" + addr)

	dns = groupcache.NewGroup("dns", 64<<20, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			// Lookup the adddress based on the hostname (key).
			addrs, err := net.LookupHost(key)
			if err != nil {
				return err
			}

			// We'll just store the first.
			dest.SetString(addrs[0])
			return nil
		}))

	// Run in the background so we can use the console.
	go http.ListenAndServe(addr, nil)
}
Beispiel #12
0
func initCacheGroup() {
	self := config.Get().CacheSelf()

	pool := groupcache.NewHTTPPool(self)
	pool.Set(config.Get().CachePeers()...)

	if self != "" {
		log.Println("Cache listen on:", strings.TrimLeft(self, "http://"))
		go http.ListenAndServe(strings.TrimLeft(self, "http://"), http.HandlerFunc(pool.ServeHTTP))
	}

	cacheGroup = groupcache.NewGroup("imagio-storage", config.Get().CacheSize(), groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			dest.SetBytes(imgproc.Do(
				Construct(new(query.Options), key).(*query.Options),
			))
			return nil
		}),
	)
}
Beispiel #13
0
func main() {

	flag.Set("logtostderr", "true")
	flag.Parse()

	me := ":8080"
	peers := groupcache.NewHTTPPool("http://localhost" + me)
	peers.Set("http://localhost:8081", "http://localhost:8082", "http://localhost:8083")

	helloworld := groupcache.NewGroup("helloworld", 1024*1024*1024*16, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			glog.Infof("%v, key = %v", me, key)
			dest.SetString(key)
			return nil
		}))

	glog.Infof("GroupName: %v", helloworld.Name())
	http.HandleFunc("/xbox/",
		func(w http.ResponseWriter, r *http.Request) {
			parts := strings.SplitN(r.URL.Path[len("/xbox/"):], "/", 1)
			glog.Infof("parts: %v", parts)
			if len(parts) != 1 {
				http.Error(w, "Bad Request", http.StatusBadRequest)
				return
			}

			var data []byte
			helloworld.Get(nil, parts[0], groupcache.AllocatingByteSliceSink(&data))
			w.Write(data)

			glog.Infof("data: %s", data)
			glog.Infof("Stats: %#v", helloworld.Stats)
			//glog.Infof("Gets: %v", helloworld.Stats.Gets.String())
			//glog.Infof("Load: %v", helloworld.Stats.Loads.String())
			//glog.Infof("LocalLoad: %v", helloworld.Stats.LocalLoads.String())
			//glog.Infof("PeerError: %v", helloworld.Stats.PeerErrors.String())
			//glog.Infof("PeerLoad: %v", helloworld.Stats.PeerLoads.String())
		})

	http.ListenAndServe(me, nil)
}
Beispiel #14
0
func ServeDir(mountpoint string, target string, limit int, me string, peerlist []string) {

	peers := groupcache.NewHTTPPool(me)

	if peerlist != nil && len(peerlist) > 0 {
		peers.Set(peerlist...)
	}

	filecache = groupcache.NewGroup("filecache", int64(limit)<<20, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			contents, err := ioutil.ReadFile(key)
			dest.SetBytes(contents)
			return err
		}))

	c, err := fuse.Mount(mountpoint)
	if err != nil {
		log.Fatal(err)
	}

	fs.Serve(c, TargetDir{target})

}
func main() {

	var port = flag.String("port", "8001", "groupcache port")
	flag.Parse()

	peers := groupcache.NewHTTPPool("http://localhost:" + *port)

	client := new(client.Client)

	var stringcache = groupcache.NewGroup("SlowDBCache", 64<<20, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			result := client.Get(key)
			fmt.Printf("asking for %s from dbserver\n", key)
			dest.SetBytes([]byte(result))
			return nil
		}))

	peers.Set("http://localhost:8001", "http://localhost:8002", "http://localhost:8003")

	frontendServer := NewServer(stringcache)

	i, err := strconv.Atoi(*port)
	if err != nil {
		// handle error
		fmt.Println(err)
		os.Exit(2)
	}
	var frontEndport = ":" + strconv.Itoa(i+1000)
	go frontendServer.Start(frontEndport)

	fmt.Println(stringcache)
	fmt.Println("cachegroup slave starting on " + *port)
	fmt.Println("frontend starting on " + frontEndport)
	http.ListenAndServe("127.0.0.1:"+*port, http.HandlerFunc(peers.ServeHTTP))

}
Beispiel #16
0
func main() {

	configUri := filters.LookupConfigStoreURI("main.json")
	filename := "main.json"
	config, err := NewConfig(configUri, filename)
	if err != nil {
		fmt.Printf("NewConfig(%#v) failed: %s", filename, err)
		return
	}

	pidfile := ""
	flag.StringVar(&pidfile, "pidfile", "", "goproxy pidfile")
	flag.StringVar(&config.Addr, "addr", config.Addr, "goproxy listen address")
	flag.StringVar(&config.GroupCache.Addr, "groupcache-addr", config.GroupCache.Addr, "groupcache listen address")
	if config.LogToStderr || runtime.GOOS == "windows" {
		logToStderr := true
		for i := 1; i < len(os.Args); i++ {
			if strings.HasPrefix(os.Args[i], "-logtostderr=") {
				logToStderr = false
				break
			}
		}
		if logToStderr {
			flag.Set("logtostderr", "true")
		}
	}
	flag.Parse()

	if runtime.GOOS != "windows" && pidfile != "" {
		if err = ioutil.WriteFile(pidfile, []byte(strconv.Itoa(os.Getpid())), 0644); err != nil {
			glog.Fatalf("Write pidfile(%s) error: %s", pidfile, err)
		}
	}

	var ln0 net.Listener
	if config.GroupCache.Addr != "" {
		peers := groupcache.NewHTTPPool("http://" + config.GroupCache.Addr)
		peers.Set(config.GroupCache.Peers...)
		ln0, err = net.Listen("tcp", config.GroupCache.Addr)
		if err != nil {
			glog.Fatalf("ListenTCP(%s) error: %s", config.GroupCache.Addr, err)
		}
		go http.Serve(ln0, peers)
	}

	fmt.Fprintf(os.Stderr, `------------------------------------------------------
GoProxy Version    : %s (go/%s %s/%s)
Listen Address     : %s
RoundTrip Filters  : %v
Pac Server         : http://%s/proxy.pac
------------------------------------------------------
`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH,
		config.Addr,
		strings.Join(config.Filters.RoundTrip, ","),
		config.Addr)

	requestFilters, roundtripFilters, responseFilters := getFilters(config)

	var tlsConfig *tls.Config
	if config.Http.Mode == "h2" {

		readPem := func(object string) []byte {
			store, err := storage.OpenURI(configUri)
			if err != nil {
				glog.Fatalf("store.OpenURI(%v) error: %s", configUri, err)
			}

			o, err := store.GetObject(object, -1, -1)
			if err != nil {
				glog.Fatalf("store.GetObject(%v) error: %s", object, err)
			}

			rc := o.Body()
			defer rc.Close()

			b, err := ioutil.ReadAll(rc)
			if err != nil {
				glog.Fatalf("ioutil.ReadAll error: %s", err)
			}
			return b
		}

		certPem := readPem(config.Http.Certificate)
		keyPem := readPem(config.Http.Certificate)
		tlsCert, err := tls.X509KeyPair(certPem, keyPem)
		if err != nil {
			glog.Fatalf("tls.X509KeyPair error: %s", err)
		}

		tlsConfig = &tls.Config{
			Certificates: []tls.Certificate{tlsCert},
		}
	}

	listenOpts := &httpproxy.ListenOptions{TLSConfig: tlsConfig}

	if strings.HasPrefix(config.Http.Mode, "h2") {
		// http2.VerboseLogs = true
		listenOpts.KeepAlivePeriod = 3 * time.Minute
	}

	ln, err := httpproxy.ListenTCP("tcp", config.Addr, listenOpts)
	if err != nil {
		glog.Fatalf("ListenTCP(%s, %#v) error: %s", config.Addr, listenOpts, err)
	}

	h := httpproxy.Handler{
		Listener:         ln,
		RequestFilters:   requestFilters,
		RoundTripFilters: roundtripFilters,
		ResponseFilters:  responseFilters,
	}

	s := &http.Server{
		Handler:        h,
		ReadTimeout:    time.Duration(config.Http.ReadTimeout) * time.Second,
		WriteTimeout:   time.Duration(config.Http.WriteTimeout) * time.Second,
		MaxHeaderBytes: 1 << 20,
	}

	switch config.Http.Mode {
	case "h2":
		s.TLSConfig = tlsConfig
		http2.ConfigureServer(s, &http2.Server{})
	case "h2c":
		s.TLSConfig = tlsConfig
		s = http2.UpgradeServer(s, &http2.Server{})
	case "h1":
		break
	default:
		glog.Fatalf("Unknow Http mode %s", config.Http.Mode)
	}

	glog.Infof("ListenAndServe on %s\n", h.Listener.Addr().String())
	go s.Serve(h.Listener)

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP)

	for {
		switch <-c {
		case os.Interrupt, syscall.SIGTERM:
			os.Exit(0)
		case syscall.SIGHUP:
			glog.Infof("os.StartProcess %#v", os.Args)

			p, err := httpproxy.StartProcess()
			if err != nil {
				glog.Warningf("StartProcess() with Listeners(%#v, %#v) error: %#v, abort", ln0, h.Listener)
				os.Exit(-1)
			} else {
				glog.Infof("Spawn child(pid=%d) OK, exit in %d seconds", p.Pid, config.Http.WriteTimeout)
				if ln0 != nil {
					ln0.Close()
				}
				h.Listener.Close()
			}

			done := make(chan struct{}, 1)
			go func(c chan<- struct{}) {
				h.Listener.Wait()
				c <- struct{}{}
			}(done)

			select {
			case <-done:
				glog.Infof("All connections were closed, graceful shutdown")
				os.Exit(0)
			case <-time.After(s.WriteTimeout):
				glog.Warningf("Graceful shutdown timeout, quit")
				os.Exit(0)
			}
		}
	}
}
func (c *DimensionsCache) Start(listenUrl string) {
	peers := groupcache.NewHTTPPool(listenUrl)
	peers.Set(listenUrl)
	http.ListenAndServe("http://localhost:8001", http.HandlerFunc(peers.ServeHTTP))
}
Beispiel #18
0
func (g *GroupCacheProxy) MakeInitialPool(url string) PeerList {
	return groupcache.NewHTTPPool(url)
}
// LoadSentinelConfigFile loads the local config file pulled from the
// environment variable "REDSKULL_SENTINELCONFIGFILE"
func (c *Constellation) LoadSentinelConfigFile() error {
	file, err := os.Open(c.SentinelConfigName)
	if err != nil {
		log.Print(err)
		return err
	}
	defer file.Close()
	bf := bufio.NewReader(file)
	for {
		rawline, err := bf.ReadString('\n')
		if err == nil || err == io.EOF {
			line := strings.TrimSpace(rawline)
			// ignore comments
			if strings.Contains(line, "#") {
				continue
			}
			entries := strings.Split(line, " ")
			//Most values are key/value pairs
			switch entries[0] {
			case "sentinel": // Have a sentinel directive
				err := c.extractSentinelDirective(entries[1:])
				if err != nil {
					// TODO: Fix this to return a different error if we can't
					// connect to the sentinel
					log.Printf("Misshapen sentinel directive: '%s'", line)
				}
			case "port":
				iport, _ := strconv.Atoi(entries[1])
				c.SentinelConfig.Port = iport
				//log.Printf("Local sentinel is bound to port %d", c.SentinelConfig.Port)
			case "dir":
				c.SentinelConfig.Dir = entries[1]
			case "bind":
				if c.LocalOverrides.BindAddress > "" {
					log.Printf("Overriding Sentinel BIND directive '%s' with '%s'", entries[1], c.LocalOverrides.BindAddress)
				} else {
					c.SentinelConfig.Host = entries[1]
					if c.Peers == nil {
						me := "http://" + c.SentinelConfig.Host + ":" + GCPORT
						c.Peers = groupcache.NewHTTPPool(me)
						c.PeerList[c.SentinelConfig.Host+fmt.Sprintf(":%d", c.SentinelConfig.Port)] = c.SentinelConfig.Host
						c.SetPeers()
						go http.ListenAndServe(c.SentinelConfig.Host+":"+GCPORT, http.HandlerFunc(c.Peers.ServeHTTP))
						c.StartCache()
					}
				}
				log.Printf("Local sentinel is listening on IP %s", c.SentinelConfig.Host)
			case "":
				if err == io.EOF {
					log.Print("File load complete?")
					if c.LocalOverrides.BindAddress > "" {
						c.SentinelConfig.Host = c.LocalOverrides.BindAddress
						log.Printf("Local sentinel is listening on IP %s", c.SentinelConfig.Host)
					} else {
						if c.Peers == nil {
							// This means the sentinel config has no bind statement
							// So we will pull the local IP and use it
							// I don't like this but dont' have a great option either.
							myhostname, err := os.Hostname()
							if err != nil {
								log.Print(err)
							}
							myip, err := net.LookupHost(myhostname)
							if err != nil {
								log.Print(err)
							}
							c.LocalSentinel.Host = myip[0]
							log.Printf("NO BIND STATEMENT FOUND. USING: '%s'", c.LocalSentinel.Host)
							me := "http://" + c.SentinelConfig.Host + ":" + GCPORT
							c.Peers = groupcache.NewHTTPPool(me)
							c.PeerList[c.SentinelConfig.Host+fmt.Sprintf(":%d", c.SentinelConfig.Port)] = c.SentinelConfig.Host
							c.SetPeers()
							go http.ListenAndServe(c.SentinelConfig.Host+":"+GCPORT, http.HandlerFunc(c.Peers.ServeHTTP))
							c.StartCache()
						}
					}
					return nil
				}
				//log.Printf("Local:config -> %+v", c.SentinelConfig)
				//log.Printf("Found %d REMOTE sentinels", len(c.RemoteSentinels))
				//return nil
			default:
				log.Printf("UNhandled Sentinel Directive: %s", line)
			}
		} else {
			log.Print("=============== LOAD FILE ERROR ===============")
			log.Fatal(err)
		}
	}
}
// extractSentinelDirective parses the sentinel directives from the
// sentinel config file
func (c *Constellation) extractSentinelDirective(entries []string) error {
	switch entries[0] {
	case "monitor":
		pname := entries[1]
		port, _ := strconv.Atoi(entries[3])
		quorum, _ := strconv.Atoi(entries[4])
		spc := SentinelPodConfig{Name: pname, IP: entries[2], Port: port, Quorum: quorum}
		spc.Sentinels = make(map[string]string)
		// normally we should not see duplicate IP:PORT combos, however it
		// can happen when people do things manually and dont' clean up.
		// We need to detect them and ignore the second one if found,
		// reporting the error condition this will require tracking
		// ip:port pairs...
		addr := fmt.Sprintf("%s:%d", entries[2], port)
		_, exists := c.SentinelConfig.ManagedPodConfigs[addr]
		if !exists {
			c.SentinelConfig.ManagedPodConfigs[entries[1]] = spc
		}
		return nil

	case "auth-pass":
		pname := entries[1]
		pc := c.SentinelConfig.ManagedPodConfigs[pname]
		pc.AuthToken = entries[2]
		c.PodAuthMap[pname] = pc.AuthToken
		c.SentinelConfig.ManagedPodConfigs[pname] = pc
		return nil

	case "known-sentinel":
		podname := entries[1]
		sentinel_address := entries[2] + ":" + entries[3]
		pc := c.SentinelConfig.ManagedPodConfigs[podname]
		pc.Sentinels[sentinel_address] = ""
		if c.Peers == nil {
			// This means the sentinel config has no bind statement
			// So we will pull the local IP and use it
			// I don't like this but dont' have a great option either.
			myhostname, err := os.Hostname()
			if err != nil {
				log.Print(err)
			}
			myip, err := net.LookupHost(myhostname)
			if err != nil {
				log.Print(err)
			}
			c.LocalSentinel.Host = myip[0]
			log.Printf("NO BIND STATEMENT FOUND. USING: '%s'", c.LocalSentinel.Host)
			me := "http://" + c.SentinelConfig.Host + ":" + GCPORT
			c.Peers = groupcache.NewHTTPPool(me)
			c.PeerList[c.SentinelConfig.Host+fmt.Sprintf(":%d", c.SentinelConfig.Port)] = c.SentinelConfig.Host
			c.SetPeers()
			go http.ListenAndServe(c.SentinelConfig.Host+":"+GCPORT, http.HandlerFunc(c.Peers.ServeHTTP))
			c.StartCache()
		}
		c.ConfiguredSentinels[sentinel_address] = sentinel_address
		return nil

	case "known-slave":
		// Currently ignoring this, but may add call to a node manager.
		return nil

	case "config-epoch", "leader-epoch", "current-epoch", "down-after-milliseconds":
		// We don't use these keys
		return nil

	default:
		err := fmt.Errorf("Unhandled sentinel directive: %+v", entries)
		log.Print(err)
		return nil
	}
}
Beispiel #21
0
func main() {
	// Parse the flags
	flag.Parse()

	// Create a new groupcache pool
	pool := groupcache.NewHTTPPool(*cachePublic)
	pool.Set(strings.Split(*cachePeers, ",")...)

	// Listen and serve the groupcache pool
	cacheServer := http.Server{
		Addr:    *cacheBind,
		Handler: pool,
	}
	go func() {
		log.Printf("Starting up the cache HTTP server on address %s", *cacheBind)

		err := cacheServer.ListenAndServe()
		if err != nil {
			log.Fatal(err)
		}
	}()

	// Create a new groupcache pool
	cache := groupcache.NewGroup("ritratt", *cacheSize, groupcache.GetterFunc(func(ctx groupcache.Context, url string, dest groupcache.Sink) error {
		// First try with https
		schema := "https://"
		resp, err := http.Head("https://" + url)
		if err != nil {
			log.Printf("[https] Error while querying %s: %s", url, err)

			// https doesn't work, try http
			schema = "http://"
			resp, err = http.Head("http://" + url)
			if err != nil {
				log.Printf("[http] Error while querying %s: %s", url, err)
			}
		}

		// Content-Type of the result has to start with image/
		// We also don't support SVGs, check out this link for more information:
		// https://www.owasp.org/images/0/03/Mario_Heiderich_OWASP_Sweden_The_image_that_called_me.pdf
		/*ct := resp.Header.Get("Content-Type")
		if !strings.HasPrefix(ct, "image/") || strings.Contains(ct, "image/svg+xml") {
			log.Printf("[head] Invalid Content-Type of %s", url)
			return ErrInvalidContentType
		}*/

		// Query the proper URL, now including the body
		resp, err = http.Get(schema + url)
		defer resp.Body.Close()
		if err != nil {
			log.Printf("[get] Error while querying %s: %s", url, err)
		} else {
			log.Printf("[get] Loaded %s", url)
		}

		// Content-Type check #2
		ct := resp.Header.Get("Content-Type")
		if !strings.HasPrefix(ct, "image/") || strings.Contains(ct, "image/svg+xml") {
			log.Printf("[get] Invalid Content-Type of %s", url)
			return ErrInvalidContentType
		}

		var body []byte

		// Read the dlenght
		cls := resp.Header.Get("Content-Length")
		if cls != "" {
			cl, err := strconv.Atoi(cls)
			if err != nil {
				log.Print(err)
			} else {
				if cl > 25*1024*1024 {
					return ErrTooBig
				} else {
					body, err = ioutil.ReadAll(resp.Body)
					if err != nil {
						return err
					}
				}
			}
		} else {
			body = []byte{}
			totalRead := 0

			for {
				chunk := make([]byte, 1024*1024)

				read, err := resp.Body.Read(chunk)
				totalRead += read

				if err == io.EOF {
					break
				}

				if err != nil {
					return err
				}

				body = append(body, chunk[:read]...)

				if totalRead > 25*1024*1024 {
					return ErrTooBig
				}
			}
		}

		// Put the body into cache with the Content-Type
		dest.SetString(ct + ";" + string(body))

		return nil
	}))

	log.Printf("Starting up the proxy HTTP server on address %s", *proxyBind)
	proxyServer := http.Server{
		Addr: *proxyBind,
		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			// Index page
			if len(r.RequestURI) < 3 || r.RequestURI[:3] != "/i/" {
				w.Write([]byte("lavab/ritratt"))
				return
			}

			// Get the data from groupcache
			var data string
			err := cache.Get(nil, r.RequestURI[3:], groupcache.StringSink(&data))
			if err != nil {
				w.Write([]byte(err.Error()))
				return
			}

			// Split the result into two parts
			parts := strings.SplitN(data, ";", 2)

			// Set the content type
			w.Header().Set("Content-Type", parts[0])

			// Write the body
			w.Write([]byte(parts[1]))
		}),
	}
	log.Fatal(proxyServer.ListenAndServe())
}
Beispiel #22
0
func InitCachePool(me string, peers []string, port int) {
	httpPool = groupcache.NewHTTPPool(fmt.Sprintf("http://%s:%d", me, port))

	SetCachePoolPeers(peers, port)
}