Example #1
0
File: map.go Project: GeoNet/mtr
/*
Init returns an initialised Map180. d must have access to the map180 tables in the
public schema.  cacheBytes is the size of the memory cache
for land and lake layers.  region must be a valid Region.

Example:

   wm, err = map180.Init(db, map180.Region(`newzealand`), 256000000)
*/
func Init(d *sql.DB, region Region, cacheBytes int64) (*Map180, error) {
	w := &Map180{}

	var err error
	if _, ok := allZoomRegions[region]; !ok {
		err = fmt.Errorf("invalid region (allZoomRegions): %s", region)
		return w, err
	}

	if _, ok := allMapBounds[region]; !ok {
		err = fmt.Errorf("invalid region (allMapBounds): %s", region)
		return w, err
	}

	if _, ok := allNamedMapBounds[region]; !ok {
		err = fmt.Errorf("invalid region (allNamedMapBounds): %s", region)
		return w, err
	}

	zoomRegion = allZoomRegions[region]
	mapBounds = allMapBounds[region]
	namedMapBounds = allNamedMapBounds[region]

	db = d

	mapLayers = groupcache.NewGroup("mapLayers", cacheBytes, groupcache.GetterFunc(layerGetter))
	landLayers = groupcache.NewGroup("landLayers", cacheBytes, groupcache.GetterFunc(landGetter))
	lakeLayers = groupcache.NewGroup("lakeLayers", cacheBytes, groupcache.GetterFunc(lakeGetter))

	return w, nil
}
Example #2
0
func main() {
	// STARTINIT OMIT
	me := "http://127.0.0.1:11211"
	peers := groupcache.NewHTTPPool(me)

	// Whenever peers change:
	peers.Set("http://127.0.0.1:11211")
	// ENDINIT OMIT

	// STARTGROUP OMIT
	var thumbNails = groupcache.NewGroup("thumbnail", 64<<20, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			dest.SetBytes(generateThumbnail(key))
			return nil
		}))
	// ENDGROUP OMIT

	var ctx groupcache.Context

	// STARTUSE OMIT
	var data []byte
	thumbNails.Get(ctx, "big-file.jpg", groupcache.AllocatingByteSliceSink(&data))

	fmt.Println(string(data))
}
// StartCache is used to start up the groupcache mechanism
func (c *Constellation) StartCache() {
	log.Print("Starting AuthCache")
	var peers []string
	if c.PeerList == nil {
		log.Print("Initializing PeerList")
		c.PeerList = make(map[string]string)
	}

	for _, peer := range c.PeerList {
		log.Printf("Assigning peer '%s'", peer)
		if peer > "" {
			peers = append(peers, "http://"+peer+":"+GCPORT)
		}
	}
	c.Peers.Set(peers...)
	var authcache = groupcache.NewGroup(c.Groupname, 64<<20, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			pod := key
			auth := c.GetAuthForPodFromConfig(pod)
			if auth > "" {
				dest.SetString(auth)
				return nil
			}
			err := fmt.Errorf("Found no value for auth on pod %s, not storing anything", pod)
			return err
		}))
	c.AuthCache = NewCache(authcache)

}
Example #4
0
// SetCacheMaxSize enables icon caching if sizeInMB > 0.
func SetCacheMaxSize(sizeInMB int64) {
	if sizeInMB > 0 {
		iconCache = groupcache.NewGroup("icons", sizeInMB<<20, groupcache.GetterFunc(generatorFunc))
	} else {
		iconCache = nil
	}
}
Example #5
0
File: main.go Project: qmdx/mygo
func main() {
	flag.Parse()

	peers := gc.NewHTTPPool("http://localhost:" + *port)
	peers.Set("http://localhost:8001", "http://localhost:8002")

	cg = gc.NewGroup("QRCache", 1<<20, gc.GetterFunc(
		func(ctx gc.Context, key string, dest gc.Sink) error {
			fmt.Printf("asking for data of %s\n", key)
			url := fmt.Sprintf("http://chart.apis.google.com/chart?chs=300x300&cht=qr&choe=UTF-8&chl=%s", key)
			resp, err := http.Get(url)
			if err != nil {
				return nil
			}
			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return nil
			}
			value := base64.StdEncoding.EncodeToString(body)
			dest.SetBytes([]byte(value))
			return nil
		}))

	// run groupcache process in goroutine
	go http.ListenAndServe("localhost:"+*port, peers)

	http.Handle("/", http.HandlerFunc(QR))
	err := http.ListenAndServe(*addr, nil)
	if err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}
Example #6
0
func main() {
	/*	me := "http://127.0.0.1"
		peers := groupcache.NewHTTPPool(me)

		// Whenever peers change:
		peers.Set("http://127.0.0.1", "http://10.0.0.2", "http://10.0.0.3")*/

	getter := groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			//			fileName := key
			bytes := []byte{'a', 'b', 'c'}
			dest.SetBytes(bytes)
			return nil
		})

func(g *igetter) Get(ctx Content, key string, dest Sink)  {

}

	grp := groupcache.NewGroup("Grp", 64<<20, getter)
	err := grp.Get(ctx groupcache.Context, key string, dest groupcache.Sink)
	//hook
	/*	hook := groupcache.RegisterNewGroupHook(func(grp1 *groupcache.Group) {
		fmt.Println(grp1.Name())
		fmt.Println("hook")
	})*/
	
	fmt.Println(grp.Name())

	fmt.Println(grp.Stats)

}
Example #7
0
func main() {
	flag.Parse()

	// Setup the doozer connection.
	d, err := doozer.Dial(daddr)
	if err != nil {
		log.Fatalf("connecting to doozer: %v\n", err)
	}
	defer d.Close()

	// Setup the cache.
	pool = groupcache.NewHTTPPool("http://" + addr)
	dict = groupcache.NewGroup("dict", 64<<20, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			def, err := query(key)
			if err != nil {
				err = fmt.Errorf("querying remote dictionary: %v", err)
				log.Println(err)
				return err
			}

			log.Println("retrieved remote definition for", key)
			dest.SetString(def)
			return nil
		}))

	// Start watching for changes and signals.
	go watch(d)

	// Add the handler for definition requests and then start the
	// server.
	http.Handle("/define/", http.HandlerFunc(handler))
	log.Println(http.ListenAndServe(addr, nil))
}
Example #8
0
func main() {
	// STARTINIT OMIT
	me := "http://10.0.0.1"
	peers := groupcache.NewHTTPPool(me)

	// Whenever peers change:
	peers.Set("http://10.0.0.1", "http://10.0.0.2", "http://10.0.0.3")
	// ENDINIT OMIT

	// STARTGROUP OMIT
	var thumbNails = groupcache.NewGroup("thumbnail", 64<<20, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			fileName := key
			dest.SetBytes(generateThumbnail(fileName))
			return nil
		}))
	// ENDGROUP OMIT

	var ctx groupcache.Context
	var w http.ResponseWriter
	var r *http.Request

	// STARTUSE OMIT
	var data []byte
	err := thumbNails.Get(ctx, "big-file.jpg",
		groupcache.AllocatingByteSliceSink(&data))
	// ...
	_ = err               // OMIT
	var modTime time.Time // OMIT
	http.ServeContent(w, r, "big-file-thumb.jpg", modTime, bytes.NewReader(data))
	// ENDUSE OMIT
}
Example #9
0
func main() {
	me := ":" + os.Args[1]
	peers := groupcache.NewHTTPPool("http://localhost" + me)
	peers.Set("http://localhost:8081", "http://localhost:8082",
		"http://localhost:8083")
	helloworld := groupcache.NewGroup("helloworld",
		1024*1024*10, groupcache.GetterFunc(
			func(ctx groupcache.Context,
				key string,
				dest groupcache.Sink) error {
				log.Println(me)
				dest.SetString(me + "->" + key)
				return nil
			}))
	fmt.Println("GroupName: ", helloworld.Name())
	http.HandleFunc("/xbox/", func(w http.ResponseWriter, r *http.Request) {
		parts := strings.SplitN(r.URL.Path[len("/xbox/"):], "/", 1)
		for _, x := range parts {
			fmt.Println("get: ", x)
		}
		if len(parts) != 1 {
			http.Error(w, "Bad Request", http.StatusBadRequest)
			return
		}
		var data []byte
		helloworld.Get(nil, parts[0], groupcache.AllocatingByteSliceSink(&data))
		w.Write(data)
		log.Println("Gets: ", helloworld.Stats.Gets.String())
		log.Println("Load: ", helloworld.Stats.Loads.String())
		log.Println("LocalLoad: ", helloworld.Stats.LocalLoads.String())
		log.Println("PeerError: ", helloworld.Stats.PeerErrors.String())
		log.Println("PeerLoad: ", helloworld.Stats.PeerLoads.String())
	})
	http.ListenAndServe(me, nil)
}
Example #10
0
func InitRadosCache(name string, size int64, chunkSize int) {
	if cacheMade {
		panic("groupcache: InitRadosCache must be called only once")
	}
	cacheMade = true

	radosCache = groupcache.NewGroup(name, size, groupcache.GetterFunc(radosGetter))
	chunkPool = sync.Pool{
		New: func() interface{} { return make([]byte, chunkSize) },
	}
}
Example #11
0
// NewGroupCache function
func NewGroupCache(cacheSize int64, redisGetFunc GetFunc) Cache {
	return &groupCacheMgr{
		group: gc.NewGroup(singleGroupName, cacheSize, gc.GetterFunc(func(ctx gc.Context, key string, dest gc.Sink) error {
			log.WithField("domain", key).Info("GroupCache:RedisRequest")

			redisRequest := api.NewValueRequest(key)
			redisGetFunc(redisRequest)
			redisResponse := <-redisRequest.Response

			log.WithField("domain", key).Info("GroupCache:RedisRequest:Done")
			return dest.SetString(redisResponse.Value)
		}))}
}
Example #12
0
func setupGroupcache(config GroupcacheConfig) error {
	if config.GB == 0 {
		return nil
	}
	var cacheBytes int64
	cacheBytes = int64(config.GB) << 30

	pool := groupcache.NewHTTPPool(config.Host)
	if pool != nil {
		dvid.Infof("Initializing groupcache with %d GB at %s...\n", config.GB, config.Host)
		manager.gcache.cache = groupcache.NewGroup("immutable", cacheBytes, groupcache.GetterFunc(
			func(c groupcache.Context, key string, dest groupcache.Sink) error {
				// Use KeyValueDB defined as context.
				gctx, ok := c.(GroupcacheCtx)
				if !ok {
					return fmt.Errorf("bad groupcache context: expected GroupcacheCtx, got %v", c)
				}

				// First four bytes of key is instance ID to isolate groupcache collisions.
				tk := TKey(key[4:])
				data, err := gctx.KeyValueDB.Get(gctx.Context, tk)
				if err != nil {
					return err
				}
				return dest.SetBytes(data)
			}))
		manager.gcache.supported = make(map[dvid.DataSpecifier]struct{})
		for _, dataspec := range config.Instances {
			name := strings.Trim(dataspec, "\"")
			parts := strings.Split(name, ":")
			switch len(parts) {
			case 2:
				dataid := dvid.GetDataSpecifier(dvid.InstanceName(parts[0]), dvid.UUID(parts[1]))
				manager.gcache.supported[dataid] = struct{}{}
			default:
				dvid.Errorf("bad data instance specification %q given for groupcache support in config file\n", dataspec)
			}
		}

		// If we have additional peers, add them and start a listener via the HTTP port.
		if len(config.Peers) > 0 {
			peers := []string{config.Host}
			peers = append(peers, config.Peers...)
			pool.Set(peers...)
			dvid.Infof("Groupcache configuration has %d peers in addition to local host.\n", len(config.Peers))
			dvid.Infof("Starting groupcache HTTP server on %s\n", config.Host)
			http.ListenAndServe(config.Host, http.HandlerFunc(pool.ServeHTTP))
		}
	}
	return nil
}
Example #13
0
func main() {
	var err error
	p = groupcache.NewHTTPPool("http://127.0.0.1:50000")
	// update p's list
	p.Set("http://127.0.0.1:50001", "http://127.0.0.1:50002")

	// create a new group by name and set a function for get a value by key
	// if you g.Get KEY at first time, it will store the value use sink, run the function
	// if you g.Get KEY not at first time, actually it NOT entry the function
	// does not support update a key's value
	g = groupcache.NewGroup("dns", 64<<20, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) (err error) {
			if ctx == nil {
				ctx = "f**k"
			}
			log.Println(key, ctx)
			err = dest.SetString(ctx.(string))
			return
		}))

	// let it listen
	go http.ListenAndServe("127.0.0.1:50000", nil)

	// get a key's value into data

	time.Sleep(2 * time.Second)
	var data []byte
	var key = "key"
	err = g.Get("aa", key, groupcache.AllocatingByteSliceSink(&data))
	if err != nil {
		log.Println("get error:", err)
	}
	log.Println(string(data))

	time.Sleep(2 * time.Second)
	key = "key1"
	err = g.Get("bb", key, groupcache.AllocatingByteSliceSink(&data))
	if err != nil {
		log.Println("get error:", err)
	}
	log.Println(string(data))

	time.Sleep(2 * time.Second)
	key = "key"
	err = g.Get("cc", key, groupcache.AllocatingByteSliceSink(&data))
	if err != nil {
		log.Println("get error:", err)
	}
	log.Println(string(data))
}
Example #14
0
func (g *GroupCacheProxy) MakeCache(c *Cluster, size int64) CacheGetter {
	return groupcache.NewGroup(
		"ReticulumCache", size, groupcache.GetterFunc(
			func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
				// get image from disk
				ri := NewImageSpecifier(key)
				img_data, err := c.RetrieveImage(ri)
				if err != nil {
					return err
				}
				dest.SetBytes([]byte(img_data))
				return nil
			}))
}
Example #15
0
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	awsAuth, err := aws.EnvAuth()
	if err != nil {
		log.Fatalf(err.Error())
	}

	s3conn := s3.New(awsAuth, getRegion())
	storage = store.NewS3Store(s3conn)

	peers = peer.DebugPool()

	peers.SetContext(func(r *http.Request) groupcache.Context {
		return fetch.RequestContext(r)
	})

	cache = groupcache.NewGroup("ImageProxyCache", 64<<20, groupcache.GetterFunc(
		func(c groupcache.Context, key string, dest groupcache.Sink) error {
			log.Printf("Cache MISS for key -> %s", key)
			// Get image data from S3
			b, err := fetch.ImageData(storage, c)
			if err != nil {
				return err
			}

			return dest.SetBytes(b)
		}))

	if !*verbose {
		logwriter, err := syslog.Dial("udp", "app_syslog:514", syslog.LOG_NOTICE, "vip")
		if err != nil {
			log.Println(err.Error())
			log.Println("using default logger")
		} else {
			log.SetOutput(logwriter)
		}
	}

	go peers.Listen()
	go listenHttp()
	go Queue.Start(4)
	log.Println("Cache listening on port :" + peers.Port())
	s := &http.Server{
		Addr:    ":" + peers.Port(),
		Handler: peers,
	}
	s.ListenAndServe()
}
func TestGetSetGC(t *testing.T) {
	tt := T{1}

	mcStorage := NewMcStorage([]string{"localhost:12000"}, "test", 0, reflect.TypeOf(&tt))
	mcStorage.Set("1", tt)
	res, _ := mcStorage.Get("1")
	defer mcStorage.Delete("1")
	if reflect.TypeOf(res) != reflect.TypeOf(tt) {
		t.Error("res type is not T")
	}
	ttRes := res.(T)
	if ttRes.A != tt.A {
		t.Error("res A field is not equals tt field")
	}

	var groupcache = groupcache.NewGroup("SlowDBCache", 64<<20, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			result, err := mcStorage.Get(key)
			if err != nil {
				return nil
			}
			bytes, err := json.Marshal(result)
			if err != nil {
				return nil
			}
			dest.SetBytes(bytes)
			return nil
		}))
	gcStorage := &GroupCacheKvStorage{groupcache, 0, reflect.TypeOf(&tt)}
	res, _ = gcStorage.Get("1")
	if reflect.TypeOf(res) != reflect.TypeOf(tt) {
		t.Error("res type is not T")
	}
	ttRes = res.(T)
	if ttRes.A != tt.A {
		t.Error("res A field is not equals tt field")
	}

	mcStorage.Delete("1")
	res, _ = gcStorage.Get("1")
	if reflect.TypeOf(res) != reflect.TypeOf(tt) {
		t.Error("res type is not T")
	}
	ttRes = res.(T)
	if ttRes.A != tt.A {
		t.Error("res A field is not equals tt field")
	}
}
func TestServerErrorImageUnmarshal(t *testing.T) {
	srv := &Server{
		Group: groupcache.NewGroup(
			newTestServerName(),
			testSize,
			groupcache.GetterFunc(func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
				dest.SetBytes(nil)
				return nil
			}),
		),
		KeyGenerator: imageserver_cache.StringKeyGenerator("test"),
	}
	_, err := srv.Get(imageserver.Params{})
	if err == nil {
		t.Fatal("no error")
	}
}
Example #18
0
func beChildForTestGRPCPool() {
	addrs := strings.Split(*peerAddrs, ",")
	server := grpc.NewServer()

	p := NewGRPCPool(addrs[*peerIndex], server)
	p.Set(addrs...)

	getter := groupcache.GetterFunc(func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
		dest.SetString(strconv.Itoa(*peerIndex) + ":" + key)
		return nil
	})
	groupcache.NewGroup("grpcPoolTest", 1<<20, getter)
	lis, err := net.Listen("tcp", addrs[*peerIndex])
	if err != nil {
		log.Fatalf("Failed to listen on %s", addrs[*peerIndex])
	}

	server.Serve(lis)
}
Example #19
0
func start() {
	pool = groupcache.NewHTTPPool("http://" + addr)

	dns = groupcache.NewGroup("dns", 64<<20, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			// Lookup the adddress based on the hostname (key).
			addrs, err := net.LookupHost(key)
			if err != nil {
				return err
			}

			// We'll just store the first.
			dest.SetString(addrs[0])
			return nil
		}))

	// Run in the background so we can use the console.
	go http.ListenAndServe(addr, nil)
}
Example #20
0
func initCacheGroup() {
	self := config.Get().CacheSelf()

	pool := groupcache.NewHTTPPool(self)
	pool.Set(config.Get().CachePeers()...)

	if self != "" {
		log.Println("Cache listen on:", strings.TrimLeft(self, "http://"))
		go http.ListenAndServe(strings.TrimLeft(self, "http://"), http.HandlerFunc(pool.ServeHTTP))
	}

	cacheGroup = groupcache.NewGroup("imagio-storage", config.Get().CacheSize(), groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			dest.SetBytes(imgproc.Do(
				Construct(new(query.Options), key).(*query.Options),
			))
			return nil
		}),
	)
}
func NewDimensionsCache() *DimensionsCache {
	g := groupcache.NewGroup("DimensionsCache", 64<<18, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			fmt.Printf("Calculating Dimensions for %s\n", key)
			u, err := url.Parse(key)
			if err != nil {
				return err
			}

			result := calculateDimensions(u)
			var buffer = bytes.Buffer{}
			binary.Write(&buffer, binary.BigEndian, result)
			dest.SetBytes(buffer.Bytes())

			fmt.Println("Filled Cache")
			return nil
		}))
	dc := DimensionsCache{g}
	return &dc
}
Example #22
0
func main() {

	flag.Set("logtostderr", "true")
	flag.Parse()

	me := ":8080"
	peers := groupcache.NewHTTPPool("http://localhost" + me)
	peers.Set("http://localhost:8081", "http://localhost:8082", "http://localhost:8083")

	helloworld := groupcache.NewGroup("helloworld", 1024*1024*1024*16, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			glog.Infof("%v, key = %v", me, key)
			dest.SetString(key)
			return nil
		}))

	glog.Infof("GroupName: %v", helloworld.Name())
	http.HandleFunc("/xbox/",
		func(w http.ResponseWriter, r *http.Request) {
			parts := strings.SplitN(r.URL.Path[len("/xbox/"):], "/", 1)
			glog.Infof("parts: %v", parts)
			if len(parts) != 1 {
				http.Error(w, "Bad Request", http.StatusBadRequest)
				return
			}

			var data []byte
			helloworld.Get(nil, parts[0], groupcache.AllocatingByteSliceSink(&data))
			w.Write(data)

			glog.Infof("data: %s", data)
			glog.Infof("Stats: %#v", helloworld.Stats)
			//glog.Infof("Gets: %v", helloworld.Stats.Gets.String())
			//glog.Infof("Load: %v", helloworld.Stats.Loads.String())
			//glog.Infof("LocalLoad: %v", helloworld.Stats.LocalLoads.String())
			//glog.Infof("PeerError: %v", helloworld.Stats.PeerErrors.String())
			//glog.Infof("PeerLoad: %v", helloworld.Stats.PeerLoads.String())
		})

	http.ListenAndServe(me, nil)
}
Example #23
0
func main() {

	if !flag.Parsed() {
		flag.Parse()
	}

	// 	etcd := os.Getenv("ETCD_ADDR")
	ip := os.Getenv("PUBLIC_HOSTIP")
	pport := os.Getenv("PUBLIC_PORT")

	cache := etcd_groupcache.New(fmt.Sprintf("http://%s:%s", ip, pport), nil, nil)
	peers := []string{fmt.Sprintf("http://%s:8080", ip), fmt.Sprintf("http://%s:8081", ip)}
	log.Infoln(peers)
	cache.Set(peers...)
	http.Handle("/_groupcache/", cache)

	var stringcache = groupcache.NewGroup("SlowDBCache", 64<<20, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			log.Infoln(pport)
			dest.SetString(key + "heyo")
			return nil
		}))

	go func() {
		var data []byte
		key := "some key"
		err := stringcache.Get(nil, key, groupcache.AllocatingByteSliceSink(&data))
		if err != nil {
			log.Errorln(err)
		}
		log.Infoln(string(data))
	}()

	log.Infoln("Groupcache Listening 8080")
	if err := http.ListenAndServe(":"+*port, nil); err != nil {
		panic(err)
	}
}
Example #24
0
func ServeDir(mountpoint string, target string, limit int, me string, peerlist []string) {

	peers := groupcache.NewHTTPPool(me)

	if peerlist != nil && len(peerlist) > 0 {
		peers.Set(peerlist...)
	}

	filecache = groupcache.NewGroup("filecache", int64(limit)<<20, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			contents, err := ioutil.ReadFile(key)
			dest.SetBytes(contents)
			return err
		}))

	c, err := fuse.Mount(mountpoint)
	if err != nil {
		log.Fatal(err)
	}

	fs.Serve(c, TargetDir{target})

}
Example #25
0
func main() {

	// setup group
	stringGroup = gc.NewGroup(stringGroupName, cacheSize, gc.GetterFunc(func(_ gc.Context, key string, dest gc.Sink) error {
		fmt.Fprintf(os.Stdout, "Setting cache key: %s\n", key)
		return dest.SetString("ECHO:" + key)
	}))

	// Get Items
	for j := 0; j < 4; j++ {
		for i := 0; i < 4; i++ {
			var s string
			if err := stringGroup.Get(dummyCtx, "TestCaching-key"+strconv.Itoa(i), gc.StringSink(&s)); err != nil {
				fmt.Fprintf(os.Stdout, "TestCaching-key value: failed%s\n", err)
				return
			}
			fmt.Fprintf(os.Stdout, "TestCaching-key value:%s\n", s)
		}
		fmt.Fprintln(os.Stdout, "---")
	}

	fmt.Fprintln(os.Stdout, "Done.")
}
func main() {

	var port = flag.String("port", "8001", "groupcache port")
	flag.Parse()

	peers := groupcache.NewHTTPPool("http://localhost:" + *port)

	client := new(client.Client)

	var stringcache = groupcache.NewGroup("SlowDBCache", 64<<20, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			result := client.Get(key)
			fmt.Printf("asking for %s from dbserver\n", key)
			dest.SetBytes([]byte(result))
			return nil
		}))

	peers.Set("http://localhost:8001", "http://localhost:8002", "http://localhost:8003")

	frontendServer := NewServer(stringcache)

	i, err := strconv.Atoi(*port)
	if err != nil {
		// handle error
		fmt.Println(err)
		os.Exit(2)
	}
	var frontEndport = ":" + strconv.Itoa(i+1000)
	go frontendServer.Start(frontEndport)

	fmt.Println(stringcache)
	fmt.Println("cachegroup slave starting on " + *port)
	fmt.Println("frontend starting on " + frontEndport)
	http.ListenAndServe("127.0.0.1:"+*port, http.HandlerFunc(peers.ServeHTTP))

}
Example #27
0
func main() {
	rpcPort := flag.Int("rpcPort", 9090, "port for groupcache ipc")
	proxyPort := flag.Int("proxyPort", 8080, "port for http proxy")
	peerPort := flag.Int("peerPort", 7070, "port for receiving peer updates")

	flag.Parse()

	me := "http://localhost:" + strconv.Itoa(*rpcPort)
	poolOpts := &groupcache.HTTPPoolOptions{
		BasePath: "/stuff",
	}

	cache := groupcache.NewHTTPPoolOpts(me, poolOpts)

	groupcache.NewGroup("", 64<<20, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			log.V(2).Info("trying to get ", key)
			client := &http.Client{
				Timeout: time.Second * 5,
			}
			res, err := client.Get(key)
			if err != nil {
				return err
			}

			defer res.Body.Close()
			if res.StatusCode != http.StatusOK {
				return errors.New("server returned: " + string(res.Status))
			}

			data, err := ioutil.ReadAll(res.Body)
			if err != nil {
				return err
			}
			dest.SetBytes(data)
			return nil
		},
	))

	cache.Set("http://localhost:9090", "http://localhost:9091", "http://localhost:9092")

	proxy := goproxy.NewProxyHttpServer()
	//proxy.Verbose = true
	proxy.OnRequest().DoFunc(
		func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
			log.V(2).Infof("Proxy got request for: %+v", *r.URL)
			r.URL = &url.URL{
				Path: "/stuff/" + r.URL.Scheme + "://" + r.URL.Host + r.URL.Path,
			}
			pr := NewProxyResponse()
			cache.ServeHTTP(pr, r)
			log.V(3).Infof("body: %v", pr.response.Body)
			data, err := ioutil.ReadAll(pr.response.Body)
			if err != nil {
				log.V(1).Infof("Failed to read response %v", err)
			}
			return r, goproxy.NewResponse(r, goproxy.ContentTypeText, http.StatusForbidden, string(data))
		},
	)

	mux := http.NewServeMux()
	mux.HandleFunc("/peers", func(w http.ResponseWriter, r *http.Request) {
		data, err := ioutil.ReadAll(r.Body)
		if err != nil {
			log.V(1).Infof("Failed to read response %v", err)
		}
		cache.Set(strings.Split(string(data), ",")...)
	})

	log.Infoln("Starting rpc instance.")
	go func() { log.Fatal(http.ListenAndServe(":"+strconv.Itoa(*rpcPort), cache)) }()

	log.Infoln("Starting peer update listener.")
	go func() { log.Fatal(http.ListenAndServe(":"+strconv.Itoa(*peerPort), mux)) }()
	log.Infoln("Starting proxy instance.")
	log.Fatal(http.ListenAndServe(":"+strconv.Itoa(*proxyPort), proxy))
}
Example #28
0
File: gp.go Project: Wuvist/gws
	"github.com/golang/groupcache"
	"io/ioutil"
	"log"
	"net/http"
)

func getter(ctx groupcache.Context, key string, dest groupcache.Sink) error {
	resp, _ := http.Get(key)
	defer resp.Body.Close()
	body, _ := ioutil.ReadAll(resp.Body)
	log.Printf("asking for %s from solr\n", key)
	dest.SetString(string(body))
	return nil
}

var stringcache = groupcache.NewGroup("Solr", 64<<20, groupcache.GetterFunc(getter))

func proxyHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("content-type", "application/json")
	url := "http://moapi.com:8080" + r.URL.Path + "?" + r.URL.RawQuery

	var s string
	stringcache.Get(nil, url, groupcache.StringSink(&s))
	fmt.Fprintf(w, s)
}

func startProxy() {
	log.Print("starting")
	handler := http.NewServeMux()
	handler.HandleFunc("/select/", proxyHandler)
	server := &http.Server{Addr: ":8889", Handler: handler}
Example #29
0
func main() {
	// Parse the flags
	flag.Parse()

	// Create a new groupcache pool
	pool := groupcache.NewHTTPPool(*cachePublic)
	pool.Set(strings.Split(*cachePeers, ",")...)

	// Listen and serve the groupcache pool
	cacheServer := http.Server{
		Addr:    *cacheBind,
		Handler: pool,
	}
	go func() {
		log.Printf("Starting up the cache HTTP server on address %s", *cacheBind)

		err := cacheServer.ListenAndServe()
		if err != nil {
			log.Fatal(err)
		}
	}()

	// Create a new groupcache pool
	cache := groupcache.NewGroup("ritratt", *cacheSize, groupcache.GetterFunc(func(ctx groupcache.Context, url string, dest groupcache.Sink) error {
		// First try with https
		schema := "https://"
		resp, err := http.Head("https://" + url)
		if err != nil {
			log.Printf("[https] Error while querying %s: %s", url, err)

			// https doesn't work, try http
			schema = "http://"
			resp, err = http.Head("http://" + url)
			if err != nil {
				log.Printf("[http] Error while querying %s: %s", url, err)
			}
		}

		// Content-Type of the result has to start with image/
		// We also don't support SVGs, check out this link for more information:
		// https://www.owasp.org/images/0/03/Mario_Heiderich_OWASP_Sweden_The_image_that_called_me.pdf
		/*ct := resp.Header.Get("Content-Type")
		if !strings.HasPrefix(ct, "image/") || strings.Contains(ct, "image/svg+xml") {
			log.Printf("[head] Invalid Content-Type of %s", url)
			return ErrInvalidContentType
		}*/

		// Query the proper URL, now including the body
		resp, err = http.Get(schema + url)
		defer resp.Body.Close()
		if err != nil {
			log.Printf("[get] Error while querying %s: %s", url, err)
		} else {
			log.Printf("[get] Loaded %s", url)
		}

		// Content-Type check #2
		ct := resp.Header.Get("Content-Type")
		if !strings.HasPrefix(ct, "image/") || strings.Contains(ct, "image/svg+xml") {
			log.Printf("[get] Invalid Content-Type of %s", url)
			return ErrInvalidContentType
		}

		var body []byte

		// Read the dlenght
		cls := resp.Header.Get("Content-Length")
		if cls != "" {
			cl, err := strconv.Atoi(cls)
			if err != nil {
				log.Print(err)
			} else {
				if cl > 25*1024*1024 {
					return ErrTooBig
				} else {
					body, err = ioutil.ReadAll(resp.Body)
					if err != nil {
						return err
					}
				}
			}
		} else {
			body = []byte{}
			totalRead := 0

			for {
				chunk := make([]byte, 1024*1024)

				read, err := resp.Body.Read(chunk)
				totalRead += read

				if err == io.EOF {
					break
				}

				if err != nil {
					return err
				}

				body = append(body, chunk[:read]...)

				if totalRead > 25*1024*1024 {
					return ErrTooBig
				}
			}
		}

		// Put the body into cache with the Content-Type
		dest.SetString(ct + ";" + string(body))

		return nil
	}))

	log.Printf("Starting up the proxy HTTP server on address %s", *proxyBind)
	proxyServer := http.Server{
		Addr: *proxyBind,
		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			// Index page
			if len(r.RequestURI) < 3 || r.RequestURI[:3] != "/i/" {
				w.Write([]byte("lavab/ritratt"))
				return
			}

			// Get the data from groupcache
			var data string
			err := cache.Get(nil, r.RequestURI[3:], groupcache.StringSink(&data))
			if err != nil {
				w.Write([]byte(err.Error()))
				return
			}

			// Split the result into two parts
			parts := strings.SplitN(data, ";", 2)

			// Set the content type
			w.Header().Set("Content-Type", parts[0])

			// Write the body
			w.Write([]byte(parts[1]))
		}),
	}
	log.Fatal(proxyServer.ListenAndServe())
}
Example #30
0
func TestGRPCPool(t *testing.T) {
	if *peerChild {
		beChildForTestGRPCPool()
		os.Exit(0)
	}

	const (
		nChild = 5
		nGets  = 5000
	)

	var childAddr []string
	for i := 0; i < nChild; i++ {
		childAddr = append(childAddr, pickFreeAddr(t))
	}

	var cmds []*exec.Cmd
	var wg sync.WaitGroup
	for i := 0; i < nChild; i++ {
		cmd := exec.Command(os.Args[0],
			"--test.run=TestGRPCPool",
			"--test_peer_child",
			"--test_peer_addrs="+strings.Join(childAddr, ","),
			"--test_peer_index="+strconv.Itoa(i),
		)
		cmds = append(cmds, cmd)
		wg.Add(1)
		if err := cmd.Start(); err != nil {
			t.Fatal("failed to start child process: ", err)
		}
		go awaitAddrReady(t, childAddr[i], &wg)
	}
	defer func() {
		for i := 0; i < nChild; i++ {
			if cmds[i].Process != nil {
				cmds[i].Process.Kill()
			}
		}
	}()
	wg.Wait()

	// Use a dummy self address so that we don't handle gets in-process.
	p := NewGRPCPool("should-be-ignored", grpc.NewServer())
	p.Set(childAddr...)

	// Dummy getter function. Gets should go to children only.
	// The only time this process will handle a get is when the
	// children can't be contacted for some reason.
	getter := groupcache.GetterFunc(func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
		return errors.New("parent getter called; something's wrong")
	})
	g := groupcache.NewGroup("grpcPoolTest", 1<<20, getter)

	for _, key := range testKeys(nGets) {
		var value string
		if err := g.Get(nil, key, groupcache.StringSink(&value)); err != nil {
			t.Fatal(err)
		}
		if suffix := ":" + key; !strings.HasSuffix(value, suffix) {
			t.Errorf("Get(%q) = %q, want value ending in %q", key, value, suffix)
		}
		t.Logf("Get key=%q, value=%q (peer:key)", key, value)
	}
}