/*
Init returns an initialised Map180. d must have access to the map180 tables in the
public schema. cacheBytes is the size of the memory cache
for land and lake layers. region must be a valid Region.
Example:
wm, err = map180.Init(db, map180.Region(`newzealand`), 256000000)
*/
func Init(d *sql.DB, region Region, cacheBytes int64) (*Map180, error) {
w := &Map180{}
var err error
if _, ok := allZoomRegions[region]; !ok {
err = fmt.Errorf("invalid region (allZoomRegions): %s", region)
return w, err
}
if _, ok := allMapBounds[region]; !ok {
err = fmt.Errorf("invalid region (allMapBounds): %s", region)
return w, err
}
if _, ok := allNamedMapBounds[region]; !ok {
err = fmt.Errorf("invalid region (allNamedMapBounds): %s", region)
return w, err
}
zoomRegion = allZoomRegions[region]
mapBounds = allMapBounds[region]
namedMapBounds = allNamedMapBounds[region]
db = d
mapLayers = groupcache.NewGroup("mapLayers", cacheBytes, groupcache.GetterFunc(layerGetter))
landLayers = groupcache.NewGroup("landLayers", cacheBytes, groupcache.GetterFunc(landGetter))
lakeLayers = groupcache.NewGroup("lakeLayers", cacheBytes, groupcache.GetterFunc(lakeGetter))
return w, nil
}
func main() {
// STARTINIT OMIT
me := "http://127.0.0.1:11211"
peers := groupcache.NewHTTPPool(me)
// Whenever peers change:
peers.Set("http://127.0.0.1:11211")
// ENDINIT OMIT
// STARTGROUP OMIT
var thumbNails = groupcache.NewGroup("thumbnail", 64<<20, groupcache.GetterFunc(
func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
dest.SetBytes(generateThumbnail(key))
return nil
}))
// ENDGROUP OMIT
var ctx groupcache.Context
// STARTUSE OMIT
var data []byte
thumbNails.Get(ctx, "big-file.jpg", groupcache.AllocatingByteSliceSink(&data))
fmt.Println(string(data))
}
// StartCache is used to start up the groupcache mechanism
func (c *Constellation) StartCache() {
log.Print("Starting AuthCache")
var peers []string
if c.PeerList == nil {
log.Print("Initializing PeerList")
c.PeerList = make(map[string]string)
}
for _, peer := range c.PeerList {
log.Printf("Assigning peer '%s'", peer)
if peer > "" {
peers = append(peers, "http://"+peer+":"+GCPORT)
}
}
c.Peers.Set(peers...)
var authcache = groupcache.NewGroup(c.Groupname, 64<<20, groupcache.GetterFunc(
func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
pod := key
auth := c.GetAuthForPodFromConfig(pod)
if auth > "" {
dest.SetString(auth)
return nil
}
err := fmt.Errorf("Found no value for auth on pod %s, not storing anything", pod)
return err
}))
c.AuthCache = NewCache(authcache)
}
// SetCacheMaxSize enables icon caching if sizeInMB > 0.
func SetCacheMaxSize(sizeInMB int64) {
if sizeInMB > 0 {
iconCache = groupcache.NewGroup("icons", sizeInMB<<20, groupcache.GetterFunc(generatorFunc))
} else {
iconCache = nil
}
}
func main() {
flag.Parse()
peers := gc.NewHTTPPool("http://localhost:" + *port)
peers.Set("http://localhost:8001", "http://localhost:8002")
cg = gc.NewGroup("QRCache", 1<<20, gc.GetterFunc(
func(ctx gc.Context, key string, dest gc.Sink) error {
fmt.Printf("asking for data of %s\n", key)
url := fmt.Sprintf("http://chart.apis.google.com/chart?chs=300x300&cht=qr&choe=UTF-8&chl=%s", key)
resp, err := http.Get(url)
if err != nil {
return nil
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil
}
value := base64.StdEncoding.EncodeToString(body)
dest.SetBytes([]byte(value))
return nil
}))
// run groupcache process in goroutine
go http.ListenAndServe("localhost:"+*port, peers)
http.Handle("/", http.HandlerFunc(QR))
err := http.ListenAndServe(*addr, nil)
if err != nil {
log.Fatal("ListenAndServe:", err)
}
}
func main() {
/* me := "http://127.0.0.1"
peers := groupcache.NewHTTPPool(me)
// Whenever peers change:
peers.Set("http://127.0.0.1", "http://10.0.0.2", "http://10.0.0.3")*/
getter := groupcache.GetterFunc(
func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
// fileName := key
bytes := []byte{'a', 'b', 'c'}
dest.SetBytes(bytes)
return nil
})
func(g *igetter) Get(ctx Content, key string, dest Sink) {
}
grp := groupcache.NewGroup("Grp", 64<<20, getter)
err := grp.Get(ctx groupcache.Context, key string, dest groupcache.Sink)
//hook
/* hook := groupcache.RegisterNewGroupHook(func(grp1 *groupcache.Group) {
fmt.Println(grp1.Name())
fmt.Println("hook")
})*/
fmt.Println(grp.Name())
fmt.Println(grp.Stats)
}
func main() {
flag.Parse()
// Setup the doozer connection.
d, err := doozer.Dial(daddr)
if err != nil {
log.Fatalf("connecting to doozer: %v\n", err)
}
defer d.Close()
// Setup the cache.
pool = groupcache.NewHTTPPool("http://" + addr)
dict = groupcache.NewGroup("dict", 64<<20, groupcache.GetterFunc(
func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
def, err := query(key)
if err != nil {
err = fmt.Errorf("querying remote dictionary: %v", err)
log.Println(err)
return err
}
log.Println("retrieved remote definition for", key)
dest.SetString(def)
return nil
}))
// Start watching for changes and signals.
go watch(d)
// Add the handler for definition requests and then start the
// server.
http.Handle("/define/", http.HandlerFunc(handler))
log.Println(http.ListenAndServe(addr, nil))
}
func main() {
// STARTINIT OMIT
me := "http://10.0.0.1"
peers := groupcache.NewHTTPPool(me)
// Whenever peers change:
peers.Set("http://10.0.0.1", "http://10.0.0.2", "http://10.0.0.3")
// ENDINIT OMIT
// STARTGROUP OMIT
var thumbNails = groupcache.NewGroup("thumbnail", 64<<20, groupcache.GetterFunc(
func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
fileName := key
dest.SetBytes(generateThumbnail(fileName))
return nil
}))
// ENDGROUP OMIT
var ctx groupcache.Context
var w http.ResponseWriter
var r *http.Request
// STARTUSE OMIT
var data []byte
err := thumbNails.Get(ctx, "big-file.jpg",
groupcache.AllocatingByteSliceSink(&data))
// ...
_ = err // OMIT
var modTime time.Time // OMIT
http.ServeContent(w, r, "big-file-thumb.jpg", modTime, bytes.NewReader(data))
// ENDUSE OMIT
}
func main() {
me := ":" + os.Args[1]
peers := groupcache.NewHTTPPool("http://localhost" + me)
peers.Set("http://localhost:8081", "http://localhost:8082",
"http://localhost:8083")
helloworld := groupcache.NewGroup("helloworld",
1024*1024*10, groupcache.GetterFunc(
func(ctx groupcache.Context,
key string,
dest groupcache.Sink) error {
log.Println(me)
dest.SetString(me + "->" + key)
return nil
}))
fmt.Println("GroupName: ", helloworld.Name())
http.HandleFunc("/xbox/", func(w http.ResponseWriter, r *http.Request) {
parts := strings.SplitN(r.URL.Path[len("/xbox/"):], "/", 1)
for _, x := range parts {
fmt.Println("get: ", x)
}
if len(parts) != 1 {
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
var data []byte
helloworld.Get(nil, parts[0], groupcache.AllocatingByteSliceSink(&data))
w.Write(data)
log.Println("Gets: ", helloworld.Stats.Gets.String())
log.Println("Load: ", helloworld.Stats.Loads.String())
log.Println("LocalLoad: ", helloworld.Stats.LocalLoads.String())
log.Println("PeerError: ", helloworld.Stats.PeerErrors.String())
log.Println("PeerLoad: ", helloworld.Stats.PeerLoads.String())
})
http.ListenAndServe(me, nil)
}
func InitRadosCache(name string, size int64, chunkSize int) {
if cacheMade {
panic("groupcache: InitRadosCache must be called only once")
}
cacheMade = true
radosCache = groupcache.NewGroup(name, size, groupcache.GetterFunc(radosGetter))
chunkPool = sync.Pool{
New: func() interface{} { return make([]byte, chunkSize) },
}
}
// NewGroupCache function
func NewGroupCache(cacheSize int64, redisGetFunc GetFunc) Cache {
return &groupCacheMgr{
group: gc.NewGroup(singleGroupName, cacheSize, gc.GetterFunc(func(ctx gc.Context, key string, dest gc.Sink) error {
log.WithField("domain", key).Info("GroupCache:RedisRequest")
redisRequest := api.NewValueRequest(key)
redisGetFunc(redisRequest)
redisResponse := <-redisRequest.Response
log.WithField("domain", key).Info("GroupCache:RedisRequest:Done")
return dest.SetString(redisResponse.Value)
}))}
}
func setupGroupcache(config GroupcacheConfig) error {
if config.GB == 0 {
return nil
}
var cacheBytes int64
cacheBytes = int64(config.GB) << 30
pool := groupcache.NewHTTPPool(config.Host)
if pool != nil {
dvid.Infof("Initializing groupcache with %d GB at %s...\n", config.GB, config.Host)
manager.gcache.cache = groupcache.NewGroup("immutable", cacheBytes, groupcache.GetterFunc(
func(c groupcache.Context, key string, dest groupcache.Sink) error {
// Use KeyValueDB defined as context.
gctx, ok := c.(GroupcacheCtx)
if !ok {
return fmt.Errorf("bad groupcache context: expected GroupcacheCtx, got %v", c)
}
// First four bytes of key is instance ID to isolate groupcache collisions.
tk := TKey(key[4:])
data, err := gctx.KeyValueDB.Get(gctx.Context, tk)
if err != nil {
return err
}
return dest.SetBytes(data)
}))
manager.gcache.supported = make(map[dvid.DataSpecifier]struct{})
for _, dataspec := range config.Instances {
name := strings.Trim(dataspec, "\"")
parts := strings.Split(name, ":")
switch len(parts) {
case 2:
dataid := dvid.GetDataSpecifier(dvid.InstanceName(parts[0]), dvid.UUID(parts[1]))
manager.gcache.supported[dataid] = struct{}{}
default:
dvid.Errorf("bad data instance specification %q given for groupcache support in config file\n", dataspec)
}
}
// If we have additional peers, add them and start a listener via the HTTP port.
if len(config.Peers) > 0 {
peers := []string{config.Host}
peers = append(peers, config.Peers...)
pool.Set(peers...)
dvid.Infof("Groupcache configuration has %d peers in addition to local host.\n", len(config.Peers))
dvid.Infof("Starting groupcache HTTP server on %s\n", config.Host)
http.ListenAndServe(config.Host, http.HandlerFunc(pool.ServeHTTP))
}
}
return nil
}
func main() {
var err error
p = groupcache.NewHTTPPool("http://127.0.0.1:50000")
// update p's list
p.Set("http://127.0.0.1:50001", "http://127.0.0.1:50002")
// create a new group by name and set a function for get a value by key
// if you g.Get KEY at first time, it will store the value use sink, run the function
// if you g.Get KEY not at first time, actually it NOT entry the function
// does not support update a key's value
g = groupcache.NewGroup("dns", 64<<20, groupcache.GetterFunc(
func(ctx groupcache.Context, key string, dest groupcache.Sink) (err error) {
if ctx == nil {
ctx = "f**k"
}
log.Println(key, ctx)
err = dest.SetString(ctx.(string))
return
}))
// let it listen
go http.ListenAndServe("127.0.0.1:50000", nil)
// get a key's value into data
time.Sleep(2 * time.Second)
var data []byte
var key = "key"
err = g.Get("aa", key, groupcache.AllocatingByteSliceSink(&data))
if err != nil {
log.Println("get error:", err)
}
log.Println(string(data))
time.Sleep(2 * time.Second)
key = "key1"
err = g.Get("bb", key, groupcache.AllocatingByteSliceSink(&data))
if err != nil {
log.Println("get error:", err)
}
log.Println(string(data))
time.Sleep(2 * time.Second)
key = "key"
err = g.Get("cc", key, groupcache.AllocatingByteSliceSink(&data))
if err != nil {
log.Println("get error:", err)
}
log.Println(string(data))
}
func (g *GroupCacheProxy) MakeCache(c *Cluster, size int64) CacheGetter {
return groupcache.NewGroup(
"ReticulumCache", size, groupcache.GetterFunc(
func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
// get image from disk
ri := NewImageSpecifier(key)
img_data, err := c.RetrieveImage(ri)
if err != nil {
return err
}
dest.SetBytes([]byte(img_data))
return nil
}))
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
awsAuth, err := aws.EnvAuth()
if err != nil {
log.Fatalf(err.Error())
}
s3conn := s3.New(awsAuth, getRegion())
storage = store.NewS3Store(s3conn)
peers = peer.DebugPool()
peers.SetContext(func(r *http.Request) groupcache.Context {
return fetch.RequestContext(r)
})
cache = groupcache.NewGroup("ImageProxyCache", 64<<20, groupcache.GetterFunc(
func(c groupcache.Context, key string, dest groupcache.Sink) error {
log.Printf("Cache MISS for key -> %s", key)
// Get image data from S3
b, err := fetch.ImageData(storage, c)
if err != nil {
return err
}
return dest.SetBytes(b)
}))
if !*verbose {
logwriter, err := syslog.Dial("udp", "app_syslog:514", syslog.LOG_NOTICE, "vip")
if err != nil {
log.Println(err.Error())
log.Println("using default logger")
} else {
log.SetOutput(logwriter)
}
}
go peers.Listen()
go listenHttp()
go Queue.Start(4)
log.Println("Cache listening on port :" + peers.Port())
s := &http.Server{
Addr: ":" + peers.Port(),
Handler: peers,
}
s.ListenAndServe()
}
func TestGetSetGC(t *testing.T) {
tt := T{1}
mcStorage := NewMcStorage([]string{"localhost:12000"}, "test", 0, reflect.TypeOf(&tt))
mcStorage.Set("1", tt)
res, _ := mcStorage.Get("1")
defer mcStorage.Delete("1")
if reflect.TypeOf(res) != reflect.TypeOf(tt) {
t.Error("res type is not T")
}
ttRes := res.(T)
if ttRes.A != tt.A {
t.Error("res A field is not equals tt field")
}
var groupcache = groupcache.NewGroup("SlowDBCache", 64<<20, groupcache.GetterFunc(
func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
result, err := mcStorage.Get(key)
if err != nil {
return nil
}
bytes, err := json.Marshal(result)
if err != nil {
return nil
}
dest.SetBytes(bytes)
return nil
}))
gcStorage := &GroupCacheKvStorage{groupcache, 0, reflect.TypeOf(&tt)}
res, _ = gcStorage.Get("1")
if reflect.TypeOf(res) != reflect.TypeOf(tt) {
t.Error("res type is not T")
}
ttRes = res.(T)
if ttRes.A != tt.A {
t.Error("res A field is not equals tt field")
}
mcStorage.Delete("1")
res, _ = gcStorage.Get("1")
if reflect.TypeOf(res) != reflect.TypeOf(tt) {
t.Error("res type is not T")
}
ttRes = res.(T)
if ttRes.A != tt.A {
t.Error("res A field is not equals tt field")
}
}
func TestServerErrorImageUnmarshal(t *testing.T) {
srv := &Server{
Group: groupcache.NewGroup(
newTestServerName(),
testSize,
groupcache.GetterFunc(func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
dest.SetBytes(nil)
return nil
}),
),
KeyGenerator: imageserver_cache.StringKeyGenerator("test"),
}
_, err := srv.Get(imageserver.Params{})
if err == nil {
t.Fatal("no error")
}
}
func beChildForTestGRPCPool() {
addrs := strings.Split(*peerAddrs, ",")
server := grpc.NewServer()
p := NewGRPCPool(addrs[*peerIndex], server)
p.Set(addrs...)
getter := groupcache.GetterFunc(func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
dest.SetString(strconv.Itoa(*peerIndex) + ":" + key)
return nil
})
groupcache.NewGroup("grpcPoolTest", 1<<20, getter)
lis, err := net.Listen("tcp", addrs[*peerIndex])
if err != nil {
log.Fatalf("Failed to listen on %s", addrs[*peerIndex])
}
server.Serve(lis)
}
func start() {
pool = groupcache.NewHTTPPool("http://" + addr)
dns = groupcache.NewGroup("dns", 64<<20, groupcache.GetterFunc(
func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
// Lookup the adddress based on the hostname (key).
addrs, err := net.LookupHost(key)
if err != nil {
return err
}
// We'll just store the first.
dest.SetString(addrs[0])
return nil
}))
// Run in the background so we can use the console.
go http.ListenAndServe(addr, nil)
}
func initCacheGroup() {
self := config.Get().CacheSelf()
pool := groupcache.NewHTTPPool(self)
pool.Set(config.Get().CachePeers()...)
if self != "" {
log.Println("Cache listen on:", strings.TrimLeft(self, "http://"))
go http.ListenAndServe(strings.TrimLeft(self, "http://"), http.HandlerFunc(pool.ServeHTTP))
}
cacheGroup = groupcache.NewGroup("imagio-storage", config.Get().CacheSize(), groupcache.GetterFunc(
func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
dest.SetBytes(imgproc.Do(
Construct(new(query.Options), key).(*query.Options),
))
return nil
}),
)
}
func NewDimensionsCache() *DimensionsCache {
g := groupcache.NewGroup("DimensionsCache", 64<<18, groupcache.GetterFunc(
func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
fmt.Printf("Calculating Dimensions for %s\n", key)
u, err := url.Parse(key)
if err != nil {
return err
}
result := calculateDimensions(u)
var buffer = bytes.Buffer{}
binary.Write(&buffer, binary.BigEndian, result)
dest.SetBytes(buffer.Bytes())
fmt.Println("Filled Cache")
return nil
}))
dc := DimensionsCache{g}
return &dc
}
func main() {
flag.Set("logtostderr", "true")
flag.Parse()
me := ":8080"
peers := groupcache.NewHTTPPool("http://localhost" + me)
peers.Set("http://localhost:8081", "http://localhost:8082", "http://localhost:8083")
helloworld := groupcache.NewGroup("helloworld", 1024*1024*1024*16, groupcache.GetterFunc(
func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
glog.Infof("%v, key = %v", me, key)
dest.SetString(key)
return nil
}))
glog.Infof("GroupName: %v", helloworld.Name())
http.HandleFunc("/xbox/",
func(w http.ResponseWriter, r *http.Request) {
parts := strings.SplitN(r.URL.Path[len("/xbox/"):], "/", 1)
glog.Infof("parts: %v", parts)
if len(parts) != 1 {
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
var data []byte
helloworld.Get(nil, parts[0], groupcache.AllocatingByteSliceSink(&data))
w.Write(data)
glog.Infof("data: %s", data)
glog.Infof("Stats: %#v", helloworld.Stats)
//glog.Infof("Gets: %v", helloworld.Stats.Gets.String())
//glog.Infof("Load: %v", helloworld.Stats.Loads.String())
//glog.Infof("LocalLoad: %v", helloworld.Stats.LocalLoads.String())
//glog.Infof("PeerError: %v", helloworld.Stats.PeerErrors.String())
//glog.Infof("PeerLoad: %v", helloworld.Stats.PeerLoads.String())
})
http.ListenAndServe(me, nil)
}
func main() {
if !flag.Parsed() {
flag.Parse()
}
// etcd := os.Getenv("ETCD_ADDR")
ip := os.Getenv("PUBLIC_HOSTIP")
pport := os.Getenv("PUBLIC_PORT")
cache := etcd_groupcache.New(fmt.Sprintf("http://%s:%s", ip, pport), nil, nil)
peers := []string{fmt.Sprintf("http://%s:8080", ip), fmt.Sprintf("http://%s:8081", ip)}
log.Infoln(peers)
cache.Set(peers...)
http.Handle("/_groupcache/", cache)
var stringcache = groupcache.NewGroup("SlowDBCache", 64<<20, groupcache.GetterFunc(
func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
log.Infoln(pport)
dest.SetString(key + "heyo")
return nil
}))
go func() {
var data []byte
key := "some key"
err := stringcache.Get(nil, key, groupcache.AllocatingByteSliceSink(&data))
if err != nil {
log.Errorln(err)
}
log.Infoln(string(data))
}()
log.Infoln("Groupcache Listening 8080")
if err := http.ListenAndServe(":"+*port, nil); err != nil {
panic(err)
}
}
func ServeDir(mountpoint string, target string, limit int, me string, peerlist []string) {
peers := groupcache.NewHTTPPool(me)
if peerlist != nil && len(peerlist) > 0 {
peers.Set(peerlist...)
}
filecache = groupcache.NewGroup("filecache", int64(limit)<<20, groupcache.GetterFunc(
func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
contents, err := ioutil.ReadFile(key)
dest.SetBytes(contents)
return err
}))
c, err := fuse.Mount(mountpoint)
if err != nil {
log.Fatal(err)
}
fs.Serve(c, TargetDir{target})
}
func main() {
// setup group
stringGroup = gc.NewGroup(stringGroupName, cacheSize, gc.GetterFunc(func(_ gc.Context, key string, dest gc.Sink) error {
fmt.Fprintf(os.Stdout, "Setting cache key: %s\n", key)
return dest.SetString("ECHO:" + key)
}))
// Get Items
for j := 0; j < 4; j++ {
for i := 0; i < 4; i++ {
var s string
if err := stringGroup.Get(dummyCtx, "TestCaching-key"+strconv.Itoa(i), gc.StringSink(&s)); err != nil {
fmt.Fprintf(os.Stdout, "TestCaching-key value: failed%s\n", err)
return
}
fmt.Fprintf(os.Stdout, "TestCaching-key value:%s\n", s)
}
fmt.Fprintln(os.Stdout, "---")
}
fmt.Fprintln(os.Stdout, "Done.")
}
func main() {
var port = flag.String("port", "8001", "groupcache port")
flag.Parse()
peers := groupcache.NewHTTPPool("http://localhost:" + *port)
client := new(client.Client)
var stringcache = groupcache.NewGroup("SlowDBCache", 64<<20, groupcache.GetterFunc(
func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
result := client.Get(key)
fmt.Printf("asking for %s from dbserver\n", key)
dest.SetBytes([]byte(result))
return nil
}))
peers.Set("http://localhost:8001", "http://localhost:8002", "http://localhost:8003")
frontendServer := NewServer(stringcache)
i, err := strconv.Atoi(*port)
if err != nil {
// handle error
fmt.Println(err)
os.Exit(2)
}
var frontEndport = ":" + strconv.Itoa(i+1000)
go frontendServer.Start(frontEndport)
fmt.Println(stringcache)
fmt.Println("cachegroup slave starting on " + *port)
fmt.Println("frontend starting on " + frontEndport)
http.ListenAndServe("127.0.0.1:"+*port, http.HandlerFunc(peers.ServeHTTP))
}
func main() {
rpcPort := flag.Int("rpcPort", 9090, "port for groupcache ipc")
proxyPort := flag.Int("proxyPort", 8080, "port for http proxy")
peerPort := flag.Int("peerPort", 7070, "port for receiving peer updates")
flag.Parse()
me := "http://localhost:" + strconv.Itoa(*rpcPort)
poolOpts := &groupcache.HTTPPoolOptions{
BasePath: "/stuff",
}
cache := groupcache.NewHTTPPoolOpts(me, poolOpts)
groupcache.NewGroup("", 64<<20, groupcache.GetterFunc(
func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
log.V(2).Info("trying to get ", key)
client := &http.Client{
Timeout: time.Second * 5,
}
res, err := client.Get(key)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return errors.New("server returned: " + string(res.Status))
}
data, err := ioutil.ReadAll(res.Body)
if err != nil {
return err
}
dest.SetBytes(data)
return nil
},
))
cache.Set("http://localhost:9090", "http://localhost:9091", "http://localhost:9092")
proxy := goproxy.NewProxyHttpServer()
//proxy.Verbose = true
proxy.OnRequest().DoFunc(
func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
log.V(2).Infof("Proxy got request for: %+v", *r.URL)
r.URL = &url.URL{
Path: "/stuff/" + r.URL.Scheme + "://" + r.URL.Host + r.URL.Path,
}
pr := NewProxyResponse()
cache.ServeHTTP(pr, r)
log.V(3).Infof("body: %v", pr.response.Body)
data, err := ioutil.ReadAll(pr.response.Body)
if err != nil {
log.V(1).Infof("Failed to read response %v", err)
}
return r, goproxy.NewResponse(r, goproxy.ContentTypeText, http.StatusForbidden, string(data))
},
)
mux := http.NewServeMux()
mux.HandleFunc("/peers", func(w http.ResponseWriter, r *http.Request) {
data, err := ioutil.ReadAll(r.Body)
if err != nil {
log.V(1).Infof("Failed to read response %v", err)
}
cache.Set(strings.Split(string(data), ",")...)
})
log.Infoln("Starting rpc instance.")
go func() { log.Fatal(http.ListenAndServe(":"+strconv.Itoa(*rpcPort), cache)) }()
log.Infoln("Starting peer update listener.")
go func() { log.Fatal(http.ListenAndServe(":"+strconv.Itoa(*peerPort), mux)) }()
log.Infoln("Starting proxy instance.")
log.Fatal(http.ListenAndServe(":"+strconv.Itoa(*proxyPort), proxy))
}
"github.com/golang/groupcache"
"io/ioutil"
"log"
"net/http"
)
func getter(ctx groupcache.Context, key string, dest groupcache.Sink) error {
resp, _ := http.Get(key)
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
log.Printf("asking for %s from solr\n", key)
dest.SetString(string(body))
return nil
}
var stringcache = groupcache.NewGroup("Solr", 64<<20, groupcache.GetterFunc(getter))
func proxyHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("content-type", "application/json")
url := "http://moapi.com:8080" + r.URL.Path + "?" + r.URL.RawQuery
var s string
stringcache.Get(nil, url, groupcache.StringSink(&s))
fmt.Fprintf(w, s)
}
func startProxy() {
log.Print("starting")
handler := http.NewServeMux()
handler.HandleFunc("/select/", proxyHandler)
server := &http.Server{Addr: ":8889", Handler: handler}
func main() {
// Parse the flags
flag.Parse()
// Create a new groupcache pool
pool := groupcache.NewHTTPPool(*cachePublic)
pool.Set(strings.Split(*cachePeers, ",")...)
// Listen and serve the groupcache pool
cacheServer := http.Server{
Addr: *cacheBind,
Handler: pool,
}
go func() {
log.Printf("Starting up the cache HTTP server on address %s", *cacheBind)
err := cacheServer.ListenAndServe()
if err != nil {
log.Fatal(err)
}
}()
// Create a new groupcache pool
cache := groupcache.NewGroup("ritratt", *cacheSize, groupcache.GetterFunc(func(ctx groupcache.Context, url string, dest groupcache.Sink) error {
// First try with https
schema := "https://"
resp, err := http.Head("https://" + url)
if err != nil {
log.Printf("[https] Error while querying %s: %s", url, err)
// https doesn't work, try http
schema = "http://"
resp, err = http.Head("http://" + url)
if err != nil {
log.Printf("[http] Error while querying %s: %s", url, err)
}
}
// Content-Type of the result has to start with image/
// We also don't support SVGs, check out this link for more information:
// https://www.owasp.org/images/0/03/Mario_Heiderich_OWASP_Sweden_The_image_that_called_me.pdf
/*ct := resp.Header.Get("Content-Type")
if !strings.HasPrefix(ct, "image/") || strings.Contains(ct, "image/svg+xml") {
log.Printf("[head] Invalid Content-Type of %s", url)
return ErrInvalidContentType
}*/
// Query the proper URL, now including the body
resp, err = http.Get(schema + url)
defer resp.Body.Close()
if err != nil {
log.Printf("[get] Error while querying %s: %s", url, err)
} else {
log.Printf("[get] Loaded %s", url)
}
// Content-Type check #2
ct := resp.Header.Get("Content-Type")
if !strings.HasPrefix(ct, "image/") || strings.Contains(ct, "image/svg+xml") {
log.Printf("[get] Invalid Content-Type of %s", url)
return ErrInvalidContentType
}
var body []byte
// Read the dlenght
cls := resp.Header.Get("Content-Length")
if cls != "" {
cl, err := strconv.Atoi(cls)
if err != nil {
log.Print(err)
} else {
if cl > 25*1024*1024 {
return ErrTooBig
} else {
body, err = ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
}
}
} else {
body = []byte{}
totalRead := 0
for {
chunk := make([]byte, 1024*1024)
read, err := resp.Body.Read(chunk)
totalRead += read
if err == io.EOF {
break
}
if err != nil {
return err
}
body = append(body, chunk[:read]...)
if totalRead > 25*1024*1024 {
return ErrTooBig
}
}
}
// Put the body into cache with the Content-Type
dest.SetString(ct + ";" + string(body))
return nil
}))
log.Printf("Starting up the proxy HTTP server on address %s", *proxyBind)
proxyServer := http.Server{
Addr: *proxyBind,
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Index page
if len(r.RequestURI) < 3 || r.RequestURI[:3] != "/i/" {
w.Write([]byte("lavab/ritratt"))
return
}
// Get the data from groupcache
var data string
err := cache.Get(nil, r.RequestURI[3:], groupcache.StringSink(&data))
if err != nil {
w.Write([]byte(err.Error()))
return
}
// Split the result into two parts
parts := strings.SplitN(data, ";", 2)
// Set the content type
w.Header().Set("Content-Type", parts[0])
// Write the body
w.Write([]byte(parts[1]))
}),
}
log.Fatal(proxyServer.ListenAndServe())
}
func TestGRPCPool(t *testing.T) {
if *peerChild {
beChildForTestGRPCPool()
os.Exit(0)
}
const (
nChild = 5
nGets = 5000
)
var childAddr []string
for i := 0; i < nChild; i++ {
childAddr = append(childAddr, pickFreeAddr(t))
}
var cmds []*exec.Cmd
var wg sync.WaitGroup
for i := 0; i < nChild; i++ {
cmd := exec.Command(os.Args[0],
"--test.run=TestGRPCPool",
"--test_peer_child",
"--test_peer_addrs="+strings.Join(childAddr, ","),
"--test_peer_index="+strconv.Itoa(i),
)
cmds = append(cmds, cmd)
wg.Add(1)
if err := cmd.Start(); err != nil {
t.Fatal("failed to start child process: ", err)
}
go awaitAddrReady(t, childAddr[i], &wg)
}
defer func() {
for i := 0; i < nChild; i++ {
if cmds[i].Process != nil {
cmds[i].Process.Kill()
}
}
}()
wg.Wait()
// Use a dummy self address so that we don't handle gets in-process.
p := NewGRPCPool("should-be-ignored", grpc.NewServer())
p.Set(childAddr...)
// Dummy getter function. Gets should go to children only.
// The only time this process will handle a get is when the
// children can't be contacted for some reason.
getter := groupcache.GetterFunc(func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
return errors.New("parent getter called; something's wrong")
})
g := groupcache.NewGroup("grpcPoolTest", 1<<20, getter)
for _, key := range testKeys(nGets) {
var value string
if err := g.Get(nil, key, groupcache.StringSink(&value)); err != nil {
t.Fatal(err)
}
if suffix := ":" + key; !strings.HasSuffix(value, suffix) {
t.Errorf("Get(%q) = %q, want value ending in %q", key, value, suffix)
}
t.Logf("Get key=%q, value=%q (peer:key)", key, value)
}
}