Пример #1
0
func (a *Application) ImageCleanup(filepath string) {
	con := a.KVStore.Connection()
	defer con.Close()

	childrenPath := filepath + ":children"

	// Get the list of items to cleanup.
	children := con.SetMembers(childrenPath)

	// Delete them right away, we don't care about them anymore.
	a.Logger.Infof("Deleting children set: %s", childrenPath)
	_ = con.Delete(childrenPath)

	// No children? Okay..
	if children == nil {
		return
	}

	for _, s := range children {
		key, err := gokvstores.String(s)
		if err != nil {
			// Should really be a string here, but if it's not, keep going...
			continue
		}

		// Now, every child is a hash which points to a key/value pair in
		// KVStore which in turn points to a file in dst storage.

		dstfile, err := gokvstores.String(con.Get(key))
		if err != nil {
			// Well, what can we do about it, let's just continue.
			continue
		}

		// And try to delete it all. Ignore errors.
		a.Logger.Infof("Deleting child %s and its KV store entry %s", dstfile, key)
		_ = a.DestStorage.Delete(dstfile)
		_ = con.Delete(key)
	}
}
Пример #2
0
func TestStorageApplicationWithURL(t *testing.T) {
	ts := newHTTPServer()
	defer ts.Close()
	defer ts.CloseClientConnections()

	tmp := os.TempDir()

	content := `{
	  "debug": true,
	  "port": 3001,
	  "kvstore": {
		"prefix": "picfit:",
		"type": "redis",
		"host": "127.0.0.1",
		"db": 0,
		"password": "",
		"port": 6379
	  },
	  "storage": {
		"src": {
		  "type": "fs",
		  "location": "%s"
		}
	  }
	}`

	content = fmt.Sprintf(content, tmp)

	app, err := NewFromConfig(content)
	assert.Nil(t, err)

	connection := app.KVStore.Connection()
	defer connection.Close()

	assert.NotNil(t, app.SourceStorage)
	assert.Equal(t, app.SourceStorage, app.DestStorage)

	filename := "avatar.png"

	u, _ := url.Parse(ts.URL + "/" + filename)

	location := fmt.Sprintf("http://example.com/display?url=%s&w=100&h=100&op=resize", u.String())

	request, _ := http.NewRequest("GET", location, nil)

	res := httptest.NewRecorder()

	handler := app.ServeHTTP(ImageHandler)

	handler.ServeHTTP(res, request)

	assert.Equal(t, res.Code, 200)

	// We wait until the goroutine to save the file on disk is finished
	timer1 := time.NewTimer(time.Second * 2)
	<-timer1.C

	etag := res.Header().Get("ETag")

	key := app.WithPrefix(etag)

	assert.True(t, connection.Exists(key))

	filepath, _ := gokvstores.String(connection.Get(key))

	parts := strings.Split(filepath, "/")

	assert.Equal(t, len(parts), 1)

	assert.True(t, app.SourceStorage.Exists(filepath))
}
Пример #3
0
func (a *Application) ImageFileFromRequest(req *Request, async bool, load bool) (*image.ImageFile, error) {
	var file *image.ImageFile = &image.ImageFile{
		Key:     req.Key,
		Storage: a.DestStorage,
		Headers: map[string]string{},
	}
	var err error

	key := a.WithPrefix(req.Key)

	// Image from the KVStore found
	stored, err := gokvstores.String(req.Connection.Get(key))

	file.Filepath = stored

	if stored != "" {
		a.Logger.Infof("Key %s found in kvstore: %s", key, stored)

		if load {
			file, err = image.FromStorage(a.DestStorage, stored)

			if err != nil {
				return nil, err
			}
		}
	} else {
		a.Logger.Infof("Key %s not found in kvstore", key)

		// Image not found from the KVStore, we need to process it
		// URL available in Query String
		if req.URL != nil {
			file, err = image.FromURL(req.URL)
		} else {
			// URL provided we use http protocol to retrieve it
			file, err = image.FromStorage(a.SourceStorage, req.Filepath)
		}

		if err != nil {
			return nil, err
		}

		file, err = a.Engine.Transform(file, req.Operation, req.QueryString)

		if err != nil {
			return nil, err
		}

		file.Filepath = fmt.Sprintf("%s.%s", a.ShardFilename(req.Key), file.Format())
	}

	file.Key = req.Key
	file.Storage = a.DestStorage

	file.Headers["Content-Type"] = file.ContentType()
	file.Headers["ETag"] = req.Key

	if stored == "" {
		if async == true {
			go a.Store(req.Filepath, file)
		} else {
			err = a.Store(req.Filepath, file)
		}
	}

	return file, err
}
Пример #4
0
func TestStorageApplicationWithPath(t *testing.T) {
	ts := newHTTPServer()
	defer ts.Close()
	defer ts.CloseClientConnections()

	tmp := os.TempDir()

	f, err := os.Open("testdata/avatar.png")
	assert.Nil(t, err)
	defer f.Close()

	body, err := ioutil.ReadAll(f)
	assert.Nil(t, err)

	// We store the image at the tmp location to access it
	// with the SourceStorage
	ioutil.WriteFile(path.Join(tmp, "avatar.png"), body, 0755)

	content := `{
	  "debug": true,
	  "port": 3001,
	  "kvstore": {
		"prefix": "picfit:",
		"type": "redis",
		"host": "127.0.0.1",
		"db": 0,
		"password": "",
		"port": 6379
	  },
	  "storage": {
		"src": {
		  "type": "fs",
		  "location": "%s",
		  "base_url": "http://img.example.com"
		}
	  },
	  "shard": {
		"width": 1,
		"depth": 2
	  }
	}`

	content = fmt.Sprintf(content, tmp)

	app, err := NewFromConfig(content)
	assert.Nil(t, err)

	negroni := app.InitRouter()

	connection := app.KVStore.Connection()
	defer connection.Close()

	location := "http://example.com/display/resize/100x100/avatar.png"

	request, _ := http.NewRequest("GET", location, nil)

	res := httptest.NewRecorder()

	negroni.ServeHTTP(res, request)

	assert.Equal(t, 200, res.Code)

	// We wait until the goroutine to save the file on disk is finished
	timer1 := time.NewTimer(time.Second * 2)
	<-timer1.C

	etag := res.Header().Get("ETag")

	key := app.WithPrefix(etag)

	assert.True(t, connection.Exists(key))

	filepath, _ := gokvstores.String(connection.Get(key))

	parts := strings.Split(filepath, "/")

	assert.Equal(t, len(parts), 3)
	assert.Equal(t, len(parts[0]), 1)
	assert.Equal(t, len(parts[1]), 1)

	assert.True(t, app.SourceStorage.Exists(filepath))

	location = "http://example.com/get/resize/100x100/avatar.png"

	request, _ = http.NewRequest("GET", location, nil)

	res = httptest.NewRecorder()

	negroni.ServeHTTP(res, request)

	assert.Equal(t, 200, res.Code)
	assert.Equal(t, "application/json", res.Header().Get("Content-Type"))

	var dat map[string]interface{}

	err = json.Unmarshal(res.Body.Bytes(), &dat)

	assert.Nil(t, err)

	expected := "http://img.example.com/" + filepath

	assert.Equal(t, expected, dat["url"].(string))

	location = "http://example.com/redirect/resize/100x100/avatar.png"

	request, _ = http.NewRequest("GET", location, nil)

	res = httptest.NewRecorder()

	negroni.ServeHTTP(res, request)

	assert.Equal(t, expected, res.Header().Get("Location"))
	assert.Equal(t, 301, res.Code)
}