func (a *Application) ImageCleanup(filepath string) {
con := a.KVStore.Connection()
defer con.Close()
childrenPath := filepath + ":children"
// Get the list of items to cleanup.
children := con.SetMembers(childrenPath)
// Delete them right away, we don't care about them anymore.
a.Logger.Infof("Deleting children set: %s", childrenPath)
_ = con.Delete(childrenPath)
// No children? Okay..
if children == nil {
return
}
for _, s := range children {
key, err := gokvstores.String(s)
if err != nil {
// Should really be a string here, but if it's not, keep going...
continue
}
// Now, every child is a hash which points to a key/value pair in
// KVStore which in turn points to a file in dst storage.
dstfile, err := gokvstores.String(con.Get(key))
if err != nil {
// Well, what can we do about it, let's just continue.
continue
}
// And try to delete it all. Ignore errors.
a.Logger.Infof("Deleting child %s and its KV store entry %s", dstfile, key)
_ = a.DestStorage.Delete(dstfile)
_ = con.Delete(key)
}
}
func TestStorageApplicationWithURL(t *testing.T) {
ts := newHTTPServer()
defer ts.Close()
defer ts.CloseClientConnections()
tmp := os.TempDir()
content := `{
"debug": true,
"port": 3001,
"kvstore": {
"prefix": "picfit:",
"type": "redis",
"host": "127.0.0.1",
"db": 0,
"password": "",
"port": 6379
},
"storage": {
"src": {
"type": "fs",
"location": "%s"
}
}
}`
content = fmt.Sprintf(content, tmp)
app, err := NewFromConfig(content)
assert.Nil(t, err)
connection := app.KVStore.Connection()
defer connection.Close()
assert.NotNil(t, app.SourceStorage)
assert.Equal(t, app.SourceStorage, app.DestStorage)
filename := "avatar.png"
u, _ := url.Parse(ts.URL + "/" + filename)
location := fmt.Sprintf("http://example.com/display?url=%s&w=100&h=100&op=resize", u.String())
request, _ := http.NewRequest("GET", location, nil)
res := httptest.NewRecorder()
handler := app.ServeHTTP(ImageHandler)
handler.ServeHTTP(res, request)
assert.Equal(t, res.Code, 200)
// We wait until the goroutine to save the file on disk is finished
timer1 := time.NewTimer(time.Second * 2)
<-timer1.C
etag := res.Header().Get("ETag")
key := app.WithPrefix(etag)
assert.True(t, connection.Exists(key))
filepath, _ := gokvstores.String(connection.Get(key))
parts := strings.Split(filepath, "/")
assert.Equal(t, len(parts), 1)
assert.True(t, app.SourceStorage.Exists(filepath))
}
func (a *Application) ImageFileFromRequest(req *Request, async bool, load bool) (*image.ImageFile, error) {
var file *image.ImageFile = &image.ImageFile{
Key: req.Key,
Storage: a.DestStorage,
Headers: map[string]string{},
}
var err error
key := a.WithPrefix(req.Key)
// Image from the KVStore found
stored, err := gokvstores.String(req.Connection.Get(key))
file.Filepath = stored
if stored != "" {
a.Logger.Infof("Key %s found in kvstore: %s", key, stored)
if load {
file, err = image.FromStorage(a.DestStorage, stored)
if err != nil {
return nil, err
}
}
} else {
a.Logger.Infof("Key %s not found in kvstore", key)
// Image not found from the KVStore, we need to process it
// URL available in Query String
if req.URL != nil {
file, err = image.FromURL(req.URL)
} else {
// URL provided we use http protocol to retrieve it
file, err = image.FromStorage(a.SourceStorage, req.Filepath)
}
if err != nil {
return nil, err
}
file, err = a.Engine.Transform(file, req.Operation, req.QueryString)
if err != nil {
return nil, err
}
file.Filepath = fmt.Sprintf("%s.%s", a.ShardFilename(req.Key), file.Format())
}
file.Key = req.Key
file.Storage = a.DestStorage
file.Headers["Content-Type"] = file.ContentType()
file.Headers["ETag"] = req.Key
if stored == "" {
if async == true {
go a.Store(req.Filepath, file)
} else {
err = a.Store(req.Filepath, file)
}
}
return file, err
}
func TestStorageApplicationWithPath(t *testing.T) {
ts := newHTTPServer()
defer ts.Close()
defer ts.CloseClientConnections()
tmp := os.TempDir()
f, err := os.Open("testdata/avatar.png")
assert.Nil(t, err)
defer f.Close()
body, err := ioutil.ReadAll(f)
assert.Nil(t, err)
// We store the image at the tmp location to access it
// with the SourceStorage
ioutil.WriteFile(path.Join(tmp, "avatar.png"), body, 0755)
content := `{
"debug": true,
"port": 3001,
"kvstore": {
"prefix": "picfit:",
"type": "redis",
"host": "127.0.0.1",
"db": 0,
"password": "",
"port": 6379
},
"storage": {
"src": {
"type": "fs",
"location": "%s",
"base_url": "http://img.example.com"
}
},
"shard": {
"width": 1,
"depth": 2
}
}`
content = fmt.Sprintf(content, tmp)
app, err := NewFromConfig(content)
assert.Nil(t, err)
negroni := app.InitRouter()
connection := app.KVStore.Connection()
defer connection.Close()
location := "http://example.com/display/resize/100x100/avatar.png"
request, _ := http.NewRequest("GET", location, nil)
res := httptest.NewRecorder()
negroni.ServeHTTP(res, request)
assert.Equal(t, 200, res.Code)
// We wait until the goroutine to save the file on disk is finished
timer1 := time.NewTimer(time.Second * 2)
<-timer1.C
etag := res.Header().Get("ETag")
key := app.WithPrefix(etag)
assert.True(t, connection.Exists(key))
filepath, _ := gokvstores.String(connection.Get(key))
parts := strings.Split(filepath, "/")
assert.Equal(t, len(parts), 3)
assert.Equal(t, len(parts[0]), 1)
assert.Equal(t, len(parts[1]), 1)
assert.True(t, app.SourceStorage.Exists(filepath))
location = "http://example.com/get/resize/100x100/avatar.png"
request, _ = http.NewRequest("GET", location, nil)
res = httptest.NewRecorder()
negroni.ServeHTTP(res, request)
assert.Equal(t, 200, res.Code)
assert.Equal(t, "application/json", res.Header().Get("Content-Type"))
var dat map[string]interface{}
err = json.Unmarshal(res.Body.Bytes(), &dat)
assert.Nil(t, err)
expected := "http://img.example.com/" + filepath
assert.Equal(t, expected, dat["url"].(string))
location = "http://example.com/redirect/resize/100x100/avatar.png"
request, _ = http.NewRequest("GET", location, nil)
res = httptest.NewRecorder()
negroni.ServeHTTP(res, request)
assert.Equal(t, expected, res.Header().Get("Location"))
assert.Equal(t, 301, res.Code)
}