// CheckBucket returns whether the owner has a bucket named as given
func (root hier) CheckBucket(owner s3intf.Owner, bucket string) bool {
dh, err := os.Open(filepath.Join(string(root), owner.ID(), bucket))
if err != nil {
return false
}
dh.Close()
return true
}
// List lists a bucket, all objects Key starts with prefix, delimiter segments
// Key, thus the returned commonprefixes (think a generalized filepath
// structure, where / is the delimiter, a commonprefix is a subdir)
func (root hier) List(owner s3intf.Owner, bucket, prefix, delimiter, marker string,
limit, skip int) (
objects []s3intf.Object, commonprefixes []string,
truncated bool, err error) {
dh, e := os.Open(filepath.Join(string(root), owner.ID(), bucket))
if e != nil {
err = e
return
}
defer dh.Close()
var (
infos []os.FileInfo
key, etag string
md5hash []byte
ok bool
)
objects = make([]s3intf.Object, 0, 64)
f := s3intf.NewListFilter(prefix, delimiter, marker, limit, skip)
OUTER:
for {
infos, e = dh.Readdir(limit)
if e != nil {
if e == io.EOF {
break
}
err = e
return
}
for _, fi := range infos {
if key, _, _, md5hash, err = decodeFilename(fi.Name()); err != nil {
return
}
if ok, e = f.Check(key); e != nil {
if e == io.EOF {
break OUTER
}
err = fmt.Errorf("error checking %s: %s", key, e)
return
} else if ok {
if md5hash != nil && len(md5hash) == 16 {
etag = hex.EncodeToString(md5hash)
} else {
etag = ""
}
objects = append(objects,
s3intf.Object{Key: string(key), Owner: owner,
ETag: etag, LastModified: fi.ModTime(), Size: fi.Size()})
}
}
}
commonprefixes, truncated = f.Result()
return
}
// CheckBucket returns whether the owner has a bucket named as given
func (m master) CheckBucket(owner s3intf.Owner, bucket string) bool {
m.Lock()
defer m.Unlock()
if o, ok := m.owners[owner.ID()]; ok {
o.Lock()
_, ok = o.buckets[bucket]
o.Unlock()
return ok
}
return false
}
// Put puts a file as a new object into the bucket
func (root hier) Put(owner s3intf.Owner, bucket, object, filename, media string,
body io.Reader, size int64, md5hash []byte) error {
fh, err := os.Create(filepath.Join(string(root), owner.ID(), bucket,
encodeFilename(object, filename, media, string(md5hash))))
if err != nil {
return err
}
_, err = io.Copy(fh, body)
fh.Close()
return err
}
// ListBuckets list all buckets owned by the given owner
func (m master) ListBuckets(owner s3intf.Owner) ([]s3intf.Bucket, error) {
m.Lock()
o, ok := m.owners[owner.ID()]
m.Unlock()
if !ok {
return nil, fmt.Errorf("unkown owner %s", owner.ID())
}
buckets := make([]s3intf.Bucket, len(o.buckets))
i := 0
for k, b := range o.buckets {
buckets[i].Name = k
buckets[i].Created = b.created
i++
}
return buckets, nil
}
// DelBucket deletes a bucket
func (root hier) DelBucket(owner s3intf.Owner, bucket string) error {
dh, err := os.Open(filepath.Join(string(root), owner.ID(), bucket))
if err != nil {
return err
}
infos, err := dh.Readdir(1)
if err != nil && err != io.EOF {
dh.Close()
return err
}
if len(infos) > 0 {
dh.Close()
return errors.New("cannot delete non-empty bucket")
}
nm := dh.Name()
dh.Close()
return os.Remove(nm)
}
// Del deletes the object from the bucket
func (m master) Del(owner s3intf.Owner, bucket, object string) (err error) {
m.Lock()
o, ok := m.owners[owner.ID()]
m.Unlock()
if !ok {
return errors.New("cannot find owner " + owner.ID())
}
o.Lock()
b, ok := o.buckets[bucket]
o.Unlock()
if !ok {
return errors.New("cannot find bucket " + bucket)
}
if err = b.db.BeginTransaction(); err != nil {
return fmt.Errorf("cannot start transaction: %s", err)
}
defer func() {
if err != nil {
b.db.Rollback()
}
}()
val, e := b.db.Extract(nil, []byte(object))
if e != nil {
err = fmt.Errorf("cannot get %s object: %s", object, e)
return
}
if val == nil {
err = s3intf.NotFound
return
}
vi := new(weedutils.ValInfo)
if err = vi.Decode(val); err != nil {
err = fmt.Errorf("error deserializing %s: %s", val, err)
return
}
if err = m.wm.Delete(vi.Fid); err != nil {
return
}
err = nil
return b.db.Commit()
}
func (root hier) findFile(owner s3intf.Owner, bucket, object string) (string, error) {
dh, err := os.Open(filepath.Join(string(root), owner.ID(), bucket))
if err != nil {
return "", err
}
defer dh.Close()
prefix := encodeFilename(object, "")
var names []string
for err == nil {
if names, err = dh.Readdirnames(1000); err != nil && err != io.EOF {
return "", err
}
for _, nm := range names {
if strings.HasPrefix(nm, prefix) {
return filepath.Join(dh.Name(), nm), nil
}
}
}
return "", nil
}
// CreateBucket creates a new bucket
func (m master) CreateBucket(owner s3intf.Owner, bucket string) error {
m.Lock()
defer m.Unlock()
o, ok := m.owners[owner.ID()]
if !ok {
dir := filepath.Join(m.baseDir, owner.ID())
err := os.MkdirAll(dir, 0750)
if err != nil {
return err
}
o = wOwner{dir: dir, buckets: make(map[string]wBucket, 1)}
m.owners[owner.ID()] = o
//} else if o.buckets == nil {
// o.buckets = make(map[string]wBucket, 1)
}
o.Lock()
defer o.Unlock()
_, ok = o.buckets[bucket]
if ok {
return nil //AlreadyExists ?
}
b := wBucket{filename: filepath.Join(o.dir, bucket+".kv"), created: time.Now()}
var err error
if b.db, err = kv.Create(b.filename, kvOptions()); err != nil {
return err
}
o.buckets[bucket] = b
return nil
}
// DelBucket deletes a bucket
func (m master) DelBucket(owner s3intf.Owner, bucket string) error {
m.Lock()
defer m.Unlock()
o, ok := m.owners[owner.ID()]
if !ok {
return fmt.Errorf("unknown owner %s", owner.ID())
}
o.Lock()
defer o.Unlock()
b, ok := o.buckets[bucket]
if !ok {
return fmt.Errorf("bucket %s not exists!", bucket)
}
if k, v, err := b.db.First(); err != nil {
return err
} else if k != nil || v != nil {
return errors.New("cannot delete non-empty bucket")
}
b.db.Close()
b.db = nil
delete(o.buckets, bucket)
return os.Remove(b.filename)
}
// Get retrieves an object from the bucket
func (m master) Get(owner s3intf.Owner, bucket, object string) (
filename, media string, body io.ReadCloser, size int64, md5 []byte, err error) {
m.Lock()
o, ok := m.owners[owner.ID()]
m.Unlock()
if !ok {
err = errors.New("cannot find owner " + owner.ID())
return
}
o.Lock()
b, ok := o.buckets[bucket]
o.Unlock()
if !ok {
err = errors.New("cannot find bucket " + bucket)
return
}
val, e := b.db.Get(nil, []byte(object))
if e != nil {
err = fmt.Errorf("cannot get %s object: %s", object, e)
return
}
if val == nil {
err = s3intf.NotFound
return
}
vi := new(weedutils.ValInfo)
if err = vi.Decode(val); err != nil {
err = fmt.Errorf("error deserializing %s: %s", val, err)
return
}
filename, media, size, md5 = vi.Filename, vi.ContentType, vi.Size, vi.MD5
body, err = m.wm.Download(vi.Fid)
return
}
// ListBuckets list all buckets owned by the given owner
func (root hier) ListBuckets(owner s3intf.Owner) ([]s3intf.Bucket, error) {
dn := filepath.Join(string(root), owner.ID())
dh, err := os.Open(dn)
if err != nil {
if _, ok := err.(*os.PathError); ok {
os.MkdirAll(dn, 0750)
dh, err = os.Open(dn)
}
if err != nil {
return nil, err
}
}
defer dh.Close()
infos, err := dh.Readdir(1000)
if err != nil && err != io.EOF {
return nil, err
}
buckets := make([]s3intf.Bucket, len(infos))
for i, fi := range infos {
buckets[i].Name = fi.Name()
buckets[i].Created = fi.ModTime()
}
return buckets, nil
}
func doReq(t *testing.T, method, path string, body io.Reader, check ResponseChecker) {
req, err := http.NewRequest(method, path, body)
if err != nil {
t.Fatalf("cannot create request: " + err.Error())
}
req.Host = serviceHost
//req.URL.Host = req.Host
var o s3intf.Owner
for i, b := range backers {
if o, err = b.GetOwner("test"); err != nil {
t.Errorf("cannot get owner for test: %s", err)
continue
}
if Debug {
log.Printf("===")
s3intf.Debug = Debug
}
bts := s3intf.GetBytesToSign(req, serviceHost)
if Debug {
log.Printf("bts: %q", bts)
log.Printf("---")
}
t.Logf("owner: %s bts=%q", o, bts)
actsign := b64.EncodeToString(o.CalcHash(bts))
req.Header.Set("Authorization", "AWS test:"+actsign)
rw := httptest.NewRecorder()
handlers[i].ServeHTTP(rw, req)
if check != nil {
if err = check(rw); err != nil {
t.Fatalf("bad response for %s %s: %s (body:%q)", method, path,
err.Error(), rw.Body.Bytes())
}
}
}
}
// CreateBucket creates a new bucket
func (root hier) CreateBucket(owner s3intf.Owner, bucket string) error {
return os.MkdirAll(filepath.Join(string(root), owner.ID(), bucket), 0750)
}
// Put puts a file as a new object into the bucket
func (m master) Put(owner s3intf.Owner, bucket, object, filename, media string,
body io.Reader, size int64, md5hash []byte) (
err error) {
m.Lock()
o, ok := m.owners[owner.ID()]
m.Unlock()
if !ok {
err = errors.New("cannot find owner " + owner.ID())
return
}
o.Lock()
b, ok := o.buckets[bucket]
o.Unlock()
if !ok {
err = errors.New("cannot find bucket " + bucket)
return
}
if err = b.db.BeginTransaction(); err != nil {
return fmt.Errorf("cannot start transaction: %s", err)
}
defer func() {
if err != nil {
b.db.Rollback()
}
}()
//upload
fid, publicURL, err := m.wm.AssignFid()
if err != nil {
err = fmt.Errorf("error getting fid: %s", err)
return
}
vi := weedutils.ValInfo{Filename: filename, ContentType: media,
Fid: fid, Created: time.Now(), Size: size, MD5: md5hash}
val, err := vi.Encode(nil)
if err != nil {
err = fmt.Errorf("error serializing %v: %s", vi, err)
return
}
if err = b.db.Set([]byte(object), val); err != nil {
err = fmt.Errorf("error storing key in db: %s", err)
return
}
//log.Printf("filename=%q", filename)
var hsh hash.Hash
if vi.MD5 == nil {
hsh = md5.New()
body = io.TeeReader(body, hsh)
}
if _, err = m.wm.UploadAssigned(fid, publicURL, filename, media, body); err != nil {
b.db.Rollback()
err = fmt.Errorf("error uploading to %s: %s", fid, err)
return
}
if vi.MD5 == nil {
vi.MD5 = hsh.Sum(nil)
if val, err = vi.Encode(nil); err != nil {
err = fmt.Errorf("error serializing %v: %s", vi, err)
return
}
if err = b.db.Set([]byte(object), val); err != nil {
err = fmt.Errorf("error storing key in db: %s", err)
return
}
}
//log.Printf("uploading %s [%d] resulted in %s", filename, size, resp)
err = nil
return b.db.Commit()
}
// List lists a bucket, all objects Key starts with prefix, delimiter segments
// Key, thus the returned commonprefixes (think a generalized filepath
// structure, where / is the delimiter, a commonprefix is a subdir)
func (m master) List(owner s3intf.Owner, bucket, prefix, delimiter, marker string,
limit, skip int) (
objects []s3intf.Object, commonprefixes []string,
truncated bool, err error) {
m.Lock()
o, ok := m.owners[owner.ID()]
if !ok {
m.Unlock()
err = fmt.Errorf("unknown owner %s", owner.ID())
return
}
o.Lock()
b, ok := o.buckets[bucket]
o.Unlock()
m.Unlock()
if !ok {
err = fmt.Errorf("unknown bucket %s", bucket)
return
}
err = nil
enum, e := b.db.SeekFirst()
if e != nil {
if e == io.EOF { //empty
return
}
err = fmt.Errorf("error getting first: %s", e)
return
}
var (
key, val []byte
vi = new(weedutils.ValInfo)
etag string
)
objects = make([]s3intf.Object, 0, 64)
f := s3intf.NewListFilter(prefix, delimiter, marker, limit, skip)
for {
if key, val, e = enum.Next(); e != nil {
if e == io.EOF {
break
}
err = fmt.Errorf("error seeking next: %s", e)
return
}
//log.Printf("key=%q", key)
if ok, e = f.Check(string(key)); e != nil {
if e == io.EOF {
break
}
err = fmt.Errorf("error checking %s: %s", key, e)
return
} else if ok {
if err = vi.Decode(val); err != nil {
return
}
if vi.MD5 != nil && len(vi.MD5) == 16 {
etag = hex.EncodeToString(vi.MD5)
} else {
etag = ""
}
objects = append(objects,
s3intf.Object{Key: string(key), Owner: owner,
ETag: etag, LastModified: vi.Created, Size: vi.Size})
}
}
commonprefixes, truncated = f.Result()
return
}