コード例 #1
0
ファイル: dir.go プロジェクト: daozhao/s3weed
// CheckBucket returns whether the owner has a bucket named as given
func (root hier) CheckBucket(owner s3intf.Owner, bucket string) bool {
	dh, err := os.Open(filepath.Join(string(root), owner.ID(), bucket))
	if err != nil {
		return false
	}
	dh.Close()
	return true
}
コード例 #2
0
ファイル: dir.go プロジェクト: daozhao/s3weed
// List lists a bucket, all objects Key starts with prefix, delimiter segments
// Key, thus the returned commonprefixes (think a generalized filepath
// structure, where / is the delimiter, a commonprefix is a subdir)
func (root hier) List(owner s3intf.Owner, bucket, prefix, delimiter, marker string,
	limit, skip int) (
	objects []s3intf.Object, commonprefixes []string,
	truncated bool, err error) {
	dh, e := os.Open(filepath.Join(string(root), owner.ID(), bucket))
	if e != nil {
		err = e
		return
	}
	defer dh.Close()
	var (
		infos     []os.FileInfo
		key, etag string
		md5hash   []byte
		ok        bool
	)

	objects = make([]s3intf.Object, 0, 64)
	f := s3intf.NewListFilter(prefix, delimiter, marker, limit, skip)
OUTER:
	for {
		infos, e = dh.Readdir(limit)
		if e != nil {
			if e == io.EOF {
				break
			}
			err = e
			return
		}

		for _, fi := range infos {
			if key, _, _, md5hash, err = decodeFilename(fi.Name()); err != nil {
				return
			}
			if ok, e = f.Check(key); e != nil {
				if e == io.EOF {
					break OUTER
				}
				err = fmt.Errorf("error checking %s: %s", key, e)
				return
			} else if ok {
				if md5hash != nil && len(md5hash) == 16 {
					etag = hex.EncodeToString(md5hash)
				} else {
					etag = ""
				}
				objects = append(objects,
					s3intf.Object{Key: string(key), Owner: owner,
						ETag: etag, LastModified: fi.ModTime(), Size: fi.Size()})
			}
		}
	}
	commonprefixes, truncated = f.Result()
	return
}
コード例 #3
0
ファイル: weed.go プロジェクト: daozhao/s3weed
// CheckBucket returns whether the owner has a bucket named as given
func (m master) CheckBucket(owner s3intf.Owner, bucket string) bool {
	m.Lock()
	defer m.Unlock()
	if o, ok := m.owners[owner.ID()]; ok {
		o.Lock()
		_, ok = o.buckets[bucket]
		o.Unlock()
		return ok
	}
	return false
}
コード例 #4
0
ファイル: dir.go プロジェクト: daozhao/s3weed
// Put puts a file as a new object into the bucket
func (root hier) Put(owner s3intf.Owner, bucket, object, filename, media string,
	body io.Reader, size int64, md5hash []byte) error {

	fh, err := os.Create(filepath.Join(string(root), owner.ID(), bucket,
		encodeFilename(object, filename, media, string(md5hash))))
	if err != nil {
		return err
	}
	_, err = io.Copy(fh, body)
	fh.Close()
	return err
}
コード例 #5
0
ファイル: weed.go プロジェクト: daozhao/s3weed
// ListBuckets list all buckets owned by the given owner
func (m master) ListBuckets(owner s3intf.Owner) ([]s3intf.Bucket, error) {
	m.Lock()
	o, ok := m.owners[owner.ID()]
	m.Unlock()
	if !ok {
		return nil, fmt.Errorf("unkown owner %s", owner.ID())
	}
	buckets := make([]s3intf.Bucket, len(o.buckets))
	i := 0
	for k, b := range o.buckets {
		buckets[i].Name = k
		buckets[i].Created = b.created
		i++
	}
	return buckets, nil
}
コード例 #6
0
ファイル: dir.go プロジェクト: daozhao/s3weed
// DelBucket deletes a bucket
func (root hier) DelBucket(owner s3intf.Owner, bucket string) error {
	dh, err := os.Open(filepath.Join(string(root), owner.ID(), bucket))
	if err != nil {
		return err
	}
	infos, err := dh.Readdir(1)
	if err != nil && err != io.EOF {
		dh.Close()
		return err
	}
	if len(infos) > 0 {
		dh.Close()
		return errors.New("cannot delete non-empty bucket")
	}
	nm := dh.Name()
	dh.Close()
	return os.Remove(nm)
}
コード例 #7
0
ファイル: weed.go プロジェクト: daozhao/s3weed
// Del deletes the object from the bucket
func (m master) Del(owner s3intf.Owner, bucket, object string) (err error) {
	m.Lock()
	o, ok := m.owners[owner.ID()]
	m.Unlock()
	if !ok {
		return errors.New("cannot find owner " + owner.ID())
	}
	o.Lock()
	b, ok := o.buckets[bucket]
	o.Unlock()
	if !ok {
		return errors.New("cannot find bucket " + bucket)
	}

	if err = b.db.BeginTransaction(); err != nil {
		return fmt.Errorf("cannot start transaction: %s", err)
	}
	defer func() {
		if err != nil {
			b.db.Rollback()
		}
	}()

	val, e := b.db.Extract(nil, []byte(object))
	if e != nil {
		err = fmt.Errorf("cannot get %s object: %s", object, e)
		return
	}
	if val == nil {
		err = s3intf.NotFound
		return
	}
	vi := new(weedutils.ValInfo)
	if err = vi.Decode(val); err != nil {
		err = fmt.Errorf("error deserializing %s: %s", val, err)
		return
	}
	if err = m.wm.Delete(vi.Fid); err != nil {
		return
	}
	err = nil
	return b.db.Commit()
}
コード例 #8
0
ファイル: dir.go プロジェクト: daozhao/s3weed
func (root hier) findFile(owner s3intf.Owner, bucket, object string) (string, error) {
	dh, err := os.Open(filepath.Join(string(root), owner.ID(), bucket))
	if err != nil {
		return "", err
	}
	defer dh.Close()
	prefix := encodeFilename(object, "")
	var names []string
	for err == nil {
		if names, err = dh.Readdirnames(1000); err != nil && err != io.EOF {
			return "", err
		}
		for _, nm := range names {
			if strings.HasPrefix(nm, prefix) {
				return filepath.Join(dh.Name(), nm), nil
			}
		}
	}
	return "", nil
}
コード例 #9
0
ファイル: weed.go プロジェクト: daozhao/s3weed
// CreateBucket creates a new bucket
func (m master) CreateBucket(owner s3intf.Owner, bucket string) error {
	m.Lock()
	defer m.Unlock()
	o, ok := m.owners[owner.ID()]
	if !ok {
		dir := filepath.Join(m.baseDir, owner.ID())
		err := os.MkdirAll(dir, 0750)
		if err != nil {
			return err
		}
		o = wOwner{dir: dir, buckets: make(map[string]wBucket, 1)}
		m.owners[owner.ID()] = o
		//} else if o.buckets == nil {
		//	o.buckets = make(map[string]wBucket, 1)
	}
	o.Lock()
	defer o.Unlock()
	_, ok = o.buckets[bucket]
	if ok {
		return nil //AlreadyExists ?
	}
	b := wBucket{filename: filepath.Join(o.dir, bucket+".kv"), created: time.Now()}
	var err error
	if b.db, err = kv.Create(b.filename, kvOptions()); err != nil {
		return err
	}
	o.buckets[bucket] = b
	return nil
}
コード例 #10
0
ファイル: weed.go プロジェクト: daozhao/s3weed
// DelBucket deletes a bucket
func (m master) DelBucket(owner s3intf.Owner, bucket string) error {
	m.Lock()
	defer m.Unlock()
	o, ok := m.owners[owner.ID()]
	if !ok {
		return fmt.Errorf("unknown owner %s", owner.ID())
	}
	o.Lock()
	defer o.Unlock()
	b, ok := o.buckets[bucket]
	if !ok {
		return fmt.Errorf("bucket %s not exists!", bucket)
	}
	if k, v, err := b.db.First(); err != nil {
		return err
	} else if k != nil || v != nil {
		return errors.New("cannot delete non-empty bucket")
	}
	b.db.Close()
	b.db = nil
	delete(o.buckets, bucket)
	return os.Remove(b.filename)
}
コード例 #11
0
ファイル: weed.go プロジェクト: daozhao/s3weed
// Get retrieves an object from the bucket
func (m master) Get(owner s3intf.Owner, bucket, object string) (
	filename, media string, body io.ReadCloser, size int64, md5 []byte, err error) {

	m.Lock()
	o, ok := m.owners[owner.ID()]
	m.Unlock()
	if !ok {
		err = errors.New("cannot find owner " + owner.ID())
		return
	}
	o.Lock()
	b, ok := o.buckets[bucket]
	o.Unlock()
	if !ok {
		err = errors.New("cannot find bucket " + bucket)
		return
	}

	val, e := b.db.Get(nil, []byte(object))
	if e != nil {
		err = fmt.Errorf("cannot get %s object: %s", object, e)
		return
	}
	if val == nil {
		err = s3intf.NotFound
		return
	}
	vi := new(weedutils.ValInfo)
	if err = vi.Decode(val); err != nil {
		err = fmt.Errorf("error deserializing %s: %s", val, err)
		return
	}
	filename, media, size, md5 = vi.Filename, vi.ContentType, vi.Size, vi.MD5

	body, err = m.wm.Download(vi.Fid)
	return
}
コード例 #12
0
ファイル: dir.go プロジェクト: daozhao/s3weed
// ListBuckets list all buckets owned by the given owner
func (root hier) ListBuckets(owner s3intf.Owner) ([]s3intf.Bucket, error) {
	dn := filepath.Join(string(root), owner.ID())
	dh, err := os.Open(dn)
	if err != nil {
		if _, ok := err.(*os.PathError); ok {
			os.MkdirAll(dn, 0750)
			dh, err = os.Open(dn)
		}
		if err != nil {
			return nil, err
		}
	}
	defer dh.Close()
	infos, err := dh.Readdir(1000)
	if err != nil && err != io.EOF {
		return nil, err
	}
	buckets := make([]s3intf.Bucket, len(infos))
	for i, fi := range infos {
		buckets[i].Name = fi.Name()
		buckets[i].Created = fi.ModTime()
	}
	return buckets, nil
}
コード例 #13
0
ファイル: impl_test.go プロジェクト: daozhao/s3weed
func doReq(t *testing.T, method, path string, body io.Reader, check ResponseChecker) {
	req, err := http.NewRequest(method, path, body)
	if err != nil {
		t.Fatalf("cannot create request: " + err.Error())
	}
	req.Host = serviceHost
	//req.URL.Host = req.Host
	var o s3intf.Owner
	for i, b := range backers {
		if o, err = b.GetOwner("test"); err != nil {
			t.Errorf("cannot get owner for test: %s", err)
			continue
		}
		if Debug {
			log.Printf("===")
			s3intf.Debug = Debug
		}
		bts := s3intf.GetBytesToSign(req, serviceHost)
		if Debug {
			log.Printf("bts: %q", bts)
			log.Printf("---")
		}
		t.Logf("owner: %s bts=%q", o, bts)
		actsign := b64.EncodeToString(o.CalcHash(bts))
		req.Header.Set("Authorization", "AWS test:"+actsign)

		rw := httptest.NewRecorder()
		handlers[i].ServeHTTP(rw, req)
		if check != nil {
			if err = check(rw); err != nil {
				t.Fatalf("bad response for %s %s: %s (body:%q)", method, path,
					err.Error(), rw.Body.Bytes())
			}
		}
	}
}
コード例 #14
0
ファイル: dir.go プロジェクト: daozhao/s3weed
// CreateBucket creates a new bucket
func (root hier) CreateBucket(owner s3intf.Owner, bucket string) error {
	return os.MkdirAll(filepath.Join(string(root), owner.ID(), bucket), 0750)
}
コード例 #15
0
ファイル: weed.go プロジェクト: daozhao/s3weed
// Put puts a file as a new object into the bucket
func (m master) Put(owner s3intf.Owner, bucket, object, filename, media string,
	body io.Reader, size int64, md5hash []byte) (
	err error) {

	m.Lock()
	o, ok := m.owners[owner.ID()]
	m.Unlock()
	if !ok {
		err = errors.New("cannot find owner " + owner.ID())
		return
	}
	o.Lock()
	b, ok := o.buckets[bucket]
	o.Unlock()
	if !ok {
		err = errors.New("cannot find bucket " + bucket)
		return
	}

	if err = b.db.BeginTransaction(); err != nil {
		return fmt.Errorf("cannot start transaction: %s", err)
	}
	defer func() {
		if err != nil {
			b.db.Rollback()
		}
	}()
	//upload
	fid, publicURL, err := m.wm.AssignFid()
	if err != nil {
		err = fmt.Errorf("error getting fid: %s", err)
		return
	}
	vi := weedutils.ValInfo{Filename: filename, ContentType: media,
		Fid: fid, Created: time.Now(), Size: size, MD5: md5hash}
	val, err := vi.Encode(nil)
	if err != nil {
		err = fmt.Errorf("error serializing %v: %s", vi, err)
		return
	}
	if err = b.db.Set([]byte(object), val); err != nil {
		err = fmt.Errorf("error storing key in db: %s", err)
		return
	}
	//log.Printf("filename=%q", filename)
	var hsh hash.Hash
	if vi.MD5 == nil {
		hsh = md5.New()
		body = io.TeeReader(body, hsh)
	}
	if _, err = m.wm.UploadAssigned(fid, publicURL, filename, media, body); err != nil {
		b.db.Rollback()
		err = fmt.Errorf("error uploading to %s: %s", fid, err)
		return
	}
	if vi.MD5 == nil {
		vi.MD5 = hsh.Sum(nil)
		if val, err = vi.Encode(nil); err != nil {
			err = fmt.Errorf("error serializing %v: %s", vi, err)
			return
		}
		if err = b.db.Set([]byte(object), val); err != nil {
			err = fmt.Errorf("error storing key in db: %s", err)
			return
		}
	}

	//log.Printf("uploading %s [%d] resulted in %s", filename, size, resp)
	err = nil
	return b.db.Commit()
}
コード例 #16
0
ファイル: weed.go プロジェクト: daozhao/s3weed
// List lists a bucket, all objects Key starts with prefix, delimiter segments
// Key, thus the returned commonprefixes (think a generalized filepath
// structure, where / is the delimiter, a commonprefix is a subdir)
func (m master) List(owner s3intf.Owner, bucket, prefix, delimiter, marker string,
	limit, skip int) (
	objects []s3intf.Object, commonprefixes []string,
	truncated bool, err error) {

	m.Lock()
	o, ok := m.owners[owner.ID()]
	if !ok {
		m.Unlock()
		err = fmt.Errorf("unknown owner %s", owner.ID())
		return
	}
	o.Lock()
	b, ok := o.buckets[bucket]
	o.Unlock()
	m.Unlock()
	if !ok {
		err = fmt.Errorf("unknown bucket %s", bucket)
		return
	}

	err = nil
	enum, e := b.db.SeekFirst()
	if e != nil {
		if e == io.EOF { //empty
			return
		}
		err = fmt.Errorf("error getting first: %s", e)
		return
	}
	var (
		key, val []byte
		vi       = new(weedutils.ValInfo)
		etag     string
	)
	objects = make([]s3intf.Object, 0, 64)
	f := s3intf.NewListFilter(prefix, delimiter, marker, limit, skip)
	for {
		if key, val, e = enum.Next(); e != nil {
			if e == io.EOF {
				break
			}
			err = fmt.Errorf("error seeking next: %s", e)
			return
		}
		//log.Printf("key=%q", key)
		if ok, e = f.Check(string(key)); e != nil {
			if e == io.EOF {
				break
			}
			err = fmt.Errorf("error checking %s: %s", key, e)
			return
		} else if ok {
			if err = vi.Decode(val); err != nil {
				return
			}
			if vi.MD5 != nil && len(vi.MD5) == 16 {
				etag = hex.EncodeToString(vi.MD5)
			} else {
				etag = ""
			}
			objects = append(objects,
				s3intf.Object{Key: string(key), Owner: owner,
					ETag: etag, LastModified: vi.Created, Size: vi.Size})
		}
	}
	commonprefixes, truncated = f.Result()
	return
}