func test_bucket_file_update(t *BackrunnerTest) error {
bname := strconv.FormatInt(rand.Int63(), 16)
meta := bucket.BucketMsgpack{
Version: 1,
Name: bname,
Groups: []uint32{t.groups[0], 12345},
Acl: make(map[string]bucket.BucketACL),
}
acl := bucket.BucketACL{
Version: 1,
User: t.all_allowed_user,
Token: t.all_allowed_token,
Flags: bucket.BucketAuthWrite,
}
meta.Acl[acl.User] = acl
_, err := bucket.WriteBucket(t.ell, &meta)
if err != nil {
log.Fatalf("Could not upload bucket: %v", err)
}
bfile, err := os.OpenFile(t.bucket_file, os.O_RDWR|os.O_APPEND, 0660)
if err != nil {
return fmt.Errorf("Could not open bucket file '%s': %v", t.bucket_file, err)
}
fmt.Fprintf(bfile, "%s\n", bname)
bfile.Close()
t.proxy_cmd.Process.Signal(syscall.SIGHUP)
// wait for statistics to update
time.Sleep(6 * time.Second)
st, err := t.parse_stat()
if err != nil {
return err
}
_, ok := st.Buckets[bname]
if !ok {
return fmt.Errorf("There is no bucket '%s' in new stats", bname)
}
t.failed_bucket = bname
return nil
}
func bmeta_read_upload(ell *etransport.Elliptics, file string) (err error) {
metaf, err := ioutil.ReadFile(file)
if err != nil {
log.Fatalf("Could not read upload bucket config file %s: %v", file, err)
}
var iface interface{}
err = json.Unmarshal(metaf, &iface)
if err != nil {
err = fmt.Errorf("could not parse data: %v", err)
return
}
generic := bucket.NewBucketMsgpack("generic")
imap := iface.(map[string]interface{})
if g, ok := imap["generic"]; ok {
generic.ExtractJson(g)
}
if biface, ok := imap["buckets"]; ok {
bmap := biface.(map[string]interface{})
for bname, iface := range bmap {
log.Printf("bucket: %s, iface: %p\n", bname, iface)
tmp := bucket.NewBucketMsgpack(bname)
*tmp = *generic
tmp.Name = bname
tmp.ExtractJson(iface)
b, err := bucket.WriteBucket(ell, tmp)
if err != nil {
log.Printf("Could not write bucket %s: %v", bname, err)
} else {
log.Printf("%s\n", b.Meta.String())
fmt.Printf("%s\n", b.Meta.String())
}
}
} else {
err = fmt.Errorf("There is no 'buckets' section in metadata file, nothing to upload")
}
return
}
func test_bucket_update(t *BackrunnerTest) error {
bname := strconv.FormatInt(rand.Int63(), 16)
user := strconv.FormatInt(rand.Int63(), 16)
token := strconv.FormatInt(rand.Int63(), 16)
key := "bucket-update-test"
meta := bucket.BucketMsgpack{
Version: 1,
Name: bname,
Groups: t.groups,
Acl: make(map[string]bucket.BucketACL),
}
acl := bucket.BucketACL{
Version: 1,
User: user,
Token: token,
Flags: bucket.BucketAuthWrite,
}
meta.Acl[acl.User] = acl
_, err := bucket.WriteBucket(t.ell, &meta)
if err != nil {
log.Fatalf("Could not upload bucket: %v", err)
}
log.Printf("test_bucket_update: new bucket: %s, meta: %v\n", bname, meta)
// bucket has been uploaded into the storage,
// let's check that reading/writing from that bucket succeeds
err = t.upload_get_helper(bname, key, user, token)
if err != nil {
return err
}
// update ACL
new_token := strconv.FormatInt(rand.Int63(), 16)
acl = bucket.BucketACL{
Version: 1,
User: user,
Token: new_token,
Flags: bucket.BucketAuthWrite,
}
meta.Acl[acl.User] = acl
_, err = bucket.WriteBucket(t.ell, &meta)
if err != nil {
log.Fatalf("Could not upload bucket: %v", err)
}
log.Printf("test_bucket_update: updated bucket: %s, meta: %v\n", bname, meta)
// trying to read data using old token, it should fail with 403 error
req := t.NewEmptyRequest("GET", "get", user, new_token, bname, key)
resp, err := t.client.Do(req)
if err != nil {
return fmt.Errorf("test_bucket_update: url: %s: could not send get request: %v", req.URL.String(), err)
}
defer resp.Body.Close()
_, err = ioutil.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("test_bucket_update: url: %s: could not read reply: %v", req.URL.String(), req.Header, err)
}
if resp.StatusCode != http.StatusForbidden {
return fmt.Errorf("test_bucket_update: url: %s, returned status: %d, must be: %d",
req.URL.String(), resp.StatusCode, http.StatusForbidden)
}
// wait for ACL to update, it should be updated once per 30 seconds or so
time.Sleep(32 * time.Second)
// trying to update key using new token
err = t.upload_get_helper(bname, "some another key", user, new_token)
if err != nil {
return err
}
return nil
}
func (t *BackrunnerTest) ACLInit() error {
user := strconv.FormatInt(rand.Int63(), 16)
meta := bucket.BucketMsgpack{
Version: 1,
Name: t.acl_bucket,
Groups: t.groups,
Acl: make(map[string]bucket.BucketACL),
}
var acl bucket.BucketACL
var flags uint64
flags = bucket.BucketAuthEmpty
acl = bucket.BucketACL{
Version: 1,
User: fmt.Sprintf("user-%s-%x", user, flags),
Token: strconv.FormatInt(rand.Int63(), 16),
Flags: flags,
}
meta.Acl[acl.User] = acl
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", acl.User, acl.Token, http.StatusOK))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", acl.User, "qwerty", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", "qwerty", "qwerty", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", "", "", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", acl.User, acl.Token, http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", acl.User, "qwerty", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", "qwerty", "qwerty", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", "", "", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", acl.User, acl.Token, http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", acl.User, "qwerty", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", "qwerty", "qwerty", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", "", "", http.StatusForbidden))
flags = bucket.BucketAuthNoToken
acl = bucket.BucketACL{
Version: 1,
User: fmt.Sprintf("user-%s-%x", user, flags),
Token: strconv.FormatInt(rand.Int63(), 16),
Flags: flags,
}
meta.Acl[acl.User] = acl
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", acl.User, acl.Token, http.StatusOK))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", acl.User, "qwerty", http.StatusOK))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", "qwerty", "qwerty", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", "", "", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", acl.User, acl.Token, http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", acl.User, "qwerty", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", "qwerty", "qwerty", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", "", "", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", acl.User, acl.Token, http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", acl.User, "qwerty", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", "qwerty", "qwerty", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", "", "", http.StatusForbidden))
flags = bucket.BucketAuthWrite
acl = bucket.BucketACL{
Version: 1,
User: fmt.Sprintf("user-%s-%x", user, flags),
Token: strconv.FormatInt(rand.Int63(), 16),
Flags: flags,
}
meta.Acl[acl.User] = acl
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", acl.User, acl.Token, http.StatusOK))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", "", "", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", acl.User, "qwerty", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", "qwerty", "qwerty", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", acl.User, acl.Token, http.StatusOK))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", acl.User, "qwerty", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", "qwerty", "qwerty", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", "", "", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", acl.User, acl.Token, http.StatusOK))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", acl.User, "qwerty", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", "qwerty", "qwerty", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", "", "", http.StatusForbidden))
flags = bucket.BucketAuthWrite | bucket.BucketAuthNoToken
acl = bucket.BucketACL{
Version: 1,
User: fmt.Sprintf("user-%s-%x", user, flags),
Token: strconv.FormatInt(rand.Int63(), 16),
Flags: flags,
}
meta.Acl[acl.User] = acl
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", acl.User, acl.Token, http.StatusOK))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", acl.User, "qwerty", http.StatusOK))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", "qwerty", "qwerty", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", "", "", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", acl.User, acl.Token, http.StatusOK))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", acl.User, "qwerty", http.StatusOK))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", "qwerty", "qwerty", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", "", "", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", acl.User, acl.Token, http.StatusOK))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", acl.User, "qwerty", http.StatusOK))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", "qwerty", "qwerty", http.StatusForbidden))
t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", "", "", http.StatusForbidden))
acl = bucket.BucketACL{
Version: 1,
User: t.all_allowed_user,
Token: t.all_allowed_token,
Flags: bucket.BucketAuthWrite | bucket.BucketAuthNoToken,
}
meta.Acl[acl.User] = acl
_, err := bucket.WriteBucket(t.ell, &meta)
if err != nil {
log.Fatalf("Could not upload bucket: %v", err)
}
return nil
}
func (bt *BackrunnerTest) StartEllipticsClientProxy(proxy_path string) {
bt.conf.Elliptics.LogFile = bt.proxy_log
bt.conf.Elliptics.LogPrefix = "backrunner: "
file := fmt.Sprintf("%s/elliptics_transport.conf", bt.base)
err := bt.conf.Save(file)
if err != nil {
log.Fatalf("Could not save client transport config: %v", err)
}
bt.ell, err = etransport.NewEllipticsTransport(&bt.conf.Elliptics)
if err != nil {
log.Fatal("Could not connect to elliptics server %v: %v", bt.elliptics_address, err)
}
bt.bucket_file = fmt.Sprintf("%s/buckets", bt.base)
fd, err := os.Create(bt.bucket_file)
if err != nil {
log.Fatalf("Could not create bucket file %s: %v\n", bt.bucket_file, err)
}
defer fd.Close()
_, err = fmt.Fprintf(fd, "%s\n", bt.acl_bucket)
if err != nil {
log.Fatalf("Could not write acl bucket into bucket file: %v\n", err)
}
for _, g := range bt.groups {
b := fmt.Sprintf("b%d", g)
_, err = fmt.Fprintf(fd, "%s\n", b)
if err != nil {
log.Fatalf("Could not write bucket into bucket file: %v\n", err)
}
meta := bucket.BucketMsgpack{
Version: 1,
Name: b,
Groups: []uint32{g},
Acl: make(map[string]bucket.BucketACL),
}
acl := bucket.BucketACL{
Version: 1,
User: bt.all_allowed_user,
Token: bt.all_allowed_token,
Flags: bucket.BucketAuthWrite | bucket.BucketAuthNoToken,
}
meta.Acl[acl.User] = acl
_, err := bucket.WriteBucket(bt.ell, &meta)
if err != nil {
log.Fatal("Could not upload bucket %s into storage: %v", b, err)
}
bt.io_buckets = append(bt.io_buckets, b)
}
bt.ACLInit()
file = fmt.Sprintf("%s/proxy.conf", bt.base)
err = bt.conf.Save(file)
if err != nil {
log.Fatalf("Could not save proxy config: %v", err)
}
cmd := exec.Command(proxy_path, "-config", file, "-buckets", bt.bucket_file)
cmd.Stdout = &bt.proxy_stdout
cmd.Stderr = &bt.proxy_stderr
err = cmd.Start()
if err != nil {
log.Fatalf("Could not start proxy process: %v\n", err)
}
bt.proxy_cmd = cmd
// wait 1 second for proxy to start
time.Sleep(1 * time.Second)
}