Beispiel #1
0
func test_bucket_file_update(t *BackrunnerTest) error {
	bname := strconv.FormatInt(rand.Int63(), 16)

	meta := bucket.BucketMsgpack{
		Version: 1,
		Name:    bname,
		Groups:  []uint32{t.groups[0], 12345},
		Acl:     make(map[string]bucket.BucketACL),
	}
	acl := bucket.BucketACL{
		Version: 1,
		User:    t.all_allowed_user,
		Token:   t.all_allowed_token,
		Flags:   bucket.BucketAuthWrite,
	}
	meta.Acl[acl.User] = acl

	_, err := bucket.WriteBucket(t.ell, &meta)
	if err != nil {
		log.Fatalf("Could not upload bucket: %v", err)
	}

	bfile, err := os.OpenFile(t.bucket_file, os.O_RDWR|os.O_APPEND, 0660)
	if err != nil {
		return fmt.Errorf("Could not open bucket file '%s': %v", t.bucket_file, err)
	}

	fmt.Fprintf(bfile, "%s\n", bname)
	bfile.Close()

	t.proxy_cmd.Process.Signal(syscall.SIGHUP)

	// wait for statistics to update
	time.Sleep(6 * time.Second)

	st, err := t.parse_stat()
	if err != nil {
		return err
	}

	_, ok := st.Buckets[bname]
	if !ok {
		return fmt.Errorf("There is no bucket '%s' in new stats", bname)
	}

	t.failed_bucket = bname
	return nil
}
Beispiel #2
0
func bmeta_read_upload(ell *etransport.Elliptics, file string) (err error) {
	metaf, err := ioutil.ReadFile(file)
	if err != nil {
		log.Fatalf("Could not read upload bucket config file %s: %v", file, err)
	}

	var iface interface{}
	err = json.Unmarshal(metaf, &iface)
	if err != nil {
		err = fmt.Errorf("could not parse data: %v", err)
		return
	}

	generic := bucket.NewBucketMsgpack("generic")

	imap := iface.(map[string]interface{})

	if g, ok := imap["generic"]; ok {
		generic.ExtractJson(g)
	}

	if biface, ok := imap["buckets"]; ok {
		bmap := biface.(map[string]interface{})
		for bname, iface := range bmap {
			log.Printf("bucket: %s, iface: %p\n", bname, iface)

			tmp := bucket.NewBucketMsgpack(bname)
			*tmp = *generic
			tmp.Name = bname

			tmp.ExtractJson(iface)

			b, err := bucket.WriteBucket(ell, tmp)
			if err != nil {
				log.Printf("Could not write bucket %s: %v", bname, err)
			} else {
				log.Printf("%s\n", b.Meta.String())
				fmt.Printf("%s\n", b.Meta.String())
			}
		}
	} else {
		err = fmt.Errorf("There is no 'buckets' section in metadata file, nothing to upload")
	}

	return
}
Beispiel #3
0
func test_bucket_update(t *BackrunnerTest) error {
	bname := strconv.FormatInt(rand.Int63(), 16)
	user := strconv.FormatInt(rand.Int63(), 16)
	token := strconv.FormatInt(rand.Int63(), 16)
	key := "bucket-update-test"

	meta := bucket.BucketMsgpack{
		Version: 1,
		Name:    bname,
		Groups:  t.groups,
		Acl:     make(map[string]bucket.BucketACL),
	}
	acl := bucket.BucketACL{
		Version: 1,
		User:    user,
		Token:   token,
		Flags:   bucket.BucketAuthWrite,
	}
	meta.Acl[acl.User] = acl

	_, err := bucket.WriteBucket(t.ell, &meta)
	if err != nil {
		log.Fatalf("Could not upload bucket: %v", err)
	}
	log.Printf("test_bucket_update: new bucket: %s, meta: %v\n", bname, meta)

	// bucket has been uploaded into the storage,
	// let's check that reading/writing from that bucket succeeds

	err = t.upload_get_helper(bname, key, user, token)
	if err != nil {
		return err
	}

	// update ACL
	new_token := strconv.FormatInt(rand.Int63(), 16)
	acl = bucket.BucketACL{
		Version: 1,
		User:    user,
		Token:   new_token,
		Flags:   bucket.BucketAuthWrite,
	}
	meta.Acl[acl.User] = acl

	_, err = bucket.WriteBucket(t.ell, &meta)
	if err != nil {
		log.Fatalf("Could not upload bucket: %v", err)
	}
	log.Printf("test_bucket_update: updated bucket: %s, meta: %v\n", bname, meta)

	// trying to read data using old token, it should fail with 403 error
	req := t.NewEmptyRequest("GET", "get", user, new_token, bname, key)

	resp, err := t.client.Do(req)
	if err != nil {
		return fmt.Errorf("test_bucket_update: url: %s: could not send get request: %v", req.URL.String(), err)
	}
	defer resp.Body.Close()

	_, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("test_bucket_update: url: %s: could not read reply: %v", req.URL.String(), req.Header, err)
	}

	if resp.StatusCode != http.StatusForbidden {
		return fmt.Errorf("test_bucket_update: url: %s, returned status: %d, must be: %d",
			req.URL.String(), resp.StatusCode, http.StatusForbidden)
	}

	// wait for ACL to update, it should be updated once per 30 seconds or so
	time.Sleep(32 * time.Second)

	// trying to update key using new token
	err = t.upload_get_helper(bname, "some another key", user, new_token)
	if err != nil {
		return err
	}

	return nil
}
Beispiel #4
0
func (t *BackrunnerTest) ACLInit() error {
	user := strconv.FormatInt(rand.Int63(), 16)

	meta := bucket.BucketMsgpack{
		Version: 1,
		Name:    t.acl_bucket,
		Groups:  t.groups,
		Acl:     make(map[string]bucket.BucketACL),
	}

	var acl bucket.BucketACL
	var flags uint64

	flags = bucket.BucketAuthEmpty
	acl = bucket.BucketACL{
		Version: 1,
		User:    fmt.Sprintf("user-%s-%x", user, flags),
		Token:   strconv.FormatInt(rand.Int63(), 16),
		Flags:   flags,
	}
	meta.Acl[acl.User] = acl
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", acl.User, acl.Token, http.StatusOK))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", acl.User, "qwerty", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", "qwerty", "qwerty", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", "", "", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", acl.User, acl.Token, http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", acl.User, "qwerty", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", "qwerty", "qwerty", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", "", "", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", acl.User, acl.Token, http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", acl.User, "qwerty", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", "qwerty", "qwerty", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", "", "", http.StatusForbidden))

	flags = bucket.BucketAuthNoToken
	acl = bucket.BucketACL{
		Version: 1,
		User:    fmt.Sprintf("user-%s-%x", user, flags),
		Token:   strconv.FormatInt(rand.Int63(), 16),
		Flags:   flags,
	}
	meta.Acl[acl.User] = acl
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", acl.User, acl.Token, http.StatusOK))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", acl.User, "qwerty", http.StatusOK))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", "qwerty", "qwerty", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", "", "", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", acl.User, acl.Token, http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", acl.User, "qwerty", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", "qwerty", "qwerty", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", "", "", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", acl.User, acl.Token, http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", acl.User, "qwerty", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", "qwerty", "qwerty", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", "", "", http.StatusForbidden))

	flags = bucket.BucketAuthWrite
	acl = bucket.BucketACL{
		Version: 1,
		User:    fmt.Sprintf("user-%s-%x", user, flags),
		Token:   strconv.FormatInt(rand.Int63(), 16),
		Flags:   flags,
	}
	meta.Acl[acl.User] = acl
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", acl.User, acl.Token, http.StatusOK))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", "", "", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", acl.User, "qwerty", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", "qwerty", "qwerty", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", acl.User, acl.Token, http.StatusOK))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", acl.User, "qwerty", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", "qwerty", "qwerty", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", "", "", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", acl.User, acl.Token, http.StatusOK))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", acl.User, "qwerty", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", "qwerty", "qwerty", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", "", "", http.StatusForbidden))

	flags = bucket.BucketAuthWrite | bucket.BucketAuthNoToken
	acl = bucket.BucketACL{
		Version: 1,
		User:    fmt.Sprintf("user-%s-%x", user, flags),
		Token:   strconv.FormatInt(rand.Int63(), 16),
		Flags:   flags,
	}
	meta.Acl[acl.User] = acl
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", acl.User, acl.Token, http.StatusOK))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", acl.User, "qwerty", http.StatusOK))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", "qwerty", "qwerty", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("GET", "get", "", "", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", acl.User, acl.Token, http.StatusOK))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", acl.User, "qwerty", http.StatusOK))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", "qwerty", "qwerty", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "upload", "", "", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", acl.User, acl.Token, http.StatusOK))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", acl.User, "qwerty", http.StatusOK))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", "qwerty", "qwerty", http.StatusForbidden))
	t.acl_requests = append(t.acl_requests, t.NewCheckRequest("POST", "delete", "", "", http.StatusForbidden))

	acl = bucket.BucketACL{
		Version: 1,
		User:    t.all_allowed_user,
		Token:   t.all_allowed_token,
		Flags:   bucket.BucketAuthWrite | bucket.BucketAuthNoToken,
	}
	meta.Acl[acl.User] = acl

	_, err := bucket.WriteBucket(t.ell, &meta)
	if err != nil {
		log.Fatalf("Could not upload bucket: %v", err)
	}

	return nil
}
Beispiel #5
0
func (bt *BackrunnerTest) StartEllipticsClientProxy(proxy_path string) {
	bt.conf.Elliptics.LogFile = bt.proxy_log
	bt.conf.Elliptics.LogPrefix = "backrunner: "

	file := fmt.Sprintf("%s/elliptics_transport.conf", bt.base)
	err := bt.conf.Save(file)
	if err != nil {
		log.Fatalf("Could not save client transport config: %v", err)
	}

	bt.ell, err = etransport.NewEllipticsTransport(&bt.conf.Elliptics)
	if err != nil {
		log.Fatal("Could not connect to elliptics server %v: %v", bt.elliptics_address, err)
	}

	bt.bucket_file = fmt.Sprintf("%s/buckets", bt.base)
	fd, err := os.Create(bt.bucket_file)
	if err != nil {
		log.Fatalf("Could not create bucket file %s: %v\n", bt.bucket_file, err)
	}
	defer fd.Close()

	_, err = fmt.Fprintf(fd, "%s\n", bt.acl_bucket)
	if err != nil {
		log.Fatalf("Could not write acl bucket into bucket file: %v\n", err)
	}
	for _, g := range bt.groups {
		b := fmt.Sprintf("b%d", g)
		_, err = fmt.Fprintf(fd, "%s\n", b)
		if err != nil {
			log.Fatalf("Could not write bucket into bucket file: %v\n", err)
		}

		meta := bucket.BucketMsgpack{
			Version: 1,
			Name:    b,
			Groups:  []uint32{g},
			Acl:     make(map[string]bucket.BucketACL),
		}

		acl := bucket.BucketACL{
			Version: 1,
			User:    bt.all_allowed_user,
			Token:   bt.all_allowed_token,
			Flags:   bucket.BucketAuthWrite | bucket.BucketAuthNoToken,
		}
		meta.Acl[acl.User] = acl

		_, err := bucket.WriteBucket(bt.ell, &meta)
		if err != nil {
			log.Fatal("Could not upload bucket %s into storage: %v", b, err)
		}

		bt.io_buckets = append(bt.io_buckets, b)
	}

	bt.ACLInit()

	file = fmt.Sprintf("%s/proxy.conf", bt.base)
	err = bt.conf.Save(file)
	if err != nil {
		log.Fatalf("Could not save proxy config: %v", err)
	}

	cmd := exec.Command(proxy_path, "-config", file, "-buckets", bt.bucket_file)
	cmd.Stdout = &bt.proxy_stdout
	cmd.Stderr = &bt.proxy_stderr

	err = cmd.Start()
	if err != nil {
		log.Fatalf("Could not start proxy process: %v\n", err)
	}

	bt.proxy_cmd = cmd

	// wait 1 second for proxy to start
	time.Sleep(1 * time.Second)
}