Exemplo n.º 1
0
func GetPaste(params martini.Params, ren render.Render, r *http.Request, cf *swift.Connection, mc *memcache.Client) {
	cachedPaste, err := mc.Get(params["pasteid"])
	format := params["format"]
	if err != nil {
		log.Println(err)
	}
	var paste Paste
	paste.PasteId = params["pasteid"]
	if cachedPaste == nil {
		log.Println("Asking swift for ", params["pasteid"])
		cfPaste, err := cf.ObjectGetBytes("go-cfpaste", params["pasteid"])
		if err != nil {
			if err.Error() == "Object Not Found" {
				ren.HTML(404, "404", paste)
				return
			} else {
				log.Println(err)
				ren.Error(500)
				return
			}
		}
		err = json.Unmarshal(cfPaste, &paste)
		PanicIf(err)
	} else {
		log.Println("Cache hit for ", params["pasteid"])
		err = json.Unmarshal(cachedPaste.Value, &paste)
		PanicIf(err)
	}
	if format == "json" {
		ren.JSON(200, paste)
	} else {
		ren.HTML(200, "paste", paste)
	}
	return
}
Exemplo n.º 2
0
func UploadImage(fileURL, fileName, containerName string) bool {
	cDallas := swift.Connection{
		UserName: "",
		ApiKey:   "",
		AuthUrl:  "https://auth.api.rackspacecloud.com/v1.0",
		Region:   "DFW",
	}
	if err := cDallas.Authenticate(); err != nil {
		log.Println("Couldn't authenticate with rackspace Dallas")
		return false
	}
	var rspaceHeader swift.Headers
	fDallas, err := cDallas.ObjectCreate(containerName, fileName, false, "", "image/jpg", rspaceHeader)
	if err != nil {
		log.Println("Failed to create rackspace object ", err)
		return false
	}
	defer fDallas.Close()
	d, err := ioutil.ReadFile(fileURL)
	if err != nil {
		log.Println("Failed to read source file ", err)
		return false
	}
	log.Println("uploading path:", fileURL)
	if _, err := fDallas.Write(d); err != nil {
		log.Println("Failed to write file to rackspace Dallas server", err)
		return false
	}
	return true
}
func main() {
	// Create a connection
	c := swift.Connection{
		UserName: os.Getenv("SWIFT_API_USER"),
		ApiKey:   os.Getenv("SWIFT_API_KEY"),
		AuthUrl:  os.Getenv("SWIFT_AUTH_URL"),
		Domain:   "", // Name of the domain (v3 auth only)
		Tenant:   "", // Name of the tenant (v2 auth only)
	}

	if len(os.Args) < 3 {
		fmt.Printf("Usage: %s project files...\n", os.Args[0])
		os.Exit(1)
	}

	println("Starting objekt storage uploader")

	var (
		project = os.Args[1]
	)

	container_name := "project-" + project

	// Authenticate
	err := c.Authenticate()
	if err != nil {
		panic(err)
	}

	files := os.Args[2:]

	uploadFiles(container_name, files, c)

}
Exemplo n.º 4
0
func TestTransport(t *testing.T) {
	UserName := os.Getenv("SWIFT_API_USER")
	ApiKey := os.Getenv("SWIFT_API_KEY")
	AuthUrl := os.Getenv("SWIFT_AUTH_URL")
	if UserName == "" || ApiKey == "" || AuthUrl == "" {
		t.Fatal("SWIFT_API_USER, SWIFT_API_KEY and SWIFT_AUTH_URL not all set")
	}
	tr := &someTransport{Transport: http.Transport{MaxIdleConnsPerHost: 2048}}
	ct := swift.Connection{
		UserName:       UserName,
		ApiKey:         ApiKey,
		AuthUrl:        AuthUrl,
		Tenant:         os.Getenv("SWIFT_TENANT"),
		TenantId:       os.Getenv("SWIFT_TENANT_ID"),
		Transport:      tr,
		ConnectTimeout: 60 * time.Second,
		Timeout:        60 * time.Second,
	}
	err := ct.Authenticate()
	if err != nil {
		t.Fatal("Auth failed", err)
	}
	if !ct.Authenticated() {
		t.Fatal("Not authenticated")
	}
}
Exemplo n.º 5
0
Arquivo: task.go Projeto: pquerna/hurl
func NewTask(ui common.UI) (workers.WorkerTask, error) {
	c := ui.ConfigGet()
	conf := c.GetSwiftConfig()
	if conf == nil {
		panic("Invalid Configuration object for swift worker")
	}

	conn := swift.Connection{
		UserName: conf.Username,
		ApiKey:   conf.ApiKey,
		AuthUrl:  conf.AuthUrl,
		// https://auth.api.rackspacecloud.com/v1.0
		Region:    conf.Region,
		UserAgent: fmt.Sprintf("hurl/1 http load tester; https://github.com/pquerna/hurl;  username=%s", conf.Username),
	}

	if authTokenCache != "" && storageUrlCache != "" {
		conn.AuthToken = authTokenCache
		conn.StorageUrl = storageUrlCache
	} else {
		err := conn.Authenticate()
		if err != nil {
			return nil, err
		}
		authTokenCache = conn.AuthToken
		storageUrlCache = conn.StorageUrl
	}

	return &Task{
		conf: conf,
		conn: &conn}, nil
}
func downloadFiles(container string, files []string, c swift.Connection) {
	wpauswiftcommons.CreatePublicContainer(container, c)
	names, err := c.ObjectNames(container, nil)

	if err != nil {
		println(err.Error())
		return
	}

	for _, name := range files {
		if !exists(names, name) {
			println("Object does not exists: " + name)
			continue
		}

		f, err := os.Create(name)
		if err != nil {
			println("Couldn't create file: " + name)
			continue
		}
		defer f.Close()

		_, err = c.ObjectGet(container, name, f, true, nil)
		if err != nil {
			println("Couldn't download file: " + name)
		} else {
			println("Downloaded file: " + name)
		}
	}
}
Exemplo n.º 7
0
func sendTempfileToArchive(tmpfilename string, archivefilename string, authenticated swift.Connection) error {
	defer os.Remove(tmpfilename)

	// read file into byte array
	bytearray, err := ioutil.ReadFile(tmpfilename)

	if err != nil {
		log.Printf("Failed to read temp file to bytearray - %v", err)
		return err
	}

	targetFilename := strings.ToLower(archivefilename)

	if targetFilename == "image.jpg" {
		targetFilename = "image_" + strconv.Itoa(randgen.Intn(1000000)+1) + ".jpg"
	}
	err2 := authenticated.ObjectPutBytes(albumfolder, targetFilename, bytearray, "image-jpeg")
	if err2 != nil {
		log.Printf("Failed to send temp file to archive - %v", err2)
		return err2
	}

	return nil

}
Exemplo n.º 8
0
func main() {
	// Create a connection
	c := swift.Connection{
		UserName: os.Getenv("SWIFT_API_USER"),
		ApiKey:   os.Getenv("SWIFT_API_KEY"),
		AuthUrl:  os.Getenv("SWIFT_AUTH_URL"),
		Domain:   "", // Name of the domain (v3 auth only)
		Tenant:   "", // Name of the tenant (v2 auth only)
	}

	println("Starting cleaner")

	// Authenticate
	err := c.Authenticate()
	if err != nil {
		panic(err)
	}

	containers := get_projects(c)

	fmt.Println("Found job containers:")
	fmt.Println(containers)

	for _, val := range containers {
		clean_old_jobs(c, val)
	}
}
Exemplo n.º 9
0
func makeConnection() (*swift.Connection, error) {
	var err error

	UserName := os.Getenv("SWIFT_API_USER")
	ApiKey := os.Getenv("SWIFT_API_KEY")
	AuthUrl := os.Getenv("SWIFT_AUTH_URL")

	Insecure := os.Getenv("SWIFT_AUTH_INSECURE")
	ConnectionChannelTimeout := os.Getenv("SWIFT_CONNECTION_CHANNEL_TIMEOUT")
	DataChannelTimeout := os.Getenv("SWIFT_DATA_CHANNEL_TIMEOUT")

	if UserName == "" || ApiKey == "" || AuthUrl == "" {
		if srv != nil {
			srv.Close()
		}
		srv, err = swifttest.NewSwiftServer("localhost")
		if err != nil {
			return nil, err
		}

		UserName = "******"
		ApiKey = "swifttest"
		AuthUrl = srv.AuthURL
	}

	transport := &http.Transport{
		Proxy:               http.ProxyFromEnvironment,
		MaxIdleConnsPerHost: 2048,
	}
	if Insecure == "1" {
		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	}

	c := swift.Connection{
		UserName:       UserName,
		ApiKey:         ApiKey,
		AuthUrl:        AuthUrl,
		Transport:      transport,
		ConnectTimeout: 60 * time.Second,
		Timeout:        60 * time.Second,
	}

	var timeout int64
	if ConnectionChannelTimeout != "" {
		timeout, err = strconv.ParseInt(ConnectionChannelTimeout, 10, 32)
		if err == nil {
			c.ConnectTimeout = time.Duration(timeout) * time.Second
		}
	}

	if DataChannelTimeout != "" {
		timeout, err = strconv.ParseInt(DataChannelTimeout, 10, 32)
		if err == nil {
			c.Timeout = time.Duration(timeout) * time.Second
		}
	}

	return &c, nil
}
Exemplo n.º 10
0
// NewPublishedStorage creates new instance of PublishedStorage with specified Swift access
// keys, tenant and tenantId
func NewPublishedStorage(username string, password string, authURL string, tenant string, tenantID string, container string, prefix string) (*PublishedStorage, error) {
	if username == "" {
		if username = os.Getenv("OS_USERNAME"); username == "" {
			username = os.Getenv("ST_USER")
		}
	}
	if password == "" {
		if password = os.Getenv("OS_PASSWORD"); password == "" {
			password = os.Getenv("ST_KEY")
		}
	}
	if authURL == "" {
		if authURL = os.Getenv("OS_AUTH_URL"); authURL == "" {
			authURL = os.Getenv("ST_AUTH")
		}
	}
	if tenant == "" {
		tenant = os.Getenv("OS_TENANT_NAME")
	}
	if tenantID == "" {
		tenantID = os.Getenv("OS_TENANT_ID")
	}

	ct := swift.Connection{
		UserName:       username,
		ApiKey:         password,
		AuthUrl:        authURL,
		UserAgent:      "aptly/" + aptly.Version,
		Tenant:         tenant,
		TenantId:       tenantID,
		ConnectTimeout: 60 * time.Second,
		Timeout:        60 * time.Second,
	}
	err := ct.Authenticate()
	if err != nil {
		return nil, fmt.Errorf("swift authentication failed: %s", err)
	}

	var bulkDelete bool
	resp, err := http.Get(filepath.Join(ct.StorageUrl, "..", "..") + "/info")
	if err == nil {
		defer resp.Body.Close()
		decoder := json.NewDecoder(resp.Body)
		var infos swiftInfo
		if decoder.Decode(&infos) == nil {
			_, bulkDelete = infos["bulk_delete"]
		}
	}

	result := &PublishedStorage{
		conn:              ct,
		container:         container,
		prefix:            prefix,
		supportBulkDelete: bulkDelete,
	}

	return result, nil
}
Exemplo n.º 11
0
func get_objects(c swift.Connection, container string, prefix string) []string {
	opts := new(swift.ObjectsOpts)
	opts.Prefix = prefix
	names, err := c.ObjectNames(container, opts)
	if err != nil {
		panic(err)
	}
	return names
}
Exemplo n.º 12
0
func get_projects(c swift.Connection) []string {
	opts := new(swift.ContainersOpts)
	opts.Prefix = "jobs"

	containers, err := c.ContainerNames(opts)
	if err != nil {
		panic(err)
	}
	return containers
}
Exemplo n.º 13
0
func remove_objects(c swift.Connection, container string, names []string) bool {
	removed_all := true
	for _, val := range names {
		fmt.Printf("Removing %s\n", val)
		err := c.ObjectDelete(container, val)
		if err != nil {
			fmt.Printf("Couldn't delete '%s' because: %s\n", val, err.Error())
			removed_all = false
		}
	}
	return removed_all
}
Exemplo n.º 14
0
func main() {
	//lol if you don't already use swiftly
	username := os.Getenv("SWIFTLY_AUTH_USER")
	apikey := os.Getenv("SWIFTLY_AUTH_KEY")
	authurl := os.Getenv("SWIFTLY_AUTH_URL")
	region := os.Getenv("SWIFTLY_REGION")
	snet := os.Getenv("SWIFTLY_SNET")
	dockerized := os.Getenv("DOCKERIZED")
	memcacheOverride := os.Getenv("MEMCACHEADDR")
	if strings.ToLower(dockerized) == "true" {
		memcacheOverride = fmt.Sprintf("%s:%s", os.Getenv("MEMCACHED_PORT_11211_TCP_ADDR"), os.Getenv("MEMCACHED_PORT_11211_TCP_PORT"))
	}
	memcacheAddr := "127.0.0.1:11211"
	if memcacheOverride != "" {
		memcacheAddr = memcacheOverride
	}
	internal := false
	if strings.ToLower(snet) == "true" {
		internal = true
	}
	//martini looks for a HOST and PORT env var to determine what to listen on
	m := martini.Classic()

	cf := swift.Connection{
		UserName: username,
		ApiKey:   apikey,
		AuthUrl:  authurl,
		Region:   region,
		Internal: internal,
	}
	// Authenticate
	err := cf.Authenticate()
	PanicIf(err)
	m.Map(&cf)
	log.Println(os.Environ())
	log.Println(memcacheAddr)
	mc := memcache.New(memcacheAddr)
	m.Map(mc)

	m.Use(render.Renderer())

	m.Get("/", IndexPage)
	m.Get("/history", GetHistory)
	m.Get("/:pasteid", GetPaste)
	m.Get("/:pasteid/:format", GetPaste)
	m.Post("/paste", binding.Json(Paste{}), binding.ErrorHandler, SavePaste)
	m.Put("/paste", binding.Json(Paste{}), binding.ErrorHandler, SavePaste)
	m.Run()
}
Exemplo n.º 15
0
func GetHistory(ren render.Render, r *http.Request, cf *swift.Connection) {
	opts := swift.ObjectsOpts{}
	opts.Prefix = "cfpaste-"
	opts.Limit = 10
	objects, err := cf.ObjectNames("go-cfpaste", &opts)
	pastes := make([]string, 10)
	for i := range objects {
		object, headers, err := cf.Object("go-cfpaste", objects[i])
		PanicIf(err)
		log.Println(object.Name)
		pastes = append(pastes, headers["X-Object-Meta-Pasteid"])
	}
	PanicIf(err)
	ren.HTML(200, "history", pastes)
	return
}
Exemplo n.º 16
0
func Example() {
	// Create a connection
	c := swift.Connection{
		UserName: "******",
		ApiKey:   "key",
		AuthUrl:  "auth_url",
	}
	// Authenticate
	err := c.Authenticate()
	if err != nil {
		panic(err)
	}
	// List all the containers
	containers, err := c.ContainerNames(nil)
	fmt.Println(containers)
	// etc...
}
Exemplo n.º 17
0
// New constructs a new Driver with the given Openstack Swift credentials and container name
func New(params DriverParameters) (*Driver, error) {
	transport := &http.Transport{
		Proxy:               http.ProxyFromEnvironment,
		MaxIdleConnsPerHost: 2048,
		TLSClientConfig:     &tls.Config{InsecureSkipVerify: params.InsecureSkipVerify},
	}

	ct := swift.Connection{
		UserName:       params.Username,
		ApiKey:         params.Password,
		AuthUrl:        params.AuthURL,
		Region:         params.Region,
		UserAgent:      "distribution",
		Tenant:         params.Tenant,
		TenantId:       params.TenantID,
		Domain:         params.Domain,
		DomainId:       params.DomainID,
		Transport:      transport,
		ConnectTimeout: 60 * time.Second,
		Timeout:        15 * 60 * time.Second,
	}
	err := ct.Authenticate()
	if err != nil {
		return nil, fmt.Errorf("Swift authentication failed: %s", err)
	}

	if err := ct.ContainerCreate(params.Container, nil); err != nil {
		return nil, fmt.Errorf("Failed to create container %s (%s)", params.Container, err)
	}

	d := &driver{
		Conn:              ct,
		Container:         params.Container,
		Prefix:            params.Prefix,
		BulkDeleteSupport: detectBulkDelete(params.AuthURL),
		ChunkSize:         params.ChunkSize,
	}

	return &Driver{
		baseEmbed: baseEmbed{
			Base: base.Base{
				StorageDriver: d,
			},
		},
	}, nil
}
Exemplo n.º 18
0
// Test a connection can be serialized and unserialized with XML
func TestSerializeConnectionXml(t *testing.T) {
	serializedConnection, err := xml.Marshal(c)
	if err != nil {
		t.Fatalf("Failed to serialize connection: %v", err)
	}
	c2 := new(swift.Connection)
	err = xml.Unmarshal(serializedConnection, &c2)
	if err != nil {
		t.Fatalf("Failed to unserialize connection: %v", err)
	}
	if !c2.Authenticated() {
		t.Fatal("Should be authenticated")
	}
	_, _, err = c2.Account()
	if err != nil {
		t.Fatalf("Failed to use unserialized connection: %v", err)
	}
}
Exemplo n.º 19
0
func ExampleConnection() {
	// Create a v1 auth connection
	c := swift.Connection{
		// This should be your username
		UserName: "******",
		// This should be your api key
		ApiKey: "key",
		// This should be a v1 auth url, eg
		//  Rackspace US        https://auth.api.rackspacecloud.com/v1.0
		//  Rackspace UK        https://lon.auth.api.rackspacecloud.com/v1.0
		//  Memset Memstore UK  https://auth.storage.memset.com/v1.0
		AuthUrl: "auth_url",
	}

	// Authenticate
	err := c.Authenticate()
	if err != nil {
		panic(err)
	}
	// List all the containers
	containers, err := c.ContainerNames(nil)
	fmt.Println(containers)
	// etc...

	// ------ or alternatively create a v2 connection ------

	// Create a v2 auth connection
	c = swift.Connection{
		// This is the sub user for the storage - eg "admin"
		UserName: "******",
		// This should be your api key
		ApiKey: "key",
		// This should be a version2 auth url, eg
		//  Rackspace v2        https://identity.api.rackspacecloud.com/v2.0
		//  Memset Memstore v2  https://auth.storage.memset.com/v2.0
		AuthUrl: "v2_auth_url",
		// Region to use - default is use first region if unset
		Region: "LON",
		// Name of the tenant - this is likely your username
		Tenant: "jim",
	}

	// as above...
}
Exemplo n.º 20
0
func main() {
	//lol if you don't already use swiftly
	username := os.Getenv("SWIFTLY_AUTH_USER")
	apikey := os.Getenv("SWIFTLY_AUTH_KEY")
	authurl := os.Getenv("SWIFTLY_AUTH_URL")
	region := os.Getenv("SWIFTLY_REGION")
	snet := os.Getenv("SWIFTLY_SNET")
	internal := false
	if strings.ToLower(snet) == "true" {
		internal = true
	}
	//martini looks for a HOST and PORT env var to determine what to listen on
	m := martini.Classic()

	cf := swift.Connection{
		UserName: username,
		ApiKey:   apikey,
		AuthUrl:  authurl,
		Region:   region,
		Internal: internal,
	}
	// Authenticate
	err := cf.Authenticate()
	PanicIf(err)
	m.Map(&cf)

	mc := memcache.New("127.0.0.1:11211")
	m.Map(mc)

	m.Use(render.Renderer())

	m.Get("/", IndexPage)
	m.Get("/history", GetHistory)
	m.Get("/:pasteid", GetPaste)
	m.Get("/:pasteid/:format", GetPaste)
	m.Post("/paste", binding.Json(Paste{}), binding.ErrorHandler, SavePaste)
	m.Put("/paste", binding.Json(Paste{}), binding.ErrorHandler, SavePaste)
	m.Run()
}
Exemplo n.º 21
0
func Main() {
	headers["X-Log-Retention"] = "true"
	headers["X-Container-Meta-fubar"] = "false"
	config := getConfig()
	err := config.Parse(os.Args[1:])

	if err != nil {
		fmt.Printf("error verifying flags, %v. See 'osync --help'.", err)
		os.Exit(1)
	}

	prefixlen = len(config.DataDir)

	if err = config.InitializeOSAuth(); nil != err {
		panic("Initial Openstack Authtication Code Failed With " + err.Error())
	}

	headers[OS_TEMP_URL_KEY] = config.OSAuth.TempURLKey

	c := swift.Connection{
		UserName: config.OSAuth.Username,
		ApiKey:   config.OSAuth.Password,
		AuthUrl:  config.OSAuth.Host,
		Tenant:   config.OSAuth.ProjectName,
	}
	// Authenticate
	err = c.Authenticate()
	if err != nil {
		panic(err)
	}

	// List all the containers
	containers, err := c.ContainerNames(nil)
	fmt.Println(containers)

	worker(c, config.DataDir, 0)
}
Exemplo n.º 22
0
func TestTransport(t *testing.T) {
	var err error
	UserName := os.Getenv("SWIFT_API_USER")
	ApiKey := os.Getenv("SWIFT_API_KEY")
	AuthUrl := os.Getenv("SWIFT_AUTH_URL")
	if UserName == "" || ApiKey == "" || AuthUrl == "" {
		srv, err = swifttest.NewSwiftServer("localhost")
		if err != nil {
			t.Fatal("Failed to create server", err)
		}
		UserName = "******"
		ApiKey = "swifttest"
		AuthUrl = srv.AuthURL
	}
	tr := &someTransport{Transport: http.Transport{MaxIdleConnsPerHost: 2048}}
	ct := swift.Connection{
		UserName:       UserName,
		ApiKey:         ApiKey,
		AuthUrl:        AuthUrl,
		Tenant:         os.Getenv("SWIFT_TENANT"),
		TenantId:       os.Getenv("SWIFT_TENANT_ID"),
		Transport:      tr,
		ConnectTimeout: 60 * time.Second,
		Timeout:        60 * time.Second,
	}
	err = ct.Authenticate()
	if err != nil {
		t.Fatal("Auth failed", err)
	}
	if !ct.Authenticated() {
		t.Fatal("Not authenticated")
	}
	if srv != nil {
		srv.Close()
	}
}
Exemplo n.º 23
0
func SavePaste(paste Paste, ren render.Render, r *http.Request, cf *swift.Connection, mc *memcache.Client) {
	paste.PasteId = genPasteId()
	payload, _ := json.Marshal(paste)
	seconds, err := getTTL(paste.PasteTtl)
	PanicIf(err)
	headers := swift.Headers{}
	now := time.Now()
	pasteIndex := 9999999999 - now.Unix()
	indexKey := fmt.Sprintf("cfpaste-%d", pasteIndex)
	headers["x-object-meta-pastetype"] = paste.PasteType
	headers["x-object-meta-pasteid"] = paste.PasteId
	headers["x-object-meta-pasteindex"] = fmt.Sprintf("%d", pasteIndex)
	if seconds != 0 {
		headers["x-delete-after"] = fmt.Sprintf("%d", seconds)
	}
	buf := bytes.NewBuffer(payload)
	_, err = cf.ObjectPut("go-cfpaste", paste.PasteId, buf, true, "", "application/json; charset=utf-8", headers)
	PanicIf(err)
	// gholt's listing index hack so that he can spy on pastes
	_, err = cf.ObjectPut("go-cfpaste", indexKey, bytes.NewBuffer([]byte("")), true, "", "application/json; charset=utf-8", headers)
	PanicIf(err)
	mc.Set(&memcache.Item{Key: paste.PasteId, Value: payload})
	ren.JSON(200, map[string]interface{}{"pasteid": paste.PasteId})
}
Exemplo n.º 24
0
func main() {
	// Create a connection
	c := swift.Connection{
		UserName: os.Getenv("SWIFT_API_USER"),
		ApiKey:   os.Getenv("SWIFT_API_KEY"),
		AuthUrl:  os.Getenv("SWIFT_AUTH_URL"),
		Domain:   "", // Name of the domain (v3 auth only)
		Tenant:   "", // Name of the tenant (v2 auth only)
	}

	if len(os.Args) < 5 {
		fmt.Printf("Usage: %s customer jobId status path\n", os.Args[0])
		os.Exit(1)
	}

	println("Starting objekt storage uploader")

	var (
		customer = os.Args[1]
		jobId    = os.Args[2]
		status   = os.Args[3]
		path     = os.Args[4]
	)

	container_name := "jobs-" + customer
	file_prefix := jobId

	// Authenticate
	err := c.Authenticate()
	if err != nil {
		panic(err)
	}

	uploadContentsInFolder(path, file_prefix, container_name, c)
	wpaumetadata.Add(c, container_name, jobId, status)
}
Exemplo n.º 25
0
func UploadFile(container string, prefix string, path string, c swift.Connection) {
	dat, err := ioutil.ReadFile(path)
	if err != nil {
		println(err.Error())
	} else {
		name := filepath.Base(path)
		if len(prefix) > 0 {
			name = prefix + "-" + name
		}
		ext := filepath.Ext(path)
		hasher := md5.New()
		hasher.Write(dat)
		md5hash := hex.EncodeToString(hasher.Sum(nil))

		fmt.Printf("Uploading %s to container %s\n", name, container)
		file, err := c.ObjectCreate(container, name, false, md5hash, ext, nil)
		if err != nil {
			println(err.Error())
		} else {
			file.Write(dat)
		}
		file.Close()
	}
}
Exemplo n.º 26
0
func TestSwiftBackend(t *testing.T) {
	if os.Getenv("OS_USERNAME") == "" || os.Getenv("OS_PASSWORD") == "" ||
		os.Getenv("OS_AUTH_URL") == "" {
		t.SkipNow()
	}
	username := os.Getenv("OS_USERNAME")
	password := os.Getenv("OS_PASSWORD")
	authUrl := os.Getenv("OS_AUTH_URL")
	tenant := os.Getenv("OS_TENANT_NAME")

	ts := time.Now().UnixNano()
	container := fmt.Sprintf("vault-test-%d", ts)

	cleaner := swift.Connection{
		UserName:  username,
		ApiKey:    password,
		AuthUrl:   authUrl,
		Tenant:    tenant,
		Transport: cleanhttp.DefaultPooledTransport(),
	}

	err := cleaner.Authenticate()
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	err = cleaner.ContainerCreate(container, nil)
	if nil != err {
		t.Fatalf("Unable to create test container '%s': %v", container, err)
	}
	defer func() {
		newObjects, err := cleaner.ObjectNamesAll(container, nil)
		if err != nil {
			t.Fatalf("err: %s", err)
		}
		for _, o := range newObjects {
			err := cleaner.ObjectDelete(container, o)
			if err != nil {
				t.Fatalf("err: %s", err)
			}
		}
		err = cleaner.ContainerDelete(container)
		if err != nil {
			t.Fatalf("err: %s", err)
		}
	}()

	logger := logformat.NewVaultLogger(log.LevelTrace)

	b, err := NewBackend("swift", logger, map[string]string{
		"username":  username,
		"password":  password,
		"container": container,
		"auth_url":  authUrl,
		"tenant":    tenant,
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	testBackend(t, b)
	testBackend_ListPrefix(t, b)

}
Exemplo n.º 27
0
// New constructs a new Driver with the given Openstack Swift credentials and container name
func New(params Parameters) (*Driver, error) {
	transport := &http.Transport{
		Proxy:               http.ProxyFromEnvironment,
		MaxIdleConnsPerHost: 2048,
		TLSClientConfig:     &tls.Config{InsecureSkipVerify: params.InsecureSkipVerify},
	}

	ct := swift.Connection{
		UserName:       params.Username,
		ApiKey:         params.Password,
		AuthUrl:        params.AuthURL,
		Region:         params.Region,
		UserAgent:      "distribution/" + version.Version,
		Tenant:         params.Tenant,
		TenantId:       params.TenantID,
		Domain:         params.Domain,
		DomainId:       params.DomainID,
		TrustId:        params.TrustID,
		Transport:      transport,
		ConnectTimeout: 60 * time.Second,
		Timeout:        15 * 60 * time.Second,
	}
	err := ct.Authenticate()
	if err != nil {
		return nil, fmt.Errorf("Swift authentication failed: %s", err)
	}

	if _, _, err := ct.Container(params.Container); err == swift.ContainerNotFound {
		if err := ct.ContainerCreate(params.Container, nil); err != nil {
			return nil, fmt.Errorf("Failed to create container %s (%s)", params.Container, err)
		}
	} else if err != nil {
		return nil, fmt.Errorf("Failed to retrieve info about container %s (%s)", params.Container, err)
	}

	d := &driver{
		Conn:           ct,
		Container:      params.Container,
		Prefix:         params.Prefix,
		ChunkSize:      params.ChunkSize,
		TempURLMethods: make([]string, 0),
		AccessKey:      params.AccessKey,
	}

	info := swiftInfo{}
	if config, err := d.Conn.QueryInfo(); err == nil {
		_, d.BulkDeleteSupport = config["bulk_delete"]

		if err := mapstructure.Decode(config, &info); err == nil {
			d.TempURLContainerKey = info.Swift.Version >= "2.3.0"
			d.TempURLMethods = info.Tempurl.Methods
		}
	} else {
		d.TempURLContainerKey = params.TempURLContainerKey
		d.TempURLMethods = params.TempURLMethods
	}

	if len(d.TempURLMethods) > 0 {
		secretKey := params.SecretKey
		if secretKey == "" {
			secretKey, _ = generateSecret()
		}

		// Since Swift 2.2.2, we can now set secret keys on containers
		// in addition to the account secret keys. Use them in preference.
		if d.TempURLContainerKey {
			_, containerHeaders, err := d.Conn.Container(d.Container)
			if err != nil {
				return nil, fmt.Errorf("Failed to fetch container info %s (%s)", d.Container, err)
			}

			d.SecretKey = containerHeaders["X-Container-Meta-Temp-Url-Key"]
			if d.SecretKey == "" || (params.SecretKey != "" && d.SecretKey != params.SecretKey) {
				m := swift.Metadata{}
				m["temp-url-key"] = secretKey
				if d.Conn.ContainerUpdate(d.Container, m.ContainerHeaders()); err == nil {
					d.SecretKey = secretKey
				}
			}
		} else {
			// Use the account secret key
			_, accountHeaders, err := d.Conn.Account()
			if err != nil {
				return nil, fmt.Errorf("Failed to fetch account info (%s)", err)
			}

			d.SecretKey = accountHeaders["X-Account-Meta-Temp-Url-Key"]
			if d.SecretKey == "" || (params.SecretKey != "" && d.SecretKey != params.SecretKey) {
				m := swift.Metadata{}
				m["temp-url-key"] = secretKey
				if err := d.Conn.AccountUpdate(m.AccountHeaders()); err == nil {
					d.SecretKey = secretKey
				}
			}
		}
	}

	return &Driver{
		baseEmbed: baseEmbed{
			Base: base.Base{
				StorageDriver: d,
			},
		},
	}, nil
}
Exemplo n.º 28
0
func CreatePublicContainer(name string, c swift.Connection) {
	headers := map[string]string{
		"X-Container-Read": ".r:*",
	}
	c.ContainerCreate(name, headers)
}
Exemplo n.º 29
0
// newSwiftBackend constructs a Swift backend using a pre-existing
// container. Credentials can be provided to the backend, sourced
// from the environment.
func newSwiftBackend(conf map[string]string, logger log.Logger) (Backend, error) {

	username := os.Getenv("OS_USERNAME")
	if username == "" {
		username = conf["username"]
		if username == "" {
			return nil, fmt.Errorf("missing username")
		}
	}
	password := os.Getenv("OS_PASSWORD")
	if password == "" {
		password = conf["password"]
		if password == "" {
			return nil, fmt.Errorf("missing password")
		}
	}
	authUrl := os.Getenv("OS_AUTH_URL")
	if authUrl == "" {
		authUrl = conf["auth_url"]
		if authUrl == "" {
			return nil, fmt.Errorf("missing auth_url")
		}
	}
	container := os.Getenv("OS_CONTAINER")
	if container == "" {
		container = conf["container"]
		if container == "" {
			return nil, fmt.Errorf("missing container")
		}
	}
	tenant := os.Getenv("OS_TENANT_NAME")
	if tenant == "" {
		tenant = conf["tenant"]
	}

	c := swift.Connection{
		UserName:  username,
		ApiKey:    password,
		AuthUrl:   authUrl,
		Tenant:    tenant,
		Transport: cleanhttp.DefaultPooledTransport(),
	}

	err := c.Authenticate()
	if err != nil {
		return nil, err
	}

	_, _, err = c.Container(container)
	if err != nil {
		return nil, fmt.Errorf("Unable to access container '%s': %v", container, err)
	}

	maxParStr, ok := conf["max_parallel"]
	var maxParInt int
	if ok {
		maxParInt, err = strconv.Atoi(maxParStr)
		if err != nil {
			return nil, errwrap.Wrapf("failed parsing max_parallel parameter: {{err}}", err)
		}
		if logger.IsDebug() {
			logger.Debug("swift: max_parallel set", "max_parallel", maxParInt)
		}
	}

	s := &SwiftBackend{
		client:     &c,
		container:  container,
		logger:     logger,
		permitPool: NewPermitPool(maxParInt),
	}
	return s, nil
}
Exemplo n.º 30
0
func worker(c swift.Connection, parent string, level int) string {
	dirs, err := ioutil.ReadDir(parent)

	if nil != err {
		fmt.Println(err.Error())
	}

	var logstr string
	for _, f0 := range dirs {
		var curdir string
		if prefixlen == len(parent) {
			curdir = f0.Name()
		} else {
			curdir = parent[prefixlen:len(parent)]
			curdir = strings.TrimLeft(curdir, "/")
		}

		//		fmt.Println(parent, level, curdir)

		switch {

		case f0.IsDir():
			if 0 == level {
				logstr = fmt.Sprint(strings.Repeat("\t", level)+"Found Container Creating...", curdir)
				fmt.Println(logstr)
				if err = c.ContainerCreate(curdir, headers); err != nil {
					panicString := fmt.Sprint("PutContainer Error:", err)
					panic(panicString)
				}
			}
			worker(c, parent+"/"+f0.Name(), level+1)
		case !f0.IsDir():
			if strings.HasPrefix(f0.Name(), ".") {
				logstr = fmt.Sprint("Found Hidden Object. Ommited ", f0.Name(), " to ", curdir)
				fmt.Println(logstr)
				continue
			}

			contents, _err := ioutil.ReadFile(parent + "/" + f0.Name())
			if nil != _err {
				panicString := fmt.Sprint("PutObject Error:", _err)
				panic(panicString)
			}

			h := sha1.New()
			h.Write(contents)
			obj_sha1hash := hex.EncodeToString(h.Sum(nil))
			logstr = fmt.Sprint(strings.Repeat("\t", level)+"Found Object. Uploading ", f0.Name(), " [ ", obj_sha1hash, " ] to ", curdir)
			fmt.Println(logstr)

			fstream, err := c.ObjectCreate(curdir, f0.Name(), false, "", "application/zip", headers)
			if nil != err {
				panicString := fmt.Sprint("PutObject Error:", err)
				panic(panicString)
			}

			if counts, err := fstream.Write(contents); nil != err {
				panicString := fmt.Sprint("PutObject Error:", err)
				panic(panicString)
			} else {
				fmt.Printf("%vBytes Total: %v, Written: %v\n", strings.Repeat("\t", level), len(contents), counts)
			}

			fstream.Close()

			return ""
		}
	}
	return ""
}