func GetPaste(params martini.Params, ren render.Render, r *http.Request, cf *swift.Connection, mc *memcache.Client) {
cachedPaste, err := mc.Get(params["pasteid"])
format := params["format"]
if err != nil {
log.Println(err)
}
var paste Paste
paste.PasteId = params["pasteid"]
if cachedPaste == nil {
log.Println("Asking swift for ", params["pasteid"])
cfPaste, err := cf.ObjectGetBytes("go-cfpaste", params["pasteid"])
if err != nil {
if err.Error() == "Object Not Found" {
ren.HTML(404, "404", paste)
return
} else {
log.Println(err)
ren.Error(500)
return
}
}
err = json.Unmarshal(cfPaste, &paste)
PanicIf(err)
} else {
log.Println("Cache hit for ", params["pasteid"])
err = json.Unmarshal(cachedPaste.Value, &paste)
PanicIf(err)
}
if format == "json" {
ren.JSON(200, paste)
} else {
ren.HTML(200, "paste", paste)
}
return
}
func UploadImage(fileURL, fileName, containerName string) bool {
cDallas := swift.Connection{
UserName: "",
ApiKey: "",
AuthUrl: "https://auth.api.rackspacecloud.com/v1.0",
Region: "DFW",
}
if err := cDallas.Authenticate(); err != nil {
log.Println("Couldn't authenticate with rackspace Dallas")
return false
}
var rspaceHeader swift.Headers
fDallas, err := cDallas.ObjectCreate(containerName, fileName, false, "", "image/jpg", rspaceHeader)
if err != nil {
log.Println("Failed to create rackspace object ", err)
return false
}
defer fDallas.Close()
d, err := ioutil.ReadFile(fileURL)
if err != nil {
log.Println("Failed to read source file ", err)
return false
}
log.Println("uploading path:", fileURL)
if _, err := fDallas.Write(d); err != nil {
log.Println("Failed to write file to rackspace Dallas server", err)
return false
}
return true
}
func main() {
// Create a connection
c := swift.Connection{
UserName: os.Getenv("SWIFT_API_USER"),
ApiKey: os.Getenv("SWIFT_API_KEY"),
AuthUrl: os.Getenv("SWIFT_AUTH_URL"),
Domain: "", // Name of the domain (v3 auth only)
Tenant: "", // Name of the tenant (v2 auth only)
}
if len(os.Args) < 3 {
fmt.Printf("Usage: %s project files...\n", os.Args[0])
os.Exit(1)
}
println("Starting objekt storage uploader")
var (
project = os.Args[1]
)
container_name := "project-" + project
// Authenticate
err := c.Authenticate()
if err != nil {
panic(err)
}
files := os.Args[2:]
uploadFiles(container_name, files, c)
}
func TestTransport(t *testing.T) {
UserName := os.Getenv("SWIFT_API_USER")
ApiKey := os.Getenv("SWIFT_API_KEY")
AuthUrl := os.Getenv("SWIFT_AUTH_URL")
if UserName == "" || ApiKey == "" || AuthUrl == "" {
t.Fatal("SWIFT_API_USER, SWIFT_API_KEY and SWIFT_AUTH_URL not all set")
}
tr := &someTransport{Transport: http.Transport{MaxIdleConnsPerHost: 2048}}
ct := swift.Connection{
UserName: UserName,
ApiKey: ApiKey,
AuthUrl: AuthUrl,
Tenant: os.Getenv("SWIFT_TENANT"),
TenantId: os.Getenv("SWIFT_TENANT_ID"),
Transport: tr,
ConnectTimeout: 60 * time.Second,
Timeout: 60 * time.Second,
}
err := ct.Authenticate()
if err != nil {
t.Fatal("Auth failed", err)
}
if !ct.Authenticated() {
t.Fatal("Not authenticated")
}
}
func NewTask(ui common.UI) (workers.WorkerTask, error) {
c := ui.ConfigGet()
conf := c.GetSwiftConfig()
if conf == nil {
panic("Invalid Configuration object for swift worker")
}
conn := swift.Connection{
UserName: conf.Username,
ApiKey: conf.ApiKey,
AuthUrl: conf.AuthUrl,
// https://auth.api.rackspacecloud.com/v1.0
Region: conf.Region,
UserAgent: fmt.Sprintf("hurl/1 http load tester; https://github.com/pquerna/hurl; username=%s", conf.Username),
}
if authTokenCache != "" && storageUrlCache != "" {
conn.AuthToken = authTokenCache
conn.StorageUrl = storageUrlCache
} else {
err := conn.Authenticate()
if err != nil {
return nil, err
}
authTokenCache = conn.AuthToken
storageUrlCache = conn.StorageUrl
}
return &Task{
conf: conf,
conn: &conn}, nil
}
func downloadFiles(container string, files []string, c swift.Connection) {
wpauswiftcommons.CreatePublicContainer(container, c)
names, err := c.ObjectNames(container, nil)
if err != nil {
println(err.Error())
return
}
for _, name := range files {
if !exists(names, name) {
println("Object does not exists: " + name)
continue
}
f, err := os.Create(name)
if err != nil {
println("Couldn't create file: " + name)
continue
}
defer f.Close()
_, err = c.ObjectGet(container, name, f, true, nil)
if err != nil {
println("Couldn't download file: " + name)
} else {
println("Downloaded file: " + name)
}
}
}
func sendTempfileToArchive(tmpfilename string, archivefilename string, authenticated swift.Connection) error {
defer os.Remove(tmpfilename)
// read file into byte array
bytearray, err := ioutil.ReadFile(tmpfilename)
if err != nil {
log.Printf("Failed to read temp file to bytearray - %v", err)
return err
}
targetFilename := strings.ToLower(archivefilename)
if targetFilename == "image.jpg" {
targetFilename = "image_" + strconv.Itoa(randgen.Intn(1000000)+1) + ".jpg"
}
err2 := authenticated.ObjectPutBytes(albumfolder, targetFilename, bytearray, "image-jpeg")
if err2 != nil {
log.Printf("Failed to send temp file to archive - %v", err2)
return err2
}
return nil
}
func main() {
// Create a connection
c := swift.Connection{
UserName: os.Getenv("SWIFT_API_USER"),
ApiKey: os.Getenv("SWIFT_API_KEY"),
AuthUrl: os.Getenv("SWIFT_AUTH_URL"),
Domain: "", // Name of the domain (v3 auth only)
Tenant: "", // Name of the tenant (v2 auth only)
}
println("Starting cleaner")
// Authenticate
err := c.Authenticate()
if err != nil {
panic(err)
}
containers := get_projects(c)
fmt.Println("Found job containers:")
fmt.Println(containers)
for _, val := range containers {
clean_old_jobs(c, val)
}
}
func makeConnection() (*swift.Connection, error) {
var err error
UserName := os.Getenv("SWIFT_API_USER")
ApiKey := os.Getenv("SWIFT_API_KEY")
AuthUrl := os.Getenv("SWIFT_AUTH_URL")
Insecure := os.Getenv("SWIFT_AUTH_INSECURE")
ConnectionChannelTimeout := os.Getenv("SWIFT_CONNECTION_CHANNEL_TIMEOUT")
DataChannelTimeout := os.Getenv("SWIFT_DATA_CHANNEL_TIMEOUT")
if UserName == "" || ApiKey == "" || AuthUrl == "" {
if srv != nil {
srv.Close()
}
srv, err = swifttest.NewSwiftServer("localhost")
if err != nil {
return nil, err
}
UserName = "******"
ApiKey = "swifttest"
AuthUrl = srv.AuthURL
}
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
MaxIdleConnsPerHost: 2048,
}
if Insecure == "1" {
transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}
c := swift.Connection{
UserName: UserName,
ApiKey: ApiKey,
AuthUrl: AuthUrl,
Transport: transport,
ConnectTimeout: 60 * time.Second,
Timeout: 60 * time.Second,
}
var timeout int64
if ConnectionChannelTimeout != "" {
timeout, err = strconv.ParseInt(ConnectionChannelTimeout, 10, 32)
if err == nil {
c.ConnectTimeout = time.Duration(timeout) * time.Second
}
}
if DataChannelTimeout != "" {
timeout, err = strconv.ParseInt(DataChannelTimeout, 10, 32)
if err == nil {
c.Timeout = time.Duration(timeout) * time.Second
}
}
return &c, nil
}
// NewPublishedStorage creates new instance of PublishedStorage with specified Swift access
// keys, tenant and tenantId
func NewPublishedStorage(username string, password string, authURL string, tenant string, tenantID string, container string, prefix string) (*PublishedStorage, error) {
if username == "" {
if username = os.Getenv("OS_USERNAME"); username == "" {
username = os.Getenv("ST_USER")
}
}
if password == "" {
if password = os.Getenv("OS_PASSWORD"); password == "" {
password = os.Getenv("ST_KEY")
}
}
if authURL == "" {
if authURL = os.Getenv("OS_AUTH_URL"); authURL == "" {
authURL = os.Getenv("ST_AUTH")
}
}
if tenant == "" {
tenant = os.Getenv("OS_TENANT_NAME")
}
if tenantID == "" {
tenantID = os.Getenv("OS_TENANT_ID")
}
ct := swift.Connection{
UserName: username,
ApiKey: password,
AuthUrl: authURL,
UserAgent: "aptly/" + aptly.Version,
Tenant: tenant,
TenantId: tenantID,
ConnectTimeout: 60 * time.Second,
Timeout: 60 * time.Second,
}
err := ct.Authenticate()
if err != nil {
return nil, fmt.Errorf("swift authentication failed: %s", err)
}
var bulkDelete bool
resp, err := http.Get(filepath.Join(ct.StorageUrl, "..", "..") + "/info")
if err == nil {
defer resp.Body.Close()
decoder := json.NewDecoder(resp.Body)
var infos swiftInfo
if decoder.Decode(&infos) == nil {
_, bulkDelete = infos["bulk_delete"]
}
}
result := &PublishedStorage{
conn: ct,
container: container,
prefix: prefix,
supportBulkDelete: bulkDelete,
}
return result, nil
}
func get_objects(c swift.Connection, container string, prefix string) []string {
opts := new(swift.ObjectsOpts)
opts.Prefix = prefix
names, err := c.ObjectNames(container, opts)
if err != nil {
panic(err)
}
return names
}
func get_projects(c swift.Connection) []string {
opts := new(swift.ContainersOpts)
opts.Prefix = "jobs"
containers, err := c.ContainerNames(opts)
if err != nil {
panic(err)
}
return containers
}
func remove_objects(c swift.Connection, container string, names []string) bool {
removed_all := true
for _, val := range names {
fmt.Printf("Removing %s\n", val)
err := c.ObjectDelete(container, val)
if err != nil {
fmt.Printf("Couldn't delete '%s' because: %s\n", val, err.Error())
removed_all = false
}
}
return removed_all
}
func main() {
//lol if you don't already use swiftly
username := os.Getenv("SWIFTLY_AUTH_USER")
apikey := os.Getenv("SWIFTLY_AUTH_KEY")
authurl := os.Getenv("SWIFTLY_AUTH_URL")
region := os.Getenv("SWIFTLY_REGION")
snet := os.Getenv("SWIFTLY_SNET")
dockerized := os.Getenv("DOCKERIZED")
memcacheOverride := os.Getenv("MEMCACHEADDR")
if strings.ToLower(dockerized) == "true" {
memcacheOverride = fmt.Sprintf("%s:%s", os.Getenv("MEMCACHED_PORT_11211_TCP_ADDR"), os.Getenv("MEMCACHED_PORT_11211_TCP_PORT"))
}
memcacheAddr := "127.0.0.1:11211"
if memcacheOverride != "" {
memcacheAddr = memcacheOverride
}
internal := false
if strings.ToLower(snet) == "true" {
internal = true
}
//martini looks for a HOST and PORT env var to determine what to listen on
m := martini.Classic()
cf := swift.Connection{
UserName: username,
ApiKey: apikey,
AuthUrl: authurl,
Region: region,
Internal: internal,
}
// Authenticate
err := cf.Authenticate()
PanicIf(err)
m.Map(&cf)
log.Println(os.Environ())
log.Println(memcacheAddr)
mc := memcache.New(memcacheAddr)
m.Map(mc)
m.Use(render.Renderer())
m.Get("/", IndexPage)
m.Get("/history", GetHistory)
m.Get("/:pasteid", GetPaste)
m.Get("/:pasteid/:format", GetPaste)
m.Post("/paste", binding.Json(Paste{}), binding.ErrorHandler, SavePaste)
m.Put("/paste", binding.Json(Paste{}), binding.ErrorHandler, SavePaste)
m.Run()
}
func GetHistory(ren render.Render, r *http.Request, cf *swift.Connection) {
opts := swift.ObjectsOpts{}
opts.Prefix = "cfpaste-"
opts.Limit = 10
objects, err := cf.ObjectNames("go-cfpaste", &opts)
pastes := make([]string, 10)
for i := range objects {
object, headers, err := cf.Object("go-cfpaste", objects[i])
PanicIf(err)
log.Println(object.Name)
pastes = append(pastes, headers["X-Object-Meta-Pasteid"])
}
PanicIf(err)
ren.HTML(200, "history", pastes)
return
}
func Example() {
// Create a connection
c := swift.Connection{
UserName: "******",
ApiKey: "key",
AuthUrl: "auth_url",
}
// Authenticate
err := c.Authenticate()
if err != nil {
panic(err)
}
// List all the containers
containers, err := c.ContainerNames(nil)
fmt.Println(containers)
// etc...
}
// New constructs a new Driver with the given Openstack Swift credentials and container name
func New(params DriverParameters) (*Driver, error) {
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
MaxIdleConnsPerHost: 2048,
TLSClientConfig: &tls.Config{InsecureSkipVerify: params.InsecureSkipVerify},
}
ct := swift.Connection{
UserName: params.Username,
ApiKey: params.Password,
AuthUrl: params.AuthURL,
Region: params.Region,
UserAgent: "distribution",
Tenant: params.Tenant,
TenantId: params.TenantID,
Domain: params.Domain,
DomainId: params.DomainID,
Transport: transport,
ConnectTimeout: 60 * time.Second,
Timeout: 15 * 60 * time.Second,
}
err := ct.Authenticate()
if err != nil {
return nil, fmt.Errorf("Swift authentication failed: %s", err)
}
if err := ct.ContainerCreate(params.Container, nil); err != nil {
return nil, fmt.Errorf("Failed to create container %s (%s)", params.Container, err)
}
d := &driver{
Conn: ct,
Container: params.Container,
Prefix: params.Prefix,
BulkDeleteSupport: detectBulkDelete(params.AuthURL),
ChunkSize: params.ChunkSize,
}
return &Driver{
baseEmbed: baseEmbed{
Base: base.Base{
StorageDriver: d,
},
},
}, nil
}
// Test a connection can be serialized and unserialized with XML
func TestSerializeConnectionXml(t *testing.T) {
serializedConnection, err := xml.Marshal(c)
if err != nil {
t.Fatalf("Failed to serialize connection: %v", err)
}
c2 := new(swift.Connection)
err = xml.Unmarshal(serializedConnection, &c2)
if err != nil {
t.Fatalf("Failed to unserialize connection: %v", err)
}
if !c2.Authenticated() {
t.Fatal("Should be authenticated")
}
_, _, err = c2.Account()
if err != nil {
t.Fatalf("Failed to use unserialized connection: %v", err)
}
}
func ExampleConnection() {
// Create a v1 auth connection
c := swift.Connection{
// This should be your username
UserName: "******",
// This should be your api key
ApiKey: "key",
// This should be a v1 auth url, eg
// Rackspace US https://auth.api.rackspacecloud.com/v1.0
// Rackspace UK https://lon.auth.api.rackspacecloud.com/v1.0
// Memset Memstore UK https://auth.storage.memset.com/v1.0
AuthUrl: "auth_url",
}
// Authenticate
err := c.Authenticate()
if err != nil {
panic(err)
}
// List all the containers
containers, err := c.ContainerNames(nil)
fmt.Println(containers)
// etc...
// ------ or alternatively create a v2 connection ------
// Create a v2 auth connection
c = swift.Connection{
// This is the sub user for the storage - eg "admin"
UserName: "******",
// This should be your api key
ApiKey: "key",
// This should be a version2 auth url, eg
// Rackspace v2 https://identity.api.rackspacecloud.com/v2.0
// Memset Memstore v2 https://auth.storage.memset.com/v2.0
AuthUrl: "v2_auth_url",
// Region to use - default is use first region if unset
Region: "LON",
// Name of the tenant - this is likely your username
Tenant: "jim",
}
// as above...
}
func main() {
//lol if you don't already use swiftly
username := os.Getenv("SWIFTLY_AUTH_USER")
apikey := os.Getenv("SWIFTLY_AUTH_KEY")
authurl := os.Getenv("SWIFTLY_AUTH_URL")
region := os.Getenv("SWIFTLY_REGION")
snet := os.Getenv("SWIFTLY_SNET")
internal := false
if strings.ToLower(snet) == "true" {
internal = true
}
//martini looks for a HOST and PORT env var to determine what to listen on
m := martini.Classic()
cf := swift.Connection{
UserName: username,
ApiKey: apikey,
AuthUrl: authurl,
Region: region,
Internal: internal,
}
// Authenticate
err := cf.Authenticate()
PanicIf(err)
m.Map(&cf)
mc := memcache.New("127.0.0.1:11211")
m.Map(mc)
m.Use(render.Renderer())
m.Get("/", IndexPage)
m.Get("/history", GetHistory)
m.Get("/:pasteid", GetPaste)
m.Get("/:pasteid/:format", GetPaste)
m.Post("/paste", binding.Json(Paste{}), binding.ErrorHandler, SavePaste)
m.Put("/paste", binding.Json(Paste{}), binding.ErrorHandler, SavePaste)
m.Run()
}
func Main() {
headers["X-Log-Retention"] = "true"
headers["X-Container-Meta-fubar"] = "false"
config := getConfig()
err := config.Parse(os.Args[1:])
if err != nil {
fmt.Printf("error verifying flags, %v. See 'osync --help'.", err)
os.Exit(1)
}
prefixlen = len(config.DataDir)
if err = config.InitializeOSAuth(); nil != err {
panic("Initial Openstack Authtication Code Failed With " + err.Error())
}
headers[OS_TEMP_URL_KEY] = config.OSAuth.TempURLKey
c := swift.Connection{
UserName: config.OSAuth.Username,
ApiKey: config.OSAuth.Password,
AuthUrl: config.OSAuth.Host,
Tenant: config.OSAuth.ProjectName,
}
// Authenticate
err = c.Authenticate()
if err != nil {
panic(err)
}
// List all the containers
containers, err := c.ContainerNames(nil)
fmt.Println(containers)
worker(c, config.DataDir, 0)
}
func TestTransport(t *testing.T) {
var err error
UserName := os.Getenv("SWIFT_API_USER")
ApiKey := os.Getenv("SWIFT_API_KEY")
AuthUrl := os.Getenv("SWIFT_AUTH_URL")
if UserName == "" || ApiKey == "" || AuthUrl == "" {
srv, err = swifttest.NewSwiftServer("localhost")
if err != nil {
t.Fatal("Failed to create server", err)
}
UserName = "******"
ApiKey = "swifttest"
AuthUrl = srv.AuthURL
}
tr := &someTransport{Transport: http.Transport{MaxIdleConnsPerHost: 2048}}
ct := swift.Connection{
UserName: UserName,
ApiKey: ApiKey,
AuthUrl: AuthUrl,
Tenant: os.Getenv("SWIFT_TENANT"),
TenantId: os.Getenv("SWIFT_TENANT_ID"),
Transport: tr,
ConnectTimeout: 60 * time.Second,
Timeout: 60 * time.Second,
}
err = ct.Authenticate()
if err != nil {
t.Fatal("Auth failed", err)
}
if !ct.Authenticated() {
t.Fatal("Not authenticated")
}
if srv != nil {
srv.Close()
}
}
func SavePaste(paste Paste, ren render.Render, r *http.Request, cf *swift.Connection, mc *memcache.Client) {
paste.PasteId = genPasteId()
payload, _ := json.Marshal(paste)
seconds, err := getTTL(paste.PasteTtl)
PanicIf(err)
headers := swift.Headers{}
now := time.Now()
pasteIndex := 9999999999 - now.Unix()
indexKey := fmt.Sprintf("cfpaste-%d", pasteIndex)
headers["x-object-meta-pastetype"] = paste.PasteType
headers["x-object-meta-pasteid"] = paste.PasteId
headers["x-object-meta-pasteindex"] = fmt.Sprintf("%d", pasteIndex)
if seconds != 0 {
headers["x-delete-after"] = fmt.Sprintf("%d", seconds)
}
buf := bytes.NewBuffer(payload)
_, err = cf.ObjectPut("go-cfpaste", paste.PasteId, buf, true, "", "application/json; charset=utf-8", headers)
PanicIf(err)
// gholt's listing index hack so that he can spy on pastes
_, err = cf.ObjectPut("go-cfpaste", indexKey, bytes.NewBuffer([]byte("")), true, "", "application/json; charset=utf-8", headers)
PanicIf(err)
mc.Set(&memcache.Item{Key: paste.PasteId, Value: payload})
ren.JSON(200, map[string]interface{}{"pasteid": paste.PasteId})
}
func main() {
// Create a connection
c := swift.Connection{
UserName: os.Getenv("SWIFT_API_USER"),
ApiKey: os.Getenv("SWIFT_API_KEY"),
AuthUrl: os.Getenv("SWIFT_AUTH_URL"),
Domain: "", // Name of the domain (v3 auth only)
Tenant: "", // Name of the tenant (v2 auth only)
}
if len(os.Args) < 5 {
fmt.Printf("Usage: %s customer jobId status path\n", os.Args[0])
os.Exit(1)
}
println("Starting objekt storage uploader")
var (
customer = os.Args[1]
jobId = os.Args[2]
status = os.Args[3]
path = os.Args[4]
)
container_name := "jobs-" + customer
file_prefix := jobId
// Authenticate
err := c.Authenticate()
if err != nil {
panic(err)
}
uploadContentsInFolder(path, file_prefix, container_name, c)
wpaumetadata.Add(c, container_name, jobId, status)
}
func UploadFile(container string, prefix string, path string, c swift.Connection) {
dat, err := ioutil.ReadFile(path)
if err != nil {
println(err.Error())
} else {
name := filepath.Base(path)
if len(prefix) > 0 {
name = prefix + "-" + name
}
ext := filepath.Ext(path)
hasher := md5.New()
hasher.Write(dat)
md5hash := hex.EncodeToString(hasher.Sum(nil))
fmt.Printf("Uploading %s to container %s\n", name, container)
file, err := c.ObjectCreate(container, name, false, md5hash, ext, nil)
if err != nil {
println(err.Error())
} else {
file.Write(dat)
}
file.Close()
}
}
func TestSwiftBackend(t *testing.T) {
if os.Getenv("OS_USERNAME") == "" || os.Getenv("OS_PASSWORD") == "" ||
os.Getenv("OS_AUTH_URL") == "" {
t.SkipNow()
}
username := os.Getenv("OS_USERNAME")
password := os.Getenv("OS_PASSWORD")
authUrl := os.Getenv("OS_AUTH_URL")
tenant := os.Getenv("OS_TENANT_NAME")
ts := time.Now().UnixNano()
container := fmt.Sprintf("vault-test-%d", ts)
cleaner := swift.Connection{
UserName: username,
ApiKey: password,
AuthUrl: authUrl,
Tenant: tenant,
Transport: cleanhttp.DefaultPooledTransport(),
}
err := cleaner.Authenticate()
if err != nil {
t.Fatalf("err: %s", err)
}
err = cleaner.ContainerCreate(container, nil)
if nil != err {
t.Fatalf("Unable to create test container '%s': %v", container, err)
}
defer func() {
newObjects, err := cleaner.ObjectNamesAll(container, nil)
if err != nil {
t.Fatalf("err: %s", err)
}
for _, o := range newObjects {
err := cleaner.ObjectDelete(container, o)
if err != nil {
t.Fatalf("err: %s", err)
}
}
err = cleaner.ContainerDelete(container)
if err != nil {
t.Fatalf("err: %s", err)
}
}()
logger := logformat.NewVaultLogger(log.LevelTrace)
b, err := NewBackend("swift", logger, map[string]string{
"username": username,
"password": password,
"container": container,
"auth_url": authUrl,
"tenant": tenant,
})
if err != nil {
t.Fatalf("err: %s", err)
}
testBackend(t, b)
testBackend_ListPrefix(t, b)
}
// New constructs a new Driver with the given Openstack Swift credentials and container name
func New(params Parameters) (*Driver, error) {
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
MaxIdleConnsPerHost: 2048,
TLSClientConfig: &tls.Config{InsecureSkipVerify: params.InsecureSkipVerify},
}
ct := swift.Connection{
UserName: params.Username,
ApiKey: params.Password,
AuthUrl: params.AuthURL,
Region: params.Region,
UserAgent: "distribution/" + version.Version,
Tenant: params.Tenant,
TenantId: params.TenantID,
Domain: params.Domain,
DomainId: params.DomainID,
TrustId: params.TrustID,
Transport: transport,
ConnectTimeout: 60 * time.Second,
Timeout: 15 * 60 * time.Second,
}
err := ct.Authenticate()
if err != nil {
return nil, fmt.Errorf("Swift authentication failed: %s", err)
}
if _, _, err := ct.Container(params.Container); err == swift.ContainerNotFound {
if err := ct.ContainerCreate(params.Container, nil); err != nil {
return nil, fmt.Errorf("Failed to create container %s (%s)", params.Container, err)
}
} else if err != nil {
return nil, fmt.Errorf("Failed to retrieve info about container %s (%s)", params.Container, err)
}
d := &driver{
Conn: ct,
Container: params.Container,
Prefix: params.Prefix,
ChunkSize: params.ChunkSize,
TempURLMethods: make([]string, 0),
AccessKey: params.AccessKey,
}
info := swiftInfo{}
if config, err := d.Conn.QueryInfo(); err == nil {
_, d.BulkDeleteSupport = config["bulk_delete"]
if err := mapstructure.Decode(config, &info); err == nil {
d.TempURLContainerKey = info.Swift.Version >= "2.3.0"
d.TempURLMethods = info.Tempurl.Methods
}
} else {
d.TempURLContainerKey = params.TempURLContainerKey
d.TempURLMethods = params.TempURLMethods
}
if len(d.TempURLMethods) > 0 {
secretKey := params.SecretKey
if secretKey == "" {
secretKey, _ = generateSecret()
}
// Since Swift 2.2.2, we can now set secret keys on containers
// in addition to the account secret keys. Use them in preference.
if d.TempURLContainerKey {
_, containerHeaders, err := d.Conn.Container(d.Container)
if err != nil {
return nil, fmt.Errorf("Failed to fetch container info %s (%s)", d.Container, err)
}
d.SecretKey = containerHeaders["X-Container-Meta-Temp-Url-Key"]
if d.SecretKey == "" || (params.SecretKey != "" && d.SecretKey != params.SecretKey) {
m := swift.Metadata{}
m["temp-url-key"] = secretKey
if d.Conn.ContainerUpdate(d.Container, m.ContainerHeaders()); err == nil {
d.SecretKey = secretKey
}
}
} else {
// Use the account secret key
_, accountHeaders, err := d.Conn.Account()
if err != nil {
return nil, fmt.Errorf("Failed to fetch account info (%s)", err)
}
d.SecretKey = accountHeaders["X-Account-Meta-Temp-Url-Key"]
if d.SecretKey == "" || (params.SecretKey != "" && d.SecretKey != params.SecretKey) {
m := swift.Metadata{}
m["temp-url-key"] = secretKey
if err := d.Conn.AccountUpdate(m.AccountHeaders()); err == nil {
d.SecretKey = secretKey
}
}
}
}
return &Driver{
baseEmbed: baseEmbed{
Base: base.Base{
StorageDriver: d,
},
},
}, nil
}
func CreatePublicContainer(name string, c swift.Connection) {
headers := map[string]string{
"X-Container-Read": ".r:*",
}
c.ContainerCreate(name, headers)
}
// newSwiftBackend constructs a Swift backend using a pre-existing
// container. Credentials can be provided to the backend, sourced
// from the environment.
func newSwiftBackend(conf map[string]string, logger log.Logger) (Backend, error) {
username := os.Getenv("OS_USERNAME")
if username == "" {
username = conf["username"]
if username == "" {
return nil, fmt.Errorf("missing username")
}
}
password := os.Getenv("OS_PASSWORD")
if password == "" {
password = conf["password"]
if password == "" {
return nil, fmt.Errorf("missing password")
}
}
authUrl := os.Getenv("OS_AUTH_URL")
if authUrl == "" {
authUrl = conf["auth_url"]
if authUrl == "" {
return nil, fmt.Errorf("missing auth_url")
}
}
container := os.Getenv("OS_CONTAINER")
if container == "" {
container = conf["container"]
if container == "" {
return nil, fmt.Errorf("missing container")
}
}
tenant := os.Getenv("OS_TENANT_NAME")
if tenant == "" {
tenant = conf["tenant"]
}
c := swift.Connection{
UserName: username,
ApiKey: password,
AuthUrl: authUrl,
Tenant: tenant,
Transport: cleanhttp.DefaultPooledTransport(),
}
err := c.Authenticate()
if err != nil {
return nil, err
}
_, _, err = c.Container(container)
if err != nil {
return nil, fmt.Errorf("Unable to access container '%s': %v", container, err)
}
maxParStr, ok := conf["max_parallel"]
var maxParInt int
if ok {
maxParInt, err = strconv.Atoi(maxParStr)
if err != nil {
return nil, errwrap.Wrapf("failed parsing max_parallel parameter: {{err}}", err)
}
if logger.IsDebug() {
logger.Debug("swift: max_parallel set", "max_parallel", maxParInt)
}
}
s := &SwiftBackend{
client: &c,
container: container,
logger: logger,
permitPool: NewPermitPool(maxParInt),
}
return s, nil
}
func worker(c swift.Connection, parent string, level int) string {
dirs, err := ioutil.ReadDir(parent)
if nil != err {
fmt.Println(err.Error())
}
var logstr string
for _, f0 := range dirs {
var curdir string
if prefixlen == len(parent) {
curdir = f0.Name()
} else {
curdir = parent[prefixlen:len(parent)]
curdir = strings.TrimLeft(curdir, "/")
}
// fmt.Println(parent, level, curdir)
switch {
case f0.IsDir():
if 0 == level {
logstr = fmt.Sprint(strings.Repeat("\t", level)+"Found Container Creating...", curdir)
fmt.Println(logstr)
if err = c.ContainerCreate(curdir, headers); err != nil {
panicString := fmt.Sprint("PutContainer Error:", err)
panic(panicString)
}
}
worker(c, parent+"/"+f0.Name(), level+1)
case !f0.IsDir():
if strings.HasPrefix(f0.Name(), ".") {
logstr = fmt.Sprint("Found Hidden Object. Ommited ", f0.Name(), " to ", curdir)
fmt.Println(logstr)
continue
}
contents, _err := ioutil.ReadFile(parent + "/" + f0.Name())
if nil != _err {
panicString := fmt.Sprint("PutObject Error:", _err)
panic(panicString)
}
h := sha1.New()
h.Write(contents)
obj_sha1hash := hex.EncodeToString(h.Sum(nil))
logstr = fmt.Sprint(strings.Repeat("\t", level)+"Found Object. Uploading ", f0.Name(), " [ ", obj_sha1hash, " ] to ", curdir)
fmt.Println(logstr)
fstream, err := c.ObjectCreate(curdir, f0.Name(), false, "", "application/zip", headers)
if nil != err {
panicString := fmt.Sprint("PutObject Error:", err)
panic(panicString)
}
if counts, err := fstream.Write(contents); nil != err {
panicString := fmt.Sprint("PutObject Error:", err)
panic(panicString)
} else {
fmt.Printf("%vBytes Total: %v, Written: %v\n", strings.Repeat("\t", level), len(contents), counts)
}
fstream.Close()
return ""
}
}
return ""
}