Example #1
0
func downloadChunk(n lib.Node, opts lib.Opts, filename string, offset int64, c chan int) {
	oh, err := os.OpenFile(filename, os.O_RDWR, 0644)
	if err != nil {
		fmt.Printf("Error open output file %s: %s\n", filename, err.Error())
	}
	defer oh.Close()

	_, err = oh.Seek(offset, os.SEEK_SET)
	if err != nil {
		fmt.Printf("Error seek output file %s: %s\n", filename, err.Error())
	}

	if ih, err := n.Download(opts); err != nil {
		fmt.Printf("Error downloading %s: %s\n", n.Id, err.Error())
	} else {
		if s, err := io.Copy(oh, ih); err != nil {
			fmt.Printf("Error writing output: %s\n", err.Error())
		} else {
			fmt.Printf("Success. Wrote %d bytes\n", s)
		}
	}
	c <- 1
}
Example #2
0
func acl(action, perm, users, id string) (err error) {
	n := lib.Node{Id: id}
	switch action {
	case "add":
		return n.AclAdd(perm, users)
	case "rm":
		return n.AclRemove(perm, users)
	case "chown":
		return n.AclChown(users)
	default:
		helpf("")
	}
	return
}
Example #3
0
func main() {
	if len(os.Args) == 1 || os.Args[1] == "help" {
		helpf("")
	}

	cmd := os.Args[1]
	args := conf.Initialize(os.Args[2:])

	setToken(false)
	switch cmd {
	case "create", "update":
		n := lib.Node{}
		if cmd == "update" {
			if len(args) != 1 {
				helpf("update requires <id>")
			} else {
				n.Id = args[0]
			}
		}
		opts := lib.Opts{}
		if ne(conf.Flags["attributes"]) {
			opts["attributes"] = (*conf.Flags["attributes"])
		}
		if t, err := fileOptions(conf.Flags); err != nil {
			helpf(err.Error())
		} else {
			if t == "part" {
				if cmd == "create" {
					helpf("part option only usable with update")
				}
				if !ne(conf.Flags["file"]) {
					helpf("part option requires file")
				}
				opts["upload_type"] = t
				opts["part"] = (*conf.Flags["part"])
				opts["file"] = (*conf.Flags["file"])
				if err := n.Update(opts); err != nil {
					handleString(fmt.Sprintf("Error updating %s: %s\n", n.Id, err.Error()))
				} else {
					n.PP()
				}
			} else {
				if t != "" {
					opts["upload_type"] = t
					opts[t] = (*conf.Flags[t])
					if cmd == "create" {
						if err := n.Create(opts); err != nil {
							handleString(fmt.Sprintf("Error creating node: %s\n", err.Error()))
						} else {
							n.PP()
						}
					} else {
						if err := n.Update(opts); err != nil {
							handleString(fmt.Sprintf("Error updating %s: %s\n", n.Id, err.Error()))
						} else {
							n.PP()
						}
					}
				} else {
					if err := n.Create(opts); err != nil {
						handleString(fmt.Sprintf("Error creating node: %s\n", err.Error()))
					} else {
						n.PP()
					}
				}
			}
		}
	case "pcreate":
		pcreate(args)

	case "get":
		if len(args) != 1 {
			helpf("get requires <id>")
		}
		n := lib.Node{Id: args[0]}
		if err := n.Get(); err != nil {
			fmt.Printf("Error retrieving %s: %s\n", n.Id, err.Error())
		} else {
			n.PP()
		}
	case "download":
		if len(args) < 1 {
			helpf("download requires <id>")
		}
		index := conf.Flags["index"]
		parts := conf.Flags["parts"]
		indexOptions := conf.Flags["index_options"]
		opts := lib.Opts{}
		if ne(index) || ne(parts) || ne(indexOptions) {
			if ne(index) && ne(parts) {
				opts["index"] = (*index)
				opts["parts"] = (*parts)
				if ne(indexOptions) {
					opts["index_options"] = (*indexOptions)
				}
			} else {
				helpf("index and parts options must be used together")
			}
		}
		n := lib.Node{Id: args[0]}
		if ih, err := n.Download(opts); err != nil {
			fmt.Printf("Error downloading %s: %s\n", n.Id, err.Error())
		} else {
			if len(args) == 3 {
				if oh, err := os.Create(args[1]); err == nil {
					if s, err := io.Copy(oh, ih); err != nil {
						handleString(fmt.Sprintf("Error writing output: %s\n", err.Error()))
					} else {
						fmt.Printf("Success. Wrote %d bytes\n", s)
					}
				} else {
					handleString(fmt.Sprintf("Error writing output: %s\n", err.Error()))
				}
			} else {
				io.Copy(os.Stdout, ih)
			}
		}
	case "pdownload":
		if len(args) < 1 {
			helpf("pdownload requires <id>")
		}
		n := lib.Node{Id: args[0]}
		if err := n.Get(); err != nil {
			handleString(fmt.Sprintf("Error retrieving %s: %s\n", n.Id, err.Error()))
		}

		totalChunk := int(n.File.Size / conf.CHUNK_SIZE)
		m := n.File.Size % conf.CHUNK_SIZE
		if m != 0 {
			totalChunk += 1
		}
		if totalChunk < 1 {
			totalChunk = 1
		}
		splits := conf.DOWNLOAD_THREADS
		if ne(conf.Flags["threads"]) {
			if th, err := strconv.Atoi(*conf.Flags["threads"]); err == nil {
				splits = th
			}
		}
		if totalChunk < splits {
			splits = totalChunk
		}

		fmt.Printf("downloading using %d threads\n", splits)

		var filename string
		if len(args) == 2 {
			filename = args[1]
		} else {
			filename = n.Id
		}

		oh, err := os.Create(filename)
		if err != nil {
			handleString(fmt.Sprintf("Error creating output file %s: %s\n", filename, err.Error()))
			return
		}
		oh.Close()

		ch := make(chan int, 1)
		split_size := totalChunk / splits
		remainder := totalChunk % splits
		//splitting, if total chunk is 10, each split will have 3,3,2,2 chunks respectively
		for i := 0; i < splits; i++ {
			var start, end int
			if i < remainder {
				start = (split_size+1)*i + 1
				end = start + split_size
			} else {
				start = (split_size+1)*remainder + split_size*(i-remainder) + 1
				end = start + split_size - 1
			}
			part_string := fmt.Sprintf("%d-%d", start, end)
			opts := lib.Opts{}
			opts["index"] = "size"
			opts["parts"] = part_string
			opts["index_options"] = fmt.Sprintf("chunk_size=%d", conf.CHUNK_SIZE)

			start_offset := (int64(start) - 1) * conf.CHUNK_SIZE
			go downloadChunk(n, opts, filename, start_offset, ch)
		}
		for i := 0; i < splits; i++ {
			<-ch
		}
	case "auth":
		if len(args) < 1 {
			helpf("auth requires show/set/set-token/unset")
		}
		switch args[0] {
		case "set":
			var username, password string
			fmt.Printf("Please authenticate to store your credentials.\nusername: ")
			fmt.Scan(&username)
			password, _ = gopass.GetPass("password: ")
			if t, err := lib.OAuthToken(username, password); err == nil {
				if err := t.Store(); err != nil {
					handleString(fmt.Sprintf("Authenticated but failed to store token: %s\n", err.Error()))
				}
				fmt.Printf("Authenticated credentials stored for user %s. Expires in %d days.\n", t.UserName, t.ExpiresInDays())
			} else {
				fmt.Printf("%s\n", err.Error())
			}
		case "set-token":
			if len(args) != 2 {
				helpf("auth set-token requires token.")
			}
			t := &lib.Token{}
			if err := json.Unmarshal([]byte(args[1]), &t); err != nil {
				handleString("Invalid auth token.\n")
			}
			if err := t.Store(); err != nil {
				handleString(fmt.Sprintf("Failed to store token: %s\n", err.Error()))
			}
			fmt.Printf("Authenticated credentials stored for user %s. Expires in %d days.\n", t.UserName, t.ExpiresInDays())
		case "unset":
			t := &lib.Token{}
			if err := t.Delete(); err != nil {
				fmt.Printf("%s\n", err.Error())
			} else {
				fmt.Printf("Stored authentication credentials have been deleted.\n")
			}
		case "show":
			t := &lib.Token{}
			if err := t.Load(); err != nil {
				handleToken(err)
			} else {
				fmt.Printf("Authenticated credentials stored for user %s. Expires in %d days.\n", t.UserName, t.ExpiresInDays())
			}
		}
	case "acl":
		if len(args) != 4 {
			helpf("acl requires 4 arguments")
		}
		if err := acl(args[0], args[1], args[2], args[3]); err != nil {
			handle(err)
		}
	case "chown":
		if len(args) != 2 {
			helpf("chown requires <user> and <id>")
		}
		if err := acl("chown", "", args[0], args[1]); err != nil {
			handle(err)
		}
	default:
		helpf("invalid command")
	}
}
Example #4
0
func pcreate(args []string) (err error) {
	n := lib.Node{}
	var filename string
	if ne(conf.Flags["full"]) {
		filename = (*conf.Flags["full"])
	} else {
		helpf("pcreate requires file path: -full=<u>")
	}

	var filesize int64
	fh, err := os.Open(filename)
	if err != nil {
		handleString(fmt.Sprintf("Error open file: %s\n", err.Error()))
	}
	if fi, _ := fh.Stat(); err == nil {
		filesize = fi.Size()
	}

	chunks := int(filesize / (conf.CHUNK_SIZE))
	if filesize%conf.CHUNK_SIZE != 0 {
		chunks += 1
	}

	if chunks == 1 {
		opts := lib.Opts{}
		opts["upload_type"] = "full"
		opts["full"] = filename
		if err := n.Create(opts); err != nil {
			handleString(fmt.Sprintf("Error creating node: %s\n", err.Error()))
		} else {
			n.PP()
		}
	} else {
		threads, _ := strconv.Atoi(*conf.Flags["threads"])
		if threads == 0 {
			threads = 1
		}

		//create node
		opts := lib.Opts{}
		opts["upload_type"] = "parts"
		opts["parts"] = strconv.Itoa(chunks)
		if err := n.Create(opts); err != nil {
			handleString(fmt.Sprintf("Error creating node: %s\n", err.Error()))
		}

		workers := pool.New(threads)
		workers.Run()
		for i := 0; i < chunks; i++ {
			size := int64(conf.CHUNK_SIZE)
			if size*(int64(i)+1) > filesize {
				size = filesize - size*(int64(i))
			}
			workers.Add(uploader, n, (i + 1), fh, size)
		}
		workers.Wait()
		maxRetries := 10
		for i := 1; i <= maxRetries; i++ {
			errCount := 0
			completed_jobs := workers.Results()
			for _, job := range completed_jobs {
				if job.Result != nil {
					err := job.Result.(error)
					println("Chunk", job.Args[1].(int), "error:", err.Error())
					workers.Add(job.F, job.Args...)
					errCount++
				}
			}
			if errCount == 0 {
				println("All chunks successfully upload.")
				break
			} else {
				println("Retry", i, "of", maxRetries)
				workers.Wait()
			}
		}
		workers.Stop()

		n.Get()
		n.PP()
	}
	return
}
Example #5
0
func main() {
	args := flag.Args()
	if *conf.Examples {
		helpf("Coming soon.")
	}
	if len(args) == 0 {
		helpf("")
	}

	switch args[0] {
	case "create", "update":
		n := lib.Node{}
		if args[0] == "update" {
			if len(args) != 2 {
				helpf("update requires <id>")
			} else {
				n.Id = args[1]
			}
		}
		opts := lib.Opts{}
		if ne(conf.Flags["attributes"]) {
			opts["attributes"] = (*conf.Flags["attributes"])
		}
		if t, err := fileOptions(conf.Flags); err != nil {
			helpf(err.Error())
		} else {
			if t == "part" {
				if args[0] == "create" {
					helpf("part option only usable with update")
				}
				if !ne(conf.Flags["file"]) {
					helpf("part option requires file")
				}
				opts["upload_type"] = t
				opts["part"] = (*conf.Flags["part"])
				opts["file"] = (*conf.Flags["file"])
				if err := n.Update(opts); err != nil {
					fmt.Printf("Error updating %s: %s\n", n.Id, err.Error())
				} else {
					n.PP()
				}
			} else {
				if t != "" {
					opts["upload_type"] = t
					opts[t] = (*conf.Flags[t])
					if args[0] == "create" {
						if err := n.Create(opts); err != nil {
							fmt.Printf("Error creating node: %s\n", err.Error())
						} else {
							n.PP()
						}
					} else {
						if err := n.Update(opts); err != nil {
							fmt.Printf("Error updating %s: %s\n", n.Id, err.Error())
						} else {
							n.PP()
						}
					}
				} else {
					if err := n.Create(opts); err != nil {
						fmt.Printf("Error creating node: %s\n", err.Error())
					} else {
						n.PP()
					}
				}
			}
		}
	case "get":
		if len(args) != 2 {
			helpf("get requires <id>")
		}
		n := lib.Node{Id: args[1]}
		if err := n.Get(); err != nil {
			fmt.Printf("Error retrieving %s: %s\n", n.Id, err.Error())
		} else {
			n.PP()
		}
	case "download":
		if len(args) < 2 {
			helpf("download requires <id>")
		}
		index := conf.Flags["index"]
		parts := conf.Flags["parts"]
		indexOptions := conf.Flags["index_options"]
		opts := lib.Opts{}
		if ne(index) || ne(parts) || ne(indexOptions) {
			if ne(index) && ne(parts) {
				opts["index"] = (*index)
				opts["parts"] = (*parts)
				if ne(indexOptions) {
					opts["index_options"] = (*indexOptions)
				}
			} else {
				helpf("index and parts options must be used together")
			}
		}
		n := lib.Node{Id: args[1]}
		if ih, err := n.Download(opts); err != nil {
			fmt.Printf("Error downloading %s: %s\n", n.Id, err.Error())
		} else {
			if len(args) == 3 {
				if oh, err := os.Create(args[2]); err == nil {
					if s, err := io.Copy(oh, ih); err != nil {
						fmt.Printf("Error writing output: %s\n", err.Error())
					} else {
						fmt.Printf("Success. Wrote %d bytes\n", s)
					}
				} else {
					fmt.Printf("Error writing output: %s\n", err.Error())
				}
			} else {
				io.Copy(os.Stdout, ih)
			}
		}

	case "auth":
		if len(args) != 2 {
			helpf("auth requires show/set/unset")
		}
		switch args[1] {
		case "set":
			var username, password string
			fmt.Printf("Please authenticate to store your credentials.\nusername: ")
			fmt.Scan(&username)
			fmt.Printf("password: ")
			fmt.Scan(&password)
			if t, err := lib.OAuthToken(username, password); err == nil {
				if err := t.Store(); err != nil {
					fmt.Printf("Authenticated but failed to store token: %s\n", err.Error())
				}
				fmt.Printf("Authenticated credentials stored for user %s. Expires in %d days.\n", t.UserName, t.ExpiresInDays())
			} else {
				fmt.Printf("%s\n", err.Error())
			}
		case "unset":
			t := &lib.Token{}
			if err := t.Delete(); err != nil {
				fmt.Printf("%s\n", err.Error())
			} else {
				fmt.Printf("Stored authentication credentials have been deleted.\n")
			}
		case "show":
			t := &lib.Token{}
			if err := t.Load(); err != nil {
				handleToken(err)
			} else {
				fmt.Printf("Authenticated credentials stored for user %s. Expires in %d days.\n", t.UserName, t.ExpiresInDays())
			}
		}
	case "acl":
		if len(args) != 5 {
			helpf("acl requires 4 arguments")
		}
		setToken(true)
		if err := acl(args[1], args[2], args[3], args[4]); err != nil {
			handle(err)
		}
	case "chown":
		if len(args) != 3 {
			helpf("chown requires <user> and <id>")
		}
		setToken(true)
		if err := acl("chown", "", args[1], args[2]); err != nil {
			handle(err)
		}
	default:
		helpf("invalid command")
	}
}