func (ts *Source) refreshToken() error {
log.Debugf("refreshing the token from %q", refreshURL)
data, err := json.Marshal(ts.token)
if err != nil {
log.Errorf("%s: %s", constants.ErrJSONEncoding, err)
return constants.ErrJSONEncoding
}
req, err := http.NewRequest("POST", refreshURL, bytes.NewBuffer(data))
if err != nil {
log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err)
return constants.ErrCreatingHTTPRequest
}
req.Header.Set("Content-Type", "application/json")
res, err := (&http.Client{}).Do(req)
if err != nil {
log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err)
return constants.ErrDoingHTTPRequest
}
defer res.Body.Close()
if err := json.NewDecoder(res.Body).Decode(ts.token); err != nil {
log.Errorf("%s: %s", constants.ErrJSONDecodingResponseBody, err)
return constants.ErrJSONDecodingResponseBody
}
log.Debug("token was refreshed successfully")
return nil
}
func loadConfig(configFile string) (*Config, error) {
// validate the config file
if err := validateFile(configFile, false); err != nil {
return nil, err
}
cf, err := os.Open(configFile)
if err != nil {
log.Errorf("%s: %s", constants.ErrOpenFile, err)
return nil, err
}
defer cf.Close()
var config Config
if err := json.NewDecoder(cf).Decode(&config); err != nil {
log.Errorf("%s: %s", constants.ErrJSONDecoding, err)
return nil, err
}
// validate the token file
if err := validateFile(config.TokenFile, true); err != nil {
return nil, err
}
return &config, nil
}
// MkdirAll creates a directory named path, along with any necessary parents,
// and returns the directory node and nil, or else returns an error. If path is
// already a directory, MkdirAll does nothing and returns the directory node
// and nil.
func (nt *Tree) MkdirAll(path string) (*Node, error) {
var (
err error
folderNode = nt.Node
logLevel = log.GetLevel()
nextNode *Node
node *Node
)
// Short-circuit if the node already exists!
{
log.SetLevel(log.DisableLogLevel)
node, err = nt.FindNode(path)
log.SetLevel(logLevel)
}
if err == nil {
if node.IsDir() {
return node, err
}
log.Errorf("%s: %s", constants.ErrFileExistsAndIsNotFolder, path)
return nil, constants.ErrFileExistsAndIsNotFolder
}
// chop off the first /.
if strings.HasPrefix(path, "/") {
path = path[1:]
}
parts := strings.Split(path, "/")
if len(parts) == 0 {
log.Errorf("%s: %s", constants.ErrCannotCreateRootNode, path)
return nil, constants.ErrCannotCreateRootNode
}
for i, part := range parts {
{
log.SetLevel(log.DisableLogLevel)
nextNode, err = nt.FindNode(strings.Join(parts[:i+1], "/"))
log.SetLevel(logLevel)
}
if err != nil && err != constants.ErrNodeNotFound {
return nil, err
}
if err == constants.ErrNodeNotFound {
nextNode, err = folderNode.CreateFolder(part)
if err != nil {
return nil, err
}
}
if !nextNode.IsDir() {
log.Errorf("%s: %s", constants.ErrCannotCreateANodeUnderAFile, strings.Join(parts[:i+1], "/"))
return nil, constants.ErrCannotCreateANodeUnderAFile
}
folderNode = nextNode
}
return folderNode, nil
}
// Sync syncs the tree with the server.
func (nt *Tree) Sync() error {
postURL := nt.client.GetMetadataURL("changes")
c := &changes{
Checkpoint: nt.Checkpoint,
}
jsonBytes, err := json.Marshal(c)
if err != nil {
log.Errorf("%s: %s", constants.ErrJSONEncoding, err)
return constants.ErrJSONEncoding
}
req, err := http.NewRequest("POST", postURL, bytes.NewBuffer(jsonBytes))
if err != nil {
log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err)
return constants.ErrCreatingHTTPRequest
}
req.Header.Set("Content-Type", "application/json")
res, err := nt.client.Do(req)
if err != nil {
log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err)
return constants.ErrDoingHTTPRequest
}
if err := nt.client.CheckResponse(res); err != nil {
return err
}
// return format should be:
// {"checkpoint": str, "reset": bool, "nodes": []}
// {"checkpoint": str, "reset": false, "nodes": []}
// {"end": true}
defer res.Body.Close()
bodyBytes, err := ioutil.ReadAll(res.Body)
if err != nil {
log.Errorf("%s: %s", constants.ErrReadingResponseBody, err)
return constants.ErrReadingResponseBody
}
for _, lineBytes := range bytes.Split(bodyBytes, []byte("\n")) {
var cr changesResponse
if err := json.Unmarshal(lineBytes, &cr); err != nil {
log.Errorf("%s: %s", constants.ErrJSONDecodingResponseBody, err)
return constants.ErrJSONDecodingResponseBody
}
if cr.Checkpoint != "" {
log.Debugf("changes returned Checkpoint: %s", cr.Checkpoint)
nt.Checkpoint = cr.Checkpoint
}
if cr.Reset {
log.Debug("reset is required")
return constants.ErrMustFetchFresh
}
if cr.End {
break
}
if err := nt.updateNodes(cr.Nodes); err != nil {
return err
}
}
return nil
}
func (n *Node) upload(url, method, metadataJSON, name string, r io.Reader) (*Node, error) {
bodyReader, bodyWriter := io.Pipe()
errChan := make(chan error)
bodyChan := make(chan io.ReadCloser)
contentTypeChan := make(chan string)
go n.bodyWriter(metadataJSON, name, r, bodyWriter, errChan, contentTypeChan)
go func() {
req, err := http.NewRequest(method, url, bodyReader)
if err != nil {
log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err)
select {
case errChan <- constants.ErrCreatingHTTPRequest:
default:
}
return
}
req.Header.Add("Content-Type", <-contentTypeChan)
res, err := n.client.Do(req) // this should block until the upload is finished.
if err != nil {
log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err)
select {
case errChan <- constants.ErrDoingHTTPRequest:
default:
}
return
}
if err := n.client.CheckResponse(res); err != nil {
select {
case errChan <- err:
default:
}
return
}
select {
case bodyChan <- res.Body:
default:
}
}()
for {
select {
case err := <-errChan:
if err != nil {
return nil, err
}
case body := <-bodyChan:
defer body.Close()
var node Node
if err := json.NewDecoder(body).Decode(&node); err != nil {
log.Errorf("%s: %s", constants.ErrJSONDecodingResponseBody, err)
return nil, constants.ErrJSONDecodingResponseBody
}
return &node, nil
}
}
}
func (nt *Tree) saveCache() error {
f, err := os.Create(nt.cacheFile)
if err != nil {
log.Errorf("%s: %s", constants.ErrCreateFile, nt.cacheFile)
return constants.ErrCreateFile
}
if err := gob.NewEncoder(f).Encode(nt); err != nil {
log.Errorf("%s: %s", constants.ErrGOBEncoding, err)
return constants.ErrGOBEncoding
}
log.Debugf("saved NodeTree to cache file %q.", nt.cacheFile)
return nil
}
func (n *Node) bodyWriter(metadataJSON, name string, r io.Reader, bodyWriter io.WriteCloser, errChan chan error, contentTypeChan chan string) {
writer := multipart.NewWriter(bodyWriter)
contentTypeChan <- writer.FormDataContentType()
if metadataJSON != "" {
if err := writer.WriteField("metadata", metadataJSON); err != nil {
log.Errorf("%s: %s", constants.ErrWritingMetadata, err)
select {
case errChan <- constants.ErrWritingMetadata:
default:
}
return
}
}
part, err := writer.CreateFormFile("content", name)
if err != nil {
log.Errorf("%s: %s", constants.ErrCreatingWriterFromFile, err)
select {
case errChan <- err:
default:
}
return
}
count, err := io.Copy(part, r)
if err != nil {
log.Errorf("%s: %s", constants.ErrWritingFileContents, err)
select {
case errChan <- constants.ErrWritingFileContents:
default:
}
return
}
if count == 0 {
select {
case errChan <- constants.ErrNoContentsToUpload:
default:
}
return
}
select {
case errChan <- writer.Close():
default:
}
select {
case errChan <- bodyWriter.Close():
default:
}
}
func (ts *Source) saveToken() error {
log.Debugf("saving the token to %s", ts.path)
f, err := os.Create(ts.path)
if err != nil {
log.Errorf("%s: %s", constants.ErrCreateFile, ts.path)
return constants.ErrCreateFile
}
if err := json.NewEncoder(f).Encode(ts.token); err != nil {
log.Errorf("%s: %s", constants.ErrJSONEncoding, err)
return constants.ErrJSONEncoding
}
log.Debug("token saved successfully")
return nil
}
func (ts *Source) readToken() error {
log.Debugf("reading the token from %s", ts.path)
f, err := os.Open(ts.path)
if err != nil {
log.Errorf("%s: %s", constants.ErrOpenFile, ts.path)
return constants.ErrOpenFile
}
if err := json.NewDecoder(f).Decode(ts.token); err != nil {
log.Errorf("%s: %s", constants.ErrJSONDecoding, err)
return constants.ErrJSONDecoding
}
log.Debug("token loaded successfully")
return nil
}
// CheckResponse validates the response from the Amazon Cloud Drive API. It
// does that by looking at the response's status code and it returns an error
// for any code lower than 200 or greater than 300
func (c *Client) CheckResponse(res *http.Response) error {
if 200 <= res.StatusCode && res.StatusCode <= 299 {
return nil
}
errBody := "no response body"
defer res.Body.Close()
if data, err := ioutil.ReadAll(res.Body); err == nil {
errBody = string(data)
}
var err error
switch res.StatusCode {
case http.StatusBadRequest:
err = constants.ErrResponseBadInput
case http.StatusUnauthorized:
err = constants.ErrResponseInvalidToken
case http.StatusForbidden:
err = constants.ErrResponseForbidden
case http.StatusConflict:
err = constants.ErrResponseDuplicateExists
case http.StatusInternalServerError:
err = constants.ErrResponseInternalServerError
case http.StatusServiceUnavailable:
err = constants.ErrResponseUnavailable
default:
err = constants.ErrResponseUnknown
}
log.Errorf("{code: %s} %s: %s", res.Status, err, errBody)
return err
}
func (n *Node) update(newNode *Node) error {
// encode the newNode to JSON.
v, err := json.Marshal(newNode)
if err != nil {
log.Errorf("error encoding the node to JSON: %s", err)
return constants.ErrJSONEncoding
}
// decode it back to n
if err := json.Unmarshal(v, n); err != nil {
log.Errorf("error decoding the node from JSON: %s", err)
return constants.ErrJSONDecoding
}
return nil
}
func cleanUp() {
if needCleaning {
// remove the test folder
if err := removeTestFolder(); err != nil {
log.Errorf("error removing the test folder: %s", err)
}
// avoid double cleaning
needCleaning = false
}
// remove all cache files
for _, cf := range cacheFiles {
os.Remove(cf)
}
// remove all config files.
for _, cf := range configFiles {
os.Remove(cf)
}
// remove all token files.
for _, cf := range tokenFiles {
os.Remove(cf)
}
}
// FindNode finds a node for a particular path.
// TODO(kalbasit): This does not perform well, this should be cached in a map
// path->node and calculated on load (fresh, cache, refresh).
func (nt *Tree) FindNode(path string) (*Node, error) {
// replace multiple n*/ with /
re := regexp.MustCompile("/[/]*")
path = string(re.ReplaceAll([]byte(path), []byte("/")))
// chop off the first /.
path = strings.TrimPrefix(path, "/")
// did we ask for the root node?
if path == "" {
return nt.Node, nil
}
// initialize our search from the root node
node := nt.Node
// iterate over the path parts until we find the path (or not).
parts := strings.Split(path, "/")
for _, part := range parts {
var found bool
for _, n := range node.Nodes {
// does node.name matches our query?
if strings.ToLower(n.Name) == strings.ToLower(part) {
node = n
found = true
break
}
}
if !found {
log.Errorf("%s: %s", constants.ErrNodeNotFound, path)
return nil, constants.ErrNodeNotFound
}
}
return node, nil
}
func validateFile(file string, checkPerms bool) error {
stat, err := os.Stat(file)
if err != nil {
if os.IsNotExist(err) {
log.Errorf("%s: %s -- %s", constants.ErrFileNotFound, err, file)
return constants.ErrFileNotFound
}
log.Errorf("%s: %s -- %s", constants.ErrStatFile, err, file)
return constants.ErrStatFile
}
if checkPerms && stat.Mode() != os.FileMode(0600) {
log.Errorf("%s: want 0600 got %s", constants.ErrWrongPermissions, stat.Mode())
return constants.ErrWrongPermissions
}
return nil
}
// DownloadFolder downloads an entire folder to a path, if recursive is true,
// it will also download all subfolders.
func (c *Client) DownloadFolder(localPath, remotePath string, recursive bool) error {
log.Debugf("downloading %q to %q", localPath, remotePath)
if err := os.Mkdir(localPath, os.FileMode(0755)); err != nil && !os.IsExist(err) {
log.Errorf("%s: %s", constants.ErrCreateFolder, err)
return constants.ErrCreateFolder
}
rootNode, err := c.GetNodeTree().FindNode(remotePath)
if err != nil {
return nil
}
for _, node := range rootNode.Nodes {
flp := path.Join(localPath, node.Name)
frp := fmt.Sprintf("%s/%s", remotePath, node.Name)
if node.IsDir() {
if recursive {
if err := c.DownloadFolder(flp, frp, recursive); err != nil {
return err
}
}
continue
}
con, err := node.Download()
if err != nil {
return err
}
f, err := os.Create(flp)
if err != nil {
log.Errorf("%s: %s", constants.ErrCreateFile, flp)
return constants.ErrCreateFile
}
log.Debugf("saving %s as %s", frp, flp)
_, err = io.Copy(f, con)
f.Close()
con.Close()
if err != nil {
log.Errorf("%s: %s", constants.ErrWritingFileContents, err)
return err
}
}
return nil
}
// FindByID returns the node identified by the ID.
func (nt *Tree) FindByID(id string) (*Node, error) {
n, found := nt.nodeMap[id]
if !found {
log.Errorf("%s: ID %q", constants.ErrNodeNotFound, id)
return nil, constants.ErrNodeNotFound
}
return n, nil
}
// Remove deletes a node from the server.
// This function does not update the NodeTree, the caller should do so!
func (n *Node) Remove() error {
putURL := n.client.GetMetadataURL(fmt.Sprintf("/trash/%s", n.ID))
req, err := http.NewRequest("PUT", putURL, nil)
if err != nil {
log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err)
return constants.ErrCreatingHTTPRequest
}
res, err := n.client.Do(req)
if err != nil {
log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err)
return constants.ErrDoingHTTPRequest
}
if err := n.client.CheckResponse(res); err != nil {
return err
}
return nil
}
// List returns nodes.Nodes for all of the nodes underneath the path. It's up
// to the caller to differentiate between a file, a folder or an asset by using
// (*node.Node).IsFile(), (*node.Node).IsDir() and/or (*node.Node).IsAsset().
// A dir has sub-nodes accessible via (*node.Node).Nodes, you do not need to
// call this this function for every sub-node.
func (c *Client) List(path string) (node.Nodes, error) {
rootNode, err := c.GetNodeTree().FindNode(path)
if err != nil {
return nil, err
}
if !rootNode.IsDir() {
log.Errorf("%s: %s", constants.ErrPathIsNotFolder, path)
return nil, constants.ErrPathIsNotFolder
}
return rootNode.Nodes, nil
}
// Download downloads the node and returns the body as io.ReadCloser or an
// error. The caller is responsible for closing the reader.
func (n *Node) Download() (io.ReadCloser, error) {
if n.IsDir() {
log.Errorf("%s: cannot download a folder", constants.ErrPathIsFolder)
return nil, constants.ErrPathIsFolder
}
url := n.client.GetContentURL(fmt.Sprintf("nodes/%s/content", n.ID))
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err)
return nil, constants.ErrCreatingHTTPRequest
}
res, err := n.client.Do(req)
if err != nil {
log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err)
return nil, constants.ErrDoingHTTPRequest
}
if err := n.client.CheckResponse(res); err != nil {
return nil, err
}
return res.Body, nil
}
// GetAccountInfo returns AccountInfo about the current account.
func (c *Client) GetAccountInfo() (*AccountInfo, error) {
var ai AccountInfo
req, err := http.NewRequest("GET", c.metadataURL+"/account/info", nil)
if err != nil {
log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err)
return nil, constants.ErrCreatingHTTPRequest
}
res, err := c.Do(req)
if err != nil {
log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err)
return nil, constants.ErrDoingHTTPRequest
}
defer res.Body.Close()
if err := json.NewDecoder(res.Body).Decode(&ai); err != nil {
log.Errorf("%s: %s", constants.ErrJSONDecodingResponseBody, err)
return nil, constants.ErrJSONDecodingResponseBody
}
return &ai, nil
}
// CreateFolder creates the named folder under the node
func (n *Node) CreateFolder(name string) (*Node, error) {
cn := &newNode{
Name: name,
Kind: "FOLDER",
Parents: []string{n.ID},
}
jsonBytes, err := json.Marshal(cn)
if err != nil {
log.Errorf("%s: %s", constants.ErrJSONEncoding, err)
return nil, constants.ErrJSONEncoding
}
req, err := http.NewRequest("POST", n.client.GetMetadataURL("nodes"), bytes.NewBuffer(jsonBytes))
if err != nil {
log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err)
return nil, constants.ErrCreatingHTTPRequest
}
req.Header.Set("Content-Type", "application/json")
res, err := n.client.Do(req)
if err != nil {
log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err)
return nil, constants.ErrDoingHTTPRequest
}
if err := n.client.CheckResponse(res); err != nil {
return nil, err
}
defer res.Body.Close()
var node Node
if err := json.NewDecoder(res.Body).Decode(&node); err != nil {
log.Errorf("%s: %s", constants.ErrJSONDecodingResponseBody, err)
return nil, constants.ErrJSONDecodingResponseBody
}
n.AddChild(&node)
return &node, nil
}
func setEndpoints(c *Client) error {
req, err := http.NewRequest("GET", endpointURL, nil)
if err != nil {
log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err)
return constants.ErrCreatingHTTPRequest
}
var er endpointResponse
res, err := c.Do(req)
if err != nil {
log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err)
return constants.ErrDoingHTTPRequest
}
defer res.Body.Close()
if err := json.NewDecoder(res.Body).Decode(&er); err != nil {
log.Errorf("%s: %s", constants.ErrJSONDecodingResponseBody, err)
return constants.ErrJSONDecodingResponseBody
}
c.contentURL = er.ContentURL
c.metadataURL = er.MetadataURL
return nil
}
// New returns a new Source implementing oauth2.TokenSource. The path must
// exist on the filesystem and must be of permissions 0600.
func New(path string) (*Source, error) {
if _, err := os.Stat(path); os.IsNotExist(err) {
log.Errorf("%s: %s", constants.ErrFileNotFound, path)
return nil, constants.ErrFileNotFound
}
ts := &Source{
path: path,
token: new(oauth2.Token),
}
ts.readToken()
return ts, nil
}
// Upload writes contents of r as name inside the current node.
func (n *Node) Upload(name string, r io.Reader) (*Node, error) {
metadata := &newNode{
Name: name,
Kind: "FILE",
Parents: []string{n.ID},
}
metadataJSON, err := json.Marshal(metadata)
if err != nil {
log.Errorf("%s: %s", constants.ErrJSONEncoding, err)
return nil, constants.ErrJSONEncoding
}
postURL := n.client.GetContentURL("nodes?suppress=deduplication")
node, err := n.upload(postURL, "POST", string(metadataJSON), name, r)
if err != nil {
return nil, err
}
n.AddChild(node)
return node, nil
}
func (nt *Tree) fetchFresh() error {
// grab the list of all of the nodes from the server.
var nextToken string
var nodes []*Node
for {
nl := nodeList{
Nodes: make([]*Node, 0, 200),
}
urlStr := nt.client.GetMetadataURL("nodes")
u, err := url.Parse(urlStr)
if err != nil {
log.Errorf("%s: %s", constants.ErrParsingURL, urlStr)
return constants.ErrParsingURL
}
v := url.Values{}
v.Set("limit", "200")
if nextToken != "" {
v.Set("startToken", nextToken)
}
u.RawQuery = v.Encode()
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err)
return constants.ErrCreatingHTTPRequest
}
req.Header.Set("Content-Type", "application/json")
res, err := nt.client.Do(req)
if err != nil {
log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err)
return constants.ErrDoingHTTPRequest
}
defer res.Body.Close()
if err := json.NewDecoder(res.Body).Decode(&nl); err != nil {
log.Errorf("%s: %s", constants.ErrJSONDecodingResponseBody, err)
return constants.ErrJSONDecodingResponseBody
}
nextToken = nl.NextToken
nodes = append(nodes, nl.Nodes...)
if nextToken == "" {
break
}
}
nodeMap := make(map[string]*Node, len(nodes))
for _, node := range nodes {
if !node.Available() {
continue
}
nt.setClient(node)
nodeMap[node.ID] = node
}
for _, node := range nodeMap {
if node.Name == "" && node.IsDir() && len(node.Parents) == 0 {
nt.Node = node
node.Root = true
}
for _, parentID := range node.Parents {
if pn, found := nodeMap[parentID]; found {
pn.Nodes = append(pn.Nodes, node)
}
}
}
nt.nodeMap = nodeMap
return nil
}