func (ts *Source) refreshToken() error { log.Debugf("refreshing the token from %q", refreshURL) data, err := json.Marshal(ts.token) if err != nil { log.Errorf("%s: %s", constants.ErrJSONEncoding, err) return constants.ErrJSONEncoding } req, err := http.NewRequest("POST", refreshURL, bytes.NewBuffer(data)) if err != nil { log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err) return constants.ErrCreatingHTTPRequest } req.Header.Set("Content-Type", "application/json") res, err := (&http.Client{}).Do(req) if err != nil { log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err) return constants.ErrDoingHTTPRequest } defer res.Body.Close() if err := json.NewDecoder(res.Body).Decode(ts.token); err != nil { log.Errorf("%s: %s", constants.ErrJSONDecodingResponseBody, err) return constants.ErrJSONDecodingResponseBody } log.Debug("token was refreshed successfully") return nil }
func loadConfig(configFile string) (*Config, error) { // validate the config file if err := validateFile(configFile, false); err != nil { return nil, err } cf, err := os.Open(configFile) if err != nil { log.Errorf("%s: %s", constants.ErrOpenFile, err) return nil, err } defer cf.Close() var config Config if err := json.NewDecoder(cf).Decode(&config); err != nil { log.Errorf("%s: %s", constants.ErrJSONDecoding, err) return nil, err } // validate the token file if err := validateFile(config.TokenFile, true); err != nil { return nil, err } return &config, nil }
// MkdirAll creates a directory named path, along with any necessary parents, // and returns the directory node and nil, or else returns an error. If path is // already a directory, MkdirAll does nothing and returns the directory node // and nil. func (nt *Tree) MkdirAll(path string) (*Node, error) { var ( err error folderNode = nt.Node logLevel = log.GetLevel() nextNode *Node node *Node ) // Short-circuit if the node already exists! { log.SetLevel(log.DisableLogLevel) node, err = nt.FindNode(path) log.SetLevel(logLevel) } if err == nil { if node.IsDir() { return node, err } log.Errorf("%s: %s", constants.ErrFileExistsAndIsNotFolder, path) return nil, constants.ErrFileExistsAndIsNotFolder } // chop off the first /. if strings.HasPrefix(path, "/") { path = path[1:] } parts := strings.Split(path, "/") if len(parts) == 0 { log.Errorf("%s: %s", constants.ErrCannotCreateRootNode, path) return nil, constants.ErrCannotCreateRootNode } for i, part := range parts { { log.SetLevel(log.DisableLogLevel) nextNode, err = nt.FindNode(strings.Join(parts[:i+1], "/")) log.SetLevel(logLevel) } if err != nil && err != constants.ErrNodeNotFound { return nil, err } if err == constants.ErrNodeNotFound { nextNode, err = folderNode.CreateFolder(part) if err != nil { return nil, err } } if !nextNode.IsDir() { log.Errorf("%s: %s", constants.ErrCannotCreateANodeUnderAFile, strings.Join(parts[:i+1], "/")) return nil, constants.ErrCannotCreateANodeUnderAFile } folderNode = nextNode } return folderNode, nil }
// Sync syncs the tree with the server. func (nt *Tree) Sync() error { postURL := nt.client.GetMetadataURL("changes") c := &changes{ Checkpoint: nt.Checkpoint, } jsonBytes, err := json.Marshal(c) if err != nil { log.Errorf("%s: %s", constants.ErrJSONEncoding, err) return constants.ErrJSONEncoding } req, err := http.NewRequest("POST", postURL, bytes.NewBuffer(jsonBytes)) if err != nil { log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err) return constants.ErrCreatingHTTPRequest } req.Header.Set("Content-Type", "application/json") res, err := nt.client.Do(req) if err != nil { log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err) return constants.ErrDoingHTTPRequest } if err := nt.client.CheckResponse(res); err != nil { return err } // return format should be: // {"checkpoint": str, "reset": bool, "nodes": []} // {"checkpoint": str, "reset": false, "nodes": []} // {"end": true} defer res.Body.Close() bodyBytes, err := ioutil.ReadAll(res.Body) if err != nil { log.Errorf("%s: %s", constants.ErrReadingResponseBody, err) return constants.ErrReadingResponseBody } for _, lineBytes := range bytes.Split(bodyBytes, []byte("\n")) { var cr changesResponse if err := json.Unmarshal(lineBytes, &cr); err != nil { log.Errorf("%s: %s", constants.ErrJSONDecodingResponseBody, err) return constants.ErrJSONDecodingResponseBody } if cr.Checkpoint != "" { log.Debugf("changes returned Checkpoint: %s", cr.Checkpoint) nt.Checkpoint = cr.Checkpoint } if cr.Reset { log.Debug("reset is required") return constants.ErrMustFetchFresh } if cr.End { break } if err := nt.updateNodes(cr.Nodes); err != nil { return err } } return nil }
func (n *Node) upload(url, method, metadataJSON, name string, r io.Reader) (*Node, error) { bodyReader, bodyWriter := io.Pipe() errChan := make(chan error) bodyChan := make(chan io.ReadCloser) contentTypeChan := make(chan string) go n.bodyWriter(metadataJSON, name, r, bodyWriter, errChan, contentTypeChan) go func() { req, err := http.NewRequest(method, url, bodyReader) if err != nil { log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err) select { case errChan <- constants.ErrCreatingHTTPRequest: default: } return } req.Header.Add("Content-Type", <-contentTypeChan) res, err := n.client.Do(req) // this should block until the upload is finished. if err != nil { log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err) select { case errChan <- constants.ErrDoingHTTPRequest: default: } return } if err := n.client.CheckResponse(res); err != nil { select { case errChan <- err: default: } return } select { case bodyChan <- res.Body: default: } }() for { select { case err := <-errChan: if err != nil { return nil, err } case body := <-bodyChan: defer body.Close() var node Node if err := json.NewDecoder(body).Decode(&node); err != nil { log.Errorf("%s: %s", constants.ErrJSONDecodingResponseBody, err) return nil, constants.ErrJSONDecodingResponseBody } return &node, nil } } }
func (nt *Tree) saveCache() error { f, err := os.Create(nt.cacheFile) if err != nil { log.Errorf("%s: %s", constants.ErrCreateFile, nt.cacheFile) return constants.ErrCreateFile } if err := gob.NewEncoder(f).Encode(nt); err != nil { log.Errorf("%s: %s", constants.ErrGOBEncoding, err) return constants.ErrGOBEncoding } log.Debugf("saved NodeTree to cache file %q.", nt.cacheFile) return nil }
func (n *Node) bodyWriter(metadataJSON, name string, r io.Reader, bodyWriter io.WriteCloser, errChan chan error, contentTypeChan chan string) { writer := multipart.NewWriter(bodyWriter) contentTypeChan <- writer.FormDataContentType() if metadataJSON != "" { if err := writer.WriteField("metadata", metadataJSON); err != nil { log.Errorf("%s: %s", constants.ErrWritingMetadata, err) select { case errChan <- constants.ErrWritingMetadata: default: } return } } part, err := writer.CreateFormFile("content", name) if err != nil { log.Errorf("%s: %s", constants.ErrCreatingWriterFromFile, err) select { case errChan <- err: default: } return } count, err := io.Copy(part, r) if err != nil { log.Errorf("%s: %s", constants.ErrWritingFileContents, err) select { case errChan <- constants.ErrWritingFileContents: default: } return } if count == 0 { select { case errChan <- constants.ErrNoContentsToUpload: default: } return } select { case errChan <- writer.Close(): default: } select { case errChan <- bodyWriter.Close(): default: } }
func (ts *Source) saveToken() error { log.Debugf("saving the token to %s", ts.path) f, err := os.Create(ts.path) if err != nil { log.Errorf("%s: %s", constants.ErrCreateFile, ts.path) return constants.ErrCreateFile } if err := json.NewEncoder(f).Encode(ts.token); err != nil { log.Errorf("%s: %s", constants.ErrJSONEncoding, err) return constants.ErrJSONEncoding } log.Debug("token saved successfully") return nil }
func (ts *Source) readToken() error { log.Debugf("reading the token from %s", ts.path) f, err := os.Open(ts.path) if err != nil { log.Errorf("%s: %s", constants.ErrOpenFile, ts.path) return constants.ErrOpenFile } if err := json.NewDecoder(f).Decode(ts.token); err != nil { log.Errorf("%s: %s", constants.ErrJSONDecoding, err) return constants.ErrJSONDecoding } log.Debug("token loaded successfully") return nil }
// CheckResponse validates the response from the Amazon Cloud Drive API. It // does that by looking at the response's status code and it returns an error // for any code lower than 200 or greater than 300 func (c *Client) CheckResponse(res *http.Response) error { if 200 <= res.StatusCode && res.StatusCode <= 299 { return nil } errBody := "no response body" defer res.Body.Close() if data, err := ioutil.ReadAll(res.Body); err == nil { errBody = string(data) } var err error switch res.StatusCode { case http.StatusBadRequest: err = constants.ErrResponseBadInput case http.StatusUnauthorized: err = constants.ErrResponseInvalidToken case http.StatusForbidden: err = constants.ErrResponseForbidden case http.StatusConflict: err = constants.ErrResponseDuplicateExists case http.StatusInternalServerError: err = constants.ErrResponseInternalServerError case http.StatusServiceUnavailable: err = constants.ErrResponseUnavailable default: err = constants.ErrResponseUnknown } log.Errorf("{code: %s} %s: %s", res.Status, err, errBody) return err }
func (n *Node) update(newNode *Node) error { // encode the newNode to JSON. v, err := json.Marshal(newNode) if err != nil { log.Errorf("error encoding the node to JSON: %s", err) return constants.ErrJSONEncoding } // decode it back to n if err := json.Unmarshal(v, n); err != nil { log.Errorf("error decoding the node from JSON: %s", err) return constants.ErrJSONDecoding } return nil }
func cleanUp() { if needCleaning { // remove the test folder if err := removeTestFolder(); err != nil { log.Errorf("error removing the test folder: %s", err) } // avoid double cleaning needCleaning = false } // remove all cache files for _, cf := range cacheFiles { os.Remove(cf) } // remove all config files. for _, cf := range configFiles { os.Remove(cf) } // remove all token files. for _, cf := range tokenFiles { os.Remove(cf) } }
// FindNode finds a node for a particular path. // TODO(kalbasit): This does not perform well, this should be cached in a map // path->node and calculated on load (fresh, cache, refresh). func (nt *Tree) FindNode(path string) (*Node, error) { // replace multiple n*/ with / re := regexp.MustCompile("/[/]*") path = string(re.ReplaceAll([]byte(path), []byte("/"))) // chop off the first /. path = strings.TrimPrefix(path, "/") // did we ask for the root node? if path == "" { return nt.Node, nil } // initialize our search from the root node node := nt.Node // iterate over the path parts until we find the path (or not). parts := strings.Split(path, "/") for _, part := range parts { var found bool for _, n := range node.Nodes { // does node.name matches our query? if strings.ToLower(n.Name) == strings.ToLower(part) { node = n found = true break } } if !found { log.Errorf("%s: %s", constants.ErrNodeNotFound, path) return nil, constants.ErrNodeNotFound } } return node, nil }
func validateFile(file string, checkPerms bool) error { stat, err := os.Stat(file) if err != nil { if os.IsNotExist(err) { log.Errorf("%s: %s -- %s", constants.ErrFileNotFound, err, file) return constants.ErrFileNotFound } log.Errorf("%s: %s -- %s", constants.ErrStatFile, err, file) return constants.ErrStatFile } if checkPerms && stat.Mode() != os.FileMode(0600) { log.Errorf("%s: want 0600 got %s", constants.ErrWrongPermissions, stat.Mode()) return constants.ErrWrongPermissions } return nil }
// DownloadFolder downloads an entire folder to a path, if recursive is true, // it will also download all subfolders. func (c *Client) DownloadFolder(localPath, remotePath string, recursive bool) error { log.Debugf("downloading %q to %q", localPath, remotePath) if err := os.Mkdir(localPath, os.FileMode(0755)); err != nil && !os.IsExist(err) { log.Errorf("%s: %s", constants.ErrCreateFolder, err) return constants.ErrCreateFolder } rootNode, err := c.GetNodeTree().FindNode(remotePath) if err != nil { return nil } for _, node := range rootNode.Nodes { flp := path.Join(localPath, node.Name) frp := fmt.Sprintf("%s/%s", remotePath, node.Name) if node.IsDir() { if recursive { if err := c.DownloadFolder(flp, frp, recursive); err != nil { return err } } continue } con, err := node.Download() if err != nil { return err } f, err := os.Create(flp) if err != nil { log.Errorf("%s: %s", constants.ErrCreateFile, flp) return constants.ErrCreateFile } log.Debugf("saving %s as %s", frp, flp) _, err = io.Copy(f, con) f.Close() con.Close() if err != nil { log.Errorf("%s: %s", constants.ErrWritingFileContents, err) return err } } return nil }
// FindByID returns the node identified by the ID. func (nt *Tree) FindByID(id string) (*Node, error) { n, found := nt.nodeMap[id] if !found { log.Errorf("%s: ID %q", constants.ErrNodeNotFound, id) return nil, constants.ErrNodeNotFound } return n, nil }
// Remove deletes a node from the server. // This function does not update the NodeTree, the caller should do so! func (n *Node) Remove() error { putURL := n.client.GetMetadataURL(fmt.Sprintf("/trash/%s", n.ID)) req, err := http.NewRequest("PUT", putURL, nil) if err != nil { log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err) return constants.ErrCreatingHTTPRequest } res, err := n.client.Do(req) if err != nil { log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err) return constants.ErrDoingHTTPRequest } if err := n.client.CheckResponse(res); err != nil { return err } return nil }
// List returns nodes.Nodes for all of the nodes underneath the path. It's up // to the caller to differentiate between a file, a folder or an asset by using // (*node.Node).IsFile(), (*node.Node).IsDir() and/or (*node.Node).IsAsset(). // A dir has sub-nodes accessible via (*node.Node).Nodes, you do not need to // call this this function for every sub-node. func (c *Client) List(path string) (node.Nodes, error) { rootNode, err := c.GetNodeTree().FindNode(path) if err != nil { return nil, err } if !rootNode.IsDir() { log.Errorf("%s: %s", constants.ErrPathIsNotFolder, path) return nil, constants.ErrPathIsNotFolder } return rootNode.Nodes, nil }
// Download downloads the node and returns the body as io.ReadCloser or an // error. The caller is responsible for closing the reader. func (n *Node) Download() (io.ReadCloser, error) { if n.IsDir() { log.Errorf("%s: cannot download a folder", constants.ErrPathIsFolder) return nil, constants.ErrPathIsFolder } url := n.client.GetContentURL(fmt.Sprintf("nodes/%s/content", n.ID)) req, err := http.NewRequest("GET", url, nil) if err != nil { log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err) return nil, constants.ErrCreatingHTTPRequest } res, err := n.client.Do(req) if err != nil { log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err) return nil, constants.ErrDoingHTTPRequest } if err := n.client.CheckResponse(res); err != nil { return nil, err } return res.Body, nil }
// GetAccountInfo returns AccountInfo about the current account. func (c *Client) GetAccountInfo() (*AccountInfo, error) { var ai AccountInfo req, err := http.NewRequest("GET", c.metadataURL+"/account/info", nil) if err != nil { log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err) return nil, constants.ErrCreatingHTTPRequest } res, err := c.Do(req) if err != nil { log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err) return nil, constants.ErrDoingHTTPRequest } defer res.Body.Close() if err := json.NewDecoder(res.Body).Decode(&ai); err != nil { log.Errorf("%s: %s", constants.ErrJSONDecodingResponseBody, err) return nil, constants.ErrJSONDecodingResponseBody } return &ai, nil }
// CreateFolder creates the named folder under the node func (n *Node) CreateFolder(name string) (*Node, error) { cn := &newNode{ Name: name, Kind: "FOLDER", Parents: []string{n.ID}, } jsonBytes, err := json.Marshal(cn) if err != nil { log.Errorf("%s: %s", constants.ErrJSONEncoding, err) return nil, constants.ErrJSONEncoding } req, err := http.NewRequest("POST", n.client.GetMetadataURL("nodes"), bytes.NewBuffer(jsonBytes)) if err != nil { log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err) return nil, constants.ErrCreatingHTTPRequest } req.Header.Set("Content-Type", "application/json") res, err := n.client.Do(req) if err != nil { log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err) return nil, constants.ErrDoingHTTPRequest } if err := n.client.CheckResponse(res); err != nil { return nil, err } defer res.Body.Close() var node Node if err := json.NewDecoder(res.Body).Decode(&node); err != nil { log.Errorf("%s: %s", constants.ErrJSONDecodingResponseBody, err) return nil, constants.ErrJSONDecodingResponseBody } n.AddChild(&node) return &node, nil }
func setEndpoints(c *Client) error { req, err := http.NewRequest("GET", endpointURL, nil) if err != nil { log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err) return constants.ErrCreatingHTTPRequest } var er endpointResponse res, err := c.Do(req) if err != nil { log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err) return constants.ErrDoingHTTPRequest } defer res.Body.Close() if err := json.NewDecoder(res.Body).Decode(&er); err != nil { log.Errorf("%s: %s", constants.ErrJSONDecodingResponseBody, err) return constants.ErrJSONDecodingResponseBody } c.contentURL = er.ContentURL c.metadataURL = er.MetadataURL return nil }
// New returns a new Source implementing oauth2.TokenSource. The path must // exist on the filesystem and must be of permissions 0600. func New(path string) (*Source, error) { if _, err := os.Stat(path); os.IsNotExist(err) { log.Errorf("%s: %s", constants.ErrFileNotFound, path) return nil, constants.ErrFileNotFound } ts := &Source{ path: path, token: new(oauth2.Token), } ts.readToken() return ts, nil }
// Upload writes contents of r as name inside the current node. func (n *Node) Upload(name string, r io.Reader) (*Node, error) { metadata := &newNode{ Name: name, Kind: "FILE", Parents: []string{n.ID}, } metadataJSON, err := json.Marshal(metadata) if err != nil { log.Errorf("%s: %s", constants.ErrJSONEncoding, err) return nil, constants.ErrJSONEncoding } postURL := n.client.GetContentURL("nodes?suppress=deduplication") node, err := n.upload(postURL, "POST", string(metadataJSON), name, r) if err != nil { return nil, err } n.AddChild(node) return node, nil }
func (nt *Tree) fetchFresh() error { // grab the list of all of the nodes from the server. var nextToken string var nodes []*Node for { nl := nodeList{ Nodes: make([]*Node, 0, 200), } urlStr := nt.client.GetMetadataURL("nodes") u, err := url.Parse(urlStr) if err != nil { log.Errorf("%s: %s", constants.ErrParsingURL, urlStr) return constants.ErrParsingURL } v := url.Values{} v.Set("limit", "200") if nextToken != "" { v.Set("startToken", nextToken) } u.RawQuery = v.Encode() req, err := http.NewRequest("GET", u.String(), nil) if err != nil { log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err) return constants.ErrCreatingHTTPRequest } req.Header.Set("Content-Type", "application/json") res, err := nt.client.Do(req) if err != nil { log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err) return constants.ErrDoingHTTPRequest } defer res.Body.Close() if err := json.NewDecoder(res.Body).Decode(&nl); err != nil { log.Errorf("%s: %s", constants.ErrJSONDecodingResponseBody, err) return constants.ErrJSONDecodingResponseBody } nextToken = nl.NextToken nodes = append(nodes, nl.Nodes...) if nextToken == "" { break } } nodeMap := make(map[string]*Node, len(nodes)) for _, node := range nodes { if !node.Available() { continue } nt.setClient(node) nodeMap[node.ID] = node } for _, node := range nodeMap { if node.Name == "" && node.IsDir() && len(node.Parents) == 0 { nt.Node = node node.Root = true } for _, parentID := range node.Parents { if pn, found := nodeMap[parentID]; found { pn.Nodes = append(pn.Nodes, node) } } } nt.nodeMap = nodeMap return nil }