Пример #1
0
func (ts *Source) refreshToken() error {
	log.Debugf("refreshing the token from %q", refreshURL)

	data, err := json.Marshal(ts.token)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrJSONEncoding, err)
		return constants.ErrJSONEncoding
	}
	req, err := http.NewRequest("POST", refreshURL, bytes.NewBuffer(data))
	if err != nil {
		log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err)
		return constants.ErrCreatingHTTPRequest
	}
	req.Header.Set("Content-Type", "application/json")
	res, err := (&http.Client{}).Do(req)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err)
		return constants.ErrDoingHTTPRequest
	}
	defer res.Body.Close()
	if err := json.NewDecoder(res.Body).Decode(ts.token); err != nil {
		log.Errorf("%s: %s", constants.ErrJSONDecodingResponseBody, err)
		return constants.ErrJSONDecodingResponseBody
	}
	log.Debug("token was refreshed successfully")

	return nil
}
Пример #2
0
func loadConfig(configFile string) (*Config, error) {
	// validate the config file
	if err := validateFile(configFile, false); err != nil {
		return nil, err
	}

	cf, err := os.Open(configFile)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrOpenFile, err)
		return nil, err
	}
	defer cf.Close()
	var config Config
	if err := json.NewDecoder(cf).Decode(&config); err != nil {
		log.Errorf("%s: %s", constants.ErrJSONDecoding, err)
		return nil, err
	}

	// validate the token file
	if err := validateFile(config.TokenFile, true); err != nil {
		return nil, err
	}

	return &config, nil
}
Пример #3
0
// MkdirAll creates a directory named path, along with any necessary parents,
// and returns the directory node and nil, or else returns an error. If path is
// already a directory, MkdirAll does nothing and returns the directory node
// and nil.
func (nt *Tree) MkdirAll(path string) (*Node, error) {
	var (
		err        error
		folderNode = nt.Node
		logLevel   = log.GetLevel()
		nextNode   *Node
		node       *Node
	)

	// Short-circuit if the node already exists!
	{
		log.SetLevel(log.DisableLogLevel)
		node, err = nt.FindNode(path)
		log.SetLevel(logLevel)
	}
	if err == nil {
		if node.IsDir() {
			return node, err
		}
		log.Errorf("%s: %s", constants.ErrFileExistsAndIsNotFolder, path)
		return nil, constants.ErrFileExistsAndIsNotFolder
	}

	// chop off the first /.
	if strings.HasPrefix(path, "/") {
		path = path[1:]
	}
	parts := strings.Split(path, "/")
	if len(parts) == 0 {
		log.Errorf("%s: %s", constants.ErrCannotCreateRootNode, path)
		return nil, constants.ErrCannotCreateRootNode
	}

	for i, part := range parts {
		{
			log.SetLevel(log.DisableLogLevel)
			nextNode, err = nt.FindNode(strings.Join(parts[:i+1], "/"))
			log.SetLevel(logLevel)
		}
		if err != nil && err != constants.ErrNodeNotFound {
			return nil, err
		}
		if err == constants.ErrNodeNotFound {
			nextNode, err = folderNode.CreateFolder(part)
			if err != nil {
				return nil, err
			}
		}

		if !nextNode.IsDir() {
			log.Errorf("%s: %s", constants.ErrCannotCreateANodeUnderAFile, strings.Join(parts[:i+1], "/"))
			return nil, constants.ErrCannotCreateANodeUnderAFile
		}

		folderNode = nextNode
	}

	return folderNode, nil
}
Пример #4
0
// Sync syncs the tree with the server.
func (nt *Tree) Sync() error {
	postURL := nt.client.GetMetadataURL("changes")
	c := &changes{
		Checkpoint: nt.Checkpoint,
	}
	jsonBytes, err := json.Marshal(c)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrJSONEncoding, err)
		return constants.ErrJSONEncoding
	}
	req, err := http.NewRequest("POST", postURL, bytes.NewBuffer(jsonBytes))
	if err != nil {
		log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err)
		return constants.ErrCreatingHTTPRequest
	}
	req.Header.Set("Content-Type", "application/json")
	res, err := nt.client.Do(req)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err)
		return constants.ErrDoingHTTPRequest
	}
	if err := nt.client.CheckResponse(res); err != nil {
		return err
	}

	// return format should be:
	// {"checkpoint": str, "reset": bool, "nodes": []}
	// {"checkpoint": str, "reset": false, "nodes": []}
	// {"end": true}
	defer res.Body.Close()
	bodyBytes, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrReadingResponseBody, err)
		return constants.ErrReadingResponseBody
	}
	for _, lineBytes := range bytes.Split(bodyBytes, []byte("\n")) {
		var cr changesResponse
		if err := json.Unmarshal(lineBytes, &cr); err != nil {
			log.Errorf("%s: %s", constants.ErrJSONDecodingResponseBody, err)
			return constants.ErrJSONDecodingResponseBody
		}
		if cr.Checkpoint != "" {
			log.Debugf("changes returned Checkpoint: %s", cr.Checkpoint)
			nt.Checkpoint = cr.Checkpoint
		}
		if cr.Reset {
			log.Debug("reset is required")
			return constants.ErrMustFetchFresh
		}
		if cr.End {
			break
		}
		if err := nt.updateNodes(cr.Nodes); err != nil {
			return err
		}
	}

	return nil
}
Пример #5
0
func (n *Node) upload(url, method, metadataJSON, name string, r io.Reader) (*Node, error) {
	bodyReader, bodyWriter := io.Pipe()
	errChan := make(chan error)
	bodyChan := make(chan io.ReadCloser)
	contentTypeChan := make(chan string)

	go n.bodyWriter(metadataJSON, name, r, bodyWriter, errChan, contentTypeChan)
	go func() {
		req, err := http.NewRequest(method, url, bodyReader)
		if err != nil {
			log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err)
			select {
			case errChan <- constants.ErrCreatingHTTPRequest:
			default:
			}
			return
		}
		req.Header.Add("Content-Type", <-contentTypeChan)
		res, err := n.client.Do(req) // this should block until the upload is finished.
		if err != nil {
			log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err)
			select {
			case errChan <- constants.ErrDoingHTTPRequest:
			default:
			}
			return
		}
		if err := n.client.CheckResponse(res); err != nil {
			select {
			case errChan <- err:
			default:
			}
			return
		}

		select {
		case bodyChan <- res.Body:
		default:
		}
	}()

	for {
		select {
		case err := <-errChan:
			if err != nil {
				return nil, err
			}
		case body := <-bodyChan:
			defer body.Close()
			var node Node
			if err := json.NewDecoder(body).Decode(&node); err != nil {
				log.Errorf("%s: %s", constants.ErrJSONDecodingResponseBody, err)
				return nil, constants.ErrJSONDecodingResponseBody
			}

			return &node, nil
		}
	}
}
Пример #6
0
func (nt *Tree) saveCache() error {
	f, err := os.Create(nt.cacheFile)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrCreateFile, nt.cacheFile)
		return constants.ErrCreateFile
	}
	if err := gob.NewEncoder(f).Encode(nt); err != nil {
		log.Errorf("%s: %s", constants.ErrGOBEncoding, err)
		return constants.ErrGOBEncoding
	}
	log.Debugf("saved NodeTree to cache file %q.", nt.cacheFile)
	return nil
}
Пример #7
0
func (n *Node) bodyWriter(metadataJSON, name string, r io.Reader, bodyWriter io.WriteCloser, errChan chan error, contentTypeChan chan string) {
	writer := multipart.NewWriter(bodyWriter)
	contentTypeChan <- writer.FormDataContentType()
	if metadataJSON != "" {
		if err := writer.WriteField("metadata", metadataJSON); err != nil {
			log.Errorf("%s: %s", constants.ErrWritingMetadata, err)
			select {
			case errChan <- constants.ErrWritingMetadata:
			default:
			}
			return
		}
	}

	part, err := writer.CreateFormFile("content", name)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrCreatingWriterFromFile, err)
		select {
		case errChan <- err:
		default:
		}
		return
	}
	count, err := io.Copy(part, r)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrWritingFileContents, err)
		select {
		case errChan <- constants.ErrWritingFileContents:
		default:
		}
		return
	}
	if count == 0 {
		select {
		case errChan <- constants.ErrNoContentsToUpload:
		default:
		}
		return
	}

	select {
	case errChan <- writer.Close():
	default:
	}
	select {
	case errChan <- bodyWriter.Close():
	default:
	}
}
Пример #8
0
func (ts *Source) saveToken() error {
	log.Debugf("saving the token to %s", ts.path)
	f, err := os.Create(ts.path)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrCreateFile, ts.path)
		return constants.ErrCreateFile
	}
	if err := json.NewEncoder(f).Encode(ts.token); err != nil {
		log.Errorf("%s: %s", constants.ErrJSONEncoding, err)
		return constants.ErrJSONEncoding
	}

	log.Debug("token saved successfully")
	return nil
}
Пример #9
0
func (ts *Source) readToken() error {
	log.Debugf("reading the token from %s", ts.path)
	f, err := os.Open(ts.path)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrOpenFile, ts.path)
		return constants.ErrOpenFile
	}
	if err := json.NewDecoder(f).Decode(ts.token); err != nil {
		log.Errorf("%s: %s", constants.ErrJSONDecoding, err)
		return constants.ErrJSONDecoding
	}

	log.Debug("token loaded successfully")
	return nil
}
Пример #10
0
// CheckResponse validates the response from the Amazon Cloud Drive API. It
// does that by looking at the response's status code and it returns an error
// for any code lower than 200 or greater than 300
func (c *Client) CheckResponse(res *http.Response) error {
	if 200 <= res.StatusCode && res.StatusCode <= 299 {
		return nil
	}
	errBody := "no response body"
	defer res.Body.Close()
	if data, err := ioutil.ReadAll(res.Body); err == nil {
		errBody = string(data)
	}
	var err error
	switch res.StatusCode {
	case http.StatusBadRequest:
		err = constants.ErrResponseBadInput
	case http.StatusUnauthorized:
		err = constants.ErrResponseInvalidToken
	case http.StatusForbidden:
		err = constants.ErrResponseForbidden
	case http.StatusConflict:
		err = constants.ErrResponseDuplicateExists
	case http.StatusInternalServerError:
		err = constants.ErrResponseInternalServerError
	case http.StatusServiceUnavailable:
		err = constants.ErrResponseUnavailable
	default:
		err = constants.ErrResponseUnknown
	}

	log.Errorf("{code: %s} %s: %s", res.Status, err, errBody)
	return err
}
Пример #11
0
func (n *Node) update(newNode *Node) error {
	// encode the newNode to JSON.
	v, err := json.Marshal(newNode)
	if err != nil {
		log.Errorf("error encoding the node to JSON: %s", err)
		return constants.ErrJSONEncoding
	}

	// decode it back to n
	if err := json.Unmarshal(v, n); err != nil {
		log.Errorf("error decoding the node from JSON: %s", err)
		return constants.ErrJSONDecoding
	}

	return nil
}
Пример #12
0
func cleanUp() {
	if needCleaning {
		// remove the test folder
		if err := removeTestFolder(); err != nil {
			log.Errorf("error removing the test folder: %s", err)
		}

		// avoid double cleaning
		needCleaning = false
	}

	// remove all cache files
	for _, cf := range cacheFiles {
		os.Remove(cf)
	}

	// remove all config files.
	for _, cf := range configFiles {
		os.Remove(cf)
	}

	// remove all token files.
	for _, cf := range tokenFiles {
		os.Remove(cf)
	}
}
Пример #13
0
// FindNode finds a node for a particular path.
// TODO(kalbasit): This does not perform well, this should be cached in a map
// path->node and calculated on load (fresh, cache, refresh).
func (nt *Tree) FindNode(path string) (*Node, error) {
	// replace multiple n*/ with /
	re := regexp.MustCompile("/[/]*")
	path = string(re.ReplaceAll([]byte(path), []byte("/")))
	// chop off the first /.
	path = strings.TrimPrefix(path, "/")
	// did we ask for the root node?
	if path == "" {
		return nt.Node, nil
	}

	// initialize our search from the root node
	node := nt.Node

	// iterate over the path parts until we find the path (or not).
	parts := strings.Split(path, "/")
	for _, part := range parts {
		var found bool
		for _, n := range node.Nodes {
			// does node.name matches our query?
			if strings.ToLower(n.Name) == strings.ToLower(part) {
				node = n
				found = true
				break
			}
		}

		if !found {
			log.Errorf("%s: %s", constants.ErrNodeNotFound, path)
			return nil, constants.ErrNodeNotFound
		}
	}

	return node, nil
}
Пример #14
0
func validateFile(file string, checkPerms bool) error {
	stat, err := os.Stat(file)
	if err != nil {
		if os.IsNotExist(err) {
			log.Errorf("%s: %s -- %s", constants.ErrFileNotFound, err, file)
			return constants.ErrFileNotFound
		}
		log.Errorf("%s: %s -- %s", constants.ErrStatFile, err, file)
		return constants.ErrStatFile
	}
	if checkPerms && stat.Mode() != os.FileMode(0600) {
		log.Errorf("%s: want 0600 got %s", constants.ErrWrongPermissions, stat.Mode())
		return constants.ErrWrongPermissions
	}

	return nil
}
Пример #15
0
// DownloadFolder downloads an entire folder to a path, if recursive is true,
// it will also download all subfolders.
func (c *Client) DownloadFolder(localPath, remotePath string, recursive bool) error {
	log.Debugf("downloading %q to %q", localPath, remotePath)

	if err := os.Mkdir(localPath, os.FileMode(0755)); err != nil && !os.IsExist(err) {
		log.Errorf("%s: %s", constants.ErrCreateFolder, err)
		return constants.ErrCreateFolder
	}
	rootNode, err := c.GetNodeTree().FindNode(remotePath)
	if err != nil {
		return nil
	}
	for _, node := range rootNode.Nodes {
		flp := path.Join(localPath, node.Name)
		frp := fmt.Sprintf("%s/%s", remotePath, node.Name)
		if node.IsDir() {
			if recursive {
				if err := c.DownloadFolder(flp, frp, recursive); err != nil {
					return err
				}
			}

			continue
		}

		con, err := node.Download()
		if err != nil {
			return err
		}
		f, err := os.Create(flp)
		if err != nil {
			log.Errorf("%s: %s", constants.ErrCreateFile, flp)
			return constants.ErrCreateFile
		}
		log.Debugf("saving %s as %s", frp, flp)
		_, err = io.Copy(f, con)
		f.Close()
		con.Close()
		if err != nil {
			log.Errorf("%s: %s", constants.ErrWritingFileContents, err)
			return err
		}
	}

	return nil
}
Пример #16
0
// FindByID returns the node identified by the ID.
func (nt *Tree) FindByID(id string) (*Node, error) {
	n, found := nt.nodeMap[id]
	if !found {
		log.Errorf("%s: ID %q", constants.ErrNodeNotFound, id)
		return nil, constants.ErrNodeNotFound
	}

	return n, nil
}
Пример #17
0
// Remove deletes a node from the server.
// This function does not update the NodeTree, the caller should do so!
func (n *Node) Remove() error {
	putURL := n.client.GetMetadataURL(fmt.Sprintf("/trash/%s", n.ID))
	req, err := http.NewRequest("PUT", putURL, nil)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err)
		return constants.ErrCreatingHTTPRequest
	}
	res, err := n.client.Do(req)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err)
		return constants.ErrDoingHTTPRequest
	}
	if err := n.client.CheckResponse(res); err != nil {
		return err
	}

	return nil
}
Пример #18
0
// List returns nodes.Nodes for all of the nodes underneath the path. It's up
// to the caller to differentiate between a file, a folder or an asset by using
// (*node.Node).IsFile(), (*node.Node).IsDir() and/or (*node.Node).IsAsset().
// A dir has sub-nodes accessible via (*node.Node).Nodes, you do not need to
// call this this function for every sub-node.
func (c *Client) List(path string) (node.Nodes, error) {
	rootNode, err := c.GetNodeTree().FindNode(path)
	if err != nil {
		return nil, err
	}
	if !rootNode.IsDir() {
		log.Errorf("%s: %s", constants.ErrPathIsNotFolder, path)
		return nil, constants.ErrPathIsNotFolder
	}

	return rootNode.Nodes, nil
}
Пример #19
0
// Download downloads the node and returns the body as io.ReadCloser or an
// error. The caller is responsible for closing the reader.
func (n *Node) Download() (io.ReadCloser, error) {
	if n.IsDir() {
		log.Errorf("%s: cannot download a folder", constants.ErrPathIsFolder)
		return nil, constants.ErrPathIsFolder
	}
	url := n.client.GetContentURL(fmt.Sprintf("nodes/%s/content", n.ID))
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err)
		return nil, constants.ErrCreatingHTTPRequest
	}
	res, err := n.client.Do(req)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err)
		return nil, constants.ErrDoingHTTPRequest
	}
	if err := n.client.CheckResponse(res); err != nil {
		return nil, err
	}

	return res.Body, nil
}
Пример #20
0
// GetAccountInfo returns AccountInfo about the current account.
func (c *Client) GetAccountInfo() (*AccountInfo, error) {
	var ai AccountInfo
	req, err := http.NewRequest("GET", c.metadataURL+"/account/info", nil)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err)
		return nil, constants.ErrCreatingHTTPRequest
	}

	res, err := c.Do(req)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err)
		return nil, constants.ErrDoingHTTPRequest
	}

	defer res.Body.Close()
	if err := json.NewDecoder(res.Body).Decode(&ai); err != nil {
		log.Errorf("%s: %s", constants.ErrJSONDecodingResponseBody, err)
		return nil, constants.ErrJSONDecodingResponseBody
	}

	return &ai, nil
}
Пример #21
0
// CreateFolder creates the named folder under the node
func (n *Node) CreateFolder(name string) (*Node, error) {
	cn := &newNode{
		Name:    name,
		Kind:    "FOLDER",
		Parents: []string{n.ID},
	}
	jsonBytes, err := json.Marshal(cn)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrJSONEncoding, err)
		return nil, constants.ErrJSONEncoding
	}

	req, err := http.NewRequest("POST", n.client.GetMetadataURL("nodes"), bytes.NewBuffer(jsonBytes))
	if err != nil {
		log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err)
		return nil, constants.ErrCreatingHTTPRequest
	}

	req.Header.Set("Content-Type", "application/json")
	res, err := n.client.Do(req)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err)
		return nil, constants.ErrDoingHTTPRequest
	}
	if err := n.client.CheckResponse(res); err != nil {
		return nil, err
	}

	defer res.Body.Close()
	var node Node
	if err := json.NewDecoder(res.Body).Decode(&node); err != nil {
		log.Errorf("%s: %s", constants.ErrJSONDecodingResponseBody, err)
		return nil, constants.ErrJSONDecodingResponseBody
	}
	n.AddChild(&node)

	return &node, nil
}
Пример #22
0
func setEndpoints(c *Client) error {
	req, err := http.NewRequest("GET", endpointURL, nil)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err)
		return constants.ErrCreatingHTTPRequest
	}

	var er endpointResponse
	res, err := c.Do(req)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err)
		return constants.ErrDoingHTTPRequest
	}
	defer res.Body.Close()
	if err := json.NewDecoder(res.Body).Decode(&er); err != nil {
		log.Errorf("%s: %s", constants.ErrJSONDecodingResponseBody, err)
		return constants.ErrJSONDecodingResponseBody
	}

	c.contentURL = er.ContentURL
	c.metadataURL = er.MetadataURL
	return nil
}
Пример #23
0
// New returns a new Source implementing oauth2.TokenSource. The path must
// exist on the filesystem and must be of permissions 0600.
func New(path string) (*Source, error) {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		log.Errorf("%s: %s", constants.ErrFileNotFound, path)
		return nil, constants.ErrFileNotFound
	}

	ts := &Source{
		path:  path,
		token: new(oauth2.Token),
	}
	ts.readToken()

	return ts, nil
}
Пример #24
0
// Upload writes contents of r as name inside the current node.
func (n *Node) Upload(name string, r io.Reader) (*Node, error) {
	metadata := &newNode{
		Name:    name,
		Kind:    "FILE",
		Parents: []string{n.ID},
	}
	metadataJSON, err := json.Marshal(metadata)
	if err != nil {
		log.Errorf("%s: %s", constants.ErrJSONEncoding, err)
		return nil, constants.ErrJSONEncoding
	}

	postURL := n.client.GetContentURL("nodes?suppress=deduplication")
	node, err := n.upload(postURL, "POST", string(metadataJSON), name, r)
	if err != nil {
		return nil, err
	}

	n.AddChild(node)
	return node, nil
}
Пример #25
0
func (nt *Tree) fetchFresh() error {
	// grab the list of all of the nodes from the server.
	var nextToken string
	var nodes []*Node
	for {
		nl := nodeList{
			Nodes: make([]*Node, 0, 200),
		}
		urlStr := nt.client.GetMetadataURL("nodes")
		u, err := url.Parse(urlStr)
		if err != nil {
			log.Errorf("%s: %s", constants.ErrParsingURL, urlStr)
			return constants.ErrParsingURL
		}

		v := url.Values{}
		v.Set("limit", "200")
		if nextToken != "" {
			v.Set("startToken", nextToken)
		}
		u.RawQuery = v.Encode()

		req, err := http.NewRequest("GET", u.String(), nil)
		if err != nil {
			log.Errorf("%s: %s", constants.ErrCreatingHTTPRequest, err)
			return constants.ErrCreatingHTTPRequest
		}
		req.Header.Set("Content-Type", "application/json")
		res, err := nt.client.Do(req)
		if err != nil {
			log.Errorf("%s: %s", constants.ErrDoingHTTPRequest, err)
			return constants.ErrDoingHTTPRequest
		}

		defer res.Body.Close()
		if err := json.NewDecoder(res.Body).Decode(&nl); err != nil {
			log.Errorf("%s: %s", constants.ErrJSONDecodingResponseBody, err)
			return constants.ErrJSONDecodingResponseBody
		}

		nextToken = nl.NextToken
		nodes = append(nodes, nl.Nodes...)

		if nextToken == "" {
			break
		}
	}

	nodeMap := make(map[string]*Node, len(nodes))
	for _, node := range nodes {
		if !node.Available() {
			continue
		}
		nt.setClient(node)
		nodeMap[node.ID] = node
	}

	for _, node := range nodeMap {
		if node.Name == "" && node.IsDir() && len(node.Parents) == 0 {
			nt.Node = node
			node.Root = true
		}

		for _, parentID := range node.Parents {
			if pn, found := nodeMap[parentID]; found {
				pn.Nodes = append(pn.Nodes, node)
			}
		}
	}

	nt.nodeMap = nodeMap
	return nil
}