Exemple #1
0
func filerUpload(localName, remoteName, checksum, moduleType, authToken string) (stat *contester_proto.FileStat, err error) {
	ec := tools.ErrorContext("upload")
	if stat, err = tools.StatFile(localName, true); err != nil || stat == nil {
		return stat, err
	}
	if checksum != "" && stat.GetChecksum() != checksum {
		return nil, fmt.Errorf("Checksum mismatch, local %s != %s", stat.GetChecksum(), checksum)
	}
	checksum = stat.GetChecksum()

	local, err := os.Open(localName)
	if err != nil {
		return nil, ec.NewError(err, "local.Open")
	}
	defer local.Close()

	req, err := http.NewRequest("PUT", remoteName, local)
	if err != nil {
		return nil, err
	}
	if moduleType != "" {
		req.Header.Add("X-FS-Module-Type", moduleType)
	}
	if authToken != "" {
		req.Header.Add("Authorization", "bearer "+authToken)
	}
	req.Header.Add("X-FS-Content-Length", strconv.FormatUint(stat.GetSize_(), 10))
	var base64sha1 string
	if checksum != "" && strings.HasPrefix(checksum, "sha1:") {
		if data, err := hex.DecodeString(strings.TrimPrefix(checksum, "sha1:")); err == nil {
			base64sha1 = base64.StdEncoding.EncodeToString(data)
			req.Header.Add("Digest", "SHA="+base64sha1)
		}
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	var st uploadStatus
	err = json.NewDecoder(resp.Body).Decode(&st)
	if err != nil {
		return nil, err
	}
	if st.Size != int64(stat.GetSize_()) || (base64sha1 != "" && base64sha1 != st.Digests["SHA"]) {
		return nil, fmt.Errorf("upload integrity verification failed")
	}
	return stat, nil
}
Exemple #2
0
func (s *Contester) Put(request *contester_proto.FileBlob, response *contester_proto.FileStat) error {
	ec := tools.ErrorContext("Put")

	resolved, sandbox, err := resolvePath(s.Sandboxes, *request.Name, true)
	if err != nil {
		return ec.NewError(err, "resolvePath")
	}

	if sandbox != nil {
		sandbox.Mutex.Lock()
		defer sandbox.Mutex.Unlock()
	}

	var destination *os.File

	for {
		destination, err = os.Create(resolved)
		loop, err := OnOsCreateError(err)

		if err != nil {
			return ec.NewError(err, "os.Create")
		}
		if !loop {
			break
		}
	}
	data, err := request.Data.Bytes()
	if err != nil {
		return ec.NewError(err, "request.Data.Bytes")
	}
	_, err = destination.Write(data)
	if err != nil {
		return ec.NewError(err, "destination.Write")
	}
	destination.Close()
	if sandbox != nil {
		return sandbox.Own(resolved)
	}

	stat, err := tools.StatFile(resolved, true)
	if err != nil {
		return ec.NewError(err, "statFile")
	}

	*response = *stat
	return nil
}
Exemple #3
0
func (s *Contester) Stat(request *contester_proto.StatRequest, response *contester_proto.FileStats) error {
	ec := tools.ErrorContext("Stat")
	if request.SandboxId != nil {
		sandbox, err := getSandboxById(s.Sandboxes, *request.SandboxId)
		if err != nil {
			return ec.NewError(err, "getSandboxById")
		}
		sandbox.Mutex.RLock()
		defer sandbox.Mutex.RUnlock()
	}

	response.Entries = make([]*contester_proto.FileStat, 0, len(request.Name))
	for _, name := range request.Name {
		resolved, _, err := resolvePath(s.Sandboxes, name, false)
		if err != nil {
			return ec.NewError(err, "resolvePath")
		}
		var expanded []string
		if request.GetExpand() {
			expanded, err = filepath.Glob(resolved)
			if err != nil {
				return ec.NewError(err, "filepath.Glob")
			}
		} else {
			expanded = []string{resolved}
		}

		for _, name := range expanded {
			stat, err := tools.StatFile(name, request.GetCalculateChecksum())
			if err != nil {
				return ec.NewError(err, "statFile")
			}
			if stat != nil {
				response.Entries = append(response.Entries, stat)
			}
		}

	}
	return nil
}
Exemple #4
0
// remoteName must be full URL.
func filerDownload(localName, remoteName, authToken string) (stat *contester_proto.FileStat, err error) {
	local, err := os.Create(localName)
	if err != nil {
		return nil, err
	}
	defer local.Close()
	req, err := http.NewRequest("GET", remoteName, nil)
	if err != nil {
		return nil, err
	}
	if authToken != "" {
		req.Header.Add("Authorization", "bearer "+authToken)
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if _, err = io.Copy(local, resp.Body); err != nil {
		return nil, err
	}
	local.Close()
	return tools.StatFile(localName, true)
}
Exemple #5
0
func (s *mongodbStorage) Copy(localName, remoteName string, toRemote bool, checksum, moduleType string) (stat *contester_proto.FileStat, err error) {
	ec := tools.ErrorContext("mongodb.Copy")

	if toRemote {
		stat, err = tools.StatFile(localName, true)
		if err != nil {
			err = ec.NewError(err, "local.CalculateChecksum")
		}
		// If file doesn't exist then stat == nil.
		if err != nil || stat == nil {
			return
		}

		if checksum != "" && *stat.Checksum != checksum {
			return nil, ec.NewError(fmt.Errorf("Checksum mismatch, local %s != %s", stat.Checksum, checksum))
		}

		checksum = *stat.Checksum
	}

	var local *os.File
	if toRemote {
		local, err = os.Open(localName)
	} else {
		local, err = os.Create(localName)
	}

	if err != nil {
		return nil, ec.NewError(err, "local.Open")
	}
	defer local.Close()

	var remote *mgo.GridFile
	if toRemote {
		// Remove all files with the same remoteName.
		if err = s.GridFS.Remove(remoteName); err != nil {
			return nil, ec.NewError(err, "remote.Remove")
		}
		remote, err = s.GridFS.Create(remoteName)
	} else {
		remote, err = s.GridFS.Open(remoteName)
	}
	if err != nil {
		return nil, ec.NewError(err, "remote.Open")
	}
	defer remote.Close()

	var source io.ReadCloser
	if toRemote {
		source = local
	} else {
		source = remote
		var meta fileMetadata
		if err = remote.GetMeta(&meta); err != nil {
			return nil, ec.NewError(err, "remote.GetMeta")
		}
		if meta.CompressionType == "ZLIB" {
			source, err = zlib.NewReader(source)
			if err != nil {
				return nil, ec.NewError(err, "zlib.NewReader")
			}
		}
	}

	var destination io.WriteCloser
	if toRemote {
		destination = zlib.NewWriter(remote)
	} else {
		destination = local
	}

	size, err := io.Copy(destination, source)
	if err != nil {
		return nil, ec.NewError(err, "io.Copy")
	}

	if toRemote {
		var meta fileMetadata
		meta.OriginalSize = uint64(size)
		meta.CompressionType = "ZLIB"
		meta.Checksum = *stat.Checksum
		meta.ModuleType = moduleType

		remote.SetMeta(meta)
	}

	if err = destination.Close(); err != nil {
		return nil, ec.NewError(err, "destination.Close")
	}

	if err = source.Close(); err != nil {
		return nil, ec.NewError(err, "source.Close")
	}

	if !toRemote {
		stat, err = tools.StatFile(localName, true)
		if err != nil {
			return nil, ec.NewError(err, "StatFile")
		}
	}

	return stat, nil
}