func filerUpload(localName, remoteName, checksum, moduleType, authToken string) (stat *contester_proto.FileStat, err error) {
ec := tools.ErrorContext("upload")
if stat, err = tools.StatFile(localName, true); err != nil || stat == nil {
return stat, err
}
if checksum != "" && stat.GetChecksum() != checksum {
return nil, fmt.Errorf("Checksum mismatch, local %s != %s", stat.GetChecksum(), checksum)
}
checksum = stat.GetChecksum()
local, err := os.Open(localName)
if err != nil {
return nil, ec.NewError(err, "local.Open")
}
defer local.Close()
req, err := http.NewRequest("PUT", remoteName, local)
if err != nil {
return nil, err
}
if moduleType != "" {
req.Header.Add("X-FS-Module-Type", moduleType)
}
if authToken != "" {
req.Header.Add("Authorization", "bearer "+authToken)
}
req.Header.Add("X-FS-Content-Length", strconv.FormatUint(stat.GetSize_(), 10))
var base64sha1 string
if checksum != "" && strings.HasPrefix(checksum, "sha1:") {
if data, err := hex.DecodeString(strings.TrimPrefix(checksum, "sha1:")); err == nil {
base64sha1 = base64.StdEncoding.EncodeToString(data)
req.Header.Add("Digest", "SHA="+base64sha1)
}
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
var st uploadStatus
err = json.NewDecoder(resp.Body).Decode(&st)
if err != nil {
return nil, err
}
if st.Size != int64(stat.GetSize_()) || (base64sha1 != "" && base64sha1 != st.Digests["SHA"]) {
return nil, fmt.Errorf("upload integrity verification failed")
}
return stat, nil
}
func (s *Contester) Put(request *contester_proto.FileBlob, response *contester_proto.FileStat) error {
ec := tools.ErrorContext("Put")
resolved, sandbox, err := resolvePath(s.Sandboxes, *request.Name, true)
if err != nil {
return ec.NewError(err, "resolvePath")
}
if sandbox != nil {
sandbox.Mutex.Lock()
defer sandbox.Mutex.Unlock()
}
var destination *os.File
for {
destination, err = os.Create(resolved)
loop, err := OnOsCreateError(err)
if err != nil {
return ec.NewError(err, "os.Create")
}
if !loop {
break
}
}
data, err := request.Data.Bytes()
if err != nil {
return ec.NewError(err, "request.Data.Bytes")
}
_, err = destination.Write(data)
if err != nil {
return ec.NewError(err, "destination.Write")
}
destination.Close()
if sandbox != nil {
return sandbox.Own(resolved)
}
stat, err := tools.StatFile(resolved, true)
if err != nil {
return ec.NewError(err, "statFile")
}
*response = *stat
return nil
}
func (s *Contester) Stat(request *contester_proto.StatRequest, response *contester_proto.FileStats) error {
ec := tools.ErrorContext("Stat")
if request.SandboxId != nil {
sandbox, err := getSandboxById(s.Sandboxes, *request.SandboxId)
if err != nil {
return ec.NewError(err, "getSandboxById")
}
sandbox.Mutex.RLock()
defer sandbox.Mutex.RUnlock()
}
response.Entries = make([]*contester_proto.FileStat, 0, len(request.Name))
for _, name := range request.Name {
resolved, _, err := resolvePath(s.Sandboxes, name, false)
if err != nil {
return ec.NewError(err, "resolvePath")
}
var expanded []string
if request.GetExpand() {
expanded, err = filepath.Glob(resolved)
if err != nil {
return ec.NewError(err, "filepath.Glob")
}
} else {
expanded = []string{resolved}
}
for _, name := range expanded {
stat, err := tools.StatFile(name, request.GetCalculateChecksum())
if err != nil {
return ec.NewError(err, "statFile")
}
if stat != nil {
response.Entries = append(response.Entries, stat)
}
}
}
return nil
}
// remoteName must be full URL.
func filerDownload(localName, remoteName, authToken string) (stat *contester_proto.FileStat, err error) {
local, err := os.Create(localName)
if err != nil {
return nil, err
}
defer local.Close()
req, err := http.NewRequest("GET", remoteName, nil)
if err != nil {
return nil, err
}
if authToken != "" {
req.Header.Add("Authorization", "bearer "+authToken)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if _, err = io.Copy(local, resp.Body); err != nil {
return nil, err
}
local.Close()
return tools.StatFile(localName, true)
}
func (s *mongodbStorage) Copy(localName, remoteName string, toRemote bool, checksum, moduleType string) (stat *contester_proto.FileStat, err error) {
ec := tools.ErrorContext("mongodb.Copy")
if toRemote {
stat, err = tools.StatFile(localName, true)
if err != nil {
err = ec.NewError(err, "local.CalculateChecksum")
}
// If file doesn't exist then stat == nil.
if err != nil || stat == nil {
return
}
if checksum != "" && *stat.Checksum != checksum {
return nil, ec.NewError(fmt.Errorf("Checksum mismatch, local %s != %s", stat.Checksum, checksum))
}
checksum = *stat.Checksum
}
var local *os.File
if toRemote {
local, err = os.Open(localName)
} else {
local, err = os.Create(localName)
}
if err != nil {
return nil, ec.NewError(err, "local.Open")
}
defer local.Close()
var remote *mgo.GridFile
if toRemote {
// Remove all files with the same remoteName.
if err = s.GridFS.Remove(remoteName); err != nil {
return nil, ec.NewError(err, "remote.Remove")
}
remote, err = s.GridFS.Create(remoteName)
} else {
remote, err = s.GridFS.Open(remoteName)
}
if err != nil {
return nil, ec.NewError(err, "remote.Open")
}
defer remote.Close()
var source io.ReadCloser
if toRemote {
source = local
} else {
source = remote
var meta fileMetadata
if err = remote.GetMeta(&meta); err != nil {
return nil, ec.NewError(err, "remote.GetMeta")
}
if meta.CompressionType == "ZLIB" {
source, err = zlib.NewReader(source)
if err != nil {
return nil, ec.NewError(err, "zlib.NewReader")
}
}
}
var destination io.WriteCloser
if toRemote {
destination = zlib.NewWriter(remote)
} else {
destination = local
}
size, err := io.Copy(destination, source)
if err != nil {
return nil, ec.NewError(err, "io.Copy")
}
if toRemote {
var meta fileMetadata
meta.OriginalSize = uint64(size)
meta.CompressionType = "ZLIB"
meta.Checksum = *stat.Checksum
meta.ModuleType = moduleType
remote.SetMeta(meta)
}
if err = destination.Close(); err != nil {
return nil, ec.NewError(err, "destination.Close")
}
if err = source.Close(); err != nil {
return nil, ec.NewError(err, "source.Close")
}
if !toRemote {
stat, err = tools.StatFile(localName, true)
if err != nil {
return nil, ec.NewError(err, "StatFile")
}
}
return stat, nil
}