func PreAuthRequest(ctx context.Context) { id := ctx.PathValue("id") if p, err := preauth.Load(id); err != nil { err_msg := "err:@preAuth load: " + err.Error() logger.Error(err_msg) responder.RespondWithError(ctx, 500, err_msg) return } else { if n, err := node.LoadUnauth(p.NodeId); err == nil { switch p.Type { case "download": filename := n.Id if fn, has := p.Options["filename"]; has { filename = fn } streamDownload(ctx, n, filename) preauth.Delete(id) return default: responder.RespondWithError(ctx, 500, "Preauthorization type not supported: "+p.Type) } } else { err_msg := "err:@preAuth loadnode: " + err.Error() logger.Error(err_msg) responder.RespondWithError(ctx, 500, err_msg) } } return }
// POST: /node func (cr *Controller) Create(cx *goweb.Context) { // Log Request and check for Auth request.Log(cx.Request) u, err := request.Authenticate(cx.Request) if err != nil && err.Error() != e.NoAuth { request.AuthError(err, cx) return } // Fake public user if u == nil { if conf.Bool(conf.Conf["anon-write"]) { u = &user.User{Uuid: ""} } else { cx.RespondWithErrorMessage(e.NoAuth, http.StatusUnauthorized) return } } // Parse uploaded form params, files, err := request.ParseMultipartForm(cx.Request) if err != nil { // If not multipart/form-data it will create an empty node. // TODO: create another request parser for non-multipart request // to handle this cleaner. if err.Error() == "request Content-Type isn't multipart/form-data" { n, err := node.CreateNodeUpload(u, params, files) if err != nil { logger.Error("Error at create empty: " + err.Error()) cx.RespondWithError(http.StatusInternalServerError) return } if n == nil { // Not sure how you could get an empty node with no error // Assume it's the user's fault cx.RespondWithError(http.StatusBadRequest) return } else { cx.RespondWithData(n) return } } else { // Some error other than request encoding. Theoretically // could be a lost db connection between user lookup and parsing. // Blame the user, Its probaby their fault anyway. logger.Error("Error parsing form: " + err.Error()) cx.RespondWithError(http.StatusBadRequest) return } } // Create node n, err := node.CreateNodeUpload(u, params, files) if err != nil { logger.Error("err@node_CreateNodeUpload: " + err.Error()) cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest) return } cx.RespondWithData(n) return }
// GET, POST, PUT, DELETE: /node/{nid}/acl/ // GET is the only action implemented here. func AclRequest(ctx context.Context) { nid := ctx.PathValue("nid") rmeth := ctx.HttpRequest().Method u, err := request.Authenticate(ctx.HttpRequest()) if err != nil && err.Error() != e.NoAuth { request.AuthError(err, ctx) return } // public user (no auth) can perform a GET operation with the proper node permissions if u == nil { if rmeth == "GET" && conf.ANON_READ { u = &user.User{Uuid: "public"} } else { responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth) return } } // Load node by id n, err := node.Load(nid) if err != nil { if err == mgo.ErrNotFound { responder.RespondWithError(ctx, http.StatusNotFound, e.NodeNotFound) return } else { // In theory the db connection could be lost between // checking user and load but seems unlikely. err_msg := "Err@node_Acl:LoadNode: " + nid + err.Error() logger.Error(err_msg) responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) return } } // Only the owner, an admin, or someone with read access can view acl's. // // NOTE: If the node is publicly owned, then anyone can view all acl's. The owner can only // be "public" when anonymous node creation (ANON_WRITE) is enabled in Shock config. rights := n.Acl.Check(u.Uuid) if n.Acl.Owner != u.Uuid && u.Admin == false && n.Acl.Owner != "public" && rights["read"] == false { responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth) return } if rmeth == "GET" { query := ctx.HttpRequest().URL.Query() verbosity := "" if _, ok := query["verbosity"]; ok { verbosity = query.Get("verbosity") } responder.RespondWithData(ctx, n.Acl.FormatDisplayAcl(verbosity)) } else { responder.RespondWithError(ctx, http.StatusNotImplemented, "This request type is not implemented.") } return }
func AuthError(err error, ctx context.Context) error { if err.Error() == e.InvalidAuth { return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid authorization header or content") } err_msg := "Error at Auth: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) }
// DELETE: /node/{id} func (cr *NodeController) Delete(id string, ctx context.Context) error { u, err := request.Authenticate(ctx.HttpRequest()) if err != nil && err.Error() != e.NoAuth { return request.AuthError(err, ctx) } // public user (no auth) can be used in some cases if u == nil { if conf.ANON_DELETE { u = &user.User{Uuid: "public"} } else { return responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth) } } // Load node by id n, err := node.Load(id) if err != nil { if err == mgo.ErrNotFound { return responder.RespondWithError(ctx, http.StatusNotFound, e.NodeNotFound) } else { // In theory the db connection could be lost between // checking user and load but seems unlikely. err_msg := "Err@node_Delete:LoadNode: " + id + ":" + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) } } rights := n.Acl.Check(u.Uuid) prights := n.Acl.Check("public") if rights["delete"] == false && u.Admin == false && n.Acl.Owner != u.Uuid && prights["delete"] == false { return responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth) } if err := n.Delete(); err == nil { return responder.RespondOK(ctx) } else { err_msg := "Err@node_Delete:Delete: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) } }
func streamDownload(ctx context.Context, n *node.Node, filename string) { nf, err := n.FileReader() defer nf.Close() if err != nil { // File not found or some sort of file read error. // Probably deserves more checking err_msg := "err:@preAuth node.FileReader: " + err.Error() logger.Error(err_msg) responder.RespondWithError(ctx, 500, err_msg) return } s := &request.Streamer{R: []file.SectionReader{nf}, W: ctx.HttpResponseWriter(), ContentType: "application/octet-stream", Filename: filename, Size: n.File.Size, Filter: nil} err = s.Stream() if err != nil { // causes "multiple response.WriteHeader calls" error but better than no response err_msg := "err:@preAuth: s.stream: " + err.Error() logger.Error(err_msg) responder.RespondWithError(ctx, 500, err_msg) } }
// handle download and its options func streamDownload(ctx context.Context, n *node.Node, options map[string]string) { nf, err := n.FileReader() defer nf.Close() if err != nil { // File not found or some sort of file read error. // Probably deserves more checking err_msg := "err:@preAuth node.FileReader: " + err.Error() logger.Error(err_msg) responder.RespondWithError(ctx, 500, err_msg) return } // set defaults filename := n.Id var filterFunc filter.FilterFunc = nil var compressionFormat string = "" // use options if exist if fn, has := options["filename"]; has { filename = fn } if fl, has := options["filter"]; has { if filter.Has(fl) { filterFunc = filter.Filter(fl) } } if cp, has := options["compression"]; has { if archive.IsValidCompress(cp) { compressionFormat = cp } } // stream it s := &request.Streamer{R: []file.SectionReader{nf}, W: ctx.HttpResponseWriter(), ContentType: "application/octet-stream", Filename: filename, Size: n.File.Size, Filter: filterFunc, Compression: compressionFormat} err = s.Stream(false) if err != nil { // causes "multiple response.WriteHeader calls" error but better than no response err_msg := "err:@preAuth: s.stream: " + err.Error() logger.Error(err_msg) responder.RespondWithError(ctx, 500, err_msg) } return }
func unZip(filePath string, unpackDir string) (fileList []FormFile, err error) { // open file with unzip zipReader, err := zip.OpenReader(filePath) if err != nil { return } // extract archive for _, zf := range zipReader.File { // set names path := filepath.Join(unpackDir, zf.Name) baseName := filepath.Base(zf.Name) // skip hidden if strings.HasPrefix(baseName, ".") { continue } if zf.FileInfo().IsDir() { // handle directory if merr := os.MkdirAll(path, 0777); merr != nil { logger.Error("err:@node_untar: " + err.Error()) } } else { // open output file writer, werr := os.Create(path) if werr != nil { return nil, werr } // open input stream zfh, zerr := zf.Open() if zerr != nil { writer.Close() return nil, zerr } // get md5 md5h := md5.New() dst := io.MultiWriter(writer, md5h) // write it _, err = io.Copy(dst, zfh) writer.Close() zfh.Close() if err != nil { return } // add to filelist ffile := FormFile{Name: baseName, Path: path, Checksum: make(map[string]string)} ffile.Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil)) fileList = append(fileList, ffile) } } return }
// DELETE: /node/{id} func (cr *NodeController) Delete(id string, ctx context.Context) error { u, err := request.Authenticate(ctx.HttpRequest()) if err != nil && err.Error() != e.NoAuth { return request.AuthError(err, ctx) } if u == nil { return responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth) } // Load node and handle user unauthorized n, err := node.Load(id, u.Uuid) if err != nil { if err.Error() == e.UnAuth { return responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth) } else if err.Error() == e.MongoDocNotFound { return responder.RespondWithError(ctx, http.StatusNotFound, "Node not found") } else { // In theory the db connection could be lost between // checking user and load but seems unlikely. err_msg := "Err@node_Read:Delete: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) } } rights := n.Acl.Check(u.Uuid) if !rights["delete"] { return responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth) } if err := n.Delete(); err == nil { return responder.RespondOK(ctx) } else { err_msg := "Err@node_Delete:Delete: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) } }
// DELETE: /node/{id} func (cr *Controller) Delete(id string, cx *goweb.Context) { request.Log(cx.Request) u, err := request.Authenticate(cx.Request) if err != nil && err.Error() != e.NoAuth { request.AuthError(err, cx) return } if u == nil { cx.RespondWithErrorMessage(e.NoAuth, http.StatusUnauthorized) return } // Load node and handle user unauthorized n, err := node.Load(id, u.Uuid) if err != nil { if err.Error() == e.UnAuth { cx.RespondWithError(http.StatusUnauthorized) return } else if err.Error() == e.MongoDocNotFound { cx.RespondWithNotFound() return } else { // In theory the db connection could be lost between // checking user and load but seems unlikely. logger.Error("Err@node_Read:Delete: " + err.Error()) cx.RespondWithError(http.StatusInternalServerError) return } } if err := n.Delete(); err == nil { cx.RespondWithOK() return } else { logger.Error("Err@node_Delet:Delete: " + err.Error()) cx.RespondWithError(http.StatusInternalServerError) } return }
// GET, POST, PUT, DELETE: /node/{nid}/acl/ // GET is the only action implemented here. func AclRequest(ctx context.Context) { nid := ctx.PathValue("nid") u, err := request.Authenticate(ctx.HttpRequest()) if err != nil && err.Error() != e.NoAuth { request.AuthError(err, ctx) return } // acl require auth even for public data if u == nil { responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth) return } // Load node and handle user unauthorized n, err := node.Load(nid, u.Uuid) if err != nil { if err.Error() == e.UnAuth { responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth) return } else if err.Error() == e.MongoDocNotFound { responder.RespondWithError(ctx, http.StatusNotFound, "Node not found") return } else { // In theory the db connection could be lost between // checking user and load but seems unlikely. err_msg := "Err@node_Read:LoadNode: " + err.Error() logger.Error(err_msg) responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) return } } rights := n.Acl.Check(u.Uuid) if ctx.HttpRequest().Method == "GET" { if u.Uuid == n.Acl.Owner || rights["read"] { responder.RespondWithData(ctx, n.Acl) } else { responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth) return } } else { responder.RespondWithError(ctx, http.StatusNotImplemented, "This request type is not implemented.") } return }
func (nr *NodeReaper) Handle() { waitDuration := time.Duration(conf.EXPIRE_WAIT) * time.Minute for { // sleep time.Sleep(waitDuration) // query to get expired nodes nodes := Nodes{} query := nr.getQuery() nodes.GetAll(query) // delete expired nodes for _, n := range nodes { logger.Info("access", "Deleting expired node: "+n.Id) if err := n.Delete(); err != nil { err_msg := "err:@node_delete: " + err.Error() logger.Error(err_msg) } } } }
// authToken validiates token by fetching user information. func authToken(t string) (u *user.User, err error) { client := &http.Client{ Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}, } req, err := http.NewRequest("GET", conf.AUTH_MGRAST_OAUTH_URL, nil) if err != nil { return nil, err } req.Header.Add("Auth", t) if resp, err := client.Do(req); err == nil { defer resp.Body.Close() if resp.StatusCode == http.StatusOK { if body, err := ioutil.ReadAll(resp.Body); err == nil { u = &user.User{} c := &credentials{} if err = json.Unmarshal(body, &c); err != nil { return nil, err } else { if c.Uname == "" { return nil, errors.New(e.InvalidAuth) } u.Username = c.Uname u.Fullname = c.Fname + " " + c.Lname u.Email = c.Email if err = u.SetMongoInfo(); err != nil { return nil, err } } } } else if resp.StatusCode == http.StatusForbidden { return nil, errors.New(e.InvalidAuth) } else { err_str := "Authentication failed: Unexpected response status: " + resp.Status logger.Error(err_str) return nil, errors.New(err_str) } } else { return nil, err } return }
func FilesFromArchive(format string, filePath string) (fileList []FormFile, unpackDir string, err error) { // set unpack dir unpackDir = fmt.Sprintf("%s/temp/%d%d", conf.PATH_DATA, rand.Int(), rand.Int()) if merr := os.Mkdir(unpackDir, 0777); merr != nil { logger.Error("err:@node_unpack: " + err.Error()) } // magic to unpack archive if format == "zip" { fileList, err = unZip(filePath, unpackDir) } else if format == "tar" { fileList, err = unTar(filePath, unpackDir, "") } else if format == "tar.gz" { fileList, err = unTar(filePath, unpackDir, "gzip") } else if format == "tar.bz2" { fileList, err = unTar(filePath, unpackDir, "bzip2") } else { return nil, unpackDir, errors.New("invalid archive format. must be one of: " + ArchiveList) } return }
// fetchProfile validiates token by using it to fetch user profile func fetchProfile(t string) (u *user.User, err error) { client := &http.Client{ Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}, } req, err := http.NewRequest("GET", conf.AUTH_GLOBUS_PROFILE_URL+"/"+clientId(t), nil) if err != nil { return nil, err } req.Header.Add("Authorization", "Globus-Goauthtoken "+t) if resp, err := client.Do(req); err == nil { defer resp.Body.Close() if resp.StatusCode == http.StatusOK { if body, err := ioutil.ReadAll(resp.Body); err == nil { u = &user.User{} if err = json.Unmarshal(body, &u); err != nil { return nil, err } else { if u.Username == "" { return nil, errors.New(e.InvalidAuth) } if err = u.SetMongoInfo(); err != nil { return nil, err } } } } else if resp.StatusCode == http.StatusForbidden { return nil, errors.New(e.InvalidAuth) } else { err_str := "Authentication failed: Unexpected response status: " + resp.Status logger.Error(err_str) return nil, errors.New(err_str) } } else { return nil, err } return }
// POST: /node func (cr *NodeController) Create(ctx context.Context) error { u, err := request.Authenticate(ctx.HttpRequest()) if err != nil && err.Error() != e.NoAuth { return request.AuthError(err, ctx) } // Fake public user if u == nil { if conf.Bool(conf.Conf["anon-write"]) { u = &user.User{Uuid: ""} } else { return responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth) } } // Parse uploaded form params, files, err := request.ParseMultipartForm(ctx.HttpRequest()) if err != nil { if err.Error() == "request Content-Type isn't multipart/form-data" { // If not multipart/form-data it will try to read the Body of the // request. If the Body is not empty it will create a file from // the Body contents. If the Body is empty it will create an empty // node. if ctx.HttpRequest().ContentLength != 0 { params, files, err = request.DataUpload(ctx.HttpRequest()) if err != nil { err_msg := "Error uploading data from request body:" + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) } } n, cn_err := node.CreateNodeUpload(u, params, files) if cn_err != nil { err_msg := "Error at create empty node: " + cn_err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) } if n == nil { // Not sure how you could get an empty node with no error // Assume it's the user's fault return responder.RespondWithError(ctx, http.StatusBadRequest, "Error, could not create node.") } else { return responder.RespondWithData(ctx, n) } } else { // Some error other than request encoding. Theoretically // could be a lost db connection between user lookup and parsing. // Blame the user, Its probaby their fault anyway. err_msg := "Error parsing form: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } } // Create node n, err := node.CreateNodeUpload(u, params, files) if err != nil { err_msg := "err@node_CreateNodeUpload: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } return responder.RespondWithData(ctx, n) }
// GET, POST, PUT, DELETE: /node/{nid}/acl/ // GET is the only action implemented here. func AclRequest(ctx context.Context) { nid := ctx.PathValue("nid") u, err := request.Authenticate(ctx.HttpRequest()) if err != nil && err.Error() != e.NoAuth { request.AuthError(err, ctx) return } // acl require auth even for public data if u == nil { responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth) return } // Load node and handle user unauthorized n, err := node.LoadUnauth(nid) if err != nil { if err.Error() == e.MongoDocNotFound { responder.RespondWithError(ctx, http.StatusNotFound, "Node not found") return } else { // In theory the db connection could be lost between // checking user and load but seems unlikely. err_msg := "Err@node_Read:LoadNode: " + err.Error() logger.Error(err_msg) responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) return } } // only the owner can view/edit acl's unless owner="" or owner=nil // note: owner can only be empty when anonymous node creation is enabled in shock config. if n.Acl.Owner != u.Uuid && n.Acl.Owner != "" { err_msg := "Only the node owner can edit/view node ACL's" logger.Error(err_msg) responder.RespondWithError(ctx, http.StatusUnauthorized, err_msg) return } if ctx.HttpRequest().Method == "GET" { if u.Uuid == n.Acl.Owner || rights["read"] { // this is pretty clumsy and I could have done it with a lot less code and cleaner // in perl or js, but I am not so familiar with GO yet (it always hurts the first time) // the return structure should probably not be a concatenated string, but rather a hash // for each entry containing Username, Fullname and Uuid // owner cu, err := user.FindByUuid(n.Acl.Owner) if err != nil { responder.RespondWithError(ctx, http.StatusInternalServerError, "Err@Uuid_Resolve: "+err.Error()) return } else { n.Acl.Owner = cu.Username + "|" + cu.Uuid } // read p1 := []string{} for _, v := range n.Acl.Read { cu, err := user.FindByUuid(v) if err != nil { responder.RespondWithError(ctx, http.StatusInternalServerError, "Err@Uuid_Resolve: "+err.Error()) return } else { p1 = append(p1, cu.Username + "|" + cu.Uuid) } } n.Acl.Read = p1 // write p2 := []string{} for _, v := range n.Acl.Write { cu, err := user.FindByUuid(v) if err != nil { responder.RespondWithError(ctx, http.StatusInternalServerError, "Err@Uuid_Resolve: "+err.Error()) return } else { p2 = append(p2, cu.Username + "|" + cu.Uuid) } } n.Acl.Write = p2 // delete p3 := []string{} for _, v := range n.Acl.Delete { cu, err := user.FindByUuid(v) if err != nil { responder.RespondWithError(ctx, http.StatusInternalServerError, "Err@Uuid_Resolve: "+err.Error()) return } else { p3 = append(p3, cu.Username + "|" + cu.Uuid) } } n.Acl.Delete = p3 responder.RespondWithData(ctx, n.Acl) } else { responder.RespondWithError(ctx, http.StatusNotImplemented, "This request type is not implemented.") } return } // GET, POST, PUT, DELETE: /node/{nid}/acl/{type} func AclTypedRequest(ctx context.Context) { nid := ctx.PathValue("nid") rtype := ctx.PathValue("type") u, err := request.Authenticate(ctx.HttpRequest()) if err != nil && err.Error() != e.NoAuth { request.AuthError(err, ctx) return } if !validAclTypes[rtype] { responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid acl type") return } // acl require auth even for public data if u == nil { responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth) return } // Load node and handle user unauthorized n, err := node.LoadUnauth(nid) if err != nil { if err.Error() == e.MongoDocNotFound { responder.RespondWithError(ctx, http.StatusNotFound, "Node not found") return } else { // In theory the db connection could be lost between // checking user and load but seems unlikely. err_msg := "Err@node_Read:LoadNode: " + err.Error() logger.Error(err_msg) responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) return } } // only the owner can view/edit acl's unless owner="" or owner=nil // note: owner can only be empty when anonymous node creation is enabled in shock config. if n.Acl.Owner != u.Uuid && n.Acl.Owner != "" { err_msg := "Only the node owner can edit/view node ACL's" logger.Error(err_msg) responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) return } requestMethod := ctx.HttpRequest().Method if requestMethod != "GET" { ids, err := parseAclRequestTyped(ctx) if err != nil { responder.RespondWithError(ctx, http.StatusBadRequest, err.Error()) return } if requestMethod == "POST" || requestMethod == "PUT" { if rtype == "owner" { if len(ids) == 1 { n.Acl.SetOwner(ids[0]) } else { responder.RespondWithError(ctx, http.StatusBadRequest, "Too many users. Nodes may have only one owner.") return } } else if rtype == "all" { for _, atype := range []string{"read", "write", "delete"} { for _, i := range ids { n.Acl.Set(i, map[string]bool{atype: true}) } } } else { for _, i := range ids { n.Acl.Set(i, map[string]bool{rtype: true}) } } n.Save() } else if requestMethod == "DELETE" { if rtype == "owner" { responder.RespondWithError(ctx, http.StatusBadRequest, "Deleting ownership is not a supported request type.") return } else if rtype == "all" { for _, atype := range []string{"read", "write", "delete"} { for _, i := range ids { n.Acl.UnSet(i, map[string]bool{atype: true}) } } } else { for _, i := range ids { n.Acl.UnSet(i, map[string]bool{rtype: true}) } } n.Save() } else { responder.RespondWithError(ctx, http.StatusNotImplemented, "This request type is not implemented.") return } } switch rtype { default: responder.RespondWithError(ctx, http.StatusNotImplemented, "This request type is not implemented.") case "read": responder.RespondWithData(ctx, map[string][]string{"read": n.Acl.Read}) case "write": responder.RespondWithData(ctx, map[string][]string{"write": n.Acl.Write}) case "delete": responder.RespondWithData(ctx, map[string][]string{"delete": n.Acl.Delete}) case "owner": responder.RespondWithData(ctx, map[string]string{"owner": n.Acl.Owner}) case "all": responder.RespondWithData(ctx, n.Acl) } return } func parseAclRequestTyped(ctx context.Context) (ids []string, err error) { var users []string query := ctx.HttpRequest().URL.Query() params, _, err := request.ParseMultipartForm(ctx.HttpRequest()) if _, ok := query["users"]; ok && err != nil && err.Error() == "request Content-Type isn't multipart/form-data" { users = strings.Split(query.Get("users"), ",") } else if params["users"] != "" { users = strings.Split(params["users"], ",") } else { return nil, errors.New("Action requires list of comma separated usernames in 'users' parameter") } for _, v := range users { if uuid.Parse(v) != nil { ids = append(ids, v) } else { u := user.User{Username: v} if err := u.SetMongoInfo(); err != nil { return nil, err } ids = append(ids, u.Uuid) } } return ids, nil }
// PUT: /node/{id} -> multipart-form func (cr *NodeController) Replace(id string, ctx context.Context) error { u, err := request.Authenticate(ctx.HttpRequest()) if err != nil && err.Error() != e.NoAuth { return request.AuthError(err, ctx) } // public user (no auth) can be used in some cases if u == nil { if conf.ANON_WRITE { u = &user.User{Uuid: "public"} } else { return responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth) } } // Load node by id n, err := node.Load(id) if err != nil { if err == mgo.ErrNotFound { return responder.RespondWithError(ctx, http.StatusNotFound, e.NodeNotFound) } else { // In theory the db connection could be lost between // checking user and load but seems unlikely. err_msg := "Err@node_Update:LoadNode: " + id + ":" + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) } } rights := n.Acl.Check(u.Uuid) prights := n.Acl.Check("public") if rights["write"] == false && u.Admin == false && n.Acl.Owner != u.Uuid && prights["write"] == false { return responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth) } if conf.LOG_PERF { logger.Perf("START PUT data: " + id) } params, files, err := request.ParseMultipartForm(ctx.HttpRequest()) // clean up temp dir !! defer node.RemoveAllFormFiles(files) if err != nil { err_msg := "err@node_ParseMultipartForm: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } // need delete rights to set expiration if _, hasExpiration := params["expiration"]; hasExpiration { if rights["delete"] == false && u.Admin == false && n.Acl.Owner != u.Uuid && prights["delete"] == false { return responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth) } } if _, hasCopyData := params["copy_data"]; hasCopyData { _, err = node.Load(params["copy_data"]) if err != nil { return request.AuthError(err, ctx) } } if _, hasParentNode := params["parent_node"]; hasParentNode { _, err = node.Load(params["parent_node"]) if err != nil { return request.AuthError(err, ctx) } } err = n.Update(params, files) if err != nil { err_msg := "err@node_Update: " + id + ": " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } responder.RespondWithData(ctx, n) if conf.LOG_PERF { logger.Perf("END PUT data: " + id) } return nil }
// PUT: /node/{id} -> multipart-form func (cr *Controller) Update(id string, cx *goweb.Context) { // Log Request and check for Auth request.Log(cx.Request) u, err := request.Authenticate(cx.Request) if err != nil && err.Error() != e.NoAuth { request.AuthError(err, cx) return } // Gather query params query := util.Q(cx.Request.URL.Query()) // Fake public user if u == nil { u = &user.User{Uuid: ""} } n, err := node.Load(id, u.Uuid) if err != nil { if err.Error() == e.UnAuth { cx.RespondWithError(http.StatusUnauthorized) return } else if err.Error() == e.MongoDocNotFound { cx.RespondWithNotFound() return } else { // In theory the db connection could be lost between // checking user and load but seems unlikely. logger.Error("Err@node_Update:LoadNode: " + err.Error()) cx.RespondWithError(http.StatusInternalServerError) return } } if query.Has("index") { if conf.Bool(conf.Conf["perf-log"]) { logger.Perf("START indexing: " + id) } if !n.HasFile() { cx.RespondWithErrorMessage("node file empty", http.StatusBadRequest) return } if query.Value("index") == "bai" { //bam index is created by the command-line tool samtools if ext := n.FileExt(); ext == ".bam" { if err := index.CreateBamIndex(n.FilePath()); err != nil { cx.RespondWithErrorMessage("Error while creating bam index", http.StatusBadRequest) return } return } else { cx.RespondWithErrorMessage("Index type bai requires .bam file", http.StatusBadRequest) return } } idxtype := query.Value("index") if _, ok := index.Indexers[idxtype]; !ok { cx.RespondWithErrorMessage("invalid index type", http.StatusBadRequest) return } newIndexer := index.Indexers[idxtype] f, _ := os.Open(n.FilePath()) defer f.Close() idxer := newIndexer(f) count, err := idxer.Create() if err != nil { logger.Error("err " + err.Error()) cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest) return } if err := idxer.Dump(n.IndexPath() + "/" + query.Value("index") + ".idx"); err != nil { logger.Error("err " + err.Error()) cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest) return } idxInfo := node.IdxInfo{ Type: query.Value("index"), TotalUnits: count, AvgUnitSize: n.File.Size / count, } if idxtype == "chunkrecord" { idxInfo.AvgUnitSize = conf.CHUNK_SIZE } if err := n.SetIndexInfo(query.Value("index"), idxInfo); err != nil { logger.Error("[email protected]: " + err.Error()) } if conf.Bool(conf.Conf["perf-log"]) { logger.Perf("END indexing: " + id) } cx.RespondWithOK() return } else { if conf.Bool(conf.Conf["perf-log"]) { logger.Perf("START PUT data: " + id) } params, files, err := request.ParseMultipartForm(cx.Request) if err != nil { logger.Error("err@node_ParseMultipartForm: " + err.Error()) cx.RespondWithError(http.StatusBadRequest) return } err = n.Update(params, files) if err != nil { errors := []string{e.FileImut, e.AttrImut, "parts cannot be less than 1"} for e := range errors { if err.Error() == errors[e] { cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest) return } } logger.Error("err@node_Update: " + id + ":" + err.Error()) cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest) return } cx.RespondWithData(n) if conf.Bool(conf.Conf["perf-log"]) { logger.Perf("END PUT data: " + id) } } return }
// GET, PUT, DELETE: /node/{nid}/index/{idxType} func IndexTypedRequest(ctx context.Context) { nid := ctx.PathValue("nid") idxType := ctx.PathValue("idxType") u, err := request.Authenticate(ctx.HttpRequest()) if err != nil && err.Error() != e.NoAuth { request.AuthError(err, ctx) return } // Fake public user if u == nil { u = &user.User{Uuid: ""} } // Load node and handle user unauthorized n, err := node.Load(nid, u.Uuid) if err != nil { if err.Error() == e.UnAuth { responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth) return } else if err.Error() == e.MongoDocNotFound { responder.RespondWithError(ctx, http.StatusNotFound, "Node not found.") return } else { // In theory the db connection could be lost between // checking user and load but seems unlikely. err_msg := "Err@index:LoadNode: " + err.Error() logger.Error(err_msg) responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) return } } switch ctx.HttpRequest().Method { case "DELETE": rights := n.Acl.Check(u.Uuid) if !rights["write"] { responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth) return } if _, has := n.Indexes[idxType]; has { if err := n.DeleteIndex(idxType); err != nil { err_msg := err.Error() logger.Error(err_msg) responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) } responder.RespondOK(ctx) } else { responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Node %s does not have index of type %s.", n.Id, idxType)) } case "GET": if v, has := n.Indexes[idxType]; has { responder.RespondWithData(ctx, map[string]interface{}{idxType: v}) } else { responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Node %s does not have index of type %s.", n.Id, idxType)) } case "PUT": rights := n.Acl.Check(u.Uuid) if !rights["write"] { responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth) return } // Gather query params query := ctx.HttpRequest().URL.Query() _, forceRebuild := query["force_rebuild"] if _, has := n.Indexes[idxType]; has { if idxType == "size" { responder.RespondOK(ctx) return } else if !forceRebuild { responder.RespondWithError(ctx, http.StatusBadRequest, "This index already exists, please add the parameter 'force_rebuild=1' to force a rebuild of the existing index.") return } } if !n.HasFile() { responder.RespondWithError(ctx, http.StatusBadRequest, "Node has no file.") return } else if idxType == "" { responder.RespondWithError(ctx, http.StatusBadRequest, "Index create requires type.") return } if _, ok := index.Indexers[idxType]; !ok && idxType != "bai" && idxType != "subset" && idxType != "column" { responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Index type %s unavailable.", idxType)) return } if idxType == "size" { responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Index type size is a virtual index and does not require index building.")) return } if conf.Bool(conf.Conf["perf-log"]) { logger.Perf("START indexing: " + nid) } if idxType == "bai" { //bam index is created by the command-line tool samtools if n.Type == "subset" { responder.RespondWithError(ctx, http.StatusBadRequest, "Shock does not support bam index creation on subset nodes.") return } if ext := n.FileExt(); ext == ".bam" { if err := index.CreateBamIndex(n.FilePath()); err != nil { responder.RespondWithError(ctx, http.StatusInternalServerError, "Error while creating bam index.") return } responder.RespondOK(ctx) return } else { responder.RespondWithError(ctx, http.StatusBadRequest, "Index type bai requires .bam file.") return } } subsetSize := int64(0) count := int64(0) indexFormat := "" subsetName := "" if idxType == "subset" { // Utilizing the multipart form parser since we need to upload a file. params, files, err := request.ParseMultipartForm(ctx.HttpRequest()) if err != nil { responder.RespondWithError(ctx, http.StatusBadRequest, err.Error()) return } parentIndex, hasParent := params["parent_index"] if !hasParent { responder.RespondWithError(ctx, http.StatusBadRequest, "Index type subset requires parent_index param.") return } else if _, has := n.Indexes[parentIndex]; !has { responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Node %s does not have index of type %s.", n.Id, parentIndex)) return } newIndex, hasName := params["index_name"] if !hasName { responder.RespondWithError(ctx, http.StatusBadRequest, "Index type subset requires index_name param.") return } else if _, reservedName := index.Indexers[newIndex]; reservedName || newIndex == "bai" { responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("%s is a reserved index name and cannot be used to create a custom subset index.", newIndex)) return } subsetName = newIndex subsetIndices, hasFile := files["subset_indices"] if !hasFile { responder.RespondWithError(ctx, http.StatusBadRequest, "Index type subset requires subset_indices file.") return } f, _ := os.Open(subsetIndices.Path) defer f.Close() idxer := index.NewSubsetIndexer(f) // we default to "array" index format for backwards compatibility indexFormat = "array" if n.Indexes[parentIndex].Format == "array" || n.Indexes[parentIndex].Format == "matrix" { indexFormat = n.Indexes[parentIndex].Format } count, subsetSize, err = index.CreateSubsetIndex(&idxer, n.IndexPath()+"/"+newIndex+".idx", n.IndexPath()+"/"+parentIndex+".idx", indexFormat, n.Indexes[parentIndex].TotalUnits) if err != nil { logger.Error("err " + err.Error()) responder.RespondWithError(ctx, http.StatusBadRequest, err.Error()) return } } else if idxType == "column" { // Gather query params query := ctx.HttpRequest().URL.Query() if n.Type == "subset" { responder.RespondWithError(ctx, http.StatusBadRequest, "Shock does not support column index creation on subset nodes.") return } if _, exists := query["number"]; !exists { err_msg := "Index type column requires a number parameter in the url." logger.Error(err_msg) responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) return } num_str := query.Get("number") idxType = idxType + num_str num, err := strconv.Atoi(num_str) if err != nil || num < 1 { err_msg := "Index type column requires a number parameter in the url of an integer greater than zero." logger.Error(err_msg) responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) return } f, _ := os.Open(n.FilePath()) defer f.Close() idxer := index.NewColumnIndexer(f) count, indexFormat, err = index.CreateColumnIndex(&idxer, num, n.IndexPath()+"/"+idxType+".idx") if err != nil { logger.Error("err " + err.Error()) responder.RespondWithError(ctx, http.StatusBadRequest, err.Error()) return } } else { if n.Type == "subset" && (idxType != "chunkrecord" || n.Subset.Parent.IndexName != "record") { responder.RespondWithError(ctx, http.StatusBadRequest, "For subset nodes, Shock currently only supports subset and chunkrecord indexes. Also, for a chunkrecord index, the subset node must have been generated from a record index.") return } newIndexer := index.Indexers[idxType] f, _ := os.Open(n.FilePath()) defer f.Close() var idxer index.Indexer if n.Type == "subset" { idxer = newIndexer(f, n.Type, n.Subset.Index.Format, n.IndexPath()+"/"+n.Subset.Parent.IndexName+".idx") } else { idxer = newIndexer(f, n.Type, "", "") } count, indexFormat, err = idxer.Create(n.IndexPath() + "/" + idxType + ".idx") if err != nil { logger.Error("err " + err.Error()) responder.RespondWithError(ctx, http.StatusBadRequest, err.Error()) return } } if count == 0 { responder.RespondWithError(ctx, http.StatusBadRequest, "Index empty.") return } idxInfo := node.IdxInfo{ Type: idxType, TotalUnits: count, AvgUnitSize: n.File.Size / count, Format: indexFormat, } //if idxType == "chunkrecord" { // idxInfo.AvgUnitSize = conf.CHUNK_SIZE //} if idxType == "subset" { idxType = subsetName idxInfo.AvgUnitSize = subsetSize / count } if err := n.SetIndexInfo(idxType, idxInfo); err != nil { logger.Error("[email protected]: " + err.Error()) } if conf.Bool(conf.Conf["perf-log"]) { logger.Perf("END indexing: " + nid) } responder.RespondOK(ctx) return default: responder.RespondWithError(ctx, http.StatusNotImplemented, "This request type is not implemented.") } return }
// GET: /node/{id} func (cr *NodeController) Read(id string, ctx context.Context) error { u, err := request.Authenticate(ctx.HttpRequest()) if err != nil && err.Error() != e.NoAuth { return request.AuthError(err, ctx) } // Fake public user if u == nil { if conf.Bool(conf.Conf["anon-read"]) { u = &user.User{Uuid: ""} } else { return responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth) } } // Gather query params query := ctx.HttpRequest().URL.Query() var fFunc filter.FilterFunc = nil if _, ok := query["filter"]; ok { if filter.Has(query.Get("filter")) { fFunc = filter.Filter(query.Get("filter")) } } // Load node and handle user unauthorized n, err := node.Load(id, u.Uuid) if err != nil { if err.Error() == e.UnAuth { return responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth) } else if err.Error() == e.MongoDocNotFound { return responder.RespondWithError(ctx, http.StatusNotFound, "Node not found") } else { // In theory the db connection could be lost between // checking user and load but seems unlikely. logger.Error("Err@node_Read:LoadNode:" + id + ":" + err.Error()) n, err = node.LoadFromDisk(id) if err != nil { err_msg := "Err@node_Read:LoadNodeFromDisk:" + id + ":" + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) } } } // Switch though param flags // ?download=1 if _, ok := query["download"]; ok { if !n.HasFile() { return responder.RespondWithError(ctx, http.StatusBadRequest, "Node has no file") } filename := n.Id if _, ok := query["filename"]; ok { filename = query.Get("filename") } if _, ok := query["index"]; ok { //handling bam file if query.Get("index") == "bai" { s := &request.Streamer{R: []file.SectionReader{}, W: ctx.HttpResponseWriter(), ContentType: "application/octet-stream", Filename: filename, Size: n.File.Size, Filter: fFunc} var region string if _, ok := query["region"]; ok { //retrieve alingments overlapped with specified region region = query.Get("region") } argv, err := request.ParseSamtoolsArgs(ctx) if err != nil { return responder.RespondWithError(ctx, http.StatusBadRequest, "Invaid args in query url") } err = s.StreamSamtools(n.FilePath(), region, argv...) if err != nil { return responder.RespondWithError(ctx, http.StatusBadRequest, "error while involking samtools") } return nil } // if forgot ?part=N if _, ok := query["part"]; !ok { return responder.RespondWithError(ctx, http.StatusBadRequest, "Index parameter requires part parameter") } // open file r, err := n.FileReader() defer r.Close() if err != nil { err_msg := "Err@node_Read:Open: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) } // load index idx, err := n.Index(query.Get("index")) if err != nil { return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index") } if idx.Type() == "virtual" { csize := conf.CHUNK_SIZE if _, ok := query["chunk_size"]; ok { csize, err = strconv.ParseInt(query.Get("chunk_size"), 10, 64) if err != nil { return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid chunk_size") } } idx.Set(map[string]interface{}{"ChunkSize": csize}) } var size int64 = 0 s := &request.Streamer{R: []file.SectionReader{}, W: ctx.HttpResponseWriter(), ContentType: "application/octet-stream", Filename: filename, Filter: fFunc} for _, p := range query["part"] { pos, length, err := idx.Part(p) if err != nil { return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index part") } size += length s.R = append(s.R, io.NewSectionReader(r, pos, length)) } s.Size = size err = s.Stream() if err != nil { // causes "multiple response.WriteHeader calls" error but better than no response err_msg := "err:@node_Read s.stream: " + err.Error() logger.Error(err_msg) responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } } else { nf, err := n.FileReader() defer nf.Close() if err != nil { // File not found or some sort of file read error. // Probably deserves more checking err_msg := "err:@node_Read node.FileReader: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } s := &request.Streamer{R: []file.SectionReader{nf}, W: ctx.HttpResponseWriter(), ContentType: "application/octet-stream", Filename: filename, Size: n.File.Size, Filter: fFunc} err = s.Stream() if err != nil { // causes "multiple response.WriteHeader calls" error but better than no response err_msg := "err:@node_Read: s.stream: " + err.Error() logger.Error(err_msg) responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } } } else if _, ok := query["download_url"]; ok { if !n.HasFile() { return responder.RespondWithError(ctx, http.StatusBadRequest, "Node has not file") } else if u.Uuid == "" { return responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth) } else { options := map[string]string{} if _, ok := query["filename"]; ok { options["filename"] = query.Get("filename") } if p, err := preauth.New(util.RandString(20), "download", n.Id, options); err != nil { err_msg := "err:@node_Read download_url: " + err.Error() logger.Error(err_msg) responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) } else { responder.RespondWithData(ctx, util.UrlResponse{Url: util.ApiUrl(ctx) + "/preauth/" + p.Id, ValidTill: p.ValidTill.Format(time.ANSIC)}) } } } else { // Base case respond with node in json responder.RespondWithData(ctx, n) } return nil }
// GET: /node // To do: // - Iterate node queries func (cr *NodeController) ReadMany(ctx context.Context) error { u, err := request.Authenticate(ctx.HttpRequest()) if err != nil && err.Error() != e.NoAuth { return request.AuthError(err, ctx) } // Gather query params query := ctx.HttpRequest().URL.Query() // Setup query and nodes objects q := bson.M{} nodes := node.Nodes{} if u != nil { // Admin sees all if !u.Admin { q["$or"] = []bson.M{bson.M{"acl.read": []string{}}, bson.M{"acl.read": u.Uuid}, bson.M{"acl.owner": u.Uuid}} } } else { if conf.Bool(conf.Conf["anon-read"]) { // select on only nodes with no read rights set q["acl.read"] = []string{} } else { return responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth) } } // Gather params to make db query. Do not include the // following list. paramlist := map[string]int{"limit": 1, "offset": 1, "query": 1, "querynode": 1, "owner": 1, "read": 1, "write": 1, "delete": 1, "public_owner": 1, "public_read": 1, "public_write": 1, "public_delete": 1} if _, ok := query["query"]; ok { for key := range query { if _, found := paramlist[key]; !found { keyStr := fmt.Sprintf("attributes.%s", key) value := query.Get(key) if value != "" { if numValue, err := strconv.Atoi(value); err == nil { q["$or"] = ListOfMaps{{keyStr: value}, {keyStr: numValue}} } else if value == "null" { q["$or"] = ListOfMaps{{keyStr: value}, {keyStr: nil}} } else { q[keyStr] = value } } else { existsMap := map[string]bool{ "$exists": true, } q[keyStr] = existsMap } } } } else if _, ok := query["querynode"]; ok { for key := range query { if _, found := paramlist[key]; !found { value := query.Get(key) if value != "" { if numValue, err := strconv.Atoi(value); err == nil { q["$or"] = ListOfMaps{{key: value}, {key: numValue}} } else if value == "null" { q["$or"] = ListOfMaps{{key: value}, {key: nil}} } else { q[key] = value } } else { existsMap := map[string]bool{ "$exists": true, } q[key] = existsMap } } } } // defaults limit := 25 offset := 0 if _, ok := query["limit"]; ok { limit = util.ToInt(query.Get("limit")) } if _, ok := query["offset"]; ok { offset = util.ToInt(query.Get("offset")) } // Allowing user to query based on ACL's with a comma-separated list of users. // Users can be written as a username or a UUID. for _, atype := range []string{"owner", "read", "write", "delete"} { if _, ok := query[atype]; ok { users := strings.Split(query.Get(atype), ",") for _, v := range users { if uuid.Parse(v) != nil { q["acl."+atype] = v } else { u := user.User{Username: v} if err := u.SetMongoInfo(); err != nil { err_msg := "err " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } q["acl."+atype] = u.Uuid } } } } if _, ok := query["public_owner"]; ok { // If search is for public_owner AND owner = non-empty string then return zero nodes, no nodes can match this query. if _, exists := q["acl.owner"]; exists { return responder.RespondWithPaginatedData(ctx, nodes, limit, offset, 0) } q["acl.owner"] = "" } // Allowing users to query based on whether ACL is public for _, atype := range []string{"read", "write", "delete"} { if _, ok := query["public_"+atype]; ok { sizeMap := map[string]int{ "$size": 0, } // Currently a node cannot be public and have an ACL at the same time, so this returns zero nodes. if _, exists := q["acl."+atype]; exists { return responder.RespondWithPaginatedData(ctx, nodes, limit, offset, 0) } else { q["acl."+atype] = sizeMap } } } // Get nodes from db count, err := nodes.GetPaginated(q, limit, offset) if err != nil { err_msg := "err " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } return responder.RespondWithPaginatedData(ctx, nodes, limit, offset, count) }
// GET: /node // To do: // - Iterate node queries func (cr *NodeController) ReadMany(ctx context.Context) error { u, err := request.Authenticate(ctx.HttpRequest()) if err != nil && err.Error() != e.NoAuth { return request.AuthError(err, ctx) } // Gather query params query := ctx.HttpRequest().URL.Query() // Setup query and nodes objects q := bson.M{} nodes := node.Nodes{} if u != nil { // Admin sees all if !u.Admin { q["$or"] = []bson.M{bson.M{"acl.read": []string{}}, bson.M{"acl.read": u.Uuid}, bson.M{"acl.owner": u.Uuid}} } } else { if conf.Bool(conf.Conf["anon-read"]) { // select on only nodes with no read rights set q["acl.read"] = []string{} } else { return responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth) } } // Gather params to make db query. Do not include the // following list. paramlist := map[string]int{"limit": 1, "offset": 1, "query": 1, "querynode": 1} if _, ok := query["query"]; ok { for key := range query { if _, found := paramlist[key]; !found { q[fmt.Sprintf("attributes.%s", key)] = query.Get(key) } } } else if _, ok := query["querynode"]; ok { for key := range query { if key == "type" { querytypes := strings.Split(query.Get(key), ",") q["type"] = bson.M{"$all": querytypes} } else { if _, found := paramlist[key]; !found { q[key] = query.Get(key) } } } } // defaults limit := 25 offset := 0 if _, ok := query["limit"]; ok { limit = util.ToInt(query.Get("limit")) } if _, ok := query["offset"]; ok { offset = util.ToInt(query.Get("offset")) } // Get nodes from db count, err := nodes.GetPaginated(q, limit, offset) if err != nil { err_msg := "err " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } return responder.RespondWithPaginatedData(ctx, nodes, limit, offset, count) }
// GET: /node // To do: // - Iterate node queries func (cr *NodeController) ReadMany(ctx context.Context) error { u, err := request.Authenticate(ctx.HttpRequest()) if err != nil && err.Error() != e.NoAuth { return request.AuthError(err, ctx) } // Gather query params query := ctx.HttpRequest().URL.Query() // Setup query and nodes objects // Note: query is composed of 3 sub-query objects: // 1) qPerm - user permissions (system-defined) // 2) qOpts - query options (user-defined) // 3) qAcls - ACL queries (user-defined) q := bson.M{} qPerm := bson.M{} qOpts := bson.M{} qAcls := bson.M{} nodes := node.Nodes{} if u != nil { // Skip this part if user is an admin if !u.Admin { qPerm["$or"] = []bson.M{bson.M{"acl.read": "public"}, bson.M{"acl.read": u.Uuid}, bson.M{"acl.owner": u.Uuid}} } } else { // User is anonymous if conf.ANON_READ { // select on only nodes that are publicly readable qPerm["acl.read"] = "public" } else { return responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth) } } // bson.M is a convenient alias for a map[string]interface{} map, useful for dealing with BSON in a native way. var OptsMArray []bson.M // Gather params to make db query. Do not include the following list. if _, ok := query["query"]; ok { paramlist := map[string]int{"query": 1, "limit": 1, "offset": 1, "order": 1, "direction": 1, "distinct": 1} for key := range query { if _, found := paramlist[key]; !found { keyStr := fmt.Sprintf("attributes.%s", key) for _, value := range query[key] { if value != "" { OptsMArray = append(OptsMArray, parseOption(keyStr, value)) } else { OptsMArray = append(OptsMArray, bson.M{keyStr: map[string]bool{"$exists": true}}) } } } } } else if _, ok := query["querynode"]; ok { paramlist := map[string]int{"querynode": 1, "limit": 1, "offset": 1, "order": 1, "direction": 1, "distinct": 1, "owner": 1, "read": 1, "write": 1, "delete": 1, "public_owner": 1, "public_read": 1, "public_write": 1, "public_delete": 1} for key := range query { if _, found := paramlist[key]; !found { for _, value := range query[key] { if value != "" { OptsMArray = append(OptsMArray, parseOption(key, value)) } else { OptsMArray = append(OptsMArray, bson.M{key: map[string]bool{"$exists": true}}) } } } } } if len(OptsMArray) > 0 { qOpts["$and"] = OptsMArray } // bson.M is a convenient alias for a map[string]interface{} map, useful for dealing with BSON in a native way. var AclsMArray []bson.M // Allowing user to query based on ACL's with a comma-separated list of users. // Restricting ACL queries to just the querynode operation. // Users can be written as a username or a UUID. if _, qok := query["querynode"]; qok { for _, atype := range []string{"owner", "read", "write", "delete"} { if _, ok := query[atype]; ok { users := strings.Split(query.Get(atype), ",") for _, v := range users { if uuid.Parse(v) != nil { AclsMArray = append(AclsMArray, bson.M{"acl." + atype: v}) } else { u := user.User{Username: v} if err := u.SetMongoInfo(); err != nil { err_msg := "err " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } AclsMArray = append(AclsMArray, bson.M{"acl." + atype: u.Uuid}) } } } } // Allowing users to query based on whether ACL is public for _, atype := range []string{"owner", "read", "write", "delete"} { if _, ok := query["public_"+atype]; ok { AclsMArray = append(AclsMArray, bson.M{"acl." + atype: "public"}) } } } if len(AclsMArray) > 0 { qAcls["$and"] = AclsMArray } // Combine permissions query with query parameters and ACL query into one AND clause q["$and"] = []bson.M{qPerm, qOpts, qAcls} // process distinct query if _, ok := query["distinct"]; ok { dField := query.Get("distinct") if !node.HasAttributeField(dField) { err_msg := "err unable to run distinct query on non-indexed attributes field: " + dField logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } results, err := node.DbFindDistinct(q, dField) if err != nil { err_msg := "err " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } return responder.RespondWithData(ctx, results) } // defaults order := "created_on" direction := "-" limit := 25 offset := 0 // get from query if _, ok := query["query"]; ok { if _, ok := query["order"]; ok { order = fmt.Sprintf("attributes.%s", query.Get("order")) } } else { if _, ok := query["order"]; ok { order = query.Get("order") } } if _, ok := query["direction"]; ok { if query.Get("direction") == "asc" { direction = "" } } if _, ok := query["limit"]; ok { limit = util.ToInt(query.Get("limit")) } if _, ok := query["offset"]; ok { offset = util.ToInt(query.Get("offset")) } // Get nodes from db order = direction + order count, err := nodes.GetPaginated(q, limit, offset, order) if err != nil { err_msg := "err " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } return responder.RespondWithPaginatedData(ctx, nodes, limit, offset, count) }
// GET, POST, PUT, DELETE: /node/{nid}/acl/{type} func AclTypedRequest(ctx context.Context) { nid := ctx.PathValue("nid") rtype := ctx.PathValue("type") u, err := request.Authenticate(ctx.HttpRequest()) if err != nil && err.Error() != e.NoAuth { request.AuthError(err, ctx) return } // acl require auth even for public data if u == nil { responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth) return } if !validAclTypes[rtype] { responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid acl type") return } // Load node and handle user unauthorized n, err := node.Load(nid, u.Uuid) if err != nil { if err.Error() == e.UnAuth { responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth) return } else if err.Error() == e.MongoDocNotFound { responder.RespondWithError(ctx, http.StatusNotFound, "Node not found") return } else { // In theory the db connection could be lost between // checking user and load but seems unlikely. err_msg := "Err@node_Read:LoadNode: " + err.Error() logger.Error(err_msg) responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) return } } rights := n.Acl.Check(u.Uuid) requestMethod := ctx.HttpRequest().Method if requestMethod != "GET" { ids, err := parseAclRequestTyped(ctx) if err != nil { responder.RespondWithError(ctx, http.StatusBadRequest, err.Error()) return } if (requestMethod == "POST" || requestMethod == "PUT") && (u.Uuid == n.Acl.Owner || rights["write"]) { if rtype == "owner" { if u.Uuid == n.Acl.Owner { if len(ids) == 1 { n.Acl.SetOwner(ids[0]) } else { responder.RespondWithError(ctx, http.StatusBadRequest, "Too many users. Nodes may have only one owner.") return } } else { responder.RespondWithError(ctx, http.StatusBadRequest, "Only owner can change ownership of Node.") return } } else if rtype == "all" { for _, atype := range []string{"read", "write", "delete"} { for _, i := range ids { n.Acl.Set(i, map[string]bool{atype: true}) } } } else { for _, i := range ids { n.Acl.Set(i, map[string]bool{rtype: true}) } } n.Save() } else if requestMethod == "DELETE" && (u.Uuid == n.Acl.Owner || rights["delete"]) { if rtype == "owner" { responder.RespondWithError(ctx, http.StatusBadRequest, "Deleting ownership is not a supported request type.") return } else if rtype == "all" { for _, atype := range []string{"read", "write", "delete"} { for _, i := range ids { n.Acl.UnSet(i, map[string]bool{atype: true}) } } } else { for _, i := range ids { n.Acl.UnSet(i, map[string]bool{rtype: true}) } } n.Save() } else { responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth) return } } if u.Uuid == n.Acl.Owner || rights["read"] { switch rtype { case "read": responder.RespondWithData(ctx, map[string][]string{"read": n.Acl.Read}) case "write": responder.RespondWithData(ctx, map[string][]string{"write": n.Acl.Write}) case "delete": responder.RespondWithData(ctx, map[string][]string{"delete": n.Acl.Delete}) case "owner": responder.RespondWithData(ctx, map[string]string{"owner": n.Acl.Owner}) case "all": responder.RespondWithData(ctx, n.Acl) } } else { responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth) return } return }
// GET: /node/{id} func (cr *NodeController) Read(id string, ctx context.Context) error { u, err := request.Authenticate(ctx.HttpRequest()) if err != nil && err.Error() != e.NoAuth { return request.AuthError(err, ctx) } // Fake public user if u == nil { if conf.Bool(conf.Conf["anon-read"]) { u = &user.User{Uuid: ""} } else { return responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth) } } // Load node and handle user unauthorized n, err := node.Load(id, u.Uuid) if err != nil { if err.Error() == e.UnAuth { return responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth) } else if err.Error() == e.MongoDocNotFound { return responder.RespondWithError(ctx, http.StatusNotFound, "Node not found") } else { // In theory the db connection could be lost between // checking user and load but seems unlikely. logger.Error("Err@node_Read:LoadNode:" + id + ":" + err.Error()) n, err = node.LoadFromDisk(id) if err.Error() == "Node does not exist" { logger.Error(err.Error()) return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error()) } else if err != nil { err_msg := "Err@node_Read:LoadNodeFromDisk:" + id + ":" + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) } } } // Gather query params query := ctx.HttpRequest().URL.Query() var fFunc filter.FilterFunc = nil if _, ok := query["filter"]; ok { if filter.Has(query.Get("filter")) { fFunc = filter.Filter(query.Get("filter")) } } // Switch though param flags // ?download=1 or ?download_raw=1 _, download_raw := query["download_raw"] if _, ok := query["download"]; ok || download_raw { if !n.HasFile() { return responder.RespondWithError(ctx, http.StatusBadRequest, "Node has no file") } filename := n.Id if _, ok := query["filename"]; ok { filename = query.Get("filename") } _, seek_ok := query["seek"] if _, length_ok := query["length"]; seek_ok || length_ok { if n.Type == "subset" { return responder.RespondWithError(ctx, http.StatusBadRequest, "subset nodes do not currently support seek/length offset retrieval") } var seek int64 var length int64 if !seek_ok { seek = 0 length_str := query.Get("length") length, err = strconv.ParseInt(length_str, 10, 0) if err != nil { return responder.RespondWithError(ctx, http.StatusBadRequest, "length must be an integer value") } } else if !length_ok { seek_str := query.Get("seek") seek, err = strconv.ParseInt(seek_str, 10, 0) if err != nil { return responder.RespondWithError(ctx, http.StatusBadRequest, "seek must be an integer value") } length = n.File.Size - seek } else { seek_str := query.Get("seek") seek, err = strconv.ParseInt(seek_str, 10, 0) if err != nil { return responder.RespondWithError(ctx, http.StatusBadRequest, "seek must be an integer value") } length_str := query.Get("length") length, err = strconv.ParseInt(length_str, 10, 0) if err != nil { return responder.RespondWithError(ctx, http.StatusBadRequest, "length must be an integer value") } } r, err := n.FileReader() defer r.Close() if err != nil { err_msg := "Err@node_Read:Open: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) } s := &request.Streamer{R: []file.SectionReader{}, W: ctx.HttpResponseWriter(), ContentType: "application/octet-stream", Filename: filename, Size: length, Filter: fFunc} s.R = append(s.R, io.NewSectionReader(r, seek, length)) if download_raw { err = s.StreamRaw() if err != nil { // causes "multiple response.WriteHeader calls" error but better than no response err_msg := "err:@node_Read s.StreamRaw: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } } else { err = s.Stream() if err != nil { // causes "multiple response.WriteHeader calls" error but better than no response err_msg := "err:@node_Read s.Stream: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } } } else if _, ok := query["index"]; ok { //handling bam file if query.Get("index") == "bai" { if n.Type == "subset" { return responder.RespondWithError(ctx, http.StatusBadRequest, "subset nodes do not support bam indices") } s := &request.Streamer{R: []file.SectionReader{}, W: ctx.HttpResponseWriter(), ContentType: "application/octet-stream", Filename: filename, Size: n.File.Size, Filter: fFunc} var region string if _, ok := query["region"]; ok { //retrieve alingments overlapped with specified region region = query.Get("region") } argv, err := request.ParseSamtoolsArgs(ctx) if err != nil { return responder.RespondWithError(ctx, http.StatusBadRequest, "Invaid args in query url") } err = s.StreamSamtools(n.FilePath(), region, argv...) if err != nil { return responder.RespondWithError(ctx, http.StatusBadRequest, "error while invoking samtools") } return nil } // open file r, err := n.FileReader() defer r.Close() if err != nil { err_msg := "Err@node_Read:Open: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) } // load index obj and info idxName := query.Get("index") idxInfo, ok := n.Indexes[idxName] if !ok { return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index") } idx, err := n.DynamicIndex(idxName) if err != nil { return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error()) } if idx.Type() == "virtual" { if n.Type == "subset" { return responder.RespondWithError(ctx, http.StatusBadRequest, "subset nodes do not currently support virtual indices") } csize := conf.CHUNK_SIZE if _, ok := query["chunk_size"]; ok { csize, err = strconv.ParseInt(query.Get("chunk_size"), 10, 64) if err != nil { return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid chunk_size") } } idx.Set(map[string]interface{}{"ChunkSize": csize}) } var size int64 = 0 s := &request.Streamer{R: []file.SectionReader{}, W: ctx.HttpResponseWriter(), ContentType: "application/octet-stream", Filename: filename, Filter: fFunc} _, hasPart := query["part"] if n.Type == "subset" && idxName == "chunkrecord" { recordIdxName := "record" recordIdxInfo, ok := n.Indexes[recordIdxName] if !ok { return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid request, record index must exist to retrieve chunkrecord index on a subset node.") } recordIdx, err := n.DynamicIndex(recordIdxName) if err != nil { return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error()) } if !hasPart { // download full subset file fullRange := "1-" + strconv.FormatInt(recordIdxInfo.TotalUnits, 10) recSlice, err := recordIdx.Range(fullRange, n.IndexPath()+"/"+recordIdxName+".idx", recordIdxInfo.TotalUnits) if err != nil { return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index subset") } for _, rec := range recSlice { size += rec[1] s.R = append(s.R, io.NewSectionReader(r, rec[0], rec[1])) } } else if hasPart { // download parts for _, p := range query["part"] { chunkRecSlice, err := idx.Range(p, n.IndexPath()+"/"+idxName+".idx", idxInfo.TotalUnits) if err != nil { return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index part") } // This gets us the parts of the chunkrecord index, but we still need to convert these to record indices. for _, chunkRec := range chunkRecSlice { start := (chunkRec[0] / 16) + 1 stop := (start - 1) + (chunkRec[1] / 16) recSlice, err := recordIdx.Range(strconv.FormatInt(start, 10)+"-"+strconv.FormatInt(stop, 10), n.IndexPath()+"/"+recordIdxName+".idx", recordIdxInfo.TotalUnits) if err != nil { return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index subset") } for _, rec := range recSlice { size += rec[1] s.R = append(s.R, io.NewSectionReader(r, rec[0], rec[1])) } } } } else { // bad request return responder.RespondWithError(ctx, http.StatusBadRequest, "Index parameter requires part parameter") } } else { if (!hasPart) && (idxInfo.Type == "subset") { // download full subset file fullRange := "1-" + strconv.FormatInt(idxInfo.TotalUnits, 10) recSlice, err := idx.Range(fullRange, n.IndexPath()+"/"+idxName+".idx", idxInfo.TotalUnits) if err != nil { return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index subset") } for _, rec := range recSlice { size += rec[1] s.R = append(s.R, io.NewSectionReader(r, rec[0], rec[1])) } } else if hasPart { // download parts for _, p := range query["part"] { // special case for subset ranges if idxInfo.Type == "subset" { recSlice, err := idx.Range(p, n.IndexPath()+"/"+idxName+".idx", idxInfo.TotalUnits) if err != nil { return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index part") } for _, rec := range recSlice { size += rec[1] s.R = append(s.R, io.NewSectionReader(r, rec[0], rec[1])) } } else { pos, length, err := idx.Part(p, n.IndexPath()+"/"+idxName+".idx", idxInfo.TotalUnits) if err != nil { return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index part") } size += length s.R = append(s.R, io.NewSectionReader(r, pos, length)) } } } else { // bad request return responder.RespondWithError(ctx, http.StatusBadRequest, "Index parameter requires part parameter") } } s.Size = size if download_raw { err = s.StreamRaw() if err != nil { // causes "multiple response.WriteHeader calls" error but better than no response err_msg := "err:@node_Read s.StreamRaw: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } } else { err = s.Stream() if err != nil { // causes "multiple response.WriteHeader calls" error but better than no response err_msg := "err:@node_Read s.Stream: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } } // download full file } else { if n.Type == "subset" { // open file r, err := n.FileReader() defer r.Close() if err != nil { err_msg := "Err@node_Read:Open: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) } idx := index.New() s := &request.Streamer{R: []file.SectionReader{}, W: ctx.HttpResponseWriter(), ContentType: "application/octet-stream", Filename: filename, Size: n.File.Size, Filter: fFunc} fullRange := "1-" + strconv.FormatInt(n.Subset.Index.TotalUnits, 10) recSlice, err := idx.Range(fullRange, n.Path()+"/"+n.Id+".subset.idx", n.Subset.Index.TotalUnits) if err != nil { return responder.RespondWithError(ctx, http.StatusInternalServerError, "Invalid data index for subset node.") } for _, rec := range recSlice { s.R = append(s.R, io.NewSectionReader(r, rec[0], rec[1])) } if download_raw { err = s.StreamRaw() if err != nil { // causes "multiple response.WriteHeader calls" error but better than no response err_msg := "err:@node_Read s.StreamRaw: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } } else { err = s.Stream() if err != nil { // causes "multiple response.WriteHeader calls" error but better than no response err_msg := "err:@node_Read s.Stream: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } } } else { nf, err := n.FileReader() defer nf.Close() if err != nil { // File not found or some sort of file read error. // Probably deserves more checking err_msg := "err:@node_Read node.FileReader: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } s := &request.Streamer{R: []file.SectionReader{nf}, W: ctx.HttpResponseWriter(), ContentType: "application/octet-stream", Filename: filename, Size: n.File.Size, Filter: fFunc} if download_raw { err = s.StreamRaw() if err != nil { // causes "multiple response.WriteHeader calls" error but better than no response err_msg := "err:@node_Read s.StreamRaw: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } } else { err = s.Stream() if err != nil { // causes "multiple response.WriteHeader calls" error but better than no response err_msg := "err:@node_Read s.Stream: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg) } } } } } else if _, ok := query["download_url"]; ok { if n.Type == "subset" { return responder.RespondWithError(ctx, http.StatusBadRequest, "subset nodes do not currently support download_url operation") } if !n.HasFile() { return responder.RespondWithError(ctx, http.StatusBadRequest, "Node has no file") } else { options := map[string]string{} if _, ok := query["filename"]; ok { options["filename"] = query.Get("filename") } if p, err := preauth.New(util.RandString(20), "download", n.Id, options); err != nil { err_msg := "err:@node_Read download_url: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) } else { return responder.RespondWithData(ctx, util.UrlResponse{Url: util.ApiUrl(ctx) + "/preauth/" + p.Id, ValidTill: p.ValidTill.Format(time.ANSIC)}) } } } else if _, ok := query["download_post"]; ok { // This is a request to post the node to another Shock server. The 'post_url' parameter is required. // By default the post operation will include the data file and attributes (these options can be set // with post_data=0/1 and post_attr=0/1). if n.Type == "subset" { return responder.RespondWithError(ctx, http.StatusBadRequest, "subset nodes do not currently support download_post operation") } post_url := "" if _, ok := query["post_url"]; ok { post_url = query.Get("post_url") } else { return responder.RespondWithError(ctx, http.StatusBadRequest, "Request type requires post_url parameter of where to post new Shock node") } post_opts := map[string]int{ "post_data": 1, "post_attr": 1, } for k, _ := range post_opts { if _, ok := query[k]; ok { if query.Get(k) == "0" { post_opts[k] = 0 } else if query.Get(k) == "1" { post_opts[k] = 1 } else { return responder.RespondWithError(ctx, http.StatusBadRequest, "Parameter "+k+" must be either 0 or 1") } } } form := client.NewForm() form.AddParam("file_name", n.File.Name) if post_opts["post_data"] == 1 { form.AddFile("upload", n.FilePath()) } if post_opts["post_attr"] == 1 && n.Attributes != nil { attr, _ := json.Marshal(n.Attributes) form.AddParam("attributes_str", string(attr[:])) } err = form.Create() if err != nil { err_msg := "could not create multipart form for posting to Shock server: " + err.Error() logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) } headers := client.Header{ "Content-Type": form.ContentType, "Content-Length": strconv.FormatInt(form.Length, 10), } if _, hasAuth := ctx.HttpRequest().Header["Authorization"]; hasAuth { headers["Authorization"] = ctx.HttpRequest().Header.Get("Authorization") } if res, err := client.Do("POST", post_url, headers, form.Reader); err == nil { if res.StatusCode == 200 { r := responseWrapper{} body, _ := ioutil.ReadAll(res.Body) if err = json.Unmarshal(body, &r); err != nil { err_msg := "err:@node_Read POST: " + err.Error() logger.Error(err_msg) return responder.WriteResponseObject(ctx, http.StatusInternalServerError, err_msg) } else { return responder.WriteResponseObject(ctx, http.StatusOK, r) } } else { r := responseWrapper{} body, _ := ioutil.ReadAll(res.Body) if err = json.Unmarshal(body, &r); err == nil { err_msg := res.Status + ": " + (*r.Error)[0] logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) } else { err_msg := "request error: " + res.Status logger.Error(err_msg) return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) } } } else { return err } } else { // Base case respond with node in json return responder.RespondWithData(ctx, n) } return nil }
func main() { // init(s) conf.Initialize() logger.Initialize() if err := db.Initialize(); err != nil { fmt.Fprintf(os.Stderr, "[email protected]: %v\n", err) logger.Error("[email protected]: " + err.Error()) os.Exit(1) } user.Initialize() node.Initialize() preauth.Initialize() auth.Initialize() node.InitReaper() if err := versions.Initialize(); err != nil { fmt.Fprintf(os.Stderr, "[email protected]: %v\n", err) logger.Error("[email protected]: " + err.Error()) os.Exit(1) } if err := versions.RunVersionUpdates(); err != nil { fmt.Fprintf(os.Stderr, "[email protected]: %v\n", err) logger.Error("[email protected]: " + err.Error()) os.Exit(1) } // After version updates have succeeded without error, we can push the configured version numbers into the mongo db // Note: configured version numbers are configured in conf.go but are NOT user configurable by design if err := versions.PushVersionsToDatabase(); err != nil { fmt.Fprintf(os.Stderr, "[email protected]: %v\n", err) logger.Error("[email protected]: " + err.Error()) os.Exit(1) } printLogo() conf.Print() if err := versions.Print(); err != nil { fmt.Fprintf(os.Stderr, "[email protected]: %v\n", err) logger.Error("[email protected]: " + err.Error()) os.Exit(1) } // check if necessary directories exist or created for _, path := range []string{conf.PATH_SITE, conf.PATH_DATA, conf.PATH_LOGS, conf.PATH_DATA + "/temp"} { if _, err := os.Stat(path); err != nil && os.IsNotExist(err) { if err := os.Mkdir(path, 0777); err != nil { fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) logger.Error("ERROR: " + err.Error()) os.Exit(1) } } } // reload if conf.RELOAD != "" { fmt.Println("####### Reloading #######") err := reload(conf.RELOAD) if err != nil { fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) logger.Error("ERROR: " + err.Error()) os.Exit(1) } fmt.Println("Done") } // setting GOMAXPROCS var procs int avail := runtime.NumCPU() if avail <= 2 { procs = 1 } else if avail == 3 { procs = 2 } else { procs = avail - 2 } fmt.Println("##### Procs #####") fmt.Printf("Number of available CPUs = %d\n", avail) if conf.GOMAXPROCS != "" { if setting, err := strconv.Atoi(conf.GOMAXPROCS); err != nil { err_msg := "ERROR: could not interpret configured GOMAXPROCS value as integer.\n" fmt.Fprintf(os.Stderr, err_msg) logger.Error("ERROR: " + err_msg) os.Exit(1) } else { procs = setting } } if procs <= avail { fmt.Printf("Running Shock server with GOMAXPROCS = %d\n\n", procs) runtime.GOMAXPROCS(procs) } else { fmt.Println("GOMAXPROCS config value is greater than available number of CPUs.") fmt.Printf("Running Shock server with GOMAXPROCS = %d\n\n", avail) runtime.GOMAXPROCS(avail) } if conf.PATH_PIDFILE != "" { f, err := os.Create(conf.PATH_PIDFILE) if err != nil { err_msg := "Could not create pid file: " + conf.PATH_PIDFILE + "\n" fmt.Fprintf(os.Stderr, err_msg) logger.Error("ERROR: " + err_msg) os.Exit(1) } defer f.Close() pid := os.Getpid() fmt.Fprintln(f, pid) fmt.Println("##### pidfile #####") fmt.Printf("pid: %d saved to file: %s\n\n", pid, conf.PATH_PIDFILE) } Address := conf.API_IP + ":" + conf.API_PORT mapRoutes() s := &http.Server{ Addr: ":" + Address, Handler: goweb.DefaultHttpHandler(), ReadTimeout: 48 * time.Hour, WriteTimeout: 48 * time.Hour, MaxHeaderBytes: 1 << 20, } c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt) listener, listenErr := net.Listen("tcp", Address) if listenErr != nil { err_msg := "Could not listen - " + listenErr.Error() + "\n" fmt.Fprintf(os.Stderr, err_msg) logger.Error("ERROR: " + err_msg) os.Exit(1) } go node.Ttl.Handle() go func() { for _ = range c { // sig is a ^C, handle it // stop the HTTP server fmt.Fprintln(os.Stderr, "Stopping the server...") listener.Close() } }() fmt.Fprintf(os.Stderr, "Error in Serve: %s\n", s.Serve(listener)) }
// GET, POST, PUT, DELETE: /node/{nid}/acl/{type} func AclTypedRequest(ctx context.Context) { nid := ctx.PathValue("nid") rtype := ctx.PathValue("type") rmeth := ctx.HttpRequest().Method query := ctx.HttpRequest().URL.Query() verbosity := "" if _, ok := query["verbosity"]; ok { verbosity = query.Get("verbosity") } u, err := request.Authenticate(ctx.HttpRequest()) if err != nil && err.Error() != e.NoAuth { request.AuthError(err, ctx) return } if !validAclTypes[rtype] { responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid acl type") return } // Load node by id n, err := node.Load(nid) if err != nil { if err == mgo.ErrNotFound { responder.RespondWithError(ctx, http.StatusNotFound, e.NodeNotFound) return } else { // In theory the db connection could be lost between // checking user and load but seems unlikely. err_msg := "Err@node_Acl:LoadNode: " + nid + err.Error() logger.Error(err_msg) responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) return } } // public user (no auth) can perform a GET operation given the proper node permissions if u == nil { rights := n.Acl.Check("public") if rmeth == "GET" && conf.ANON_READ && (rights["read"] || n.Acl.Owner == "public") { responder.RespondWithData(ctx, n.Acl.FormatDisplayAcl(verbosity)) return } else { responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth) return } } // Users that are not an admin or the node owner can only delete themselves from an ACL. if n.Acl.Owner != u.Uuid && u.Admin == false { // Users that are not an admin or the node owner cannot remove public from ACL's. if rtype == "public_read" || rtype == "public_write" || rtype == "public_delete" || rtype == "public_all" { responder.RespondWithError(ctx, http.StatusBadRequest, "Users that are not node owners can only delete themselves from ACLs.") return } // Parse user list ids, err := parseAclRequestTyped(ctx) if err != nil { responder.RespondWithError(ctx, http.StatusBadRequest, err.Error()) return } if rmeth == "DELETE" { if len(ids) != 1 || (len(ids) == 1 && ids[0] != u.Uuid) { responder.RespondWithError(ctx, http.StatusBadRequest, "Users that are not node owners can delete only themselves from ACLs.") return } if rtype == "owner" { responder.RespondWithError(ctx, http.StatusBadRequest, "Deleting node ownership is not a supported request type.") return } if rtype == "all" { n.Acl.UnSet(ids[0], map[string]bool{"read": true, "write": true, "delete": true}) } else { n.Acl.UnSet(ids[0], map[string]bool{rtype: true}) } n.Save() responder.RespondWithData(ctx, n.Acl.FormatDisplayAcl(verbosity)) return } responder.RespondWithError(ctx, http.StatusBadRequest, "Users that are not node owners can only delete themselves from ACLs.") return } // At this point we know we're dealing with an admin or the node owner. // Admins and node owners can view/edit/delete ACLs if rmeth == "GET" { responder.RespondWithData(ctx, n.Acl.FormatDisplayAcl(verbosity)) return } else if rmeth == "POST" || rmeth == "PUT" { if rtype == "public_read" || rtype == "public_write" || rtype == "public_delete" || rtype == "public_all" { if rtype == "public_read" { n.Acl.Set("public", map[string]bool{"read": true}) } else if rtype == "public_write" { n.Acl.Set("public", map[string]bool{"write": true}) } else if rtype == "public_delete" { n.Acl.Set("public", map[string]bool{"delete": true}) } else if rtype == "public_all" { n.Acl.Set("public", map[string]bool{"read": true, "write": true, "delete": true}) } n.Save() responder.RespondWithData(ctx, n.Acl.FormatDisplayAcl(verbosity)) return } // Parse user list ids, err := parseAclRequestTyped(ctx) if err != nil { responder.RespondWithError(ctx, http.StatusBadRequest, err.Error()) return } if rtype == "owner" { if len(ids) == 1 { n.Acl.SetOwner(ids[0]) } else { responder.RespondWithError(ctx, http.StatusBadRequest, "Too many users. Nodes may have only one owner.") return } } else if rtype == "all" { for _, i := range ids { n.Acl.Set(i, map[string]bool{"read": true, "write": true, "delete": true}) } } else { for _, i := range ids { n.Acl.Set(i, map[string]bool{rtype: true}) } } n.Save() responder.RespondWithData(ctx, n.Acl.FormatDisplayAcl(verbosity)) return } else if rmeth == "DELETE" { if rtype == "public_read" || rtype == "public_write" || rtype == "public_delete" || rtype == "public_all" { if rtype == "public_read" { n.Acl.UnSet("public", map[string]bool{"read": true}) } else if rtype == "public_write" { n.Acl.UnSet("public", map[string]bool{"write": true}) } else if rtype == "public_delete" { n.Acl.UnSet("public", map[string]bool{"delete": true}) } else if rtype == "public_all" { n.Acl.UnSet("public", map[string]bool{"read": true, "write": true, "delete": true}) } n.Save() responder.RespondWithData(ctx, n.Acl.FormatDisplayAcl(verbosity)) return } // Parse user list ids, err := parseAclRequestTyped(ctx) if err != nil { responder.RespondWithError(ctx, http.StatusBadRequest, err.Error()) return } if rtype == "owner" { responder.RespondWithError(ctx, http.StatusBadRequest, "Deleting ownership is not a supported request type.") return } else if rtype == "all" { for _, i := range ids { n.Acl.UnSet(i, map[string]bool{"read": true, "write": true, "delete": true}) } } else { for _, i := range ids { n.Acl.UnSet(i, map[string]bool{rtype: true}) } } n.Save() responder.RespondWithData(ctx, n.Acl.FormatDisplayAcl(verbosity)) return } else { responder.RespondWithError(ctx, http.StatusNotImplemented, "This request type is not implemented.") return } }
} // Load node and handle user unauthorized id := cx.PathParams["nid"] n, err := node.Load(id, u.Uuid) if err != nil { if err.Error() == e.UnAuth { cx.RespondWithError(http.StatusUnauthorized) return } else if err.Error() == e.MongoDocNotFound { cx.RespondWithNotFound() return } else { // In theory the db connection could be lost between // checking user and load but seems unlikely. logger.Error("Err@index:LoadNode: " + err.Error()) cx.RespondWithError(http.StatusInternalServerError) return } } idxType, hasType := cx.PathParams["type"] query := util.Q(cx.Request.URL.Query()) switch cx.Request.Method { case "GET": if hasType { if v, has := n.Indexes[idxType]; has { cx.RespondWithData(map[string]interface{}{idxType: v}) } else { cx.RespondWithErrorMessage(fmt.Sprintf("Node %s does not have index of type %s.", n.Id, idxType), http.StatusBadRequest) }
func main() { // init(s) conf.Initialize() logger.Initialize() if err := db.Initialize(); err != nil { fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) logger.Error("ERROR: " + err.Error()) os.Exit(1) } user.Initialize() node.Initialize() preauth.Initialize() auth.Initialize() // print conf printLogo() conf.Print() // check if necessary directories exist or created for _, path := range []string{conf.Conf["site-path"], conf.Conf["data-path"], conf.Conf["logs-path"], conf.Conf["data-path"] + "/temp"} { if _, err := os.Stat(path); err != nil && os.IsNotExist(err) { if err := os.Mkdir(path, 0777); err != nil { fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) logger.Error("ERROR: " + err.Error()) os.Exit(1) } } } // reload if conf.RELOAD != "" { fmt.Println("####### Reloading #######") err := reload(conf.RELOAD) if err != nil { fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) logger.Error("ERROR: " + err.Error()) os.Exit(1) } fmt.Println("Done") } // setting GOMAXPROCS var procs int avail := runtime.NumCPU() if avail <= 2 { procs = 1 } else if avail == 3 { procs = 2 } else { procs = avail - 2 } fmt.Println("##### Procs #####") fmt.Printf("Number of available CPUs = %d\n", avail) if conf.Conf["GOMAXPROCS"] != "" { if setting, err := strconv.Atoi(conf.Conf["GOMAXPROCS"]); err != nil { err_msg := "ERROR: could not interpret configured GOMAXPROCS value as integer.\n" fmt.Fprintf(os.Stderr, err_msg) logger.Error("ERROR: " + err_msg) os.Exit(1) } else { procs = setting } } if procs <= avail { fmt.Printf("Running Shock server with GOMAXPROCS = %d\n\n", procs) runtime.GOMAXPROCS(procs) } else { fmt.Println("GOMAXPROCS config value is greater than available number of CPUs.") fmt.Printf("Running Shock server with GOMAXPROCS = %d\n\n", avail) runtime.GOMAXPROCS(avail) } if conf.Conf["pidfile"] != "" { f, err := os.Create(conf.Conf["pidfile"]) if err != nil { err_msg := "Could not create pid file: " + conf.Conf["pidfile"] + "\n" fmt.Fprintf(os.Stderr, err_msg) logger.Error("ERROR: " + err_msg) os.Exit(1) } defer f.Close() pid := os.Getpid() fmt.Fprintln(f, pid) fmt.Println("##### pidfile #####") fmt.Printf("pid: %d saved to file: %s\n\n", pid, conf.Conf["pidfile"]) } Address := conf.Conf["api-ip"] + ":" + conf.Conf["api-port"] mapRoutes() s := &http.Server{ Addr: ":" + Address, Handler: goweb.DefaultHttpHandler(), ReadTimeout: 48 * time.Hour, WriteTimeout: 48 * time.Hour, MaxHeaderBytes: 1 << 20, } c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt) listener, listenErr := net.Listen("tcp", Address) if listenErr != nil { err_msg := "Could not listen - " + listenErr.Error() + "\n" fmt.Fprintf(os.Stderr, err_msg) logger.Error("ERROR: " + err_msg) os.Exit(1) } go func() { for _ = range c { // sig is a ^C, handle it // stop the HTTP server fmt.Fprintln(os.Stderr, "Stopping the server...") listener.Close() } }() fmt.Fprintf(os.Stderr, "Error in Serve: %s\n", s.Serve(listener)) }