//helper function to translate args in URL query to samtools args
//manual: http://samtools.sourceforge.net/samtools.shtml
func ParseSamtoolsArgs(ctx context.Context) (argv []string, err error) {
query := ctx.HttpRequest().URL.Query()
var (
filter_options = map[string]string{
"head": "-h",
"headonly": "-H",
"count": "-c",
}
valued_options = map[string]string{
"flag": "-f",
"lib": "-l",
"mapq": "-q",
"readgroup": "-r",
}
)
for src, des := range filter_options {
if _, ok := query[src]; ok {
argv = append(argv, des)
}
}
for src, des := range valued_options {
if _, ok := query[src]; ok {
if val := query.Get(src); val != "" {
argv = append(argv, des)
argv = append(argv, val)
} else {
return nil, errors.New(fmt.Sprintf("required value not found for query arg: %s ", src))
}
}
}
return argv, nil
}
func PreAuthRequest(ctx context.Context) {
id := ctx.PathValue("id")
if p, err := preauth.Load(id); err != nil {
err_msg := "err:@preAuth load: " + err.Error()
logger.Error(err_msg)
responder.RespondWithError(ctx, 500, err_msg)
return
} else {
if n, err := node.LoadUnauth(p.NodeId); err == nil {
switch p.Type {
case "download":
filename := n.Id
if fn, has := p.Options["filename"]; has {
filename = fn
}
streamDownload(ctx, n, filename)
preauth.Delete(id)
return
default:
responder.RespondWithError(ctx, 500, "Preauthorization type not supported: "+p.Type)
}
} else {
err_msg := "err:@preAuth loadnode: " + err.Error()
logger.Error(err_msg)
responder.RespondWithError(ctx, 500, err_msg)
}
}
return
}
// Handle writes the error from the context into the HttpResponseWriter with a
// 500 http.StatusInternalServerError status code.
func (h *DefaultErrorHandler) Handle(ctx context.Context) (stop bool, err error) {
var handlerError HandlerError = ctx.Data().Get(DataKeyForError).Data().(HandlerError)
hostname, _ := os.Hostname()
w := ctx.HttpResponseWriter()
// write the error out
w.Header().Set("Content-Type", "text/html")
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("<!DOCTYPE html><html><head>"))
w.Write([]byte("<style>"))
w.Write([]byte("h1 { font-size: 17px }"))
w.Write([]byte("h1 strong {text-decoration:underline}"))
w.Write([]byte("h2 { background-color: #ffd; padding: 20px }"))
w.Write([]byte("footer { margin-top: 20px; border-top:1px solid black; padding:10px; font-size:0.9em }"))
w.Write([]byte("</style>"))
w.Write([]byte("</head><body>"))
w.Write([]byte(fmt.Sprintf("<h1>Error in <code>%s</code></h1><h2>%s</h2>", handlerError.Handler, handlerError)))
w.Write([]byte(fmt.Sprintf("<h3><code>%s</code> error in Handler <code>%v</code></h3> <code><pre>%s</pre></code>", reflect.TypeOf(handlerError.OriginalError), &handlerError.Handler, handlerError.Handler)))
w.Write([]byte(fmt.Sprintf("on %s", hostname)))
w.Write([]byte("<footer>Learn more about <a href='http://github.com/stretchr/goweb' target='_blank'>Goweb</a></footer>"))
w.Write([]byte("</body></html>"))
// responses are actually ignored
return false, nil
}
// With writes a response to the request in the specified context.
func (r *GowebHTTPResponder) With(ctx context.Context, httpStatus int, body []byte) error {
r.WithStatus(ctx, httpStatus)
_, writeErr := ctx.HttpResponseWriter().Write(body)
return writeErr
}
// WithStatus writes the specified HTTP Status Code to the Context's ResponseWriter.
//
// If the Always200ParamName parameter is present, it will ignore the httpStatus argument,
// and always write net/http.StatusOK (200).
func (r *GowebHTTPResponder) WithStatus(ctx context.Context, httpStatus int) error {
// check for always200
if len(ctx.FormValue(Always200ParamName)) > 0 {
// always return OK
httpStatus = http.StatusOK
}
ctx.HttpResponseWriter().WriteHeader(httpStatus)
return nil
}
// WithStatusText writes the specified HTTP Status Code to the Context's ResponseWriter and
// includes a body with the default status text.
func (r *GowebHTTPResponder) WithStatusText(ctx context.Context, httpStatus int) error {
writeStatusErr := r.WithStatus(ctx, httpStatus)
if writeStatusErr != nil {
return writeStatusErr
}
// write the body header
_, writeErr := ctx.HttpResponseWriter().Write([]byte(http.StatusText(httpStatus)))
return writeErr
}
// GET, POST, PUT, DELETE: /node/{nid}/acl/
// GET is the only action implemented here.
func AclRequest(ctx context.Context) {
nid := ctx.PathValue("nid")
rmeth := ctx.HttpRequest().Method
u, err := request.Authenticate(ctx.HttpRequest())
if err != nil && err.Error() != e.NoAuth {
request.AuthError(err, ctx)
return
}
// public user (no auth) can perform a GET operation with the proper node permissions
if u == nil {
if rmeth == "GET" && conf.ANON_READ {
u = &user.User{Uuid: "public"}
} else {
responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth)
return
}
}
// Load node by id
n, err := node.Load(nid)
if err != nil {
if err == mgo.ErrNotFound {
responder.RespondWithError(ctx, http.StatusNotFound, e.NodeNotFound)
return
} else {
// In theory the db connection could be lost between
// checking user and load but seems unlikely.
err_msg := "Err@node_Acl:LoadNode: " + nid + err.Error()
logger.Error(err_msg)
responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
return
}
}
// Only the owner, an admin, or someone with read access can view acl's.
//
// NOTE: If the node is publicly owned, then anyone can view all acl's. The owner can only
// be "public" when anonymous node creation (ANON_WRITE) is enabled in Shock config.
rights := n.Acl.Check(u.Uuid)
if n.Acl.Owner != u.Uuid && u.Admin == false && n.Acl.Owner != "public" && rights["read"] == false {
responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth)
return
}
if rmeth == "GET" {
query := ctx.HttpRequest().URL.Query()
verbosity := ""
if _, ok := query["verbosity"]; ok {
verbosity = query.Get("verbosity")
}
responder.RespondWithData(ctx, n.Acl.FormatDisplayAcl(verbosity))
} else {
responder.RespondWithError(ctx, http.StatusNotImplemented, "This request type is not implemented.")
}
return
}
// DELETE: /node/{id}
func (cr *NodeController) Delete(id string, ctx context.Context) error {
u, err := request.Authenticate(ctx.HttpRequest())
if err != nil && err.Error() != e.NoAuth {
return request.AuthError(err, ctx)
}
// public user (no auth) can be used in some cases
if u == nil {
if conf.ANON_DELETE {
u = &user.User{Uuid: "public"}
} else {
return responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth)
}
}
// Load node by id
n, err := node.Load(id)
if err != nil {
if err == mgo.ErrNotFound {
return responder.RespondWithError(ctx, http.StatusNotFound, e.NodeNotFound)
} else {
// In theory the db connection could be lost between
// checking user and load but seems unlikely.
err_msg := "Err@node_Delete:LoadNode: " + id + ":" + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
}
}
rights := n.Acl.Check(u.Uuid)
prights := n.Acl.Check("public")
if rights["delete"] == false && u.Admin == false && n.Acl.Owner != u.Uuid && prights["delete"] == false {
return responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth)
}
if err := n.Delete(); err == nil {
return responder.RespondOK(ctx)
} else {
err_msg := "Err@node_Delete:Delete: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
}
}
// Responds to the Context with the specified status, data and errors.
func (a *GowebAPIResponder) Respond(ctx context.Context, status int, data interface{}, errors []string) error {
if data != nil {
var dataErr error
data, dataErr = codecs.PublicData(data, nil)
if dataErr != nil {
return dataErr
}
}
// make the standard response object
if (a.AlwaysEnvelopResponse && ctx.QueryValue("envelop") != "false") || ctx.QueryValue("envelop") == "true" {
sro := map[string]interface{}{
a.StandardFieldStatusKey: status,
}
if data != nil {
sro[a.StandardFieldDataKey] = data
}
if len(errors) > 0 {
sro[a.StandardFieldErrorsKey] = errors
}
data = sro
}
// transform the object
var transformErr error
data, transformErr = a.TransformStandardResponseObject(ctx, data)
if transformErr != nil {
return transformErr
}
return a.WriteResponseObject(ctx, status, data)
}
func streamDownload(ctx context.Context, n *node.Node, filename string) {
nf, err := n.FileReader()
defer nf.Close()
if err != nil {
// File not found or some sort of file read error.
// Probably deserves more checking
err_msg := "err:@preAuth node.FileReader: " + err.Error()
logger.Error(err_msg)
responder.RespondWithError(ctx, 500, err_msg)
return
}
s := &request.Streamer{R: []file.SectionReader{nf}, W: ctx.HttpResponseWriter(), ContentType: "application/octet-stream", Filename: filename, Size: n.File.Size, Filter: nil}
err = s.Stream()
if err != nil {
// causes "multiple response.WriteHeader calls" error but better than no response
err_msg := "err:@preAuth: s.stream: " + err.Error()
logger.Error(err_msg)
responder.RespondWithError(ctx, 500, err_msg)
}
return
}
// handle download and its options
func streamDownload(ctx context.Context, n *node.Node, options map[string]string) {
nf, err := n.FileReader()
defer nf.Close()
if err != nil {
// File not found or some sort of file read error.
// Probably deserves more checking
err_msg := "err:@preAuth node.FileReader: " + err.Error()
logger.Error(err_msg)
responder.RespondWithError(ctx, 500, err_msg)
return
}
// set defaults
filename := n.Id
var filterFunc filter.FilterFunc = nil
var compressionFormat string = ""
// use options if exist
if fn, has := options["filename"]; has {
filename = fn
}
if fl, has := options["filter"]; has {
if filter.Has(fl) {
filterFunc = filter.Filter(fl)
}
}
if cp, has := options["compression"]; has {
if archive.IsValidCompress(cp) {
compressionFormat = cp
}
}
// stream it
s := &request.Streamer{R: []file.SectionReader{nf}, W: ctx.HttpResponseWriter(), ContentType: "application/octet-stream", Filename: filename, Size: n.File.Size, Filter: filterFunc, Compression: compressionFormat}
err = s.Stream(false)
if err != nil {
// causes "multiple response.WriteHeader calls" error but better than no response
err_msg := "err:@preAuth: s.stream: " + err.Error()
logger.Error(err_msg)
responder.RespondWithError(ctx, 500, err_msg)
}
return
}
// DELETE: /node/{id}
func (cr *NodeController) Delete(id string, ctx context.Context) error {
u, err := request.Authenticate(ctx.HttpRequest())
if err != nil && err.Error() != e.NoAuth {
return request.AuthError(err, ctx)
}
if u == nil {
return responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth)
}
// Load node and handle user unauthorized
n, err := node.Load(id, u.Uuid)
if err != nil {
if err.Error() == e.UnAuth {
return responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth)
} else if err.Error() == e.MongoDocNotFound {
return responder.RespondWithError(ctx, http.StatusNotFound, "Node not found")
} else {
// In theory the db connection could be lost between
// checking user and load but seems unlikely.
err_msg := "Err@node_Read:Delete: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
}
}
rights := n.Acl.Check(u.Uuid)
if !rights["delete"] {
return responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth)
}
if err := n.Delete(); err == nil {
return responder.RespondOK(ctx)
} else {
err_msg := "Err@node_Delete:Delete: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
}
}
func parseAclRequestTyped(ctx context.Context) (ids []string, err error) {
var users []string
query := ctx.HttpRequest().URL.Query()
params, _, err := request.ParseMultipartForm(ctx.HttpRequest())
if _, ok := query["users"]; ok && err != nil && err.Error() == "request Content-Type isn't multipart/form-data" {
users = strings.Split(query.Get("users"), ",")
} else if params["users"] != "" {
users = strings.Split(params["users"], ",")
} else {
return nil, errors.New("Action requires list of comma separated usernames in 'users' parameter")
}
for _, v := range users {
if uuid.Parse(v) != nil {
ids = append(ids, v)
} else {
u := user.User{Username: v}
if err := u.SetMongoInfo(); err != nil {
return nil, err
}
ids = append(ids, u.Uuid)
}
}
return ids, nil
}
// POST: /node
func (cr *NodeController) Create(ctx context.Context) error {
u, err := request.Authenticate(ctx.HttpRequest())
if err != nil && err.Error() != e.NoAuth {
return request.AuthError(err, ctx)
}
// Fake public user
if u == nil {
if conf.Bool(conf.Conf["anon-write"]) {
u = &user.User{Uuid: ""}
} else {
return responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth)
}
}
// Parse uploaded form
params, files, err := request.ParseMultipartForm(ctx.HttpRequest())
if err != nil {
if err.Error() == "request Content-Type isn't multipart/form-data" {
// If not multipart/form-data it will try to read the Body of the
// request. If the Body is not empty it will create a file from
// the Body contents. If the Body is empty it will create an empty
// node.
if ctx.HttpRequest().ContentLength != 0 {
params, files, err = request.DataUpload(ctx.HttpRequest())
if err != nil {
err_msg := "Error uploading data from request body:" + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
}
}
n, cn_err := node.CreateNodeUpload(u, params, files)
if cn_err != nil {
err_msg := "Error at create empty node: " + cn_err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
}
if n == nil {
// Not sure how you could get an empty node with no error
// Assume it's the user's fault
return responder.RespondWithError(ctx, http.StatusBadRequest, "Error, could not create node.")
} else {
return responder.RespondWithData(ctx, n)
}
} else {
// Some error other than request encoding. Theoretically
// could be a lost db connection between user lookup and parsing.
// Blame the user, Its probaby their fault anyway.
err_msg := "Error parsing form: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
}
}
// Create node
n, err := node.CreateNodeUpload(u, params, files)
if err != nil {
err_msg := "err@node_CreateNodeUpload: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
}
return responder.RespondWithData(ctx, n)
}
// PUT: /node/{id} -> multipart-form
func (cr *NodeController) Replace(id string, ctx context.Context) error {
u, err := request.Authenticate(ctx.HttpRequest())
if err != nil && err.Error() != e.NoAuth {
return request.AuthError(err, ctx)
}
// public user (no auth) can be used in some cases
if u == nil {
if conf.ANON_WRITE {
u = &user.User{Uuid: "public"}
} else {
return responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth)
}
}
// Load node by id
n, err := node.Load(id)
if err != nil {
if err == mgo.ErrNotFound {
return responder.RespondWithError(ctx, http.StatusNotFound, e.NodeNotFound)
} else {
// In theory the db connection could be lost between
// checking user and load but seems unlikely.
err_msg := "Err@node_Update:LoadNode: " + id + ":" + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
}
}
rights := n.Acl.Check(u.Uuid)
prights := n.Acl.Check("public")
if rights["write"] == false && u.Admin == false && n.Acl.Owner != u.Uuid && prights["write"] == false {
return responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth)
}
if conf.LOG_PERF {
logger.Perf("START PUT data: " + id)
}
params, files, err := request.ParseMultipartForm(ctx.HttpRequest())
// clean up temp dir !!
defer node.RemoveAllFormFiles(files)
if err != nil {
err_msg := "err@node_ParseMultipartForm: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
}
// need delete rights to set expiration
if _, hasExpiration := params["expiration"]; hasExpiration {
if rights["delete"] == false && u.Admin == false && n.Acl.Owner != u.Uuid && prights["delete"] == false {
return responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth)
}
}
if _, hasCopyData := params["copy_data"]; hasCopyData {
_, err = node.Load(params["copy_data"])
if err != nil {
return request.AuthError(err, ctx)
}
}
if _, hasParentNode := params["parent_node"]; hasParentNode {
_, err = node.Load(params["parent_node"])
if err != nil {
return request.AuthError(err, ctx)
}
}
err = n.Update(params, files)
if err != nil {
err_msg := "err@node_Update: " + id + ": " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
}
responder.RespondWithData(ctx, n)
if conf.LOG_PERF {
logger.Perf("END PUT data: " + id)
}
return nil
}
// GET: /node
// To do:
// - Iterate node queries
func (cr *NodeController) ReadMany(ctx context.Context) error {
u, err := request.Authenticate(ctx.HttpRequest())
if err != nil && err.Error() != e.NoAuth {
return request.AuthError(err, ctx)
}
// Gather query params
query := ctx.HttpRequest().URL.Query()
// Setup query and nodes objects
q := bson.M{}
nodes := node.Nodes{}
if u != nil {
// Admin sees all
if !u.Admin {
q["$or"] = []bson.M{bson.M{"acl.read": []string{}}, bson.M{"acl.read": u.Uuid}, bson.M{"acl.owner": u.Uuid}}
}
} else {
if conf.Bool(conf.Conf["anon-read"]) {
// select on only nodes with no read rights set
q["acl.read"] = []string{}
} else {
return responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth)
}
}
// Gather params to make db query. Do not include the
// following list.
paramlist := map[string]int{"limit": 1, "offset": 1, "query": 1, "querynode": 1, "owner": 1, "read": 1, "write": 1, "delete": 1, "public_owner": 1, "public_read": 1, "public_write": 1, "public_delete": 1}
if _, ok := query["query"]; ok {
for key := range query {
if _, found := paramlist[key]; !found {
keyStr := fmt.Sprintf("attributes.%s", key)
value := query.Get(key)
if value != "" {
if numValue, err := strconv.Atoi(value); err == nil {
q["$or"] = ListOfMaps{{keyStr: value}, {keyStr: numValue}}
} else if value == "null" {
q["$or"] = ListOfMaps{{keyStr: value}, {keyStr: nil}}
} else {
q[keyStr] = value
}
} else {
existsMap := map[string]bool{
"$exists": true,
}
q[keyStr] = existsMap
}
}
}
} else if _, ok := query["querynode"]; ok {
for key := range query {
if _, found := paramlist[key]; !found {
value := query.Get(key)
if value != "" {
if numValue, err := strconv.Atoi(value); err == nil {
q["$or"] = ListOfMaps{{key: value}, {key: numValue}}
} else if value == "null" {
q["$or"] = ListOfMaps{{key: value}, {key: nil}}
} else {
q[key] = value
}
} else {
existsMap := map[string]bool{
"$exists": true,
}
q[key] = existsMap
}
}
}
}
// defaults
limit := 25
offset := 0
if _, ok := query["limit"]; ok {
limit = util.ToInt(query.Get("limit"))
}
if _, ok := query["offset"]; ok {
offset = util.ToInt(query.Get("offset"))
}
// Allowing user to query based on ACL's with a comma-separated list of users.
// Users can be written as a username or a UUID.
for _, atype := range []string{"owner", "read", "write", "delete"} {
if _, ok := query[atype]; ok {
users := strings.Split(query.Get(atype), ",")
for _, v := range users {
if uuid.Parse(v) != nil {
q["acl."+atype] = v
} else {
u := user.User{Username: v}
if err := u.SetMongoInfo(); err != nil {
err_msg := "err " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
}
q["acl."+atype] = u.Uuid
}
}
}
}
if _, ok := query["public_owner"]; ok {
// If search is for public_owner AND owner = non-empty string then return zero nodes, no nodes can match this query.
if _, exists := q["acl.owner"]; exists {
return responder.RespondWithPaginatedData(ctx, nodes, limit, offset, 0)
}
q["acl.owner"] = ""
}
// Allowing users to query based on whether ACL is public
for _, atype := range []string{"read", "write", "delete"} {
if _, ok := query["public_"+atype]; ok {
sizeMap := map[string]int{
"$size": 0,
}
// Currently a node cannot be public and have an ACL at the same time, so this returns zero nodes.
if _, exists := q["acl."+atype]; exists {
return responder.RespondWithPaginatedData(ctx, nodes, limit, offset, 0)
} else {
q["acl."+atype] = sizeMap
}
}
}
// Get nodes from db
count, err := nodes.GetPaginated(q, limit, offset)
if err != nil {
err_msg := "err " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
}
return responder.RespondWithPaginatedData(ctx, nodes, limit, offset, count)
}
// WithPermanentRedirect responds with a redirection to the specific path or URL with the
// http.StatusMovedPermanently status.
func (r *GowebHTTPResponder) WithPermanentRedirect(ctx context.Context, pathOrURLSegments ...interface{}) error {
ctx.HttpResponseWriter().Header().Set("Location", paths.PathFromSegments(pathOrURLSegments...))
return r.WithStatus(ctx, http.StatusMovedPermanently)
}
// WriteResponseObject writes the status code and response object to the HttpResponseWriter in
// the specified context, in the format best suited based on the request.
//
// Goweb uses the WebCodecService to decide which codec to use when responding
// see http://godoc.org/github.com/stretchr/codecs/services#WebCodecService for more information.
//
// This method should be used when the Goweb Standard Response Object does not satisfy the needs of
// the API, but other Respond* methods are recommended.
func (a *GowebAPIResponder) WriteResponseObject(ctx context.Context, status int, responseObject interface{}) error {
service := a.GetCodecService()
acceptHeader := ctx.HttpRequest().Header.Get("Accept")
extension := ctx.FileExtension()
hasCallback := len(ctx.QueryValue(CallbackParameter)) > 0
codec, codecError := service.GetCodecForResponding(acceptHeader, extension, hasCallback)
if codecError != nil {
return codecError
}
options := ctx.CodecOptions()
// do we need to add some options?
if _, exists := options[constants.OptionKeyClientCallback]; hasCallback && !exists {
options[constants.OptionKeyClientCallback] = ctx.QueryValue(CallbackParameter)
}
output, marshalErr := service.MarshalWithCodec(codec, responseObject, options)
if marshalErr != nil {
return marshalErr
}
// use the HTTP responder to respond
ctx.HttpResponseWriter().Header().Set("Content-Type", codec.ContentType()) // TODO: test me
a.httpResponder.With(ctx, status, output)
return nil
}
// GET, PUT, DELETE: /node/{nid}/index/{idxType}
func IndexTypedRequest(ctx context.Context) {
nid := ctx.PathValue("nid")
idxType := ctx.PathValue("idxType")
u, err := request.Authenticate(ctx.HttpRequest())
if err != nil && err.Error() != e.NoAuth {
request.AuthError(err, ctx)
return
}
// Fake public user
if u == nil {
u = &user.User{Uuid: ""}
}
// Load node and handle user unauthorized
n, err := node.Load(nid, u.Uuid)
if err != nil {
if err.Error() == e.UnAuth {
responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth)
return
} else if err.Error() == e.MongoDocNotFound {
responder.RespondWithError(ctx, http.StatusNotFound, "Node not found.")
return
} else {
// In theory the db connection could be lost between
// checking user and load but seems unlikely.
err_msg := "Err@index:LoadNode: " + err.Error()
logger.Error(err_msg)
responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
return
}
}
switch ctx.HttpRequest().Method {
case "DELETE":
rights := n.Acl.Check(u.Uuid)
if !rights["write"] {
responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth)
return
}
if _, has := n.Indexes[idxType]; has {
if err := n.DeleteIndex(idxType); err != nil {
err_msg := err.Error()
logger.Error(err_msg)
responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
}
responder.RespondOK(ctx)
} else {
responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Node %s does not have index of type %s.", n.Id, idxType))
}
case "GET":
if v, has := n.Indexes[idxType]; has {
responder.RespondWithData(ctx, map[string]interface{}{idxType: v})
} else {
responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Node %s does not have index of type %s.", n.Id, idxType))
}
case "PUT":
rights := n.Acl.Check(u.Uuid)
if !rights["write"] {
responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth)
return
}
// Gather query params
query := ctx.HttpRequest().URL.Query()
_, forceRebuild := query["force_rebuild"]
if _, has := n.Indexes[idxType]; has {
if idxType == "size" {
responder.RespondOK(ctx)
return
} else if !forceRebuild {
responder.RespondWithError(ctx, http.StatusBadRequest, "This index already exists, please add the parameter 'force_rebuild=1' to force a rebuild of the existing index.")
return
}
}
if !n.HasFile() {
responder.RespondWithError(ctx, http.StatusBadRequest, "Node has no file.")
return
} else if idxType == "" {
responder.RespondWithError(ctx, http.StatusBadRequest, "Index create requires type.")
return
}
if _, ok := index.Indexers[idxType]; !ok && idxType != "bai" && idxType != "subset" && idxType != "column" {
responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Index type %s unavailable.", idxType))
return
}
if idxType == "size" {
responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Index type size is a virtual index and does not require index building."))
return
}
if conf.Bool(conf.Conf["perf-log"]) {
logger.Perf("START indexing: " + nid)
}
if idxType == "bai" {
//bam index is created by the command-line tool samtools
if n.Type == "subset" {
responder.RespondWithError(ctx, http.StatusBadRequest, "Shock does not support bam index creation on subset nodes.")
return
}
if ext := n.FileExt(); ext == ".bam" {
if err := index.CreateBamIndex(n.FilePath()); err != nil {
responder.RespondWithError(ctx, http.StatusInternalServerError, "Error while creating bam index.")
return
}
responder.RespondOK(ctx)
return
} else {
responder.RespondWithError(ctx, http.StatusBadRequest, "Index type bai requires .bam file.")
return
}
}
subsetSize := int64(0)
count := int64(0)
indexFormat := ""
subsetName := ""
if idxType == "subset" {
// Utilizing the multipart form parser since we need to upload a file.
params, files, err := request.ParseMultipartForm(ctx.HttpRequest())
if err != nil {
responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
return
}
parentIndex, hasParent := params["parent_index"]
if !hasParent {
responder.RespondWithError(ctx, http.StatusBadRequest, "Index type subset requires parent_index param.")
return
} else if _, has := n.Indexes[parentIndex]; !has {
responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Node %s does not have index of type %s.", n.Id, parentIndex))
return
}
newIndex, hasName := params["index_name"]
if !hasName {
responder.RespondWithError(ctx, http.StatusBadRequest, "Index type subset requires index_name param.")
return
} else if _, reservedName := index.Indexers[newIndex]; reservedName || newIndex == "bai" {
responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("%s is a reserved index name and cannot be used to create a custom subset index.", newIndex))
return
}
subsetName = newIndex
subsetIndices, hasFile := files["subset_indices"]
if !hasFile {
responder.RespondWithError(ctx, http.StatusBadRequest, "Index type subset requires subset_indices file.")
return
}
f, _ := os.Open(subsetIndices.Path)
defer f.Close()
idxer := index.NewSubsetIndexer(f)
// we default to "array" index format for backwards compatibility
indexFormat = "array"
if n.Indexes[parentIndex].Format == "array" || n.Indexes[parentIndex].Format == "matrix" {
indexFormat = n.Indexes[parentIndex].Format
}
count, subsetSize, err = index.CreateSubsetIndex(&idxer, n.IndexPath()+"/"+newIndex+".idx", n.IndexPath()+"/"+parentIndex+".idx", indexFormat, n.Indexes[parentIndex].TotalUnits)
if err != nil {
logger.Error("err " + err.Error())
responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
return
}
} else if idxType == "column" {
// Gather query params
query := ctx.HttpRequest().URL.Query()
if n.Type == "subset" {
responder.RespondWithError(ctx, http.StatusBadRequest, "Shock does not support column index creation on subset nodes.")
return
}
if _, exists := query["number"]; !exists {
err_msg := "Index type column requires a number parameter in the url."
logger.Error(err_msg)
responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
return
}
num_str := query.Get("number")
idxType = idxType + num_str
num, err := strconv.Atoi(num_str)
if err != nil || num < 1 {
err_msg := "Index type column requires a number parameter in the url of an integer greater than zero."
logger.Error(err_msg)
responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
return
}
f, _ := os.Open(n.FilePath())
defer f.Close()
idxer := index.NewColumnIndexer(f)
count, indexFormat, err = index.CreateColumnIndex(&idxer, num, n.IndexPath()+"/"+idxType+".idx")
if err != nil {
logger.Error("err " + err.Error())
responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
return
}
} else {
if n.Type == "subset" && (idxType != "chunkrecord" || n.Subset.Parent.IndexName != "record") {
responder.RespondWithError(ctx, http.StatusBadRequest, "For subset nodes, Shock currently only supports subset and chunkrecord indexes. Also, for a chunkrecord index, the subset node must have been generated from a record index.")
return
}
newIndexer := index.Indexers[idxType]
f, _ := os.Open(n.FilePath())
defer f.Close()
var idxer index.Indexer
if n.Type == "subset" {
idxer = newIndexer(f, n.Type, n.Subset.Index.Format, n.IndexPath()+"/"+n.Subset.Parent.IndexName+".idx")
} else {
idxer = newIndexer(f, n.Type, "", "")
}
count, indexFormat, err = idxer.Create(n.IndexPath() + "/" + idxType + ".idx")
if err != nil {
logger.Error("err " + err.Error())
responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
return
}
}
if count == 0 {
responder.RespondWithError(ctx, http.StatusBadRequest, "Index empty.")
return
}
idxInfo := node.IdxInfo{
Type: idxType,
TotalUnits: count,
AvgUnitSize: n.File.Size / count,
Format: indexFormat,
}
//if idxType == "chunkrecord" {
// idxInfo.AvgUnitSize = conf.CHUNK_SIZE
//}
if idxType == "subset" {
idxType = subsetName
idxInfo.AvgUnitSize = subsetSize / count
}
if err := n.SetIndexInfo(idxType, idxInfo); err != nil {
logger.Error("[email protected]: " + err.Error())
}
if conf.Bool(conf.Conf["perf-log"]) {
logger.Perf("END indexing: " + nid)
}
responder.RespondOK(ctx)
return
default:
responder.RespondWithError(ctx, http.StatusNotImplemented, "This request type is not implemented.")
}
return
}
// WillHandle checks whether this handler will be used to handle the specified
// request or not.
func (p *PathMatchHandler) WillHandle(c context.Context) (bool, error) {
// check each matcher func
matcherFuncMatches := true
matcherFuncDecisionMade := false
for _, matcherFunc := range p.MatcherFuncs {
decision, matcherFuncErr := matcherFunc(c)
if matcherFuncErr != nil {
return false, matcherFuncErr
}
switch decision {
case NoMatch:
matcherFuncMatches = false
matcherFuncDecisionMade = true
break
case Match:
matcherFuncMatches = true
matcherFuncDecisionMade = true
break
}
if matcherFuncDecisionMade {
break
}
}
// check HTTP methods
var httpMethodMatch bool = false
if len(p.HttpMethods) == 0 {
// no specific HTTP methods
httpMethodMatch = true
} else {
for _, httpMethod := range p.HttpMethods {
if httpMethod == c.MethodString() {
httpMethodMatch = true
break
}
}
}
// cancel early if we didn't get an HTTP Method match
if !httpMethodMatch {
return false, nil
}
// check path match
pathMatch := p.PathPattern.GetPathMatch(c.Path())
var allMatch bool
if matcherFuncDecisionMade {
allMatch = matcherFuncMatches
} else {
allMatch = pathMatch.Matches
}
if allMatch {
// save the match parameters for later
c.Data().Set(context.DataKeyPathParameters, pathMatch.Parameters)
}
return allMatch, nil
}
// GET: /node
// To do:
// - Iterate node queries
func (cr *NodeController) ReadMany(ctx context.Context) error {
u, err := request.Authenticate(ctx.HttpRequest())
if err != nil && err.Error() != e.NoAuth {
return request.AuthError(err, ctx)
}
// Gather query params
query := ctx.HttpRequest().URL.Query()
// Setup query and nodes objects
// Note: query is composed of 3 sub-query objects:
// 1) qPerm - user permissions (system-defined)
// 2) qOpts - query options (user-defined)
// 3) qAcls - ACL queries (user-defined)
q := bson.M{}
qPerm := bson.M{}
qOpts := bson.M{}
qAcls := bson.M{}
nodes := node.Nodes{}
if u != nil {
// Skip this part if user is an admin
if !u.Admin {
qPerm["$or"] = []bson.M{bson.M{"acl.read": "public"}, bson.M{"acl.read": u.Uuid}, bson.M{"acl.owner": u.Uuid}}
}
} else {
// User is anonymous
if conf.ANON_READ {
// select on only nodes that are publicly readable
qPerm["acl.read"] = "public"
} else {
return responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth)
}
}
// bson.M is a convenient alias for a map[string]interface{} map, useful for dealing with BSON in a native way.
var OptsMArray []bson.M
// Gather params to make db query. Do not include the following list.
if _, ok := query["query"]; ok {
paramlist := map[string]int{"query": 1, "limit": 1, "offset": 1, "order": 1, "direction": 1, "distinct": 1}
for key := range query {
if _, found := paramlist[key]; !found {
keyStr := fmt.Sprintf("attributes.%s", key)
for _, value := range query[key] {
if value != "" {
OptsMArray = append(OptsMArray, parseOption(keyStr, value))
} else {
OptsMArray = append(OptsMArray, bson.M{keyStr: map[string]bool{"$exists": true}})
}
}
}
}
} else if _, ok := query["querynode"]; ok {
paramlist := map[string]int{"querynode": 1, "limit": 1, "offset": 1, "order": 1, "direction": 1, "distinct": 1, "owner": 1, "read": 1, "write": 1, "delete": 1, "public_owner": 1, "public_read": 1, "public_write": 1, "public_delete": 1}
for key := range query {
if _, found := paramlist[key]; !found {
for _, value := range query[key] {
if value != "" {
OptsMArray = append(OptsMArray, parseOption(key, value))
} else {
OptsMArray = append(OptsMArray, bson.M{key: map[string]bool{"$exists": true}})
}
}
}
}
}
if len(OptsMArray) > 0 {
qOpts["$and"] = OptsMArray
}
// bson.M is a convenient alias for a map[string]interface{} map, useful for dealing with BSON in a native way.
var AclsMArray []bson.M
// Allowing user to query based on ACL's with a comma-separated list of users.
// Restricting ACL queries to just the querynode operation.
// Users can be written as a username or a UUID.
if _, qok := query["querynode"]; qok {
for _, atype := range []string{"owner", "read", "write", "delete"} {
if _, ok := query[atype]; ok {
users := strings.Split(query.Get(atype), ",")
for _, v := range users {
if uuid.Parse(v) != nil {
AclsMArray = append(AclsMArray, bson.M{"acl." + atype: v})
} else {
u := user.User{Username: v}
if err := u.SetMongoInfo(); err != nil {
err_msg := "err " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
}
AclsMArray = append(AclsMArray, bson.M{"acl." + atype: u.Uuid})
}
}
}
}
// Allowing users to query based on whether ACL is public
for _, atype := range []string{"owner", "read", "write", "delete"} {
if _, ok := query["public_"+atype]; ok {
AclsMArray = append(AclsMArray, bson.M{"acl." + atype: "public"})
}
}
}
if len(AclsMArray) > 0 {
qAcls["$and"] = AclsMArray
}
// Combine permissions query with query parameters and ACL query into one AND clause
q["$and"] = []bson.M{qPerm, qOpts, qAcls}
// process distinct query
if _, ok := query["distinct"]; ok {
dField := query.Get("distinct")
if !node.HasAttributeField(dField) {
err_msg := "err unable to run distinct query on non-indexed attributes field: " + dField
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
}
results, err := node.DbFindDistinct(q, dField)
if err != nil {
err_msg := "err " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
}
return responder.RespondWithData(ctx, results)
}
// defaults
order := "created_on"
direction := "-"
limit := 25
offset := 0
// get from query
if _, ok := query["query"]; ok {
if _, ok := query["order"]; ok {
order = fmt.Sprintf("attributes.%s", query.Get("order"))
}
} else {
if _, ok := query["order"]; ok {
order = query.Get("order")
}
}
if _, ok := query["direction"]; ok {
if query.Get("direction") == "asc" {
direction = ""
}
}
if _, ok := query["limit"]; ok {
limit = util.ToInt(query.Get("limit"))
}
if _, ok := query["offset"]; ok {
offset = util.ToInt(query.Get("offset"))
}
// Get nodes from db
order = direction + order
count, err := nodes.GetPaginated(q, limit, offset, order)
if err != nil {
err_msg := "err " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
}
return responder.RespondWithPaginatedData(ctx, nodes, limit, offset, count)
}
func ApiUrl(ctx context.Context) string {
if conf.Conf["api-url"] != "" {
return conf.Conf["api-url"]
}
return "http://" + ctx.HttpRequest().Host
}
// GET: /node/{id}
func (cr *NodeController) Read(id string, ctx context.Context) error {
u, err := request.Authenticate(ctx.HttpRequest())
if err != nil && err.Error() != e.NoAuth {
return request.AuthError(err, ctx)
}
// Fake public user
if u == nil {
if conf.Bool(conf.Conf["anon-read"]) {
u = &user.User{Uuid: ""}
} else {
return responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth)
}
}
// Load node and handle user unauthorized
n, err := node.Load(id, u.Uuid)
if err != nil {
if err.Error() == e.UnAuth {
return responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth)
} else if err.Error() == e.MongoDocNotFound {
return responder.RespondWithError(ctx, http.StatusNotFound, "Node not found")
} else {
// In theory the db connection could be lost between
// checking user and load but seems unlikely.
logger.Error("Err@node_Read:LoadNode:" + id + ":" + err.Error())
n, err = node.LoadFromDisk(id)
if err.Error() == "Node does not exist" {
logger.Error(err.Error())
return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
} else if err != nil {
err_msg := "Err@node_Read:LoadNodeFromDisk:" + id + ":" + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
}
}
}
// Gather query params
query := ctx.HttpRequest().URL.Query()
var fFunc filter.FilterFunc = nil
if _, ok := query["filter"]; ok {
if filter.Has(query.Get("filter")) {
fFunc = filter.Filter(query.Get("filter"))
}
}
// Switch though param flags
// ?download=1 or ?download_raw=1
_, download_raw := query["download_raw"]
if _, ok := query["download"]; ok || download_raw {
if !n.HasFile() {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Node has no file")
}
filename := n.Id
if _, ok := query["filename"]; ok {
filename = query.Get("filename")
}
_, seek_ok := query["seek"]
if _, length_ok := query["length"]; seek_ok || length_ok {
if n.Type == "subset" {
return responder.RespondWithError(ctx, http.StatusBadRequest, "subset nodes do not currently support seek/length offset retrieval")
}
var seek int64
var length int64
if !seek_ok {
seek = 0
length_str := query.Get("length")
length, err = strconv.ParseInt(length_str, 10, 0)
if err != nil {
return responder.RespondWithError(ctx, http.StatusBadRequest, "length must be an integer value")
}
} else if !length_ok {
seek_str := query.Get("seek")
seek, err = strconv.ParseInt(seek_str, 10, 0)
if err != nil {
return responder.RespondWithError(ctx, http.StatusBadRequest, "seek must be an integer value")
}
length = n.File.Size - seek
} else {
seek_str := query.Get("seek")
seek, err = strconv.ParseInt(seek_str, 10, 0)
if err != nil {
return responder.RespondWithError(ctx, http.StatusBadRequest, "seek must be an integer value")
}
length_str := query.Get("length")
length, err = strconv.ParseInt(length_str, 10, 0)
if err != nil {
return responder.RespondWithError(ctx, http.StatusBadRequest, "length must be an integer value")
}
}
r, err := n.FileReader()
defer r.Close()
if err != nil {
err_msg := "Err@node_Read:Open: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
}
s := &request.Streamer{R: []file.SectionReader{}, W: ctx.HttpResponseWriter(), ContentType: "application/octet-stream", Filename: filename, Size: length, Filter: fFunc}
s.R = append(s.R, io.NewSectionReader(r, seek, length))
if download_raw {
err = s.StreamRaw()
if err != nil {
// causes "multiple response.WriteHeader calls" error but better than no response
err_msg := "err:@node_Read s.StreamRaw: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
}
} else {
err = s.Stream()
if err != nil {
// causes "multiple response.WriteHeader calls" error but better than no response
err_msg := "err:@node_Read s.Stream: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
}
}
} else if _, ok := query["index"]; ok {
//handling bam file
if query.Get("index") == "bai" {
if n.Type == "subset" {
return responder.RespondWithError(ctx, http.StatusBadRequest, "subset nodes do not support bam indices")
}
s := &request.Streamer{R: []file.SectionReader{}, W: ctx.HttpResponseWriter(), ContentType: "application/octet-stream", Filename: filename, Size: n.File.Size, Filter: fFunc}
var region string
if _, ok := query["region"]; ok {
//retrieve alingments overlapped with specified region
region = query.Get("region")
}
argv, err := request.ParseSamtoolsArgs(ctx)
if err != nil {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Invaid args in query url")
}
err = s.StreamSamtools(n.FilePath(), region, argv...)
if err != nil {
return responder.RespondWithError(ctx, http.StatusBadRequest, "error while invoking samtools")
}
return nil
}
// open file
r, err := n.FileReader()
defer r.Close()
if err != nil {
err_msg := "Err@node_Read:Open: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
}
// load index obj and info
idxName := query.Get("index")
idxInfo, ok := n.Indexes[idxName]
if !ok {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index")
}
idx, err := n.DynamicIndex(idxName)
if err != nil {
return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
}
if idx.Type() == "virtual" {
if n.Type == "subset" {
return responder.RespondWithError(ctx, http.StatusBadRequest, "subset nodes do not currently support virtual indices")
}
csize := conf.CHUNK_SIZE
if _, ok := query["chunk_size"]; ok {
csize, err = strconv.ParseInt(query.Get("chunk_size"), 10, 64)
if err != nil {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid chunk_size")
}
}
idx.Set(map[string]interface{}{"ChunkSize": csize})
}
var size int64 = 0
s := &request.Streamer{R: []file.SectionReader{}, W: ctx.HttpResponseWriter(), ContentType: "application/octet-stream", Filename: filename, Filter: fFunc}
_, hasPart := query["part"]
if n.Type == "subset" && idxName == "chunkrecord" {
recordIdxName := "record"
recordIdxInfo, ok := n.Indexes[recordIdxName]
if !ok {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid request, record index must exist to retrieve chunkrecord index on a subset node.")
}
recordIdx, err := n.DynamicIndex(recordIdxName)
if err != nil {
return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
}
if !hasPart {
// download full subset file
fullRange := "1-" + strconv.FormatInt(recordIdxInfo.TotalUnits, 10)
recSlice, err := recordIdx.Range(fullRange, n.IndexPath()+"/"+recordIdxName+".idx", recordIdxInfo.TotalUnits)
if err != nil {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index subset")
}
for _, rec := range recSlice {
size += rec[1]
s.R = append(s.R, io.NewSectionReader(r, rec[0], rec[1]))
}
} else if hasPart {
// download parts
for _, p := range query["part"] {
chunkRecSlice, err := idx.Range(p, n.IndexPath()+"/"+idxName+".idx", idxInfo.TotalUnits)
if err != nil {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index part")
}
// This gets us the parts of the chunkrecord index, but we still need to convert these to record indices.
for _, chunkRec := range chunkRecSlice {
start := (chunkRec[0] / 16) + 1
stop := (start - 1) + (chunkRec[1] / 16)
recSlice, err := recordIdx.Range(strconv.FormatInt(start, 10)+"-"+strconv.FormatInt(stop, 10), n.IndexPath()+"/"+recordIdxName+".idx", recordIdxInfo.TotalUnits)
if err != nil {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index subset")
}
for _, rec := range recSlice {
size += rec[1]
s.R = append(s.R, io.NewSectionReader(r, rec[0], rec[1]))
}
}
}
} else {
// bad request
return responder.RespondWithError(ctx, http.StatusBadRequest, "Index parameter requires part parameter")
}
} else {
if (!hasPart) && (idxInfo.Type == "subset") {
// download full subset file
fullRange := "1-" + strconv.FormatInt(idxInfo.TotalUnits, 10)
recSlice, err := idx.Range(fullRange, n.IndexPath()+"/"+idxName+".idx", idxInfo.TotalUnits)
if err != nil {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index subset")
}
for _, rec := range recSlice {
size += rec[1]
s.R = append(s.R, io.NewSectionReader(r, rec[0], rec[1]))
}
} else if hasPart {
// download parts
for _, p := range query["part"] {
// special case for subset ranges
if idxInfo.Type == "subset" {
recSlice, err := idx.Range(p, n.IndexPath()+"/"+idxName+".idx", idxInfo.TotalUnits)
if err != nil {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index part")
}
for _, rec := range recSlice {
size += rec[1]
s.R = append(s.R, io.NewSectionReader(r, rec[0], rec[1]))
}
} else {
pos, length, err := idx.Part(p, n.IndexPath()+"/"+idxName+".idx", idxInfo.TotalUnits)
if err != nil {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index part")
}
size += length
s.R = append(s.R, io.NewSectionReader(r, pos, length))
}
}
} else {
// bad request
return responder.RespondWithError(ctx, http.StatusBadRequest, "Index parameter requires part parameter")
}
}
s.Size = size
if download_raw {
err = s.StreamRaw()
if err != nil {
// causes "multiple response.WriteHeader calls" error but better than no response
err_msg := "err:@node_Read s.StreamRaw: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
}
} else {
err = s.Stream()
if err != nil {
// causes "multiple response.WriteHeader calls" error but better than no response
err_msg := "err:@node_Read s.Stream: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
}
}
// download full file
} else {
if n.Type == "subset" {
// open file
r, err := n.FileReader()
defer r.Close()
if err != nil {
err_msg := "Err@node_Read:Open: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
}
idx := index.New()
s := &request.Streamer{R: []file.SectionReader{}, W: ctx.HttpResponseWriter(), ContentType: "application/octet-stream", Filename: filename, Size: n.File.Size, Filter: fFunc}
fullRange := "1-" + strconv.FormatInt(n.Subset.Index.TotalUnits, 10)
recSlice, err := idx.Range(fullRange, n.Path()+"/"+n.Id+".subset.idx", n.Subset.Index.TotalUnits)
if err != nil {
return responder.RespondWithError(ctx, http.StatusInternalServerError, "Invalid data index for subset node.")
}
for _, rec := range recSlice {
s.R = append(s.R, io.NewSectionReader(r, rec[0], rec[1]))
}
if download_raw {
err = s.StreamRaw()
if err != nil {
// causes "multiple response.WriteHeader calls" error but better than no response
err_msg := "err:@node_Read s.StreamRaw: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
}
} else {
err = s.Stream()
if err != nil {
// causes "multiple response.WriteHeader calls" error but better than no response
err_msg := "err:@node_Read s.Stream: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
}
}
} else {
nf, err := n.FileReader()
defer nf.Close()
if err != nil {
// File not found or some sort of file read error.
// Probably deserves more checking
err_msg := "err:@node_Read node.FileReader: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
}
s := &request.Streamer{R: []file.SectionReader{nf}, W: ctx.HttpResponseWriter(), ContentType: "application/octet-stream", Filename: filename, Size: n.File.Size, Filter: fFunc}
if download_raw {
err = s.StreamRaw()
if err != nil {
// causes "multiple response.WriteHeader calls" error but better than no response
err_msg := "err:@node_Read s.StreamRaw: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
}
} else {
err = s.Stream()
if err != nil {
// causes "multiple response.WriteHeader calls" error but better than no response
err_msg := "err:@node_Read s.Stream: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
}
}
}
}
} else if _, ok := query["download_url"]; ok {
if n.Type == "subset" {
return responder.RespondWithError(ctx, http.StatusBadRequest, "subset nodes do not currently support download_url operation")
}
if !n.HasFile() {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Node has no file")
} else {
options := map[string]string{}
if _, ok := query["filename"]; ok {
options["filename"] = query.Get("filename")
}
if p, err := preauth.New(util.RandString(20), "download", n.Id, options); err != nil {
err_msg := "err:@node_Read download_url: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
} else {
return responder.RespondWithData(ctx, util.UrlResponse{Url: util.ApiUrl(ctx) + "/preauth/" + p.Id, ValidTill: p.ValidTill.Format(time.ANSIC)})
}
}
} else if _, ok := query["download_post"]; ok {
// This is a request to post the node to another Shock server. The 'post_url' parameter is required.
// By default the post operation will include the data file and attributes (these options can be set
// with post_data=0/1 and post_attr=0/1).
if n.Type == "subset" {
return responder.RespondWithError(ctx, http.StatusBadRequest, "subset nodes do not currently support download_post operation")
}
post_url := ""
if _, ok := query["post_url"]; ok {
post_url = query.Get("post_url")
} else {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Request type requires post_url parameter of where to post new Shock node")
}
post_opts := map[string]int{
"post_data": 1,
"post_attr": 1,
}
for k, _ := range post_opts {
if _, ok := query[k]; ok {
if query.Get(k) == "0" {
post_opts[k] = 0
} else if query.Get(k) == "1" {
post_opts[k] = 1
} else {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Parameter "+k+" must be either 0 or 1")
}
}
}
form := client.NewForm()
form.AddParam("file_name", n.File.Name)
if post_opts["post_data"] == 1 {
form.AddFile("upload", n.FilePath())
}
if post_opts["post_attr"] == 1 && n.Attributes != nil {
attr, _ := json.Marshal(n.Attributes)
form.AddParam("attributes_str", string(attr[:]))
}
err = form.Create()
if err != nil {
err_msg := "could not create multipart form for posting to Shock server: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
}
headers := client.Header{
"Content-Type": form.ContentType,
"Content-Length": strconv.FormatInt(form.Length, 10),
}
if _, hasAuth := ctx.HttpRequest().Header["Authorization"]; hasAuth {
headers["Authorization"] = ctx.HttpRequest().Header.Get("Authorization")
}
if res, err := client.Do("POST", post_url, headers, form.Reader); err == nil {
if res.StatusCode == 200 {
r := responseWrapper{}
body, _ := ioutil.ReadAll(res.Body)
if err = json.Unmarshal(body, &r); err != nil {
err_msg := "err:@node_Read POST: " + err.Error()
logger.Error(err_msg)
return responder.WriteResponseObject(ctx, http.StatusInternalServerError, err_msg)
} else {
return responder.WriteResponseObject(ctx, http.StatusOK, r)
}
} else {
r := responseWrapper{}
body, _ := ioutil.ReadAll(res.Body)
if err = json.Unmarshal(body, &r); err == nil {
err_msg := res.Status + ": " + (*r.Error)[0]
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
} else {
err_msg := "request error: " + res.Status
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
}
}
} else {
return err
}
} else {
// Base case respond with node in json
return responder.RespondWithData(ctx, n)
}
return nil
}
// GET, POST, PUT, DELETE: /node/{nid}/acl/{type}
func AclTypedRequest(ctx context.Context) {
nid := ctx.PathValue("nid")
rtype := ctx.PathValue("type")
rmeth := ctx.HttpRequest().Method
query := ctx.HttpRequest().URL.Query()
verbosity := ""
if _, ok := query["verbosity"]; ok {
verbosity = query.Get("verbosity")
}
u, err := request.Authenticate(ctx.HttpRequest())
if err != nil && err.Error() != e.NoAuth {
request.AuthError(err, ctx)
return
}
if !validAclTypes[rtype] {
responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid acl type")
return
}
// Load node by id
n, err := node.Load(nid)
if err != nil {
if err == mgo.ErrNotFound {
responder.RespondWithError(ctx, http.StatusNotFound, e.NodeNotFound)
return
} else {
// In theory the db connection could be lost between
// checking user and load but seems unlikely.
err_msg := "Err@node_Acl:LoadNode: " + nid + err.Error()
logger.Error(err_msg)
responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
return
}
}
// public user (no auth) can perform a GET operation given the proper node permissions
if u == nil {
rights := n.Acl.Check("public")
if rmeth == "GET" && conf.ANON_READ && (rights["read"] || n.Acl.Owner == "public") {
responder.RespondWithData(ctx, n.Acl.FormatDisplayAcl(verbosity))
return
} else {
responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth)
return
}
}
// Users that are not an admin or the node owner can only delete themselves from an ACL.
if n.Acl.Owner != u.Uuid && u.Admin == false {
// Users that are not an admin or the node owner cannot remove public from ACL's.
if rtype == "public_read" || rtype == "public_write" || rtype == "public_delete" || rtype == "public_all" {
responder.RespondWithError(ctx, http.StatusBadRequest, "Users that are not node owners can only delete themselves from ACLs.")
return
}
// Parse user list
ids, err := parseAclRequestTyped(ctx)
if err != nil {
responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
return
}
if rmeth == "DELETE" {
if len(ids) != 1 || (len(ids) == 1 && ids[0] != u.Uuid) {
responder.RespondWithError(ctx, http.StatusBadRequest, "Users that are not node owners can delete only themselves from ACLs.")
return
}
if rtype == "owner" {
responder.RespondWithError(ctx, http.StatusBadRequest, "Deleting node ownership is not a supported request type.")
return
}
if rtype == "all" {
n.Acl.UnSet(ids[0], map[string]bool{"read": true, "write": true, "delete": true})
} else {
n.Acl.UnSet(ids[0], map[string]bool{rtype: true})
}
n.Save()
responder.RespondWithData(ctx, n.Acl.FormatDisplayAcl(verbosity))
return
}
responder.RespondWithError(ctx, http.StatusBadRequest, "Users that are not node owners can only delete themselves from ACLs.")
return
}
// At this point we know we're dealing with an admin or the node owner.
// Admins and node owners can view/edit/delete ACLs
if rmeth == "GET" {
responder.RespondWithData(ctx, n.Acl.FormatDisplayAcl(verbosity))
return
} else if rmeth == "POST" || rmeth == "PUT" {
if rtype == "public_read" || rtype == "public_write" || rtype == "public_delete" || rtype == "public_all" {
if rtype == "public_read" {
n.Acl.Set("public", map[string]bool{"read": true})
} else if rtype == "public_write" {
n.Acl.Set("public", map[string]bool{"write": true})
} else if rtype == "public_delete" {
n.Acl.Set("public", map[string]bool{"delete": true})
} else if rtype == "public_all" {
n.Acl.Set("public", map[string]bool{"read": true, "write": true, "delete": true})
}
n.Save()
responder.RespondWithData(ctx, n.Acl.FormatDisplayAcl(verbosity))
return
}
// Parse user list
ids, err := parseAclRequestTyped(ctx)
if err != nil {
responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
return
}
if rtype == "owner" {
if len(ids) == 1 {
n.Acl.SetOwner(ids[0])
} else {
responder.RespondWithError(ctx, http.StatusBadRequest, "Too many users. Nodes may have only one owner.")
return
}
} else if rtype == "all" {
for _, i := range ids {
n.Acl.Set(i, map[string]bool{"read": true, "write": true, "delete": true})
}
} else {
for _, i := range ids {
n.Acl.Set(i, map[string]bool{rtype: true})
}
}
n.Save()
responder.RespondWithData(ctx, n.Acl.FormatDisplayAcl(verbosity))
return
} else if rmeth == "DELETE" {
if rtype == "public_read" || rtype == "public_write" || rtype == "public_delete" || rtype == "public_all" {
if rtype == "public_read" {
n.Acl.UnSet("public", map[string]bool{"read": true})
} else if rtype == "public_write" {
n.Acl.UnSet("public", map[string]bool{"write": true})
} else if rtype == "public_delete" {
n.Acl.UnSet("public", map[string]bool{"delete": true})
} else if rtype == "public_all" {
n.Acl.UnSet("public", map[string]bool{"read": true, "write": true, "delete": true})
}
n.Save()
responder.RespondWithData(ctx, n.Acl.FormatDisplayAcl(verbosity))
return
}
// Parse user list
ids, err := parseAclRequestTyped(ctx)
if err != nil {
responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
return
}
if rtype == "owner" {
responder.RespondWithError(ctx, http.StatusBadRequest, "Deleting ownership is not a supported request type.")
return
} else if rtype == "all" {
for _, i := range ids {
n.Acl.UnSet(i, map[string]bool{"read": true, "write": true, "delete": true})
}
} else {
for _, i := range ids {
n.Acl.UnSet(i, map[string]bool{rtype: true})
}
}
n.Save()
responder.RespondWithData(ctx, n.Acl.FormatDisplayAcl(verbosity))
return
} else {
responder.RespondWithError(ctx, http.StatusNotImplemented, "This request type is not implemented.")
return
}
}
// PUT: /node/{id} -> multipart-form
func (cr *NodeController) Replace(id string, ctx context.Context) error {
u, err := request.Authenticate(ctx.HttpRequest())
if err != nil && err.Error() != e.NoAuth {
return request.AuthError(err, ctx)
}
// Fake public user
if u == nil {
u = &user.User{Uuid: ""}
}
n, err := node.Load(id, u.Uuid)
if err != nil {
if err.Error() == e.UnAuth {
return responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth)
} else if err.Error() == e.MongoDocNotFound {
return responder.RespondWithError(ctx, http.StatusNotFound, "Node not found")
} else {
// In theory the db connection could be lost between
// checking user and load but seems unlikely.
err_msg := "Err@node_Update:LoadNode: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
}
}
rights := n.Acl.Check(u.Uuid)
if !rights["write"] {
return responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth)
}
if conf.Bool(conf.Conf["perf-log"]) {
logger.Perf("START PUT data: " + id)
}
params, files, err := request.ParseMultipartForm(ctx.HttpRequest())
if err != nil {
err_msg := "err@node_ParseMultipartForm: " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
}
if _, hasCopyData := params["copy_data"]; hasCopyData {
_, err = node.Load(params["copy_data"], u.Uuid)
if err != nil {
return request.AuthError(err, ctx)
}
}
if _, hasParentNode := params["parent_node"]; hasParentNode {
_, err = node.Load(params["parent_node"], u.Uuid)
if err != nil {
return request.AuthError(err, ctx)
}
}
err = n.Update(params, files)
if err != nil {
err_msg := "err@node_Update: " + id + ": " + err.Error()
logger.Error(err_msg)
return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
}
responder.RespondWithData(ctx, n)
if conf.Bool(conf.Conf["perf-log"]) {
logger.Perf("END PUT data: " + id)
}
return nil
}
func addResponseHeaders(ctx context.Context) {
ctx.HttpResponseWriter().Header().Set("Connection", "close")
ctx.HttpResponseWriter().Header().Set("Access-Control-Allow-Headers", "Authorization")
ctx.HttpResponseWriter().Header().Set("Access-Control-Allow-Methods", "POST, GET, PUT, DELETE, OPTIONS")
ctx.HttpResponseWriter().Header().Set("Access-Control-Allow-Origin", "*")
}
func ApiUrl(ctx context.Context) string {
if conf.API_URL != "" {
return conf.API_URL
}
return "http://" + ctx.HttpRequest().Host
}