func (sh *Handler) serveFiles(rw http.ResponseWriter, req *http.Request) {
ret := jsonMap()
defer httputil.ReturnJson(rw, ret)
br := blobref.Parse(req.FormValue("wholedigest"))
if br == nil {
ret["error"] = "Missing or invalid 'wholedigest' param"
ret["errorType"] = "input"
return
}
files, err := sh.index.ExistingFileSchemas(br)
if err != nil {
ret["error"] = err.String()
ret["errorType"] = "server"
return
}
strList := []string{}
for _, br := range files {
strList = append(strList, br.String())
}
ret["files"] = strList
return
}
func (h *JSONSignHandler) handleVerify(rw http.ResponseWriter, req *http.Request) {
req.ParseForm()
sjson := req.FormValue("sjson")
if sjson == "" {
http.Error(rw, "missing \"sjson\" parameter", http.StatusBadRequest)
return
}
m := make(map[string]interface{})
// TODO: use a different fetcher here that checks memory, disk,
// the internet, etc.
fetcher := h.pubKeyFetcher
vreq := jsonsign.NewVerificationRequest(sjson, fetcher)
if vreq.Verify() {
m["signatureValid"] = 1
m["signerKeyId"] = vreq.SignerKeyId
m["verifiedData"] = vreq.PayloadMap
} else {
errStr := vreq.Err.String()
m["signatureValid"] = 0
m["errorMessage"] = errStr
}
rw.WriteHeader(http.StatusOK) // no HTTP response code fun, error info in JSON
httputil.ReturnJson(rw, m)
}
func (sh *Handler) serveSignerPaths(rw http.ResponseWriter, req *http.Request) {
ret := jsonMap()
defer httputil.ReturnJson(rw, ret)
defer setPanicError(ret)
signer := blobref.MustParse(mustGet(req, "signer"))
target := blobref.MustParse(mustGet(req, "target"))
paths, err := sh.index.PathsOfSignerTarget(signer, target)
if err != nil {
ret["error"] = err.String()
} else {
jpaths := []map[string]interface{}{}
for _, path := range paths {
jpaths = append(jpaths, map[string]interface{}{
"claimRef": path.Claim.String(),
"baseRef": path.Base.String(),
"suffix": path.Suffix,
})
}
ret["paths"] = jpaths
dr := sh.NewDescribeRequest()
for _, path := range paths {
dr.Describe(path.Base, 2)
}
dr.PopulateJSON(ret)
}
}
func handleVerify(conn http.ResponseWriter, req *http.Request) {
if !(req.Method == "POST" && req.URL.Path == "/camli/sig/verify") {
httputil.BadRequestError(conn, "Inconfigured handler.")
return
}
req.ParseForm()
sjson := req.FormValue("sjson")
if sjson == "" {
httputil.BadRequestError(conn, "Missing sjson parameter.")
return
}
m := make(map[string]interface{})
vreq := jsonsign.NewVerificationRequest(sjson, pubKeyFetcher)
if vreq.Verify() {
m["signatureValid"] = 1
m["verifiedData"] = vreq.PayloadMap
} else {
errStr := vreq.Err.String()
m["signatureValid"] = 0
m["errorMessage"] = errStr
}
conn.WriteHeader(http.StatusOK) // no HTTP response code fun, error info in JSON
httputil.ReturnJson(conn, m)
}
func (sh *Handler) serveRecentPermanodes(rw http.ResponseWriter, req *http.Request) {
ret := jsonMap()
defer httputil.ReturnJson(rw, ret)
ch := make(chan *Result)
errch := make(chan os.Error)
go func() {
errch <- sh.index.GetRecentPermanodes(ch, []*blobref.BlobRef{sh.owner}, 50)
}()
dr := sh.NewDescribeRequest()
recent := jsonMapList()
for res := range ch {
dr.Describe(res.BlobRef, 2)
jm := jsonMap()
jm["blobref"] = res.BlobRef.String()
jm["owner"] = res.Signer.String()
t := time.SecondsToUTC(res.LastModTime)
jm["modtime"] = t.Format(time.RFC3339)
recent = append(recent, jm)
}
err := <-errch
if err != nil {
// TODO: return error status code
ret["error"] = err.String()
return
}
ret["recent"] = recent
dr.PopulateJSON(ret)
}
func handleRemove(conn http.ResponseWriter, req *http.Request, storage blobserver.Storage) {
if w, ok := storage.(blobserver.ContextWrapper); ok {
storage = w.WrapContext(req)
}
if req.Method != "POST" {
log.Fatalf("Invalid method; handlers misconfigured")
}
configer, ok := storage.(blobserver.Configer)
if !ok {
conn.WriteHeader(http.StatusForbidden)
fmt.Fprintf(conn, "Remove handler's blobserver.Storage isn't a blobserver.Configuer; can't remove")
return
}
if !configer.Config().IsQueue {
conn.WriteHeader(http.StatusForbidden)
fmt.Fprintf(conn, "Can only remove blobs from a queue.\n")
return
}
n := 0
toRemove := make([]*blobref.BlobRef, 0)
toRemoveStr := make([]string, 0)
for {
n++
if n > maxRemovesPerRequest {
httputil.BadRequestError(conn,
fmt.Sprintf("Too many removes in this request; max is %d", maxRemovesPerRequest))
return
}
key := fmt.Sprintf("blob%v", n)
value := req.FormValue(key)
if value == "" {
break
}
ref := blobref.Parse(value)
if ref == nil {
httputil.BadRequestError(conn, "Bogus blobref for key "+key)
return
}
toRemove = append(toRemove, ref)
toRemoveStr = append(toRemoveStr, ref.String())
}
err := storage.RemoveBlobs(toRemove)
if err != nil {
conn.WriteHeader(http.StatusInternalServerError)
log.Printf("Server error during remove: %v", err)
fmt.Fprintf(conn, "Server error")
return
}
reply := make(map[string]interface{}, 0)
reply["removed"] = toRemoveStr
httputil.ReturnJson(conn, reply)
}
func (ui *UIHandler) serveUploadHelper(rw http.ResponseWriter, req *http.Request) {
rollSum := req.URL.Query().Get("rollsum") == "1"
ret := make(map[string]interface{})
defer httputil.ReturnJson(rw, ret)
if ui.Storage == nil {
ret["error"] = "No BlobRoot configured"
ret["errorType"] = "server"
return
}
mr, err := req.MultipartReader()
if err != nil {
ret["error"] = "reading body: " + err.String()
ret["errorType"] = "server"
return
}
got := make([]map[string]interface{}, 0)
for {
part, err := mr.NextPart()
if err == os.EOF {
break
}
if err != nil {
ret["error"] = "reading body: " + err.String()
ret["errorType"] = "server"
break
}
fileName := part.FileName()
if fileName == "" {
continue
}
writeFn := schema.WriteFileFromReader
if rollSum {
writeFn = schema.WriteFileFromReaderRolling
}
br, err := writeFn(ui.Storage, fileName, part)
if err == nil {
got = append(got, map[string]interface{}{
"filename": part.FileName(),
"formname": part.FormName(),
"fileref": br.String(),
})
} else {
ret["error"] = "writing to blobserver: " + err.String()
return
}
}
ret["got"] = got
}
func (sh *Handler) serveDescribe(rw http.ResponseWriter, req *http.Request) {
ret := jsonMap()
defer httputil.ReturnJson(rw, ret)
br := blobref.Parse(req.FormValue("blobref"))
if br == nil {
ret["error"] = "Missing or invalid 'blobref' param"
ret["errorType"] = "input"
return
}
dr := sh.NewDescribeRequest()
dr.Describe(br, 4)
dr.PopulateJSON(ret)
}
func (sh *Handler) serveSignerAttrValue(rw http.ResponseWriter, req *http.Request) {
ret := jsonMap()
defer httputil.ReturnJson(rw, ret)
defer setPanicError(ret)
signer := blobref.MustParse(mustGet(req, "signer"))
attr := mustGet(req, "attr")
value := mustGet(req, "value")
pn, err := sh.index.PermanodeOfSignerAttrValue(signer, attr, value)
if err != nil {
ret["error"] = err.String()
} else {
ret["permanode"] = pn.String()
dr := sh.NewDescribeRequest()
dr.Describe(pn, 2)
dr.PopulateJSON(ret)
}
}
func (h *JSONSignHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
base := req.Header.Get("X-PrefixHandler-PathBase")
subPath := req.Header.Get("X-PrefixHandler-PathSuffix")
switch req.Method {
case "GET":
switch subPath {
case "":
http.Redirect(rw, req, base+"camli/sig/discovery", http.StatusFound)
return
case h.pubKeyBlobRefServeSuffix:
h.pubKeyHandler.ServeHTTP(rw, req)
return
case "camli/sig/sign":
fallthrough
case "camli/sig/verify":
http.Error(rw, "POST required", 400)
return
case "camli/sig/discovery":
m := map[string]interface{}{
"publicKeyId": h.entity.PrimaryKey.KeyIdString(),
"signHandler": base + "camli/sig/sign",
"verifyHandler": base + "camli/sig/verify",
}
if h.pubKeyBlobRef != nil {
m["publicKeyBlobRef"] = h.pubKeyBlobRef.String()
m["publicKey"] = base + h.pubKeyBlobRefServeSuffix
}
httputil.ReturnJson(rw, m)
return
}
case "POST":
switch subPath {
case "camli/sig/sign":
h.handleSign(rw, req)
return
case "camli/sig/verify":
h.handleVerify(rw, req)
return
}
}
http.Error(rw, "Unsupported path or method.", http.StatusBadRequest)
}
func (fth *FileTreeHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if req.Method != "GET" && req.Method != "HEAD" {
http.Error(rw, "Invalid method", 400)
return
}
ret := make(map[string]interface{})
defer httputil.ReturnJson(rw, ret)
fetchSeeker, err := fth.storageSeekFetcher()
if err != nil {
http.Error(rw, "No storageSeekFetcher", 500)
log.Printf("getting fetcher: %v\n", err)
return
}
de, err := schema.NewDirectoryEntryFromBlobRef(fetchSeeker, fth.file)
dir, err := de.Directory()
if err != nil {
http.Error(rw, "Error reading directory", 500)
log.Printf("Error reading directory from blobref %s: %v\n", fth.file, err)
return
}
entries, err := dir.Readdir(-1)
if err != nil {
http.Error(rw, "Error reading directory", 500)
log.Printf("reading dir from blobref %s: %v\n", fth.file, err)
return
}
children := make([]map[string]interface{}, 0)
for _, v := range entries {
child := map[string]interface{}{
"name": v.FileName(),
"type": v.CamliType(),
"blobRef": v.BlobRef(),
}
children = append(children, child)
}
ret["children"] = children
}
func (sh *Handler) serveClaims(rw http.ResponseWriter, req *http.Request) {
ret := jsonMap()
pn := blobref.Parse(req.FormValue("permanode"))
if pn == nil {
http.Error(rw, "Missing or invalid 'permanode' param", 400)
return
}
// TODO: rename GetOwnerClaims to GetClaims?
claims, err := sh.index.GetOwnerClaims(pn, sh.owner)
if err != nil {
log.Printf("Error getting claims of %s: %v", pn.String(), err)
} else {
sort.Sort(claims)
jclaims := jsonMapList()
for _, claim := range claims {
jclaim := jsonMap()
jclaim["blobref"] = claim.BlobRef.String()
jclaim["signer"] = claim.Signer.String()
jclaim["permanode"] = claim.Permanode.String()
jclaim["date"] = claim.Date.Format(time.RFC3339)
jclaim["type"] = claim.Type
if claim.Attr != "" {
jclaim["attr"] = claim.Attr
}
if claim.Value != "" {
jclaim["value"] = claim.Value
}
jclaims = append(jclaims, jclaim)
}
ret["claims"] = jclaims
}
httputil.ReturnJson(rw, ret)
}
func (sh *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
ret := jsonMap()
_ = req.Header.Get("X-PrefixHandler-PathBase")
suffix := req.Header.Get("X-PrefixHandler-PathSuffix")
if req.Method == "GET" {
switch suffix {
case "camli/search/recent":
sh.serveRecentPermanodes(rw, req)
return
case "camli/search/permanodeattr":
sh.servePermanodesWithAttr(rw, req)
return
case "camli/search/describe":
sh.serveDescribe(rw, req)
return
case "camli/search/claims":
sh.serveClaims(rw, req)
return
case "camli/search/files":
sh.serveFiles(rw, req)
return
case "camli/search/signerattrvalue":
sh.serveSignerAttrValue(rw, req)
return
case "camli/search/signerpaths":
sh.serveSignerPaths(rw, req)
return
}
}
// TODO: discovery for the endpoints & better error message with link to discovery info
ret["error"] = "Unsupported search path or method"
ret["errorType"] = "input"
httputil.ReturnJson(rw, ret)
}
func handleStat(conn http.ResponseWriter, req *http.Request, storage blobserver.BlobStatter) {
if w, ok := storage.(blobserver.ContextWrapper); ok {
storage = w.WrapContext(req)
}
toStat := make([]*blobref.BlobRef, 0)
switch req.Method {
case "POST":
fallthrough
case "GET":
camliVersion := req.FormValue("camliversion")
if camliVersion == "" {
httputil.BadRequestError(conn, "No camliversion")
return
}
n := 0
for {
n++
key := fmt.Sprintf("blob%v", n)
value := req.FormValue(key)
if value == "" {
n--
break
}
if n > maxStatBlobs {
httputil.BadRequestError(conn, "Too many stat blob checks")
return
}
ref := blobref.Parse(value)
if ref == nil {
httputil.BadRequestError(conn, "Bogus blobref for key "+key)
return
}
toStat = append(toStat, ref)
}
default:
httputil.BadRequestError(conn, "Invalid method.")
return
}
waitSeconds := 0
if waitStr := req.FormValue("maxwaitsec"); waitStr != "" {
waitSeconds, _ = strconv.Atoi(waitStr)
switch {
case waitSeconds < 0:
waitSeconds = 0
case waitSeconds > 30:
// TODO: don't hard-code 30. push this up into a blobserver interface
// for getting the configuration of the server (ultimately a flag in
// in the binary)
waitSeconds = 30
}
}
statRes := make([]map[string]interface{}, 0)
if len(toStat) > 0 {
blobch := make(chan blobref.SizedBlobRef)
resultch := make(chan os.Error, 1)
go func() {
err := storage.StatBlobs(blobch, toStat, waitSeconds)
close(blobch)
resultch <- err
}()
for sb := range blobch {
ah := make(map[string]interface{})
ah["blobRef"] = sb.BlobRef.String()
ah["size"] = sb.Size
statRes = append(statRes, ah)
}
err := <-resultch
if err != nil {
log.Printf("Stat error: %v", err)
conn.WriteHeader(http.StatusInternalServerError)
return
}
}
configer, _ := storage.(blobserver.Configer)
ret := commonUploadResponse(configer, req)
ret["stat"] = statRes
ret["canLongPoll"] = true
httputil.ReturnJson(conn, ret)
}
func handleMultiPartUpload(conn http.ResponseWriter, req *http.Request, blobReceiver blobserver.BlobReceiveConfiger) {
if w, ok := blobReceiver.(blobserver.ContextWrapper); ok {
blobReceiver = w.WrapContext(req).(blobserver.BlobReceiveConfiger)
}
if !(req.Method == "POST" && strings.Contains(req.URL.Path, "/camli/upload")) {
log.Printf("Inconfigured handler upload handler")
httputil.BadRequestError(conn, "Inconfigured handler.")
return
}
receivedBlobs := make([]blobref.SizedBlobRef, 0, 10)
multipart, err := req.MultipartReader()
if multipart == nil {
httputil.BadRequestError(conn, fmt.Sprintf(
"Expected multipart/form-data POST request; %v", err))
return
}
var errText string
addError := func(s string) {
log.Printf("Client error: %s", s)
if errText == "" {
errText = s
return
}
errText = errText + "\n" + s
}
for {
mimePart, err := multipart.NextPart()
if err == os.EOF {
break
}
if err != nil {
addError(fmt.Sprintf("Error reading multipart section: %v", err))
break
}
//POST-r60:
//contentDisposition, params, err := mime.ParseMediaType(mimePart.Header.Get("Content-Disposition"))
//if err != nil {
// addError(err.String())
// break
//}
// r60:
contentDisposition, params := mime.ParseMediaType(mimePart.Header.Get("Content-Disposition"))
if contentDisposition == "" {
addError("invalid Content-Disposition")
break
}
if contentDisposition != "form-data" {
addError(fmt.Sprintf("Expected Content-Disposition of \"form-data\"; got %q", contentDisposition))
break
}
formName := params["name"]
ref := blobref.Parse(formName)
if ref == nil {
addError(fmt.Sprintf("Ignoring form key %q", formName))
continue
}
if oldAppEngineHappySpec {
_, hasContentType := mimePart.Header["Content-Type"]
if !hasContentType {
addError(fmt.Sprintf("Expected Content-Type header for blobref %s; see spec", ref))
continue
}
_, hasFileName := params["filename"]
if !hasFileName {
addError(fmt.Sprintf("Expected 'filename' Content-Disposition parameter for blobref %s; see spec", ref))
continue
}
}
blobGot, err := blobReceiver.ReceiveBlob(ref, mimePart)
if err != nil {
addError(fmt.Sprintf("Error receiving blob %v: %v\n", ref, err))
break
}
log.Printf("Received blob %v\n", blobGot)
receivedBlobs = append(receivedBlobs, blobGot)
}
log.Println("Done reading multipart body.")
ret := commonUploadResponse(blobReceiver, req)
received := make([]map[string]interface{}, 0)
for _, got := range receivedBlobs {
log.Printf("Got blob: %v\n", got)
blob := make(map[string]interface{})
blob["blobRef"] = got.BlobRef.String()
blob["size"] = got.Size
received = append(received, blob)
}
ret["received"] = received
if errText != "" {
ret["errorText"] = errText
}
httputil.ReturnJson(conn, ret)
}
func handleMultiPartUpload(conn http.ResponseWriter, req *http.Request) {
if !(req.Method == "POST" && req.URL.Path == "/camli/upload") {
httputil.BadRequestError(conn, "Inconfigured handler.")
return
}
receivedBlobs := make([]*receivedBlob, 0, 10)
multipart, err := req.MultipartReader()
if multipart == nil {
httputil.BadRequestError(conn, fmt.Sprintf(
"Expected multipart/form-data POST request; %v", err))
return
}
var errText string
addError := func(s string) {
log.Printf("Client error: %s", s)
if errText == "" {
errText = s
return
}
errText = errText + "\n" + s
}
for {
part, err := multipart.NextPart()
if err != nil {
addError(fmt.Sprintf("Error reading multipart section: %v", err))
break
}
if part == nil {
break
}
contentDisposition, params := mime.ParseMediaType(part.Header["Content-Disposition"])
if contentDisposition != "form-data" {
addError(fmt.Sprintf("Expected Content-Disposition of \"form-data\"; got %q", contentDisposition))
break
}
formName := params["name"]
ref := blobref.Parse(formName)
if ref == nil {
addError(fmt.Sprintf("Ignoring form key %q", formName))
continue
}
_, hasContentType := part.Header["Content-Type"]
if !hasContentType {
addError(fmt.Sprintf("Expected Content-Type header for blobref %s; see spec", ref))
continue
}
_, hasFileName := params["filename"]
if !hasFileName {
addError(fmt.Sprintf("Expected 'filename' Content-Disposition parameter for blobref %s; see spec", ref))
continue
}
blobGot, err := receiveBlob(ref, part)
if err != nil {
addError(fmt.Sprintf("Error receiving blob %v: %v\n", ref, err))
break
}
log.Printf("Received blob %v\n", blobGot)
receivedBlobs = append(receivedBlobs, blobGot)
}
log.Println("Done reading multipart body.")
ret := commonUploadResponse(req)
received := make([]map[string]interface{}, 0)
for _, got := range receivedBlobs {
log.Printf("Got blob: %v\n", got)
blob := make(map[string]interface{})
blob["blobRef"] = got.blobRef.String()
blob["size"] = got.size
received = append(received, blob)
}
ret["received"] = received
if errText != "" {
ret["errorText"] = errText
}
httputil.ReturnJson(conn, ret)
}
// TODO(mpl): configure and/or document the name of the possible attributes in the http request
func (sh *Handler) servePermanodesWithAttr(rw http.ResponseWriter, req *http.Request) {
ret := jsonMap()
defer httputil.ReturnJson(rw, ret)
defer setPanicError(ret)
signer := blobref.MustParse(mustGet(req, "signer"))
value := mustGet(req, "value")
fuzzy := req.FormValue("fuzzy") // exact match if empty
fuzzyMatch := false
if fuzzy != "" {
lowered := strings.ToLower(fuzzy)
if lowered == "true" || lowered == "t" {
fuzzyMatch = true
}
}
attr := req.FormValue("attr") // all attributes if empty
if attr == "" { // and force fuzzy in that case.
fuzzyMatch = true
}
maxResults := maxPermanodes
max := req.FormValue("max")
if max != "" {
maxR, err := strconv.Atoi(max)
if err != nil {
log.Printf("Invalid specified max results 'max': " + err.String())
return
}
if maxR < maxResults {
maxResults = maxR
}
}
ch := make(chan *blobref.BlobRef, buffered)
errch := make(chan os.Error)
go func() {
errch <- sh.index.SearchPermanodesWithAttr(ch,
&PermanodeByAttrRequest{Attribute: attr,
Query: value,
Signer: signer,
FuzzyMatch: fuzzyMatch,
MaxResults: maxResults})
}()
dr := sh.NewDescribeRequest()
withAttr := jsonMapList()
for res := range ch {
dr.Describe(res, 2)
jm := jsonMap()
jm["permanode"] = res.String()
withAttr = append(withAttr, jm)
}
err := <-errch
if err != nil {
ret["error"] = err.String()
ret["errorType"] = "server"
return
}
ret["withAttr"] = withAttr
dr.PopulateJSON(ret)
}
func handlePreUpload(conn http.ResponseWriter, req *http.Request) {
if !(req.Method == "POST" && req.URL.Path == "/camli/preupload") {
httputil.BadRequestError(conn, "Inconfigured handler.")
return
}
req.ParseForm()
camliVersion := req.FormValue("camliversion")
if camliVersion == "" {
httputil.BadRequestError(conn, "No camliversion")
return
}
n := 0
haveVector := new(vector.Vector)
haveChan := make(chan *map[string]interface{})
for {
key := fmt.Sprintf("blob%v", n+1)
value := req.FormValue(key)
if value == "" {
break
}
ref := blobref.Parse(value)
if ref == nil {
httputil.BadRequestError(conn, "Bogus blobref for key "+key)
return
}
if !ref.IsSupported() {
httputil.BadRequestError(conn, "Unsupported or bogus blobref "+key)
}
n++
// Parallel stat all the files...
go func() {
fi, err := os.Stat(BlobFileName(ref))
if err == nil && fi.IsRegular() {
info := make(map[string]interface{})
info["blobRef"] = ref.String()
info["size"] = fi.Size
haveChan <- &info
} else {
haveChan <- nil
}
}()
}
if n > 0 {
for have := range haveChan {
if have != nil {
haveVector.Push(have)
}
n--
if n == 0 {
break
}
}
}
ret := commonUploadResponse(req)
ret["alreadyHave"] = haveVector.Copy()
httputil.ReturnJson(conn, ret)
}