// This function seeds the queue from a web request func seedQueue(hkidhex string) (err error) { h, hexerr := objects.HcidFromHex(hkidhex) if hexerr == nil { err = crawlList(objects.HCID(h.Bytes())) if err != nil { log.Println("seed was not a List") } err = crawlBlob(objects.HCID(h.Bytes())) if err != nil { log.Println("seed was not a Blob") } err = crawlCommit(objects.HKID(h.Bytes())) if err != nil { log.Println("seed was not a Commit") } err = crawlhcidCommit(objects.HCID(h.Bytes())) if err != nil { log.Println("seed was not a Commit's hcid") } err = crawlhcidTag(objects.HCID(h.Bytes())) if err != nil { log.Println("seed was not a Tag's hcid") } } else { log.Println(hexerr) } return hexerr }
func parseMessage(message string) (hkid objects.HKID, hcid objects.HCID, typeString string, nameSegment string, url string) { var Message map[string]interface{} err := json.Unmarshal([]byte(message), &Message) if err != nil { log.Printf("Error %s\n", err) } if Message["hcid"] != nil { hcid, err = objects.HcidFromHex(Message["hcid"].(string)) } if err != nil { log.Printf("Error with hex to string %s", err) } if Message["hkid"] != nil { hkid, err = objects.HkidFromHex(Message["hkid"].(string)) } if err != nil { log.Printf("Error with hex to string %s", err) } if Message["type"] != nil { typeString = Message["type"].(string) } if Message["namesegment"] != nil { nameSegment = Message["namesegment"].(string) } if Message["URL"] != nil { url = Message["URL"].(string) } return hkid, hcid, typeString, nameSegment, url }
func TestLowLevel(t *testing.T) { indata := objects.Blob([]byte("TestPostData")) services.PostBlob(indata) blobhcid, err := objects.HcidFromHex( "ca4c4244cee2bd8b8a35feddcd0ba36d775d68637b7f0b4d2558728d0752a2a2", ) b, err := services.GetBlob(blobhcid) if !bytes.Equal(b.Hash(), blobhcid) { t.Fatalf("GetBlob Fail\nExpected: %s\nGot: %s\n", blobhcid, b.Hash()) //Changed } //6dedf7e580671bd90bc9d1f735c75a4f3692b697f8979a147e8edd64fab56e85 testhkid := objects.HkidFromDString( "6523237356270560228617783789728329416595512649112249373497830592"+ "0722414168936112160694238047304378604753005642729767620850685191"+ "88612732562106886379081213385", 10) testCommit := objects.NewCommit(b.Hash(), testhkid) services.PostCommit(testCommit) c, err := services.GetCommit(testhkid) if err != nil { t.Fatalf("Get Commit Fail: %s", err) } else if !bytes.Equal(c.Hkid, testhkid) { t.Fatalf("Expected: %s Got: %s", testhkid, c.Hkid) } else if !c.Verify() { t.Fatalf("Commit Signature Invalid.") } //ede7bec713c93929751f18b1db46d4be3c95286bd5f2d92b9759ff02115dc312 taghkid := objects.HkidFromDString( "4813315537186321970719165779488475377688633084782731170482174374"+ "7256947679650787426167575073363006751150073195493002048627629373"+ "76227751462258339344895829332", 10) lowlvlTag := objects.NewTag( objects.HID(b.Hash()), "blob", "testBlob", nil, taghkid, ) //gen test tag services.PostTag(lowlvlTag) testtag, err := services.GetTag(taghkid, lowlvlTag.NameSegment) if err != nil { t.Fatalf("GetTag Fail. error: %s", err) } else if !bytes.Equal(testtag.Hkid, taghkid) { t.Fatalf("GetTag Fail.\n\t expected: %v\n\t got: %v", taghkid, testtag.Hkid) } else if !testtag.Verify() { t.Fatalf("GetTag Verify Fail") } prikey, err := services.GetKey(taghkid) if err != nil || !bytes.Equal(prikey.Hkid(), taghkid) || !prikey.Verify() { t.Logf("GetKey Fail") t.Fail() } }
func TestAppsscriptservice_getBlob(t *testing.T) { //t.Skip() h, err := objects.HcidFromHex( "ca4c4244cee2bd8b8a35feddcd0ba36d775d68637b7f0b4d2558728d0752a2a2", ) b, err := Instance.GetBlob(h) if err != nil || !bytes.Equal(b.Hash(), h) { log.Println(string(b)) t.FailNow() } }
func TestGoogledriveservice_GetBlob(t *testing.T) { t.Skip("skipping google drive tests") //googledriveservice_setup() hb, err := objects.HcidFromHex("42cc3a4c4a9d9d3ee7de9322b45acb0e5a5c33550d9ad4791df6ae937a869e12") if err != nil { t.Fail() } b, err := Instance.GetBlob(hb) if err != nil || !bytes.Equal(b.Hash(), hb) { t.Fail() } }
func webSearchHandler(w http.ResponseWriter, r *http.Request) { ResponseStruct := struct { Query string NameSegmentInfos map[string]map[nameSegmentIndexEntry]int BlobInfo blobIndexEntry BlobInfoPresent bool CommitInfo commitIndexEntry CommitInfoPresent bool TagInfo tagIndexEntry TagInfoPresent bool Text map[string][]string }{} parsedURL, err := url.Parse(r.RequestURI) if err != nil { log.Println(err) } ResponseStruct.Query = parsedURL.Query().Get("q") hcid, err := objects.HcidFromHex(ResponseStruct.Query) if err == nil { ResponseStruct.BlobInfo, ResponseStruct.BlobInfoPresent = blobIndex[ResponseStruct.Query] if !ResponseStruct.BlobInfoPresent { log.Println("HID is not present") } else { switch ResponseStruct.BlobInfo.TypeString { case "Blob": case "List": case "Commit": commit, err := services.GetCommitForHcid(hcid) if err == nil { ResponseStruct.CommitInfo, ResponseStruct.CommitInfoPresent = commitIndex[commit.Hkid.Hex()] } case "Tag": tag, err := services.GetTagForHcid(hcid) if err == nil { ResponseStruct.TagInfo, ResponseStruct.TagInfoPresent = tagIndex[tag.Hkid.Hex()] } case "Repository": ResponseStruct.CommitInfo, ResponseStruct.CommitInfoPresent = commitIndex[ResponseStruct.Query] case "Domain": ResponseStruct.TagInfo, ResponseStruct.TagInfoPresent = tagIndex[ResponseStruct.Query] default: log.Printf("Unrecognized Type %s", ResponseStruct.BlobInfo.TypeString, ) } } log.Println(err) } queryTokens := strings.FieldsFunc(ResponseStruct.Query, isSeperator) ResponseStruct.NameSegmentInfos = make(map[string]map[nameSegmentIndexEntry]int) for nameSegment, nameSegEntry := range nameSegmentIndex { for _, queryToken := range queryTokens { if present := strings.Contains(nameSegment, queryToken); present { ResponseStruct.NameSegmentInfos[nameSegment] = nameSegEntry break } } } tempMap := make(map[string][]objects.HCID) for _, queryToken := range queryTokens { if _, present := textIndex[queryToken]; present { tempMap[queryToken] = textIndex[queryToken] } } ResponseStruct.Text = make(map[string][]string) for token, tempHCIDs := range tempMap { for _, tempHCID := range tempHCIDs { if _, present := ResponseStruct.Text[tempHCID.Hex()]; !present { ResponseStruct.Text[tempHCID.Hex()] = []string{token} } else { ResponseStruct.Text[tempHCID.Hex()] = append( ResponseStruct.Text[tempHCID.Hex()], token, ) } } } t, err := template.New("WebSearch template").Funcs( template.FuncMap{ "GetCuratorsofBlob": GetCuratorsofBlob, "GetPathsForHCID": GetPathsForHCID, }, ).Parse(` {{define "NameSegTemp"}} <dl> {{range $key, $value := .}} <dt> {{$key}}: </dt> {{range $key1:= $value}} <dd> <a href= "/b/{{$key1}}">{{$key1}}</a> </dd> {{end}} {{end}} </dl> {{end}} {{define "VersionTemp"}} <dl> {{range $key, $value := .}} <dt> <a href= "/search/?q={{$value}}"> {{$key}}</a>: <a href= "/b/{{$value}}">{{$value}}</a> </dt> {{end}} </dl> {{end}} {{define "BlobInfoTemp"}} {{.TypeString}}[{{.Size}}]: {{with .SignedBy}} <a href= "/search/?q={{.}}">{{.}}</a>: {{end}} {{template "NameSegTemp" .NameSeg}} {{template "VersionTemp" .Descendants}} {{range $key, $value := GetCuratorsofBlob .HCID}} Curators: <a href= "/{{$value}}/{{$key}}">{{$key}}</a> {{end}} <dl> Paths: {{range $key, $value := GetPathsForHCID .HCID}} <dt><a href= "/{{$key}}">{{$key}}</a></dt> {{end}} </dl> {{end}} <html> <head> <title> Search - CCFS </title> </head> <body> </br> <form action = "./" method="get"> <input type = "text" name = "q" value="{{.Query}}"> <input type = "submit" value="Search"> </form> </br> {{with .Query}} Search results for: {{.}} {{end}} </br> <dl> {{with .NameSegmentInfos}} {{range $key1, $value1:= .}} {{$key1}} <ul> {{range $key, $value:= $value1}} <li> <a href= "/search/?q={{$key.Hash}}"> {{$key.TypeString}} </a>: <a href= "/b/{{$key.Hash}}"> {{$key.Hash}} </a> {{$value}} </li> {{end}} </ul> {{end}} {{end}} {{with .Text}} <br> Full Text Search Results: <ul> {{range $key1, $value1:= .}} <a href= "/b/{{$key1}}"> {{$key1}} </a> {{range $key:= $value1}} <li> {{$key}} </li> {{end}} {{end}} </ul> {{end}} {{if .BlobInfoPresent}} </br> {{template "BlobInfoTemp" .BlobInfo}} {{end}} {{if .CommitInfoPresent}} {{template "NameSegTemp" .CommitInfo.NameSeg}} {{template "VersionTemp" .CommitInfo.Version}} {{end}} {{if .TagInfoPresent}} <dl> <dt>Aliases: </dt> <dd>{{template "NameSegTemp" .TagInfo.NameSeg}}</dd> <dt>Sub Domains: </dt> <dd> <dl> {{range $key, $value := .TagInfo.Version}} <dt>{{$key}}: </dt> <dd>{{template "VersionTemp" $value}}</dd> {{end}} </dl> </dd> </dl> {{end}} </dl> </body> </html> `) if err != nil { log.Println(err) http.Error( w, fmt.Sprintf("HTTP Error 500 Internal Search server error\n%s\n", err, ), 500, ) } else { t.Execute(w, ResponseStruct) } }