Esempio n. 1
0
func doKeyStuff(b *testing.B) keyStuff {
	camliRootPath, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		b.Fatal("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
	}
	secretRingFile := filepath.Join(camliRootPath, "pkg", "jsonsign", "testdata", "test-secring.gpg")
	pubKey := `-----BEGIN PGP PUBLIC KEY BLOCK-----

xsBNBEzgoVsBCAC/56aEJ9BNIGV9FVP+WzenTAkg12k86YqlwJVAB/VwdMlyXxvi
bCT1RVRfnYxscs14LLfcMWF3zMucw16mLlJCBSLvbZ0jn4h+/8vK5WuAdjw2YzLs
WtBcjWn3lV6tb4RJz5gtD/o1w8VWxwAnAVIWZntKAWmkcChCRgdUeWso76+plxE5
aRYBJqdT1mctGqNEISd/WYPMgwnWXQsVi3x4z1dYu2tD9uO1dkAff12z1kyZQIBQ
rexKYRRRh9IKAayD4kgS0wdlULjBU98aeEaMz1ckuB46DX3lAYqmmTEL/Rl9cOI0
Enpn/oOOfYFa5h0AFndZd1blMvruXfdAobjVABEBAAE=
=28/7
-----END PGP PUBLIC KEY BLOCK-----`
	return keyStuff{
		secretRingFile: secretRingFile,
		pubKey:         pubKey,
		pubKeyRef:      blob.SHA1FromString(pubKey),
		entityFetcher: &jsonsign.CachingEntityFetcher{
			Fetcher: &jsonsign.FileEntityFetcher{File: secretRingFile},
		},
	}
}
Esempio n. 2
0
func gitShortlog() *exec.Cmd {
	if !*gitContainer {
		return exec.Command("/bin/bash", "-c", "git log | git shortlog -sen")
	}
	args := []string{"run", "--rm"}
	if inProd {
		args = append(args,
			"-v", "/var/camweb:/var/camweb",
			"--workdir="+prodSrcDir,
		)
	} else {
		hostRoot, err := osutil.GoPackagePath("camlistore.org")
		if err != nil {
			log.Fatal(err)
		}
		log.Printf("Using bind root of %q", hostRoot)
		args = append(args,
			"-v", hostRoot+":"+prodSrcDir,
			"--workdir="+prodSrcDir,
		)
	}
	args = append(args, "camlistore/git", "/bin/bash", "-c", "git log | git shortlog -sen")
	cmd := exec.Command("docker", args...)
	cmd.Stderr = os.Stderr
	return cmd
}
Esempio n. 3
0
func TestExpansionsInHighlevelConfig(t *testing.T) {
	camroot, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		t.Fatalf("failed to find camlistore.org GOPATH root: %v", err)
	}
	const keyID = "26F5ABDA"
	os.Setenv("TMP_EXPANSION_TEST", keyID)
	os.Setenv("TMP_EXPANSION_SECRING", filepath.Join(camroot, filepath.FromSlash("pkg/jsonsign/testdata/test-secring.gpg")))
	// Setting CAMLI_CONFIG_DIR to avoid triggering failInTests in osutil.CamliConfigDir
	defer os.Setenv("CAMLI_CONFIG_DIR", os.Getenv("CAMLI_CONFIG_DIR")) // restore after test
	os.Setenv("CAMLI_CONFIG_DIR", "whatever")
	conf, err := serverinit.Load([]byte(`
{
    "auth": "localhost",
    "listen": ":4430",
    "https": false,
    "identity": ["_env", "${TMP_EXPANSION_TEST}"],
    "identitySecretRing": ["_env", "${TMP_EXPANSION_SECRING}"],
    "googlecloudstorage": ":camlistore-dev-blobs",
    "kvIndexFile": "/tmp/camli-index.kvdb"
}
`))
	if err != nil {
		t.Fatal(err)
	}
	got := fmt.Sprintf("%#v", conf)
	if !strings.Contains(got, keyID) {
		t.Errorf("Expected key %s in resulting low-level config. Got: %s", keyID, got)
	}
}
Esempio n. 4
0
// setup checks if the camlistore root can be found,
// then sets up closureGitDir and destDir, and returns whether
// we should clone or update in closureGitDir (depending on
// if a .git dir was found).
func setup() string {
	camliRootPath, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		log.Fatal("Package camlistore.org not found in $GOPATH (or $GOPATH not defined).")
	}
	destDir = filepath.Join(camliRootPath, "third_party", "closure", "lib")
	closureGitDir = filepath.Join(camliRootPath, "tmp", "closure-lib")
	op := "update"
	_, err = os.Stat(closureGitDir)
	if err != nil {
		if os.IsNotExist(err) {
			err = os.MkdirAll(closureGitDir, 0755)
			if err != nil {
				log.Fatalf("Could not create %v: %v", closureGitDir, err)
			}
			op = "clone"
		} else {
			log.Fatalf("Could not stat %v: %v", closureGitDir, err)
		}
	}
	dotGitPath := filepath.Join(closureGitDir, ".git")
	_, err = os.Stat(dotGitPath)
	if err != nil {
		if os.IsNotExist(err) {
			op = "clone"
		} else {
			log.Fatalf("Could not stat %v: %v", dotGitPath, err)
		}
	}
	return op
}
Esempio n. 5
0
// camliClosurePage checks if filename is a .js file using closure
// and if yes, if it provides a page in the camlistore namespace.
// It returns that page name, or the empty string otherwise.
func camliClosurePage(filename string) string {
	camliRootPath, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		return ""
	}
	fullpath := filepath.Join(camliRootPath, "server", "camlistored", "ui", filename)
	f, err := os.Open(fullpath)
	if err != nil {
		return ""
	}
	defer f.Close()
	br := bufio.NewReader(f)
	for {
		l, err := br.ReadString('\n')
		if err != nil {
			return ""
		}
		if !strings.HasPrefix(l, "goog.") {
			continue
		}
		m := provCamliRx.FindStringSubmatch(l)
		if m != nil {
			return m[2]
		}
	}
	return ""
}
Esempio n. 6
0
// fileList parses deps.js from the closure repo, as well as the similar
// dependencies generated for the UI js files, and compiles the list of
// js files from the closure lib required for the UI.
func fileList() ([]string, error) {
	camliRootPath, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		log.Fatal("Package camlistore.org not found in $GOPATH (or $GOPATH not defined).")
	}
	uiDir := filepath.Join(camliRootPath, "server", "camlistored", "ui")
	closureDepsFile := filepath.Join(closureGitDir, "closure", "goog", "deps.js")

	f, err := os.Open(closureDepsFile)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	allClosureDeps, err := closure.DeepParseDeps(f)
	if err != nil {
		return nil, err
	}

	uiDeps, err := closure.GenDeps(http.Dir(uiDir))
	if err != nil {
		return nil, err
	}
	_, requ, err := closure.ParseDeps(bytes.NewReader(uiDeps))
	if err != nil {
		return nil, err
	}

	nameDone := make(map[string]bool)
	jsfilesDone := make(map[string]bool)
	for _, deps := range requ {
		for _, dep := range deps {
			if _, ok := nameDone[dep]; ok {
				continue
			}
			jsfiles := allClosureDeps[dep]
			for _, filename := range jsfiles {
				if _, ok := jsfilesDone[filename]; ok {
					continue
				}
				jsfilesDone[filename] = true
			}
			nameDone[dep] = true
		}
	}
	jsfiles := []string{
		"AUTHORS",
		"LICENSE",
		"README",
		filepath.Join("closure", "goog", "base.js"),
		filepath.Join("closure", "goog", "css", "common.css"),
		filepath.Join("closure", "goog", "css", "toolbar.css"),
		filepath.Join("closure", "goog", "deps.js"),
	}
	prefix := filepath.Join("closure", "goog")
	for k, _ := range jsfilesDone {
		jsfiles = append(jsfiles, filepath.Join(prefix, k))
	}
	sort.Strings(jsfiles)
	return jsfiles, nil
}
Esempio n. 7
0
func main() {
	flag.Usage = usage
	flag.Parse()
	checkFlags()

	camDir, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		log.Fatalf("Error looking up camlistore.org dir: %v", err)
	}
	dockDir = filepath.Join(camDir, "misc", "docker")

	if *doBuildServer {
		buildDockerImage("go", goDockerImage)
		buildDockerImage("djpeg-static", djpegDockerImage)

		// ctxDir is where we run "docker build" to produce the final
		// "FROM scratch" Docker image.
		ctxDir, err := ioutil.TempDir("", "camli-build")
		if err != nil {
			log.Fatal(err)
		}
		defer os.RemoveAll(ctxDir)

		genCamlistore(ctxDir)
		genDjpeg(ctxDir)
		buildServer(ctxDir)
	}

	if *doUpload {
		uploadDockerImage()
	}
}
Esempio n. 8
0
func camSrcDir() string {
	if inProd {
		return prodSrcDir
	}
	dir, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		log.Fatalf("Failed to find the root of the Camlistore source code via osutil.GoPackagePath: %v", err)
	}
	return dir
}
Esempio n. 9
0
// TODO(mpl): refactor with twitter
func fakePhoto() string {
	camliDir, err := osutil.GoPackagePath("camlistore.org")
	if err == os.ErrNotExist {
		log.Fatal("Directory \"camlistore.org\" not found under GOPATH/src; are you not running with devcam?")
	}
	if err != nil {
		log.Fatalf("Error searching for \"camlistore.org\" under GOPATH: %v", err)
	}
	return filepath.Join(camliDir, filepath.FromSlash("third_party/glitch/npc_piggy__x1_walk_png_1354829432.png"))
}
Esempio n. 10
0
// CamliSourceRoot returns the root of the source tree, or an error.
func camliSourceRoot() (string, error) {
	if os.Getenv("GOPATH") == "" {
		return "", errors.New("GOPATH environment variable isn't set; required to run Camlistore integration tests")
	}
	root, err := osutil.GoPackagePath("camlistore.org")
	if err == os.ErrNotExist {
		return "", errors.New("Directory \"camlistore.org\" not found under GOPATH/src; can't run Camlistore integration tests.")
	}
	return root, nil
}
Esempio n. 11
0
func (c *serverCmd) RunCommand(args []string) error {
	err := c.checkFlags(args)
	if err != nil {
		return cmdmain.UsageError(fmt.Sprint(err))
	}
	c.camliSrcRoot, err = osutil.GoPackagePath("camlistore.org")
	if err != nil {
		return errors.New("Package camlistore.org not found in $GOPATH (or $GOPATH not defined).")
	}
	err = os.Chdir(c.camliSrcRoot)
	if err != nil {
		return fmt.Errorf("Could not chdir to %v: %v", c.camliSrcRoot, err)
	}
	if !c.noBuild {
		for _, name := range []string{"camlistored", "camtool"} {
			err := c.build(name)
			if err != nil {
				return fmt.Errorf("Could not build %v: %v", name, err)
			}
		}
	}
	if err := c.setCamliRoot(); err != nil {
		return fmt.Errorf("Could not setup the camli root: %v", err)
	}
	if err := c.setEnvVars(); err != nil {
		return fmt.Errorf("Could not setup the env vars: %v", err)
	}
	if err := c.setupIndexer(); err != nil {
		return fmt.Errorf("Could not setup the indexer: %v", err)
	}
	if err := c.syncTemplateBlobs(); err != nil {
		return fmt.Errorf("Could not copy the template blobs: %v", err)
	}
	if err := c.setFullClosure(); err != nil {
		return fmt.Errorf("Could not setup the closure lib: %v", err)
	}

	log.Printf("Starting dev server on %v/ui/ with password \"pass%v\"\n",
		os.Getenv("CAMLI_BASEURL"), c.port)

	camliBin := filepath.Join(c.camliSrcRoot, "bin", "camlistored")
	cmdArgs := []string{
		"-configfile=" + filepath.Join(c.camliSrcRoot, "config", "dev-server-config.json"),
		"-listen=" + c.listen}
	cmd := exec.Command(camliBin, cmdArgs...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Start(); err != nil {
		return fmt.Errorf("Could not start camlistored: %v", err)
	}
	go handleKillCamliSignal(cmd.Process)
	cmd.Wait()
	return nil
}
Esempio n. 12
0
func main() {
	flag.Usage = usage
	flag.Parse()
	checkFlags()

	camDir, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		log.Fatalf("Error looking up camlistore.org dir: %v", err)
	}
	dockDir = filepath.Join(camDir, "misc", "docker")

	if *doImage {
		buildDockerImage("go", goDockerImage)
		buildDockerImage("djpeg-static", djpegDockerImage)
		// ctxDir is where we run "docker build" to produce the final
		// "FROM scratch" Docker image.
		ctxDir, err := ioutil.TempDir("", "camli-build")
		if err != nil {
			log.Fatal(err)
		}
		defer os.RemoveAll(ctxDir)

		genCamlistore(ctxDir)
		genDjpeg(ctxDir)
		buildServer(ctxDir)
	}

	// TODO(mpl): maybe *doBinaries should be done by a separate go program,
	// because the end product is not a docker image. However, we're still
	// using docker all along, and it's convenient for now for code reuse. I
	// can refactor it all out of dock.go afterwards if we like the process.
	if *doBinaries {
		// TODO(mpl): consider using an "official" or trusted existing
		// Go docker image, since we don't do anything special anymore in
		// ours?
		buildDockerImage("go", goDockerImage+"-linux")
		ctxDir, err := ioutil.TempDir("", "camli-build")
		if err != nil {
			log.Fatal(err)
		}
		defer os.RemoveAll(ctxDir)
		genBinaries(ctxDir)
		packBinaries(ctxDir)
	}

	if *doUpload {
		if *doImage {
			uploadDockerImage()
		} else if *doBinaries {
			uploadReleaseTarball()
		}
	}
}
Esempio n. 13
0
func TestQueryPermanodeLocation(t *testing.T) {
	testQuery(t, func(qt *queryTest) {
		id := qt.id

		p1 := id.NewPlannedPermanode("1")
		p2 := id.NewPlannedPermanode("2")
		p3 := id.NewPlannedPermanode("3")
		id.SetAttribute(p1, "latitude", "51.5")
		id.SetAttribute(p1, "longitude", "0")
		id.SetAttribute(p2, "latitude", "51.5")
		id.SetAttribute(p3, "longitude", "0")

		p4 := id.NewPlannedPermanode("checkin")
		p5 := id.NewPlannedPermanode("venue")
		id.SetAttribute(p4, "camliNodeType", "foursquare.com:checkin")
		id.SetAttribute(p4, "foursquareVenuePermanode", p5.String())
		id.SetAttribute(p5, "latitude", "1.0")
		id.SetAttribute(p5, "longitude", "2.0")

		// Upload a basic image
		camliRootPath, err := osutil.GoPackagePath("camlistore.org")
		if err != nil {
			panic("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
		}
		uploadFile := func(file string, modTime time.Time) blob.Ref {
			fileName := filepath.Join(camliRootPath, "pkg", "search", "testdata", file)
			contents, err := ioutil.ReadFile(fileName)
			if err != nil {
				panic(err)
			}
			br, _ := id.UploadFile(file, string(contents), modTime)
			return br
		}
		fileRef := uploadFile("dude-gps.jpg", time.Time{})

		p6 := id.NewPlannedPermanode("photo")
		id.SetAttribute(p6, "camliContent", fileRef.String())

		sq := &SearchQuery{
			Constraint: &Constraint{
				Permanode: &PermanodeConstraint{
					Location: &LocationConstraint{
						Any: true,
					},
				},
			},
		}
		qt.wantRes(sq, p1, p4, p5, p6)
	})
}
Esempio n. 14
0
// newTestServer creates a new test server with in memory storage for use in upload tests
func newTestServer(t *testing.T) *httptest.Server {
	camroot, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		t.Fatalf("failed to find camlistore.org GOPATH root: %v", err)
	}

	conf := serverconfig.Config{
		Listen:             ":3179",
		HTTPS:              false,
		Auth:               "localhost",
		Identity:           "26F5ABDA",
		IdentitySecretRing: filepath.Join(camroot, filepath.FromSlash("pkg/jsonsign/testdata/test-secring.gpg")),
		MemoryStorage:      true,
		MemoryIndex:        true,
	}

	confData, err := json.MarshalIndent(conf, "", "    ")
	if err != nil {
		t.Fatalf("Could not json encode config: %v", err)
	}

	// Setting CAMLI_CONFIG_DIR to avoid triggering failInTests in osutil.CamliConfigDir
	defer os.Setenv("CAMLI_CONFIG_DIR", os.Getenv("CAMLI_CONFIG_DIR")) // restore after test
	os.Setenv("CAMLI_CONFIG_DIR", "whatever")
	lowConf, err := serverinit.Load(confData)
	if err != nil {
		t.Fatal(err)
	}
	// because these two are normally consumed in camlistored.go
	// TODO(mpl): serverinit.Load should consume these 2 as well. Once
	// consumed, we should keep all the answers as private fields, and then we
	// put accessors on serverinit.Config. Maybe we even stop embedding
	// jsonconfig.Obj in serverinit.Config too, so none of those methods are
	// accessible.
	lowConf.OptionalBool("https", true)
	lowConf.OptionalString("listen", "")

	reindex := false
	var context *http.Request // only used by App Engine. See handlerLoader in serverinit.go
	hi := http.NewServeMux()
	address := "http://" + conf.Listen
	_, err = lowConf.InstallHandlers(hi, address, reindex, context)
	if err != nil {
		t.Fatal(err)
	}

	return httptest.NewServer(hi)
}
Esempio n. 15
0
func startEmailCommitLoop(errc chan<- error) {
	if *emailsTo == "" {
		return
	}
	if *emailNow != "" {
		dir, err := osutil.GoPackagePath("camlistore.org")
		if err != nil {
			log.Fatal(err)
		}
		if err := emailCommit(dir, *emailNow); err != nil {
			log.Fatal(err)
		}
		os.Exit(0)
	}
	go func() {
		errc <- commitEmailLoop()
	}()
}
Esempio n. 16
0
func (c *gaeCmd) RunCommand(args []string) error {
	err := c.checkFlags(args)
	if err != nil {
		return cmdmain.UsageError(fmt.Sprint(err))
	}
	c.camliSrcRoot, err = osutil.GoPackagePath("camlistore.org")
	if err != nil {
		return errors.New("Package camlistore.org not found in $GOPATH (or $GOPATH not defined).")
	}
	c.applicationDir = filepath.Join(c.camliSrcRoot, "server", "appengine")
	if _, err := os.Stat(c.applicationDir); err != nil {
		return fmt.Errorf("Appengine application dir not found at %s", c.applicationDir)
	}
	if err = c.checkSDK(); err != nil {
		return err
	}
	if err = c.mirrorSourceRoot(); err != nil {
		return err
	}

	devAppServerBin := filepath.Join(c.sdk, "dev_appserver.py")
	cmdArgs := []string{
		"--skip_sdk_update_check",
		fmt.Sprintf("--port=%s", c.port),
	}
	if c.all {
		cmdArgs = append(cmdArgs, "--host", "0.0.0.0")
	}
	if c.wipe {
		cmdArgs = append(cmdArgs, "--clear_datastore")
	}
	cmdArgs = append(cmdArgs, args...)
	cmdArgs = append(cmdArgs, c.applicationDir)
	cmd := exec.Command(devAppServerBin, cmdArgs...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Start(); err != nil {
		return fmt.Errorf("Could not start dev_appserver.py: %v", err)
	}
	go handleSignals(cmd.Process)
	cmd.Wait()
	return nil
}
Esempio n. 17
0
func main() {
	flag.Usage = usage
	flag.Parse()
	checkFlags()

	camDir, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		log.Fatalf("Error looking up camlistore.org dir: %v", err)
	}
	dockDir = filepath.Join(camDir, "misc", "docker")

	buildDockerImage("go", goDockerImage)
	// ctxDir is where we run "docker build" to produce the final
	// "FROM scratch" Docker image.
	ctxDir, err := ioutil.TempDir("", "camli-build")
	if err != nil {
		log.Fatal(err)
	}
	defer os.RemoveAll(ctxDir)
	switch {
	case *doImage:
		buildDockerImage("djpeg-static", djpegDockerImage)
		buildDockerImage("zoneinfo", zoneinfoDockerImage)
		genCamlistore(ctxDir)
		genDjpeg(ctxDir)
		genZoneinfo(ctxDir)
		buildServer(ctxDir)
	case *doBinaries:
		genBinaries(ctxDir)
		packBinaries(ctxDir)
	case *doZipSource:
		zipSource(ctxDir)
	}

	if !*doUpload {
		return
	}
	if *doImage {
		uploadDockerImage()
	} else {
		uploadReleaseTarball()
	}
}
Esempio n. 18
0
func commitEmailLoop() error {
	http.HandleFunc("/mailnow", mailNowHandler)

	go func() {
		for {
			select {
			case tokenc <- true:
			default:
			}
			time.Sleep(15 * time.Second)
		}
	}()

	dir, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		return err
	}

	hashes, err := recentCommits(dir)
	if err != nil {
		return err
	}
	for _, commit := range hashes {
		knownCommit[commit] = true
	}
	latestHash.Lock()
	latestHash.s = hashes[0]
	latestHash.Unlock()
	http.HandleFunc("/latesthash", latestHashHandler)

	for {
		pollCommits(dir)

		// Poll every minute or whenever we're forced with the
		// /mailnow handler.
		select {
		case <-time.After(1 * time.Minute):
		case <-fetchc:
			log.Printf("Polling git due to explicit trigger.")
		}
	}
}
Esempio n. 19
0
// NewWorld returns a new test world.
// It requires that GOPATH is set to find the "camlistore.org" root.
func NewWorld() (*World, error) {
	if os.Getenv("GOPATH") == "" {
		return nil, errors.New("GOPATH environment variable isn't set; required to run Camlistore integration tests")
	}
	root, err := osutil.GoPackagePath("camlistore.org")
	if err == os.ErrNotExist {
		return nil, errors.New("Directory \"camlistore.org\" not found under GOPATH/src; can't run Camlistore integration tests.")
	}
	if err != nil {
		return nil, fmt.Errorf("Error searching for \"camlistore.org\" under GOPATH: %v", err)
	}
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		return nil, err
	}

	return &World{
		camRoot:  root,
		listener: ln,
		port:     ln.Addr().(*net.TCPAddr).Port,
	}, nil
}
Esempio n. 20
0
func main() {
	flag.Usage = usage
	flag.Parse()
	checkFlags()

	var err error
	camDir, err = osutil.GoPackagePath("camlistore.org")
	if err != nil {
		log.Fatalf("Error looking up camlistore.org dir: %v", err)
	}

	if err := genDownloads(); err != nil {
		log.Fatal(err)
	}

	releaseData, err := listDownloads()
	if err != nil {
		log.Fatal(err)
	}

	if *flagStatsFrom != "" && !isWIP() {
		commitStats, err := genCommitStats()
		if err != nil {
			log.Fatal(err)
		}
		releaseData.Stats = commitStats

		notes, err := genReleaseNotes()
		if err != nil {
			log.Fatal(err)
		}
		releaseData.ReleaseNotes = notes
	}

	if err := genMonthlyPage(releaseData); err != nil {
		log.Fatal(err)
	}
}
Esempio n. 21
0
// NewIndexDeps returns an IndexDeps helper for populating and working
// with the provided index for tests.
func NewIndexDeps(index *index.Index) *IndexDeps {
	camliRootPath, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		log.Fatal("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
	}
	secretRingFile := filepath.Join(camliRootPath, "pkg", "jsonsign", "testdata", "test-secring.gpg")
	pubKey := &test.Blob{Contents: `-----BEGIN PGP PUBLIC KEY BLOCK-----

xsBNBEzgoVsBCAC/56aEJ9BNIGV9FVP+WzenTAkg12k86YqlwJVAB/VwdMlyXxvi
bCT1RVRfnYxscs14LLfcMWF3zMucw16mLlJCBSLvbZ0jn4h+/8vK5WuAdjw2YzLs
WtBcjWn3lV6tb4RJz5gtD/o1w8VWxwAnAVIWZntKAWmkcChCRgdUeWso76+plxE5
aRYBJqdT1mctGqNEISd/WYPMgwnWXQsVi3x4z1dYu2tD9uO1dkAff12z1kyZQIBQ
rexKYRRRh9IKAayD4kgS0wdlULjBU98aeEaMz1ckuB46DX3lAYqmmTEL/Rl9cOI0
Enpn/oOOfYFa5h0AFndZd1blMvruXfdAobjVABEBAAE=
=28/7
-----END PGP PUBLIC KEY BLOCK-----`}

	id := &IndexDeps{
		Index:            index,
		BlobSource:       new(test.Fetcher),
		PublicKeyFetcher: new(test.Fetcher),
		EntityFetcher: &jsonsign.CachingEntityFetcher{
			Fetcher: &jsonsign.FileEntityFetcher{File: secretRingFile},
		},
		SignerBlobRef: pubKey.BlobRef(),
		now:           time.Unix(1322443956, 123456),
		Fataler:       logFataler{},
	}
	// Add dev-camput's test key public key, keyid 26F5ABDA,
	// blobref sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007
	if g, w := id.SignerBlobRef.String(), "sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007"; g != w {
		id.Fatalf("unexpected signer blobref; got signer = %q; want %q", g, w)
	}
	id.PublicKeyFetcher.AddBlob(pubKey)
	id.Index.KeyFetcher = id.PublicKeyFetcher
	id.Index.BlobSource = id.BlobSource
	return id
}
Esempio n. 22
0
func main() {
	flag.Usage = usage
	flag.Parse()
	checkFlags()

	var err error
	camDir, err = osutil.GoPackagePath("camlistore.org")
	if err != nil {
		log.Fatalf("Error looking up camlistore.org dir: %v", err)
	}

	if err := genDownloads(); err != nil {
		log.Fatal(err)
	}

	releaseData, err := listDownloads()
	if err != nil {
		log.Fatal(err)
	}

	if err := genMonthlyPage(releaseData); err != nil {
		log.Fatal(err)
	}
}
Esempio n. 23
0
func TestAppEngineBuilds(t *testing.T) {
	camRoot, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		t.Errorf("No camlistore.org package in GOPATH: %v", err)
	}
	sdkLink := filepath.Join(camRoot, "appengine-sdk")
	if _, err := os.Lstat(sdkLink); os.IsNotExist(err) {
		t.Skipf("Skipping test; no App Engine SDK symlink at %s pointing to App Engine SDK.", sdkLink)
	}
	sdk, err := os.Readlink(sdkLink)
	if err != nil {
		t.Fatal(err)
	}

	td, err := ioutil.TempDir("", "camli-appengine")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(td)

	gab := filepath.Join(sdk, "goroot", "bin", "go-app-builder")
	if runtime.GOOS == "windows" {
		gab += ".exe"
	}

	appBase := filepath.Join(camRoot, "server", "appengine")
	f, err := os.Open(filepath.Join(appBase, "camli"))
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()
	srcFilesAll, err := f.Readdirnames(-1)
	if err != nil {
		t.Fatal(err)
	}

	cmd := exec.Command(gab,
		"-app_base", appBase,
		"-arch", archChar(),
		"-binary_name", "_go_app",
		"-dynamic",
		"-extra_imports", "appengine_internal/init",
		"-goroot", filepath.Join(sdk, "goroot"),
		"-nobuild_files", "^^$",
		"-unsafe",
		"-work_dir", td,
		"-gopath", os.Getenv("GOPATH"),
	)
	for _, f := range srcFilesAll {
		if strings.HasSuffix(f, ".go") {
			cmd.Args = append(cmd.Args, filepath.Join("camli", f))
		}
	}

	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("Error: %v\n%s", err, out)
	}
	target := filepath.Join(td, "_go_app")
	if _, err := os.Stat(target); os.IsNotExist(err) {
		t.Errorf("target binary doesn't exist")
	}
}
func main() {

	// check JRE presence
	_, err := exec.LookPath("java")
	if err != nil {
		log.Fatal("Didn't find 'java' in $PATH. The Java Runtime Environment is needed to run the closure compiler.\n")
	}

	camliRootPath, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		log.Fatal("Package camlistore.org not found in $GOPATH (or $GOPATH not defined).")
	}
	destDir := filepath.Join(camliRootPath, "tmp", "closure-compiler")
	// check if compiler already exists
	jarFile := filepath.Join(destDir, "compiler.jar")
	_, err = os.Stat(jarFile)
	if err == nil {
		// if compiler exists, check version
		cmd := exec.Command("java", "-jar", jarFile, "--version", "--help", "2>&1")
		output, _ := cmd.CombinedOutput()
		m := rgxVersion.FindStringSubmatch(string(output))
		if m == nil {
			log.Fatalf("Could not find compiler version in %q", output)
		}
		if m[1] == compilerVersion {
			log.Printf("compiler already at version %v , nothing to do.", compilerVersion)
			os.Exit(0)
		}
		if err := os.Remove(jarFile); err != nil {
			log.Fatalf("Could not remove %v: %v", jarFile, err)
		}
	} else {
		if !os.IsNotExist(err) {
			log.Fatalf("Could not stat %v: %v", jarFile, err)
		}
	}

	// otherwise, download compiler
	log.Printf("Getting closure compiler version %s.\n", compilerVersion)
	if err := os.MkdirAll(destDir, 0755); err != nil {
		log.Fatal(err)
	}
	if err := os.Chdir(destDir); err != nil {
		log.Fatal(err)
	}
	zipFilename := "compiler-" + compilerVersion + ".zip"
	compilerURL := compilerDirURL + zipFilename
	resp, err := http.Get(compilerURL)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	f, err := os.Create(zipFilename)
	if err != nil {
		log.Fatal(err)
	}
	if _, err := io.Copy(f, resp.Body); err != nil {
		log.Fatal(err)
	}
	if err := f.Close(); err != nil {
		log.Fatal(err)
	}

	r, err := zip.OpenReader(zipFilename)
	if err != nil {
		log.Fatal(err)
	}
	for x, f := range r.File {
		if f.FileHeader.Name != "compiler.jar" {
			if x == len(r.File)-1 {
				log.Fatal("compiler.jar was not found in the zip archive")
			}
			continue
		}
		rc, err := f.Open()
		if err != nil {
			log.Fatal(err)
		}
		g, err := os.Create(jarFile)
		if err != nil {
			log.Fatal(err)
		}
		defer g.Close()
		if _, err = io.Copy(g, rc); err != nil {
			log.Fatal(err)
		}
		rc.Close()
		break
	}

	if err := r.Close(); err != nil {
		log.Fatal(err)
	}
	if err := os.Remove(zipFilename); err != nil {
		log.Fatal(err)
	}
	log.Printf("Success. Installed at %v", jarFile)
}
Esempio n. 25
0
func Index(t *testing.T, initIdx func() *index.Index) {
	id := NewIndexDeps(initIdx())
	id.Fataler = t
	pn := id.NewPermanode()
	t.Logf("uploaded permanode %q", pn)
	br1 := id.SetAttribute(pn, "tag", "foo1")
	br1Time := id.lastTime()
	t.Logf("set attribute %q", br1)
	br2 := id.SetAttribute(pn, "tag", "foo2")
	br2Time := id.lastTime()
	t.Logf("set attribute %q", br2)
	rootClaim := id.SetAttribute(pn, "camliRoot", "rootval")
	rootClaimTime := id.lastTime()
	t.Logf("set attribute %q", rootClaim)

	pnChild := id.NewPermanode()
	br3 := id.SetAttribute(pnChild, "tag", "bar")
	br3Time := id.lastTime()
	t.Logf("set attribute %q", br3)
	memberRef := id.AddAttribute(pn, "camliMember", pnChild.String())
	t.Logf("add-attribute claim %q points to member permanode %q", memberRef, pnChild)
	memberRefTime := id.lastTime()

	// TODO(bradfitz): add EXIF tests here, once that stuff is ready.
	if false {
		camliRootPath, err := osutil.GoPackagePath("camlistore.org")
		if err != nil {
			t.Fatal("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
		}
		for i := 1; i <= 8; i++ {
			fileBase := fmt.Sprintf("f%d-exif.jpg", i)
			fileName := filepath.Join(camliRootPath, "pkg", "images", "testdata", fileBase)
			contents, err := ioutil.ReadFile(fileName)
			if err != nil {
				t.Fatal(err)
			}
			id.UploadFile(fileBase, string(contents))
		}
	}

	// Upload a basic image.
	var jpegFileRef *blobref.BlobRef
	{
		camliRootPath, err := osutil.GoPackagePath("camlistore.org")
		if err != nil {
			t.Fatal("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
		}
		fileName := filepath.Join(camliRootPath, "pkg", "index", "indextest", "testdata", "dude.jpg")
		contents, err := ioutil.ReadFile(fileName)
		if err != nil {
			t.Fatal(err)
		}
		jpegFileRef, _ = id.UploadFile("dude.jpg", string(contents))
	}

	lastPermanodeMutation := id.lastTime()
	id.dumpIndex(t)

	key := "signerkeyid:sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007"
	if g, e := id.Get(key), "2931A67C26F5ABDA"; g != e {
		t.Fatalf("%q = %q, want %q", key, g, e)
	}

	key = "imagesize|" + jpegFileRef.String()
	if g, e := id.Get(key), "50|100"; g != e {
		t.Errorf("JPEG dude.jpg key %q = %q; want %q", key, g, e)
	}

	key = "have:" + pn.String()
	pnSizeStr := id.Get(key)
	if pnSizeStr == "" {
		t.Fatalf("missing key %q", key)
	}

	key = "meta:" + pn.String()
	if g, e := id.Get(key), pnSizeStr+"|application/json; camliType=permanode"; g != e {
		t.Errorf("key %q = %q, want %q", key, g, e)
	}

	key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:62.999876543Z|" + br1.String()
	if g, e := id.Get(key), pn.String(); g != e {
		t.Fatalf("%q = %q, want %q (permanode)", key, g, e)
	}

	key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:61.999876543Z|" + br2.String()
	if g, e := id.Get(key), pn.String(); g != e {
		t.Fatalf("%q = %q, want %q (permanode)", key, g, e)
	}

	key = fmt.Sprintf("edgeback|%s|%s|%s", pnChild, pn, memberRef)
	if g, e := id.Get(key), "permanode|"; g != e {
		t.Fatalf("edgeback row %q = %q, want %q", key, g, e)
	}

	// PermanodeOfSignerAttrValue
	{
		gotPN, err := id.Index.PermanodeOfSignerAttrValue(id.SignerBlobRef, "camliRoot", "rootval")
		if err != nil {
			t.Fatalf("id.Index.PermanodeOfSignerAttrValue = %v", err)
		}
		if gotPN.String() != pn.String() {
			t.Errorf("id.Index.PermanodeOfSignerAttrValue = %q, want %q", gotPN, pn)
		}
		_, err = id.Index.PermanodeOfSignerAttrValue(id.SignerBlobRef, "camliRoot", "MISSING")
		if err == nil {
			t.Errorf("expected an error from PermanodeOfSignerAttrValue on missing value")
		}
	}

	// SearchPermanodesWithAttr - match attr type "tag" and value "foo1"
	{
		ch := make(chan *blobref.BlobRef, 10)
		req := &search.PermanodeByAttrRequest{
			Signer:    id.SignerBlobRef,
			Attribute: "tag",
			Query:     "foo1"}
		err := id.Index.SearchPermanodesWithAttr(ch, req)
		if err != nil {
			t.Fatalf("SearchPermanodesWithAttr = %v", err)
		}
		var got []*blobref.BlobRef
		for r := range ch {
			got = append(got, r)
		}
		want := []*blobref.BlobRef{pn}
		if len(got) < 1 || got[0].String() != want[0].String() {
			t.Errorf("id.Index.SearchPermanodesWithAttr gives %q, want %q", got, want)
		}
	}

	// SearchPermanodesWithAttr - match all with attr type "tag"
	{
		ch := make(chan *blobref.BlobRef, 10)
		req := &search.PermanodeByAttrRequest{
			Signer:    id.SignerBlobRef,
			Attribute: "tag"}
		err := id.Index.SearchPermanodesWithAttr(ch, req)
		if err != nil {
			t.Fatalf("SearchPermanodesWithAttr = %v", err)
		}
		var got []*blobref.BlobRef
		for r := range ch {
			got = append(got, r)
		}
		want := []*blobref.BlobRef{pn, pnChild}
		if len(got) != len(want) {
			t.Errorf("SearchPermanodesWithAttr results differ.\n got: %q\nwant: %q",
				got, want)
		}
		for _, w := range want {
			found := false
			for _, g := range got {
				if g.String() == w.String() {
					found = true
					break
				}
			}
			if !found {
				t.Errorf("SearchPermanodesWithAttr: %v was not found.\n", w)
			}
		}
	}

	// GetRecentPermanodes
	{
		ch := make(chan *search.Result, 10) // expect 2 results, but maybe more if buggy.
		err := id.Index.GetRecentPermanodes(ch, id.SignerBlobRef, 50)
		if err != nil {
			t.Fatalf("GetRecentPermanodes = %v", err)
		}
		got := []*search.Result{}
		for r := range ch {
			got = append(got, r)
		}
		want := []*search.Result{
			&search.Result{
				BlobRef:     pn,
				Signer:      id.SignerBlobRef,
				LastModTime: lastPermanodeMutation.Unix(),
			},
			&search.Result{
				BlobRef:     pnChild,
				Signer:      id.SignerBlobRef,
				LastModTime: br3Time.Unix(),
			},
		}
		if len(got) != len(want) {
			t.Errorf("GetRecentPermanode results differ.\n got: %v\nwant: %v",
				search.Results(got), search.Results(want))
		}
		for _, w := range want {
			found := false
			for _, g := range got {
				if reflect.DeepEqual(g, w) {
					found = true
					break
				}
			}
			if !found {
				t.Errorf("GetRecentPermanode: %v was not found.\n got: %v\nwant: %v",
					w, search.Results(got), search.Results(want))
			}
		}
	}

	// GetBlobMimeType
	{
		mime, size, err := id.Index.GetBlobMimeType(pn)
		if err != nil {
			t.Errorf("GetBlobMimeType(%q) = %v", pn, err)
		} else {
			if e := "application/json; camliType=permanode"; mime != e {
				t.Errorf("GetBlobMimeType(%q) mime = %q, want %q", pn, mime, e)
			}
			if size == 0 {
				t.Errorf("GetBlobMimeType(%q) size is zero", pn)
			}
		}
		_, _, err = id.Index.GetBlobMimeType(blobref.Parse("abc-123"))
		if err != os.ErrNotExist {
			t.Errorf("GetBlobMimeType(dummy blobref) = %v; want os.ErrNotExist", err)
		}
	}

	// GetOwnerClaims
	{
		claims, err := id.Index.GetOwnerClaims(pn, id.SignerBlobRef)
		if err != nil {
			t.Errorf("GetOwnerClaims = %v", err)
		} else {
			want := search.ClaimList([]*search.Claim{
				&search.Claim{
					BlobRef:   br1,
					Permanode: pn,
					Signer:    id.SignerBlobRef,
					Date:      br1Time.UTC(),
					Type:      "set-attribute",
					Attr:      "tag",
					Value:     "foo1",
				},
				&search.Claim{
					BlobRef:   br2,
					Permanode: pn,
					Signer:    id.SignerBlobRef,
					Date:      br2Time.UTC(),
					Type:      "set-attribute",
					Attr:      "tag",
					Value:     "foo2",
				},
				&search.Claim{
					BlobRef:   rootClaim,
					Permanode: pn,
					Signer:    id.SignerBlobRef,
					Date:      rootClaimTime.UTC(),
					Type:      "set-attribute",
					Attr:      "camliRoot",
					Value:     "rootval",
				},
				&search.Claim{
					BlobRef:   memberRef,
					Permanode: pn,
					Signer:    id.SignerBlobRef,
					Date:      memberRefTime.UTC(),
					Type:      "add-attribute",
					Attr:      "camliMember",
					Value:     pnChild.String(),
				},
			})
			if !reflect.DeepEqual(claims, want) {
				t.Errorf("GetOwnerClaims results differ.\n got: %v\nwant: %v",
					claims, want)
			}
		}
	}
}
Esempio n. 26
0
func uiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) {
	ui := &UIHandler{
		prefix:     ld.MyPrefix(),
		sourceRoot: conf.OptionalString("sourceRoot", ""),
		resizeSem: syncutil.NewSem(int64(conf.OptionalInt("maxResizeBytes",
			constants.DefaultMaxResizeMem))),
	}
	cachePrefix := conf.OptionalString("cache", "")
	scaledImageConf := conf.OptionalObject("scaledImage")
	if err = conf.Validate(); err != nil {
		return
	}

	scaledImageKV, err := newKVOrNil(scaledImageConf)
	if err != nil {
		return nil, fmt.Errorf("in UI handler's scaledImage: %v", err)
	}
	if scaledImageKV != nil && cachePrefix == "" {
		return nil, fmt.Errorf("in UI handler, can't specify scaledImage without cache")
	}
	if cachePrefix != "" {
		bs, err := ld.GetStorage(cachePrefix)
		if err != nil {
			return nil, fmt.Errorf("UI handler's cache of %q error: %v", cachePrefix, err)
		}
		ui.Cache = bs
		ui.thumbMeta = NewThumbMeta(scaledImageKV)
	}

	if ui.sourceRoot == "" {
		ui.sourceRoot = os.Getenv("CAMLI_DEV_CAMLI_ROOT")
		if uistatic.IsAppEngine {
			if _, err = os.Stat(filepath.Join(uistatic.GaeSourceRoot,
				filepath.FromSlash("server/camlistored/ui/index.html"))); err != nil {
				hint := fmt.Sprintf("\"sourceRoot\" was not specified in the config,"+
					" and the default sourceRoot dir %v does not exist or does not contain"+
					" \"server/camlistored/ui/index.html\". devcam appengine can do that for you.",
					uistatic.GaeSourceRoot)
				log.Print(hint)
				return nil, errors.New("No sourceRoot found; UI not available.")
			}
			log.Printf("Using the default \"%v\" as the sourceRoot for AppEngine", uistatic.GaeSourceRoot)
			ui.sourceRoot = uistatic.GaeSourceRoot
		}
		if ui.sourceRoot == "" && uistatic.Files.IsEmpty() {
			ui.sourceRoot, err = osutil.GoPackagePath("camlistore.org")
			if err != nil {
				log.Printf("Warning: server not compiled with linked-in UI resources (HTML, JS, CSS), and camlistore.org not found in GOPATH.")
			} else {
				log.Printf("Using UI resources (HTML, JS, CSS) from disk, under %v", ui.sourceRoot)
			}
		}
	}
	if ui.sourceRoot != "" {
		ui.uiDir = filepath.Join(ui.sourceRoot, filepath.FromSlash("server/camlistored/ui"))
		// Ignore any fileembed files:
		Files = &fileembed.Files{
			DirFallback: filepath.Join(ui.sourceRoot, filepath.FromSlash("pkg/server")),
		}
		uistatic.Files = &fileembed.Files{
			DirFallback: ui.uiDir,
			Listable:    true,
			// In dev_appserver, allow edit-and-reload without
			// restarting. In production, though, it's faster to just
			// slurp it in.
			SlurpToMemory: uistatic.IsProdAppEngine,
		}
	}

	ui.closureHandler, err = ui.makeClosureHandler(ui.sourceRoot)
	if err != nil {
		return nil, fmt.Errorf(`Invalid "sourceRoot" value of %q: %v"`, ui.sourceRoot, err)
	}

	if ui.sourceRoot != "" {
		ui.fileReactHandler, err = makeFileServer(ui.sourceRoot, filepath.Join(vendorEmbed, "react"), "react.js")
		if err != nil {
			return nil, fmt.Errorf("Could not make react handler: %s", err)
		}
		ui.fileGlitchHandler, err = makeFileServer(ui.sourceRoot, filepath.Join(vendorEmbed, "glitch"), "npc_piggy__x1_walk_png_1354829432.png")
		if err != nil {
			return nil, fmt.Errorf("Could not make glitch handler: %s", err)
		}
		ui.fileFontawesomeHandler, err = makeFileServer(ui.sourceRoot, filepath.Join(vendorEmbed, "fontawesome"), "css/font-awesome.css")
		if err != nil {
			return nil, fmt.Errorf("Could not make fontawesome handler: %s", err)
		}
		ui.fileLessHandler, err = makeFileServer(ui.sourceRoot, filepath.Join(vendorEmbed, "less"), "less.js")
		if err != nil {
			return nil, fmt.Errorf("Could not make less handler: %s", err)
		}
	}

	rootPrefix, _, err := ld.FindHandlerByType("root")
	if err != nil {
		return nil, errors.New("No root handler configured, which is necessary for the ui handler")
	}
	if h, err := ld.GetHandler(rootPrefix); err == nil {
		ui.root = h.(*RootHandler)
		ui.root.registerUIHandler(ui)
	} else {
		return nil, errors.New("failed to find the 'root' handler")
	}

	return ui, nil
}
Esempio n. 27
0
					 }
				 }
			   }`),
	},

	// Test recent permanode of a file
	{
		name: "recent-file",
		setup: func(*test.FakeIndex) index.Interface {
			// Ignore the fakeindex and use the real (but in-memory) implementation,
			// using IndexDeps to populate it.
			idx := index.NewMemoryIndex()
			id := indextest.NewIndexDeps(idx)

			// Upload a basic image
			camliRootPath, err := osutil.GoPackagePath("camlistore.org")
			if err != nil {
				panic("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
			}
			uploadFile := func(file string, modTime time.Time) blob.Ref {
				fileName := filepath.Join(camliRootPath, "pkg", "index", "indextest", "testdata", file)
				contents, err := ioutil.ReadFile(fileName)
				if err != nil {
					panic(err)
				}
				br, _ := id.UploadFile(file, string(contents), modTime)
				return br
			}
			dudeFileRef := uploadFile("dude.jpg", time.Time{})

			pn := id.NewPlannedPermanode("pn1")
Esempio n. 28
0
func TestInstallHandlers(t *testing.T) {
	camroot, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		t.Fatalf("failed to find camlistore.org GOPATH root: %v", err)
	}
	conf := serverinit.DefaultBaseConfig
	conf.Identity = "26F5ABDA"
	conf.IdentitySecretRing = filepath.Join(camroot, filepath.FromSlash("pkg/jsonsign/testdata/test-secring.gpg"))
	conf.MemoryStorage = true
	conf.MemoryIndex = true

	confData, err := json.MarshalIndent(conf, "", "    ")
	if err != nil {
		t.Fatalf("Could not json encode config: %v", err)
	}

	// Setting CAMLI_CONFIG_DIR to avoid triggering failInTests in osutil.CamliConfigDir
	defer os.Setenv("CAMLI_CONFIG_DIR", os.Getenv("CAMLI_CONFIG_DIR")) // restore after test
	os.Setenv("CAMLI_CONFIG_DIR", "whatever")
	lowConf, err := serverinit.Load(confData)
	if err != nil {
		t.Fatal(err)
	}
	// because these two are normally consumed in camlistored.go
	// TODO(mpl): serverinit.Load should consume these 2 as well. Once
	// consumed, we should keep all the answers as private fields, and then we
	// put accessors on serverinit.Config. Maybe we even stop embedding
	// jsonconfig.Obj in serverinit.Config too, so none of those methods are
	// accessible.
	lowConf.OptionalBool("https", true)
	lowConf.OptionalString("listen", "")

	reindex := false
	var context *http.Request // only used by App Engine. See handlerLoader in serverinit.go
	hi := http.NewServeMux()
	address := "http://" + conf.Listen
	_, err = lowConf.InstallHandlers(hi, address, reindex, context)
	if err != nil {
		t.Fatal(err)
	}

	tests := []struct {
		prefix        string
		authWrapped   bool
		prefixWrapped bool
		handlerType   reflect.Type
	}{
		{
			prefix:        "/",
			handlerType:   reflect.TypeOf(&server.RootHandler{}),
			prefixWrapped: true,
		},

		{
			prefix:        "/sync/",
			handlerType:   reflect.TypeOf(&server.SyncHandler{}),
			prefixWrapped: true,
			authWrapped:   true,
		},

		{
			prefix:        "/my-search/",
			handlerType:   reflect.TypeOf(&search.Handler{}),
			prefixWrapped: true,
			authWrapped:   true,
		},

		{
			prefix:        "/ui/",
			handlerType:   reflect.TypeOf(&server.UIHandler{}),
			prefixWrapped: true,
			authWrapped:   true,
		},

		{
			prefix:        "/importer/",
			handlerType:   reflect.TypeOf(&importer.Host{}),
			prefixWrapped: true,
			authWrapped:   true,
		},

		{
			prefix:        "/sighelper/",
			handlerType:   reflect.TypeOf(&signhandler.Handler{}),
			prefixWrapped: true,
			authWrapped:   true,
		},

		{
			prefix:        "/status/",
			handlerType:   reflect.TypeOf(&server.StatusHandler{}),
			prefixWrapped: true,
			authWrapped:   true,
		},

		{
			prefix:        "/help/",
			handlerType:   reflect.TypeOf(&server.HelpHandler{}),
			prefixWrapped: true,
			authWrapped:   true,
		},

		{
			prefix:        "/setup/",
			handlerType:   reflect.TypeOf(&server.SetupHandler{}),
			prefixWrapped: true,
		},

		{
			prefix:      "/bs/camli/",
			handlerType: reflect.TypeOf(http.HandlerFunc(nil)),
		},

		{
			prefix:      "/index/camli/",
			handlerType: reflect.TypeOf(http.HandlerFunc(nil)),
		},

		{
			prefix:      "/bs-and-index/camli/",
			handlerType: reflect.TypeOf(http.HandlerFunc(nil)),
		},

		{
			prefix:      "/bs-and-maybe-also-index/camli/",
			handlerType: reflect.TypeOf(http.HandlerFunc(nil)),
		},

		{
			prefix:      "/cache/camli/",
			handlerType: reflect.TypeOf(http.HandlerFunc(nil)),
		},
	}
	for _, v := range tests {
		req, err := http.NewRequest("GET", address+v.prefix, nil)
		if err != nil {
			t.Error(err)
			continue
		}
		h, _ := hi.Handler(req)
		if v.authWrapped {
			ah, ok := h.(auth.Handler)
			if !ok {
				t.Errorf("handler for %v should be auth wrapped", v.prefix)
				continue
			}
			h = ah.Handler
		}
		if v.prefixWrapped {
			ph, ok := h.(*httputil.PrefixHandler)
			if !ok {
				t.Errorf("handler for %v should be prefix wrapped", v.prefix)
				continue
			}
			h = ph.Handler
		}
		if reflect.TypeOf(h) != v.handlerType {
			t.Errorf("for %v: want %v, got %v", v.prefix, v.handlerType, reflect.TypeOf(h))
		}
	}
}
Esempio n. 29
0
func newUIFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) {
	ui := &UIHandler{
		prefix:       ld.MyPrefix(),
		JSONSignRoot: conf.OptionalString("jsonSignRoot", ""),
	}
	pubRoots := conf.OptionalList("publishRoots")
	cachePrefix := conf.OptionalString("cache", "")
	scType := conf.OptionalString("scaledImage", "")
	if err = conf.Validate(); err != nil {
		return
	}

	if ui.JSONSignRoot != "" {
		h, _ := ld.GetHandler(ui.JSONSignRoot)
		if sigh, ok := h.(*signhandler.Handler); ok {
			ui.sigh = sigh
		}
	}

	ui.PublishRoots = make(map[string]*PublishHandler)
	for _, pubRoot := range pubRoots {
		h, err := ld.GetHandler(pubRoot)
		if err != nil {
			return nil, fmt.Errorf("UI handler's publishRoots references invalid %q", pubRoot)
		}
		pubh, ok := h.(*PublishHandler)
		if !ok {
			return nil, fmt.Errorf("UI handler's publishRoots references invalid %q; not a PublishHandler", pubRoot)
		}
		ui.PublishRoots[pubRoot] = pubh
	}

	checkType := func(key string, htype string) {
		v := conf.OptionalString(key, "")
		if v == "" {
			return
		}
		ct := ld.GetHandlerType(v)
		if ct == "" {
			err = fmt.Errorf("UI handler's %q references non-existant %q", key, v)
		} else if ct != htype {
			err = fmt.Errorf("UI handler's %q references %q of type %q; expected type %q", key, v, ct, htype)
		}
	}
	checkType("searchRoot", "search")
	checkType("jsonSignRoot", "jsonsign")
	if err != nil {
		return
	}

	if cachePrefix != "" {
		bs, err := ld.GetStorage(cachePrefix)
		if err != nil {
			return nil, fmt.Errorf("UI handler's cache of %q error: %v", cachePrefix, err)
		}
		ui.Cache = bs
		switch scType {
		case "lrucache":
			ui.sc = NewScaledImageLru()
		default:
			return nil, fmt.Errorf("unsupported ui handler's scType: %q ", scType)
		}
	}

	camliRootPath, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		log.Printf("Package camlistore.org not found in $GOPATH (or $GOPATH not defined)." +
			" Closure will not be used.")
	} else {
		closureDir := filepath.Join(camliRootPath, "tmp", "closure-lib", "closure")
		ui.closureHandler = http.FileServer(http.Dir(closureDir))
	}

	rootPrefix, _, err := ld.FindHandlerByType("root")
	if err != nil {
		return nil, errors.New("No root handler configured, which is necessary for the ui handler")
	}
	if h, err := ld.GetHandler(rootPrefix); err == nil {
		ui.root = h.(*RootHandler)
		ui.root.registerUIHandler(ui)
	} else {
		return nil, errors.New("failed to find the 'root' handler")
	}

	return ui, nil
}
Esempio n. 30
0
func Index(t *testing.T, initIdx func() *index.Index) {
	oldLocal := time.Local
	time.Local = time.UTC
	defer func() { time.Local = oldLocal }()

	id := NewIndexDeps(initIdx())
	id.Fataler = t
	defer id.DumpIndex(t)
	pn := id.NewPermanode()
	t.Logf("uploaded permanode %q", pn)
	br1 := id.SetAttribute(pn, "tag", "foo1")
	br1Time := id.LastTime()
	t.Logf("set attribute %q", br1)
	br2 := id.SetAttribute(pn, "tag", "foo2")
	br2Time := id.LastTime()
	t.Logf("set attribute %q", br2)
	rootClaim := id.SetAttribute(pn, "camliRoot", "rootval")
	rootClaimTime := id.LastTime()
	t.Logf("set attribute %q", rootClaim)

	pnChild := id.NewPermanode()
	br3 := id.SetAttribute(pnChild, "tag", "bar")
	br3Time := id.LastTime()
	t.Logf("set attribute %q", br3)
	memberRef := id.AddAttribute(pn, "camliMember", pnChild.String())
	t.Logf("add-attribute claim %q points to member permanode %q", memberRef, pnChild)
	memberRefTime := id.LastTime()

	// TODO(bradfitz): add EXIF tests here, once that stuff is ready.
	if false {
		camliRootPath, err := osutil.GoPackagePath("camlistore.org")
		if err != nil {
			t.Fatal("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
		}
		for i := 1; i <= 8; i++ {
			fileBase := fmt.Sprintf("f%d-exif.jpg", i)
			fileName := filepath.Join(camliRootPath, "pkg", "images", "testdata", fileBase)
			contents, err := ioutil.ReadFile(fileName)
			if err != nil {
				t.Fatal(err)
			}
			id.UploadFile(fileBase, string(contents), noTime)
		}
	}

	// Upload some files.
	var jpegFileRef, exifFileRef, mediaFileRef, mediaWholeRef blob.Ref
	{
		camliRootPath, err := osutil.GoPackagePath("camlistore.org")
		if err != nil {
			t.Fatal("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
		}
		uploadFile := func(file string, modTime time.Time) (fileRef, wholeRef blob.Ref) {
			fileName := filepath.Join(camliRootPath, "pkg", "index", "indextest", "testdata", file)
			contents, err := ioutil.ReadFile(fileName)
			if err != nil {
				t.Fatal(err)
			}
			fileRef, wholeRef = id.UploadFile(file, string(contents), modTime)
			return
		}
		jpegFileRef, _ = uploadFile("dude.jpg", noTime)
		exifFileRef, _ = uploadFile("dude-exif.jpg", time.Unix(1361248796, 0))
		mediaFileRef, mediaWholeRef = uploadFile("0s.mp3", noTime)
	}

	// Upload the dir containing the previous files.
	imagesDirRef := id.UploadDir(
		"testdata",
		[]blob.Ref{jpegFileRef, exifFileRef, mediaFileRef},
		time.Now(),
	)

	lastPermanodeMutation := id.LastTime()

	key := "signerkeyid:sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007"
	if g, e := id.Get(key), "2931A67C26F5ABDA"; g != e {
		t.Fatalf("%q = %q, want %q", key, g, e)
	}

	key = "imagesize|" + jpegFileRef.String()
	if g, e := id.Get(key), "50|100"; g != e {
		t.Errorf("JPEG dude.jpg key %q = %q; want %q", key, g, e)
	}
	key = "filetimes|" + jpegFileRef.String()
	if g, e := id.Get(key), ""; g != e {
		t.Errorf("JPEG dude.jpg key %q = %q; want %q", key, g, e)
	}

	key = "filetimes|" + exifFileRef.String()
	if g, e := id.Get(key), "2013-02-18T01%3A11%3A20Z%2C2013-02-19T04%3A39%3A56Z"; g != e {
		t.Errorf("EXIF dude-exif.jpg key %q = %q; want %q", key, g, e)
	}

	key = "have:" + pn.String()
	pnSizeStr := strings.TrimSuffix(id.Get(key), "|indexed")
	if pnSizeStr == "" {
		t.Fatalf("missing key %q", key)
	}

	key = "meta:" + pn.String()
	if g, e := id.Get(key), pnSizeStr+"|application/json; camliType=permanode"; g != e {
		t.Errorf("key %q = %q, want %q", key, g, e)
	}

	key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:62.999876543Z|" + br1.String()
	if g, e := id.Get(key), pn.String(); g != e {
		t.Fatalf("%q = %q, want %q (permanode)", key, g, e)
	}

	key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:61.999876543Z|" + br2.String()
	if g, e := id.Get(key), pn.String(); g != e {
		t.Fatalf("%q = %q, want %q (permanode)", key, g, e)
	}

	key = fmt.Sprintf("edgeback|%s|%s|%s", pnChild, pn, memberRef)
	if g, e := id.Get(key), "permanode|"; g != e {
		t.Fatalf("edgeback row %q = %q, want %q", key, g, e)
	}

	mediaTests := []struct {
		prop, exp string
	}{
		{"title", "Zero Seconds"},
		{"artist", "Test Artist"},
		{"album", "Test Album"},
		{"genre", "(20)Alternative"},
		{"musicbrainzalbumid", "00000000-0000-0000-0000-000000000000"},
		{"year", "1992"},
		{"track", "1"},
		{"disc", "2"},
		{"mediaref", "sha1-fefac74a1d5928316d7131747107c8a61b71ffe4"},
		{"durationms", "26"},
	}
	for _, tt := range mediaTests {
		key = fmt.Sprintf("mediatag|%s|%s", mediaWholeRef.String(), tt.prop)
		if g, _ := url.QueryUnescape(id.Get(key)); g != tt.exp {
			t.Errorf("0s.mp3 key %q = %q; want %q", key, g, tt.exp)
		}
	}

	// PermanodeOfSignerAttrValue
	{
		gotPN, err := id.Index.PermanodeOfSignerAttrValue(id.SignerBlobRef, "camliRoot", "rootval")
		if err != nil {
			t.Fatalf("id.Index.PermanodeOfSignerAttrValue = %v", err)
		}
		if gotPN.String() != pn.String() {
			t.Errorf("id.Index.PermanodeOfSignerAttrValue = %q, want %q", gotPN, pn)
		}
		_, err = id.Index.PermanodeOfSignerAttrValue(id.SignerBlobRef, "camliRoot", "MISSING")
		if err == nil {
			t.Errorf("expected an error from PermanodeOfSignerAttrValue on missing value")
		}
	}

	// SearchPermanodesWithAttr - match attr type "tag" and value "foo1"
	{
		ch := make(chan blob.Ref, 10)
		req := &camtypes.PermanodeByAttrRequest{
			Signer:    id.SignerBlobRef,
			Attribute: "tag",
			Query:     "foo1",
		}
		err := id.Index.SearchPermanodesWithAttr(ch, req)
		if err != nil {
			t.Fatalf("SearchPermanodesWithAttr = %v", err)
		}
		var got []blob.Ref
		for r := range ch {
			got = append(got, r)
		}
		want := []blob.Ref{pn}
		if len(got) < 1 || got[0].String() != want[0].String() {
			t.Errorf("id.Index.SearchPermanodesWithAttr gives %q, want %q", got, want)
		}
	}

	// SearchPermanodesWithAttr - match all with attr type "tag"
	{
		ch := make(chan blob.Ref, 10)
		req := &camtypes.PermanodeByAttrRequest{
			Signer:    id.SignerBlobRef,
			Attribute: "tag",
		}
		err := id.Index.SearchPermanodesWithAttr(ch, req)
		if err != nil {
			t.Fatalf("SearchPermanodesWithAttr = %v", err)
		}
		var got []blob.Ref
		for r := range ch {
			got = append(got, r)
		}
		want := []blob.Ref{pn, pnChild}
		if len(got) != len(want) {
			t.Errorf("SearchPermanodesWithAttr results differ.\n got: %q\nwant: %q",
				got, want)
		}
		for _, w := range want {
			found := false
			for _, g := range got {
				if g.String() == w.String() {
					found = true
					break
				}
			}
			if !found {
				t.Errorf("SearchPermanodesWithAttr: %v was not found.\n", w)
			}
		}
	}

	// Delete value "pony" of type "title" (which does not actually exist) for pn
	br4 := id.DelAttribute(pn, "title", "pony")
	br4Time := id.LastTime()
	// and verify it is not found when searching by attr
	{
		ch := make(chan blob.Ref, 10)
		req := &camtypes.PermanodeByAttrRequest{
			Signer:    id.SignerBlobRef,
			Attribute: "title",
			Query:     "pony",
		}
		err := id.Index.SearchPermanodesWithAttr(ch, req)
		if err != nil {
			t.Fatalf("SearchPermanodesWithAttr = %v", err)
		}
		var got []blob.Ref
		for r := range ch {
			got = append(got, r)
		}
		want := []blob.Ref{}
		if len(got) != len(want) {
			t.Errorf("SearchPermanodesWithAttr results differ.\n got: %q\nwant: %q",
				got, want)
		}
	}

	// GetRecentPermanodes
	{
		verify := func(prefix string, want []camtypes.RecentPermanode, before time.Time) {
			ch := make(chan camtypes.RecentPermanode, 10) // expect 2 results, but maybe more if buggy.
			err := id.Index.GetRecentPermanodes(ch, id.SignerBlobRef, 50, before)
			if err != nil {
				t.Fatalf("[%s] GetRecentPermanodes = %v", prefix, err)
			}
			got := []camtypes.RecentPermanode{}
			for r := range ch {
				got = append(got, r)
			}
			if len(got) != len(want) {
				t.Errorf("[%s] GetRecentPermanode results differ.\n got: %v\nwant: %v",
					prefix, searchResults(got), searchResults(want))
			}
			for _, w := range want {
				found := false
				for _, g := range got {
					if g.Equal(w) {
						found = true
						break
					}
				}
				if !found {
					t.Errorf("[%s] GetRecentPermanode: %v was not found.\n got: %v\nwant: %v",
						prefix, w, searchResults(got), searchResults(want))
				}
			}
		}

		want := []camtypes.RecentPermanode{
			{
				Permanode:   pn,
				Signer:      id.SignerBlobRef,
				LastModTime: br4Time,
			},
			{
				Permanode:   pnChild,
				Signer:      id.SignerBlobRef,
				LastModTime: br3Time,
			},
		}

		before := time.Time{}
		verify("Zero before", want, before)

		before = lastPermanodeMutation
		t.Log("lastPermanodeMutation", lastPermanodeMutation,
			lastPermanodeMutation.Unix())
		verify("Non-zero before", want[1:], before)
	}
	// GetDirMembers
	{
		ch := make(chan blob.Ref, 10) // expect 2 results
		err := id.Index.GetDirMembers(imagesDirRef, ch, 50)
		if err != nil {
			t.Fatalf("GetDirMembers = %v", err)
		}
		got := []blob.Ref{}
		for r := range ch {
			got = append(got, r)
		}
		want := []blob.Ref{jpegFileRef, exifFileRef, mediaFileRef}
		if len(got) != len(want) {
			t.Errorf("GetDirMembers results differ.\n got: %v\nwant: %v",
				got, want)
		}
		for _, w := range want {
			found := false
			for _, g := range got {
				if w == g {
					found = true
					break
				}
			}
			if !found {
				t.Errorf("GetDirMembers: %v was not found.", w)
			}
		}
	}

	// GetBlobMeta
	{
		meta, err := id.Index.GetBlobMeta(pn)
		if err != nil {
			t.Errorf("GetBlobMeta(%q) = %v", pn, err)
		} else {
			if e := "permanode"; meta.CamliType != e {
				t.Errorf("GetBlobMeta(%q) mime = %q, want %q", pn, meta.CamliType, e)
			}
			if meta.Size == 0 {
				t.Errorf("GetBlobMeta(%q) size is zero", pn)
			}
		}
		_, err = id.Index.GetBlobMeta(blob.ParseOrZero("abc-123"))
		if err != os.ErrNotExist {
			t.Errorf("GetBlobMeta(dummy blobref) = %v; want os.ErrNotExist", err)
		}
	}

	// AppendClaims
	{
		claims, err := id.Index.AppendClaims(nil, pn, id.SignerBlobRef, "")
		if err != nil {
			t.Errorf("AppendClaims = %v", err)
		} else {
			want := []camtypes.Claim{
				{
					BlobRef:   br1,
					Permanode: pn,
					Signer:    id.SignerBlobRef,
					Date:      br1Time.UTC(),
					Type:      "set-attribute",
					Attr:      "tag",
					Value:     "foo1",
				},
				{
					BlobRef:   br2,
					Permanode: pn,
					Signer:    id.SignerBlobRef,
					Date:      br2Time.UTC(),
					Type:      "set-attribute",
					Attr:      "tag",
					Value:     "foo2",
				},
				{
					BlobRef:   rootClaim,
					Permanode: pn,
					Signer:    id.SignerBlobRef,
					Date:      rootClaimTime.UTC(),
					Type:      "set-attribute",
					Attr:      "camliRoot",
					Value:     "rootval",
				},
				{
					BlobRef:   memberRef,
					Permanode: pn,
					Signer:    id.SignerBlobRef,
					Date:      memberRefTime.UTC(),
					Type:      "add-attribute",
					Attr:      "camliMember",
					Value:     pnChild.String(),
				},
				{
					BlobRef:   br4,
					Permanode: pn,
					Signer:    id.SignerBlobRef,
					Date:      br4Time.UTC(),
					Type:      "del-attribute",
					Attr:      "title",
					Value:     "pony",
				},
			}
			if !reflect.DeepEqual(claims, want) {
				t.Errorf("AppendClaims results differ.\n got: %v\nwant: %v",
					claims, want)
			}
		}
	}
}