func doKeyStuff(b *testing.B) keyStuff {
camliRootPath, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
b.Fatal("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
}
secretRingFile := filepath.Join(camliRootPath, "pkg", "jsonsign", "testdata", "test-secring.gpg")
pubKey := `-----BEGIN PGP PUBLIC KEY BLOCK-----
xsBNBEzgoVsBCAC/56aEJ9BNIGV9FVP+WzenTAkg12k86YqlwJVAB/VwdMlyXxvi
bCT1RVRfnYxscs14LLfcMWF3zMucw16mLlJCBSLvbZ0jn4h+/8vK5WuAdjw2YzLs
WtBcjWn3lV6tb4RJz5gtD/o1w8VWxwAnAVIWZntKAWmkcChCRgdUeWso76+plxE5
aRYBJqdT1mctGqNEISd/WYPMgwnWXQsVi3x4z1dYu2tD9uO1dkAff12z1kyZQIBQ
rexKYRRRh9IKAayD4kgS0wdlULjBU98aeEaMz1ckuB46DX3lAYqmmTEL/Rl9cOI0
Enpn/oOOfYFa5h0AFndZd1blMvruXfdAobjVABEBAAE=
=28/7
-----END PGP PUBLIC KEY BLOCK-----`
return keyStuff{
secretRingFile: secretRingFile,
pubKey: pubKey,
pubKeyRef: blob.SHA1FromString(pubKey),
entityFetcher: &jsonsign.CachingEntityFetcher{
Fetcher: &jsonsign.FileEntityFetcher{File: secretRingFile},
},
}
}
func gitShortlog() *exec.Cmd {
if !*gitContainer {
return exec.Command("/bin/bash", "-c", "git log | git shortlog -sen")
}
args := []string{"run", "--rm"}
if inProd {
args = append(args,
"-v", "/var/camweb:/var/camweb",
"--workdir="+prodSrcDir,
)
} else {
hostRoot, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
log.Fatal(err)
}
log.Printf("Using bind root of %q", hostRoot)
args = append(args,
"-v", hostRoot+":"+prodSrcDir,
"--workdir="+prodSrcDir,
)
}
args = append(args, "camlistore/git", "/bin/bash", "-c", "git log | git shortlog -sen")
cmd := exec.Command("docker", args...)
cmd.Stderr = os.Stderr
return cmd
}
func TestExpansionsInHighlevelConfig(t *testing.T) {
camroot, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
t.Fatalf("failed to find camlistore.org GOPATH root: %v", err)
}
const keyID = "26F5ABDA"
os.Setenv("TMP_EXPANSION_TEST", keyID)
os.Setenv("TMP_EXPANSION_SECRING", filepath.Join(camroot, filepath.FromSlash("pkg/jsonsign/testdata/test-secring.gpg")))
// Setting CAMLI_CONFIG_DIR to avoid triggering failInTests in osutil.CamliConfigDir
defer os.Setenv("CAMLI_CONFIG_DIR", os.Getenv("CAMLI_CONFIG_DIR")) // restore after test
os.Setenv("CAMLI_CONFIG_DIR", "whatever")
conf, err := serverinit.Load([]byte(`
{
"auth": "localhost",
"listen": ":4430",
"https": false,
"identity": ["_env", "${TMP_EXPANSION_TEST}"],
"identitySecretRing": ["_env", "${TMP_EXPANSION_SECRING}"],
"googlecloudstorage": ":camlistore-dev-blobs",
"kvIndexFile": "/tmp/camli-index.kvdb"
}
`))
if err != nil {
t.Fatal(err)
}
got := fmt.Sprintf("%#v", conf)
if !strings.Contains(got, keyID) {
t.Errorf("Expected key %s in resulting low-level config. Got: %s", keyID, got)
}
}
// setup checks if the camlistore root can be found,
// then sets up closureGitDir and destDir, and returns whether
// we should clone or update in closureGitDir (depending on
// if a .git dir was found).
func setup() string {
camliRootPath, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
log.Fatal("Package camlistore.org not found in $GOPATH (or $GOPATH not defined).")
}
destDir = filepath.Join(camliRootPath, "third_party", "closure", "lib")
closureGitDir = filepath.Join(camliRootPath, "tmp", "closure-lib")
op := "update"
_, err = os.Stat(closureGitDir)
if err != nil {
if os.IsNotExist(err) {
err = os.MkdirAll(closureGitDir, 0755)
if err != nil {
log.Fatalf("Could not create %v: %v", closureGitDir, err)
}
op = "clone"
} else {
log.Fatalf("Could not stat %v: %v", closureGitDir, err)
}
}
dotGitPath := filepath.Join(closureGitDir, ".git")
_, err = os.Stat(dotGitPath)
if err != nil {
if os.IsNotExist(err) {
op = "clone"
} else {
log.Fatalf("Could not stat %v: %v", dotGitPath, err)
}
}
return op
}
// camliClosurePage checks if filename is a .js file using closure
// and if yes, if it provides a page in the camlistore namespace.
// It returns that page name, or the empty string otherwise.
func camliClosurePage(filename string) string {
camliRootPath, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
return ""
}
fullpath := filepath.Join(camliRootPath, "server", "camlistored", "ui", filename)
f, err := os.Open(fullpath)
if err != nil {
return ""
}
defer f.Close()
br := bufio.NewReader(f)
for {
l, err := br.ReadString('\n')
if err != nil {
return ""
}
if !strings.HasPrefix(l, "goog.") {
continue
}
m := provCamliRx.FindStringSubmatch(l)
if m != nil {
return m[2]
}
}
return ""
}
// fileList parses deps.js from the closure repo, as well as the similar
// dependencies generated for the UI js files, and compiles the list of
// js files from the closure lib required for the UI.
func fileList() ([]string, error) {
camliRootPath, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
log.Fatal("Package camlistore.org not found in $GOPATH (or $GOPATH not defined).")
}
uiDir := filepath.Join(camliRootPath, "server", "camlistored", "ui")
closureDepsFile := filepath.Join(closureGitDir, "closure", "goog", "deps.js")
f, err := os.Open(closureDepsFile)
if err != nil {
return nil, err
}
defer f.Close()
allClosureDeps, err := closure.DeepParseDeps(f)
if err != nil {
return nil, err
}
uiDeps, err := closure.GenDeps(http.Dir(uiDir))
if err != nil {
return nil, err
}
_, requ, err := closure.ParseDeps(bytes.NewReader(uiDeps))
if err != nil {
return nil, err
}
nameDone := make(map[string]bool)
jsfilesDone := make(map[string]bool)
for _, deps := range requ {
for _, dep := range deps {
if _, ok := nameDone[dep]; ok {
continue
}
jsfiles := allClosureDeps[dep]
for _, filename := range jsfiles {
if _, ok := jsfilesDone[filename]; ok {
continue
}
jsfilesDone[filename] = true
}
nameDone[dep] = true
}
}
jsfiles := []string{
"AUTHORS",
"LICENSE",
"README",
filepath.Join("closure", "goog", "base.js"),
filepath.Join("closure", "goog", "css", "common.css"),
filepath.Join("closure", "goog", "css", "toolbar.css"),
filepath.Join("closure", "goog", "deps.js"),
}
prefix := filepath.Join("closure", "goog")
for k, _ := range jsfilesDone {
jsfiles = append(jsfiles, filepath.Join(prefix, k))
}
sort.Strings(jsfiles)
return jsfiles, nil
}
func main() {
flag.Usage = usage
flag.Parse()
checkFlags()
camDir, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
log.Fatalf("Error looking up camlistore.org dir: %v", err)
}
dockDir = filepath.Join(camDir, "misc", "docker")
if *doBuildServer {
buildDockerImage("go", goDockerImage)
buildDockerImage("djpeg-static", djpegDockerImage)
// ctxDir is where we run "docker build" to produce the final
// "FROM scratch" Docker image.
ctxDir, err := ioutil.TempDir("", "camli-build")
if err != nil {
log.Fatal(err)
}
defer os.RemoveAll(ctxDir)
genCamlistore(ctxDir)
genDjpeg(ctxDir)
buildServer(ctxDir)
}
if *doUpload {
uploadDockerImage()
}
}
func camSrcDir() string {
if inProd {
return prodSrcDir
}
dir, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
log.Fatalf("Failed to find the root of the Camlistore source code via osutil.GoPackagePath: %v", err)
}
return dir
}
// TODO(mpl): refactor with twitter
func fakePhoto() string {
camliDir, err := osutil.GoPackagePath("camlistore.org")
if err == os.ErrNotExist {
log.Fatal("Directory \"camlistore.org\" not found under GOPATH/src; are you not running with devcam?")
}
if err != nil {
log.Fatalf("Error searching for \"camlistore.org\" under GOPATH: %v", err)
}
return filepath.Join(camliDir, filepath.FromSlash("third_party/glitch/npc_piggy__x1_walk_png_1354829432.png"))
}
// CamliSourceRoot returns the root of the source tree, or an error.
func camliSourceRoot() (string, error) {
if os.Getenv("GOPATH") == "" {
return "", errors.New("GOPATH environment variable isn't set; required to run Camlistore integration tests")
}
root, err := osutil.GoPackagePath("camlistore.org")
if err == os.ErrNotExist {
return "", errors.New("Directory \"camlistore.org\" not found under GOPATH/src; can't run Camlistore integration tests.")
}
return root, nil
}
func (c *serverCmd) RunCommand(args []string) error {
err := c.checkFlags(args)
if err != nil {
return cmdmain.UsageError(fmt.Sprint(err))
}
c.camliSrcRoot, err = osutil.GoPackagePath("camlistore.org")
if err != nil {
return errors.New("Package camlistore.org not found in $GOPATH (or $GOPATH not defined).")
}
err = os.Chdir(c.camliSrcRoot)
if err != nil {
return fmt.Errorf("Could not chdir to %v: %v", c.camliSrcRoot, err)
}
if !c.noBuild {
for _, name := range []string{"camlistored", "camtool"} {
err := c.build(name)
if err != nil {
return fmt.Errorf("Could not build %v: %v", name, err)
}
}
}
if err := c.setCamliRoot(); err != nil {
return fmt.Errorf("Could not setup the camli root: %v", err)
}
if err := c.setEnvVars(); err != nil {
return fmt.Errorf("Could not setup the env vars: %v", err)
}
if err := c.setupIndexer(); err != nil {
return fmt.Errorf("Could not setup the indexer: %v", err)
}
if err := c.syncTemplateBlobs(); err != nil {
return fmt.Errorf("Could not copy the template blobs: %v", err)
}
if err := c.setFullClosure(); err != nil {
return fmt.Errorf("Could not setup the closure lib: %v", err)
}
log.Printf("Starting dev server on %v/ui/ with password \"pass%v\"\n",
os.Getenv("CAMLI_BASEURL"), c.port)
camliBin := filepath.Join(c.camliSrcRoot, "bin", "camlistored")
cmdArgs := []string{
"-configfile=" + filepath.Join(c.camliSrcRoot, "config", "dev-server-config.json"),
"-listen=" + c.listen}
cmd := exec.Command(camliBin, cmdArgs...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Start(); err != nil {
return fmt.Errorf("Could not start camlistored: %v", err)
}
go handleKillCamliSignal(cmd.Process)
cmd.Wait()
return nil
}
func main() {
flag.Usage = usage
flag.Parse()
checkFlags()
camDir, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
log.Fatalf("Error looking up camlistore.org dir: %v", err)
}
dockDir = filepath.Join(camDir, "misc", "docker")
if *doImage {
buildDockerImage("go", goDockerImage)
buildDockerImage("djpeg-static", djpegDockerImage)
// ctxDir is where we run "docker build" to produce the final
// "FROM scratch" Docker image.
ctxDir, err := ioutil.TempDir("", "camli-build")
if err != nil {
log.Fatal(err)
}
defer os.RemoveAll(ctxDir)
genCamlistore(ctxDir)
genDjpeg(ctxDir)
buildServer(ctxDir)
}
// TODO(mpl): maybe *doBinaries should be done by a separate go program,
// because the end product is not a docker image. However, we're still
// using docker all along, and it's convenient for now for code reuse. I
// can refactor it all out of dock.go afterwards if we like the process.
if *doBinaries {
// TODO(mpl): consider using an "official" or trusted existing
// Go docker image, since we don't do anything special anymore in
// ours?
buildDockerImage("go", goDockerImage+"-linux")
ctxDir, err := ioutil.TempDir("", "camli-build")
if err != nil {
log.Fatal(err)
}
defer os.RemoveAll(ctxDir)
genBinaries(ctxDir)
packBinaries(ctxDir)
}
if *doUpload {
if *doImage {
uploadDockerImage()
} else if *doBinaries {
uploadReleaseTarball()
}
}
}
func TestQueryPermanodeLocation(t *testing.T) {
testQuery(t, func(qt *queryTest) {
id := qt.id
p1 := id.NewPlannedPermanode("1")
p2 := id.NewPlannedPermanode("2")
p3 := id.NewPlannedPermanode("3")
id.SetAttribute(p1, "latitude", "51.5")
id.SetAttribute(p1, "longitude", "0")
id.SetAttribute(p2, "latitude", "51.5")
id.SetAttribute(p3, "longitude", "0")
p4 := id.NewPlannedPermanode("checkin")
p5 := id.NewPlannedPermanode("venue")
id.SetAttribute(p4, "camliNodeType", "foursquare.com:checkin")
id.SetAttribute(p4, "foursquareVenuePermanode", p5.String())
id.SetAttribute(p5, "latitude", "1.0")
id.SetAttribute(p5, "longitude", "2.0")
// Upload a basic image
camliRootPath, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
panic("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
}
uploadFile := func(file string, modTime time.Time) blob.Ref {
fileName := filepath.Join(camliRootPath, "pkg", "search", "testdata", file)
contents, err := ioutil.ReadFile(fileName)
if err != nil {
panic(err)
}
br, _ := id.UploadFile(file, string(contents), modTime)
return br
}
fileRef := uploadFile("dude-gps.jpg", time.Time{})
p6 := id.NewPlannedPermanode("photo")
id.SetAttribute(p6, "camliContent", fileRef.String())
sq := &SearchQuery{
Constraint: &Constraint{
Permanode: &PermanodeConstraint{
Location: &LocationConstraint{
Any: true,
},
},
},
}
qt.wantRes(sq, p1, p4, p5, p6)
})
}
// newTestServer creates a new test server with in memory storage for use in upload tests
func newTestServer(t *testing.T) *httptest.Server {
camroot, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
t.Fatalf("failed to find camlistore.org GOPATH root: %v", err)
}
conf := serverconfig.Config{
Listen: ":3179",
HTTPS: false,
Auth: "localhost",
Identity: "26F5ABDA",
IdentitySecretRing: filepath.Join(camroot, filepath.FromSlash("pkg/jsonsign/testdata/test-secring.gpg")),
MemoryStorage: true,
MemoryIndex: true,
}
confData, err := json.MarshalIndent(conf, "", " ")
if err != nil {
t.Fatalf("Could not json encode config: %v", err)
}
// Setting CAMLI_CONFIG_DIR to avoid triggering failInTests in osutil.CamliConfigDir
defer os.Setenv("CAMLI_CONFIG_DIR", os.Getenv("CAMLI_CONFIG_DIR")) // restore after test
os.Setenv("CAMLI_CONFIG_DIR", "whatever")
lowConf, err := serverinit.Load(confData)
if err != nil {
t.Fatal(err)
}
// because these two are normally consumed in camlistored.go
// TODO(mpl): serverinit.Load should consume these 2 as well. Once
// consumed, we should keep all the answers as private fields, and then we
// put accessors on serverinit.Config. Maybe we even stop embedding
// jsonconfig.Obj in serverinit.Config too, so none of those methods are
// accessible.
lowConf.OptionalBool("https", true)
lowConf.OptionalString("listen", "")
reindex := false
var context *http.Request // only used by App Engine. See handlerLoader in serverinit.go
hi := http.NewServeMux()
address := "http://" + conf.Listen
_, err = lowConf.InstallHandlers(hi, address, reindex, context)
if err != nil {
t.Fatal(err)
}
return httptest.NewServer(hi)
}
func startEmailCommitLoop(errc chan<- error) {
if *emailsTo == "" {
return
}
if *emailNow != "" {
dir, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
log.Fatal(err)
}
if err := emailCommit(dir, *emailNow); err != nil {
log.Fatal(err)
}
os.Exit(0)
}
go func() {
errc <- commitEmailLoop()
}()
}
func (c *gaeCmd) RunCommand(args []string) error {
err := c.checkFlags(args)
if err != nil {
return cmdmain.UsageError(fmt.Sprint(err))
}
c.camliSrcRoot, err = osutil.GoPackagePath("camlistore.org")
if err != nil {
return errors.New("Package camlistore.org not found in $GOPATH (or $GOPATH not defined).")
}
c.applicationDir = filepath.Join(c.camliSrcRoot, "server", "appengine")
if _, err := os.Stat(c.applicationDir); err != nil {
return fmt.Errorf("Appengine application dir not found at %s", c.applicationDir)
}
if err = c.checkSDK(); err != nil {
return err
}
if err = c.mirrorSourceRoot(); err != nil {
return err
}
devAppServerBin := filepath.Join(c.sdk, "dev_appserver.py")
cmdArgs := []string{
"--skip_sdk_update_check",
fmt.Sprintf("--port=%s", c.port),
}
if c.all {
cmdArgs = append(cmdArgs, "--host", "0.0.0.0")
}
if c.wipe {
cmdArgs = append(cmdArgs, "--clear_datastore")
}
cmdArgs = append(cmdArgs, args...)
cmdArgs = append(cmdArgs, c.applicationDir)
cmd := exec.Command(devAppServerBin, cmdArgs...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Start(); err != nil {
return fmt.Errorf("Could not start dev_appserver.py: %v", err)
}
go handleSignals(cmd.Process)
cmd.Wait()
return nil
}
func main() {
flag.Usage = usage
flag.Parse()
checkFlags()
camDir, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
log.Fatalf("Error looking up camlistore.org dir: %v", err)
}
dockDir = filepath.Join(camDir, "misc", "docker")
buildDockerImage("go", goDockerImage)
// ctxDir is where we run "docker build" to produce the final
// "FROM scratch" Docker image.
ctxDir, err := ioutil.TempDir("", "camli-build")
if err != nil {
log.Fatal(err)
}
defer os.RemoveAll(ctxDir)
switch {
case *doImage:
buildDockerImage("djpeg-static", djpegDockerImage)
buildDockerImage("zoneinfo", zoneinfoDockerImage)
genCamlistore(ctxDir)
genDjpeg(ctxDir)
genZoneinfo(ctxDir)
buildServer(ctxDir)
case *doBinaries:
genBinaries(ctxDir)
packBinaries(ctxDir)
case *doZipSource:
zipSource(ctxDir)
}
if !*doUpload {
return
}
if *doImage {
uploadDockerImage()
} else {
uploadReleaseTarball()
}
}
func commitEmailLoop() error {
http.HandleFunc("/mailnow", mailNowHandler)
go func() {
for {
select {
case tokenc <- true:
default:
}
time.Sleep(15 * time.Second)
}
}()
dir, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
return err
}
hashes, err := recentCommits(dir)
if err != nil {
return err
}
for _, commit := range hashes {
knownCommit[commit] = true
}
latestHash.Lock()
latestHash.s = hashes[0]
latestHash.Unlock()
http.HandleFunc("/latesthash", latestHashHandler)
for {
pollCommits(dir)
// Poll every minute or whenever we're forced with the
// /mailnow handler.
select {
case <-time.After(1 * time.Minute):
case <-fetchc:
log.Printf("Polling git due to explicit trigger.")
}
}
}
// NewWorld returns a new test world.
// It requires that GOPATH is set to find the "camlistore.org" root.
func NewWorld() (*World, error) {
if os.Getenv("GOPATH") == "" {
return nil, errors.New("GOPATH environment variable isn't set; required to run Camlistore integration tests")
}
root, err := osutil.GoPackagePath("camlistore.org")
if err == os.ErrNotExist {
return nil, errors.New("Directory \"camlistore.org\" not found under GOPATH/src; can't run Camlistore integration tests.")
}
if err != nil {
return nil, fmt.Errorf("Error searching for \"camlistore.org\" under GOPATH: %v", err)
}
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return nil, err
}
return &World{
camRoot: root,
listener: ln,
port: ln.Addr().(*net.TCPAddr).Port,
}, nil
}
func main() {
flag.Usage = usage
flag.Parse()
checkFlags()
var err error
camDir, err = osutil.GoPackagePath("camlistore.org")
if err != nil {
log.Fatalf("Error looking up camlistore.org dir: %v", err)
}
if err := genDownloads(); err != nil {
log.Fatal(err)
}
releaseData, err := listDownloads()
if err != nil {
log.Fatal(err)
}
if *flagStatsFrom != "" && !isWIP() {
commitStats, err := genCommitStats()
if err != nil {
log.Fatal(err)
}
releaseData.Stats = commitStats
notes, err := genReleaseNotes()
if err != nil {
log.Fatal(err)
}
releaseData.ReleaseNotes = notes
}
if err := genMonthlyPage(releaseData); err != nil {
log.Fatal(err)
}
}
// NewIndexDeps returns an IndexDeps helper for populating and working
// with the provided index for tests.
func NewIndexDeps(index *index.Index) *IndexDeps {
camliRootPath, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
log.Fatal("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
}
secretRingFile := filepath.Join(camliRootPath, "pkg", "jsonsign", "testdata", "test-secring.gpg")
pubKey := &test.Blob{Contents: `-----BEGIN PGP PUBLIC KEY BLOCK-----
xsBNBEzgoVsBCAC/56aEJ9BNIGV9FVP+WzenTAkg12k86YqlwJVAB/VwdMlyXxvi
bCT1RVRfnYxscs14LLfcMWF3zMucw16mLlJCBSLvbZ0jn4h+/8vK5WuAdjw2YzLs
WtBcjWn3lV6tb4RJz5gtD/o1w8VWxwAnAVIWZntKAWmkcChCRgdUeWso76+plxE5
aRYBJqdT1mctGqNEISd/WYPMgwnWXQsVi3x4z1dYu2tD9uO1dkAff12z1kyZQIBQ
rexKYRRRh9IKAayD4kgS0wdlULjBU98aeEaMz1ckuB46DX3lAYqmmTEL/Rl9cOI0
Enpn/oOOfYFa5h0AFndZd1blMvruXfdAobjVABEBAAE=
=28/7
-----END PGP PUBLIC KEY BLOCK-----`}
id := &IndexDeps{
Index: index,
BlobSource: new(test.Fetcher),
PublicKeyFetcher: new(test.Fetcher),
EntityFetcher: &jsonsign.CachingEntityFetcher{
Fetcher: &jsonsign.FileEntityFetcher{File: secretRingFile},
},
SignerBlobRef: pubKey.BlobRef(),
now: time.Unix(1322443956, 123456),
Fataler: logFataler{},
}
// Add dev-camput's test key public key, keyid 26F5ABDA,
// blobref sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007
if g, w := id.SignerBlobRef.String(), "sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007"; g != w {
id.Fatalf("unexpected signer blobref; got signer = %q; want %q", g, w)
}
id.PublicKeyFetcher.AddBlob(pubKey)
id.Index.KeyFetcher = id.PublicKeyFetcher
id.Index.BlobSource = id.BlobSource
return id
}
func main() {
flag.Usage = usage
flag.Parse()
checkFlags()
var err error
camDir, err = osutil.GoPackagePath("camlistore.org")
if err != nil {
log.Fatalf("Error looking up camlistore.org dir: %v", err)
}
if err := genDownloads(); err != nil {
log.Fatal(err)
}
releaseData, err := listDownloads()
if err != nil {
log.Fatal(err)
}
if err := genMonthlyPage(releaseData); err != nil {
log.Fatal(err)
}
}
func TestAppEngineBuilds(t *testing.T) {
camRoot, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
t.Errorf("No camlistore.org package in GOPATH: %v", err)
}
sdkLink := filepath.Join(camRoot, "appengine-sdk")
if _, err := os.Lstat(sdkLink); os.IsNotExist(err) {
t.Skipf("Skipping test; no App Engine SDK symlink at %s pointing to App Engine SDK.", sdkLink)
}
sdk, err := os.Readlink(sdkLink)
if err != nil {
t.Fatal(err)
}
td, err := ioutil.TempDir("", "camli-appengine")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(td)
gab := filepath.Join(sdk, "goroot", "bin", "go-app-builder")
if runtime.GOOS == "windows" {
gab += ".exe"
}
appBase := filepath.Join(camRoot, "server", "appengine")
f, err := os.Open(filepath.Join(appBase, "camli"))
if err != nil {
t.Fatal(err)
}
defer f.Close()
srcFilesAll, err := f.Readdirnames(-1)
if err != nil {
t.Fatal(err)
}
cmd := exec.Command(gab,
"-app_base", appBase,
"-arch", archChar(),
"-binary_name", "_go_app",
"-dynamic",
"-extra_imports", "appengine_internal/init",
"-goroot", filepath.Join(sdk, "goroot"),
"-nobuild_files", "^^$",
"-unsafe",
"-work_dir", td,
"-gopath", os.Getenv("GOPATH"),
)
for _, f := range srcFilesAll {
if strings.HasSuffix(f, ".go") {
cmd.Args = append(cmd.Args, filepath.Join("camli", f))
}
}
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("Error: %v\n%s", err, out)
}
target := filepath.Join(td, "_go_app")
if _, err := os.Stat(target); os.IsNotExist(err) {
t.Errorf("target binary doesn't exist")
}
}
func main() {
// check JRE presence
_, err := exec.LookPath("java")
if err != nil {
log.Fatal("Didn't find 'java' in $PATH. The Java Runtime Environment is needed to run the closure compiler.\n")
}
camliRootPath, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
log.Fatal("Package camlistore.org not found in $GOPATH (or $GOPATH not defined).")
}
destDir := filepath.Join(camliRootPath, "tmp", "closure-compiler")
// check if compiler already exists
jarFile := filepath.Join(destDir, "compiler.jar")
_, err = os.Stat(jarFile)
if err == nil {
// if compiler exists, check version
cmd := exec.Command("java", "-jar", jarFile, "--version", "--help", "2>&1")
output, _ := cmd.CombinedOutput()
m := rgxVersion.FindStringSubmatch(string(output))
if m == nil {
log.Fatalf("Could not find compiler version in %q", output)
}
if m[1] == compilerVersion {
log.Printf("compiler already at version %v , nothing to do.", compilerVersion)
os.Exit(0)
}
if err := os.Remove(jarFile); err != nil {
log.Fatalf("Could not remove %v: %v", jarFile, err)
}
} else {
if !os.IsNotExist(err) {
log.Fatalf("Could not stat %v: %v", jarFile, err)
}
}
// otherwise, download compiler
log.Printf("Getting closure compiler version %s.\n", compilerVersion)
if err := os.MkdirAll(destDir, 0755); err != nil {
log.Fatal(err)
}
if err := os.Chdir(destDir); err != nil {
log.Fatal(err)
}
zipFilename := "compiler-" + compilerVersion + ".zip"
compilerURL := compilerDirURL + zipFilename
resp, err := http.Get(compilerURL)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
f, err := os.Create(zipFilename)
if err != nil {
log.Fatal(err)
}
if _, err := io.Copy(f, resp.Body); err != nil {
log.Fatal(err)
}
if err := f.Close(); err != nil {
log.Fatal(err)
}
r, err := zip.OpenReader(zipFilename)
if err != nil {
log.Fatal(err)
}
for x, f := range r.File {
if f.FileHeader.Name != "compiler.jar" {
if x == len(r.File)-1 {
log.Fatal("compiler.jar was not found in the zip archive")
}
continue
}
rc, err := f.Open()
if err != nil {
log.Fatal(err)
}
g, err := os.Create(jarFile)
if err != nil {
log.Fatal(err)
}
defer g.Close()
if _, err = io.Copy(g, rc); err != nil {
log.Fatal(err)
}
rc.Close()
break
}
if err := r.Close(); err != nil {
log.Fatal(err)
}
if err := os.Remove(zipFilename); err != nil {
log.Fatal(err)
}
log.Printf("Success. Installed at %v", jarFile)
}
func Index(t *testing.T, initIdx func() *index.Index) {
id := NewIndexDeps(initIdx())
id.Fataler = t
pn := id.NewPermanode()
t.Logf("uploaded permanode %q", pn)
br1 := id.SetAttribute(pn, "tag", "foo1")
br1Time := id.lastTime()
t.Logf("set attribute %q", br1)
br2 := id.SetAttribute(pn, "tag", "foo2")
br2Time := id.lastTime()
t.Logf("set attribute %q", br2)
rootClaim := id.SetAttribute(pn, "camliRoot", "rootval")
rootClaimTime := id.lastTime()
t.Logf("set attribute %q", rootClaim)
pnChild := id.NewPermanode()
br3 := id.SetAttribute(pnChild, "tag", "bar")
br3Time := id.lastTime()
t.Logf("set attribute %q", br3)
memberRef := id.AddAttribute(pn, "camliMember", pnChild.String())
t.Logf("add-attribute claim %q points to member permanode %q", memberRef, pnChild)
memberRefTime := id.lastTime()
// TODO(bradfitz): add EXIF tests here, once that stuff is ready.
if false {
camliRootPath, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
t.Fatal("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
}
for i := 1; i <= 8; i++ {
fileBase := fmt.Sprintf("f%d-exif.jpg", i)
fileName := filepath.Join(camliRootPath, "pkg", "images", "testdata", fileBase)
contents, err := ioutil.ReadFile(fileName)
if err != nil {
t.Fatal(err)
}
id.UploadFile(fileBase, string(contents))
}
}
// Upload a basic image.
var jpegFileRef *blobref.BlobRef
{
camliRootPath, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
t.Fatal("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
}
fileName := filepath.Join(camliRootPath, "pkg", "index", "indextest", "testdata", "dude.jpg")
contents, err := ioutil.ReadFile(fileName)
if err != nil {
t.Fatal(err)
}
jpegFileRef, _ = id.UploadFile("dude.jpg", string(contents))
}
lastPermanodeMutation := id.lastTime()
id.dumpIndex(t)
key := "signerkeyid:sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007"
if g, e := id.Get(key), "2931A67C26F5ABDA"; g != e {
t.Fatalf("%q = %q, want %q", key, g, e)
}
key = "imagesize|" + jpegFileRef.String()
if g, e := id.Get(key), "50|100"; g != e {
t.Errorf("JPEG dude.jpg key %q = %q; want %q", key, g, e)
}
key = "have:" + pn.String()
pnSizeStr := id.Get(key)
if pnSizeStr == "" {
t.Fatalf("missing key %q", key)
}
key = "meta:" + pn.String()
if g, e := id.Get(key), pnSizeStr+"|application/json; camliType=permanode"; g != e {
t.Errorf("key %q = %q, want %q", key, g, e)
}
key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:62.999876543Z|" + br1.String()
if g, e := id.Get(key), pn.String(); g != e {
t.Fatalf("%q = %q, want %q (permanode)", key, g, e)
}
key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:61.999876543Z|" + br2.String()
if g, e := id.Get(key), pn.String(); g != e {
t.Fatalf("%q = %q, want %q (permanode)", key, g, e)
}
key = fmt.Sprintf("edgeback|%s|%s|%s", pnChild, pn, memberRef)
if g, e := id.Get(key), "permanode|"; g != e {
t.Fatalf("edgeback row %q = %q, want %q", key, g, e)
}
// PermanodeOfSignerAttrValue
{
gotPN, err := id.Index.PermanodeOfSignerAttrValue(id.SignerBlobRef, "camliRoot", "rootval")
if err != nil {
t.Fatalf("id.Index.PermanodeOfSignerAttrValue = %v", err)
}
if gotPN.String() != pn.String() {
t.Errorf("id.Index.PermanodeOfSignerAttrValue = %q, want %q", gotPN, pn)
}
_, err = id.Index.PermanodeOfSignerAttrValue(id.SignerBlobRef, "camliRoot", "MISSING")
if err == nil {
t.Errorf("expected an error from PermanodeOfSignerAttrValue on missing value")
}
}
// SearchPermanodesWithAttr - match attr type "tag" and value "foo1"
{
ch := make(chan *blobref.BlobRef, 10)
req := &search.PermanodeByAttrRequest{
Signer: id.SignerBlobRef,
Attribute: "tag",
Query: "foo1"}
err := id.Index.SearchPermanodesWithAttr(ch, req)
if err != nil {
t.Fatalf("SearchPermanodesWithAttr = %v", err)
}
var got []*blobref.BlobRef
for r := range ch {
got = append(got, r)
}
want := []*blobref.BlobRef{pn}
if len(got) < 1 || got[0].String() != want[0].String() {
t.Errorf("id.Index.SearchPermanodesWithAttr gives %q, want %q", got, want)
}
}
// SearchPermanodesWithAttr - match all with attr type "tag"
{
ch := make(chan *blobref.BlobRef, 10)
req := &search.PermanodeByAttrRequest{
Signer: id.SignerBlobRef,
Attribute: "tag"}
err := id.Index.SearchPermanodesWithAttr(ch, req)
if err != nil {
t.Fatalf("SearchPermanodesWithAttr = %v", err)
}
var got []*blobref.BlobRef
for r := range ch {
got = append(got, r)
}
want := []*blobref.BlobRef{pn, pnChild}
if len(got) != len(want) {
t.Errorf("SearchPermanodesWithAttr results differ.\n got: %q\nwant: %q",
got, want)
}
for _, w := range want {
found := false
for _, g := range got {
if g.String() == w.String() {
found = true
break
}
}
if !found {
t.Errorf("SearchPermanodesWithAttr: %v was not found.\n", w)
}
}
}
// GetRecentPermanodes
{
ch := make(chan *search.Result, 10) // expect 2 results, but maybe more if buggy.
err := id.Index.GetRecentPermanodes(ch, id.SignerBlobRef, 50)
if err != nil {
t.Fatalf("GetRecentPermanodes = %v", err)
}
got := []*search.Result{}
for r := range ch {
got = append(got, r)
}
want := []*search.Result{
&search.Result{
BlobRef: pn,
Signer: id.SignerBlobRef,
LastModTime: lastPermanodeMutation.Unix(),
},
&search.Result{
BlobRef: pnChild,
Signer: id.SignerBlobRef,
LastModTime: br3Time.Unix(),
},
}
if len(got) != len(want) {
t.Errorf("GetRecentPermanode results differ.\n got: %v\nwant: %v",
search.Results(got), search.Results(want))
}
for _, w := range want {
found := false
for _, g := range got {
if reflect.DeepEqual(g, w) {
found = true
break
}
}
if !found {
t.Errorf("GetRecentPermanode: %v was not found.\n got: %v\nwant: %v",
w, search.Results(got), search.Results(want))
}
}
}
// GetBlobMimeType
{
mime, size, err := id.Index.GetBlobMimeType(pn)
if err != nil {
t.Errorf("GetBlobMimeType(%q) = %v", pn, err)
} else {
if e := "application/json; camliType=permanode"; mime != e {
t.Errorf("GetBlobMimeType(%q) mime = %q, want %q", pn, mime, e)
}
if size == 0 {
t.Errorf("GetBlobMimeType(%q) size is zero", pn)
}
}
_, _, err = id.Index.GetBlobMimeType(blobref.Parse("abc-123"))
if err != os.ErrNotExist {
t.Errorf("GetBlobMimeType(dummy blobref) = %v; want os.ErrNotExist", err)
}
}
// GetOwnerClaims
{
claims, err := id.Index.GetOwnerClaims(pn, id.SignerBlobRef)
if err != nil {
t.Errorf("GetOwnerClaims = %v", err)
} else {
want := search.ClaimList([]*search.Claim{
&search.Claim{
BlobRef: br1,
Permanode: pn,
Signer: id.SignerBlobRef,
Date: br1Time.UTC(),
Type: "set-attribute",
Attr: "tag",
Value: "foo1",
},
&search.Claim{
BlobRef: br2,
Permanode: pn,
Signer: id.SignerBlobRef,
Date: br2Time.UTC(),
Type: "set-attribute",
Attr: "tag",
Value: "foo2",
},
&search.Claim{
BlobRef: rootClaim,
Permanode: pn,
Signer: id.SignerBlobRef,
Date: rootClaimTime.UTC(),
Type: "set-attribute",
Attr: "camliRoot",
Value: "rootval",
},
&search.Claim{
BlobRef: memberRef,
Permanode: pn,
Signer: id.SignerBlobRef,
Date: memberRefTime.UTC(),
Type: "add-attribute",
Attr: "camliMember",
Value: pnChild.String(),
},
})
if !reflect.DeepEqual(claims, want) {
t.Errorf("GetOwnerClaims results differ.\n got: %v\nwant: %v",
claims, want)
}
}
}
}
func uiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) {
ui := &UIHandler{
prefix: ld.MyPrefix(),
sourceRoot: conf.OptionalString("sourceRoot", ""),
resizeSem: syncutil.NewSem(int64(conf.OptionalInt("maxResizeBytes",
constants.DefaultMaxResizeMem))),
}
cachePrefix := conf.OptionalString("cache", "")
scaledImageConf := conf.OptionalObject("scaledImage")
if err = conf.Validate(); err != nil {
return
}
scaledImageKV, err := newKVOrNil(scaledImageConf)
if err != nil {
return nil, fmt.Errorf("in UI handler's scaledImage: %v", err)
}
if scaledImageKV != nil && cachePrefix == "" {
return nil, fmt.Errorf("in UI handler, can't specify scaledImage without cache")
}
if cachePrefix != "" {
bs, err := ld.GetStorage(cachePrefix)
if err != nil {
return nil, fmt.Errorf("UI handler's cache of %q error: %v", cachePrefix, err)
}
ui.Cache = bs
ui.thumbMeta = NewThumbMeta(scaledImageKV)
}
if ui.sourceRoot == "" {
ui.sourceRoot = os.Getenv("CAMLI_DEV_CAMLI_ROOT")
if uistatic.IsAppEngine {
if _, err = os.Stat(filepath.Join(uistatic.GaeSourceRoot,
filepath.FromSlash("server/camlistored/ui/index.html"))); err != nil {
hint := fmt.Sprintf("\"sourceRoot\" was not specified in the config,"+
" and the default sourceRoot dir %v does not exist or does not contain"+
" \"server/camlistored/ui/index.html\". devcam appengine can do that for you.",
uistatic.GaeSourceRoot)
log.Print(hint)
return nil, errors.New("No sourceRoot found; UI not available.")
}
log.Printf("Using the default \"%v\" as the sourceRoot for AppEngine", uistatic.GaeSourceRoot)
ui.sourceRoot = uistatic.GaeSourceRoot
}
if ui.sourceRoot == "" && uistatic.Files.IsEmpty() {
ui.sourceRoot, err = osutil.GoPackagePath("camlistore.org")
if err != nil {
log.Printf("Warning: server not compiled with linked-in UI resources (HTML, JS, CSS), and camlistore.org not found in GOPATH.")
} else {
log.Printf("Using UI resources (HTML, JS, CSS) from disk, under %v", ui.sourceRoot)
}
}
}
if ui.sourceRoot != "" {
ui.uiDir = filepath.Join(ui.sourceRoot, filepath.FromSlash("server/camlistored/ui"))
// Ignore any fileembed files:
Files = &fileembed.Files{
DirFallback: filepath.Join(ui.sourceRoot, filepath.FromSlash("pkg/server")),
}
uistatic.Files = &fileembed.Files{
DirFallback: ui.uiDir,
Listable: true,
// In dev_appserver, allow edit-and-reload without
// restarting. In production, though, it's faster to just
// slurp it in.
SlurpToMemory: uistatic.IsProdAppEngine,
}
}
ui.closureHandler, err = ui.makeClosureHandler(ui.sourceRoot)
if err != nil {
return nil, fmt.Errorf(`Invalid "sourceRoot" value of %q: %v"`, ui.sourceRoot, err)
}
if ui.sourceRoot != "" {
ui.fileReactHandler, err = makeFileServer(ui.sourceRoot, filepath.Join(vendorEmbed, "react"), "react.js")
if err != nil {
return nil, fmt.Errorf("Could not make react handler: %s", err)
}
ui.fileGlitchHandler, err = makeFileServer(ui.sourceRoot, filepath.Join(vendorEmbed, "glitch"), "npc_piggy__x1_walk_png_1354829432.png")
if err != nil {
return nil, fmt.Errorf("Could not make glitch handler: %s", err)
}
ui.fileFontawesomeHandler, err = makeFileServer(ui.sourceRoot, filepath.Join(vendorEmbed, "fontawesome"), "css/font-awesome.css")
if err != nil {
return nil, fmt.Errorf("Could not make fontawesome handler: %s", err)
}
ui.fileLessHandler, err = makeFileServer(ui.sourceRoot, filepath.Join(vendorEmbed, "less"), "less.js")
if err != nil {
return nil, fmt.Errorf("Could not make less handler: %s", err)
}
}
rootPrefix, _, err := ld.FindHandlerByType("root")
if err != nil {
return nil, errors.New("No root handler configured, which is necessary for the ui handler")
}
if h, err := ld.GetHandler(rootPrefix); err == nil {
ui.root = h.(*RootHandler)
ui.root.registerUIHandler(ui)
} else {
return nil, errors.New("failed to find the 'root' handler")
}
return ui, nil
}
}
}
}`),
},
// Test recent permanode of a file
{
name: "recent-file",
setup: func(*test.FakeIndex) index.Interface {
// Ignore the fakeindex and use the real (but in-memory) implementation,
// using IndexDeps to populate it.
idx := index.NewMemoryIndex()
id := indextest.NewIndexDeps(idx)
// Upload a basic image
camliRootPath, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
panic("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
}
uploadFile := func(file string, modTime time.Time) blob.Ref {
fileName := filepath.Join(camliRootPath, "pkg", "index", "indextest", "testdata", file)
contents, err := ioutil.ReadFile(fileName)
if err != nil {
panic(err)
}
br, _ := id.UploadFile(file, string(contents), modTime)
return br
}
dudeFileRef := uploadFile("dude.jpg", time.Time{})
pn := id.NewPlannedPermanode("pn1")
func TestInstallHandlers(t *testing.T) {
camroot, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
t.Fatalf("failed to find camlistore.org GOPATH root: %v", err)
}
conf := serverinit.DefaultBaseConfig
conf.Identity = "26F5ABDA"
conf.IdentitySecretRing = filepath.Join(camroot, filepath.FromSlash("pkg/jsonsign/testdata/test-secring.gpg"))
conf.MemoryStorage = true
conf.MemoryIndex = true
confData, err := json.MarshalIndent(conf, "", " ")
if err != nil {
t.Fatalf("Could not json encode config: %v", err)
}
// Setting CAMLI_CONFIG_DIR to avoid triggering failInTests in osutil.CamliConfigDir
defer os.Setenv("CAMLI_CONFIG_DIR", os.Getenv("CAMLI_CONFIG_DIR")) // restore after test
os.Setenv("CAMLI_CONFIG_DIR", "whatever")
lowConf, err := serverinit.Load(confData)
if err != nil {
t.Fatal(err)
}
// because these two are normally consumed in camlistored.go
// TODO(mpl): serverinit.Load should consume these 2 as well. Once
// consumed, we should keep all the answers as private fields, and then we
// put accessors on serverinit.Config. Maybe we even stop embedding
// jsonconfig.Obj in serverinit.Config too, so none of those methods are
// accessible.
lowConf.OptionalBool("https", true)
lowConf.OptionalString("listen", "")
reindex := false
var context *http.Request // only used by App Engine. See handlerLoader in serverinit.go
hi := http.NewServeMux()
address := "http://" + conf.Listen
_, err = lowConf.InstallHandlers(hi, address, reindex, context)
if err != nil {
t.Fatal(err)
}
tests := []struct {
prefix string
authWrapped bool
prefixWrapped bool
handlerType reflect.Type
}{
{
prefix: "/",
handlerType: reflect.TypeOf(&server.RootHandler{}),
prefixWrapped: true,
},
{
prefix: "/sync/",
handlerType: reflect.TypeOf(&server.SyncHandler{}),
prefixWrapped: true,
authWrapped: true,
},
{
prefix: "/my-search/",
handlerType: reflect.TypeOf(&search.Handler{}),
prefixWrapped: true,
authWrapped: true,
},
{
prefix: "/ui/",
handlerType: reflect.TypeOf(&server.UIHandler{}),
prefixWrapped: true,
authWrapped: true,
},
{
prefix: "/importer/",
handlerType: reflect.TypeOf(&importer.Host{}),
prefixWrapped: true,
authWrapped: true,
},
{
prefix: "/sighelper/",
handlerType: reflect.TypeOf(&signhandler.Handler{}),
prefixWrapped: true,
authWrapped: true,
},
{
prefix: "/status/",
handlerType: reflect.TypeOf(&server.StatusHandler{}),
prefixWrapped: true,
authWrapped: true,
},
{
prefix: "/help/",
handlerType: reflect.TypeOf(&server.HelpHandler{}),
prefixWrapped: true,
authWrapped: true,
},
{
prefix: "/setup/",
handlerType: reflect.TypeOf(&server.SetupHandler{}),
prefixWrapped: true,
},
{
prefix: "/bs/camli/",
handlerType: reflect.TypeOf(http.HandlerFunc(nil)),
},
{
prefix: "/index/camli/",
handlerType: reflect.TypeOf(http.HandlerFunc(nil)),
},
{
prefix: "/bs-and-index/camli/",
handlerType: reflect.TypeOf(http.HandlerFunc(nil)),
},
{
prefix: "/bs-and-maybe-also-index/camli/",
handlerType: reflect.TypeOf(http.HandlerFunc(nil)),
},
{
prefix: "/cache/camli/",
handlerType: reflect.TypeOf(http.HandlerFunc(nil)),
},
}
for _, v := range tests {
req, err := http.NewRequest("GET", address+v.prefix, nil)
if err != nil {
t.Error(err)
continue
}
h, _ := hi.Handler(req)
if v.authWrapped {
ah, ok := h.(auth.Handler)
if !ok {
t.Errorf("handler for %v should be auth wrapped", v.prefix)
continue
}
h = ah.Handler
}
if v.prefixWrapped {
ph, ok := h.(*httputil.PrefixHandler)
if !ok {
t.Errorf("handler for %v should be prefix wrapped", v.prefix)
continue
}
h = ph.Handler
}
if reflect.TypeOf(h) != v.handlerType {
t.Errorf("for %v: want %v, got %v", v.prefix, v.handlerType, reflect.TypeOf(h))
}
}
}
func newUIFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) {
ui := &UIHandler{
prefix: ld.MyPrefix(),
JSONSignRoot: conf.OptionalString("jsonSignRoot", ""),
}
pubRoots := conf.OptionalList("publishRoots")
cachePrefix := conf.OptionalString("cache", "")
scType := conf.OptionalString("scaledImage", "")
if err = conf.Validate(); err != nil {
return
}
if ui.JSONSignRoot != "" {
h, _ := ld.GetHandler(ui.JSONSignRoot)
if sigh, ok := h.(*signhandler.Handler); ok {
ui.sigh = sigh
}
}
ui.PublishRoots = make(map[string]*PublishHandler)
for _, pubRoot := range pubRoots {
h, err := ld.GetHandler(pubRoot)
if err != nil {
return nil, fmt.Errorf("UI handler's publishRoots references invalid %q", pubRoot)
}
pubh, ok := h.(*PublishHandler)
if !ok {
return nil, fmt.Errorf("UI handler's publishRoots references invalid %q; not a PublishHandler", pubRoot)
}
ui.PublishRoots[pubRoot] = pubh
}
checkType := func(key string, htype string) {
v := conf.OptionalString(key, "")
if v == "" {
return
}
ct := ld.GetHandlerType(v)
if ct == "" {
err = fmt.Errorf("UI handler's %q references non-existant %q", key, v)
} else if ct != htype {
err = fmt.Errorf("UI handler's %q references %q of type %q; expected type %q", key, v, ct, htype)
}
}
checkType("searchRoot", "search")
checkType("jsonSignRoot", "jsonsign")
if err != nil {
return
}
if cachePrefix != "" {
bs, err := ld.GetStorage(cachePrefix)
if err != nil {
return nil, fmt.Errorf("UI handler's cache of %q error: %v", cachePrefix, err)
}
ui.Cache = bs
switch scType {
case "lrucache":
ui.sc = NewScaledImageLru()
default:
return nil, fmt.Errorf("unsupported ui handler's scType: %q ", scType)
}
}
camliRootPath, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
log.Printf("Package camlistore.org not found in $GOPATH (or $GOPATH not defined)." +
" Closure will not be used.")
} else {
closureDir := filepath.Join(camliRootPath, "tmp", "closure-lib", "closure")
ui.closureHandler = http.FileServer(http.Dir(closureDir))
}
rootPrefix, _, err := ld.FindHandlerByType("root")
if err != nil {
return nil, errors.New("No root handler configured, which is necessary for the ui handler")
}
if h, err := ld.GetHandler(rootPrefix); err == nil {
ui.root = h.(*RootHandler)
ui.root.registerUIHandler(ui)
} else {
return nil, errors.New("failed to find the 'root' handler")
}
return ui, nil
}
func Index(t *testing.T, initIdx func() *index.Index) {
oldLocal := time.Local
time.Local = time.UTC
defer func() { time.Local = oldLocal }()
id := NewIndexDeps(initIdx())
id.Fataler = t
defer id.DumpIndex(t)
pn := id.NewPermanode()
t.Logf("uploaded permanode %q", pn)
br1 := id.SetAttribute(pn, "tag", "foo1")
br1Time := id.LastTime()
t.Logf("set attribute %q", br1)
br2 := id.SetAttribute(pn, "tag", "foo2")
br2Time := id.LastTime()
t.Logf("set attribute %q", br2)
rootClaim := id.SetAttribute(pn, "camliRoot", "rootval")
rootClaimTime := id.LastTime()
t.Logf("set attribute %q", rootClaim)
pnChild := id.NewPermanode()
br3 := id.SetAttribute(pnChild, "tag", "bar")
br3Time := id.LastTime()
t.Logf("set attribute %q", br3)
memberRef := id.AddAttribute(pn, "camliMember", pnChild.String())
t.Logf("add-attribute claim %q points to member permanode %q", memberRef, pnChild)
memberRefTime := id.LastTime()
// TODO(bradfitz): add EXIF tests here, once that stuff is ready.
if false {
camliRootPath, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
t.Fatal("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
}
for i := 1; i <= 8; i++ {
fileBase := fmt.Sprintf("f%d-exif.jpg", i)
fileName := filepath.Join(camliRootPath, "pkg", "images", "testdata", fileBase)
contents, err := ioutil.ReadFile(fileName)
if err != nil {
t.Fatal(err)
}
id.UploadFile(fileBase, string(contents), noTime)
}
}
// Upload some files.
var jpegFileRef, exifFileRef, mediaFileRef, mediaWholeRef blob.Ref
{
camliRootPath, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
t.Fatal("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
}
uploadFile := func(file string, modTime time.Time) (fileRef, wholeRef blob.Ref) {
fileName := filepath.Join(camliRootPath, "pkg", "index", "indextest", "testdata", file)
contents, err := ioutil.ReadFile(fileName)
if err != nil {
t.Fatal(err)
}
fileRef, wholeRef = id.UploadFile(file, string(contents), modTime)
return
}
jpegFileRef, _ = uploadFile("dude.jpg", noTime)
exifFileRef, _ = uploadFile("dude-exif.jpg", time.Unix(1361248796, 0))
mediaFileRef, mediaWholeRef = uploadFile("0s.mp3", noTime)
}
// Upload the dir containing the previous files.
imagesDirRef := id.UploadDir(
"testdata",
[]blob.Ref{jpegFileRef, exifFileRef, mediaFileRef},
time.Now(),
)
lastPermanodeMutation := id.LastTime()
key := "signerkeyid:sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007"
if g, e := id.Get(key), "2931A67C26F5ABDA"; g != e {
t.Fatalf("%q = %q, want %q", key, g, e)
}
key = "imagesize|" + jpegFileRef.String()
if g, e := id.Get(key), "50|100"; g != e {
t.Errorf("JPEG dude.jpg key %q = %q; want %q", key, g, e)
}
key = "filetimes|" + jpegFileRef.String()
if g, e := id.Get(key), ""; g != e {
t.Errorf("JPEG dude.jpg key %q = %q; want %q", key, g, e)
}
key = "filetimes|" + exifFileRef.String()
if g, e := id.Get(key), "2013-02-18T01%3A11%3A20Z%2C2013-02-19T04%3A39%3A56Z"; g != e {
t.Errorf("EXIF dude-exif.jpg key %q = %q; want %q", key, g, e)
}
key = "have:" + pn.String()
pnSizeStr := strings.TrimSuffix(id.Get(key), "|indexed")
if pnSizeStr == "" {
t.Fatalf("missing key %q", key)
}
key = "meta:" + pn.String()
if g, e := id.Get(key), pnSizeStr+"|application/json; camliType=permanode"; g != e {
t.Errorf("key %q = %q, want %q", key, g, e)
}
key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:62.999876543Z|" + br1.String()
if g, e := id.Get(key), pn.String(); g != e {
t.Fatalf("%q = %q, want %q (permanode)", key, g, e)
}
key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:61.999876543Z|" + br2.String()
if g, e := id.Get(key), pn.String(); g != e {
t.Fatalf("%q = %q, want %q (permanode)", key, g, e)
}
key = fmt.Sprintf("edgeback|%s|%s|%s", pnChild, pn, memberRef)
if g, e := id.Get(key), "permanode|"; g != e {
t.Fatalf("edgeback row %q = %q, want %q", key, g, e)
}
mediaTests := []struct {
prop, exp string
}{
{"title", "Zero Seconds"},
{"artist", "Test Artist"},
{"album", "Test Album"},
{"genre", "(20)Alternative"},
{"musicbrainzalbumid", "00000000-0000-0000-0000-000000000000"},
{"year", "1992"},
{"track", "1"},
{"disc", "2"},
{"mediaref", "sha1-fefac74a1d5928316d7131747107c8a61b71ffe4"},
{"durationms", "26"},
}
for _, tt := range mediaTests {
key = fmt.Sprintf("mediatag|%s|%s", mediaWholeRef.String(), tt.prop)
if g, _ := url.QueryUnescape(id.Get(key)); g != tt.exp {
t.Errorf("0s.mp3 key %q = %q; want %q", key, g, tt.exp)
}
}
// PermanodeOfSignerAttrValue
{
gotPN, err := id.Index.PermanodeOfSignerAttrValue(id.SignerBlobRef, "camliRoot", "rootval")
if err != nil {
t.Fatalf("id.Index.PermanodeOfSignerAttrValue = %v", err)
}
if gotPN.String() != pn.String() {
t.Errorf("id.Index.PermanodeOfSignerAttrValue = %q, want %q", gotPN, pn)
}
_, err = id.Index.PermanodeOfSignerAttrValue(id.SignerBlobRef, "camliRoot", "MISSING")
if err == nil {
t.Errorf("expected an error from PermanodeOfSignerAttrValue on missing value")
}
}
// SearchPermanodesWithAttr - match attr type "tag" and value "foo1"
{
ch := make(chan blob.Ref, 10)
req := &camtypes.PermanodeByAttrRequest{
Signer: id.SignerBlobRef,
Attribute: "tag",
Query: "foo1",
}
err := id.Index.SearchPermanodesWithAttr(ch, req)
if err != nil {
t.Fatalf("SearchPermanodesWithAttr = %v", err)
}
var got []blob.Ref
for r := range ch {
got = append(got, r)
}
want := []blob.Ref{pn}
if len(got) < 1 || got[0].String() != want[0].String() {
t.Errorf("id.Index.SearchPermanodesWithAttr gives %q, want %q", got, want)
}
}
// SearchPermanodesWithAttr - match all with attr type "tag"
{
ch := make(chan blob.Ref, 10)
req := &camtypes.PermanodeByAttrRequest{
Signer: id.SignerBlobRef,
Attribute: "tag",
}
err := id.Index.SearchPermanodesWithAttr(ch, req)
if err != nil {
t.Fatalf("SearchPermanodesWithAttr = %v", err)
}
var got []blob.Ref
for r := range ch {
got = append(got, r)
}
want := []blob.Ref{pn, pnChild}
if len(got) != len(want) {
t.Errorf("SearchPermanodesWithAttr results differ.\n got: %q\nwant: %q",
got, want)
}
for _, w := range want {
found := false
for _, g := range got {
if g.String() == w.String() {
found = true
break
}
}
if !found {
t.Errorf("SearchPermanodesWithAttr: %v was not found.\n", w)
}
}
}
// Delete value "pony" of type "title" (which does not actually exist) for pn
br4 := id.DelAttribute(pn, "title", "pony")
br4Time := id.LastTime()
// and verify it is not found when searching by attr
{
ch := make(chan blob.Ref, 10)
req := &camtypes.PermanodeByAttrRequest{
Signer: id.SignerBlobRef,
Attribute: "title",
Query: "pony",
}
err := id.Index.SearchPermanodesWithAttr(ch, req)
if err != nil {
t.Fatalf("SearchPermanodesWithAttr = %v", err)
}
var got []blob.Ref
for r := range ch {
got = append(got, r)
}
want := []blob.Ref{}
if len(got) != len(want) {
t.Errorf("SearchPermanodesWithAttr results differ.\n got: %q\nwant: %q",
got, want)
}
}
// GetRecentPermanodes
{
verify := func(prefix string, want []camtypes.RecentPermanode, before time.Time) {
ch := make(chan camtypes.RecentPermanode, 10) // expect 2 results, but maybe more if buggy.
err := id.Index.GetRecentPermanodes(ch, id.SignerBlobRef, 50, before)
if err != nil {
t.Fatalf("[%s] GetRecentPermanodes = %v", prefix, err)
}
got := []camtypes.RecentPermanode{}
for r := range ch {
got = append(got, r)
}
if len(got) != len(want) {
t.Errorf("[%s] GetRecentPermanode results differ.\n got: %v\nwant: %v",
prefix, searchResults(got), searchResults(want))
}
for _, w := range want {
found := false
for _, g := range got {
if g.Equal(w) {
found = true
break
}
}
if !found {
t.Errorf("[%s] GetRecentPermanode: %v was not found.\n got: %v\nwant: %v",
prefix, w, searchResults(got), searchResults(want))
}
}
}
want := []camtypes.RecentPermanode{
{
Permanode: pn,
Signer: id.SignerBlobRef,
LastModTime: br4Time,
},
{
Permanode: pnChild,
Signer: id.SignerBlobRef,
LastModTime: br3Time,
},
}
before := time.Time{}
verify("Zero before", want, before)
before = lastPermanodeMutation
t.Log("lastPermanodeMutation", lastPermanodeMutation,
lastPermanodeMutation.Unix())
verify("Non-zero before", want[1:], before)
}
// GetDirMembers
{
ch := make(chan blob.Ref, 10) // expect 2 results
err := id.Index.GetDirMembers(imagesDirRef, ch, 50)
if err != nil {
t.Fatalf("GetDirMembers = %v", err)
}
got := []blob.Ref{}
for r := range ch {
got = append(got, r)
}
want := []blob.Ref{jpegFileRef, exifFileRef, mediaFileRef}
if len(got) != len(want) {
t.Errorf("GetDirMembers results differ.\n got: %v\nwant: %v",
got, want)
}
for _, w := range want {
found := false
for _, g := range got {
if w == g {
found = true
break
}
}
if !found {
t.Errorf("GetDirMembers: %v was not found.", w)
}
}
}
// GetBlobMeta
{
meta, err := id.Index.GetBlobMeta(pn)
if err != nil {
t.Errorf("GetBlobMeta(%q) = %v", pn, err)
} else {
if e := "permanode"; meta.CamliType != e {
t.Errorf("GetBlobMeta(%q) mime = %q, want %q", pn, meta.CamliType, e)
}
if meta.Size == 0 {
t.Errorf("GetBlobMeta(%q) size is zero", pn)
}
}
_, err = id.Index.GetBlobMeta(blob.ParseOrZero("abc-123"))
if err != os.ErrNotExist {
t.Errorf("GetBlobMeta(dummy blobref) = %v; want os.ErrNotExist", err)
}
}
// AppendClaims
{
claims, err := id.Index.AppendClaims(nil, pn, id.SignerBlobRef, "")
if err != nil {
t.Errorf("AppendClaims = %v", err)
} else {
want := []camtypes.Claim{
{
BlobRef: br1,
Permanode: pn,
Signer: id.SignerBlobRef,
Date: br1Time.UTC(),
Type: "set-attribute",
Attr: "tag",
Value: "foo1",
},
{
BlobRef: br2,
Permanode: pn,
Signer: id.SignerBlobRef,
Date: br2Time.UTC(),
Type: "set-attribute",
Attr: "tag",
Value: "foo2",
},
{
BlobRef: rootClaim,
Permanode: pn,
Signer: id.SignerBlobRef,
Date: rootClaimTime.UTC(),
Type: "set-attribute",
Attr: "camliRoot",
Value: "rootval",
},
{
BlobRef: memberRef,
Permanode: pn,
Signer: id.SignerBlobRef,
Date: memberRefTime.UTC(),
Type: "add-attribute",
Attr: "camliMember",
Value: pnChild.String(),
},
{
BlobRef: br4,
Permanode: pn,
Signer: id.SignerBlobRef,
Date: br4Time.UTC(),
Type: "del-attribute",
Attr: "title",
Value: "pony",
},
}
if !reflect.DeepEqual(claims, want) {
t.Errorf("AppendClaims results differ.\n got: %v\nwant: %v",
claims, want)
}
}
}
}
