// Test that `camget -o' can restore a fifo correctly.
func TestCamgetFIFO(t *testing.T) {
if runtime.GOOS == "windows" {
t.SkipNow()
}
fifo, cleanup := mkTmpFIFO(t)
defer cleanup()
// Upload the fifo
w := test.GetWorld(t)
out := test.MustRunCmd(t, w.Cmd("camput", "file", fifo))
br := strings.Split(out, "\n")[0]
// Try and get it back
tdir, err := ioutil.TempDir("", "fifo-test-")
if err != nil {
t.Fatalf("ioutil.TempDir(): %v", err)
}
defer os.RemoveAll(tdir)
test.MustRunCmd(t, w.Cmd("camget", "-o", tdir, br))
// Ensure it is actually a fifo
name := filepath.Join(tdir, filepath.Base(fifo))
fi, err := os.Lstat(name)
if err != nil {
t.Fatalf("os.Lstat(): %v", err)
}
if mask := fi.Mode() & os.ModeNamedPipe; mask == 0 {
t.Fatalf("Retrieved file %s: Not a FIFO", name)
}
}
// Test that camput twice on the same file only uploads once.
func TestCamputUploadOnce(t *testing.T) {
w := test.GetWorld(t)
camputCmd := func() *exec.Cmd {
// Use --contents_only because if test is run from devcam,
// server-config.json is going to be the one from within the fake gopath,
// hence with a different cTime and with a different blobRef everytime.
// Also, CAMLI_DEBUG is needed for --contents_only flag.
return w.CmdWithEnv("camput", append(os.Environ(), "CAMLI_DEBUG=1"), "file", "--contents_only=true", filepath.FromSlash("../testdata/server-config.json"))
}
wantBlobRef := "sha1-46d4023ef523d6a19e45183ae9dab575a496f51f"
cmd := camputCmd()
out := test.MustRunCmd(t, cmd)
out = strings.TrimSpace(out)
if out != wantBlobRef {
t.Fatalf("wrong camput output; wanted %v, got %v", wantBlobRef, out)
}
cmd = camputCmd()
var stderr bytes.Buffer
cmd.Stderr = &stderr
output, err := cmd.Output()
if err != nil {
t.Fatalf("second camput failed: %v, stdout: %v, stderr: %v", err, output, stderr.String())
}
out = strings.TrimSpace(string(output))
if out != wantBlobRef {
t.Fatalf("wrong 2nd camput output; wanted %v, got %v", wantBlobRef, out)
}
wantStats := `[uploadRequests=[blobs=0 bytes=0] uploads=[blobs=0 bytes=0]]`
if !strings.Contains(stderr.String(), wantStats) {
t.Fatalf("Wrong stats for 2nd camput upload; wanted %v, got %v", wantStats, out)
}
}
func TestWebsocketQuery(t *testing.T) {
w := test.GetWorld(t)
pn := w.NewPermanode(t)
test.MustRunCmd(t, w.Cmd("camput", "attr", pn.String(), "tag", "foo"))
check := func(err error) {
if err != nil {
t.Fatal(err)
}
}
const bufSize = 1 << 20
c, err := net.Dial("tcp", w.Addr())
if err != nil {
t.Fatalf("Dial: %v", err)
}
defer c.Close()
wc, _, err := websocket.NewClient(c, &url.URL{Host: w.Addr(), Path: w.SearchHandlerPath() + "ws"}, nil, bufSize, bufSize)
check(err)
msg, err := wc.NextWriter(websocket.TextMessage)
check(err)
_, err = msg.Write([]byte(`{"tag": "foo", "query": { "expression": "tag:foo" }}`))
check(err)
check(msg.Close())
errc := make(chan error, 1)
go func() {
inType, inMsg, err := wc.ReadMessage()
if err != nil {
errc <- err
return
}
if !strings.HasPrefix(string(inMsg), `{"tag":"_status"`) {
errc <- fmt.Errorf("unexpected message type=%d msg=%q, wanted status update", inType, inMsg)
return
}
inType, inMsg, err = wc.ReadMessage()
if err != nil {
errc <- err
return
}
if strings.Contains(string(inMsg), pn.String()) {
errc <- nil
return
}
errc <- fmt.Errorf("unexpected message type=%d msg=%q", inType, inMsg)
}()
select {
case err := <-errc:
if err != nil {
t.Error(err)
}
case <-time.After(5 * time.Second):
t.Error("timeout")
}
}
func cammountTest(t *testing.T, fn func(env *mountEnv)) {
dupLog := io.MultiWriter(os.Stderr, testLog{t})
log.SetOutput(dupLog)
defer log.SetOutput(os.Stderr)
w := test.GetWorld(t)
mountPoint, err := ioutil.TempDir("", "fs-test-mount")
if err != nil {
t.Fatal(err)
}
verbose := "false"
var stderrDest io.Writer = ioutil.Discard
if v, _ := strconv.ParseBool(os.Getenv("VERBOSE_FUSE")); v {
verbose = "true"
stderrDest = testLog{t}
}
if v, _ := strconv.ParseBool(os.Getenv("VERBOSE_FUSE_STDERR")); v {
stderrDest = io.MultiWriter(stderrDest, os.Stderr)
}
mount := w.Cmd("cammount", "--debug="+verbose, mountPoint)
mount.Stderr = stderrDest
mount.Env = append(mount.Env, "CAMLI_TRACK_FS_STATS=1")
stdin, err := mount.StdinPipe()
if err != nil {
t.Fatal(err)
}
if err := mount.Start(); err != nil {
t.Fatal(err)
}
waitc := make(chan error, 1)
go func() { waitc <- mount.Wait() }()
defer func() {
log.Printf("Sending quit")
stdin.Write([]byte("q\n"))
select {
case <-time.After(5 * time.Second):
log.Printf("timeout waiting for cammount to finish")
mount.Process.Kill()
Unmount(mountPoint)
case err := <-waitc:
log.Printf("cammount exited: %v", err)
}
if !test.WaitFor(not(dirToBeFUSE(mountPoint)), 5*time.Second, 1*time.Second) {
// It didn't unmount. Try again.
Unmount(mountPoint)
}
}()
if !test.WaitFor(dirToBeFUSE(mountPoint), 5*time.Second, 100*time.Millisecond) {
t.Fatalf("error waiting for %s to be mounted", mountPoint)
}
fn(&mountEnv{
t: t,
mountPoint: mountPoint,
process: mount.Process,
})
}
// Test that we can camput and camget a file whose name is not utf8,
// that we don't panic in the process and that the results are
// correct.
func TestNonUTF8FileName(t *testing.T) {
srcDir, cleanup := tempDir(t)
defer cleanup()
base, err := hex.DecodeString(nonUTF8)
if err != nil {
t.Fatalf("hex.DecodeString(): %v", err)
}
fd, err := os.Create(filepath.Join(srcDir, string(base)))
if err != nil {
t.Fatalf("os.Create(): %v", err)
}
fd.Close()
w := test.GetWorld(t)
out := test.MustRunCmd(t, w.Cmd("camput", "file", fd.Name()))
br := strings.Split(out, "\n")[0]
// camput was a success. Can we get the file back in another directory?
dstDir, cleanup := tempDir(t)
defer cleanup()
_ = test.MustRunCmd(t, w.Cmd("camget", "-o", dstDir, br))
_, err = os.Lstat(filepath.Join(dstDir, string(base)))
if err != nil {
t.Fatalf("Failed to stat file %s in directory %s",
fd.Name(), dstDir)
}
}
// ImporterTest sets up the environment for an importer integration test.
func ImporterTest(t *testing.T, importerName string, transport http.RoundTripper, fn func(*importer.RunContext)) {
const importerPrefix = "/importer/"
w := test.GetWorld(t)
defer w.Stop()
baseURL := w.ServerBaseURL()
// TODO(mpl): add a utility in integration package to provide a client that
// just works with World.
cl, err := setupClient(w)
if err != nil {
t.Fatal(err)
}
signer, err := cl.Signer()
if err != nil {
t.Fatal(err)
}
clientId := map[string]string{
importerName: "fakeStaticClientId",
}
clientSecret := map[string]string{
importerName: "fakeStaticClientSecret",
}
httpClient := &http.Client{
Transport: transport,
}
hc := importer.HostConfig{
BaseURL: baseURL,
Prefix: importerPrefix,
Target: cl,
BlobSource: cl,
Signer: signer,
Search: cl,
ClientId: clientId,
ClientSecret: clientSecret,
HTTPClient: httpClient,
}
host, err := importer.NewHost(hc)
if err != nil {
t.Fatal(err)
}
rc, err := importer.CreateAccount(host, importerName)
if err != nil {
t.Fatal(err)
}
fn(rc)
}
// Test that:
// 1) `camget -contents' can restore a regular file correctly.
// 2) if the file already exists, and has the same size as the one held by the server,
// stop early and do not even fetch it from the server.
func TestCamgetFile(t *testing.T) {
dirName, err := ioutil.TempDir("", "camli-TestCamgetFile")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dirName)
f, err := os.Create(filepath.Join(dirName, "test.txt"))
if err != nil {
t.Fatal(err)
}
filename := f.Name()
contents := "not empty anymore"
if _, err := f.Write([]byte(contents)); err != nil {
t.Fatal(err)
}
if err := f.Close(); err != nil {
t.Fatal(err)
}
outDir := filepath.Join(dirName, "fetched")
if err := os.Mkdir(outDir, 0700); err != nil {
t.Fatal(err)
}
w := test.GetWorld(t)
out := test.MustRunCmd(t, w.Cmd("camput", "file", filename))
br := strings.Split(out, "\n")[0]
_ = test.MustRunCmd(t, w.Cmd("camget", "-o", outDir, "-contents", br))
fetchedName := filepath.Join(outDir, "test.txt")
b, err := ioutil.ReadFile(fetchedName)
if err != nil {
t.Fatal(err)
}
if string(b) != contents {
t.Fatalf("fetched file different from original file, got contents %q, wanted %q", b, contents)
}
var stderr bytes.Buffer
c := w.Cmd("camget", "-o", outDir, "-contents", "-verbose", br)
c.Stderr = &stderr
if err := c.Run(); err != nil {
t.Fatalf("running second camget: %v", err)
}
if !strings.Contains(stderr.String(), fmt.Sprintf("Skipping %s; already exists.", fetchedName)) {
t.Fatal(errors.New("Was expecting info message about local file already existing"))
}
}
// Test that `camput' can upload sockets correctly.
func TestCamputSocket(t *testing.T) {
if runtime.GOOS == "windows" {
t.SkipNow()
}
socket, cleanup := mkTmpSocket(t)
defer cleanup()
// Can we successfully upload a socket?
w := test.GetWorld(t)
out := test.MustRunCmd(t, w.Cmd("camput", "file", socket))
br := strings.Split(out, "\n")[0]
out = test.MustRunCmd(t, w.Cmd("camget", br))
t.Logf("Retrieved stored socket schema: %s", out)
}
func share(t *testing.T, file string) {
w := test.GetWorld(t)
out := test.MustRunCmd(t, w.Cmd("camput", "file", file))
fileRef := strings.Split(out, "\n")[0]
out = test.MustRunCmd(t, w.Cmd("camput", "share", "-transitive", fileRef))
shareRef := strings.Split(out, "\n")[0]
testDir, err := ioutil.TempDir("", "camli-share-test-")
if err != nil {
t.Fatalf("ioutil.TempDir(): %v", err)
}
defer os.RemoveAll(testDir)
// test that we can get it through the share
test.MustRunCmd(t, w.Cmd("camget", "-o", testDir, "-shared", fmt.Sprintf("%v/share/%v", w.ServerBaseURL(), shareRef)))
filePath := filepath.Join(testDir, filepath.Base(file))
fi, err := os.Stat(filePath)
if err != nil {
t.Fatalf("camget -shared failed to get %v: %v", file, err)
}
if fi.IsDir() {
// test that we also get the dir contents
d, err := os.Open(filePath)
if err != nil {
t.Fatal(err)
}
defer d.Close()
names, err := d.Readdirnames(-1)
if err != nil {
t.Fatal(err)
}
if len(names) == 0 {
t.Fatalf("camget did not fetch contents of directory %v", file)
}
}
// test that we're not allowed to get it directly
fileURL := fmt.Sprintf("%v/share/%v", w.ServerBaseURL(), fileRef)
_, err = test.RunCmd(w.Cmd("camget", "-shared", fileURL))
if err == nil {
t.Fatal("Was expecting error for 'camget -shared " + fileURL + "'")
}
if !strings.Contains(err.Error(), "client: got status code 401") {
t.Fatalf("'camget -shared %v': got error %v, was expecting 401", fileURL, err)
}
}
// Test that running:
// $ camput permanode
// ... creates and uploads a permanode, and that we can camget it back.
func TestCamputPermanode(t *testing.T) {
w := test.GetWorld(t)
br := w.NewPermanode(t)
out := test.MustRunCmd(t, w.Cmd("camget", br.String()))
mustHave := []string{
`{"camliVersion": 1,`,
`"camliSigner": "`,
`"camliType": "permanode",`,
`random": "`,
`,"camliSig":"`,
}
for _, str := range mustHave {
if !strings.Contains(out, str) {
t.Errorf("Expected permanode response to contain %q; it didn't. Got: %s", str, out)
}
}
}
func TestInternalHandler(t *testing.T) {
w := test.GetWorld(t)
tests := map[string]int{
"/no-http-storage/": 401,
"/no-http-handler/": 401,
"/good-status/": 200,
"/bs-and-maybe-also-index/camli": 400,
"/bs/camli/sha1-b2201302e129a4396a323cb56283cddeef11bbe8": 404,
"/no-http-storage/camli/sha1-b2201302e129a4396a323cb56283cddeef11bbe8": 401,
}
for suffix, want := range tests {
res, err := http.Get(w.ServerBaseURL() + suffix)
if err != nil {
t.Fatalf("On %s: %v", suffix, err)
}
if res.StatusCode != want {
t.Errorf("For %s: Status = %d; want %d", suffix, res.StatusCode, want)
}
res.Body.Close()
}
}
// Test that running:
// $ camput permanode
// ... creates and uploads a permanode, and that we can camget it back.
func TestCamputPermanode(t *testing.T) {
w := test.GetWorld(t)
out := test.MustRunCmd(t, w.Cmd("camput", "permanode"))
br, ok := blob.Parse(strings.TrimSpace(out))
if !ok {
t.Fatalf("Expected permanode in stdout; got %q", out)
}
out = test.MustRunCmd(t, w.Cmd("camget", br.String()))
mustHave := []string{
`{"camliVersion": 1,`,
`"camliSigner": "`,
`"camliType": "permanode",`,
`random": "`,
`,"camliSig":"`,
}
for _, str := range mustHave {
if !strings.Contains(out, str) {
t.Errorf("Expected permanode response to contain %q; it didn't. Got: %s", str, out)
}
}
}
// Test that we can camput and camget a symbolic link whose target is
// not utf8, that we do no panic in the process and that the results
// are correct.
func TestNonUTF8SymlinkTarget(t *testing.T) {
srcDir, cleanup := tempDir(t)
defer cleanup()
base, err := hex.DecodeString(nonUTF8)
if err != nil {
t.Fatalf("hex.DecodeString(): %v", err)
}
fd, err := os.Create(filepath.Join(srcDir, string(base)))
if err != nil {
t.Fatalf("os.Create(): %v", err)
}
defer fd.Close()
err = os.Symlink(string(base), filepath.Join(srcDir, "link"))
if err != nil {
t.Fatalf("os.Symlink(): %v", err)
}
w := test.GetWorld(t)
out := test.MustRunCmd(t, w.Cmd("camput", "file", filepath.Join(srcDir, "link")))
br := strings.Split(out, "\n")[0]
// See if we can camget it back correctly
dstDir, cleanup := tempDir(t)
defer cleanup()
_ = test.MustRunCmd(t, w.Cmd("camget", "-o", dstDir, br))
target, err := os.Readlink(filepath.Join(dstDir, "link"))
if err != nil {
t.Fatalf("os.Readlink(): %v", err)
}
if !bytes.Equal([]byte(target), base) {
t.Fatalf("Retrieved symlink contains points to unexpected target")
}
}
// Verify that a batch import of 3 posts works
func TestIntegrationRun(t *testing.T) {
const importerPrefix = "/importer/"
const authToken = "gina:foo"
const attrKey = "key"
const attrValue = "value"
w := test.GetWorld(t)
defer w.Stop()
baseURL := w.ServerBaseURL()
// TODO(mpl): add a utility in integration package to provide a client that
// just works with World.
cl, err := setupClient(w)
if err != nil {
t.Fatal(err)
}
signer, err := cl.Signer()
if err != nil {
t.Fatal(err)
}
clientId := map[string]string{
"pinboard": "fakeStaticClientId",
}
clientSecret := map[string]string{
"pinboard": "fakeStaticClientSecret",
}
responder := httputil.FileResponder("testdata/batchresponse.json")
transport, err := httputil.NewRegexpFakeTransport([]*httputil.Matcher{
&httputil.Matcher{`^https\://api\.pinboard\.in/v1/posts/all\?auth_token=gina:foo&format=json&results=10000&todt=\d\d\d\d.*`, responder},
})
if err != nil {
t.Fatal(err)
}
httpClient := &http.Client{
Transport: transport,
}
hc := importer.HostConfig{
BaseURL: baseURL,
Prefix: importerPrefix,
Target: cl,
BlobSource: cl,
Signer: signer,
Search: cl,
ClientId: clientId,
ClientSecret: clientSecret,
HTTPClient: httpClient,
}
host, err := importer.NewHost(hc)
if err != nil {
t.Fatal(err)
}
rc, err := importer.CreateAccount(host, "pinboard")
if err != nil {
t.Fatal(err)
}
err = rc.AccountNode().SetAttrs(attrAuthToken, authToken)
if err != nil {
t.Fatal(err)
}
testee := imp{}
if err := testee.Run(rc); err != nil {
t.Fatal(err)
}
postsNode, err := getRequiredChildPathObj(rc.RootNode(), "posts")
if err != nil {
t.Fatal(err)
}
childRefs, err := findChildRefs(postsNode)
if err != nil {
t.Fatal(err)
}
expectedPosts := map[string]string{
`https://wiki.archlinux.org/index.php/xorg#Display_size_and_DPI`: "Xorg - ArchWiki",
`http://www.harihareswara.net/sumana/2014/08/17/0`: "One Way Confidence Will Look",
`http://www.wikiart.org/en/marcus-larson/fishing-near-the-fjord-by-moonlight-1862`: "Fishing Near The Fjord By Moonlight - Marcus Larson - WikiArt.org",
}
if len(childRefs) != len(expectedPosts) {
t.Fatalf("After import, found %d child refs, want %d: %v", len(childRefs), len(expectedPosts), childRefs)
}
for _, ref := range childRefs {
childNode, err := host.ObjectFromRef(ref)
if err != nil {
t.Fatal(err)
}
foundURL := childNode.Attr(nodeattr.URL)
expectedTitle, ok := expectedPosts[foundURL]
if !ok {
t.Fatalf("Found unexpected child node %v with url %q", childNode, foundURL)
}
foundTitle := childNode.Attr(nodeattr.Title)
if foundTitle != expectedTitle {
t.Fatalf("Found unexpected child node %v with title %q when we want %q", childNode, foundTitle, expectedTitle)
}
delete(expectedPosts, foundURL)
}
if len(expectedPosts) != 0 {
t.Fatalf("The following entries were expected but not found: %#v", expectedPosts)
}
}
// Test that `camget -o' can restore a symlink correctly.
func TestCamgetSymlink(t *testing.T) {
w := test.GetWorld(t)
srcDir, err := ioutil.TempDir("", "camget-test-")
if err != nil {
t.Fatalf("ioutil.TempDir(): %v", err)
}
defer os.RemoveAll(srcDir)
targetBase := "a"
target := filepath.Join(srcDir, targetBase)
targetFD, err := os.Create(target)
if err != nil {
t.Fatalf("os.Create(): %v", err)
}
targetFD.Close()
subdirBase := "child"
subdirName := filepath.Join(srcDir, subdirBase)
linkBase := "b"
linkName := filepath.Join(subdirName, linkBase)
err = os.Mkdir(subdirName, 0777)
if err != nil {
t.Fatalf("os.Mkdir(): %v", err)
}
err = os.Symlink("../"+targetBase, linkName)
if err != nil {
t.Fatalf("os.Symlink(): %v", err)
}
out := test.MustRunCmd(t, w.Cmd("camput", "file", srcDir))
asserts.ExpectBool(t, true, out != "", "camput")
br := strings.Split(out, "\n")[0]
dstDir, err := ioutil.TempDir("", "camget-test-")
if err != nil {
t.Fatalf("ioutil.TempDir(): %v", err)
}
defer os.RemoveAll(dstDir)
// Now restore the symlink
_ = test.MustRunCmd(t, w.Cmd("camget", "-o", dstDir, br))
symlink := filepath.Join(dstDir, filepath.Base(srcDir), subdirBase,
linkBase)
link, err := os.Readlink(symlink)
if err != nil {
t.Fatalf("os.Readlink(): %v", err)
}
expected := "../a"
if expected != link {
t.Fatalf("os.Readlink(): Expected: %s, got %s", expected,
link)
}
// Ensure that the link is not broken
_, err = os.Stat(symlink)
if err != nil {
t.Fatalf("os.Stat(): %v", err)
}
}
// Run camput in the environment it runs in under the Android app.
// This matches how camput is used in UploadThread.java.
func TestAndroidCamputFile(t *testing.T) {
w := test.GetWorld(t)
// UploadThread.java sets:
// CAMLI_AUTH (set by w.CmdWithEnv)
// CAMLI_TRUSTED_CERT (not needed)
// CAMLI_CACHE_DIR
// CAMPUT_ANDROID_OUTPUT=1
cacheDir, clean := mustTempDir(t)
defer clean()
env := []string{
"CAMPUT_ANDROID_OUTPUT=1",
"CAMLI_CACHE_DIR=" + cacheDir,
}
cmd := w.CmdWithEnv("camput",
env,
"--server="+w.ServerBaseURL(),
"file",
"-stdinargs",
"-vivify")
cmd.Stderr = os.Stderr
in, err := cmd.StdinPipe()
if err != nil {
t.Fatal(err)
}
out, err := cmd.StdoutPipe()
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
defer cmd.Process.Kill()
srcDir, clean := mustTempDir(t)
defer clean()
file1 := filepath.Join(srcDir, "file1.txt")
mustWriteFile(t, file1, "contents 1")
file2 := filepath.Join(srcDir, "file2.txt")
mustWriteFile(t, file2, "contents 2 longer length")
go func() {
fmt.Fprintf(in, "%s\n", file1)
fmt.Fprintf(in, "%s\n", file2)
}()
waitc := make(chan error)
go func() {
sc := bufio.NewScanner(out)
fileUploaded := 0
for sc.Scan() {
t.Logf("Got: %q", sc.Text())
f := strings.Fields(sc.Text())
if len(f) == 0 {
t.Logf("empty text?")
continue
}
if f[0] == "FILE_UPLOADED" {
fileUploaded++
if fileUploaded == 2 {
break
}
}
}
in.Close()
if err := sc.Err(); err != nil {
t.Error(err)
}
}()
defer cmd.Process.Kill()
go func() {
waitc <- cmd.Wait()
}()
select {
case <-time.After(5 * time.Second):
t.Fatal("timeout waiting for camput to end")
case err := <-waitc:
if err != nil {
t.Errorf("camput exited uncleanly: %v", err)
}
}
}
// like TestFileSharing, but with a file large enough to have several parts,
// including some bytesRef parts.
func TestFileWithBytesSharing(t *testing.T) {
share(t, test.GetWorld(t).ServerBinary())
}