func testOpen(t *testing.T, fs rwvfs.FileSystem) {
const path = "testOpen"
var buf bytes.Buffer
for i := uint8(0); i < 255; i++ {
for j := uint8(0); j < 255; j++ {
buf.Write([]byte{i, j})
}
}
fullData := []byte(base64.StdEncoding.EncodeToString(buf.Bytes()))[10:]
fullLen := int64(len(fullData))
createFile(t, fs, path, fullData)
{
// Full reads.
f, err := fs.Open(path)
if err != nil {
t.Fatal(err)
}
b, err := ioutil.ReadAll(f)
if err != nil {
t.Fatal(err)
}
if err := f.Close(); err != nil {
t.Fatal(err)
}
if !bytes.Equal(b, fullData) {
t.Errorf("full read: got %q, want %q", b, fullData)
}
}
{
// Partial reads.
rrt := &rangeRecordingTransport{}
fs.(*S3FS).config.Client = &http.Client{Transport: rrt}
var f vfs.ReadSeekCloser
cases := [][2]int64{
{0, 0},
{0, 1},
{0, 2},
{1, 1},
{0, 3},
{1, 3},
{2, 3},
{0, 2},
{0, 3},
{3, 4},
{0, fullLen / 2},
{1, fullLen / 2},
{fullLen / 3, fullLen / 2},
{0, fullLen - 1},
{1, fullLen - 1},
{fullLen / 2, fullLen/2 + 1333},
{fullLen / 2, fullLen/2 + 1},
{fullLen / 2, fullLen/2 + 2},
{fullLen / 2, fullLen / 2},
{fullLen - 10, fullLen - 1},
}
for _, autofetch := range []bool{false, true} {
for _, reuse := range []bool{false, true} {
for i, c := range cases {
if !reuse || i == 0 {
var err error
f, err = fs.(rwvfs.FetcherOpener).OpenFetcher(path)
if err != nil {
t.Fatal(err)
}
}
f.(interface {
SetAutofetch(bool)
}).SetAutofetch(true)
rrt.reset()
start, end := c[0], c[1]
label := fmt.Sprintf("range %d-%d (autofetch=%v, reuse=%v)", start, end, autofetch, reuse)
fetchEnd := end
if autofetch {
// Short fetch.
fetchEnd = (start + end) / 2
if fetchEnd < start {
fetchEnd = end
}
}
if err := f.(rwvfs.Fetcher).Fetch(start, fetchEnd); err != nil {
t.Error(err)
continue
}
n, err := f.Seek(start, 0)
if err != nil {
t.Errorf("%s: %s", label, err)
continue
}
if n != start {
t.Errorf("got post-Seek offset %d, want %d", n, start)
}
b, err := ioutil.ReadAll(io.LimitReader(f, end-start))
if err != nil {
t.Errorf("%s: ReadAll: %s", label, err)
continue
}
trunc := func(b []byte) string {
if len(b) > 75 {
return string(b[:75]) + "..." + string(b[len(b)-5:]) + fmt.Sprintf(" (%d bytes total)", len(b))
}
return string(b)
}
if want := fullData[start:end]; !bytes.Equal(b, want) {
t.Errorf("%s: full read: got %q, want %q", label, trunc(b), trunc(want))
continue
}
if start != end && !reuse {
if len(rrt.readRanges) == 0 {
t.Errorf("%s: no read ranges, want range %d-%d", label, start, end)
}
}
if !autofetch {
if err := rrt.checkOnlyReadRange(start, end); err != nil {
t.Errorf("%s: %s", label, err)
}
}
if !reuse || i == len(cases)-1 {
if err := f.Close(); err != nil {
t.Fatal(err)
}
}
}
}
}
}
}
func main() {
log.SetFlags(0)
flag.Parse()
if flag.NArg() != 2 {
log.Fatal("error: usage: httpvfs-client [opts] <cat|ls|put|rm> <path>")
}
op := flag.Arg(0)
path := path.Clean(flag.Arg(1))
url, err := url.Parse(*urlStr)
if err != nil {
log.Fatal(err)
}
fs := rwvfs.HTTP(url, nil)
switch strings.ToLower(op) {
case "cat":
if *startByte != 0 && *endByte < *startByte {
log.Fatal("error: -end-byte must be greater than -start-byte")
}
var f vfs.ReadSeekCloser
var err error
if *startByte != 0 {
f, err = fs.(rwvfs.FetcherOpener).OpenFetcher(path)
} else {
f, err = fs.Open(path)
}
if err != nil {
log.Fatal(err)
}
defer func() {
if err := f.Close(); err != nil {
log.Fatal(err)
}
}()
rdr := io.Reader(f)
if *startByte != 0 {
if err := f.(rwvfs.Fetcher).Fetch(int64(*startByte), int64(*endByte)); err != nil {
log.Fatalf("Fetch bytes=%d-%d: %s", *startByte, *endByte, err)
}
}
if *startByte != 0 {
if _, err := f.Seek(int64(*startByte), 0); err != nil {
log.Fatalln("Seek:", err)
}
}
if *endByte != -1 {
byteLen := *endByte - *startByte
rdr = io.LimitReader(f, int64(byteLen))
}
if _, err := io.Copy(os.Stdout, rdr); err != nil {
log.Fatalln("Copy:", err)
}
case "ls":
fis, err := fs.ReadDir(path)
if err != nil {
log.Fatal(err)
}
var longestNameLen int
for _, fi := range fis {
if len(fi.Name()) > longestNameLen {
longestNameLen = len(fi.Name())
}
}
longestNameLen++ // account for "/" suffix on dirs
for _, fi := range fis {
name := fi.Name()
if fi.IsDir() {
name += "/"
}
mtime := fi.ModTime().Round(time.Second)
fmt.Printf("%-*s %s %d\n", longestNameLen, name, mtime, fi.Size())
}
case "put":
f, err := fs.Create(path)
if err != nil {
log.Fatal(err)
}
defer func() {
if err := f.Close(); err != nil {
log.Fatal(err)
}
}()
log.Println("(reading file data on stdin...)")
if _, err := io.Copy(f, os.Stdin); err != nil {
log.Fatal(err)
}
case "rm":
if err := fs.Remove(path); err != nil {
log.Fatal(err)
}
default:
log.Fatal("error: invalid op (see -h)")
}
}