func main() {
comment := flag.String("comment", "", "comment to add to commit's meta data")
spec.RegisterDatabaseFlags(flag.CommandLine)
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "Fetches a URL (or file) into a noms blob\n\nUsage: %s <url-or-local-path> <dataset>:\n", os.Args[0])
flag.PrintDefaults()
}
flag.Parse(true)
if flag.NArg() != 2 {
d.CheckErrorNoUsage(errors.New("expected dataset and url arguments"))
}
ds, err := spec.GetDataset(flag.Arg(1))
d.CheckErrorNoUsage(err)
defer ds.Database().Close()
url := flag.Arg(0)
fileOrUrl := "file"
start = time.Now()
var pr io.Reader
if strings.HasPrefix(url, "http") {
resp, err := http.Get(url)
if err != nil {
fmt.Fprintf(os.Stderr, "Could not fetch url %s, error: %s\n", url, err)
return
}
switch resp.StatusCode / 100 {
case 4, 5:
fmt.Fprintf(os.Stderr, "Could not fetch url %s, error: %d (%s)\n", url, resp.StatusCode, resp.Status)
return
}
pr = progressreader.New(resp.Body, getStatusPrinter(resp.ContentLength))
fileOrUrl = "url"
} else {
// assume it's a file
f, err := os.Open(url)
if err != nil {
fmt.Fprintf(os.Stderr, "Invalid URL %s - does not start with 'http' and isn't local file either. fopen error: %s", url, err)
return
}
s, err := f.Stat()
if err != nil {
fmt.Fprintf(os.Stderr, "Could not stat file %s: %s", url, err)
return
}
pr = progressreader.New(f, getStatusPrinter(s.Size()))
fileOrUrl = "file"
}
b := types.NewStreamingBlob(pr, ds.Database())
mi := metaInfoForCommit(fileOrUrl, url, *comment)
ds, err = ds.Commit(b, dataset.CommitOptions{Meta: mi})
if err != nil {
d.Chk.Equal(datas.ErrMergeNeeded, err)
fmt.Fprintf(os.Stderr, "Could not commit, optimistic concurrency failed.")
return
}
status.Done()
fmt.Println("Done")
}
func main() {
comment := flag.String("comment", "", "comment to add to commit's meta data")
stdin := flag.Bool("stdin", false, "read blob from stdin")
spec.RegisterDatabaseFlags(flag.CommandLine)
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "Fetches a URL, file, or stdin into a noms blob\n\nUsage: %s [--stdin?] [url-or-local-path?] [dataset]\n", os.Args[0])
flag.PrintDefaults()
}
flag.Parse(true)
if !(*stdin && flag.NArg() == 1) && flag.NArg() != 2 {
flag.Usage()
os.Exit(-1)
}
start = time.Now()
ds, err := spec.GetDataset(flag.Arg(flag.NArg() - 1))
d.CheckErrorNoUsage(err)
defer ds.Database().Close()
var r io.Reader
var contentLength int64
var sourceType, sourceVal string
if *stdin {
r = os.Stdin
contentLength = -1
} else if url := flag.Arg(0); strings.HasPrefix(url, "http") {
resp, err := http.Get(url)
if err != nil {
fmt.Fprintf(os.Stderr, "Could not fetch url %s, error: %s\n", url, err)
return
}
switch resp.StatusCode / 100 {
case 4, 5:
fmt.Fprintf(os.Stderr, "Could not fetch url %s, error: %d (%s)\n", url, resp.StatusCode, resp.Status)
return
}
r = resp.Body
contentLength = resp.ContentLength
sourceType, sourceVal = "url", url
} else {
// assume it's a file
f, err := os.Open(url)
if err != nil {
fmt.Fprintf(os.Stderr, "Invalid URL %s - does not start with 'http' and isn't local file either. fopen error: %s", url, err)
return
}
s, err := f.Stat()
if err != nil {
fmt.Fprintf(os.Stderr, "Could not stat file %s: %s", url, err)
return
}
r = f
contentLength = s.Size()
sourceType, sourceVal = "file", url
}
pr := progressreader.New(r, getStatusPrinter(contentLength))
b := types.NewStreamingBlob(pr, ds.Database())
mi := metaInfoForCommit(sourceType, sourceVal, *comment)
ds, err = ds.Commit(b, dataset.CommitOptions{Meta: mi})
if err != nil {
d.Chk.Equal(datas.ErrMergeNeeded, err)
fmt.Fprintf(os.Stderr, "Could not commit, optimistic concurrency failed.")
return
}
status.Done()
fmt.Println("Done")
}