func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
url := config.RequiredString("url")
auth := config.RequiredString("auth")
skipStartupCheck := config.OptionalBool("skipStartupCheck", false)
if err := config.Validate(); err != nil {
return nil, err
}
client := client.New(url)
if err = client.SetupAuthFromString(auth); err != nil {
return nil, err
}
client.SetLogger(log.New(os.Stderr, "remote", log.LstdFlags))
sto := &remoteStorage{
client: client,
}
if !skipStartupCheck {
// Do a quick dummy operation to check that our credentials are
// correct.
// TODO(bradfitz,mpl): skip this operation smartly if it turns out this is annoying/slow for whatever reason.
c := make(chan blob.SizedRef, 1)
err = sto.EnumerateBlobs(context.TODO(), c, "", 1)
if err != nil {
return nil, err
}
}
return sto, nil
}
func TestIsolation(t *testing.T) {
ld := test.NewLoader()
master, _ := ld.GetStorage("/good-storage/")
ns1 := newNamespace(t, ld)
ns2 := newNamespace(t, ld)
stoMap := map[string]blobserver.Storage{
"ns1": ns1,
"ns2": ns2,
"master": master,
}
want := func(src string, want ...blob.Ref) {
if _, ok := stoMap[src]; !ok {
t.Fatalf("undefined storage %q", src)
}
sort.Sort(blob.ByRef(want))
var got []blob.Ref
if err := blobserver.EnumerateAll(context.TODO(), stoMap[src], func(sb blob.SizedRef) error {
got = append(got, sb.Ref)
return nil
}); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(got, want) {
t.Errorf("server %q = %q; want %q", src, got, want)
}
}
b1 := &test.Blob{Contents: "Blob 1"}
b1r := b1.BlobRef()
b2 := &test.Blob{Contents: "Blob 2"}
b2r := b2.BlobRef()
b3 := &test.Blob{Contents: "Shared Blob"}
b3r := b3.BlobRef()
b1.MustUpload(t, ns1)
want("ns1", b1r)
want("ns2")
want("master", b1r)
b2.MustUpload(t, ns2)
want("ns1", b1r)
want("ns2", b2r)
want("master", b1r, b2r)
b3.MustUpload(t, ns2)
want("ns1", b1r)
want("ns2", b2r, b3r)
want("master", b1r, b2r, b3r)
b3.MustUpload(t, ns1)
want("ns1", b1r, b3r)
want("ns2", b2r, b3r)
want("master", b1r, b2r, b3r)
if _, _, err := ns2.FetchStreaming(b1r); err == nil {
t.Errorf("b1 shouldn't be accessible via ns2")
}
}
// Reindex rewrites the index files of the diskpacked .pack files
func Reindex(root string, overwrite bool, indexConf jsonconfig.Obj) (err error) {
// there is newStorage, but that may open a file for writing
var s = &storage{root: root}
index, err := newIndex(root, indexConf)
if err != nil {
return err
}
defer func() {
closeErr := index.Close()
// just returning the first error - if the index or disk is corrupt
// and can't close, it's very likely these two errors are related and
// have the same root cause.
if err == nil {
err = closeErr
}
}()
ctx := context.TODO() // TODO(tgulacsi): get the verbosity from context
for i := 0; i >= 0; i++ {
fh, err := os.Open(s.filename(i))
if err != nil {
if os.IsNotExist(err) {
break
}
return err
}
err = s.reindexOne(ctx, index, overwrite, i)
fh.Close()
if err != nil {
return err
}
}
return nil
}
func TestParseExpression(t *testing.T) {
qj := func(sq *SearchQuery) []byte {
v, err := json.MarshalIndent(sq, "", " ")
if err != nil {
panic(err)
}
return v
}
for _, tt := range parseExpressionTests {
ins := tt.inList
if len(ins) == 0 {
ins = []string{tt.in}
}
for _, in := range ins {
got, err := parseExpression(context.TODO(), in)
if err != nil {
if tt.errContains != "" && strings.Contains(err.Error(), tt.errContains) {
continue
}
t.Errorf("%s: parseExpression(%q) error: %v", tt.name, in, err)
continue
}
if tt.errContains != "" {
t.Errorf("%s: parseExpression(%q) succeeded; want error containing %q", tt.name, in, tt.errContains)
continue
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("%s: parseExpression(%q) got:\n%s\n\nwant:%s\n", tt.name, in, qj(got), qj(tt.want))
}
}
}
}
// Serves oauth callback at http://host/importer/TYPE/callback
func (h *Host) serveImporterAcctCallback(w http.ResponseWriter, r *http.Request, imp *importer) {
if r.Method != "GET" {
http.Error(w, "invalid method", 400)
return
}
acctRef, err := imp.impl.CallbackRequestAccount(r)
if err != nil {
httputil.ServeError(w, r, err)
return
}
if !acctRef.Valid() {
httputil.ServeError(w, r, errors.New("No valid blobref returned from CallbackRequestAccount(r)"))
return
}
ia, err := imp.account(acctRef)
if err != nil {
http.Error(w, "invalid 'acct' param: "+err.Error(), 400)
return
}
imp.impl.ServeCallback(w, r, &SetupContext{
Context: context.TODO(),
Host: h,
AccountNode: ia.acct,
ia: ia,
})
}
func (ia *importerAcct) setup(w http.ResponseWriter, r *http.Request) {
ia.im.impl.ServeSetup(w, r, &SetupContext{
Context: context.TODO(),
Host: ia.im.host,
AccountNode: ia.acct,
ia: ia,
})
}
func TestParseOperand(t *testing.T) {
for _, tt := range parseOperandTests {
p := newParser(tt.in)
got, err := p.parseOperand(context.TODO())
doChecking("parseOperand", t, tt, got, err, p)
}
}
func TestParseConjuction(t *testing.T) {
for _, tt := range parseAndRHSTests {
p := newParser(tt.in)
got, err := p.parseAndRHS(context.TODO(), tt.lhs)
doSticherChecking("parseAndRHS", t, tt, got, err, p)
}
}
func (ia *importerAcct) setup(w http.ResponseWriter, r *http.Request) {
if err := ia.im.impl.ServeSetup(w, r, &SetupContext{
Context: context.TODO(),
Host: ia.im.host,
AccountNode: ia.acct,
ia: ia,
}); err != nil {
log.Printf("%v", err)
}
}
func (c *gceCmd) RunCommand(args []string) error {
if c.verbose {
gce.Verbose = true
}
if c.project == "" {
return cmdmain.UsageError("Missing --project flag.")
}
if (c.certFile == "") != (c.keyFile == "") {
return cmdmain.UsageError("--cert and --key must both be given together.")
}
if c.certFile == "" && c.hostname == "" {
return cmdmain.UsageError("Either --hostname, or --cert & --key must provided.")
}
config := gce.NewOAuthConfig(readFile(clientIdDat), readFile(clientSecretDat))
config.RedirectURL = "urn:ietf:wg:oauth:2.0:oob"
instConf := &gce.InstanceConf{
Name: c.instName,
Project: c.project,
Machine: c.machine,
Zone: c.zone,
CertFile: c.certFile,
KeyFile: c.keyFile,
Hostname: c.hostname,
}
if c.sshPub != "" {
instConf.SSHPub = strings.TrimSpace(readFile(c.sshPub))
}
depl := &gce.Deployer{
Client: oauth2.NewClient(oauth2.NoContext, oauth2.ReuseTokenSource(nil, &oauthutil.TokenSource{
Config: config,
CacheFile: c.project + "-token.json",
AuthCode: func() string {
fmt.Println("Get auth code from:")
fmt.Printf("%v\n", config.AuthCodeURL("my-state", oauth2.AccessTypeOffline, oauth2.ApprovalForce))
fmt.Println("Enter auth code:")
sc := bufio.NewScanner(os.Stdin)
sc.Scan()
return strings.TrimSpace(sc.Text())
},
})),
Conf: instConf,
}
inst, err := depl.Create(context.TODO())
if err != nil {
return err
}
log.Printf("Instance is up at %s", inst.NetworkInterfaces[0].AccessConfigs[0].NatIP)
return nil
}
// Serves oauth callback at http://host/importer/TYPE/callback
func (h *Host) serveImporterAcctCallback(w http.ResponseWriter, r *http.Request, imp *importer) {
if r.Method != "GET" {
http.Error(w, "invalid method", 400)
return
}
acctRef, ok := blob.Parse(r.FormValue("acct"))
if !ok {
http.Error(w, "missing 'acct' blobref param", 400)
return
}
ia, err := imp.account(acctRef)
if err != nil {
http.Error(w, "invalid 'acct' param: "+err.Error(), 400)
return
}
imp.impl.ServeCallback(w, r, &SetupContext{
Context: context.TODO(),
Host: h,
AccountNode: ia.acct,
ia: ia,
})
}
func (h *DeployHandler) serveCallback(w http.ResponseWriter, r *http.Request) {
ck, err := r.Cookie("user")
if err != nil {
http.Error(w,
fmt.Sprintf("Cookie expired, or CSRF attempt. Restart from %s%s%s", h.scheme, h.host, h.prefix),
http.StatusBadRequest)
h.Printf("Cookie expired, or CSRF attempt on callback.")
return
}
code := r.FormValue("code")
if code == "" {
httputil.ServeError(w, r, errors.New("No oauth code parameter in callback URL"))
return
}
h.Printf("successful authentication: %v", r.URL.RawQuery)
br, tk, err := fromState(r)
if err != nil {
httputil.ServeError(w, r, err)
return
}
if !xsrftoken.Valid(tk, h.xsrfKey, ck.Value, br.String()) {
httputil.ServeError(w, r, fmt.Errorf("Invalid xsrf token: %q", tk))
return
}
oAuthConf := h.oAuthConfig()
tok, err := oAuthConf.Exchange(oauth2.NoContext, code)
if err != nil {
httputil.ServeError(w, r, fmt.Errorf("could not obtain a token: %v", err))
return
}
h.Printf("successful authorization with token: %v", tok)
instConf, err := h.instanceConf(br)
if err != nil {
httputil.ServeError(w, r, err)
return
}
depl := &Deployer{
Client: oAuthConf.Client(oauth2.NoContext, tok),
Conf: instConf,
}
if found := h.serveOldInstance(w, br, depl); found {
return
}
if err := h.recordState(br, &creationState{
InstConf: br,
}); err != nil {
httputil.ServeError(w, r, err)
return
}
if h.debug {
// We simulate an instance creation, without actually ever doing anything on Google Cloud,
// by sleeping for a while. Then, as we would do in the real case, we record a creation
// state (but a made-up one). In the meantime, the progress page/animation is served as
// usual.
go func() {
time.Sleep(7 * time.Second)
if err := h.recordState(br, &creationState{
InstConf: br,
InstAddr: "fake.instance.com",
Success: true,
CertFingerprintSHA1: "XXXXXXXXXXXXXXXXXXXX",
CertFingerprintSHA256: "YYYYYYYYYYYYYYYYYYYY",
}); err != nil {
h.Printf("Could not record creation state for %v: %v", br, err)
h.recordStateErrMu.Lock()
defer h.recordStateErrMu.Unlock()
h.recordStateErr[br.String()] = err
}
}()
h.serveProgress(w, br)
return
}
go func() {
inst, err := depl.Create(context.TODO())
state := &creationState{
InstConf: br,
}
if err != nil {
h.Printf("could not create instance: %v", err)
switch e := err.(type) {
case instanceExistsError:
state.Err = fmt.Sprintf("%v %v", e, helpDeleteInstance)
case projectIDError:
state.Err = fmt.Sprintf("%v", e)
default:
state.Err = fmt.Sprintf("%v. %v", err, fileIssue(br.String()))
}
} else {
state.InstAddr = addr(inst)
state.Success = true
state.CertFingerprintSHA1 = depl.certFingerprints["SHA-1"]
state.CertFingerprintSHA256 = depl.certFingerprints["SHA-256"]
}
if err := h.recordState(br, state); err != nil {
h.Printf("Could not record creation state for %v: %v", br, err)
h.recordStateErrMu.Lock()
defer h.recordStateErrMu.Unlock()
h.recordStateErr[br.String()] = err
}
}()
h.serveProgress(w, br)
}
func newSyncFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) {
var (
from = conf.RequiredString("from")
to = conf.RequiredString("to")
fullSync = conf.OptionalBool("fullSyncOnStart", false)
blockFullSync = conf.OptionalBool("blockingFullSyncOnStart", false)
idle = conf.OptionalBool("idle", false)
queueConf = conf.OptionalObject("queue")
copierPoolSize = conf.OptionalInt("copierPoolSize", 5)
validate = conf.OptionalBool("validateOnStart", validateOnStartDefault)
)
if err := conf.Validate(); err != nil {
return nil, err
}
if idle {
return newIdleSyncHandler(from, to), nil
}
if len(queueConf) == 0 {
return nil, errors.New(`Missing required "queue" object`)
}
q, err := sorted.NewKeyValue(queueConf)
if err != nil {
return nil, err
}
isToIndex := false
fromBs, err := ld.GetStorage(from)
if err != nil {
return nil, err
}
toBs, err := ld.GetStorage(to)
if err != nil {
return nil, err
}
if _, ok := fromBs.(*index.Index); !ok {
if _, ok := toBs.(*index.Index); ok {
isToIndex = true
}
}
sh := newSyncHandler(from, to, fromBs, toBs, q)
sh.toIndex = isToIndex
sh.copierPoolSize = copierPoolSize
if err := sh.readQueueToMemory(); err != nil {
return nil, fmt.Errorf("Error reading sync queue to memory: %v", err)
}
if fullSync || blockFullSync {
sh.logf("Doing full sync")
didFullSync := make(chan bool, 1)
go func() {
for {
n := sh.runSync("queue", sh.enumeratePendingBlobs)
if n > 0 {
sh.logf("Queue sync copied %d blobs", n)
continue
}
break
}
n := sh.runSync("full", blobserverEnumerator(context.TODO(), fromBs))
sh.logf("Full sync copied %d blobs", n)
didFullSync <- true
sh.syncLoop()
}()
if blockFullSync {
sh.logf("Blocking startup, waiting for full sync from %q to %q", from, to)
<-didFullSync
sh.logf("Full sync complete.")
}
} else {
go sh.syncLoop()
}
if validate {
go sh.startFullValidation()
}
blobserver.GetHub(fromBs).AddReceiveHook(sh.enqueue)
return sh, nil
}
// src: non-nil source
// dest: non-nil destination
// thirdLeg: optional third-leg client. if not nil, anything on src
// but not on dest will instead be copied to thirdLeg, instead of
// directly to dest. (sneakernet mode, copying to a portable drive
// and transporting thirdLeg to dest)
func (c *syncCmd) doPass(src, dest, thirdLeg blobserver.Storage) (stats SyncStats, retErr error) {
srcBlobs := make(chan blob.SizedRef, 100)
destBlobs := make(chan blob.SizedRef, 100)
srcErr := make(chan error, 1)
destErr := make(chan error, 1)
ctx := context.TODO()
defer ctx.Cancel()
go func() {
srcErr <- enumerateAllBlobs(ctx, src, srcBlobs)
}()
checkSourceError := func() {
if err := <-srcErr; err != nil {
retErr = fmt.Errorf("Enumerate error from source: %v", err)
}
}
if c.dest == "stdout" {
for sb := range srcBlobs {
fmt.Printf("%s %d\n", sb.Ref, sb.Size)
}
checkSourceError()
return
}
go func() {
destErr <- enumerateAllBlobs(ctx, dest, destBlobs)
}()
checkDestError := func() {
if err := <-destErr; err != nil {
retErr = errors.New(fmt.Sprintf("Enumerate error from destination: %v", err))
}
}
destNotHaveBlobs := make(chan blob.SizedRef)
sizeMismatch := make(chan blob.Ref)
readSrcBlobs := srcBlobs
if c.verbose {
readSrcBlobs = loggingBlobRefChannel(srcBlobs)
}
mismatches := []blob.Ref{}
go client.ListMissingDestinationBlobs(destNotHaveBlobs, sizeMismatch, readSrcBlobs, destBlobs)
// Handle three-legged mode if tc is provided.
checkThirdError := func() {} // default nop
syncBlobs := destNotHaveBlobs
firstHopDest := dest
if thirdLeg != nil {
thirdBlobs := make(chan blob.SizedRef, 100)
thirdErr := make(chan error, 1)
go func() {
thirdErr <- enumerateAllBlobs(ctx, thirdLeg, thirdBlobs)
}()
checkThirdError = func() {
if err := <-thirdErr; err != nil {
retErr = fmt.Errorf("Enumerate error from third leg: %v", err)
}
}
thirdNeedBlobs := make(chan blob.SizedRef)
go client.ListMissingDestinationBlobs(thirdNeedBlobs, sizeMismatch, destNotHaveBlobs, thirdBlobs)
syncBlobs = thirdNeedBlobs
firstHopDest = thirdLeg
}
For:
for {
select {
case br := <-sizeMismatch:
// TODO(bradfitz): check both sides and repair, carefully. For now, fail.
log.Printf("WARNING: blobref %v has differing sizes on source and dest", br)
stats.ErrorCount++
mismatches = append(mismatches, br)
case sb, ok := <-syncBlobs:
if !ok {
break For
}
fmt.Printf("Destination needs blob: %s\n", sb)
blobReader, size, err := src.FetchStreaming(sb.Ref)
if err != nil {
stats.ErrorCount++
log.Printf("Error fetching %s: %v", sb.Ref, err)
continue
}
if size != sb.Size {
stats.ErrorCount++
log.Printf("Source blobserver's enumerate size of %d for blob %s doesn't match its Get size of %d",
sb.Size, sb.Ref, size)
continue
}
if _, err := blobserver.Receive(firstHopDest, sb.Ref, blobReader); err != nil {
stats.ErrorCount++
log.Printf("Upload of %s to destination blobserver failed: %v", sb.Ref, err)
continue
}
stats.BlobsCopied++
stats.BytesCopied += size
if c.removeSrc {
if err = src.RemoveBlobs([]blob.Ref{sb.Ref}); err != nil {
stats.ErrorCount++
log.Printf("Failed to delete %s from source: %v", sb.Ref, err)
}
}
}
}
checkSourceError()
checkDestError()
checkThirdError()
if retErr == nil && stats.ErrorCount > 0 {
retErr = fmt.Errorf("%d errors during sync", stats.ErrorCount)
}
return stats, retErr
}
func TestParseCoreAtom(t *testing.T) {
for _, tt := range parseCoreAtomTests {
got, err := parseCoreAtom(context.TODO(), tt.in)
doAtomChecking("parseCoreAtom", t, tt, got, err)
}
}
func newSyncFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) {
var (
from = conf.RequiredString("from")
to = conf.RequiredString("to")
fullSync = conf.OptionalBool("fullSyncOnStart", false)
blockFullSync = conf.OptionalBool("blockingFullSyncOnStart", false)
idle = conf.OptionalBool("idle", false)
queueConf = conf.OptionalObject("queue")
)
if err := conf.Validate(); err != nil {
return nil, err
}
if idle {
synch, err := createIdleSyncHandler(from, to)
if err != nil {
return nil, err
}
return synch, nil
}
if len(queueConf) == 0 {
return nil, errors.New(`Missing required "queue" object`)
}
q, err := sorted.NewKeyValue(queueConf)
if err != nil {
return nil, err
}
isToIndex := false
fromBs, err := ld.GetStorage(from)
if err != nil {
return nil, err
}
toBs, err := ld.GetStorage(to)
if err != nil {
return nil, err
}
if _, ok := fromBs.(*index.Index); !ok {
if _, ok := toBs.(*index.Index); ok {
isToIndex = true
}
}
sh, err := createSyncHandler(from, to, fromBs, toBs, q, isToIndex)
if err != nil {
return nil, err
}
if fullSync || blockFullSync {
didFullSync := make(chan bool, 1)
go func() {
n := sh.runSync("queue", sh.enumerateQueuedBlobs)
sh.logf("Queue sync copied %d blobs", n)
n = sh.runSync("full", blobserverEnumerator(context.TODO(), fromBs))
sh.logf("Full sync copied %d blobs", n)
didFullSync <- true
sh.syncQueueLoop()
}()
if blockFullSync {
sh.logf("Blocking startup, waiting for full sync from %q to %q", from, to)
<-didFullSync
sh.logf("Full sync complete.")
}
} else {
go sh.syncQueueLoop()
}
blobserver.GetHub(fromBs).AddReceiveHook(sh.enqueue)
return sh, nil
}
func handleEnumerateBlobs(conn http.ResponseWriter, req *http.Request, storage blobserver.BlobEnumerator) {
// Potential input parameters
formValueLimit := req.FormValue("limit")
formValueMaxWaitSec := req.FormValue("maxwaitsec")
formValueAfter := req.FormValue("after")
maxEnumerate := defaultMaxEnumerate
if config, ok := storage.(blobserver.MaxEnumerateConfig); ok {
maxEnumerate = config.MaxEnumerate() - 1 // Since we'll add one below.
}
limit := defaultEnumerateSize
if formValueLimit != "" {
n, err := strconv.ParseUint(formValueLimit, 10, 32)
if err != nil || n > uint64(maxEnumerate) {
limit = maxEnumerate
} else {
limit = int(n)
}
}
waitSeconds := 0
if formValueMaxWaitSec != "" {
waitSeconds, _ = strconv.Atoi(formValueMaxWaitSec)
if waitSeconds != 0 && formValueAfter != "" {
conn.WriteHeader(http.StatusBadRequest)
fmt.Fprintf(conn, errMsgMaxWaitSecWithAfter)
return
}
switch {
case waitSeconds < 0:
waitSeconds = 0
case waitSeconds > 30:
// TODO: don't hard-code 30. push this up into a blobserver interface
// for getting the configuration of the server (ultimately a flag in
// in the binary)
waitSeconds = 30
}
}
conn.Header().Set("Content-Type", "text/javascript; charset=utf-8")
fmt.Fprintf(conn, "{\n \"blobs\": [\n")
loop := true
needsComma := false
deadline := time.Now().Add(time.Duration(waitSeconds) * time.Second)
after := ""
for loop && (waitSeconds == 0 || time.Now().After(deadline)) {
if waitSeconds == 0 {
loop = false
}
blobch := make(chan blob.SizedRef, 100)
resultch := make(chan error, 1)
go func() {
resultch <- storage.EnumerateBlobs(context.TODO(), blobch, formValueAfter, limit+1)
}()
endsReached := 0
gotBlobs := 0
for endsReached < 2 {
select {
case sb, ok := <-blobch:
if !ok {
endsReached++
if gotBlobs <= limit {
after = ""
}
continue
}
gotBlobs++
loop = false
if gotBlobs > limit {
// We requested one more from storage than the user asked for.
// Now we know to return a "continueAfter" response key.
// But we don't return this blob.
continue
}
blobName := sb.Ref.String()
if needsComma {
fmt.Fprintf(conn, ",\n")
}
fmt.Fprintf(conn, " {\"blobRef\": \"%s\", \"size\": %d}",
blobName, sb.Size)
after = blobName
needsComma = true
case err := <-resultch:
if err != nil {
log.Printf("Error during enumerate: %v", err)
fmt.Fprintf(conn, "{{{ SERVER ERROR }}}")
return
}
endsReached++
}
}
if loop {
blobserver.WaitForBlob(storage, deadline, nil)
}
}
fmt.Fprintf(conn, "\n ]")
if after != "" {
fmt.Fprintf(conn, ",\n \"continueAfter\": \"%s\"", after)
}
const longPollSupported = true
if longPollSupported {
fmt.Fprintf(conn, ",\n \"canLongPoll\": true")
}
fmt.Fprintf(conn, "\n}\n")
}
func (h *Handler) Query(rawq *SearchQuery) (*SearchResult, error) {
ctx := context.TODO() // TODO: set from rawq
exprResult, err := rawq.checkValid(ctx)
if err != nil {
return nil, fmt.Errorf("Invalid SearchQuery: %v", err)
}
q := rawq.plannedQuery(exprResult)
res := new(SearchResult)
s := &search{
h: h,
q: q,
res: res,
ctx: context.TODO(),
}
defer s.ctx.Cancel()
corpus := h.corpus
var unlockOnce sync.Once
if corpus != nil {
corpus.RLock()
defer unlockOnce.Do(corpus.RUnlock)
}
ch := make(chan camtypes.BlobMeta, buffered)
errc := make(chan error, 1)
cands := q.pickCandidateSource(s)
if candSourceHook != nil {
candSourceHook(cands.name)
}
sendCtx := s.ctx.New()
defer sendCtx.Cancel()
go func() { errc <- cands.send(sendCtx, s, ch) }()
wantAround, foundAround := false, false
if q.Around.Valid() {
wantAround = true
}
blobMatches := q.Constraint.matcher()
for meta := range ch {
match, err := blobMatches(s, meta.Ref, meta)
if err != nil {
return nil, err
}
if match {
res.Blobs = append(res.Blobs, &SearchResultBlob{
Blob: meta.Ref,
})
if q.Limit <= 0 || !cands.sorted {
continue
}
if !wantAround || foundAround {
if len(res.Blobs) == q.Limit {
sendCtx.Cancel()
break
}
continue
}
if q.Around == meta.Ref {
foundAround = true
if len(res.Blobs)*2 > q.Limit {
// If we've already collected more than half of the Limit when Around is found,
// we ditch the surplus from the beginning of the slice of results.
// If Limit is even, and the number of results before and after Around
// are both greater than half the limit, then there will be one more result before
// than after.
discard := len(res.Blobs) - q.Limit/2 - 1
if discard < 0 {
discard = 0
}
res.Blobs = res.Blobs[discard:]
}
if len(res.Blobs) == q.Limit {
sendCtx.Cancel()
break
}
continue
}
if len(res.Blobs) == q.Limit {
n := copy(res.Blobs, res.Blobs[len(res.Blobs)/2:])
res.Blobs = res.Blobs[:n]
}
}
}
if err := <-errc; err != nil && err != context.ErrCanceled {
return nil, err
}
if q.Limit > 0 && cands.sorted && wantAround && !foundAround {
// results are ignored if Around was not found
res.Blobs = nil
}
if !cands.sorted {
switch q.Sort {
case UnspecifiedSort, Unsorted:
// Nothing to do.
case BlobRefAsc:
sort.Sort(sortSearchResultBlobs{res.Blobs, func(a, b *SearchResultBlob) bool {
return a.Blob.Less(b.Blob)
}})
case CreatedDesc, CreatedAsc:
if corpus == nil {
return nil, errors.New("TODO: Sorting without a corpus unsupported")
}
var err error
corpus.RLock()
sort.Sort(sortSearchResultBlobs{res.Blobs, func(a, b *SearchResultBlob) bool {
if err != nil {
return false
}
ta, ok := corpus.PermanodeAnyTimeLocked(a.Blob)
if !ok {
err = fmt.Errorf("no ctime or modtime found for %v", a.Blob)
return false
}
tb, ok := corpus.PermanodeAnyTimeLocked(b.Blob)
if !ok {
err = fmt.Errorf("no ctime or modtime found for %v", b.Blob)
return false
}
if q.Sort == CreatedAsc {
return ta.Before(tb)
}
return tb.Before(ta)
}})
corpus.RUnlock()
if err != nil {
return nil, err
}
// TODO(mpl): LastModifiedDesc, LastModifiedAsc
default:
return nil, errors.New("TODO: unsupported sort+query combination.")
}
if q.Limit > 0 && len(res.Blobs) > q.Limit {
res.Blobs = res.Blobs[:q.Limit]
}
}
if corpus != nil {
if !wantAround {
q.setResultContinue(corpus, res)
}
unlockOnce.Do(corpus.RUnlock)
}
if q.Describe != nil {
q.Describe.BlobRef = blob.Ref{} // zero this out, if caller set it
blobs := make([]blob.Ref, 0, len(res.Blobs))
for _, srb := range res.Blobs {
blobs = append(blobs, srb.Blob)
}
q.Describe.BlobRefs = blobs
res, err := s.h.Describe(q.Describe)
if err != nil {
return nil, err
}
s.res.Describe = res
}
return s.res, nil
}
func (s *storage) readAllMetaBlobs() error {
type metaBlob struct {
br blob.Ref
dat []byte // encrypted blob
err error
}
metac := make(chan metaBlob, 16)
const maxInFlight = 50
var gate = make(chan bool, maxInFlight)
var stopEnumerate = make(chan bool) // closed on error
enumErrc := make(chan error, 1)
go func() {
var wg sync.WaitGroup
enumErrc <- blobserver.EnumerateAll(context.TODO(), s.meta, func(sb blob.SizedRef) error {
select {
case <-stopEnumerate:
return errors.New("enumeration stopped")
default:
}
wg.Add(1)
gate <- true
go func() {
defer wg.Done()
defer func() { <-gate }()
rc, _, err := s.meta.Fetch(sb.Ref)
var all []byte
if err == nil {
all, err = ioutil.ReadAll(rc)
rc.Close()
}
metac <- metaBlob{sb.Ref, all, err}
}()
return nil
})
wg.Wait()
close(metac)
}()
for mi := range metac {
err := mi.err
if err == nil {
err = s.processEncryptedMetaBlob(mi.br, mi.dat)
}
if err != nil {
close(stopEnumerate)
go func() {
for _ = range metac {
}
}()
// TODO: advertise in this error message a new option or environment variable
// to skip a certain or all meta blobs, to allow partial recovery, if some
// are corrupt. For now, require all to be correct.
return fmt.Errorf("Error with meta blob %v: %v", mi.br, err)
}
}
return <-enumErrc
}
func (h *DeployHandler) serveCallback(w http.ResponseWriter, r *http.Request) {
ck, err := r.Cookie("user")
if err != nil {
http.Error(w,
fmt.Sprintf("Cookie expired, or CSRF attempt. Restart from %s%s%s", h.scheme, h.host, h.prefix),
http.StatusBadRequest)
h.Printf("Cookie expired, or CSRF attempt on callback.")
return
}
code := r.FormValue("code")
if code == "" {
httputil.ServeError(w, r, errors.New("No oauth code parameter in callback URL"))
return
}
h.Printf("successful authentication: %v", r.URL.RawQuery)
br, tk, err := fromState(r)
if err != nil {
httputil.ServeError(w, r, err)
return
}
if !xsrftoken.Valid(tk, h.xsrfKey, ck.Value, br.String()) {
httputil.ServeError(w, r, fmt.Errorf("Invalid xsrf token: %q", tk))
return
}
oAuthConf := h.oAuthConfig()
tok, err := oAuthConf.Exchange(oauth2.NoContext, code)
if err != nil {
httputil.ServeError(w, r, fmt.Errorf("could not obtain a token: %v", err))
return
}
h.Printf("successful authorization with token: %v", tok)
instConf, err := h.instanceConf(br)
if err != nil {
httputil.ServeError(w, r, err)
return
}
depl := &Deployer{
Client: oAuthConf.Client(oauth2.NoContext, tok),
Conf: instConf,
}
if found := h.serveOldInstance(w, br, depl); found {
return
}
if err := h.recordState(br, &creationState{
InstConf: br,
}); err != nil {
httputil.ServeError(w, r, err)
return
}
go func() {
inst, err := depl.Create(context.TODO())
state := &creationState{
InstConf: br,
}
if err != nil {
h.Printf("could not create instance: %v", err)
switch e := err.(type) {
case instanceExistsError:
state.Err = fmt.Sprintf("%v %v", e, helpDeleteInstance)
case projectIDError:
state.Err = fmt.Sprintf("%v", e)
default:
state.Err = fmt.Sprintf("%v. %v", err, fileIssue(br.String()))
}
} else {
state.InstAddr = addr(inst)
state.Success = true
state.CertFingerprintSHA1 = depl.certFingerprints["SHA-1"]
state.CertFingerprintSHA256 = depl.certFingerprints["SHA-256"]
}
if err := h.recordState(br, state); err != nil {
h.Printf("Could not record creation state for %v: %v", br, err)
h.recordStateErrMu.Lock()
defer h.recordStateErrMu.Unlock()
h.recordStateErr[br.String()] = err
}
}()
h.serveProgress(w, br)
}
func (x *Index) Reindex() error {
ctx := context.TODO()
wiper, ok := x.s.(sorted.Wiper)
if !ok {
return fmt.Errorf("index's storage type %T doesn't support sorted.Wiper", x.s)
}
log.Printf("Wiping index storage type %T ...", x.s)
if err := wiper.Wipe(); err != nil {
return fmt.Errorf("error wiping index's sorted key/value type %T: %v", x.s, err)
}
log.Printf("Index wiped. Rebuilding...")
reindexStart, _ := blob.Parse(os.Getenv("CAMLI_REINDEX_START"))
err := x.s.Set(keySchemaVersion.name, fmt.Sprintf("%d", requiredSchemaVersion))
if err != nil {
return err
}
var nerrmu sync.Mutex
nerr := 0
blobc := make(chan blob.Ref, 32)
enumCtx := ctx.New()
enumErr := make(chan error, 1)
go func() {
defer close(blobc)
donec := enumCtx.Done()
var lastTick time.Time
enumErr <- blobserver.EnumerateAll(enumCtx, x.BlobSource, func(sb blob.SizedRef) error {
now := time.Now()
if lastTick.Before(now.Add(-1 * time.Second)) {
log.Printf("Reindexing at %v", sb.Ref)
lastTick = now
}
if reindexStart.Valid() && sb.Ref.Less(reindexStart) {
return nil
}
select {
case <-donec:
return context.ErrCanceled
case blobc <- sb.Ref:
return nil
}
})
}()
const j = 4 // arbitrary concurrency level
var wg sync.WaitGroup
for i := 0; i < j; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for br := range blobc {
if err := x.reindex(br); err != nil {
log.Printf("Error reindexing %v: %v", br, err)
nerrmu.Lock()
nerr++
nerrmu.Unlock()
// TODO: flag (or default?) to stop the EnumerateAll above once
// there's any error with reindexing?
}
}
}()
}
if err := <-enumErr; err != nil {
return err
}
wg.Wait()
log.Printf("Index rebuild complete.")
nerrmu.Lock() // no need to unlock
if nerr != 0 {
return fmt.Errorf("%d blobs failed to re-index", nerr)
}
if err := x.initDeletesCache(); err != nil {
return err
}
return nil
}
func TestDeletePermanode(t *testing.T) {
idx := index.NewMemoryIndex()
idxd := indextest.NewIndexDeps(idx)
foopn := idxd.NewPlannedPermanode("foo")
idxd.SetAttribute(foopn, "tag", "foo")
barpn := idxd.NewPlannedPermanode("bar")
idxd.SetAttribute(barpn, "tag", "bar")
bazpn := idxd.NewPlannedPermanode("baz")
idxd.SetAttribute(bazpn, "tag", "baz")
idxd.Delete(barpn)
c, err := idxd.Index.KeepInMemory()
if err != nil {
t.Fatalf("error slurping index to memory: %v", err)
}
// check that we initially only find permanodes foo and baz,
// because bar is already marked as deleted.
want := []blob.Ref{foopn, bazpn}
ch := make(chan camtypes.BlobMeta, 10)
var got []camtypes.BlobMeta
errc := make(chan error, 1)
c.RLock()
go func() { errc <- c.EnumeratePermanodesLastModifiedLocked(context.TODO(), ch) }()
for blobMeta := range ch {
got = append(got, blobMeta)
}
err = <-errc
c.RUnlock()
if err != nil {
t.Fatalf("Could not enumerate permanodes: %v", err)
}
if len(got) != len(want) {
t.Fatalf("Saw %d permanodes in corpus; want %d", len(got), len(want))
}
for _, bm := range got {
found := false
for _, perm := range want {
if bm.Ref == perm {
found = true
break
}
}
if !found {
t.Fatalf("permanode %v was not found in corpus", bm.Ref)
}
}
// now add a delete claim for permanode baz, and check that we're only left with foo permanode
delbaz := idxd.Delete(bazpn)
want = []blob.Ref{foopn}
got = got[:0]
ch = make(chan camtypes.BlobMeta, 10)
c.RLock()
go func() { errc <- c.EnumeratePermanodesLastModifiedLocked(context.TODO(), ch) }()
for blobMeta := range ch {
got = append(got, blobMeta)
}
err = <-errc
c.RUnlock()
if err != nil {
t.Fatalf("Could not enumerate permanodes: %v", err)
}
if len(got) != len(want) {
t.Fatalf("Saw %d permanodes in corpus; want %d", len(got), len(want))
}
if got[0].Ref != foopn {
t.Fatalf("Wrong permanode found in corpus. Wanted %v, got %v", foopn, got[0].Ref)
}
// baz undeletion. delete delbaz.
idxd.Delete(delbaz)
want = []blob.Ref{foopn, bazpn}
got = got[:0]
ch = make(chan camtypes.BlobMeta, 10)
c.RLock()
go func() { errc <- c.EnumeratePermanodesLastModifiedLocked(context.TODO(), ch) }()
for blobMeta := range ch {
got = append(got, blobMeta)
}
err = <-errc
c.RUnlock()
if err != nil {
t.Fatalf("Could not enumerate permanodes: %v", err)
}
if len(got) != len(want) {
t.Fatalf("Saw %d permanodes in corpus; want %d", len(got), len(want))
}
for _, bm := range got {
found := false
for _, perm := range want {
if bm.Ref == perm {
found = true
break
}
}
if !found {
t.Fatalf("permanode %v was not found in corpus", bm.Ref)
}
}
}
func (h *Handler) Query(rawq *SearchQuery) (*SearchResult, error) {
ctx := context.TODO() // TODO: set from rawq
exprResult, err := rawq.checkValid(ctx)
if err != nil {
return nil, fmt.Errorf("Invalid SearchQuery: %v", err)
}
q := rawq.plannedQuery(exprResult)
res := new(SearchResult)
s := &search{
h: h,
q: q,
res: res,
ctx: context.TODO(),
}
defer s.ctx.Cancel()
corpus := h.corpus
var unlockOnce sync.Once
if corpus != nil {
corpus.RLock()
defer unlockOnce.Do(corpus.RUnlock)
}
ch := make(chan camtypes.BlobMeta, buffered)
errc := make(chan error, 1)
cands := q.pickCandidateSource(s)
if candSourceHook != nil {
candSourceHook(cands.name)
}
sendCtx := s.ctx.New()
defer sendCtx.Cancel()
go func() { errc <- cands.send(sendCtx, s, ch) }()
blobMatches := q.Constraint.matcher()
for meta := range ch {
match, err := blobMatches(s, meta.Ref, meta)
if err != nil {
return nil, err
}
if match {
res.Blobs = append(res.Blobs, &SearchResultBlob{
Blob: meta.Ref,
})
if q.Limit > 0 && len(res.Blobs) == q.Limit && cands.sorted {
sendCtx.Cancel()
break
}
}
}
if err := <-errc; err != nil && err != context.ErrCanceled {
return nil, err
}
if !cands.sorted {
// TODO(bradfitz): sort them
if q.Limit > 0 && len(res.Blobs) > q.Limit {
res.Blobs = res.Blobs[:q.Limit]
}
}
if corpus != nil {
q.setResultContinue(corpus, res)
unlockOnce.Do(corpus.RUnlock)
}
if q.Describe != nil {
q.Describe.BlobRef = blob.Ref{} // zero this out, if caller set it
blobs := make([]blob.Ref, 0, len(res.Blobs))
for _, srb := range res.Blobs {
blobs = append(blobs, srb.Blob)
}
q.Describe.BlobRefs = blobs
res, err := s.h.Describe(q.Describe)
if err != nil {
return nil, err
}
s.res.Describe = res
}
return s.res, nil
}
// src: non-nil source
// dest: non-nil destination
// thirdLeg: optional third-leg client. if not nil, anything on src
// but not on dest will instead be copied to thirdLeg, instead of
// directly to dest. (sneakernet mode, copying to a portable drive
// and transporting thirdLeg to dest)
func (c *syncCmd) doPass(src, dest, thirdLeg blobserver.Storage) (stats SyncStats, retErr error) {
srcBlobs := make(chan blob.SizedRef, 100)
destBlobs := make(chan blob.SizedRef, 100)
srcErr := make(chan error, 1)
destErr := make(chan error, 1)
ctx := context.TODO()
enumCtx := ctx.New() // used for all (2 or 3) enumerates
defer enumCtx.Cancel()
enumerate := func(errc chan<- error, sto blobserver.Storage, blobc chan<- blob.SizedRef) {
err := enumerateAllBlobs(enumCtx, sto, blobc)
if err != nil {
enumCtx.Cancel()
}
errc <- err
}
go enumerate(srcErr, src, srcBlobs)
checkSourceError := func() {
if err := <-srcErr; err != nil && err != context.ErrCanceled {
retErr = fmt.Errorf("Enumerate error from source: %v", err)
}
}
if c.dest == "stdout" {
for sb := range srcBlobs {
fmt.Fprintf(cmdmain.Stdout, "%s %d\n", sb.Ref, sb.Size)
}
checkSourceError()
return
}
if c.wipe {
// TODO(mpl): dest is a client. make it send a "wipe" request?
// upon reception its server then wipes itself if it is a wiper.
log.Print("Index wiping not yet supported.")
}
go enumerate(destErr, dest, destBlobs)
checkDestError := func() {
if err := <-destErr; err != nil && err != context.ErrCanceled {
retErr = fmt.Errorf("Enumerate error from destination: %v", err)
}
}
destNotHaveBlobs := make(chan blob.SizedRef)
readSrcBlobs := srcBlobs
if c.verbose {
readSrcBlobs = loggingBlobRefChannel(srcBlobs)
}
mismatches := []blob.Ref{}
onMismatch := func(br blob.Ref) {
// TODO(bradfitz): check both sides and repair, carefully. For now, fail.
log.Printf("WARNING: blobref %v has differing sizes on source and dest", br)
stats.ErrorCount++
mismatches = append(mismatches, br)
}
go blobserver.ListMissingDestinationBlobs(destNotHaveBlobs, onMismatch, readSrcBlobs, destBlobs)
// Handle three-legged mode if tc is provided.
checkThirdError := func() {} // default nop
syncBlobs := destNotHaveBlobs
firstHopDest := dest
if thirdLeg != nil {
thirdBlobs := make(chan blob.SizedRef, 100)
thirdErr := make(chan error, 1)
go enumerate(thirdErr, thirdLeg, thirdBlobs)
checkThirdError = func() {
if err := <-thirdErr; err != nil && err != context.ErrCanceled {
retErr = fmt.Errorf("Enumerate error from third leg: %v", err)
}
}
thirdNeedBlobs := make(chan blob.SizedRef)
go blobserver.ListMissingDestinationBlobs(thirdNeedBlobs, onMismatch, destNotHaveBlobs, thirdBlobs)
syncBlobs = thirdNeedBlobs
firstHopDest = thirdLeg
}
for sb := range syncBlobs {
fmt.Fprintf(cmdmain.Stdout, "Destination needs blob: %s\n", sb)
blobReader, size, err := src.Fetch(sb.Ref)
if err != nil {
stats.ErrorCount++
log.Printf("Error fetching %s: %v", sb.Ref, err)
continue
}
if size != sb.Size {
stats.ErrorCount++
log.Printf("Source blobserver's enumerate size of %d for blob %s doesn't match its Get size of %d",
sb.Size, sb.Ref, size)
continue
}
if _, err := blobserver.Receive(firstHopDest, sb.Ref, blobReader); err != nil {
stats.ErrorCount++
log.Printf("Upload of %s to destination blobserver failed: %v", sb.Ref, err)
continue
}
stats.BlobsCopied++
stats.BytesCopied += int64(size)
if c.removeSrc {
if err = src.RemoveBlobs([]blob.Ref{sb.Ref}); err != nil {
stats.ErrorCount++
log.Printf("Failed to delete %s from source: %v", sb.Ref, err)
}
}
}
checkSourceError()
checkDestError()
checkThirdError()
if retErr == nil && stats.ErrorCount > 0 {
retErr = fmt.Errorf("%d errors during sync", stats.ErrorCount)
}
return stats, retErr
}