예제 #1
0
func (ds *DiskStore) Put(name string, vn IVnode, r io.Reader) (written int64, err os.Error) {
	written = 0
	dir_path, full_path, err := ds.buildBlobPath(vn, name)
	if err != nil {
		return
	}

	err = os.MkdirAll(dir_path, 0700)
	if err != nil {
		return
	}

	_, err = os.Stat(full_path)
	if err == nil {
		// blob with same name exists.
		// means we are done.
		// TODO: check to verify size of the blobs are same
		//       and log that we tried to add existing blob.
		l4g.Info("Blob already exists at %s", full_path)
		return
	}
	f, err := os.Create(full_path)
	defer f.Close()
	if err != nil {
		return
	}

	written, err = io.Copy(f, r)
	return
}
예제 #2
0
파일: main.go 프로젝트: srijak/blobstore
func serve(configs *ConfigOpts) os.Error {
	ks := NewKeySpace(configs.ZkRootNode, configs.ZkHosts, 5e6)
	ks.Connect()
	rs := &SimpleRep{N: configs.ReplicationFactor}
	ls := NewDiskStore(configs.RootDir)
	rsf := &RemoteStoreFactory{}
	b := NewBlobStore(ks, rs, ls, rsf, configs.Port)
	rpc.Register(b)
	rpc.HandleHTTP()
	addr := fmt.Sprintf(":%d", configs.Port)
	l4g.Info("Serving on %s", addr)
	http.ListenAndServe(addr, nil)
	return nil
}
예제 #3
0
func (k *KeySpace) Connect() os.Error {
	zk, session, err := gozk.Init(k.zkServers, k.zkTimeout)
	if err != nil {
		return os.NewError("Couldn't connect: " + err.String())
	}

	event := <-session
	if event.State != gozk.STATE_CONNECTED {
		return os.NewError("Couldn't connect to zookeeper\n")
	}

	k.zk = zk

	stat, err := k.zk.Exists(k.zkRoot)
	if stat == nil {
		path, err := k.zk.Create(k.zkRoot, "root", 0, gozk.WorldACL(gozk.PERM_ALL))
		if err != nil {
			l4g.Info("Created root node: %s\n", path)
		}
	}

	return nil
}
예제 #4
0
func (b *BlobStore) Get(key *string, w io.Writer) os.Error {
	// get replica list
	// TODO: remove servers that aren't currently up.
	// If we are in the replica list:
	//    return data
	//      if data not available locally, get data from other replicas
	//        (readrepair: store locally so next time we can field the request)
	// If not on us:
	//    hit one of the replicas for data, it will find and return data.
	vnodes, err := b.ks.GetVnodes()
	if err != nil {
		return err
	}
	replicas, err := b.rs.Replicas(*key, vnodes)
	if err != nil {
		return err
	}
	copyLocally := false
	var local_vnode_idx int
	for i, replica := range replicas {
		// look for a local one first.

		if isLocalVnode(replica) {
			// this code assumes that there is only one matching local vnode.
			// i.e that replicas aren't on the same server.
			l4g.Debug("Key should be on local vnode: %s", replica)
			_, err = b.ls.Get(*key, replica, w)
			if err != nil {
				// err is assumed to be coz data was missing.
				// so, copy it locally once we get it.
				l4g.Debug("Key not found on local vnode. Will read repair vode %s for key: %s", replica, *key)
				copyLocally = true
				local_vnode_idx = i
			} else {
				// if no errors getting data locally,
				// we are done
				//TODO: verify data:
				//if hash doesn't match, get it from another replica.
				l4g.Debug("Found data for key %s locally.", *key)
				return nil
			}
		}
	}
	// the fact that we are here means data wasn't found locally.
	// if copyLocally is true, it means that we should store
	// the data locally coz it should have been here.
	var local_vnode IVnode
	if copyLocally {
		//remove local vnode from list so it isn't tried again.
		local_vnode = replicas[local_vnode_idx]
		replicas = append(replicas[:local_vnode_idx], replicas[local_vnode_idx+1:]...)
	}
	l4g.Debug("Data for key %s not found locally. Trying other replicas.", *key)
	for _, replica := range replicas {
		err = b.getRemoteBlob(key, replica, w)
		if err == nil {
			l4g.Debug("Got data for key %s from remote: %s", *key, replica.String())
			if copyLocally {
				l4g.Debug("ReadRepair: Copying key %s to local vnode: %s", *key, local_vnode.String())
				// go b.ls.Put(blob, *key, local_vnode)
				l4g.Error("ReadRepaid not implemented")
			}
			return nil
		} else {
			l4g.Info("Couldnt get key %s from remote replica %s. Error: %s", *key, replica, err.String())
		}
	}

	return os.NewError("Could not get from any of the replicas")
}