Beispiel #1
0
func (b *BlobStore) Put(key *string, r io.Reader) os.Error {
	// TODO: get and put are very similiar. Refactor to reuse logic
	vnodes, err := b.ks.GetVnodes()
	if err != nil {
		return err
	}
	replicas, err := b.rs.Replicas(*key, vnodes)
	if err != nil {
		return err
	}
	for _, replica := range replicas {
		// look for a local one first.
		if isLocalVnode(replica) {
			l4g.Debug("Data should be on local vnode: %s", replica)
			_, err := b.ls.Put(*key, replica, r)
			if err != nil {
				// local put failed for some reason.
				// TODO: how to handle?
				return err
			} else {
				// if no errors putting data locally,
				return nil
			}
		}
	}
	// the fact that we are here means it's not a local put or that the local
	// put failed.
	// Try to store on any replica. Note that we retry local here.
	// also, we require one synchronous successfull write.
	// the number of writes that need to succeed will be customizable.
	l4g.Debug("Data should be on remote vnode.")
	one_passed := false
	for _, replica := range replicas {

		l4g.Debug("Trying to store to %s.", replica.String())
		if !one_passed {
			err = b.putRemoteBlob(key, replica, r)
			if err == nil {
				l4g.Debug("Data stored on vnode %s.", replica.String())
				one_passed = true
			}
		} else {
			go b.putRemoteBlob(key, replica, r)
		}
	}

	if one_passed {
		return nil
	}
	return os.NewError("Could not put to any responsible nodes. TODO: hinted handoff")
}
Beispiel #2
0
func prettyPrint(printMap map[string]map[string]string) {
	for key, tmpMap := range printMap {
		for key2, value := range tmpMap {
			l4g.Debug("%s ->\t%s -> %s", key, key2, value)
		}
	}
}
Beispiel #3
0
//Move these into a different package?
func loadConfig(file string) map[string]map[string]string {
	config, err := conf.ReadConfigFile(file)
	if err != nil {
		l4g.Error("error reading config: %s\n", err.String())
		os.Exit(1)
	}

	//Setup configuration values
	values := make(map[string]map[string]string)

	//Setup some default values - these are in the default namespace
	//There are 3 configuration namespaces default, database, and stomp
	//If the default value is "" that means it is required in the config file.
	//An empty value in the config will not overwrite the default value
	values["default"] = map[string]string{"host": "localhost", "port": "80", "read_timeout": "0", "write_timeout": "0"}
	values["database"] = map[string]string{"host": "localhost", "port": "3306", "user": "", "pass": "", "name": ""}
	values["stomp"] = map[string]string{"host": ""}

	l4g.Debug("Default vals: %v", values)

	//Read values from config
	for namespace := range values {
		//If there is a default value it's ok if the config key doesn't exist
		for key, value := range values[namespace] {
			if value != "" {
				set := getValue(config, key, namespace, false)
				if set != "" {
					values[namespace][key] = set
				}
			} else {
				values[namespace][key] = getValue(config, key, namespace, true)
			}
		}
	}

	l4g.Debug("Final values: %v", values)
	return values
}
Beispiel #4
0
func (b *BlobStore) Get(key *string, w io.Writer) os.Error {
	// get replica list
	// TODO: remove servers that aren't currently up.
	// If we are in the replica list:
	//    return data
	//      if data not available locally, get data from other replicas
	//        (readrepair: store locally so next time we can field the request)
	// If not on us:
	//    hit one of the replicas for data, it will find and return data.
	vnodes, err := b.ks.GetVnodes()
	if err != nil {
		return err
	}
	replicas, err := b.rs.Replicas(*key, vnodes)
	if err != nil {
		return err
	}
	copyLocally := false
	var local_vnode_idx int
	for i, replica := range replicas {
		// look for a local one first.

		if isLocalVnode(replica) {
			// this code assumes that there is only one matching local vnode.
			// i.e that replicas aren't on the same server.
			l4g.Debug("Key should be on local vnode: %s", replica)
			_, err = b.ls.Get(*key, replica, w)
			if err != nil {
				// err is assumed to be coz data was missing.
				// so, copy it locally once we get it.
				l4g.Debug("Key not found on local vnode. Will read repair vode %s for key: %s", replica, *key)
				copyLocally = true
				local_vnode_idx = i
			} else {
				// if no errors getting data locally,
				// we are done
				//TODO: verify data:
				//if hash doesn't match, get it from another replica.
				l4g.Debug("Found data for key %s locally.", *key)
				return nil
			}
		}
	}
	// the fact that we are here means data wasn't found locally.
	// if copyLocally is true, it means that we should store
	// the data locally coz it should have been here.
	var local_vnode IVnode
	if copyLocally {
		//remove local vnode from list so it isn't tried again.
		local_vnode = replicas[local_vnode_idx]
		replicas = append(replicas[:local_vnode_idx], replicas[local_vnode_idx+1:]...)
	}
	l4g.Debug("Data for key %s not found locally. Trying other replicas.", *key)
	for _, replica := range replicas {
		err = b.getRemoteBlob(key, replica, w)
		if err == nil {
			l4g.Debug("Got data for key %s from remote: %s", *key, replica.String())
			if copyLocally {
				l4g.Debug("ReadRepair: Copying key %s to local vnode: %s", *key, local_vnode.String())
				// go b.ls.Put(blob, *key, local_vnode)
				l4g.Error("ReadRepaid not implemented")
			}
			return nil
		} else {
			l4g.Info("Couldnt get key %s from remote replica %s. Error: %s", *key, replica, err.String())
		}
	}

	return os.NewError("Could not get from any of the replicas")
}
Beispiel #5
0
func handleRequest(w http.ResponseWriter, req *http.Request) {
	apiKey := req.URL.Path[1:]

	if req.Method == "GET" {
		//Need to have some cleanup and validation here
		solrUrl := "http://" + solrServers[apiKey]["server"] + "/solr/" + solrServers[apiKey]["core"] + "/select/?" + req.URL.RawQuery
		l4g.Debug("Proxying Request to %s for %s", solrUrl, apiKey)

		r, err := http.Get(solrUrl)
		if err != nil {
			//Set actual error page here
			l4g.Error("Error: %s\n", err.String())
			http.Error(w, "500 Internal Server Error", 500)
			return
		}
		l4g.Debug("Tomcat response: %s", r.Status)
		r.Write(w)
		r.Body.Close()
	}
	if req.Method == "POST" {
		header := req.Header

		//Reject non-json data - This check can probably be done at the processing layer
		ct := header["Content-Type"][0]
		if ct != "application/json" {
			l4g.Error("Unsupported Content type %s", ct)
			http.Error(w, "Unsupported content type", 400)
			return
		}

		//Handle length check
		length, err := strconv.Atoi(header["Content-Length"][0])
		if err != nil {
			l4g.Error("Error converting content-length header: %s", err.String())
			http.Error(w, "Internal Server Error", 500)
			return
		}
		if length > 1024*1024 {
			l4g.Error("Post too large: %d", length)
			http.Error(w, "Post too large", 400)
			return
		}
		l4g.Debug("Post content-length: %d", length)

		body := make([]byte, length)
		len, ok := io.ReadFull(req.Body, body)
		if ok != nil {
			l4g.Debug("error reading io.ReadFull: %s", ok.String())
			http.Error(w, "Internal Server Error", 500)
			return
		}
		l4g.Debug("Content-length of %d, but read %d bytes from body", length, len)

		var message SolrPost
		if ok := json.Unmarshal(body, &message); ok != nil {
			l4g.Debug("Error unmarshalling json: %s", ok.String())
			http.Error(w, "Internal Server Error", 500)
			return
		}

		l4g.Debug("JSON message body: %s", message.Body)

		//Handle Auth
		if solrServers[apiKey]["authstring"] != message.Authkey {
			l4g.Error("Incorrect authkey")
			http.Error(w, "Unauthorized", 401)
			return
		}

		//Post message to queue
		nc, err := net.Dial("tcp", ""+config["stomp"]["host"])
		if err != nil {
			l4g.Error("Error conneceting to queue %s", err.String())
			http.Error(w, "Internal Server Error", 500)
			return
		}

		//Note, the stomp module doesn't return os.Error on Send()
		c, _ := stomp.Connect(nc, nil)
		queue := "/queue/" + apiKey
		l4g.Debug("Posting %s to queue %s", message.Body, queue)
		nh := stomp.Header{}
		c.Send(queue, nh, message.Body)
		c.Disconnect(nh)

		//Return result to client
		http.Error(w, "ok", 200)
	}
}