Exemple #1
0
func getValue(config *conf.ConfigFile, key string, namespace string, fail bool) string {
	str, err := config.GetString(namespace, key)
	if err != nil && fail {
		//Exit if we can't find an expected value
		l4g.Error("Error getting %s: %s\n", key, err.String())
		os.Exit(1)
	}
	return str
}
Exemple #2
0
func main() {
	//Load logging configuration
	l4g.LoadConfiguration("logging.xml")

	//Load default config file
	var configPath string
	flag.StringVar(&configPath, "config", "/etc/gosolr/gosolr.cfg", "Path to the configuration file")
	flag.Parse()
	config = loadConfig(configPath)

	//Load solr servers/cores from mysql db
	solrServers = loadSolrServers(config)
	prettyPrint(solrServers)

	//Setup the http proxy stuff
	for apiKey := range solrServers {
		urlPath := "/" + apiKey
		http.HandleFunc(urlPath, handleRequest)
	}

	var err os.Error
	var srv http.Server
	srv.Addr = config["default"]["host"] + ":" + config["default"]["port"]
	srv.Handler = nil
	if srv.ReadTimeout, err = strconv.Atoi64(config["default"]["read_timeout"]); err != nil {
		l4g.Error("Configuration error. Bad read_timout value")
		os.Exit(1)
	}
	if srv.WriteTimeout, err = strconv.Atoi64(config["default"]["write_timeout"]); err != nil {
		l4g.Error("Configuration error. Bad write_timeout value")
		os.Exit(1)
	}

	//If this were real, this should be TLS
	if err := srv.ListenAndServe(); err != nil {
		l4g.Error("Error starting server: %s", err.String())
		os.Exit(1)
	}
}
Exemple #3
0
//Pull information from MySQL to find out which API keys we should handle and how they map
//  i.e. servers and cores
func loadSolrServers(config map[string]map[string]string) map[string]map[string]string {
	db, err := mysql.DialTCP(config["database"]["host"], config["database"]["user"], config["database"]["pass"], config["database"]["name"])
	if err != nil {
		l4g.Error("Error connecting to db: %s\n", err.String())
		os.Exit(1)
	}
	stmt, err := db.Prepare("Select apistring,core,server,authstring from cores where gosolr = ?")
	if err != nil {
		l4g.Error("Error preparing statement: %s", err.String())
		os.Exit(1)
	}

	stmt.BindParams(config["default"]["host"])

	err = stmt.Execute()
	if err != nil {
		l4g.Error("error executing stmt: %s", err.String())
		os.Exit(1)
	}

	var solrrow SolrData
	stmt.BindResult(&solrrow.apistring, &solrrow.core, &solrrow.server, &solrrow.authstring)
	solr_values := make(map[string]map[string]string)

	for {
		eof, err := stmt.Fetch()
		if err != nil {
			l4g.Error("Error fetching row: %s", err.String())
			os.Exit(1)
		}

		solr_values[solrrow.apistring] = map[string]string{"core": solrrow.core, "server": solrrow.server, "authstring": solrrow.authstring}
		if eof {
			break
		}
	}
	return solr_values
}
Exemple #4
0
func main() {
	if len(os.Args) < 3 {
		usage()
		return
	}
	flag.Parse()

	rs, err := NewRemoteStore(*hostname, *port)
	if err != nil {
		l4g.Error(err.String())
		return
	}
	runCommand(flag.Arg(0), rs, flag.Args()[1:])
}
Exemple #5
0
func runCommand(command string, rs IRemoteStore, args []string) {
	var err os.Error
	switch {
	case command == "put":
		err = put(rs, args)
	case command == "get":
		err = get(rs, args)
	default:
		usage()
	}
	if err != nil {
		l4g.Error(err)
	}
}
Exemple #6
0
func runCommand(configs *ConfigOpts, command string, args []string) {
	var err os.Error
	switch {
	case command == "serve":
		err = serve(configs)
	case command == "add-vnode":
		err = addVnode(configs, args)
	case command == "rm-vnode":
		err = rmVnode(configs, args)
	case command == "ls-vnodes":
		err = lsVnodes(configs)
	default:
		usage()
	}
	if err != nil {
		l4g.Error(err)
	}
}
Exemple #7
0
//Move these into a different package?
func loadConfig(file string) map[string]map[string]string {
	config, err := conf.ReadConfigFile(file)
	if err != nil {
		l4g.Error("error reading config: %s\n", err.String())
		os.Exit(1)
	}

	//Setup configuration values
	values := make(map[string]map[string]string)

	//Setup some default values - these are in the default namespace
	//There are 3 configuration namespaces default, database, and stomp
	//If the default value is "" that means it is required in the config file.
	//An empty value in the config will not overwrite the default value
	values["default"] = map[string]string{"host": "localhost", "port": "80", "read_timeout": "0", "write_timeout": "0"}
	values["database"] = map[string]string{"host": "localhost", "port": "3306", "user": "", "pass": "", "name": ""}
	values["stomp"] = map[string]string{"host": ""}

	l4g.Debug("Default vals: %v", values)

	//Read values from config
	for namespace := range values {
		//If there is a default value it's ok if the config key doesn't exist
		for key, value := range values[namespace] {
			if value != "" {
				set := getValue(config, key, namespace, false)
				if set != "" {
					values[namespace][key] = set
				}
			} else {
				values[namespace][key] = getValue(config, key, namespace, true)
			}
		}
	}

	l4g.Debug("Final values: %v", values)
	return values
}
Exemple #8
0
func (b *BlobStore) Get(key *string, w io.Writer) os.Error {
	// get replica list
	// TODO: remove servers that aren't currently up.
	// If we are in the replica list:
	//    return data
	//      if data not available locally, get data from other replicas
	//        (readrepair: store locally so next time we can field the request)
	// If not on us:
	//    hit one of the replicas for data, it will find and return data.
	vnodes, err := b.ks.GetVnodes()
	if err != nil {
		return err
	}
	replicas, err := b.rs.Replicas(*key, vnodes)
	if err != nil {
		return err
	}
	copyLocally := false
	var local_vnode_idx int
	for i, replica := range replicas {
		// look for a local one first.

		if isLocalVnode(replica) {
			// this code assumes that there is only one matching local vnode.
			// i.e that replicas aren't on the same server.
			l4g.Debug("Key should be on local vnode: %s", replica)
			_, err = b.ls.Get(*key, replica, w)
			if err != nil {
				// err is assumed to be coz data was missing.
				// so, copy it locally once we get it.
				l4g.Debug("Key not found on local vnode. Will read repair vode %s for key: %s", replica, *key)
				copyLocally = true
				local_vnode_idx = i
			} else {
				// if no errors getting data locally,
				// we are done
				//TODO: verify data:
				//if hash doesn't match, get it from another replica.
				l4g.Debug("Found data for key %s locally.", *key)
				return nil
			}
		}
	}
	// the fact that we are here means data wasn't found locally.
	// if copyLocally is true, it means that we should store
	// the data locally coz it should have been here.
	var local_vnode IVnode
	if copyLocally {
		//remove local vnode from list so it isn't tried again.
		local_vnode = replicas[local_vnode_idx]
		replicas = append(replicas[:local_vnode_idx], replicas[local_vnode_idx+1:]...)
	}
	l4g.Debug("Data for key %s not found locally. Trying other replicas.", *key)
	for _, replica := range replicas {
		err = b.getRemoteBlob(key, replica, w)
		if err == nil {
			l4g.Debug("Got data for key %s from remote: %s", *key, replica.String())
			if copyLocally {
				l4g.Debug("ReadRepair: Copying key %s to local vnode: %s", *key, local_vnode.String())
				// go b.ls.Put(blob, *key, local_vnode)
				l4g.Error("ReadRepaid not implemented")
			}
			return nil
		} else {
			l4g.Info("Couldnt get key %s from remote replica %s. Error: %s", *key, replica, err.String())
		}
	}

	return os.NewError("Could not get from any of the replicas")
}
Exemple #9
0
func handleRequest(w http.ResponseWriter, req *http.Request) {
	apiKey := req.URL.Path[1:]

	if req.Method == "GET" {
		//Need to have some cleanup and validation here
		solrUrl := "http://" + solrServers[apiKey]["server"] + "/solr/" + solrServers[apiKey]["core"] + "/select/?" + req.URL.RawQuery
		l4g.Debug("Proxying Request to %s for %s", solrUrl, apiKey)

		r, err := http.Get(solrUrl)
		if err != nil {
			//Set actual error page here
			l4g.Error("Error: %s\n", err.String())
			http.Error(w, "500 Internal Server Error", 500)
			return
		}
		l4g.Debug("Tomcat response: %s", r.Status)
		r.Write(w)
		r.Body.Close()
	}
	if req.Method == "POST" {
		header := req.Header

		//Reject non-json data - This check can probably be done at the processing layer
		ct := header["Content-Type"][0]
		if ct != "application/json" {
			l4g.Error("Unsupported Content type %s", ct)
			http.Error(w, "Unsupported content type", 400)
			return
		}

		//Handle length check
		length, err := strconv.Atoi(header["Content-Length"][0])
		if err != nil {
			l4g.Error("Error converting content-length header: %s", err.String())
			http.Error(w, "Internal Server Error", 500)
			return
		}
		if length > 1024*1024 {
			l4g.Error("Post too large: %d", length)
			http.Error(w, "Post too large", 400)
			return
		}
		l4g.Debug("Post content-length: %d", length)

		body := make([]byte, length)
		len, ok := io.ReadFull(req.Body, body)
		if ok != nil {
			l4g.Debug("error reading io.ReadFull: %s", ok.String())
			http.Error(w, "Internal Server Error", 500)
			return
		}
		l4g.Debug("Content-length of %d, but read %d bytes from body", length, len)

		var message SolrPost
		if ok := json.Unmarshal(body, &message); ok != nil {
			l4g.Debug("Error unmarshalling json: %s", ok.String())
			http.Error(w, "Internal Server Error", 500)
			return
		}

		l4g.Debug("JSON message body: %s", message.Body)

		//Handle Auth
		if solrServers[apiKey]["authstring"] != message.Authkey {
			l4g.Error("Incorrect authkey")
			http.Error(w, "Unauthorized", 401)
			return
		}

		//Post message to queue
		nc, err := net.Dial("tcp", ""+config["stomp"]["host"])
		if err != nil {
			l4g.Error("Error conneceting to queue %s", err.String())
			http.Error(w, "Internal Server Error", 500)
			return
		}

		//Note, the stomp module doesn't return os.Error on Send()
		c, _ := stomp.Connect(nc, nil)
		queue := "/queue/" + apiKey
		l4g.Debug("Posting %s to queue %s", message.Body, queue)
		nh := stomp.Header{}
		c.Send(queue, nh, message.Body)
		c.Disconnect(nh)

		//Return result to client
		http.Error(w, "ok", 200)
	}
}