func getValue(config *conf.ConfigFile, key string, namespace string, fail bool) string {
str, err := config.GetString(namespace, key)
if err != nil && fail {
//Exit if we can't find an expected value
l4g.Error("Error getting %s: %s\n", key, err.String())
os.Exit(1)
}
return str
}
func main() {
//Load logging configuration
l4g.LoadConfiguration("logging.xml")
//Load default config file
var configPath string
flag.StringVar(&configPath, "config", "/etc/gosolr/gosolr.cfg", "Path to the configuration file")
flag.Parse()
config = loadConfig(configPath)
//Load solr servers/cores from mysql db
solrServers = loadSolrServers(config)
prettyPrint(solrServers)
//Setup the http proxy stuff
for apiKey := range solrServers {
urlPath := "/" + apiKey
http.HandleFunc(urlPath, handleRequest)
}
var err os.Error
var srv http.Server
srv.Addr = config["default"]["host"] + ":" + config["default"]["port"]
srv.Handler = nil
if srv.ReadTimeout, err = strconv.Atoi64(config["default"]["read_timeout"]); err != nil {
l4g.Error("Configuration error. Bad read_timout value")
os.Exit(1)
}
if srv.WriteTimeout, err = strconv.Atoi64(config["default"]["write_timeout"]); err != nil {
l4g.Error("Configuration error. Bad write_timeout value")
os.Exit(1)
}
//If this were real, this should be TLS
if err := srv.ListenAndServe(); err != nil {
l4g.Error("Error starting server: %s", err.String())
os.Exit(1)
}
}
//Pull information from MySQL to find out which API keys we should handle and how they map
// i.e. servers and cores
func loadSolrServers(config map[string]map[string]string) map[string]map[string]string {
db, err := mysql.DialTCP(config["database"]["host"], config["database"]["user"], config["database"]["pass"], config["database"]["name"])
if err != nil {
l4g.Error("Error connecting to db: %s\n", err.String())
os.Exit(1)
}
stmt, err := db.Prepare("Select apistring,core,server,authstring from cores where gosolr = ?")
if err != nil {
l4g.Error("Error preparing statement: %s", err.String())
os.Exit(1)
}
stmt.BindParams(config["default"]["host"])
err = stmt.Execute()
if err != nil {
l4g.Error("error executing stmt: %s", err.String())
os.Exit(1)
}
var solrrow SolrData
stmt.BindResult(&solrrow.apistring, &solrrow.core, &solrrow.server, &solrrow.authstring)
solr_values := make(map[string]map[string]string)
for {
eof, err := stmt.Fetch()
if err != nil {
l4g.Error("Error fetching row: %s", err.String())
os.Exit(1)
}
solr_values[solrrow.apistring] = map[string]string{"core": solrrow.core, "server": solrrow.server, "authstring": solrrow.authstring}
if eof {
break
}
}
return solr_values
}
func main() {
if len(os.Args) < 3 {
usage()
return
}
flag.Parse()
rs, err := NewRemoteStore(*hostname, *port)
if err != nil {
l4g.Error(err.String())
return
}
runCommand(flag.Arg(0), rs, flag.Args()[1:])
}
func runCommand(command string, rs IRemoteStore, args []string) {
var err os.Error
switch {
case command == "put":
err = put(rs, args)
case command == "get":
err = get(rs, args)
default:
usage()
}
if err != nil {
l4g.Error(err)
}
}
func runCommand(configs *ConfigOpts, command string, args []string) {
var err os.Error
switch {
case command == "serve":
err = serve(configs)
case command == "add-vnode":
err = addVnode(configs, args)
case command == "rm-vnode":
err = rmVnode(configs, args)
case command == "ls-vnodes":
err = lsVnodes(configs)
default:
usage()
}
if err != nil {
l4g.Error(err)
}
}
//Move these into a different package?
func loadConfig(file string) map[string]map[string]string {
config, err := conf.ReadConfigFile(file)
if err != nil {
l4g.Error("error reading config: %s\n", err.String())
os.Exit(1)
}
//Setup configuration values
values := make(map[string]map[string]string)
//Setup some default values - these are in the default namespace
//There are 3 configuration namespaces default, database, and stomp
//If the default value is "" that means it is required in the config file.
//An empty value in the config will not overwrite the default value
values["default"] = map[string]string{"host": "localhost", "port": "80", "read_timeout": "0", "write_timeout": "0"}
values["database"] = map[string]string{"host": "localhost", "port": "3306", "user": "", "pass": "", "name": ""}
values["stomp"] = map[string]string{"host": ""}
l4g.Debug("Default vals: %v", values)
//Read values from config
for namespace := range values {
//If there is a default value it's ok if the config key doesn't exist
for key, value := range values[namespace] {
if value != "" {
set := getValue(config, key, namespace, false)
if set != "" {
values[namespace][key] = set
}
} else {
values[namespace][key] = getValue(config, key, namespace, true)
}
}
}
l4g.Debug("Final values: %v", values)
return values
}
func (b *BlobStore) Get(key *string, w io.Writer) os.Error {
// get replica list
// TODO: remove servers that aren't currently up.
// If we are in the replica list:
// return data
// if data not available locally, get data from other replicas
// (readrepair: store locally so next time we can field the request)
// If not on us:
// hit one of the replicas for data, it will find and return data.
vnodes, err := b.ks.GetVnodes()
if err != nil {
return err
}
replicas, err := b.rs.Replicas(*key, vnodes)
if err != nil {
return err
}
copyLocally := false
var local_vnode_idx int
for i, replica := range replicas {
// look for a local one first.
if isLocalVnode(replica) {
// this code assumes that there is only one matching local vnode.
// i.e that replicas aren't on the same server.
l4g.Debug("Key should be on local vnode: %s", replica)
_, err = b.ls.Get(*key, replica, w)
if err != nil {
// err is assumed to be coz data was missing.
// so, copy it locally once we get it.
l4g.Debug("Key not found on local vnode. Will read repair vode %s for key: %s", replica, *key)
copyLocally = true
local_vnode_idx = i
} else {
// if no errors getting data locally,
// we are done
//TODO: verify data:
//if hash doesn't match, get it from another replica.
l4g.Debug("Found data for key %s locally.", *key)
return nil
}
}
}
// the fact that we are here means data wasn't found locally.
// if copyLocally is true, it means that we should store
// the data locally coz it should have been here.
var local_vnode IVnode
if copyLocally {
//remove local vnode from list so it isn't tried again.
local_vnode = replicas[local_vnode_idx]
replicas = append(replicas[:local_vnode_idx], replicas[local_vnode_idx+1:]...)
}
l4g.Debug("Data for key %s not found locally. Trying other replicas.", *key)
for _, replica := range replicas {
err = b.getRemoteBlob(key, replica, w)
if err == nil {
l4g.Debug("Got data for key %s from remote: %s", *key, replica.String())
if copyLocally {
l4g.Debug("ReadRepair: Copying key %s to local vnode: %s", *key, local_vnode.String())
// go b.ls.Put(blob, *key, local_vnode)
l4g.Error("ReadRepaid not implemented")
}
return nil
} else {
l4g.Info("Couldnt get key %s from remote replica %s. Error: %s", *key, replica, err.String())
}
}
return os.NewError("Could not get from any of the replicas")
}
func handleRequest(w http.ResponseWriter, req *http.Request) {
apiKey := req.URL.Path[1:]
if req.Method == "GET" {
//Need to have some cleanup and validation here
solrUrl := "http://" + solrServers[apiKey]["server"] + "/solr/" + solrServers[apiKey]["core"] + "/select/?" + req.URL.RawQuery
l4g.Debug("Proxying Request to %s for %s", solrUrl, apiKey)
r, err := http.Get(solrUrl)
if err != nil {
//Set actual error page here
l4g.Error("Error: %s\n", err.String())
http.Error(w, "500 Internal Server Error", 500)
return
}
l4g.Debug("Tomcat response: %s", r.Status)
r.Write(w)
r.Body.Close()
}
if req.Method == "POST" {
header := req.Header
//Reject non-json data - This check can probably be done at the processing layer
ct := header["Content-Type"][0]
if ct != "application/json" {
l4g.Error("Unsupported Content type %s", ct)
http.Error(w, "Unsupported content type", 400)
return
}
//Handle length check
length, err := strconv.Atoi(header["Content-Length"][0])
if err != nil {
l4g.Error("Error converting content-length header: %s", err.String())
http.Error(w, "Internal Server Error", 500)
return
}
if length > 1024*1024 {
l4g.Error("Post too large: %d", length)
http.Error(w, "Post too large", 400)
return
}
l4g.Debug("Post content-length: %d", length)
body := make([]byte, length)
len, ok := io.ReadFull(req.Body, body)
if ok != nil {
l4g.Debug("error reading io.ReadFull: %s", ok.String())
http.Error(w, "Internal Server Error", 500)
return
}
l4g.Debug("Content-length of %d, but read %d bytes from body", length, len)
var message SolrPost
if ok := json.Unmarshal(body, &message); ok != nil {
l4g.Debug("Error unmarshalling json: %s", ok.String())
http.Error(w, "Internal Server Error", 500)
return
}
l4g.Debug("JSON message body: %s", message.Body)
//Handle Auth
if solrServers[apiKey]["authstring"] != message.Authkey {
l4g.Error("Incorrect authkey")
http.Error(w, "Unauthorized", 401)
return
}
//Post message to queue
nc, err := net.Dial("tcp", ""+config["stomp"]["host"])
if err != nil {
l4g.Error("Error conneceting to queue %s", err.String())
http.Error(w, "Internal Server Error", 500)
return
}
//Note, the stomp module doesn't return os.Error on Send()
c, _ := stomp.Connect(nc, nil)
queue := "/queue/" + apiKey
l4g.Debug("Posting %s to queue %s", message.Body, queue)
nh := stomp.Header{}
c.Send(queue, nh, message.Body)
c.Disconnect(nh)
//Return result to client
http.Error(w, "ok", 200)
}
}