// main entry point
func main() {
fog.Info("back starts")
adminAddress := os.Getenv("NIMBUSIO_MANAGEMENT_API_REQUEST_DEST")
server := NewServer("admin", adminAddress)
server.Serve()
fog.Info("back ends")
}
// main entry point
func main() {
serviceDomain := os.Getenv("NIMBUS_IO_SERVICE_DOMAIN")
useTLS := os.Getenv("NIMBUS_IO_SERVICE_SSL") == "1"
fog.Info("front starts; service domain = %s, TLS = %t", serviceDomain, useTLS)
// testGET(serviceDomain, useTLS)
testPOST(serviceDomain, useTLS)
fog.Info("front ends")
}
// main entry point for data writer
func main() {
var err error
var writerSocket *zmq4.Socket
var eventSubSocket *zmq4.Socket
fog.Info("program starts")
if writerSocket, err = createWriterSocket(); err != nil {
fog.Critical("createWriterSocket %s", err)
}
defer writerSocket.Close()
fog.Info("binding writer socket to %s", dataWriterAddress)
if err = writerSocket.Bind(dataWriterAddress); err != nil {
fog.Critical("Bind(%s) %s", dataWriterAddress, err)
}
if eventSubSocket, err = createEventSubSocket(); err != nil {
fog.Critical("createEventSubSocket %s", err)
}
defer eventSubSocket.Close()
fog.Info("connecting event sub socket to %s", eventAggregatorPubAddress)
if err = eventSubSocket.Connect(eventAggregatorPubAddress); err != nil {
fog.Critical("Connect(%s) %s", eventAggregatorPubAddress, err)
}
messageChan := NewMessageHandler()
reactor := zmq4.NewReactor()
reactor.AddChannel(tools.NewSignalWatcher(), 1, tools.SigtermHandler)
reactor.AddSocket(writerSocket, zmq4.POLLIN,
NewWriterSocketHandler(writerSocket, messageChan))
reactor.AddSocket(eventSubSocket, zmq4.POLLIN,
NewEventSubSocketHandler(eventSubSocket))
fog.Debug("starting reactor.Run")
reactor.SetVerbose(true)
err = reactor.Run(reactorPollingInterval)
if err == tools.SigtermError {
fog.Info("program terminates normally due to SIGTERM")
} else if errno, ok := err.(syscall.Errno); ok {
// we can get 'interrupted system call' if we get SIGTERM while
// a socket is waiting on a read. That's not too bad.
if errno == syscall.EINTR {
fog.Warn("reactor.Run returns '%s' assuming SIGTERM", errno)
} else {
fog.Error("reactor.Run returns %T '%s'", errno, errno)
}
} else {
fog.Error("reactor.Run returns %T %s", err, err)
}
}
func init() {
// Issue #2
// if max procs is specfied in the environment, leave it
// otherwise set it to one less than the number of available cores
maxProcsStr := os.Getenv("GOMAXPROCS")
if maxProcsStr == "" {
maxProcs := runtime.NumCPU() - 1
if maxProcs < 1 {
maxProcs = 1
}
fog.Info("setting GOMAXPROCS to %d internally", maxProcs)
runtime.GOMAXPROCS(maxProcs)
} else {
fog.Info("GOMAXPROCS set to %s in environment", maxProcsStr)
}
}
func handleSignoff(message types.Message) {
returnAddress, err := msg.UnmarshalReturnAddress(message.Marshalled)
if err != nil {
fog.Error("handleSignoff: unable to unmarshall %s", err)
return
}
fog.Info("signoff from %s", returnAddress.ClientAddress)
}
func loadCertificate() (cert tls.Certificate, err error) {
certPath := os.Getenv("NIMBUSIO_WILDCARD_SSL_CERT")
keyPath := os.Getenv("NIMBUSIO_WILDCARD_SSL_KEY")
fog.Info("LoadX509KeyPair: certPath = '%s', keyPath = '%s'",
certPath, keyPath)
return tls.LoadX509KeyPair(certPath, keyPath)
}
func getListenAddress() (string, error) {
// set up the listener port
listenHost := os.Getenv("NIMBUSIO_WEB_DIRECTOR_ADDR")
listenPort := os.Getenv("NIMBUSIO_WEB_DIRECTOR_PORT")
fog.Info("NIMBUSIO_WEB_DIRECTOR_ADDR = '%s', NIMBUSIO_WEB_DIRECTOR_PORT = '%s'",
listenHost, listenPort)
return listenHost + ":" + listenPort, nil
}
func NewMemcacheClient() *memcache.Client {
memcacheHost := os.Getenv("NIMBUSIO_MEMCACHED_HOST")
if memcacheHost == "" {
memcacheHost = "localhost"
}
memcachePort := os.Getenv("NIMBUSIO_MEMCACHED_PORT")
if memcachePort == "" {
memcachePort = "11211"
}
memcacheAddress := fmt.Sprintf("%s:%s", memcacheHost, memcachePort)
fog.Info("NewMemcacheClient connecting to %s", memcacheAddress)
return memcache.New(memcacheAddress)
}
// NewManagementAPIDestinations returns an entity that implements the
// ManagementAPIDestinations interface
func NewManagementAPIDestinations() (ManagementAPIDestinations, error) {
var d managementAPIDestinations
destString := os.Getenv("NIMBUSIO_MANAGEMENT_API_REQUEST_DEST")
if destString == "" {
return nil, fmt.Errorf("No value for NIMBUSIO_MANAGEMENT_API_REQUEST_DEST")
}
d.destHosts = strings.Split(destString, " ")
if len(d.destHosts) == 0 {
return nil, fmt.Errorf("too few NIMBUSIO_MANAGEMENT_API_REQUEST_DEST")
}
fog.Info("NIMBUSIO_MANAGEMENT_API_REQUEST_DEST = '%s'", destString)
return &d, nil
}
func NewReplyHandler() chan<- *ReplyMessage {
replyChan := make(chan *ReplyMessage, replyChanCapacity)
pushSockets := make(map[string]*zmq4.Socket)
go func() {
for replyMessage := range replyChan {
marshalledReply, err := json.Marshal(replyMessage.Content)
if err != nil {
fog.Error("unable to marshall reply %s %s", replyMessage, err)
continue
}
var pushSocket *zmq4.Socket
var ok bool
pushSocket, ok = pushSockets[replyMessage.ClientAddress]
if !ok {
fog.Info("creating PUSH socket to %s", replyMessage.ClientAddress)
if pushSocket, err = createPushSocket(); err != nil {
fog.Error("Unable to create PUSH socket for %s %s",
replyMessage.ClientAddress, err)
continue
}
if err = pushSocket.Connect(replyMessage.ClientAddress); err != nil {
fog.Error("Unable to Connect PUSH socket to %s %s",
replyMessage.ClientAddress, err)
pushSocket.Close()
continue
}
pushSockets[replyMessage.ClientAddress] = pushSocket
}
if _, err = pushSocket.SendMessage(marshalledReply); err != nil {
fog.Error("pushSocket SendMessage to %s failed %s",
replyMessage.ClientAddress, err)
pushSocket.Close()
delete(pushSockets, replyMessage.ClientAddress)
continue
}
}
}()
return replyChan
}
// Availability returns an entity that implements the Availability
// interface. .
func NewAvailability() (Availability, error) {
var a availability
var hostName string
var err error
if hostName, err = os.Hostname(); err != nil {
return a, err
}
a.RedisWebMonitorHash = fmt.Sprintf("nimbus.io.web_monitor.%s", hostName)
a.HostResolver = tools.NewHostResolver()
if a.HostAddress, err = a.HostResolver.Lookup(hostName); err != nil {
return a, err
}
fog.Info("Availability for host %s %s", hostName, a.HostAddress)
return a, nil
}
// NewOutputValueFile creates an entity implmenting the OutputValueFile interface
func NewOutputValueFile(fileSpaceInfo tools.FileSpaceInfo) (OutputValueFile, error) {
var valueFile outputValueFile
var err error
valueFile.creationTime = tools.Timestamp()
valueFile.md5Sum = md5.New()
valueFile.collectionIDSet = make(map[uint32]struct{})
repositoryPath := os.Getenv("NIMBUSIO_REPOSITORY_PATH")
if valueFile.spaceID, err = fileSpaceInfo.FindMaxAvailSpaceID(tools.FileSpaceJournal); err != nil {
return nil, err
}
if err = valueFile.insertValueFileRow(); err != nil {
return nil, err
}
valueFile.filePath = tools.ComputeValueFilePath(repositoryPath, valueFile.spaceID,
valueFile.valueFileID)
fog.Debug("NewOutputValueFile %s", valueFile.filePath)
dirPath := path.Dir(valueFile.filePath)
if err = os.MkdirAll(dirPath, os.ModeDir|0755); err != nil {
return nil, fmt.Errorf("os.MkdirAll(%s...", err)
}
valueFile.fileHandle, err = os.Create(valueFile.filePath)
if err != nil {
return nil, fmt.Errorf("os.Create(%s) %s", valueFile.filePath, err)
}
err = syscall.Fallocate(int(valueFile.fileHandle.Fd()), 0, 0,
int64(MaxValueFileSize))
if err != nil {
return nil, fmt.Errorf("Fallocate failed %s", err)
}
valueFile.enableFsync = os.Getenv("NIMBUSIO_ENABLE_FSYNC") == "1"
fog.Info("NewOutputValueFile: NIMBUSIO_ENABLE_FSYNC = %t", valueFile.enableFsync)
return &valueFile, nil
}
func (s serverImpl) Serve() {
fog.Info("(%s) listening to %s", s.Name, s.Address)
http.HandleFunc(fmt.Sprintf("/%s", s.Name), s.handleAll)
http.ListenAndServe(s.Address, nil)
}
func handleStoreSequence(state *writerState, request requestStoreSequence) {
userRequestID := request.UserRequestID
segment := request.Segment
sequence := request.Sequence
data := request.Data
var err error
var md5Digest []byte
var offset uint64
fog.Debug("%s StoreSequence #%d", userRequestID, sequence.SequenceNum)
if state.ValueFile.Size()+sequence.SegmentSize >= MaxValueFileSize {
fog.Info("value file full")
if state.SyncTimer != nil {
state.SyncTimer.Stop()
}
state.SyncTimer = nil
if err = state.ValueFile.Close(); err != nil {
request.resultChan <- storeSequenceResult{Err: fmt.Errorf("error closing value file %s", err)}
return
}
if state.ValueFile, err = NewOutputValueFile(state.FileSpaceInfo); err != nil {
request.resultChan <- storeSequenceResult{Err: fmt.Errorf("error opening value file %s", err)}
return
}
startSyncTimer(state)
}
md5Digest, err = base64.StdEncoding.DecodeString(sequence.EncodedSegmentMD5Digest)
if err != nil {
request.resultChan <- storeSequenceResult{Err: err}
return
}
key := segmentKey{segment.UnifiedID, segment.ConjoinedPart,
segment.SegmentNum}
entry, ok := state.SegmentMap[key]
if !ok {
request.resultChan <- storeSequenceResult{Err: fmt.Errorf("StoreSequence unknown segment %s", key)}
return
}
offset, err = state.ValueFile.Store(segment.CollectionID, entry.SegmentID,
data)
if err != nil {
request.resultChan <- storeSequenceResult{Err: fmt.Errorf("ValueFile.Store %s", err)}
return
}
stmt := nodedb.Stmts["new-segment-sequence"]
_, err = stmt.Exec(
segment.CollectionID,
entry.SegmentID,
sequence.ZfecPaddingSize,
state.ValueFile.ID(),
sequence.SequenceNum,
offset,
sequence.SegmentSize,
md5Digest,
sequence.SegmentAdler32)
if err != nil {
request.resultChan <- storeSequenceResult{Err: fmt.Errorf("new-segment-sequence %s", err)}
}
state.StatGrabber.Accumulate("nimbusio_write_requests", 1)
state.StatGrabber.Accumulate("nimbusio_write_bytes", len(data))
entry.LastActionTime = tools.Timestamp()
state.SegmentMap[key] = entry
request.resultChan <- storeSequenceResult{ValueFileID: state.ValueFile.ID()}
}
// handleConnection manages one HTTP connection
// expected to be run in a goroutine
func handleConnection(router routing.Router, conn net.Conn) {
defer conn.Close()
const bufferSize = 64 * 1024
var err error
requestID, err := tools.CreateUUID()
if err != nil {
fog.Error("%s tools.CreateUUID(): %s", conn.RemoteAddr().String(), err)
return
}
fog.Info("%s starts %s", requestID, conn.RemoteAddr().String())
request, err := http.ReadRequest(bufio.NewReaderSize(conn, bufferSize))
if err != nil {
fog.Error("%s %s ReadRequest failed: %s", requestID,
conn.RemoteAddr().String(), err)
fog.Info("%s aborts", requestID)
return
}
// change the URL to point to our internal host
request.URL.Host, err = router.Route(requestID, request)
if err != nil {
routerErr, ok := err.(routing.RouterError)
if ok {
fog.Error("%s %s, %s router error: %s",
requestID, request.Method, request.URL, err)
sendErrorReply(conn, routerErr.HTTPCode(), routerErr.ErrorMessage())
} else {
fog.Error("%s %s, %s Unexpected error type: %T %s",
requestID, request.Method, request.URL, err, err)
}
fog.Info("%s aborts", requestID)
return
}
request.URL.Scheme = "http"
// heave the incoming RequestURI: can't be set in a client request
request.RequestURI = ""
modifyHeaders(request, conn.RemoteAddr().String(), requestID)
fog.Debug("%s routing %s %s", requestID, request.Method, request.URL)
// TODO: cache the connection to the internal server
internalConn, err := net.Dial("tcp", request.URL.Host)
if err != nil {
fog.Error("%s %s, %s unable to dial internal server: %s",
requestID, request.Method, request.URL, err)
sendErrorReply(conn, http.StatusInternalServerError, err.Error())
fog.Info("%s aborts", requestID)
return
}
defer internalConn.Close()
err = request.Write(bufio.NewWriterSize(internalConn, bufferSize))
if err != nil {
fog.Error("%s %s, %s request.Write: %s",
requestID, request.Method, request.URL, err)
sendErrorReply(conn, http.StatusInternalServerError, err.Error())
fog.Info("%s aborts", requestID)
return
}
request.Body.Close()
response, err := http.ReadResponse(bufio.NewReaderSize(internalConn, bufferSize),
request)
if err != nil {
fog.Error("%s %s, %s http.ReadResponse: %s",
requestID, request.Method, request.URL, err)
sendErrorReply(conn, http.StatusInternalServerError, err.Error())
fog.Info("%s aborts", requestID)
return
}
if err := response.Write(bufio.NewWriterSize(conn, bufferSize)); err != nil {
fog.Error("%s %s, %s error sending response: %s",
requestID, request.Method, request.URL, err)
}
response.Body.Close()
fog.Info("%s ends (%d) %s", requestID, response.StatusCode, response.Status)
}
// main entry point for webdirector
func main() {
var err error
fog.Info("program starts")
// set up a signal handling channel
signalChannel := make(chan os.Signal)
signal.Notify(signalChannel, os.Interrupt, syscall.SIGTERM)
if profilePath := os.Getenv("WEBDIRECTOR_CPU_PROFILE_PATH"); profilePath != "" {
profileFile, err := os.Create(profilePath)
if err != nil {
fog.Critical("os.Create(%s) failed %s", profilePath, err)
}
fog.Info("writing CPU profile data to %s", profilePath)
pprof.StartCPUProfile(profileFile)
defer pprof.StopCPUProfile()
}
useTLS := os.Getenv("NIMBUS_IO_SERVICE_SSL") == "1"
fog.Info("TLS = %t", useTLS)
listenAddress, err := getListenAddress()
if err != nil {
fog.Critical("error getListenAddress %s", err)
}
fog.Info("webdirector listens to %s", listenAddress)
var listener net.Listener
if useTLS {
if listener, err = getTLSListener(listenAddress); err != nil {
fog.Critical("Unable to create TLS listener: %s", err)
}
} else {
if listener, err = getTCPListener(listenAddress); err != nil {
fog.Critical("Unable to create TCP listener: %s", err)
}
}
managmentAPIDests, err := mgmtapi.NewManagementAPIDestinations()
if err != nil {
fog.Critical("NewManagementAPIDestinations: %s", err)
}
centralDB := centraldb.NewCentralDB()
availableHosts, err := avail.NewAvailability()
if err != nil {
fog.Critical("NewAvailability: %s", err)
}
router := routing.NewRouter(managmentAPIDests, centralDB, availableHosts)
fog.Info("NIMBUS_IO_SERVICE_DOMAIN = '%s'",
os.Getenv("NIMBUS_IO_SERVICE_DOMAIN"))
fog.Info("NIMBUSIO_WEB_PUBLIC_READER_PORT = '%s'",
os.Getenv("NIMBUSIO_WEB_PUBLIC_READER_PORT"))
fog.Info("NIMBUSIO_WEB_WRITER_PORT = '%s'",
os.Getenv("NIMBUSIO_WEB_WRITER_PORT"))
listenerChan := make(chan net.Conn, listenerChanCapacity)
go func() {
for {
connection, err := listener.Accept()
if err != nil {
fog.Error("listener.Accept() %s", err)
close(listenerChan)
break
}
listenerChan <- connection
}
}()
for running := true; running; {
select {
case signal := <-signalChannel:
fog.Info("terminated by signal: %v", signal)
running = false
case conn, ok := <-listenerChan:
if ok {
fog.Info("connection from %s", conn.RemoteAddr().String())
go handleConnection(router, conn)
} else {
running = false
}
}
}
listener.Close()
fog.Info("program terminates")
}
// Info prepends literal 'INFO' to log message
func (l logData) Info(text string, args ...interface{}) {
fog.Info(l.prefix()+text, args...)
}
