// openPorts create ZMQ sockets and start socket monitoring loops
func openPorts() {
filePort, err = utils.CreateInputPort("readfile.file", *fileEndpoint, fileCh)
utils.AssertError(err)
outPort, err = utils.CreateOutputPort("readfile.out", *outputEndpoint, outCh)
utils.AssertError(err)
if *errorEndpoint != "" {
errPort, err = utils.CreateOutputPort("readfile.err", *errorEndpoint, errCh)
utils.AssertError(err)
}
}
// openPorts create ZMQ sockets and start socket monitoring loops
func openPorts() {
inPort, err = utils.CreateInputPort("fs/walk.dir", *inputEndpoint, inCh)
utils.AssertError(err)
outPort, err = utils.CreateOutputPort("fs/walk.file", *outputEndpoint, outCh)
utils.AssertError(err)
if *errorEndpoint != "" {
errPort, err = utils.CreateOutputPort("fs/walk.err", *errorEndpoint, errCh)
utils.AssertError(err)
}
}
// openPorts create ZMQ sockets and start socket monitoring loops
func openPorts() {
cmdPort, err = utils.CreateInputPort("exec.cmd", *cmdEndpoint, cmdCh)
utils.AssertError(err)
if *outputEndpoint != "" {
outPort, err = utils.CreateOutputPort("exec.out", *outputEndpoint, outCh)
utils.AssertError(err)
}
if *errorEndpoint != "" {
errPort, err = utils.CreateOutputPort("exec.err", *errorEndpoint, errCh)
utils.AssertError(err)
}
}
func openPorts() {
optsPort, err = utils.CreateInputPort("mqtt-property.options", *optionsEndpoint, nil)
utils.AssertError(err)
tmplPort, err = utils.CreateInputPort("mqtt-property.template", *templateEndpoint, nil)
utils.AssertError(err)
propPort, err = utils.CreateOutputPort("mqtt-property.property", *propertyEndpoint, outCh)
utils.AssertError(err)
if *errorEndpoint != "" {
errPort, err = utils.CreateOutputPort("mqtt-property.err", *errorEndpoint, nil)
utils.AssertError(err)
}
}
// openPorts create ZMQ sockets and start socket monitoring loops
func openPorts() {
inPort, err = utils.CreateInputPort("passthru.in", *inputEndpoint, inCh)
utils.AssertError(err)
outPort, err = utils.CreateOutputPort("passthru.out", *outputEndpoint, outCh)
utils.AssertError(err)
}
// openPorts create ZMQ sockets and start socket monitoring loops
func openPorts() {
inPort, err = utils.CreateInputPort("debug/crasher.in", *inputEndpoint, inCh)
utils.AssertError(err)
outPort, err = utils.CreateOutputPort("debug/crasher.out", *outputEndpoint, outCh)
utils.AssertError(err)
}
// openPorts create ZMQ sockets and start socket monitoring loops
func openPorts() {
intervalPort, err = utils.CreateInputPort("ticker.interval", *intervalEndpoint, nil)
utils.AssertError(err)
outPort, err = utils.CreateOutputPort("ticker.out", *outputEndpoint, outCh)
utils.AssertError(err)
}
// openPorts create ZMQ sockets and start socket monitoring loops
func openPorts() {
inPort, err = utils.CreateInputPort("", *inputEndpoint, nil)
utils.AssertError(err)
outPort, err = utils.CreateOutputPort("bonjour/discover.out", *outputEndpoint, outCh)
utils.AssertError(err)
}
// openPorts create ZMQ sockets and start socket monitoring loops
func openPorts() {
reqPort, err = utils.CreateInputPort("http/client.req", *requestEndpoint, reqCh)
utils.AssertError(err)
if *responseEndpoint != "" {
respPort, err = utils.CreateOutputPort("http/client.resp", *responseEndpoint, respCh)
utils.AssertError(err)
}
if *bodyEndpoint != "" {
bodyPort, err = utils.CreateOutputPort("http/client.body", *bodyEndpoint, bodyCh)
utils.AssertError(err)
}
if *errorEndpoint != "" {
errPort, err = utils.CreateOutputPort("http/client.err", *errorEndpoint, errCh)
utils.AssertError(err)
}
}
// openPorts create ZMQ sockets and start socket monitoring loops
func openPorts() {
inPort, err = utils.CreateInputPort("fs/watchdog.in", *inputEndpoint, nil)
utils.AssertError(err)
if *errorEndpoint != "" {
errPort, err = utils.CreateOutputPort("fs/watchdog.err", *errorEndpoint, errCh)
utils.AssertError(err)
}
}
func openPorts() {
requestPort, err = utils.CreateInputPort(*requestEndpoint)
utils.AssertError(err)
failPort, err = utils.CreateOutputPort(*failEndpoint)
utils.AssertError(err)
successes := strings.Split(*successEndpoint, ",")
successPorts = make(map[string]*zmq.Socket, len(successes))
var port *zmq.Socket
for i, endpoint := range patterns {
port, err = utils.CreateOutputPort(strings.TrimSpace(successes[i]))
utils.AssertError(err)
successPorts
}
}
// openPorts create ZMQ sockets and start socket monitoring loops
func openPorts() {
optionsPort, err = utils.CreateInputPort("distinct.options", *optionsEndpoint, nil)
utils.AssertError(err)
inPort, err = utils.CreateInputPort("distinct.in", *inputEndpoint, inCh)
utils.AssertError(err)
outPort, err = utils.CreateOutputPort("distinct.out", *outputEndpoint, outCh)
utils.AssertError(err)
}
func openPorts() {
// Data
dataports := strings.Split(*dataEndpoint, ",")
if len(dataports) == 0 {
flag.Usage()
os.Exit(1)
}
dataPortsArray = []*zmq.Socket{}
poller = zmq.NewPoller()
for i, endpoint := range dataports {
endpoint = strings.TrimSpace(endpoint)
log.Printf("Connecting DATA[%v]=%s", i, endpoint)
port, err := utils.CreateInputPort(fmt.Sprintf("context.data-%s", i), endpoint, inCh)
utils.AssertError(err)
dataPortsArray = append(dataPortsArray, port)
poller.Add(port, zmq.POLLIN)
}
// Template
tmplPort, err = utils.CreateInputPort("context.tmpl", *templateEndpoint, nil)
utils.AssertError(err)
// Update
if *updatedEndpoint != "" {
updPort, err = utils.CreateOutputPort("context.update", *updatedEndpoint, outCh)
utils.AssertError(err)
}
// Match
if *matchedEndpoint != "" {
matchPort, err = utils.CreateOutputPort("context.match", *matchedEndpoint, outCh)
utils.AssertError(err)
}
// Error
if *errorEndpoint != "" {
errPort, err = utils.CreateOutputPort("context.err", *errorEndpoint, nil)
utils.AssertError(err)
}
}
// openPorts create ZMQ sockets and start socket monitoring loops
func openPorts() {
delayPort, err = utils.CreateInputPort("delay.delay", *delayEndpoint, nil)
utils.AssertError(err)
inPort, err = utils.CreateInputPort("delay.in", *inputEndpoint, inCh)
utils.AssertError(err)
outPort, err = utils.CreateOutputPort("delay.out", *outputEndpoint, outCh)
utils.AssertError(err)
}
// openPorts create ZMQ sockets and start socket monitoring loops
func openPorts() {
patternPort, err = utils.CreateInputPort("submatch.pattern", *patternEndpoint, nil)
utils.AssertError(err)
inPort, err = utils.CreateInputPort("submatch.in", *inputEndpoint, inCh)
utils.AssertError(err)
mapPort, err = utils.CreateOutputPort("submatch.map", *mapEndpoint, mapCh)
utils.AssertError(err)
}
// openPorts create ZMQ sockets and start socket monitoring loops
func openPorts() {
tplPort, err = utils.CreateInputPort("template.tpl", *tplEndpoint, nil)
utils.AssertError(err)
inPort, err = utils.CreateInputPort("template.in", *inputEndpoint, inCh)
utils.AssertError(err)
outPort, err = utils.CreateOutputPort("template.out", *outputEndpoint, outCh)
utils.AssertError(err)
}
func openPorts() {
var err error
// Input sockets
// Interval socket
intPort, err = utils.CreateInputPort("http-property.int", *intervalEndpoint, nil)
utils.AssertError(err)
// Request socket
reqPort, err = utils.CreateInputPort("http-property.req", *requestEndpoint, nil)
utils.AssertError(err)
// Property template socket
tmplPort, err = utils.CreateInputPort("http-property.tmpl", *templateEndpoint, nil)
utils.AssertError(err)
// Output sockets
// Property socket
if *propertyEndpoint != "" {
propPort, err = utils.CreateOutputPort("http-property.prop", *propertyEndpoint, outCh)
utils.AssertError(err)
}
// Response socket
if *responseEndpoint != "" {
respPort, err = utils.CreateOutputPort("http-property.resp", *responseEndpoint, outCh)
utils.AssertError(err)
}
// Response body socket
if *bodyEndpoint != "" {
bodyPort, err = utils.CreateOutputPort("http-property.body", *bodyEndpoint, outCh)
utils.AssertError(err)
}
// Error socket
if *errorEndpoint != "" {
errPort, err = utils.CreateOutputPort("http-property.err", *errorEndpoint, outCh)
utils.AssertError(err)
}
}
// openPorts create ZMQ sockets and start socket monitoring loops
func openPorts() {
optionsPort, err = utils.CreateInputPort("influxdb/write.options", *optionsEndpoint, nil)
utils.AssertError(err)
inPort, err = utils.CreateInputPort("influxdb/write.in", *inputEndpoint, inCh)
utils.AssertError(err)
if *errorEndpoint != "" {
errPort, err = utils.CreateOutputPort("influxdb/write.err", *errorEndpoint, errCh)
utils.AssertError(err)
}
}
// openPorts create ZMQ sockets and start socket monitoring loops
func openPorts() {
outports := strings.Split(*outputEndpoint, ",")
if len(outports) == 0 {
flag.Usage()
os.Exit(1)
}
inPort, err = utils.CreateInputPort("splitter.in", *inputEndpoint, inCh)
utils.AssertError(err)
outPortArray = []*zmq.Socket{}
for i, endpoint := range outports {
endpoint = strings.TrimSpace(endpoint)
log.Printf("Connecting OUT[%v]=%s", i, endpoint)
port, err = utils.CreateOutputPort(fmt.Sprintf("splitter.out[%v]", i), endpoint, outCh)
outPortArray = append(outPortArray, port)
}
}
// openPorts create ZMQ sockets and start socket monitoring loops
func openPorts() {
inports := strings.Split(*inputEndpoint, ",")
if len(inports) == 0 {
flag.Usage()
exitCh <- syscall.SIGTERM
return
}
inPortArray = []*zmq.Socket{}
poller = zmq.NewPoller()
for i, endpoint := range inports {
endpoint = strings.TrimSpace(endpoint)
log.Printf("Binding IN[%v]=%s", i, endpoint)
port, err = utils.CreateInputPort(fmt.Sprintf("joiner.in[%v]", i), endpoint, inCh)
utils.AssertError(err)
inPortArray = append(inPortArray, port)
poller.Add(port, zmq.POLLIN)
}
outPort, err = utils.CreateOutputPort("joiner.out", *outputEndpoint, outCh)
utils.AssertError(err)
}
func main() {
flag.Parse()
if *jsonFlag {
doc, _ := registryEntry.JSON()
fmt.Println(string(doc))
os.Exit(0)
}
log.SetFlags(0)
if *debug {
log.SetOutput(os.Stdout)
} else {
log.SetOutput(ioutil.Discard)
}
validateArgs()
openPorts()
defer closePorts()
exitCh := utils.HandleInterruption()
// Wait for the configuration on the options port
var bindAddr string
for {
log.Println("Waiting for configuration...")
ip, err := optionsPort.RecvMultipart(0)
if err != nil {
log.Println("Error receiving IP:", err.Error())
continue
}
if !runtime.IsValidIP(ip) || !runtime.IsPacket(ip) {
continue
}
bindAddr = string(ip[1])
break
}
optionsPort.Close()
// Data from http handler and data to http handler
inCh := make(chan httputils.HTTPResponse)
outCh := make(chan HandlerRequest)
go func(ctx *zmq.Context, endpoint string) {
outPort, err = utils.CreateOutputPort(context, endpoint)
utils.AssertError(err)
// Map of uuid to requests
dataMap := make(map[string]chan httputils.HTTPResponse)
// Start listening in/out channels
for {
select {
case data := <-outCh:
dataMap[data.Request.Id] = data.ResponseCh
ip, _ := httputils.Request2IP(data.Request)
outPort.SendMultipart(ip, 0)
case resp := <-inCh:
if respCh, ok := dataMap[resp.Id]; ok {
log.Println("Resolved channel for response", resp.Id)
respCh <- resp
delete(dataMap, resp.Id)
continue
}
log.Println("Didn't find request handler mapping for a given ID", resp.Id)
}
}
}(context, *outputEndpoint)
// Web server goroutine
go func() {
mux := http.NewServeMux()
mux.HandleFunc("/", Handler(outCh))
s := &http.Server{
Handler: mux,
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
MaxHeaderBytes: 1 << 20,
}
ln, err := net.Listen("tcp", bindAddr)
if err != nil {
log.Println(err.Error())
exitCh <- syscall.SIGTERM
return
}
log.Printf("Starting listening %v", bindAddr)
err = s.Serve(ln)
if err != nil {
log.Println(err.Error())
exitCh <- syscall.SIGTERM
return
}
}()
// Process incoming message forever
for {
ip, err := inPort.RecvMultipart(0)
if err != nil {
log.Println("Error receiving message:", err.Error())
continue
}
if !runtime.IsValidIP(ip) {
log.Println("Received invalid IP")
continue
}
resp, err := httputils.IP2Response(ip)
if err != nil {
log.Printf("Error converting IP to response: %s", err.Error())
continue
}
inCh <- *resp
}
}
// mainLoop initiates all ports and handles the traffic
func mainLoop() {
service := NewService()
openPorts()
defer closePorts()
go func() {
outPort, err = utils.CreateOutputPort("tcp/server.out", *outputEndpoint, outCh)
utils.AssertError(err)
for data := range service.Output {
outPort.SendMessage(runtime.NewOpenBracket())
outPort.SendMessage(runtime.NewPacket(data[0]))
outPort.SendMessage(runtime.NewPacket(data[1]))
outPort.SendMessage(runtime.NewCloseBracket())
}
}()
waitCh := make(chan bool)
go func() {
total := 0
for {
select {
case v := <-inCh:
if !v {
log.Println("IN port is closed. Interrupting execution")
exitCh <- syscall.SIGTERM
break
} else {
total++
}
case v := <-outCh:
if !v {
log.Println("OUT port is closed. Interrupting execution")
exitCh <- syscall.SIGTERM
break
} else {
total++
}
}
if total >= 2 && waitCh != nil {
waitCh <- true
}
}
}()
log.Println("Waiting for port connections to establish... ")
select {
case <-waitCh:
log.Println("Ports connected")
waitCh = nil
case <-time.Tick(30 * time.Second):
log.Println("Timeout: port connections were not established within provided interval")
exitCh <- syscall.SIGTERM
return
}
// Wait for the configuration on the options port
log.Println("Waiting for configuration...")
var bindAddr string
for {
ip, err := optionsPort.RecvMessageBytes(0)
if err != nil {
log.Println("Error receiving IP:", err.Error())
continue
}
if !runtime.IsValidIP(ip) || !runtime.IsPacket(ip) {
continue
}
bindAddr = string(ip[1])
break
}
optionsPort.Close()
// Create binding address listener
laddr, err := net.ResolveTCPAddr("tcp", bindAddr)
if err != nil {
log.Fatalln(err)
}
listener, err := net.ListenTCP("tcp", laddr)
if err != nil {
log.Fatalln(err)
}
log.Println("Listening on", listener.Addr())
go service.Serve(listener)
log.Println("Started...")
var (
connID string
data []byte
)
for {
ip, err := inPort.RecvMessageBytes(0)
if err != nil {
log.Println("Error receiving message:", err.Error())
continue
}
if !runtime.IsValidIP(ip) {
continue
}
switch {
case runtime.IsOpenBracket(ip):
connID = ""
data = nil
case runtime.IsPacket(ip):
if connID == "" {
connID = string(ip[1])
} else {
data = ip[1]
}
case runtime.IsCloseBracket(ip):
service.Dispatch(connID, data)
}
}
}
// mainLoop initiates all ports and handles the traffic
func mainLoop() {
// Setup watcher
watcher, err := fsnotify.NewWatcher()
if err != nil {
exitCh <- syscall.SIGTERM
return
}
defer watcher.Close()
openPorts()
defer closePorts()
// Process events
go func() {
// Socket to send messages to task sink
outPort, err = utils.CreateOutputPort("fs/watchdog.out", *outputEndpoint, errCh)
if err != nil {
exitCh <- syscall.SIGTERM
return
}
for {
select {
case ev := <-watcher.Event:
log.Println("Event:", ev)
if ev.IsCreate() {
if isDir(ev.Name) {
err = filepath.Walk(ev.Name, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
// we need to watch every subdirectory explicitely
watcher.Watch(path)
log.Println("Added to watch:", path)
} else {
// Consider every file found in the created directory as just created
outPort.SendMessage(runtime.NewPacket([]byte(path)))
}
return nil
})
if err != nil {
log.Println("Error walking directory:", err.Error())
}
} else {
outPort.SendMessage(runtime.NewPacket([]byte(ev.Name)))
}
} else if ev.IsDelete() && isDir(ev.Name) {
watcher.RemoveWatch(ev.Name)
log.Println("Removed from watch:", ev.Name)
}
case err := <-watcher.Error:
log.Println("Error:", err)
}
}
}()
go func() {
for {
select {
case v := <-outCh:
if !v {
log.Println("CREATED port is closed. Interrupting execution")
exitCh <- syscall.SIGTERM
break
}
case v := <-errCh:
if !v {
log.Println("ERR port is closed. Interrupting execution")
exitCh <- syscall.SIGTERM
break
}
}
}
}()
// Main loop
log.Println("Started")
for {
ip, err := inPort.RecvMessageBytes(0)
if err != nil {
continue
}
if !runtime.IsValidIP(ip) {
continue
}
dir := string(ip[1])
err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
watcher.Watch(path)
log.Println("Added to watch:", path)
}
return nil
})
if err != nil {
log.Printf("ERROR opening file %s: %s", dir, err.Error())
if errPort != nil {
errPort.SendMessage(runtime.NewPacket([]byte(err.Error())))
}
continue
}
}
}
// mainLoop initiates all ports and handles the traffic
func mainLoop() {
openPorts()
defer closePorts()
// Receiver routine
go func() {
inPort, err = utils.CreateInputPort("websocket/server.in", *inputEndpoint, inCh)
utils.AssertError(err)
defer inPort.Close()
for {
ip, err := inPort.RecvMessageBytes(0)
if err != nil {
continue
}
if !runtime.IsValidIP(ip) {
continue
}
msg, err := wsutils.IP2Message(ip)
if err != nil {
log.Println("Failed to convert IP to Message:", err.Error())
continue
}
log.Printf("Received response: %#v\n", msg)
DefaultHub.Outgoing <- *msg
}
}()
// Sender routine
go func() {
outPort, err = utils.CreateOutputPort("websocket/server.out", *outputEndpoint, outCh)
utils.AssertError(err)
defer outPort.Close()
for msg := range DefaultHub.Incoming {
log.Printf("Received data from connection: %#v\n", msg)
ip, err := wsutils.Message2IP(&msg)
if err != nil {
log.Println("Failed to convert Message to IP:", err.Error())
continue
}
outPort.SendMessageDontwait(ip)
}
}()
waitCh := make(chan bool)
go func() {
total := 0
for {
select {
case v := <-inCh:
if !v {
log.Println("IN port is closed. Interrupting execution")
exitCh <- syscall.SIGTERM
break
} else {
total++
}
case v := <-outCh:
if !v {
log.Println("OUT port is closed. Interrupting execution")
exitCh <- syscall.SIGTERM
break
} else {
total++
}
}
if total >= 2 && waitCh != nil {
waitCh <- true
}
}
}()
log.Println("Waiting for port connections to establish... ")
select {
case <-waitCh:
log.Println("Ports connected")
waitCh = nil
case <-time.Tick(30 * time.Second):
log.Println("Timeout: port connections were not established within provided interval")
exitCh <- syscall.SIGTERM
return
}
log.Println("Waiting for configuration...")
var bindAddr string
for {
ip, err := optionsPort.RecvMessageBytes(0)
if err != nil {
continue
}
if !runtime.IsValidIP(ip) || !runtime.IsPacket(ip) {
continue
}
bindAddr = string(ip[1])
break
}
optionsPort.Close()
// Configure & start websocket server
http.Handle("/", websocket.Handler(WebHandler))
go DefaultHub.Start()
// Listen & serve
log.Printf("Listening %v", bindAddr)
log.Printf("Web-socket endpoint: ws://%s/", bindAddr)
if err := http.ListenAndServe(bindAddr, nil); err != nil {
log.Fatal("ListenAndServe Error:", err)
}
}
