func TestWebSocket_Query(t *testing.T) {
mux := http.NewServeMux()
h := Handler{Tx: newTransceiver()}
<-h.Register(mux)
s := httptest.NewServer(mux)
defer s.Close()
url := strings.Replace(s.URL, "http:", "ws:", 1)
ws, err := websocket.Dial(url+"/v1/ws/jsonrpc", "", "http://localhost")
if err != nil {
t.Fatal(err)
}
defer ws.Close()
cli := rpc.NewClientWithCodec(jsonrpc.NewClientCodec(ws))
args := &QueryMessage{
Src: "nobody",
MessageID: "foobar",
}
var resp QueryMessageResp
err = cli.Call("SM.Query", args, &resp)
if err != nil {
t.Fatal(err)
}
want := "DELIVERED"
if resp.MsgState != want {
t.Fatalf("unexpected message state: want %q, have %q",
want, resp.MsgState)
}
}
// RpcCodecClientWithPort creates a new RPC client codec with a given port.
func RpcCodecClientWithPort(port string) (rpc.ClientCodec, error) {
conn, err := net.Dial("tcp", "localhost:"+port)
if err != nil {
return nil, err
}
return jsonrpc.NewClientCodec(conn), nil
}
func TestWebSocket_Send(t *testing.T) {
mux := http.NewServeMux()
h := Handler{Tx: newTransceiver()}
<-h.Register(mux)
s := httptest.NewServer(mux)
defer s.Close()
url := strings.Replace(s.URL, "http:", "ws:", -1)
ws, err := websocket.Dial(url+"/v1/ws/jsonrpc", "", "http://localhost")
if err != nil {
t.Fatal(err)
}
defer ws.Close()
cli := rpc.NewClientWithCodec(jsonrpc.NewClientCodec(ws))
args := &ShortMessage{
Dst: "root",
Text: "hello world",
}
var resp ShortMessageResp
err = cli.Call("SM.Submit", args, &resp)
if err != nil {
t.Fatal(err)
}
want := "foobar"
if resp.MessageID != want {
t.Fatalf("unexpected message id: want %q, have %q",
want, resp.MessageID)
}
}
func sendBlock(dn *DataNodeState, blockID BlockID, peers []string) {
if err := dn.Manager.LockRead(blockID); err != nil {
log.Println("Couldn't lock", blockID)
return
}
defer dn.Manager.UnlockRead(blockID)
var peerConn net.Conn
var forwardTo []string
var err error
// Find an online peer
for i, addr := range peers {
peerConn, err = net.Dial("tcp", addr)
if err == nil {
forwardTo = append(peers[:i], peers[i+1:]...)
break
}
}
if peerConn == nil {
log.Println("Couldn't forward block",
blockID,
"to any DataNodes in:",
strings.Join(peers, " "))
return
}
peerCodec := jsonrpc.NewClientCodec(peerConn)
if Debug {
peerCodec = LoggingClientCodec(
peerConn.RemoteAddr().String(),
peerCodec)
}
peer := rpc.NewClientWithCodec(peerCodec)
defer peer.Close()
size, err := dn.Store.BlockSize(blockID)
if err != nil {
log.Fatal("Stat error: ", err)
}
err = peer.Call("Forward",
&ForwardBlock{blockID, forwardTo, size},
nil)
if err != nil {
log.Fatal("Forward error: ", err)
}
err = dn.Store.ReadBlock(blockID, peerConn)
if err != nil {
log.Fatal("Copying error: ", err)
}
hash, err := dn.Store.ReadChecksum(blockID)
if err != nil {
log.Fatalln("Reading checksum:", err)
}
err = peer.Call("Confirm", hash, nil)
if err != nil {
log.Fatal("Confirm error: ", err)
}
}
// WebSocket handler for JSON RPC events, we call the client.
func (h *Handler) wsrpcEvents() http.Handler {
type conn struct {
io.Reader
io.WriteCloser
}
f := func(ws *websocket.Conn) {
id, dr := h.pool.Register()
defer h.pool.Unregister(id)
stop := make(chan struct{})
r, w := io.Pipe()
defer w.Close()
go func() {
io.Copy(w, ws)
close(stop)
}()
rwc := &conn{Reader: r, WriteCloser: ws}
cli := rpc.NewClientWithCodec(jsonrpc.NewClientCodec(rwc))
for {
select {
case r := <-dr:
err := cli.Call("SM.Deliver", r, nil)
if err != nil {
return
}
case <-stop:
return
}
}
}
return auth(cors(websocket.Handler(f).ServeHTTP, "GET"))
}
func main() {
conn, err := net.Dial("unix", "/tmp/navc.sock")
if err != nil {
log.Fatal("dial socket", err)
}
defer conn.Close()
codec := jsonrpc.NewClientCodec(conn)
defer codec.Close()
client := rpc.NewClientWithCodec(codec)
defer client.Close()
// sample call
args := Symbol{"", "", "sample/a.c", 16, 2}
var reply Symbol
err = client.Call("RequestHandler.GetSymbolDecl",
&args,
&reply)
if err != nil {
log.Fatal("calling ", err)
}
log.Println(reply)
}
func publish(message *Message, address string) (int, error) {
conn, err := net.Dial("tcp", address)
if err != nil {
return 0, err
}
client := rpc.NewClientWithCodec(jsonrpc.NewClientCodec(conn))
if err != nil {
return 0, err
}
var reply int
err = client.Call("Registry.Announce", message, &reply)
client.Close()
return reply, err
}
// Fetch the initial registry from the address
func connect(registry *Registry, address string) error {
conn, err := net.Dial("tcp", address)
if err != nil {
return err
}
client := rpc.NewClientWithCodec(jsonrpc.NewClientCodec(conn))
var reply *NodeList
err = client.Call("Registry.Query", "", &reply)
if err != nil {
return err
}
client.Close()
MergeRegistries(registry, reply)
return nil
}
//
// Start a docker container, and create a connection to /attach to it and send
// and receive RPC commands.
//
func (d *Client) Start() (err error) {
path := os.Getenv("DOCKER_CERT_PATH")
if path != "" {
ca := fmt.Sprintf("%s/ca.pem", path)
cert := fmt.Sprintf("%s/cert.pem", path)
key := fmt.Sprintf("%s/key.pem", path)
d.dockerClient, err = docker.NewTLSClient(d.endpoint, cert, key, ca)
} else {
d.dockerClient, err = docker.NewClient(d.endpoint)
}
if err != nil {
return err
}
defaultConfig := &docker.Config{
OpenStdin: true,
Image: d.dockerImage,
}
defaultHostConfig := &docker.HostConfig{}
opts := docker.CreateContainerOptions{
Config: defaultConfig,
HostConfig: defaultHostConfig,
}
if d.name != "" {
opts.Name = d.name
}
c, err := d.dockerClient.CreateContainer(opts)
if err != nil {
return err
}
d.ID = c.ID
err = d.dockerClient.StartContainer(c.ID, defaultHostConfig)
if err != nil {
return err
}
attachOpts := docker.AttachToContainerOptions{
Container: d.ID,
Stdout: true,
Stdin: true,
Stderr: true,
Stream: true,
}
err = d.AttachStreamingContainer(attachOpts)
if err != nil {
return err
}
pipes := &dockerPipes{
d.clientConn,
&d.stdErrBuf,
0,
0,
}
d.rpcClient = rpc.NewClientWithCodec(jsonrpc.NewClientCodec(pipes))
return nil
}
func Upload(file *os.File, debug bool, leaderAddress string) string {
localFileInfo, err := file.Stat()
if err != nil {
log.Fatal("Stat error: ", err)
}
localFileSize := localFileInfo.Size()
conn, err := net.Dial("tcp", leaderAddress)
if err != nil {
log.Fatal("Dial error:", err)
}
defer conn.Close()
codec := jsonrpc.NewClientCodec(conn)
if debug {
codec = LoggingClientCodec(
conn.RemoteAddr().String(),
codec)
}
client := rpc.NewClientWithCodec(codec)
var blobId string
err = client.Call("CreateBlob", nil, &blobId)
if err != nil {
panic(err)
log.Fatalln("CreateBlob error:", err)
}
bytesLeft := localFileSize
for bytesLeft > 0 {
var nodesMsg ForwardBlock
err = client.Call("Append", nil, &nodesMsg)
if err != nil {
log.Fatal("Append error:", err)
}
blockSize := nodesMsg.Size
var dataNode net.Conn
var forwardTo []string
// Find a DataNode
for i, addr := range nodesMsg.Nodes {
dataNode, err = net.Dial("tcp", addr)
if err == nil {
forwardTo = append(nodesMsg.Nodes[:i], nodesMsg.Nodes[i+1:]...)
break
}
dataNode = nil
}
// TODO: Can't compare an interface to nil
if dataNode == nil {
log.Fatalln("Couldn't connect to any DataNodes in:", strings.Join(nodesMsg.Nodes, " "))
}
defer dataNode.Close()
dataNodeCodec := jsonrpc.NewClientCodec(dataNode)
if debug {
dataNodeCodec = LoggingClientCodec(
dataNode.RemoteAddr().String(),
dataNodeCodec)
}
dataNodeClient := rpc.NewClientWithCodec(dataNodeCodec)
var size int64
if blockSize > bytesLeft {
size = bytesLeft
bytesLeft = 0
} else {
size = blockSize
bytesLeft = bytesLeft - blockSize
}
err = dataNodeClient.Call("Forward",
&ForwardBlock{nodesMsg.BlockID, forwardTo, size},
nil)
if err != nil {
log.Fatal("ForwardBlock error: ", err)
}
hash := crc32.NewIEEE()
io.CopyN(dataNode, io.TeeReader(file, hash), size)
log.Println("Uploading block with checksum", fmt.Sprint(hash.Sum32()))
err = dataNodeClient.Call("Confirm", fmt.Sprint(hash.Sum32()), nil)
if err != nil {
log.Fatal("Confirm error: ", err)
}
}
err = client.Call("Commit", nil, nil)
if err != nil {
log.Fatal("Commit error:", err)
}
fmt.Println("Blob ID:", blobId)
return blobId
}
func (t *testClientCodec) NewClientCodec(r io.ReadWriteCloser) rpc.ClientCodec {
t.called = true
return jsonrpc.NewClientCodec(r)
}
func (control *Control) barrier() {
control.client_err = <-control.client_res
control.client_codec = jsonrpc.NewClientCodec(control.proxy)
control.client = rpc.NewClientWithCodec(control.client_codec)
}
func tick(dn *DataNodeState) {
conn, err := net.Dial("tcp", dn.LeaderAddress)
if err != nil {
log.Println("Couldn't connect to leader at", dn.LeaderAddress)
dn.NodeID = ""
return
}
codec := jsonrpc.NewClientCodec(conn)
if Debug {
codec = LoggingClientCodec(
conn.RemoteAddr().String(),
codec)
}
client := rpc.NewClientWithCodec(codec)
defer client.Close()
log.Println("Heartbeat...")
if len(dn.NodeID) == 0 {
log.Println("Re-reading blocklist")
blocks, err := dn.Store.ReadBlockList()
if err != nil {
log.Fatalln("Getting blocklist:", err)
}
for _, b := range blocks {
// Seems hacky
dn.Manager.exists[b] = true
}
err = client.Call("Register", &RegistrationMsg{dn.Addr, blocks}, &dn.NodeID)
if err != nil {
log.Println("Registration error:", err)
return
}
log.Println("Registered with ID:", dn.NodeID)
return
}
// Could be cached so we don't have to hit the filesystem
blocks, err := dn.Store.ReadBlockList()
if err != nil {
log.Fatalln("Getting utilization:", err)
}
spaceUsed := len(blocks)
newBlocks := dn.DrainNewBlocks()
deadBlocks := dn.DrainDeadBlocks()
var resp HeartbeatResponse
err = client.Call("Heartbeat",
HeartbeatMsg{dn.NodeID, spaceUsed, newBlocks, deadBlocks},
&resp)
if err != nil {
log.Println("Heartbeat error:", err)
dn.HaveBlocks(newBlocks)
dn.DontHaveBlocks(deadBlocks)
return
}
if resp.NeedToRegister {
log.Println("Re-registering with leader...")
dn.NodeID = ""
dn.HaveBlocks(newBlocks) // Try again next heartbeat
dn.DontHaveBlocks(deadBlocks)
return
}
for _, blockID := range resp.InvalidateBlocks {
dn.RemoveBlock(blockID)
}
go func() {
for _, fwd := range resp.ToReplicate {
log.Println("Will replicate '"+string(fwd.BlockID)+"' to", fwd.Nodes)
dn.forwardingBlocks <- fwd
}
}()
}
func main() {
flag.Parse()
search := make(map[string]interface{})
if *searchjson == "" {
if *query == "" {
fmt.Println("You must provide the --query param.")
os.Exit(1)
}
if *cod_conta == "" {
fmt.Println("You must provide the --cod_conta param.")
os.Exit(1)
}
if *cod_monitoramento == "0" && *cod_campanha == "0" {
fmt.Println("You must provide the --cod_monitoramento param.")
os.Exit(1)
}
if *cod_busca == "" {
fmt.Println("You must provide the --cod_busca param.")
os.Exit(1)
}
if *media == "" {
fmt.Println("You must provide the --media param.")
os.Exit(1)
}
search["LastItem"] = ""
search["FormattedResult"] = "0"
search["First"] = "1"
search["Feed"] = "http://livebuzz.com.br/dev/importapi/search/store"
search["Extid"] = fmt.Sprintf("%s.%s.%s.%s.%s", *cod_conta, *cod_monitoramento, *cod_campanha, *cod_busca, *media)
search["Limit"] = *limit
search["Media"] = *media
switch search["Media"] {
case "twitter":
search["Query"] = *query
search["Lang"] = "pt"
search["Geocode"] = ""
// search["__resource"] = `{"klout_api_key":"cxyaz95kx93ep6bf4zzt6676"}` // fabriciojs
case "1_2":
case "1_1":
{
// Twitter DMs retrieving
}
case "youtube":
search["Query"] = *query
search["Lang"] = "pt"
case "3":
search["Query"] = *query
// search["Locale"] = "pt_BR"
search["Locale"] = ""
case "flickr":
search["Query"] = *query
case "facebook":
search["Query"] = *query
search["Locale"] = "pt_BR"
search["Until"] = ""
case "6":
// Orkut
search["Settings"] = `{"Cmm":["4249"]}` // Brasil
// search["Settings"] = `{"Cmm":["33383"]}` // Unesp
search["Query"] = *query
case "7":
search["Query"] = *query
search["Locale"] = "br"
case "google_plus":
search["Query"] = *query
search["Lang"] = "pt"
case "rss":
search["Url"] = "{\"Urls\":[\"http://rss.terra.com.br/0,,EI306,00.xml\",\"http://rss.terra.com.br/0,,EI14416,00.xml\"]}"
search["Query"] = *query
case "facebook_pages":
// FB pages
search["Id"] = *query
case "foursquare":
// Foursquare
search["TypeSearch"] = `{"types":["tips","photos"]}`
search["Venue"] = "4d29b8ef3c795481133ada9b"
case "linkedin":
// Linkedin
search["Settings"] = `{"GroupId":["3937610"]}`
case "slideshare":
// Slideshare
search["Query"] = "Plano"
case "reclame_aqui":
// search["Query"] = "Plano"
search["Id"] = "7712"
case "instagram":
search["Query"] = *query
}
jsonBytes, _ := json.Marshal(search)
*searchjson = string(jsonBytes)
} else {
err := json.Unmarshal([]byte(*searchjson), &search)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
}
for {
// conn, err := net.Dial("tcp", "localhost:" + )
// fmt.Println("75.126.48.226:" + *search_port)
conn, err := net.Dial("tcp", "75.126.48.226:"+*search_port)
// conn, err := net.Dial("tcp", "localhost:" + *search_port)
if err != nil {
fmt.Printf("Could not connect: %s\n", err)
time.Sleep(time.Second)
continue
}
client := rpc.NewClientWithCodec(jsonrpc.NewClientCodec(conn))
var responseString string
client.Call("RPCSender.AddExternalSearch", searchjson, &responseString)
responseData := strings.Split(responseString, ";")
searchSpeed, _ := strconv.Atoi(responseData[2])
fmt.Println(responseString)
fmt.Println(responseData)
search["LastItem"] = responseData[1]
search["First"] = "0"
fmt.Printf("\n%s - sleep: %v\n", search["Extid"], time.Second*time.Duration(searchSpeed))
time.Sleep(time.Second * time.Duration(searchSpeed))
}
}
// Here's a test. It uploads 18 small blobs onto 2 data nodes, then starts
// 2 additional datanodes, then downloads the blobs.
// TODO:
// - Decommission nodes
// - Check that files are the same
// - Random data
// - Bigger blobs / more blocks
func TestIntegration(*testing.T) {
mdnClientListener, err := net.Listen("tcp", "[::1]:0")
if err != nil {
log.Fatal(err)
}
mdnClusterListener, err := net.Listen("tcp", "[::1]:0")
if err != nil {
log.Fatal(err)
}
_, _ = metadatanode.Create(metadatanode.Config{
ClientListener: mdnClientListener,
ClusterListener: mdnClusterListener,
ReplicationFactor: 2,
DatabaseFile: "metadata.test.db",
})
log.Println(mdnClusterListener.Addr().String())
dnListener1, err := net.Listen("tcp", ":0")
if err != nil {
log.Fatal(err)
}
_, _ = datanode.Create(datanode.Config{
Listener: dnListener1,
LeaderAddress: mdnClusterListener.Addr().String(),
DataDir: "_data",
HeartbeatInterval: 1 * time.Second,
})
dnListener2, err := net.Listen("tcp", ":0")
if err != nil {
log.Fatal(err)
}
_, _ = datanode.Create(datanode.Config{
Listener: dnListener2,
LeaderAddress: mdnClusterListener.Addr().String(),
DataDir: "_data2",
HeartbeatInterval: 1 * time.Second,
})
wg := new(sync.WaitGroup)
wg2 := new(sync.WaitGroup)
doneBalancing := new(sync.WaitGroup)
for _, _ = range make([]bool, 18) {
wg.Add(1)
wg2.Add(1)
doneBalancing.Add(1)
go func() {
file, err := os.Open("Makefile")
if err != nil {
panic(err)
}
blobID := upload.Upload(file, false, mdnClientListener.Addr().String())
wg.Done()
doneBalancing.Wait()
conn, err := net.Dial("tcp", mdnClientListener.Addr().String())
if err != nil {
log.Fatal("Dial error:", err)
}
defer conn.Close()
codec := jsonrpc.NewClientCodec(conn)
client := rpc.NewClientWithCodec(codec)
var blocks []BlockID
fmt.Println(blobID)
if err := client.Call("GetBlob", blobID, &blocks); err != nil {
log.Fatal("GetBlob error:", err)
}
fmt.Println(blocks)
for _, b := range blocks {
for {
conn, err := net.Dial("tcp", mdnClientListener.Addr().String())
if err != nil {
log.Fatal("Dial error:", err)
}
defer conn.Close()
codec := jsonrpc.NewClientCodec(conn)
client := rpc.NewClientWithCodec(codec)
var nodes []string
if err := client.Call("GetBlock", b, &nodes); err != nil {
log.Fatal("GetBlock error:", err)
}
fmt.Println(nodes)
sort.Sort(ByRandom(nodes))
conn, err = net.Dial("tcp", nodes[0])
if err != nil {
log.Fatalln("Dial error:", err)
}
defer conn.Close()
codec = jsonrpc.NewClientCodec(conn)
client = rpc.NewClientWithCodec(codec)
if err := client.Call("Get", b, nil); err != nil {
if fmt.Sprint(err) == "Couldn't get read lock" {
time.Sleep(100 * time.Millisecond)
continue
}
log.Fatalln(err)
}
if _, err := io.Copy(ioutil.Discard, conn); err != nil {
log.Fatalln(err)
}
break
}
}
wg2.Done()
}()
}
wg.Wait()
dnListener3, err := net.Listen("tcp", ":0")
if err != nil {
log.Fatal(err)
}
_, _ = datanode.Create(datanode.Config{
Listener: dnListener3,
LeaderAddress: mdnClusterListener.Addr().String(),
DataDir: "_data3",
HeartbeatInterval: 1 * time.Second,
})
dnListener4, err := net.Listen("tcp", ":0")
if err != nil {
log.Fatal(err)
}
_, _ = datanode.Create(datanode.Config{
Listener: dnListener4,
LeaderAddress: mdnClusterListener.Addr().String(),
DataDir: "_data4",
HeartbeatInterval: 1 * time.Second,
})
time.Sleep(5 * time.Second)
for _, _ = range make([]bool, 18) {
doneBalancing.Done()
}
wg2.Wait()
}