Exemplo n.º 1
0
func TestWebSocket_Query(t *testing.T) {
	mux := http.NewServeMux()
	h := Handler{Tx: newTransceiver()}
	<-h.Register(mux)
	s := httptest.NewServer(mux)
	defer s.Close()
	url := strings.Replace(s.URL, "http:", "ws:", 1)
	ws, err := websocket.Dial(url+"/v1/ws/jsonrpc", "", "http://localhost")
	if err != nil {
		t.Fatal(err)
	}
	defer ws.Close()
	cli := rpc.NewClientWithCodec(jsonrpc.NewClientCodec(ws))
	args := &QueryMessage{
		Src:       "nobody",
		MessageID: "foobar",
	}
	var resp QueryMessageResp
	err = cli.Call("SM.Query", args, &resp)
	if err != nil {
		t.Fatal(err)
	}
	want := "DELIVERED"
	if resp.MsgState != want {
		t.Fatalf("unexpected message state: want %q, have %q",
			want, resp.MsgState)
	}
}
Exemplo n.º 2
0
// RpcCodecClientWithPort creates a new RPC client codec with a given port.
func RpcCodecClientWithPort(port string) (rpc.ClientCodec, error) {
	conn, err := net.Dial("tcp", "localhost:"+port)
	if err != nil {
		return nil, err
	}
	return jsonrpc.NewClientCodec(conn), nil
}
Exemplo n.º 3
0
func TestWebSocket_Send(t *testing.T) {
	mux := http.NewServeMux()
	h := Handler{Tx: newTransceiver()}
	<-h.Register(mux)
	s := httptest.NewServer(mux)
	defer s.Close()
	url := strings.Replace(s.URL, "http:", "ws:", -1)
	ws, err := websocket.Dial(url+"/v1/ws/jsonrpc", "", "http://localhost")
	if err != nil {
		t.Fatal(err)
	}
	defer ws.Close()
	cli := rpc.NewClientWithCodec(jsonrpc.NewClientCodec(ws))
	args := &ShortMessage{
		Dst:  "root",
		Text: "hello world",
	}
	var resp ShortMessageResp
	err = cli.Call("SM.Submit", args, &resp)
	if err != nil {
		t.Fatal(err)
	}
	want := "foobar"
	if resp.MessageID != want {
		t.Fatalf("unexpected message id: want %q, have %q",
			want, resp.MessageID)
	}
}
func sendBlock(dn *DataNodeState, blockID BlockID, peers []string) {
	if err := dn.Manager.LockRead(blockID); err != nil {
		log.Println("Couldn't lock", blockID)
		return
	}
	defer dn.Manager.UnlockRead(blockID)

	var peerConn net.Conn
	var forwardTo []string
	var err error
	// Find an online peer
	for i, addr := range peers {
		peerConn, err = net.Dial("tcp", addr)
		if err == nil {
			forwardTo = append(peers[:i], peers[i+1:]...)
			break
		}
	}
	if peerConn == nil {
		log.Println("Couldn't forward block",
			blockID,
			"to any DataNodes in:",
			strings.Join(peers, " "))
		return
	}
	peerCodec := jsonrpc.NewClientCodec(peerConn)
	if Debug {
		peerCodec = LoggingClientCodec(
			peerConn.RemoteAddr().String(),
			peerCodec)
	}
	peer := rpc.NewClientWithCodec(peerCodec)
	defer peer.Close()

	size, err := dn.Store.BlockSize(blockID)
	if err != nil {
		log.Fatal("Stat error: ", err)
	}

	err = peer.Call("Forward",
		&ForwardBlock{blockID, forwardTo, size},
		nil)
	if err != nil {
		log.Fatal("Forward error: ", err)
	}

	err = dn.Store.ReadBlock(blockID, peerConn)
	if err != nil {
		log.Fatal("Copying error: ", err)
	}

	hash, err := dn.Store.ReadChecksum(blockID)
	if err != nil {
		log.Fatalln("Reading checksum:", err)
	}
	err = peer.Call("Confirm", hash, nil)
	if err != nil {
		log.Fatal("Confirm error: ", err)
	}
}
Exemplo n.º 5
0
// WebSocket handler for JSON RPC events, we call the client.
func (h *Handler) wsrpcEvents() http.Handler {
	type conn struct {
		io.Reader
		io.WriteCloser
	}
	f := func(ws *websocket.Conn) {
		id, dr := h.pool.Register()
		defer h.pool.Unregister(id)
		stop := make(chan struct{})
		r, w := io.Pipe()
		defer w.Close()
		go func() {
			io.Copy(w, ws)
			close(stop)
		}()
		rwc := &conn{Reader: r, WriteCloser: ws}
		cli := rpc.NewClientWithCodec(jsonrpc.NewClientCodec(rwc))
		for {
			select {
			case r := <-dr:
				err := cli.Call("SM.Deliver", r, nil)
				if err != nil {
					return
				}
			case <-stop:
				return
			}
		}
	}
	return auth(cors(websocket.Handler(f).ServeHTTP, "GET"))
}
Exemplo n.º 6
0
func main() {
	conn, err := net.Dial("unix", "/tmp/navc.sock")
	if err != nil {
		log.Fatal("dial socket", err)
	}
	defer conn.Close()

	codec := jsonrpc.NewClientCodec(conn)
	defer codec.Close()

	client := rpc.NewClientWithCodec(codec)
	defer client.Close()

	// sample call
	args := Symbol{"", "", "sample/a.c", 16, 2}
	var reply Symbol
	err = client.Call("RequestHandler.GetSymbolDecl",
		&args,
		&reply)
	if err != nil {
		log.Fatal("calling ", err)
	}

	log.Println(reply)
}
Exemplo n.º 7
0
func publish(message *Message, address string) (int, error) {
	conn, err := net.Dial("tcp", address)
	if err != nil {
		return 0, err
	}

	client := rpc.NewClientWithCodec(jsonrpc.NewClientCodec(conn))
	if err != nil {
		return 0, err
	}

	var reply int
	err = client.Call("Registry.Announce", message, &reply)

	client.Close()
	return reply, err
}
Exemplo n.º 8
0
// Fetch the initial registry from the address
func connect(registry *Registry, address string) error {
	conn, err := net.Dial("tcp", address)
	if err != nil {
		return err
	}
	client := rpc.NewClientWithCodec(jsonrpc.NewClientCodec(conn))

	var reply *NodeList
	err = client.Call("Registry.Query", "", &reply)
	if err != nil {
		return err
	}

	client.Close()

	MergeRegistries(registry, reply)
	return nil
}
Exemplo n.º 9
0
//
// Start a docker container, and create a connection to /attach to it and send
// and receive RPC commands.
//
func (d *Client) Start() (err error) {

	path := os.Getenv("DOCKER_CERT_PATH")
	if path != "" {
		ca := fmt.Sprintf("%s/ca.pem", path)
		cert := fmt.Sprintf("%s/cert.pem", path)
		key := fmt.Sprintf("%s/key.pem", path)
		d.dockerClient, err = docker.NewTLSClient(d.endpoint, cert, key, ca)
	} else {
		d.dockerClient, err = docker.NewClient(d.endpoint)
	}

	if err != nil {
		return err
	}

	defaultConfig := &docker.Config{
		OpenStdin: true,
		Image:     d.dockerImage,
	}

	defaultHostConfig := &docker.HostConfig{}

	opts := docker.CreateContainerOptions{
		Config:     defaultConfig,
		HostConfig: defaultHostConfig,
	}

	if d.name != "" {
		opts.Name = d.name
	}

	c, err := d.dockerClient.CreateContainer(opts)

	if err != nil {
		return err
	}

	d.ID = c.ID
	err = d.dockerClient.StartContainer(c.ID, defaultHostConfig)

	if err != nil {
		return err
	}

	attachOpts := docker.AttachToContainerOptions{
		Container: d.ID,
		Stdout:    true,
		Stdin:     true,
		Stderr:    true,
		Stream:    true,
	}

	err = d.AttachStreamingContainer(attachOpts)

	if err != nil {
		return err
	}

	pipes := &dockerPipes{
		d.clientConn,
		&d.stdErrBuf,
		0,
		0,
	}

	d.rpcClient = rpc.NewClientWithCodec(jsonrpc.NewClientCodec(pipes))

	return nil
}
func Upload(file *os.File, debug bool, leaderAddress string) string {
	localFileInfo, err := file.Stat()
	if err != nil {
		log.Fatal("Stat error: ", err)
	}
	localFileSize := localFileInfo.Size()

	conn, err := net.Dial("tcp", leaderAddress)
	if err != nil {
		log.Fatal("Dial error:", err)
	}
	defer conn.Close()
	codec := jsonrpc.NewClientCodec(conn)
	if debug {
		codec = LoggingClientCodec(
			conn.RemoteAddr().String(),
			codec)
	}
	client := rpc.NewClientWithCodec(codec)

	var blobId string
	err = client.Call("CreateBlob", nil, &blobId)
	if err != nil {
		panic(err)
		log.Fatalln("CreateBlob error:", err)
	}

	bytesLeft := localFileSize
	for bytesLeft > 0 {
		var nodesMsg ForwardBlock
		err = client.Call("Append", nil, &nodesMsg)
		if err != nil {
			log.Fatal("Append error:", err)
		}
		blockSize := nodesMsg.Size

		var dataNode net.Conn
		var forwardTo []string
		// Find a DataNode
		for i, addr := range nodesMsg.Nodes {
			dataNode, err = net.Dial("tcp", addr)
			if err == nil {
				forwardTo = append(nodesMsg.Nodes[:i], nodesMsg.Nodes[i+1:]...)
				break
			}
			dataNode = nil
		}
		// TODO: Can't compare an interface to nil
		if dataNode == nil {
			log.Fatalln("Couldn't connect to any DataNodes in:", strings.Join(nodesMsg.Nodes, " "))
		}
		defer dataNode.Close()

		dataNodeCodec := jsonrpc.NewClientCodec(dataNode)
		if debug {
			dataNodeCodec = LoggingClientCodec(
				dataNode.RemoteAddr().String(),
				dataNodeCodec)
		}
		dataNodeClient := rpc.NewClientWithCodec(dataNodeCodec)

		var size int64
		if blockSize > bytesLeft {
			size = bytesLeft
			bytesLeft = 0
		} else {
			size = blockSize
			bytesLeft = bytesLeft - blockSize
		}

		err = dataNodeClient.Call("Forward",
			&ForwardBlock{nodesMsg.BlockID, forwardTo, size},
			nil)
		if err != nil {
			log.Fatal("ForwardBlock error: ", err)
		}

		hash := crc32.NewIEEE()
		io.CopyN(dataNode, io.TeeReader(file, hash), size)

		log.Println("Uploading block with checksum", fmt.Sprint(hash.Sum32()))
		err = dataNodeClient.Call("Confirm", fmt.Sprint(hash.Sum32()), nil)
		if err != nil {
			log.Fatal("Confirm error: ", err)
		}
	}

	err = client.Call("Commit", nil, nil)
	if err != nil {
		log.Fatal("Commit error:", err)
	}
	fmt.Println("Blob ID:", blobId)

	return blobId
}
Exemplo n.º 11
0
func (t *testClientCodec) NewClientCodec(r io.ReadWriteCloser) rpc.ClientCodec {
	t.called = true
	return jsonrpc.NewClientCodec(r)
}
Exemplo n.º 12
0
func (control *Control) barrier() {
	control.client_err = <-control.client_res
	control.client_codec = jsonrpc.NewClientCodec(control.proxy)
	control.client = rpc.NewClientWithCodec(control.client_codec)
}
func tick(dn *DataNodeState) {
	conn, err := net.Dial("tcp", dn.LeaderAddress)
	if err != nil {
		log.Println("Couldn't connect to leader at", dn.LeaderAddress)
		dn.NodeID = ""
		return
	}
	codec := jsonrpc.NewClientCodec(conn)
	if Debug {
		codec = LoggingClientCodec(
			conn.RemoteAddr().String(),
			codec)
	}
	client := rpc.NewClientWithCodec(codec)
	defer client.Close()

	log.Println("Heartbeat...")
	if len(dn.NodeID) == 0 {
		log.Println("Re-reading blocklist")
		blocks, err := dn.Store.ReadBlockList()
		if err != nil {
			log.Fatalln("Getting blocklist:", err)
		}
		for _, b := range blocks {
			// Seems hacky
			dn.Manager.exists[b] = true
		}
		err = client.Call("Register", &RegistrationMsg{dn.Addr, blocks}, &dn.NodeID)
		if err != nil {
			log.Println("Registration error:", err)
			return
		}
		log.Println("Registered with ID:", dn.NodeID)
		return
	}

	// Could be cached so we don't have to hit the filesystem
	blocks, err := dn.Store.ReadBlockList()
	if err != nil {
		log.Fatalln("Getting utilization:", err)
	}
	spaceUsed := len(blocks)
	newBlocks := dn.DrainNewBlocks()
	deadBlocks := dn.DrainDeadBlocks()
	var resp HeartbeatResponse

	err = client.Call("Heartbeat",
		HeartbeatMsg{dn.NodeID, spaceUsed, newBlocks, deadBlocks},
		&resp)
	if err != nil {
		log.Println("Heartbeat error:", err)
		dn.HaveBlocks(newBlocks)
		dn.DontHaveBlocks(deadBlocks)
		return
	}
	if resp.NeedToRegister {
		log.Println("Re-registering with leader...")
		dn.NodeID = ""
		dn.HaveBlocks(newBlocks) // Try again next heartbeat
		dn.DontHaveBlocks(deadBlocks)
		return
	}
	for _, blockID := range resp.InvalidateBlocks {
		dn.RemoveBlock(blockID)
	}
	go func() {
		for _, fwd := range resp.ToReplicate {
			log.Println("Will replicate '"+string(fwd.BlockID)+"' to", fwd.Nodes)
			dn.forwardingBlocks <- fwd
		}
	}()
}
Exemplo n.º 14
0
func main() {
	flag.Parse()

	search := make(map[string]interface{})

	if *searchjson == "" {
		if *query == "" {
			fmt.Println("You must provide the --query param.")
			os.Exit(1)
		}

		if *cod_conta == "" {
			fmt.Println("You must provide the --cod_conta param.")
			os.Exit(1)
		}

		if *cod_monitoramento == "0" && *cod_campanha == "0" {
			fmt.Println("You must provide the --cod_monitoramento param.")
			os.Exit(1)
		}

		if *cod_busca == "" {
			fmt.Println("You must provide the --cod_busca param.")
			os.Exit(1)
		}

		if *media == "" {
			fmt.Println("You must provide the --media param.")
			os.Exit(1)
		}

		search["LastItem"] = ""
		search["FormattedResult"] = "0"
		search["First"] = "1"
		search["Feed"] = "http://livebuzz.com.br/dev/importapi/search/store"
		search["Extid"] = fmt.Sprintf("%s.%s.%s.%s.%s", *cod_conta, *cod_monitoramento, *cod_campanha, *cod_busca, *media)
		search["Limit"] = *limit
		search["Media"] = *media

		switch search["Media"] {
		case "twitter":
			search["Query"] = *query
			search["Lang"] = "pt"
			search["Geocode"] = ""
			// search["__resource"] = `{"klout_api_key":"cxyaz95kx93ep6bf4zzt6676"}` // fabriciojs
		case "1_2":
		case "1_1":
			{
				// Twitter DMs retrieving
			}
		case "youtube":
			search["Query"] = *query
			search["Lang"] = "pt"
		case "3":
			search["Query"] = *query
			// search["Locale"] = "pt_BR"
			search["Locale"] = ""
		case "flickr":
			search["Query"] = *query
		case "facebook":
			search["Query"] = *query
			search["Locale"] = "pt_BR"
			search["Until"] = ""
		case "6":
			// Orkut
			search["Settings"] = `{"Cmm":["4249"]}` // Brasil
			// search["Settings"] = `{"Cmm":["33383"]}` // Unesp
			search["Query"] = *query
		case "7":
			search["Query"] = *query
			search["Locale"] = "br"
		case "google_plus":
			search["Query"] = *query
			search["Lang"] = "pt"
		case "rss":
			search["Url"] = "{\"Urls\":[\"http://rss.terra.com.br/0,,EI306,00.xml\",\"http://rss.terra.com.br/0,,EI14416,00.xml\"]}"
			search["Query"] = *query
		case "facebook_pages":
			// FB pages
			search["Id"] = *query
		case "foursquare":
			// Foursquare
			search["TypeSearch"] = `{"types":["tips","photos"]}`
			search["Venue"] = "4d29b8ef3c795481133ada9b"
		case "linkedin":
			// Linkedin
			search["Settings"] = `{"GroupId":["3937610"]}`
		case "slideshare":
			// Slideshare
			search["Query"] = "Plano"
		case "reclame_aqui":
			// search["Query"] = "Plano"
			search["Id"] = "7712"
		case "instagram":
			search["Query"] = *query
		}

		jsonBytes, _ := json.Marshal(search)
		*searchjson = string(jsonBytes)
	} else {
		err := json.Unmarshal([]byte(*searchjson), &search)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	}

	for {
		// conn, err := net.Dial("tcp", "localhost:" + )
		// fmt.Println("75.126.48.226:" + *search_port)
		conn, err := net.Dial("tcp", "75.126.48.226:"+*search_port)
		// conn, err := net.Dial("tcp", "localhost:" + *search_port)

		if err != nil {
			fmt.Printf("Could not connect: %s\n", err)
			time.Sleep(time.Second)
			continue
		}

		client := rpc.NewClientWithCodec(jsonrpc.NewClientCodec(conn))

		var responseString string

		client.Call("RPCSender.AddExternalSearch", searchjson, &responseString)

		responseData := strings.Split(responseString, ";")

		searchSpeed, _ := strconv.Atoi(responseData[2])

		fmt.Println(responseString)
		fmt.Println(responseData)

		search["LastItem"] = responseData[1]
		search["First"] = "0"

		fmt.Printf("\n%s - sleep: %v\n", search["Extid"], time.Second*time.Duration(searchSpeed))

		time.Sleep(time.Second * time.Duration(searchSpeed))
	}
}
// Here's a test. It uploads 18 small blobs onto 2 data nodes, then starts
// 2 additional datanodes, then downloads the blobs.
// TODO:
//   - Decommission nodes
//   - Check that files are the same
//   - Random data
//   - Bigger blobs / more blocks
func TestIntegration(*testing.T) {

	mdnClientListener, err := net.Listen("tcp", "[::1]:0")
	if err != nil {
		log.Fatal(err)
	}
	mdnClusterListener, err := net.Listen("tcp", "[::1]:0")
	if err != nil {
		log.Fatal(err)
	}

	_, _ = metadatanode.Create(metadatanode.Config{
		ClientListener:    mdnClientListener,
		ClusterListener:   mdnClusterListener,
		ReplicationFactor: 2,
		DatabaseFile:      "metadata.test.db",
	})

	log.Println(mdnClusterListener.Addr().String())

	dnListener1, err := net.Listen("tcp", ":0")
	if err != nil {
		log.Fatal(err)
	}
	_, _ = datanode.Create(datanode.Config{
		Listener:          dnListener1,
		LeaderAddress:     mdnClusterListener.Addr().String(),
		DataDir:           "_data",
		HeartbeatInterval: 1 * time.Second,
	})

	dnListener2, err := net.Listen("tcp", ":0")
	if err != nil {
		log.Fatal(err)
	}
	_, _ = datanode.Create(datanode.Config{
		Listener:          dnListener2,
		LeaderAddress:     mdnClusterListener.Addr().String(),
		DataDir:           "_data2",
		HeartbeatInterval: 1 * time.Second,
	})

	wg := new(sync.WaitGroup)
	wg2 := new(sync.WaitGroup)
	doneBalancing := new(sync.WaitGroup)
	for _, _ = range make([]bool, 18) {
		wg.Add(1)
		wg2.Add(1)
		doneBalancing.Add(1)
		go func() {
			file, err := os.Open("Makefile")
			if err != nil {
				panic(err)
			}
			blobID := upload.Upload(file, false, mdnClientListener.Addr().String())

			wg.Done()
			doneBalancing.Wait()

			conn, err := net.Dial("tcp", mdnClientListener.Addr().String())
			if err != nil {
				log.Fatal("Dial error:", err)
			}
			defer conn.Close()
			codec := jsonrpc.NewClientCodec(conn)
			client := rpc.NewClientWithCodec(codec)

			var blocks []BlockID
			fmt.Println(blobID)
			if err := client.Call("GetBlob", blobID, &blocks); err != nil {
				log.Fatal("GetBlob error:", err)
			}
			fmt.Println(blocks)

			for _, b := range blocks {
				for {
					conn, err := net.Dial("tcp", mdnClientListener.Addr().String())
					if err != nil {
						log.Fatal("Dial error:", err)
					}
					defer conn.Close()
					codec := jsonrpc.NewClientCodec(conn)
					client := rpc.NewClientWithCodec(codec)

					var nodes []string
					if err := client.Call("GetBlock", b, &nodes); err != nil {
						log.Fatal("GetBlock error:", err)
					}
					fmt.Println(nodes)
					sort.Sort(ByRandom(nodes))

					conn, err = net.Dial("tcp", nodes[0])
					if err != nil {
						log.Fatalln("Dial error:", err)
					}
					defer conn.Close()
					codec = jsonrpc.NewClientCodec(conn)
					client = rpc.NewClientWithCodec(codec)
					if err := client.Call("Get", b, nil); err != nil {
						if fmt.Sprint(err) == "Couldn't get read lock" {
							time.Sleep(100 * time.Millisecond)
							continue
						}
						log.Fatalln(err)
					}

					if _, err := io.Copy(ioutil.Discard, conn); err != nil {
						log.Fatalln(err)
					}
					break
				}
			}

			wg2.Done()
		}()
	}

	wg.Wait()
	dnListener3, err := net.Listen("tcp", ":0")
	if err != nil {
		log.Fatal(err)
	}
	_, _ = datanode.Create(datanode.Config{
		Listener:          dnListener3,
		LeaderAddress:     mdnClusterListener.Addr().String(),
		DataDir:           "_data3",
		HeartbeatInterval: 1 * time.Second,
	})
	dnListener4, err := net.Listen("tcp", ":0")
	if err != nil {
		log.Fatal(err)
	}
	_, _ = datanode.Create(datanode.Config{
		Listener:          dnListener4,
		LeaderAddress:     mdnClusterListener.Addr().String(),
		DataDir:           "_data4",
		HeartbeatInterval: 1 * time.Second,
	})
	time.Sleep(5 * time.Second)
	for _, _ = range make([]bool, 18) {
		doneBalancing.Done()
	}
	wg2.Wait()
}