Example #1
0
func TestClient(t *testing.T) {
	t.Skipf("Test cluster is not set up properly")
	zookeeper.EnsureZkFatjar()
	basePath := ""
	tc, err := zklib.StartTestCluster(1, nil, nil)
	if err != nil {
		t.Fatalf("could not start test zk cluster: %s", err)
	}
	defer os.RemoveAll(tc.Path)
	defer tc.Stop()
	time.Sleep(time.Second)

	servers := []string{fmt.Sprintf("127.0.0.1:%d", tc.Servers[0].Port)}

	dsnBytes, err := json.Marshal(zookeeper.DSN{Servers: servers, Timeout: time.Second * 15})
	if err != nil {
		t.Fatal("unexpected error creating zk DSN: %s", err)
	}
	dsn := string(dsnBytes)
	zClient, err := client.New("zookeeper", dsn, basePath, nil)

	zzk.InitializeLocalClient(zClient)

	conn, err := zzk.GetLocalConnection("/")
	if err != nil {
		t.Fatal("unexpected error getting connection")
	}

	h := host.New()
	h.ID = "nodeID"
	h.IPAddr = "192.168.1.5"
	h.PoolID = "default1"
	defer func(old func(string, os.FileMode) error) {
		mkdirAll = old
	}(mkdirAll)
	dir, err := ioutil.TempDir("", "serviced_var_")
	if err != nil {
		t.Fatalf("could not create tempdir: %s", err)
	}
	defer os.RemoveAll(dir)
	c, err := NewClient(h, dir)
	if err != nil {
		t.Fatalf("unexpected error creating client: %s", err)
	}
	defer c.Close()
	time.Sleep(time.Second * 5)

	// therefore, we need to check that the client was added under the pool from root
	nodePath := fmt.Sprintf("/storage/clients/%s", h.IPAddr)
	glog.Infof("about to check for %s", nodePath)
	if exists, err := conn.Exists(nodePath); err != nil {
		t.Fatalf("did not expect error checking for existence of %s: %s", nodePath, err)
	} else {
		if !exists {
			t.Fatalf("could not find %s", nodePath)
		}
	}
}
//SetUpSuite is run before the tests to ensure elastic, zookeeper etc. are running.
func (dt *DaoTest) SetUpSuite(c *C) {
	docker.SetUseRegistry(true)

	dt.Port = 9202
	isvcs.Init()
	isvcs.Mgr.SetVolumesDir("/tmp/serviced-test")
	esServicedClusterName, _ := utils.NewUUID36()
	if err := isvcs.Mgr.SetConfigurationOption("elasticsearch-serviced", "cluster", esServicedClusterName); err != nil {
		c.Fatalf("Could not set elasticsearch-serviced clustername: %s", err)
	}
	esLogstashClusterName, _ := utils.NewUUID36()
	if err := isvcs.Mgr.SetConfigurationOption("elasticsearch-logstash", "cluster", esLogstashClusterName); err != nil {
		c.Fatalf("Could not set elasticsearch-logstash clustername: %s", err)
	}
	isvcs.Mgr.Wipe()
	if err := isvcs.Mgr.Start(); err != nil {
		c.Fatalf("Could not start es container: %s", err)
	}
	dt.MappingsFile = "controlplane.json"
	dt.FacadeTest.SetUpSuite(c)

	dsn := coordzk.NewDSN([]string{"127.0.0.1:2181"}, time.Second*15).String()
	glog.Infof("zookeeper dsn: %s", dsn)
	zClient, err := coordclient.New("zookeeper", dsn, "", nil)
	if err != nil {
		glog.Fatalf("Could not start es container: %s", err)
	}

	zzk.InitializeLocalClient(zClient)

	dt.zkConn, err = zzk.GetLocalConnection("/")
	if err != nil {
		c.Fatalf("could not get zk connection %v", err)
	}

	dt.Dao, err = NewControlSvc("localhost", int(dt.Port), dt.Facade, "/tmp", "rsync", 4979, time.Minute*5, "localhost:5000", MockStorageDriver{})
	if err != nil {
		glog.Fatalf("Could not start es container: %s", err)
	} else {
		for i := 0; i < 10; i += 1 {
			id := strconv.Itoa(i)
			dt.Dao.RemoveService(id, &unused)
		}
		for i := 100; i < 110; i += 1 {
			id := strconv.Itoa(i)
			dt.Dao.RemoveService(id, &unused)
		}
	}
}
Example #3
0
func TestServer(t *testing.T) {
	t.Skip() // the zookeeper part doesnt work in this test, but does work in real life
	zookeeper.EnsureZkFatjar()
	basePath := ""
	tc, err := zklib.StartTestCluster(1, nil, nil)
	if err != nil {
		t.Fatalf("could not start test zk cluster: %s", err)
	}
	defer os.RemoveAll(tc.Path)
	defer tc.Stop()
	time.Sleep(time.Second)

	servers := []string{fmt.Sprintf("127.0.0.1:%d", tc.Servers[0].Port)}

	dsnBytes, err := json.Marshal(zookeeper.DSN{Servers: servers, Timeout: time.Second * 15})
	if err != nil {
		t.Fatal("unexpected error creating zk DSN: %s", err)
	}
	dsn := string(dsnBytes)

	zClient, err := client.New("zookeeper", dsn, basePath, nil)
	if err != nil {
		t.Fatal("unexpected error getting zk client")
	}
	zzk.InitializeLocalClient(zClient)

	defer func(orig func(nfs.Driver, string, string) error) {
		nfsMount = orig
	}(nfsMount)

	var local, remote string
	nfsMount = func(driver nfs.Driver, a, b string) error {
		glog.Infof("client is mounting %s to %s", a, b)
		remote = a
		local = b
		return nil
	}

	// creating a UUID in order to make a unique poolID
	// the poolID is somehow being saved on the filesystem (zookeeper config somewhere?)
	// making the poolID unique on every run will ensure it is stateless
	uuid, err := utils.NewUUID()
	if err != nil {
		t.Fatal("New UUID could not be created")
	}

	hostServer := host.New()
	hostServer.ID = "nodeID"
	hostServer.IPAddr = "192.168.1.50"
	hostServer.PoolID = uuid

	hostClient1 := host.New()
	hostClient1.ID = "nodeID_client1"
	hostClient1.IPAddr = "192.168.1.100"
	hostClient1.PoolID = uuid

	mockNfsDriver := &mockNfsDriverT{
		exportPath: "/exports",
		exportName: "serviced_var",
	}

	// TODO: this gets stuck at server.go:90 call to conn.CreateDir hangs
	s, err := NewServer(mockNfsDriver, hostServer, path.Join(mockNfsDriver.exportPath, mockNfsDriver.exportName))
	if err != nil {
		t.Fatalf("unexpected error creating Server: %s", err)
	}

	conn, err := zzk.GetLocalConnection("/")
	if err != nil {
		t.Fatalf("unexpected error getting connection: %s", err)
	}
	shutdown := make(chan interface{})
	defer close(shutdown)
	go s.Run(shutdown, conn)

	// give it some time
	time.Sleep(time.Second * 5)

	if !mockNfsDriver.syncCalled {
		t.Fatalf("sync() should have been called by now")
	}
	if len(mockNfsDriver.clients) != 0 {
		t.Fatalf("Expected number of clients: 0 --- Found: %v (%v)", len(mockNfsDriver.clients), mockNfsDriver.clients)
	}
	mockNfsDriver.syncCalled = false
	tmpVar, err := ioutil.TempDir("", "serviced_var")
	if err != nil {
		t.Fatalf("could not create tempdir: %s", err)
	}
	defer os.RemoveAll(tmpVar)
	c1, err := NewClient(hostClient1, tmpVar)
	if err != nil {
		t.Fatalf("could not create client: %s", err)
	}
	// give it some time
	time.Sleep(time.Second * 2)
	if !mockNfsDriver.syncCalled {
		t.Fatalf("sync() should have been called by now")
	}

	if len(mockNfsDriver.clients) != 1 {
		t.Fatalf("expecting 1 client, got %d", len(mockNfsDriver.clients))
	}
	if mockNfsDriver.clients[0] != hostClient1.IPAddr {
		t.Fatalf("expecting '%s', got '%s'", hostServer.IPAddr, mockNfsDriver.clients[0])
	}

	shareName := fmt.Sprintf("%s:%s", hostServer.IPAddr, mockNfsDriver.ExportPath())
	if remote != shareName {
		t.Fatalf("remote should be %s, not %s", remote, shareName)
	}

	glog.Info("about to call c1.Close()")
	c1.Close()
}
Example #4
0
func (d *daemon) startAgent() error {
	muxListener, err := createMuxListener()
	if err != nil {
		return err
	}
	mux, err := proxy.NewTCPMux(muxListener)
	if err != nil {
		return err
	}

	agentIP := options.OutboundIP
	if agentIP == "" {
		var err error
		agentIP, err = utils.GetIPAddress()
		if err != nil {
			glog.Fatalf("Failed to acquire ip address: %s", err)
		}
	}

	rpcPort := "0"
	parts := strings.Split(options.Listen, ":")
	if len(parts) > 1 {
		rpcPort = parts[1]
	}

	thisHost, err := host.Build(agentIP, rpcPort, "unknown")
	if err != nil {
		panic(err)
	}

	myHostID, err := utils.HostID()
	if err != nil {
		return fmt.Errorf("HostID failed: %v", err)
	} else if err := validation.ValidHostID(myHostID); err != nil {
		glog.Errorf("invalid hostid: %s", myHostID)
	}

	go func() {
		var poolID string
		for {
			poolID = func() string {
				glog.Infof("Trying to discover my pool...")
				var myHost *host.Host
				masterClient, err := master.NewClient(d.servicedEndpoint)
				if err != nil {
					glog.Errorf("master.NewClient failed (endpoint %+v) : %v", d.servicedEndpoint, err)
					return ""
				}
				defer masterClient.Close()
				myHost, err = masterClient.GetHost(myHostID)
				if err != nil {
					glog.Warningf("masterClient.GetHost %v failed: %v (has this host been added?)", myHostID, err)
					return ""
				}
				poolID = myHost.PoolID
				glog.Infof(" My PoolID: %v", poolID)
				//send updated host info
				updatedHost, err := host.UpdateHostInfo(*myHost)
				if err != nil {
					glog.Infof("Could not send updated host information: %v", err)
					return poolID
				}
				err = masterClient.UpdateHost(updatedHost)
				if err != nil {
					glog.Warningf("Could not update host information: %v", err)
					return poolID
				}
				glog.V(2).Infof("Sent updated host info %#v", updatedHost)
				return poolID
			}()
			if poolID != "" {
				break
			}
			select {
			case <-d.shutdown:
				return
			case <-time.After(5 * time.Second):
				continue
			}
		}

		thisHost.PoolID = poolID

		basePoolPath := "/pools/" + poolID
		dsn := coordzk.NewDSN(options.Zookeepers, time.Second*15).String()
		glog.Infof("zookeeper dsn: %s", dsn)
		zClient, err := coordclient.New("zookeeper", dsn, basePoolPath, nil)
		if err != nil {
			glog.Errorf("failed create a new coordclient: %v", err)
		}
		zzk.InitializeLocalClient(zClient)

		poolBasedConn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(poolID))
		if err != nil {
			glog.Errorf("Error in getting a connection based on pool %v: %v", poolID, err)
		}

		if options.NFSClient != "0" {
			nfsClient, err := storage.NewClient(thisHost, path.Join(options.VarPath, "volumes"))
			if err != nil {
				glog.Fatalf("could not create an NFS client: %s", err)
			}

			go func() {
				<-d.shutdown
				glog.Infof("shutting down storage client")
				nfsClient.Close()
			}()

			//loop and log waiting for Storage Leader
			nfsDone := make(chan struct{})
			go func() {
				defer close(nfsDone)
				nfsClient.Wait()
			}()
			//wait indefinitely(?) for storage to work before starting
			glog.Info("Waiting for Storage Leader")
			nfsUp := false
			for !nfsUp {
				select {
				case <-nfsDone:
					nfsUp = true
					glog.Info("Found Storage Leader")
					break
				case <-time.After(time.Second * 30):
					glog.Info("Waiting for Storage Leader, will not be available for running services. ")
					continue
				}
			}
		} else {
			glog.Info("NFS Client disabled")
		}

		agentOptions := node.AgentOptions{
			PoolID:               thisHost.PoolID,
			Master:               options.Endpoint,
			UIPort:               options.UIPort,
			RPCPort:              options.RPCPort,
			DockerDNS:            options.DockerDNS,
			VarPath:              options.VarPath,
			Mount:                options.Mount,
			FSType:               options.FSType,
			Zookeepers:           options.Zookeepers,
			Mux:                  mux,
			UseTLS:               options.TLS,
			DockerRegistry:       dockerRegistry,
			MaxContainerAge:      time.Duration(int(time.Second) * options.MaxContainerAge),
			VirtualAddressSubnet: options.VirtualAddressSubnet,
		}
		// creates a zClient that is not pool based!
		hostAgent, err := node.NewHostAgent(agentOptions)
		d.hostAgent = hostAgent

		d.waitGroup.Add(1)
		go func() {
			hostAgent.Start(d.shutdown)
			glog.Info("Host Agent has shutdown")
			d.waitGroup.Done()
		}()

		// register the API
		glog.V(0).Infoln("registering ControlPlaneAgent service")
		if err = d.rpcServer.RegisterName("ControlPlaneAgent", hostAgent); err != nil {
			glog.Fatalf("could not register ControlPlaneAgent RPC server: %v", err)
		}

		if options.ReportStats {
			statsdest := fmt.Sprintf("http://%s/api/metrics/store", options.HostStats)
			statsduration := time.Duration(options.StatsPeriod) * time.Second
			glog.V(1).Infoln("Staring container statistics reporter")
			statsReporter, err := stats.NewStatsReporter(statsdest, statsduration, poolBasedConn)
			if err != nil {
				glog.Errorf("Error kicking off stats reporter %v", err)
			} else {
				go func() {
					defer statsReporter.Close()
					<-d.shutdown
				}()
			}
		}
	}()

	glog.Infof("agent start staticips: %v [%d]", d.staticIPs, len(d.staticIPs))
	if err = d.rpcServer.RegisterName("Agent", agent.NewServer(d.staticIPs)); err != nil {
		glog.Fatalf("could not register Agent RPC server: %v", err)
	}
	if err != nil {
		glog.Fatalf("Could not start ControlPlane agent: %v", err)
	}

	// TODO: Integrate this server into the rpc server, or something.
	// Currently its only use is for command execution.
	go func() {
		sio := shell.NewProcessExecutorServer(options.Endpoint, dockerRegistry)
		http.ListenAndServe(":50000", sio)
	}()

	return nil
}
Example #5
0
func (d *daemon) startMaster() error {
	agentIP := options.OutboundIP
	if agentIP == "" {
		var err error
		agentIP, err = utils.GetIPAddress()
		if err != nil {
			glog.Fatalf("Failed to acquire ip address: %s", err)
		}
	}

	// This is storage related
	rpcPort := "0"
	parts := strings.Split(options.Listen, ":")
	if len(parts) > 1 {
		rpcPort = parts[1]
	}

	thisHost, err := host.Build(agentIP, rpcPort, d.masterPoolID)
	if err != nil {
		glog.Errorf("could not build host for agent IP %s: %v", agentIP, err)
		return err
	}

	if err := os.MkdirAll(options.VarPath, 0755); err != nil {
		glog.Errorf("could not create varpath %s: %s", options.VarPath, err)
		return err
	}

	volumesPath := path.Join(options.VarPath, "volumes")
	if d.networkDriver, err = nfs.NewServer(volumesPath, "serviced_var_volumes", "0.0.0.0/0"); err != nil {
		return err
	} else {
		d.storageHandler, err = storage.NewServer(d.networkDriver, thisHost, volumesPath)
		if err != nil {
			return err
		}
	}

	if d.dsDriver, err = d.initDriver(); err != nil {
		return err
	}

	if d.dsContext, err = d.initContext(); err != nil {
		return err
	}

	localClient, err := d.initZK(options.Zookeepers)
	if err != nil {
		glog.Errorf("failed to create a local coordclient: %v", err)
		return err
	}
	zzk.InitializeLocalClient(localClient)

	if len(options.RemoteZookeepers) > 0 {
		remoteClient, err := d.initZK(options.RemoteZookeepers)
		if err != nil {
			glog.Warningf("failed to create a remote coordclient; running in disconnected mode: %v", err)
		} else {
			zzk.InitializeRemoteClient(remoteClient)
		}
	}

	d.facade = d.initFacade()

	if d.cpDao, err = d.initDAO(); err != nil {
		return err
	}

	health.SetDao(d.cpDao)
	go health.Cleanup(d.shutdown)

	if err = d.facade.CreateDefaultPool(d.dsContext, d.masterPoolID); err != nil {
		return err
	}

	if err = d.registerMasterRPC(); err != nil {
		return err
	}

	d.initWeb()
	d.addTemplates()
	d.startScheduler()

	return nil
}
Example #6
0
// getEndpoints builds exportedEndpoints and importedEndpoints
func (c *Controller) getEndpoints(service *service.Service) error {
	var err error
	c.zkInfo, err = getAgentZkInfo(c.options.ServicedEndpoint)
	if err != nil {
		glog.Errorf("Invalid zk info: %v", err)
		return err //ErrInvalidZkInfo
	}
	glog.Infof(" c.zkInfo: %+v", c.zkInfo)

	// endpoints are created at the root level (not pool aware)
	rootBasePath := ""
	zClient, err := coordclient.New("zookeeper", c.zkInfo.ZkDSN, rootBasePath, nil)
	if err != nil {
		glog.Errorf("failed create a new coordclient: %v", err)
		return err
	}

	zzk.InitializeLocalClient(zClient)

	// get zookeeper connection
	conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(service.PoolID))
	if err != nil {
		return fmt.Errorf("getEndpoints zzk.GetLocalConnection failed: %v", err)
	}

	if os.Getenv("SERVICED_IS_SERVICE_SHELL") == "true" {
		// this is not a running service, i.e. serviced shell/run
		if hostname, err := os.Hostname(); err != nil {
			glog.Errorf("could not get hostname: %s", err)
			return fmt.Errorf("getEndpoints failed could not get hostname: %v", err)
		} else {
			c.dockerID = hostname
		}

		// TODO: deal with exports in the future when there is a use case for it

		sstate, err := servicestate.BuildFromService(service, c.hostID)
		if err != nil {
			return fmt.Errorf("Unable to create temporary service state")
		}
		// initialize importedEndpoints
		c.importedEndpoints, err = buildImportedEndpoints(conn, c.tenantID, sstate)
		if err != nil {
			glog.Errorf("Invalid ImportedEndpoints")
			return ErrInvalidImportedEndpoints
		}
	} else {
		// get service state
		glog.Infof("getting service state: %s %v", c.options.Service.ID, c.options.Service.InstanceID)
		sstate, err := getServiceState(conn, c.options.Service.ID, c.options.Service.InstanceID)
		if err != nil {
			return fmt.Errorf("getEndpoints getServiceState failed: %v", err)
		}
		c.dockerID = sstate.DockerID

		// keep a copy of the service EndPoint exports
		c.exportedEndpoints, err = buildExportedEndpoints(conn, c.tenantID, sstate)
		if err != nil {
			glog.Errorf("Invalid ExportedEndpoints")
			return ErrInvalidExportedEndpoints
		}

		// initialize importedEndpoints
		c.importedEndpoints, err = buildImportedEndpoints(conn, c.tenantID, sstate)
		if err != nil {
			glog.Errorf("Invalid ImportedEndpoints")
			return ErrInvalidImportedEndpoints
		}
	}

	return nil
}