Example #1
0
func (tk *TestKeeper) GetKeeperInfo(timeout time.Duration) (*cluster.KeeperInfo, error) {
	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()
	req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/info", net.JoinHostPort(tk.listenAddress, tk.port)), nil)
	if err != nil {
		return nil, err
	}
	var data cluster.KeeperInfo
	err = httpDo(ctx, req, nil, func(resp *http.Response, err error) error {
		if err != nil {
			return err
		}
		defer resp.Body.Close()

		if resp.StatusCode != http.StatusOK {
			return fmt.Errorf("http error code: %d, error: %s", resp.StatusCode, resp.Status)
		}
		if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		return nil, err
	}
	return &data, nil
}
Example #2
0
func (e *EtcdManager) SetKeeperDiscoveryInfo(id string, ms *cluster.KeeperDiscoveryInfo) (*etcd.Response, error) {
	msj, err := json.Marshal(ms)
	if err != nil {
		return nil, err
	}
	return e.kAPI.Set(context.Background(), filepath.Join(e.etcdPath, keepersDiscoveryInfoDir, id), string(msj), nil)
}
Example #3
0
// AtomicDelete deletes a value at "key" if the key
// has not been modified in the meantime, throws an
// error if this is the case
func (s *Etcd) AtomicDelete(key string, previous *store.KVPair) (bool, error) {
	if previous == nil {
		return false, store.ErrPreviousNotSpecified
	}

	delOpts := &etcd.DeleteOptions{}

	if previous != nil {
		delOpts.PrevIndex = previous.LastIndex
		if previous.Value != nil {
			delOpts.PrevValue = string(previous.Value)
		}
	}

	_, err := s.client.Delete(context.Background(), s.normalize(key), delOpts)
	if err != nil {
		if etcdError, ok := err.(etcd.Error); ok {
			// Key Not Found
			if etcdError.Code == etcd.ErrorCodeKeyNotFound {
				return false, store.ErrKeyNotFound
			}
			// Compare failed
			if etcdError.Code == etcd.ErrorCodeTestFailed {
				return false, store.ErrKeyModified
			}
		}
		return false, err
	}

	return true, nil
}
Example #4
0
func (s *Sentinel) Start() {
	endCh := make(chan struct{})
	endApiCh := make(chan error)

	router := s.NewRouter()
	go func() {
		endApiCh <- http.ListenAndServe(fmt.Sprintf("%s:%s", s.listenAddress, s.port), router)
	}()

	ctx, cancel := context.WithCancel(context.Background())
	timerCh := time.NewTimer(0).C

	for true {
		select {
		case <-s.stop:
			log.Debugf("stopping stolon sentinel")
			cancel()
			s.end <- true
			return
		case <-timerCh:
			go func() {
				s.clusterSentinelCheck(ctx)
				endCh <- struct{}{}
			}()
		case <-endCh:
			timerCh = time.NewTimer(s.clusterConfig.SleepInterval).C
		case err := <-endApiCh:
			if err != nil {
				log.Fatal("ListenAndServe: ", err)
			}
			close(s.stop)
		}
	}
}
Example #5
0
func (p *Manager) GetRoleFromDB() (common.Role, error) {
	db, err := sql.Open("postgres", p.connString)
	if err != nil {
		return 0, err
	}
	defer db.Close()

	ctx, cancel := context.WithTimeout(context.Background(), p.requestTimeout)
	rows, err := Query(ctx, db, "SELECT pg_is_in_recovery from pg_is_in_recovery()")
	cancel()
	if err != nil {
		return 0, err
	}
	defer rows.Close()
	for rows.Next() {
		var isInRecovery bool
		if err := rows.Scan(&isInRecovery); err != nil {
			return 0, err
		}
		if isInRecovery {
			return common.StandbyRole, nil
		}
		return common.MasterRole, nil
	}
	return 0, fmt.Errorf("cannot get pg role from db: no rows returned")
}
Example #6
0
func (p *Manager) GetReplicatinSlots() ([]string, error) {
	db, err := sql.Open("postgres", p.connString)
	if err != nil {
		return nil, err
	}
	defer db.Close()

	replSlots := []string{}

	ctx, cancel := context.WithTimeout(context.Background(), p.requestTimeout)
	rows, err := Query(ctx, db, "SELECT slot_name from pg_replication_slots")
	cancel()
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	for rows.Next() {
		var slotName string
		if err := rows.Scan(&slotName); err != nil {
			return nil, err
		}
		replSlots = append(replSlots, slotName)
	}

	return replSlots, nil
}
Example #7
0
// List child nodes of a given directory
func (s *Etcd) List(directory string) ([]*store.KVPair, error) {
	getOpts := &etcd.GetOptions{
		Quorum:    true,
		Recursive: true,
		Sort:      true,
	}

	resp, err := s.client.Get(context.Background(), s.normalize(directory), getOpts)
	if err != nil {
		if keyNotFound(err) {
			return nil, store.ErrKeyNotFound
		}
		return nil, err
	}

	kv := []*store.KVPair{}
	for _, n := range resp.Node.Nodes {
		kv = append(kv, &store.KVPair{
			Key:       n.Key,
			Value:     []byte(n.Value),
			LastIndex: n.ModifiedIndex,
		})
	}
	return kv, nil
}
Example #8
0
func (e *EtcdManager) SetLeaderSentinelInfo(si *cluster.SentinelInfo, ttl time.Duration) (*etcd.Response, error) {
	sij, err := json.Marshal(si)
	if err != nil {
		return nil, err
	}
	opts := &etcd.SetOptions{TTL: ttl}
	return e.kAPI.Set(context.Background(), filepath.Join(e.etcdPath, leaderSentinelInfoFile), string(sij), opts)
}
Example #9
0
func (e *EtcdManager) SetProxyInfo(pi *cluster.ProxyInfo, ttl time.Duration) (*etcd.Response, error) {
	pij, err := json.Marshal(pi)
	if err != nil {
		return nil, err
	}
	opts := &etcd.SetOptions{TTL: ttl}
	return e.kAPI.Set(context.Background(), filepath.Join(e.etcdPath, proxiesInfoDir, pi.ID), string(pij), opts)
}
Example #10
0
func (e *EtcdManager) SetClusterConfig(cfg *cluster.Config) (*etcd.Response, error) {
	cfgj, err := json.Marshal(cfg)
	if err != nil {
		return nil, err
	}
	path := filepath.Join(e.etcdPath, configFile)
	opts := &etcd.SetOptions{}
	return e.kAPI.Set(context.Background(), path, string(cfgj), opts)
}
Example #11
0
func (c *httpClusterClient) getLeaderEndpoint() (string, error) {
	mAPI := NewMembersAPI(c)
	leader, err := mAPI.Leader(context.Background())
	if err != nil {
		return "", err
	}

	return leader.ClientURLs[0], nil // TODO: how to handle multiple client URLs?
}
Example #12
0
func (te *TestEtcd) GetEtcdNode(timeout time.Duration, path string) (*etcd.Node, error) {
	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()

	res, err := te.kAPI.Get(ctx, path, &etcd.GetOptions{Quorum: true})
	if err != nil {
		return nil, err
	}
	return res.Node, nil
}
Example #13
0
func (p *PostgresKeeper) pgStateHandler(w http.ResponseWriter, req *http.Request) {
	pgState := &cluster.PostgresState{}
	p.cvMutex.Lock()
	defer p.cvMutex.Unlock()

	initialized, err := p.pgm.IsInitialized()
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	if !initialized {
		pgState.Initialized = false
	} else {
		var err error
		ctx, cancel := context.WithTimeout(context.Background(), p.clusterConfig.RequestTimeout)
		pgState, err = pg.GetPGState(ctx, p.getOurReplConnString())
		cancel()
		if err != nil {
			log.Errorf("error getting pg state: %v", err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		pgState.Initialized = true

		// if timeline <= 1 then no timeline history file exists.
		pgState.TimelinesHistory = cluster.PostgresTimeLinesHistory{}
		if pgState.TimelineID > 1 {
			ctx, cancel = context.WithTimeout(context.Background(), p.clusterConfig.RequestTimeout)
			tlsh, err := pg.GetTimelinesHistory(ctx, pgState.TimelineID, p.getOurReplConnString())
			cancel()
			if err != nil {
				log.Errorf("error getting timeline history: %v", err)
				w.WriteHeader(http.StatusInternalServerError)
				return
			}
			pgState.TimelinesHistory = tlsh
		}
	}
	if err := json.NewEncoder(w).Encode(&pgState); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
	}
}
Example #14
0
// DeleteTree deletes a range of keys under a given directory
func (s *Etcd) DeleteTree(directory string) error {
	delOpts := &etcd.DeleteOptions{
		Recursive: true,
	}

	_, err := s.client.Delete(context.Background(), s.normalize(directory), delOpts)
	if keyNotFound(err) {
		return store.ErrKeyNotFound
	}
	return err
}
Example #15
0
// Delete a value at "key"
func (s *Etcd) Delete(key string) error {
	opts := &etcd.DeleteOptions{
		Recursive: false,
	}

	_, err := s.client.Delete(context.Background(), s.normalize(key), opts)
	if keyNotFound(err) {
		return store.ErrKeyNotFound
	}
	return err
}
Example #16
0
func (p *Manager) DropReplicationSlot(name string) error {
	db, err := sql.Open("postgres", p.connString)
	if err != nil {
		return err
	}
	defer db.Close()

	ctx, cancel := context.WithTimeout(context.Background(), p.requestTimeout)
	_, err = Exec(ctx, db, fmt.Sprintf("select pg_drop_replication_slot('%s')", name))
	cancel()
	return err
}
Example #17
0
// Put a value at "key"
func (s *Etcd) Put(key string, value []byte, opts *store.WriteOptions) error {
	setOpts := &etcd.SetOptions{}

	// Set options
	if opts != nil {
		setOpts.Dir = opts.IsDir
		setOpts.TTL = opts.TTL
	}

	_, err := s.client.Set(context.Background(), s.normalize(key), string(value), setOpts)
	return err
}
Example #18
0
func (p *Manager) CreateReplUser() error {
	db, err := sql.Open("postgres", p.connString)
	if err != nil {
		return err
	}
	defer db.Close()

	ctx, cancel := context.WithTimeout(context.Background(), p.requestTimeout)
	_, err = Exec(ctx, db, fmt.Sprintf(`CREATE USER "%s" WITH REPLICATION ENCRYPTED PASSWORD '%s';`, p.replUser, p.replPassword))
	cancel()
	return err
}
Example #19
0
func (p *PostgresKeeper) Start() {
	endSMCh := make(chan struct{})
	endPgStatecheckerCh := make(chan struct{})
	endApiCh := make(chan error)

	err := p.loadCVVersion()
	if err != nil {
		p.end <- err
		return
	}

	p.pgm.Stop(true)

	http.HandleFunc("/info", p.infoHandler)
	http.HandleFunc("/pgstate", p.pgStateHandler)
	go func() {
		endApiCh <- http.ListenAndServe(fmt.Sprintf("%s:%s", p.listenAddress, p.port), nil)
	}()

	ctx, cancel := context.WithCancel(context.Background())
	smTimerCh := time.NewTimer(0).C
	updatePGStateTimerCh := time.NewTimer(0).C
	for true {
		select {
		case <-p.stop:
			log.Debugf("stopping postgres keeper")
			cancel()
			p.pgm.Stop(true)
			p.end <- nil
			return
		case <-smTimerCh:
			go func() {
				p.postgresKeeperSM(ctx)
				endSMCh <- struct{}{}
			}()
		case <-endSMCh:
			smTimerCh = time.NewTimer(p.clusterConfig.SleepInterval).C
		case <-updatePGStateTimerCh:
			go func() {
				p.updatePGState(ctx)
				endPgStatecheckerCh <- struct{}{}
			}()
		case <-endPgStatecheckerCh:
			updatePGStateTimerCh = time.NewTimer(p.clusterConfig.SleepInterval).C
		case err := <-endApiCh:
			if err != nil {
				log.Fatal("ListenAndServe: ", err)
			}
			close(p.stop)
		}
	}
}
Example #20
0
func (e *EtcdManager) GetClusterConfig() (*cluster.Config, *etcd.Response, error) {
	path := filepath.Join(e.etcdPath, configFile)
	res, err := e.kAPI.Get(context.Background(), path, &etcd.GetOptions{Quorum: true})
	if err != nil && !IsEtcdNotFound(err) {
		log.Errorf("err: %v", err)
		return nil, nil, err
	} else if !IsEtcdNotFound(err) {
		cfg, err := cluster.ParseConfig([]byte(res.Node.Value))
		if err != nil {
			return nil, nil, err
		}
		return cfg, res, nil
	}
	return cluster.NewDefaultConfig(), res, nil
}
Example #21
0
func (e *EtcdManager) GetClusterData() (*cluster.ClusterData, *etcd.Response, error) {
	var cd *cluster.ClusterData
	path := filepath.Join(e.etcdPath, clusterDataFile)
	res, err := e.kAPI.Get(context.Background(), path, &etcd.GetOptions{Quorum: true})
	if err != nil && !IsEtcdNotFound(err) {
		return nil, nil, err
	} else if !IsEtcdNotFound(err) {
		err = json.Unmarshal([]byte(res.Node.Value), &cd)
		if err != nil {
			return nil, nil, err
		}
		return cd, res, nil
	}
	return nil, nil, nil
}
Example #22
0
func (e *EtcdManager) GetLeaderSentinelInfo() (*cluster.SentinelInfo, *etcd.Response, error) {
	var si *cluster.SentinelInfo
	path := filepath.Join(e.etcdPath, leaderSentinelInfoFile)
	res, err := e.kAPI.Get(context.Background(), path, &etcd.GetOptions{Quorum: true})
	if err != nil && !IsEtcdNotFound(err) {
		return nil, nil, err
	} else if !IsEtcdNotFound(err) {
		err = json.Unmarshal([]byte(res.Node.Value), &si)
		if err != nil {
			return nil, nil, err
		}
		return si, res, nil
	}
	return nil, nil, nil
}
Example #23
0
// Unlock the "key". Calling unlock while
// not holding the lock will throw an error
func (l *etcdLock) Unlock() error {
	if l.stopLock != nil {
		l.stopLock <- struct{}{}
	}
	if l.last != nil {
		delOpts := &etcd.DeleteOptions{
			PrevIndex: l.last.Node.ModifiedIndex,
		}
		_, err := l.client.Delete(context.Background(), l.key, delOpts)
		if err != nil {
			return err
		}
	}
	return nil
}
Example #24
0
func (e *EtcdManager) GetProxyView() (*cluster.ProxyView, *etcd.Response, error) {
	var pv *cluster.ProxyView
	path := filepath.Join(e.etcdPath, proxyViewFile)
	res, err := e.kAPI.Get(context.Background(), path, &etcd.GetOptions{Quorum: true})
	if err != nil && !IsEtcdNotFound(err) {
		log.Errorf("err: %v", err)
		return nil, nil, err
	} else if !IsEtcdNotFound(err) {
		err = json.Unmarshal([]byte(res.Node.Value), &pv)
		if err != nil {
			return nil, nil, err
		}
		return pv, res, nil
	}
	return nil, nil, nil
}
Example #25
0
// WaitLock simply waits for the key to be available for creation
func (l *etcdLock) waitLock(key string, errorCh chan error, stopWatchCh chan bool, free chan<- bool) {
	opts := &etcd.WatcherOptions{Recursive: false}
	watcher := l.client.Watcher(key, opts)

	for {
		event, err := watcher.Next(context.Background())
		if err != nil {
			errorCh <- err
			return
		}
		if event.Action == "delete" || event.Action == "expire" {
			free <- true
			return
		}
	}
}
Example #26
0
// AtomicPut puts a value at "key" if the key has not been
// modified in the meantime, throws an error if this is the case
func (s *Etcd) AtomicPut(key string, value []byte, previous *store.KVPair, opts *store.WriteOptions) (bool, *store.KVPair, error) {
	var (
		meta *etcd.Response
		err  error
	)

	setOpts := &etcd.SetOptions{}

	if previous != nil {
		setOpts.PrevExist = etcd.PrevExist
		setOpts.PrevIndex = previous.LastIndex
		if previous.Value != nil {
			setOpts.PrevValue = string(previous.Value)
		}
	} else {
		setOpts.PrevExist = etcd.PrevNoExist
	}

	if opts != nil {
		if opts.TTL > 0 {
			setOpts.TTL = opts.TTL
		}
	}

	meta, err = s.client.Set(context.Background(), s.normalize(key), string(value), setOpts)
	if err != nil {
		if etcdError, ok := err.(etcd.Error); ok {
			// Compare failed
			if etcdError.Code == etcd.ErrorCodeTestFailed {
				return false, nil, store.ErrKeyModified
			}
			// Node exists error (when PrevNoExist)
			if etcdError.Code == etcd.ErrorCodeNodeExist {
				return false, nil, store.ErrKeyExists
			}
		}
		return false, nil, err
	}

	updated := &store.KVPair{
		Key:       key,
		Value:     value,
		LastIndex: meta.Node.ModifiedIndex,
	}

	return true, updated, nil
}
Example #27
0
func (e *EtcdManager) SetProxyView(pv *cluster.ProxyView, prevIndex uint64) (*etcd.Response, error) {
	log.Debugf("prevIndex: %d", prevIndex)
	// write cluster view
	pvj, err := json.Marshal(pv)
	if err != nil {
		return nil, err
	}
	path := filepath.Join(e.etcdPath, proxyViewFile)
	opts := &etcd.SetOptions{}
	if prevIndex == 0 {
		opts.PrevExist = etcd.PrevNoExist
	} else {
		opts.PrevExist = etcd.PrevExist
		opts.PrevIndex = prevIndex
	}
	return e.kAPI.Set(context.Background(), path, string(pvj), opts)
}
Example #28
0
func (e *EtcdManager) GetProxiesInfo() (cluster.ProxiesInfo, error) {
	psi := cluster.ProxiesInfo{}
	res, err := e.kAPI.Get(context.Background(), filepath.Join(e.etcdPath, proxiesInfoDir), &etcd.GetOptions{Recursive: true, Quorum: true})
	if err != nil && !IsEtcdNotFound(err) {
		return nil, err
	} else if !IsEtcdNotFound(err) {
		for _, node := range res.Node.Nodes {
			var pi cluster.ProxyInfo
			err = json.Unmarshal([]byte(node.Value), &pi)
			if err != nil {
				return nil, err
			}
			psi = append(psi, &pi)
		}
	}
	return psi, nil
}
Example #29
0
func (e *EtcdManager) GetProxyInfo(id string) (*cluster.ProxyInfo, bool, error) {
	if id == "" {
		return nil, false, fmt.Errorf("empty proxy id")
	}
	var pi cluster.ProxyInfo
	res, err := e.kAPI.Get(context.Background(), filepath.Join(e.etcdPath, proxiesInfoDir, id), &etcd.GetOptions{Quorum: true})
	if err != nil && !IsEtcdNotFound(err) {
		return nil, false, err
	} else if !IsEtcdNotFound(err) {
		err = json.Unmarshal([]byte(res.Node.Value), &pi)
		if err != nil {
			return nil, false, err
		}
		return &pi, true, nil
	}
	return nil, false, nil
}
Example #30
0
func (e *EtcdManager) GetKeepersDiscoveryInfo() (cluster.KeepersDiscoveryInfo, error) {
	keepers := cluster.KeepersDiscoveryInfo{}
	res, err := e.kAPI.Get(context.Background(), filepath.Join(e.etcdPath, keepersDiscoveryInfoDir), &etcd.GetOptions{Recursive: true, Quorum: true})
	if err != nil && !IsEtcdNotFound(err) {
		return nil, err
	} else if !IsEtcdNotFound(err) {
		for _, node := range res.Node.Nodes {
			var keeper cluster.KeeperDiscoveryInfo
			err = json.Unmarshal([]byte(node.Value), &keeper)
			if err != nil {
				return nil, err
			}
			keepers = append(keepers, &keeper)
		}
	}
	return keepers, nil
}