func (o *Server) ObtainConfig() (err error) {
e := NewEnvGetter(fmt.Sprintf("OORT_%s", strings.ToUpper(o.serviceName)), "_")
envSkipSRV := e.Get("SKIP_SRV")
// Check whether we're supposed to skip loading via srv method
if strings.ToLower(envSkipSRV) != "true" {
s := &srvconf.SRVLoader{
SyndicateURL: e.Get("SYNDICATE_OVERRIDE"),
}
s.Record, err = GenServiceID(o.serviceName, "syndicate", "tcp")
if err != nil {
if e.Get("SYNDICATE_OVERRIDE") == "" {
log.Println(err)
} else {
log.Fatalln("No SYNDICATE_OVERRIDE provided and", err)
}
}
if e.Get("SYNDICATE_OVERRIDE") != "" {
log.Println("Over wrote syndicate url with url from env!", e.Get("SYNDICATE_OVERRIDE"))
}
nc, err := s.Load()
if err != nil {
return err
}
o.ring, err = ring.LoadRing(bytes.NewReader(nc.Ring))
if err != nil {
return fmt.Errorf("Error while loading ring for config get via srv lookup: %s", err)
}
err = ring.PersistRingOrBuilder(o.ring, nil, fmt.Sprintf("%s/ring/%d-%s.ring", o.cwd, o.ring.Version(), o.serviceName))
if err != nil {
return err
}
o.localID = nc.Localid
o.ring.SetLocalNode(o.localID)
o.ringFile = fmt.Sprintf("%s/ring/%d-%s.ring", o.cwd, o.ring.Version(), o.serviceName)
err = o.loadCmdCtrlConfig()
if err != nil {
return err
}
} else {
// if you skip the srv load you have to provide all of the info in env vars!
log.Println("Skipped SRV Config attempting to load from env")
s, err := strconv.ParseUint(e.Get("LOCALID"), 10, 64)
if err != nil {
return fmt.Errorf("Unable to load env specified local id")
}
o.localID = s
o.ringFile = e.Get("RING_FILE")
o.ring, _, err = ring.RingOrBuilder(o.ringFile)
if err != nil {
return fmt.Errorf("Unable to road env specified ring: %s", err)
}
o.ring.SetLocalNode(o.localID)
err = o.loadCmdCtrlConfig()
if err != nil {
return err
}
}
return nil
}
func (o *Server) RingUpdate(newversion int64, ringBytes []byte) int64 {
o.cmdCtrlLock.Lock()
defer o.cmdCtrlLock.Unlock()
log.Println("Got ring update notification. Trying to update to version:", newversion)
newring, err := ring.LoadRing(bytes.NewReader(ringBytes))
if err != nil {
log.Println("Error loading ring during update:", err)
return o.Ring().Version()
}
if newring.Version() != newversion {
log.Println("Provided ring version != version in ring")
return o.Ring().Version()
}
fname := fmt.Sprintf("%s/ring/%d-%s.ring", o.cwd, newring.Version(), o.serviceName)
writeBytes(fname, &ringBytes)
o.SetRing(newring, fname)
return o.Ring().Version()
}
func NewReplValueStore(c *ReplValueStoreConfig) *ReplValueStore {
cfg := resolveReplValueStoreConfig(c)
rs := &ReplValueStore{
logError: cfg.LogError,
logDebug: cfg.LogDebug,
logDebugOn: cfg.LogDebug != nil,
addressIndex: cfg.AddressIndex,
valueCap: int(cfg.ValueCap),
concurrentRequestsPerStore: cfg.ConcurrentRequestsPerStore,
failedConnectRetryDelay: cfg.FailedConnectRetryDelay,
ftlsConfig: cfg.StoreFTLSConfig,
grpcOpts: cfg.GRPCOpts,
stores: make(map[string]*replValueStoreAndTicketChan),
ringServer: cfg.RingServer,
ringServerGRPCOpts: cfg.RingServerGRPCOpts,
ringCachePath: cfg.RingCachePath,
ringClientID: cfg.RingClientID,
}
if rs.logError == nil {
rs.logError = flog.Default.ErrorPrintf
}
if rs.logDebug == nil {
rs.logDebug = func(string, ...interface{}) {}
}
if rs.ringCachePath != "" {
if fp, err := os.Open(rs.ringCachePath); err != nil {
rs.logDebug("replValueStore: error loading cached ring %q: %s", rs.ringCachePath, err)
} else if r, err := ring.LoadRing(fp); err != nil {
fp.Close()
rs.logDebug("replValueStore: error loading cached ring %q: %s", rs.ringCachePath, err)
} else {
fp.Close()
rs.ring = r
}
}
return rs
}
func (rs *ReplValueStore) ringServerConnector(exitChan chan struct{}) {
sleeperTicks := 2
sleeperTicker := time.NewTicker(time.Second)
sleeper := func() {
for i := sleeperTicks; i > 0; i-- {
select {
case <-exitChan:
break
case <-sleeperTicker.C:
}
}
if sleeperTicks < 60 {
sleeperTicks *= 2
}
}
for {
select {
case <-exitChan:
break
default:
}
ringServer := rs.ringServer
if ringServer == "" {
var err error
ringServer, err = oort.GetRingServer("value")
if err != nil {
rs.logError("replValueStore: error resolving ring service: %s", err)
sleeper()
continue
}
}
conn, err := grpc.Dial(ringServer, rs.ringServerGRPCOpts...)
if err != nil {
rs.logError("replValueStore: error connecting to ring service %q: %s", ringServer, err)
sleeper()
continue
}
stream, err := synpb.NewSyndicateClient(conn).GetRingStream(context.Background(), &synpb.SubscriberID{Id: rs.ringClientID})
if err != nil {
rs.logError("replValueStore: error creating stream with ring service %q: %s", ringServer, err)
sleeper()
continue
}
connDoneChan := make(chan struct{})
somethingICanTakeAnAddressOf := int32(0)
activity := &somethingICanTakeAnAddressOf
// This goroutine will detect when the exitChan is closed so it can
// close the conn so that the blocking stream.Recv will get an error
// and everything will unwind properly.
// However, if the conn errors out on its own and exitChan isn't
// closed, we're going to loop back around and try a new conn, but we
// need to clear out this goroutine, which is what the connDoneChan is
// for.
// One last thing is that if nothing happens for fifteen minutes, we
// can assume the conn has gone stale and close it, causing a loop
// around to try a new conn.
// It would be so much easier if Recv could use a timeout Context...
go func(c *grpc.ClientConn, a *int32, cdc chan struct{}) {
for {
select {
case <-exitChan:
case <-cdc:
case <-time.After(15 * time.Minute):
// I'm comfortable with time.After here since it's just
// once per fifteen minutes or new conn.
v := atomic.LoadInt32(a)
if v != 0 {
atomic.AddInt32(a, -v)
continue
}
}
break
}
c.Close()
}(conn, activity, connDoneChan)
for {
select {
case <-exitChan:
break
default:
}
res, err := stream.Recv()
if err != nil {
rs.logDebug("replValueStore: error with stream to ring service %q: %s", ringServer, err)
break
}
atomic.AddInt32(activity, 1)
if res != nil {
if r, err := ring.LoadRing(bytes.NewBuffer(res.Ring)); err != nil {
rs.logDebug("replValueStore: error with ring received from stream to ring service %q: %s", ringServer, err)
} else {
// This will cache the ring if ringCachePath is not empty.
rs.SetRing(r)
// Resets the exponential sleeper since we had success.
sleeperTicks = 2
rs.logDebug("replValueStore: got new ring from stream to ring service %q: %d", ringServer, res.Version)
}
}
}
close(connDoneChan)
sleeper()
}
}
func ringOrBuilder(fileName string) (r ring.Ring, b *ring.Builder, err error) {
var f *os.File
if f, err = os.Open(fileName); err != nil {
return
}
var gf *gzip.Reader
if gf, err = gzip.NewReader(f); err != nil {
return
}
header := make([]byte, 16)
if _, err = io.ReadFull(gf, header); err != nil {
return
}
if string(header[:5]) == "RINGv" {
gf.Close()
if _, err = f.Seek(0, 0); err != nil {
return
}
r, err = ring.LoadRing(f)
} else if string(header[:12]) == "RINGBUILDERv" {
gf.Close()
if _, err = f.Seek(0, 0); err != nil {
return
}
b, err = ring.LoadBuilder(f)
}
return
}