func (self *log) appendRequest(request *protocol.Request, shardId uint32) error {
bytes, err := request.Encode()
if err != nil {
return err
}
// every request is preceded with the length, shard id and the request number
hdr := &entryHeader{
shardId: shardId,
requestNumber: request.GetRequestNumber(),
length: uint32(len(bytes)),
}
writtenHdrBytes, err := hdr.Write(self.file)
if err != nil {
logger.Error("Error while writing header: %s", err)
return err
}
written, err := self.file.Write(bytes)
if err != nil {
logger.Error("Error while writing request: %s", err)
return err
}
if written < len(bytes) {
err = fmt.Errorf("Couldn't write entire request")
logger.Error("Error while writing request: %s", err)
return err
}
self.fileSize += uint64(writtenHdrBytes + written)
return nil
}
func (self *log) appendRequest(request *protocol.Request, shardId uint32) (uint32, error) {
self.assignSequenceNumbers(shardId, request)
bytes, err := request.Encode()
if err != nil {
return 0, err
}
requestNumber := self.state.getNextRequestNumber()
// every request is preceded with the length, shard id and the request number
hdr := &entryHeader{
shardId: shardId,
requestNumber: requestNumber,
length: uint32(len(bytes)),
}
writtenHdrBytes, err := hdr.Write(self.file)
if err != nil {
logger.Error("Error while writing header: %s", err)
return 0, err
}
written, err := self.file.Write(bytes)
if err != nil {
logger.Error("Error while writing request: %s", err)
return 0, err
}
if written < len(bytes) {
err = fmt.Errorf("Couldn't write entire request")
logger.Error("Error while writing request: %s", err)
return 0, err
}
self.fileSize += uint64(writtenHdrBytes + written)
self.conditionalBookmarkAndIndex()
return requestNumber, nil
}
func (self *ProtobufRequestHandler) HandleRequest(request *protocol.Request, conn net.Conn) error {
if *request.Type == protocol.Request_WRITE {
shard := self.clusterConfig.GetLocalShardById(*request.ShardId)
log.Debug("HANDLE: (%d):%d:%v", self.clusterConfig.LocalServer.Id, request.GetId(), shard)
err := shard.WriteLocalOnly(request)
if err != nil {
log.Error("ProtobufRequestHandler: error writing local shard: ", err)
return err
}
response := &protocol.Response{RequestId: request.Id, Type: &self.writeOk}
return self.WriteResponse(conn, response)
} else if *request.Type == protocol.Request_DROP_DATABASE {
go self.handleDropDatabase(request, conn)
return nil
} else if *request.Type == protocol.Request_QUERY {
go self.handleQuery(request, conn)
} else if *request.Type == protocol.Request_HEARTBEAT {
response := &protocol.Response{RequestId: request.Id, Type: &heartbeatResponse}
return self.WriteResponse(conn, response)
} else {
log.Error("unknown request type: %v", request)
return errors.New("Unknown request type")
}
return nil
}
func (self *CoordinatorImpl) ReplicateWrite(request *protocol.Request) error {
id := atomic.AddUint32(&self.requestId, uint32(1))
request.Id = &id
location := common.RingLocation(request.Database, request.Series.Name, request.Series.Points[0].Timestamp)
replicas := self.clusterConfiguration.GetServersByRingLocation(request.Database, &location)
request.Type = &replicateWrite
self.sendRequestToReplicas(request, replicas)
return nil
}
func (self *CoordinatorImpl) ReplicateDelete(request *protocol.Request) error {
id := atomic.AddUint32(&self.requestId, uint32(1))
request.Id = &id
server := self.clusterConfiguration.GetServerById(request.OwnerServerId)
_, replicas := self.clusterConfiguration.GetReplicas(server, request.Database)
request.Type = &replicateDelete
self.sendRequestToReplicas(request, replicas)
return nil
}
func (self *CoordinatorImpl) sendRequestToReplicas(request *protocol.Request, replicas []*ClusterServer) {
for _, server := range replicas {
if server.Id != self.clusterConfiguration.localServerId {
err := server.MakeRequest(request, nil)
if err != nil {
log.Warn("REPLICATION ERROR: ", request.GetSequenceNumber(), err)
}
}
}
}
func (self *ClusterServer) Write(request *protocol.Request) error {
responseChan := make(chan *protocol.Response)
err := self.connection.MakeRequest(request, responseChan)
if err != nil {
return err
}
log.Debug("Waiting for response to %d", request.GetRequestNumber())
response := <-responseChan
if response.ErrorMessage != nil {
return errors.New(*response.ErrorMessage)
}
return nil
}
func (self *CoordinatorImpl) proxyWrite(clusterServer *ClusterServer, request *protocol.Request) error {
originatingServerId := request.OriginatingServerId
request.OriginatingServerId = nil
defer func() { request.OriginatingServerId = originatingServerId }()
responseChan := make(chan *protocol.Response, 1)
clusterServer.protobufClient.MakeRequest(request, responseChan)
response := <-responseChan
if *response.Type == protocol.Response_WRITE_OK {
return nil
} else {
return errors.New(response.GetErrorMessage())
}
}
func (self *ProtobufRequestHandler) handleWrites(request *protocol.Request, conn net.Conn) {
shard := self.clusterConfig.GetLocalShardById(*request.ShardId)
log.Debug("HANDLE: (%d):%d:%v", self.clusterConfig.LocalServer.Id, request.GetId(), shard)
err := shard.WriteLocalOnly(request)
var errorMsg *string
if err != nil {
log.Error("ProtobufRequestHandler: error writing local shard: %s", err)
errorMsg = protocol.String(err.Error())
}
response := &protocol.Response{RequestId: request.Id, Type: &self.writeOk, ErrorMessage: errorMsg}
if err := self.WriteResponse(conn, response); err != nil {
log.Error("ProtobufRequestHandler: error writing local shard: %s", err)
}
}
// Makes a request to the server. If the responseStream chan is not nil it will expect a response from the server
// with a matching request.Id. The REQUEST_RETRY_ATTEMPTS constant of 3 and the RECONNECT_RETRY_WAIT of 100ms means
// that an attempt to make a request to a downed server will take 300ms to time out.
func (self *ProtobufClient) MakeRequest(request *protocol.Request, responseStream chan *protocol.Response) error {
if request.Id == nil {
id := atomic.AddUint32(&self.lastRequestId, uint32(1))
request.Id = &id
}
if responseStream != nil {
self.requestBufferLock.Lock()
// this should actually never happen. The sweeper should clear out dead requests
// before the uint32 ids roll over.
if oldReq, alreadyHasRequestById := self.requestBuffer[*request.Id]; alreadyHasRequestById {
message := "already has a request with this id, must have timed out"
log.Error(message)
oldReq.responseChan <- &protocol.Response{Type: &endStreamResponse, ErrorMessage: &message}
}
self.requestBuffer[*request.Id] = &runningRequest{timeMade: time.Now(), responseChan: responseStream, request: request}
self.requestBufferLock.Unlock()
}
data, err := request.Encode()
if err != nil {
return err
}
conn := self.getConnection()
if conn == nil {
conn = self.reconnect()
if conn == nil {
return fmt.Errorf("Failed to connect to server %s", self.hostAndPort)
}
}
if self.writeTimeout > 0 {
conn.SetWriteDeadline(time.Now().Add(self.writeTimeout))
}
buff := bytes.NewBuffer(make([]byte, 0, len(data)+8))
binary.Write(buff, binary.LittleEndian, uint32(len(data)))
_, err = conn.Write(append(buff.Bytes(), data...))
if err == nil {
return nil
}
// if we got here it errored out, clear out the request
self.requestBufferLock.Lock()
delete(self.requestBuffer, *request.Id)
self.requestBufferLock.Unlock()
self.reconnect()
return err
}
func (self *ShardData) forwardRequest(request *p.Request) ([]<-chan *p.Response, []uint32, error) {
ids := []uint32{}
responses := []<-chan *p.Response{}
for _, server := range self.clusterServers {
responseChan := make(chan *p.Response, 1)
// do this so that a new id will get assigned
request.Id = nil
log.Debug("Forwarding request %s to %d", request.GetDescription(), server.Id)
server.MakeRequest(request, responseChan)
responses = append(responses, responseChan)
ids = append(ids, server.Id)
}
return responses, ids, nil
}
// This method never blocks. It'll buffer writes until they fill the buffer then drop the on the
// floor and let the background goroutine replay from the WAL
func (self *WriteBuffer) Write(request *protocol.Request) {
self.shardLastRequestNumber[request.GetShardId()] = request.GetRequestNumber()
select {
case self.writes <- request:
return
default:
select {
case self.stoppedWrites <- *request.RequestNumber:
return
default:
return
}
}
}
func (self *LevelDbDatastore) LogRequestAndAssignSequenceNumber(request *protocol.Request, replicationFactor *uint8, ownerServerId *uint32) error {
// log to this key structure on a different DB sharded by day: <cluster version><owner id><sequence server id><replication sequence>
var numberKey []byte
if request.SequenceNumber == nil {
sequenceNumber, err := self.AtomicIncrement(self.keyForOwnerAndServerSequenceNumber(request.ClusterVersion, replicationFactor, ownerServerId, request.OriginatingServerId), 1)
if err != nil {
return err
}
request.SequenceNumber = &sequenceNumber
} else {
// this is for a replicated write, ensure that it's the next in line for this owner and server
name := self.keyForOwnerAndServerSequenceNumber(request.ClusterVersion, replicationFactor, ownerServerId, request.OriginatingServerId)
numberKey = append(ATOMIC_INCREMENT_PREFIX, []byte(name)...)
numberBytes, err := self.db.Get(self.readOptions, numberKey)
if err != nil {
return err
}
previousSequenceNumber := self.bytesToCurrentNumber(numberBytes)
if previousSequenceNumber+uint64(1) != *request.SequenceNumber {
return SequenceMissingRequestsError{"Missing requests between last seen and this one.", previousSequenceNumber}
}
}
self.requestLogLock.RLock()
requestLog := self.currentRequestLog
self.requestLogLock.RUnlock()
// proxied writes should be logged as replicated ones. That's what is expected if they're replayed later
if *request.Type == protocol.Request_PROXY_WRITE {
request.Type = &replicateWrite
}
data, err := request.Encode()
if err != nil {
return err
}
key := self.requestLogKey(request.ClusterVersion, request.OriginatingServerId, ownerServerId, request.SequenceNumber, replicationFactor)
err = requestLog.db.Put(self.writeOptions, key, data)
if err != nil {
return err
}
if numberKey != nil {
currentNumberBuffer := bytes.NewBuffer(make([]byte, 0, 8))
binary.Write(currentNumberBuffer, binary.BigEndian, *request.SequenceNumber)
self.db.Put(self.writeOptions, numberKey, currentNumberBuffer.Bytes())
}
return nil
}
func (self *ShardData) Write(request *protocol.Request) error {
request.ShardId = &self.id
requestNumber, err := self.wal.AssignSequenceNumbersAndLog(request, self)
if err != nil {
return err
}
request.RequestNumber = &requestNumber
if self.store != nil {
self.store.BufferWrite(request)
}
for _, server := range self.clusterServers {
server.BufferWrite(request)
}
return nil
}
func (self *LevelDbDatastore) LogRequestAndAssignSequenceNumber(request *protocol.Request, replicationFactor *uint8, ownerServerId *uint32) error {
// log to this key structure on a different DB sharded by day: <cluster version><owner id><sequence server id><replication sequence>
updateSequenceNumber := false
if request.SequenceNumber == nil {
sequenceNumber, err := self.AtomicIncrement(self.keyForOwnerAndServerSequenceNumber(request.ClusterVersion, replicationFactor, ownerServerId, request.OriginatingServerId), 1)
if err != nil {
return err
}
request.SequenceNumber = &sequenceNumber
} else {
updateSequenceNumber = true
previousSequenceNumber, err := self.CurrentSequenceNumber(request.ClusterVersion, replicationFactor, ownerServerId, request.OriginatingServerId)
if err != nil {
return err
}
// Do a less than comparison because it's ok if we're just getting the same write again. As long as we haven't missed one.
if previousSequenceNumber+uint64(1) < *request.SequenceNumber {
log.Warn("MISSING REQUESTS: %d, %d", previousSequenceNumber, *request.SequenceNumber)
return SequenceMissingRequestsError{"Missing requests between last seen and this one.", previousSequenceNumber, *request.SequenceNumber}
}
}
self.requestLogLock.RLock()
requestLog := self.currentRequestLog
self.requestLogLock.RUnlock()
// proxied writes should be logged as replicated ones. That's what is expected if they're replayed later
if *request.Type == protocol.Request_PROXY_WRITE {
request.Type = &replicateWrite
}
data, err := request.Encode()
if err != nil {
return err
}
key := NewWALKey(request.ClusterVersion, request.OriginatingServerId, ownerServerId, request.SequenceNumber, replicationFactor)
err = requestLog.db.Put(self.writeOptions, key, data)
if err != nil {
return err
}
if updateSequenceNumber {
return self.updateSequenceNumber(request.ClusterVersion, replicationFactor, ownerServerId, request.OriginatingServerId, *request.SequenceNumber)
}
return nil
}
func (self *ShardData) Write(request *p.Request) error {
request.ShardId = &self.id
requestNumber, err := self.wal.AssignSequenceNumbersAndLog(request, self)
if err != nil {
return err
}
request.RequestNumber = &requestNumber
if self.store != nil {
self.store.BufferWrite(request)
}
for _, server := range self.clusterServers {
// we have to create a new reqeust object because the ID gets assigned on each server.
requestWithoutId := &p.Request{Type: request.Type, Database: request.Database, MultiSeries: request.MultiSeries, ShardId: &self.id, RequestNumber: request.RequestNumber}
server.BufferWrite(requestWithoutId)
}
return nil
}
// Makes a request to the server. If the responseStream chan is not nil it will expect a response from the server
// with a matching request.Id. The REQUEST_RETRY_ATTEMPTS constant of 3 and the RECONNECT_RETRY_WAIT of 100ms means
// that an attempt to make a request to a downed server will take 300ms to time out.
func (self *ProtobufClient) MakeRequest(request *protocol.Request, responseStream chan *protocol.Response) error {
if responseStream != nil {
self.requestBufferLock.Lock()
// this should actually never happen. The sweeper should clear out dead requests
// before the uint32 ids roll over.
if oldReq, alreadyHasRequestById := self.requestBuffer[*request.Id]; alreadyHasRequestById {
log.Error("already has a request with this id, must have timed out")
close(oldReq.responseChan)
}
self.requestBuffer[*request.Id] = &runningRequest{time.Now(), responseStream}
self.requestBufferLock.Unlock()
}
data, err := request.Encode()
if err != nil {
return err
}
// retry sending this at least a few times
for attempts := 0; attempts < REQUEST_RETRY_ATTEMPTS; attempts++ {
conn := self.getConnection()
if conn == nil {
self.reconnect()
continue
}
err = binary.Write(conn, binary.LittleEndian, uint32(len(data)))
if err == nil {
_, err = conn.Write(data)
if err == nil {
return nil
}
}
log.Error("ProtobufClient: error making request: %s", err)
// TODO: do something smarter here based on whatever the error is.
// failed to make the request, reconnect and try again.
self.reconnect()
}
// if we got here it errored out, clear out the request
self.requestBufferLock.Lock()
delete(self.requestBuffer, *request.Id)
self.requestBufferLock.Unlock()
return err
}
func (self *ShardData) WriteLocalOnly(request *protocol.Request) error {
requestNumber, err := self.wal.AssignSequenceNumbersAndLog(request, self)
if err != nil {
return err
}
request.RequestNumber = &requestNumber
self.store.BufferWrite(request)
return nil
}
func (self *CoordinatorImpl) proxyWrite(clusterServer *ClusterServer, request *protocol.Request) error {
originatingServerId := request.OriginatingServerId
request.OriginatingServerId = nil
defer func() { request.OriginatingServerId = originatingServerId }()
responseChan := make(chan *protocol.Response, 1)
err := clusterServer.MakeRequest(request, responseChan)
if err != nil {
log.Warn("PROXY WRITE ERROR: ", err)
return err
}
response := <-responseChan
if *response.Type == protocol.Response_WRITE_OK {
return nil
} else {
return errors.New(response.GetErrorMessage())
}
}
func (self *WriteBuffer) write(request *protocol.Request) {
attempts := 0
for {
self.shardIds[*request.ShardId] = true
requestNumber := *request.RequestNumber
err := self.writer.Write(request)
if err == nil {
self.shardCommitedRequestNumber[request.GetShardId()] = request.GetRequestNumber()
self.wal.Commit(requestNumber, self.serverId)
return
}
if attempts%100 == 0 {
log.Error("%s: WriteBuffer: error on write to server %d: %s", self.writerInfo, self.serverId, err)
}
attempts += 1
// backoff happens in the writer, just sleep for a small fixed amount of time before retrying
time.Sleep(time.Millisecond * 100)
}
}
func (self *ShardData) LogAndHandleDestructiveQuery(querySpec *parser.QuerySpec, request *protocol.Request, response chan *protocol.Response, runLocalOnly bool) error {
requestNumber, err := self.wal.AssignSequenceNumbersAndLog(request, self)
if err != nil {
return err
}
var localResponses chan *protocol.Response
if self.localShard != nil {
localResponses = make(chan *protocol.Response, 1)
// this doesn't really apply at this point since destructive queries don't output anything, but it may later
maxPointsFromDestructiveQuery := 1000
processor := engine.NewPassthroughEngine(localResponses, maxPointsFromDestructiveQuery)
err := self.localShard.Query(querySpec, processor)
processor.Close()
if err != nil {
return err
}
}
if !runLocalOnly {
responses := make([]chan *protocol.Response, len(self.clusterServers), len(self.clusterServers))
for i, server := range self.clusterServers {
responseChan := make(chan *protocol.Response, 1)
responses[i] = responseChan
// do this so that a new id will get assigned
request.Id = nil
server.MakeRequest(request, responseChan)
}
for i, responseChan := range responses {
for {
res := <-responseChan
if *res.Type == endStreamResponse {
self.wal.Commit(requestNumber, self.clusterServers[i].Id)
break
}
response <- res
}
}
}
if localResponses != nil {
for {
res := <-localResponses
if *res.Type == endStreamResponse {
self.wal.Commit(requestNumber, self.localServerId)
break
}
response <- res
}
}
response <- &protocol.Response{Type: &endStreamResponse}
return nil
}
func (self *ProtobufRequestHandler) handleSequenceNumberRequest(request *protocol.Request, conn net.Conn) {
replicationFactor := uint8(*request.ReplicationFactor)
var err error
lastKnownSequenceNumber, err := self.coordinator.GetLastSequenceNumber(replicationFactor, *request.OriginatingServerId, *request.OwnerServerId)
var response *protocol.Response
if err != nil {
response = &protocol.Response{Type: &sequenceNumberResponse, Request: request, RequestId: request.Id, ErrorCode: &internalError}
} else {
response = &protocol.Response{Type: &sequenceNumberResponse, Request: request, RequestId: request.Id}
request.LastKnownSequenceNumber = &lastKnownSequenceNumber
}
self.WriteResponse(conn, response)
}
// This method never blocks. It'll buffer writes until they fill the buffer then drop the on the
// floor and let the background goroutine replay from the WAL
func (self *WriteBuffer) Write(request *protocol.Request) {
self.shardLastRequestNumber[request.GetShardId()] = request.GetRequestNumber()
select {
case self.writes <- request:
log.Debug("Buffering %d:%d for %s", request.GetRequestNumber(), request.GetShardId(), self.writerInfo)
return
default:
select {
case self.stoppedWrites <- *request.RequestNumber:
return
default:
return
}
}
}
func (self *ShardData) SyncWrite(request *p.Request) error {
request.ShardId = &self.id
for _, server := range self.clusterServers {
if err := server.Write(request); err != nil {
return err
}
}
if self.store == nil {
return nil
}
return self.store.Write(request)
}
func (self *ShardData) HandleDestructiveQuery(querySpec *parser.QuerySpec, request *p.Request, response chan *p.Response, runLocalOnly bool) {
if !self.IsLocal && runLocalOnly {
panic("WTF islocal is false and runLocalOnly is true")
}
responseChannels := []<-chan *p.Response{}
serverIds := []uint32{}
if self.IsLocal {
channel, err := self.deleteDataLocally(querySpec)
if err != nil {
msg := err.Error()
response <- &p.Response{Type: &endStreamResponse, ErrorMessage: &msg}
log.Error(msg)
return
}
responseChannels = append(responseChannels, channel)
serverIds = append(serverIds, self.localServerId)
}
log.Debug("request %s, runLocalOnly: %v", request.GetDescription(), runLocalOnly)
if !runLocalOnly {
responses, ids, _ := self.forwardRequest(request)
serverIds = append(serverIds, ids...)
responseChannels = append(responseChannels, responses...)
}
accessDenied := false
for idx, channel := range responseChannels {
serverId := serverIds[idx]
log.Debug("Waiting for response to %s from %d", request.GetDescription(), serverId)
for {
res := <-channel
log.Debug("Received %s response from %d for %s", res.GetType(), serverId, request.GetDescription())
if *res.Type == endStreamResponse {
break
}
// don't send the access denied response until the end so the readers don't close out before the other responses.
// See https://github.com/influxdb/influxdb/issues/316 for more info.
if *res.Type != accessDeniedResponse {
response <- res
} else {
accessDenied = true
}
}
}
if accessDenied {
response <- &p.Response{Type: &accessDeniedResponse}
}
response <- &p.Response{Type: &endStreamResponse}
}
func (self *ShardData) SyncWrite(request *p.Request, assignSeqNum bool) error {
if assignSeqNum {
self.wal.AssignSequenceNumbers(request)
}
request.ShardId = &self.id
for _, server := range self.clusterServers {
if err := server.Write(request); err != nil {
return err
}
}
if self.store == nil {
return nil
}
return self.store.Write(request)
}
func (self *DatastoreMock) LogRequestAndAssignSequenceNumber(request *protocol.Request, replicationFactor *uint8, ownerServerId *uint32) error {
id := uint64(1)
request.SequenceNumber = &id
return nil
}
func (self *ProtobufRequestHandler) HandleRequest(request *protocol.Request, conn net.Conn) error {
if *request.Type == protocol.Request_PROXY_WRITE {
response := &protocol.Response{RequestId: request.Id, Type: &self.writeOk}
location := common.RingLocation(request.Database, request.Series.Name, request.Series.Points[0].Timestamp)
ownerId := self.clusterConfig.GetOwnerIdByLocation(&location)
request.OriginatingServerId = &self.clusterConfig.localServerId
// TODO: make request logging and datastore write atomic
replicationFactor := self.clusterConfig.GetReplicationFactor(request.Database)
err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, ownerId)
if err != nil {
return err
}
err = self.db.WriteSeriesData(*request.Database, request.Series)
if err != nil {
return err
}
err = self.WriteResponse(conn, response)
// TODO: add quorum writes?
self.coordinator.ReplicateWrite(request)
return err
} else if *request.Type == protocol.Request_PROXY_DELETE {
response := &protocol.Response{RequestId: request.Id, Type: &self.writeOk}
request.OriginatingServerId = &self.clusterConfig.localServerId
// TODO: make request logging and datastore write atomic
replicationFactor := self.clusterConfig.GetReplicationFactor(request.Database)
err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
if err != nil {
return err
}
query, _ := parser.ParseQuery(*request.Query)
err = self.db.DeleteSeriesData(*request.Database, query[0].DeleteQuery)
if err != nil {
return err
}
err = self.WriteResponse(conn, response)
// TODO: add quorum writes?
self.coordinator.ReplicateDelete(request)
return err
} else if *request.Type == protocol.Request_REPLICATION_WRITE {
replicationFactor := self.clusterConfig.GetReplicationFactor(request.Database)
// TODO: make request logging and datastore write atomic
err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
if err != nil {
switch err := err.(type) {
case datastore.SequenceMissingRequestsError:
go self.coordinator.ReplayReplication(request, &replicationFactor, request.OwnerServerId, &err.LastKnownRequestSequence)
return nil
default:
return err
}
}
self.db.WriteSeriesData(*request.Database, request.Series)
return nil
} else if *request.Type == protocol.Request_REPLICATION_DELETE {
replicationFactor := self.clusterConfig.GetReplicationFactor(request.Database)
// TODO: make request logging and datastore write atomic
err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
if err != nil {
switch err := err.(type) {
case datastore.SequenceMissingRequestsError:
go self.coordinator.ReplayReplication(request, &replicationFactor, request.OwnerServerId, &err.LastKnownRequestSequence)
return nil
default:
return err
}
}
query, _ := parser.ParseQuery(*request.Query)
return self.db.DeleteSeriesData(*request.Database, query[0].DeleteQuery)
} else if *request.Type == protocol.Request_QUERY {
go self.handleQuery(request, conn)
} else if *request.Type == protocol.Request_REPLICATION_REPLAY {
self.handleReplay(request, conn)
} else {
log.Error("unknown request type: %v", request)
return errors.New("Unknown request type")
}
return nil
}
func (self *WriteBuffer) replayAndRecover(missedRequest uint32) {
var req *protocol.Request
for {
log.Info("%s: REPLAY: Replaying dropped requests...", self.writerInfo)
// empty out the buffer before the replay so new writes can buffer while we're replaying
channelLen := len(self.writes)
// if req is nil, this is the first run through the replay. Start from the start of the write queue
if req == nil {
for i := 0; i < channelLen; i++ {
r := <-self.writes
if req == nil {
req = r
}
}
}
if req == nil {
log.Error("%s: REPLAY: emptied channel, but no request set", self.writerInfo)
return
}
log.Debug("%s: REPLAY: Emptied out channel", self.writerInfo)
shardIds := make([]uint32, 0)
for shardId, _ := range self.shardIds {
shardIds = append(shardIds, shardId)
}
log.Debug("%s: REPLAY: from request %d. Shards: ", self.writerInfo, req.GetRequestNumber(), shardIds)
self.wal.RecoverServerFromRequestNumber(*req.RequestNumber, shardIds, func(request *protocol.Request, shardId uint32) error {
log.Debug("%s: REPLAY: writing request number: %d", self.writerInfo, request.GetRequestNumber())
req = request
request.ShardId = &shardId
self.write(request)
return nil
})
log.Info("%s: REPLAY: Emptying out reqeusts from buffer that we've already replayed", self.writerInfo)
RequestLoop:
for {
select {
case newReq := <-self.writes:
if *newReq.RequestNumber == *req.RequestNumber {
break RequestLoop
}
default:
log.Error("%s: REPLAY: Got to the end of the write buffer without getting to the last written request.", self.writerInfo)
break RequestLoop
}
}
log.Info("%s: REPLAY: done.", self.writerInfo)
// now make sure that no new writes were dropped. If so, do the replay again from this place.
select {
case <-self.stoppedWrites:
log.Info("%s: REPLAY: Buffer backed up while replaying, going again.", self.writerInfo)
continue
default:
return
}
}
}
func (self *ProtobufRequestHandler) HandleRequest(request *protocol.Request, conn net.Conn) error {
if *request.Type == protocol.Request_PROXY_WRITE {
response := &protocol.Response{RequestId: request.Id, Type: &self.writeOk}
request.OriginatingServerId = &self.clusterConfig.localServerId
// TODO: make request logging and datastore write atomic
replicationFactor := self.clusterConfig.GetReplicationFactor(request.Database)
err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
if err != nil {
return err
}
err = self.db.WriteSeriesData(*request.Database, request.Series)
if err != nil {
return err
}
err = self.WriteResponse(conn, response)
// TODO: add quorum writes?
self.coordinator.ReplicateWrite(request)
return err
} else if *request.Type == protocol.Request_PROXY_DROP_SERIES {
response := &protocol.Response{RequestId: request.Id, Type: &self.writeOk}
request.OriginatingServerId = &self.clusterConfig.localServerId
replicationFactor := uint8(*request.ReplicationFactor)
// TODO: make request logging and datastore write atomic
err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
if err != nil {
return err
}
err = self.db.DropSeries(*request.Database, *request.Series.Name)
if err != nil {
return err
}
err = self.WriteResponse(conn, response)
self.coordinator.ReplicateWrite(request)
return err
} else if *request.Type == protocol.Request_REPLICATION_DROP_SERIES {
replicationFactor := uint8(*request.ReplicationFactor)
// TODO: make request logging and datastore write atomic
err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
if err != nil {
switch err := err.(type) {
case datastore.SequenceMissingRequestsError:
log.Warn("Missing sequence number error: Request SN: %v Last Known SN: %v", request.GetSequenceNumber(), err.LastKnownRequestSequence)
go self.coordinator.ReplayReplication(request, &replicationFactor, request.OwnerServerId, &err.LastKnownRequestSequence)
return nil
default:
return err
}
}
return self.db.DropSeries(*request.Database, *request.Series.Name)
} else if *request.Type == protocol.Request_PROXY_DROP_DATABASE {
response := &protocol.Response{RequestId: request.Id, Type: &self.writeOk}
request.OriginatingServerId = &self.clusterConfig.localServerId
replicationFactor := uint8(*request.ReplicationFactor)
// TODO: make request logging and datastore write atomic
err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
if err != nil {
return err
}
err = self.db.DropDatabase(*request.Database)
if err != nil {
return err
}
err = self.WriteResponse(conn, response)
self.coordinator.ReplicateWrite(request)
return err
} else if *request.Type == protocol.Request_REPLICATION_DROP_DATABASE {
replicationFactor := uint8(*request.ReplicationFactor)
// TODO: make request logging and datastore write atomic
err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
if err != nil {
switch err := err.(type) {
case datastore.SequenceMissingRequestsError:
log.Warn("Missing sequence number error: Request SN: %v Last Known SN: %v", request.GetSequenceNumber(), err.LastKnownRequestSequence)
go self.coordinator.ReplayReplication(request, &replicationFactor, request.OwnerServerId, &err.LastKnownRequestSequence)
return nil
default:
return err
}
}
return self.db.DropDatabase(*request.Database)
} else if *request.Type == protocol.Request_PROXY_DELETE {
response := &protocol.Response{RequestId: request.Id, Type: &self.writeOk}
request.OriginatingServerId = &self.clusterConfig.localServerId
// TODO: make request logging and datastore write atomic
replicationFactor := self.clusterConfig.GetReplicationFactor(request.Database)
err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
if err != nil {
return err
}
query, err := parser.ParseQuery(*request.Query)
if err != nil {
return err
}
err = self.db.DeleteSeriesData(*request.Database, query[0].DeleteQuery)
if err != nil {
return err
}
err = self.WriteResponse(conn, response)
// TODO: add quorum writes?
self.coordinator.ReplicateDelete(request)
return err
} else if *request.Type == protocol.Request_REPLICATION_WRITE {
replicationFactor := self.clusterConfig.GetReplicationFactor(request.Database)
// TODO: make request logging and datastore write atomic
err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
if err != nil {
switch err := err.(type) {
case datastore.SequenceMissingRequestsError:
log.Warn("Missing sequence number error: Request SN: %v Last Known SN: %v", request.GetSequenceNumber(), err.LastKnownRequestSequence)
go self.coordinator.ReplayReplication(request, &replicationFactor, request.OwnerServerId, &err.LastKnownRequestSequence)
return nil
default:
return err
}
}
self.db.WriteSeriesData(*request.Database, request.Series)
return nil
} else if *request.Type == protocol.Request_REPLICATION_DELETE {
replicationFactor := self.clusterConfig.GetReplicationFactor(request.Database)
// TODO: make request logging and datastore write atomic
err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
if err != nil {
switch err := err.(type) {
case datastore.SequenceMissingRequestsError:
go self.coordinator.ReplayReplication(request, &replicationFactor, request.OwnerServerId, &err.LastKnownRequestSequence)
return nil
default:
return err
}
}
query, err := parser.ParseQuery(*request.Query)
if err != nil {
return err
}
return self.db.DeleteSeriesData(*request.Database, query[0].DeleteQuery)
} else if *request.Type == protocol.Request_QUERY {
go self.handleQuery(request, conn)
} else if *request.Type == protocol.Request_LIST_SERIES {
go self.handleListSeries(request, conn)
} else if *request.Type == protocol.Request_REPLICATION_REPLAY {
self.handleReplay(request, conn)
} else if *request.Type == protocol.Request_SEQUENCE_NUMBER {
self.handleSequenceNumberRequest(request, conn)
} else {
log.Error("unknown request type: %v", request)
return errors.New("Unknown request type")
}
return nil
}