func (self *log) appendRequest(request *protocol.Request, shardId uint32) error {
bytes, err := request.Encode()
if err != nil {
return err
}
// every request is preceded with the length, shard id and the request number
hdr := &entryHeader{
shardId: shardId,
requestNumber: request.GetRequestNumber(),
length: uint32(len(bytes)),
}
writtenHdrBytes, err := hdr.Write(self.file)
if err != nil {
logger.Error("Error while writing header: %s", err)
return err
}
written, err := self.file.Write(bytes)
if err != nil {
logger.Error("Error while writing request: %s", err)
return err
}
if written < len(bytes) {
err = fmt.Errorf("Couldn't write entire request")
logger.Error("Error while writing request: %s", err)
return err
}
self.fileSize += uint64(writtenHdrBytes + written)
return nil
}
func (self *log) appendRequest(request *protocol.Request, shardId uint32) (uint32, error) {
self.assignSequenceNumbers(shardId, request)
bytes, err := request.Encode()
if err != nil {
return 0, err
}
requestNumber := self.state.getNextRequestNumber()
// every request is preceded with the length, shard id and the request number
hdr := &entryHeader{
shardId: shardId,
requestNumber: requestNumber,
length: uint32(len(bytes)),
}
writtenHdrBytes, err := hdr.Write(self.file)
if err != nil {
logger.Error("Error while writing header: %s", err)
return 0, err
}
written, err := self.file.Write(bytes)
if err != nil {
logger.Error("Error while writing request: %s", err)
return 0, err
}
if written < len(bytes) {
err = fmt.Errorf("Couldn't write entire request")
logger.Error("Error while writing request: %s", err)
return 0, err
}
self.fileSize += uint64(writtenHdrBytes + written)
self.conditionalBookmarkAndIndex()
return requestNumber, nil
}
// Makes a request to the server. If the responseStream chan is not nil it will expect a response from the server
// with a matching request.Id. The REQUEST_RETRY_ATTEMPTS constant of 3 and the RECONNECT_RETRY_WAIT of 100ms means
// that an attempt to make a request to a downed server will take 300ms to time out.
func (self *ProtobufClient) MakeRequest(request *protocol.Request, responseStream chan *protocol.Response) error {
if request.Id == nil {
id := atomic.AddUint32(&self.lastRequestId, uint32(1))
request.Id = &id
}
if responseStream != nil {
self.requestBufferLock.Lock()
// this should actually never happen. The sweeper should clear out dead requests
// before the uint32 ids roll over.
if oldReq, alreadyHasRequestById := self.requestBuffer[*request.Id]; alreadyHasRequestById {
message := "already has a request with this id, must have timed out"
log.Error(message)
oldReq.responseChan <- &protocol.Response{Type: &endStreamResponse, ErrorMessage: &message}
}
self.requestBuffer[*request.Id] = &runningRequest{timeMade: time.Now(), responseChan: responseStream, request: request}
self.requestBufferLock.Unlock()
}
data, err := request.Encode()
if err != nil {
return err
}
conn := self.getConnection()
if conn == nil {
conn = self.reconnect()
if conn == nil {
return fmt.Errorf("Failed to connect to server %s", self.hostAndPort)
}
}
if self.writeTimeout > 0 {
conn.SetWriteDeadline(time.Now().Add(self.writeTimeout))
}
buff := bytes.NewBuffer(make([]byte, 0, len(data)+8))
binary.Write(buff, binary.LittleEndian, uint32(len(data)))
_, err = conn.Write(append(buff.Bytes(), data...))
if err == nil {
return nil
}
// if we got here it errored out, clear out the request
self.requestBufferLock.Lock()
delete(self.requestBuffer, *request.Id)
self.requestBufferLock.Unlock()
self.reconnect()
return err
}
func (self *LevelDbDatastore) LogRequestAndAssignSequenceNumber(request *protocol.Request, replicationFactor *uint8, ownerServerId *uint32) error {
// log to this key structure on a different DB sharded by day: <cluster version><owner id><sequence server id><replication sequence>
var numberKey []byte
if request.SequenceNumber == nil {
sequenceNumber, err := self.AtomicIncrement(self.keyForOwnerAndServerSequenceNumber(request.ClusterVersion, replicationFactor, ownerServerId, request.OriginatingServerId), 1)
if err != nil {
return err
}
request.SequenceNumber = &sequenceNumber
} else {
// this is for a replicated write, ensure that it's the next in line for this owner and server
name := self.keyForOwnerAndServerSequenceNumber(request.ClusterVersion, replicationFactor, ownerServerId, request.OriginatingServerId)
numberKey = append(ATOMIC_INCREMENT_PREFIX, []byte(name)...)
numberBytes, err := self.db.Get(self.readOptions, numberKey)
if err != nil {
return err
}
previousSequenceNumber := self.bytesToCurrentNumber(numberBytes)
if previousSequenceNumber+uint64(1) != *request.SequenceNumber {
return SequenceMissingRequestsError{"Missing requests between last seen and this one.", previousSequenceNumber}
}
}
self.requestLogLock.RLock()
requestLog := self.currentRequestLog
self.requestLogLock.RUnlock()
// proxied writes should be logged as replicated ones. That's what is expected if they're replayed later
if *request.Type == protocol.Request_PROXY_WRITE {
request.Type = &replicateWrite
}
data, err := request.Encode()
if err != nil {
return err
}
key := self.requestLogKey(request.ClusterVersion, request.OriginatingServerId, ownerServerId, request.SequenceNumber, replicationFactor)
err = requestLog.db.Put(self.writeOptions, key, data)
if err != nil {
return err
}
if numberKey != nil {
currentNumberBuffer := bytes.NewBuffer(make([]byte, 0, 8))
binary.Write(currentNumberBuffer, binary.BigEndian, *request.SequenceNumber)
self.db.Put(self.writeOptions, numberKey, currentNumberBuffer.Bytes())
}
return nil
}
func (self *LevelDbDatastore) LogRequestAndAssignSequenceNumber(request *protocol.Request, replicationFactor *uint8, ownerServerId *uint32) error {
// log to this key structure on a different DB sharded by day: <cluster version><owner id><sequence server id><replication sequence>
updateSequenceNumber := false
if request.SequenceNumber == nil {
sequenceNumber, err := self.AtomicIncrement(self.keyForOwnerAndServerSequenceNumber(request.ClusterVersion, replicationFactor, ownerServerId, request.OriginatingServerId), 1)
if err != nil {
return err
}
request.SequenceNumber = &sequenceNumber
} else {
updateSequenceNumber = true
previousSequenceNumber, err := self.CurrentSequenceNumber(request.ClusterVersion, replicationFactor, ownerServerId, request.OriginatingServerId)
if err != nil {
return err
}
// Do a less than comparison because it's ok if we're just getting the same write again. As long as we haven't missed one.
if previousSequenceNumber+uint64(1) < *request.SequenceNumber {
log.Warn("MISSING REQUESTS: %d, %d", previousSequenceNumber, *request.SequenceNumber)
return SequenceMissingRequestsError{"Missing requests between last seen and this one.", previousSequenceNumber, *request.SequenceNumber}
}
}
self.requestLogLock.RLock()
requestLog := self.currentRequestLog
self.requestLogLock.RUnlock()
// proxied writes should be logged as replicated ones. That's what is expected if they're replayed later
if *request.Type == protocol.Request_PROXY_WRITE {
request.Type = &replicateWrite
}
data, err := request.Encode()
if err != nil {
return err
}
key := NewWALKey(request.ClusterVersion, request.OriginatingServerId, ownerServerId, request.SequenceNumber, replicationFactor)
err = requestLog.db.Put(self.writeOptions, key, data)
if err != nil {
return err
}
if updateSequenceNumber {
return self.updateSequenceNumber(request.ClusterVersion, replicationFactor, ownerServerId, request.OriginatingServerId, *request.SequenceNumber)
}
return nil
}
// Makes a request to the server. If the responseStream chan is not nil it will expect a response from the server
// with a matching request.Id. The REQUEST_RETRY_ATTEMPTS constant of 3 and the RECONNECT_RETRY_WAIT of 100ms means
// that an attempt to make a request to a downed server will take 300ms to time out.
func (self *ProtobufClient) MakeRequest(request *protocol.Request, responseStream chan *protocol.Response) error {
if responseStream != nil {
self.requestBufferLock.Lock()
// this should actually never happen. The sweeper should clear out dead requests
// before the uint32 ids roll over.
if oldReq, alreadyHasRequestById := self.requestBuffer[*request.Id]; alreadyHasRequestById {
log.Error("already has a request with this id, must have timed out")
close(oldReq.responseChan)
}
self.requestBuffer[*request.Id] = &runningRequest{time.Now(), responseStream}
self.requestBufferLock.Unlock()
}
data, err := request.Encode()
if err != nil {
return err
}
// retry sending this at least a few times
for attempts := 0; attempts < REQUEST_RETRY_ATTEMPTS; attempts++ {
conn := self.getConnection()
if conn == nil {
self.reconnect()
continue
}
err = binary.Write(conn, binary.LittleEndian, uint32(len(data)))
if err == nil {
_, err = conn.Write(data)
if err == nil {
return nil
}
}
log.Error("ProtobufClient: error making request: %s", err)
// TODO: do something smarter here based on whatever the error is.
// failed to make the request, reconnect and try again.
self.reconnect()
}
// if we got here it errored out, clear out the request
self.requestBufferLock.Lock()
delete(self.requestBuffer, *request.Id)
self.requestBufferLock.Unlock()
return err
}
