func (self *log) appendRequest(request *protocol.Request, shardId uint32) error { bytes, err := request.Encode() if err != nil { return err } // every request is preceded with the length, shard id and the request number hdr := &entryHeader{ shardId: shardId, requestNumber: request.GetRequestNumber(), length: uint32(len(bytes)), } writtenHdrBytes, err := hdr.Write(self.file) if err != nil { logger.Error("Error while writing header: %s", err) return err } written, err := self.file.Write(bytes) if err != nil { logger.Error("Error while writing request: %s", err) return err } if written < len(bytes) { err = fmt.Errorf("Couldn't write entire request") logger.Error("Error while writing request: %s", err) return err } self.fileSize += uint64(writtenHdrBytes + written) return nil }
func (self *log) appendRequest(request *protocol.Request, shardId uint32) (uint32, error) { self.assignSequenceNumbers(shardId, request) bytes, err := request.Encode() if err != nil { return 0, err } requestNumber := self.state.getNextRequestNumber() // every request is preceded with the length, shard id and the request number hdr := &entryHeader{ shardId: shardId, requestNumber: requestNumber, length: uint32(len(bytes)), } writtenHdrBytes, err := hdr.Write(self.file) if err != nil { logger.Error("Error while writing header: %s", err) return 0, err } written, err := self.file.Write(bytes) if err != nil { logger.Error("Error while writing request: %s", err) return 0, err } if written < len(bytes) { err = fmt.Errorf("Couldn't write entire request") logger.Error("Error while writing request: %s", err) return 0, err } self.fileSize += uint64(writtenHdrBytes + written) self.conditionalBookmarkAndIndex() return requestNumber, nil }
// Makes a request to the server. If the responseStream chan is not nil it will expect a response from the server // with a matching request.Id. The REQUEST_RETRY_ATTEMPTS constant of 3 and the RECONNECT_RETRY_WAIT of 100ms means // that an attempt to make a request to a downed server will take 300ms to time out. func (self *ProtobufClient) MakeRequest(request *protocol.Request, responseStream chan *protocol.Response) error { if request.Id == nil { id := atomic.AddUint32(&self.lastRequestId, uint32(1)) request.Id = &id } if responseStream != nil { self.requestBufferLock.Lock() // this should actually never happen. The sweeper should clear out dead requests // before the uint32 ids roll over. if oldReq, alreadyHasRequestById := self.requestBuffer[*request.Id]; alreadyHasRequestById { message := "already has a request with this id, must have timed out" log.Error(message) oldReq.responseChan <- &protocol.Response{Type: &endStreamResponse, ErrorMessage: &message} } self.requestBuffer[*request.Id] = &runningRequest{timeMade: time.Now(), responseChan: responseStream, request: request} self.requestBufferLock.Unlock() } data, err := request.Encode() if err != nil { return err } conn := self.getConnection() if conn == nil { conn = self.reconnect() if conn == nil { return fmt.Errorf("Failed to connect to server %s", self.hostAndPort) } } if self.writeTimeout > 0 { conn.SetWriteDeadline(time.Now().Add(self.writeTimeout)) } buff := bytes.NewBuffer(make([]byte, 0, len(data)+8)) binary.Write(buff, binary.LittleEndian, uint32(len(data))) _, err = conn.Write(append(buff.Bytes(), data...)) if err == nil { return nil } // if we got here it errored out, clear out the request self.requestBufferLock.Lock() delete(self.requestBuffer, *request.Id) self.requestBufferLock.Unlock() self.reconnect() return err }
func (self *LevelDbDatastore) LogRequestAndAssignSequenceNumber(request *protocol.Request, replicationFactor *uint8, ownerServerId *uint32) error { // log to this key structure on a different DB sharded by day: <cluster version><owner id><sequence server id><replication sequence> var numberKey []byte if request.SequenceNumber == nil { sequenceNumber, err := self.AtomicIncrement(self.keyForOwnerAndServerSequenceNumber(request.ClusterVersion, replicationFactor, ownerServerId, request.OriginatingServerId), 1) if err != nil { return err } request.SequenceNumber = &sequenceNumber } else { // this is for a replicated write, ensure that it's the next in line for this owner and server name := self.keyForOwnerAndServerSequenceNumber(request.ClusterVersion, replicationFactor, ownerServerId, request.OriginatingServerId) numberKey = append(ATOMIC_INCREMENT_PREFIX, []byte(name)...) numberBytes, err := self.db.Get(self.readOptions, numberKey) if err != nil { return err } previousSequenceNumber := self.bytesToCurrentNumber(numberBytes) if previousSequenceNumber+uint64(1) != *request.SequenceNumber { return SequenceMissingRequestsError{"Missing requests between last seen and this one.", previousSequenceNumber} } } self.requestLogLock.RLock() requestLog := self.currentRequestLog self.requestLogLock.RUnlock() // proxied writes should be logged as replicated ones. That's what is expected if they're replayed later if *request.Type == protocol.Request_PROXY_WRITE { request.Type = &replicateWrite } data, err := request.Encode() if err != nil { return err } key := self.requestLogKey(request.ClusterVersion, request.OriginatingServerId, ownerServerId, request.SequenceNumber, replicationFactor) err = requestLog.db.Put(self.writeOptions, key, data) if err != nil { return err } if numberKey != nil { currentNumberBuffer := bytes.NewBuffer(make([]byte, 0, 8)) binary.Write(currentNumberBuffer, binary.BigEndian, *request.SequenceNumber) self.db.Put(self.writeOptions, numberKey, currentNumberBuffer.Bytes()) } return nil }
func (self *LevelDbDatastore) LogRequestAndAssignSequenceNumber(request *protocol.Request, replicationFactor *uint8, ownerServerId *uint32) error { // log to this key structure on a different DB sharded by day: <cluster version><owner id><sequence server id><replication sequence> updateSequenceNumber := false if request.SequenceNumber == nil { sequenceNumber, err := self.AtomicIncrement(self.keyForOwnerAndServerSequenceNumber(request.ClusterVersion, replicationFactor, ownerServerId, request.OriginatingServerId), 1) if err != nil { return err } request.SequenceNumber = &sequenceNumber } else { updateSequenceNumber = true previousSequenceNumber, err := self.CurrentSequenceNumber(request.ClusterVersion, replicationFactor, ownerServerId, request.OriginatingServerId) if err != nil { return err } // Do a less than comparison because it's ok if we're just getting the same write again. As long as we haven't missed one. if previousSequenceNumber+uint64(1) < *request.SequenceNumber { log.Warn("MISSING REQUESTS: %d, %d", previousSequenceNumber, *request.SequenceNumber) return SequenceMissingRequestsError{"Missing requests between last seen and this one.", previousSequenceNumber, *request.SequenceNumber} } } self.requestLogLock.RLock() requestLog := self.currentRequestLog self.requestLogLock.RUnlock() // proxied writes should be logged as replicated ones. That's what is expected if they're replayed later if *request.Type == protocol.Request_PROXY_WRITE { request.Type = &replicateWrite } data, err := request.Encode() if err != nil { return err } key := NewWALKey(request.ClusterVersion, request.OriginatingServerId, ownerServerId, request.SequenceNumber, replicationFactor) err = requestLog.db.Put(self.writeOptions, key, data) if err != nil { return err } if updateSequenceNumber { return self.updateSequenceNumber(request.ClusterVersion, replicationFactor, ownerServerId, request.OriginatingServerId, *request.SequenceNumber) } return nil }
// Makes a request to the server. If the responseStream chan is not nil it will expect a response from the server // with a matching request.Id. The REQUEST_RETRY_ATTEMPTS constant of 3 and the RECONNECT_RETRY_WAIT of 100ms means // that an attempt to make a request to a downed server will take 300ms to time out. func (self *ProtobufClient) MakeRequest(request *protocol.Request, responseStream chan *protocol.Response) error { if responseStream != nil { self.requestBufferLock.Lock() // this should actually never happen. The sweeper should clear out dead requests // before the uint32 ids roll over. if oldReq, alreadyHasRequestById := self.requestBuffer[*request.Id]; alreadyHasRequestById { log.Error("already has a request with this id, must have timed out") close(oldReq.responseChan) } self.requestBuffer[*request.Id] = &runningRequest{time.Now(), responseStream} self.requestBufferLock.Unlock() } data, err := request.Encode() if err != nil { return err } // retry sending this at least a few times for attempts := 0; attempts < REQUEST_RETRY_ATTEMPTS; attempts++ { conn := self.getConnection() if conn == nil { self.reconnect() continue } err = binary.Write(conn, binary.LittleEndian, uint32(len(data))) if err == nil { _, err = conn.Write(data) if err == nil { return nil } } log.Error("ProtobufClient: error making request: %s", err) // TODO: do something smarter here based on whatever the error is. // failed to make the request, reconnect and try again. self.reconnect() } // if we got here it errored out, clear out the request self.requestBufferLock.Lock() delete(self.requestBuffer, *request.Id) self.requestBufferLock.Unlock() return err }