func UnmarshalPQMsgMetaData(sn uint64, buf []byte) *PQMsgMetaData {
p := PQMsgMetaData{SerialNumber: sn}
if err := p.Unmarshal(buf); err != nil {
log.Error("Could not unmarshal message: %d", sn)
return nil
}
return &p
}
// SaveServiceConfig saves service config into database.
func SaveServiceConfig(serviceId string, conf apis.BinaryMarshaller) error {
db := db.DatabaseInstance()
data, _ := conf.Marshal()
err := db.StoreData(cfgKey(serviceId), data)
if err != nil {
log.Error("Failed to save config: %s", err.Error())
return mpqerr.ServerError("Can not save service data: " + serviceId)
}
return nil
}
func LoadServiceConfig(serviceId string, cfg apis.BinaryMarshaller) error {
db := db.DatabaseInstance()
data := db.GetData(cfgKey(serviceId))
if len(data) == 0 {
return mpqerr.NotFoundRequest("No service settings found: " + serviceId)
}
if err := cfg.Unmarshal(data); err != nil {
log.Error("Error in '%s' service settings: %s", serviceId, err.Error())
return mpqerr.ServerError("Service settings error: " + serviceId)
}
return nil
}
func DeleteServiceData(serviceId string) {
desc := GetServiceDescription(serviceId)
if desc == nil {
log.Error("Attempt to delete unknown service id: %s", serviceId)
return
}
desc.ToDelete = true
SaveServiceDescription(desc)
db := db.DatabaseInstance()
db.DeleteDataWithPrefix(serviceId)
db.DeleteData(cfgKey(serviceId))
db.DeleteData(descKey(serviceId))
}
// Close flushes data on disk and closes database.
func (ds *LevelDBStorage) Close() {
ds.flushLock.Lock()
defer ds.flushLock.Unlock()
if !ds.closed {
log.Info("Flushing database cache")
ds.FlushCache()
ds.closed = true
log.Info("Closing the database")
ds.db.Close()
log.Info("The database has been closed.")
} else {
log.Error("Attempt to close database more than once!")
}
}
func getDatabase() *LevelDBStorage {
var err error
if database == nil {
database, err = NewLevelDBStorage(conf.CFG)
if err != nil {
log.Error("Cannot initialize FireMPQ database: %s", err)
os.Exit(255)
}
}
if database.closed {
database = nil
return getDatabase()
}
return database
}
func (s *ServiceManager) loadService(desc *queue_info.ServiceDescription) (apis.ISvc, bool) {
if desc.Disabled {
log.Error("Service is disabled. Skipping: %s", desc.Name)
return nil, false
}
if desc.ToDelete {
log.Warning("Service should be deleted: %s", desc.Name)
queue_info.DeleteServiceData(desc.Name)
return nil, false
}
log.Debug("Loading service data for: %s", desc.Name)
serviceLoader, ok := GetServiceLoader(desc.SType)
if !ok {
log.Error("Unknown service '%s' type: %s", desc.Name, desc.SType)
return nil, false
}
svcInstance, err := serviceLoader(s, desc)
if err != nil {
log.Error("Service '%s' was not loaded because of: %s", desc.Name, err)
return nil, false
}
return svcInstance, true
}
// GetServiceDescriptions Loads all service descriptions prefixed with ServiceDescPrefix
func GetServiceDescriptions() ServiceDescriptionList {
sdList := make(ServiceDescriptionList, 0, 16)
db := db.DatabaseInstance()
descIter := db.IterData(ServiceDescPrefix)
defer descIter.Close()
for ; descIter.Valid(); descIter.Next() {
svcDesc, err := UnmarshalServiceDesc([]byte(descIter.GetValue()))
if err != nil {
log.Error("Coudn't read service '%s' description: %s", descIter.GetTrimKey(), err.Error())
continue
}
sdList = append(sdList, svcDesc)
}
return sdList
}
func getDatabase() apis.DataStorage {
if database == nil {
var err error
database, err = ldb.NewLevelDBStorage(conf.CFG)
if err != nil {
log.Error("Cannot initialize FireMPQ database: %s", err)
os.Exit(255)
}
return database
}
if database.IsClosed() {
database = nil
return getDatabase()
}
return database
}
func main() {
// Initialize logging to a default INFO level to be able to log config error.
conf.ParseConfigParameters()
if len(conf.CFG.Profiler) > 0 {
go func() {
if err := http.ListenAndServe(conf.CFG.Profiler, nil); err != nil {
log.Error("Could not initialize profiler: %v", err)
}
}()
}
// Reinitialize log level according to the config data.
log.InitLogging()
srv, err := server.Server(server.SimpleServerType, conf.CFG.FMPQServerInterface)
if err != nil {
log.Critical("Error: %s", err.Error())
return
}
srv.Start()
}
func CheckAvailableQueues(
svcMgr *qmgr.ServiceManager,
attr *QueueAttributes,
sqsQuery *urlutils.SQSQuery) sqs_response.SQSResponse {
svc, ok := svcMgr.GetService(sqsQuery.QueueName)
if ok {
if svc.Info().Type != apis.ServiceTypePriorityQueue {
return sqserr.QueueAlreadyExistsError("Queue already exists for a different type of service")
}
pq, _ := svc.(*pqueue.PQueue)
pqConfig := pq.Config()
if !ok {
log.Error("Unexpected config type from the found service!")
return sqserr.ServerSideError("Queue config data error")
}
if attr.VisibilityTimeout >= 0 && attr.VisibilityTimeout != pqConfig.PopLockTimeout {
return sqserr.QueueAlreadyExistsError(errQueueExists + AttrVisibilityTimeout)
}
if attr.DelaySeconds >= 0 && attr.DelaySeconds != pqConfig.DeliveryDelay {
return sqserr.QueueAlreadyExistsError(errQueueExists + AttrDelaySeconds)
}
if attr.MaximumMessageSize >= 0 && attr.MaximumMessageSize != pqConfig.MaxMsgSize {
return sqserr.QueueAlreadyExistsError(errQueueExists + AttrMaximumMessageSize)
}
if attr.MessageRetentionPeriod >= 0 && attr.MessageRetentionPeriod != pqConfig.MsgTtl {
return sqserr.QueueAlreadyExistsError(errQueueExists + AttrMessageRetentionPeriod)
}
if attr.ReceiveMessageWaitTimeSeconds >= 0 && attr.ReceiveMessageWaitTimeSeconds != pqConfig.PopWaitTimeout {
return sqserr.QueueAlreadyExistsError(errQueueExists + AttrReceiveMessageWaitTimeSeconds)
}
return &CreateQueueResponse{
QueueUrl: sqsQuery.Host + "/queue/" + sqsQuery.QueueName,
RequestId: "1111-2222-3333",
}
}
return nil
}
func (tm *TopicManager) ListTopics(offset int) ([]string, int) {
tm.Lock()
defer tm.Unlock()
var res []string
maxLen := offset + ReturnBatchSize
ret_offset := maxLen
if maxLen >= len(tm.TopicList) {
ret_offset = 0
maxLen = len(tm.TopicList)
}
for i := offset; i < maxLen; i++ {
topicName := tm.TopicList[i]
if t := tm.Topics[topicName]; t == nil {
log.Error("Inconsistent data! No data for topic name: %s", topicName)
} else {
res = append(res, t.Arn)
}
}
return res, ret_offset
}
func MakeMessageResponse(iMsg apis.IResponseItem, opts *ReceiveMessageOptions,
sqsQuery *urlutils.SQSQuery) *MessageResponse {
msg, ok := iMsg.(*pqueue.MsgResponseItem)
if !ok {
return nil
}
sqsMsg := &sqsmsg.SQSMessagePayload{}
payload := msg.Payload()
if err := sqsMsg.Unmarshal([]byte(payload)); err != nil {
// Recovering from error. Non SQS messages will be filled with bulk info.
sqsMsg.Payload = string(msg.Payload())
sqsMsg.SenderId = "unknown"
sqsMsg.SentTimestamp = strconv.FormatInt(utils.Uts(), 10)
sqsMsg.MD5OfMessageAttributes = fmt.Sprintf("%x", md5.Sum(nil))
sqsMsg.MD5OfMessageBody = fmt.Sprintf("%x", md5.Sum(enc.UnsafeStringToBytes(sqsMsg.Payload)))
}
msgMeta := msg.GetMeta()
output := &MessageResponse{
MD5OfMessageAttributes: sqsMsg.MD5OfMessageAttributes,
MD5OfMessageBody: sqsMsg.MD5OfMessageBody,
ReceiptHandle: msg.Receipt(),
MessageId: msg.ID(),
Body: sqsMsg.Payload,
}
if opts.AllSysAttributes {
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrSenderId, Value: sqsMsg.SenderId,
})
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrSentTimestamp, Value: sqsMsg.SentTimestamp,
})
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrApproximateReceiveCount, Value: strconv.FormatInt(msgMeta.PopCount, 10),
})
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrApproximateFirstReceiveTimestamp, Value: strconv.FormatInt(utils.Uts(), 10),
})
} else {
for _, k := range opts.Attributes {
switch k {
case AttrSenderId:
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrSenderId, Value: sqsMsg.SenderId,
})
case AttrSentTimestamp:
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrSentTimestamp, Value: sqsMsg.SentTimestamp,
})
case AttrApproximateReceiveCount:
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrApproximateReceiveCount, Value: strconv.FormatInt(msgMeta.PopCount, 10),
})
case AttrApproximateFirstReceiveTimestamp:
output.Attributes = append(output.Attributes, &SysAttribute{
Name: AttrApproximateFirstReceiveTimestamp, Value: strconv.FormatInt(utils.Uts(), 10),
})
}
}
}
if opts.AllMessageAttributes {
for attrName, attrData := range sqsMsg.UserAttributes {
output.MessageAttributes = append(
output.MessageAttributes, MakeMessageAttr(attrName, attrData))
}
} else {
for _, attrName := range opts.MessageAttributes {
if v, ok := sqsMsg.UserAttributes[attrName]; ok {
output.MessageAttributes = append(
output.MessageAttributes, MakeMessageAttr(attrName, v))
}
}
}
if !ok {
log.Error("Failed to cast response item")
return nil
}
return output
}
func PushAMessage(pq *pqueue.PQueue, senderId string, paramList []string) sqs_response.SQSResponse {
out := &MessageParams{
DelaySeconds: -1,
MessageBody: "",
}
attrs, err := urlutils.ParseNNotationAttr("MessageAttribute.", paramList, out.Parse, NewReqQueueAttr)
if err != nil {
return err
}
attrsLen := len(attrs)
outAttrs := make(map[string]*sqsmsg.UserAttribute)
for i := 1; i <= attrsLen; i++ {
a, ok := attrs[i]
if !ok {
return sqserr.InvalidParameterValueError("The request must contain non-empty message attribute name.")
}
reqMsgAttr, _ := a.(*ReqMsgAttr)
sqs_err := validation.ValidateMessageAttrName(reqMsgAttr.Name)
if sqs_err != nil {
return sqs_err
}
sqs_err = validation.ValidateMessageAttrName(reqMsgAttr.DataType)
if sqs_err != nil {
return sqs_err
}
if reqMsgAttr.BinaryValue != "" && reqMsgAttr.StringValue != "" {
return sqserr.InvalidParameterValueError(
"Message attribute name '%s' has multiple values.", reqMsgAttr.Name)
}
if _, ok := outAttrs[reqMsgAttr.Name]; ok {
return sqserr.InvalidParameterValueError(
"Message attribute name '%s' already exists.", reqMsgAttr.Name)
}
if strings.HasPrefix(reqMsgAttr.DataType, "Number") {
if _, err := strconv.Atoi(reqMsgAttr.StringValue); err != nil {
return sqserr.InvalidParameterValueError(
"Could not cast message attribute '%s' value to number.", reqMsgAttr.Name)
}
}
if reqMsgAttr.BinaryValue != "" {
if reqMsgAttr.DataType != "Binary" {
return sqserr.InvalidParameterValueError(
"The message attribute '%s' with type 'Binary' must use field 'Binary'", reqMsgAttr.Name)
}
outAttrs[reqMsgAttr.Name] = &sqsmsg.UserAttribute{
Type: reqMsgAttr.DataType,
Value: reqMsgAttr.BinaryValue,
}
continue
}
if reqMsgAttr.StringValue != "" {
if reqMsgAttr.DataType != "String" && reqMsgAttr.DataType != "Number" {
return sqserr.InvalidParameterValueError(
"The message attribute '%s' with type 'String' must use field 'String'", reqMsgAttr.Name)
}
outAttrs[reqMsgAttr.Name] = &sqsmsg.UserAttribute{
Type: reqMsgAttr.DataType,
Value: reqMsgAttr.StringValue,
}
}
}
msgId := IdGen.RandId()
if out.DelaySeconds < 0 {
out.DelaySeconds = pq.Config().DeliveryDelay
} else if out.DelaySeconds > conf.CFG_PQ.MaxDeliveryDelay {
return sqserr.InvalidParameterValueError(
"Delay secods must be between 0 and %d", conf.CFG_PQ.MaxDeliveryDelay/1000)
}
bodyMd5str := fmt.Sprintf("%x", md5.Sum(enc.UnsafeStringToBytes(out.MessageBody)))
attrMd5 := CalcAttrMd5(outAttrs)
msgPayload := sqsmsg.SQSMessagePayload{
UserAttributes: outAttrs,
MD5OfMessageBody: bodyMd5str,
MD5OfMessageAttributes: attrMd5,
SenderId: senderId,
SentTimestamp: strconv.FormatInt(utils.Uts(), 10),
Payload: out.MessageBody,
}
d, marshalErr := msgPayload.Marshal()
if marshalErr != nil {
log.Error("Failed to serialize message payload: %v", err)
}
payload := enc.UnsafeBytesToString(d)
resp := pq.Push(msgId, payload, pq.Config().MsgTtl, out.DelaySeconds, 1)
if resp.IsError() {
e, _ := resp.(error)
return sqserr.InvalidParameterValueError(e.Error())
}
return &SendMessageResponse{
MessageId: msgId,
MD5OfMessageBody: bodyMd5str,
MD5OfMessageAttributes: attrMd5,
RequestId: "req",
}
}