// NewEventSubSocketHandler returns a function that handles event notifications
func NewEventSubSocketHandler(eventSubSocket *zmq4.Socket) func(zmq4.State) error {
var nodeIDMap map[string]uint32
var err error
if nodeIDMap, err = centraldb.GetNodeIDMap(); err != nil {
fog.Critical("centraldb.GetNodeIDMap() failed %s", err)
}
return func(_ zmq4.State) error {
var err error
var ok bool
var webWriterStart msg.WebWriterStart
var timestamp time.Time
var sourceNodeID uint32
marshalledMessage, err := eventSubSocket.RecvMessage(0)
if err != nil {
return fmt.Errorf("RecvMessage %s", err)
}
// the 0th part should be the topic, we skip that
err = json.Unmarshal([]byte(marshalledMessage[1]), &webWriterStart)
if err != nil {
return fmt.Errorf("Unmarshal %s", err)
}
if webWriterStart.MessageType != "web-writer-start" {
return fmt.Errorf("unknown message type '%s'",
webWriterStart.MessageType)
}
timestamp, err = tools.ParseTimestampRepr(webWriterStart.TimestampRepr)
if err != nil {
return fmt.Errorf("unable to parse %s %s",
webWriterStart.TimestampRepr, err)
}
sourceNodeID, ok = nodeIDMap[webWriterStart.SourceNodeName]
if !ok {
return fmt.Errorf("unknown source_node_name %s",
webWriterStart.SourceNodeName)
}
fog.Debug("cancel-segments-from-node %s", webWriterStart.SourceNodeName)
// cancel all all segment rows
// * from a specifiic source node
// * are in active status
// * with a timestamp earlier than the specified time.
// This is triggered by a web server restart
stmt := nodedb.Stmts["cancel-segments-from-node"]
if _, err = stmt.Exec(sourceNodeID, timestamp); err != nil {
return fmt.Errorf("cancel-segments-from-node %s", err)
}
return nil
}
}
// FinishConjoinedArchive completes a conjoined archive
func handleFinishConjoinedArchive(state *writerState, request requestFinishConjoinedArchive) {
conjoined := request.Conjoined
var err error
var ok bool
var handoffNodeID uint32
var timestamp time.Time
fog.Debug("%s FinishConjoinedArchive %s", conjoined.UserRequestID, conjoined)
if timestamp, err = tools.ParseTimestampRepr(conjoined.TimestampRepr); err != nil {
request.resultChan <- fmt.Errorf("unable to parse timestamp %s", err)
return
}
if conjoined.HandoffNodeName != "" {
if handoffNodeID, ok = state.NodeIDMap[conjoined.HandoffNodeName]; !ok {
request.resultChan <- fmt.Errorf("unknown handoff node %s", conjoined.HandoffNodeName)
}
stmt := nodedb.Stmts["finish-conjoined-for-handoff"]
_, err = stmt.Exec(
timestamp,
conjoined.CollectionID,
conjoined.Key,
conjoined.UnifiedID,
handoffNodeID)
if err != nil {
request.resultChan <- fmt.Errorf("finish-conjoined-for-handoff %s", err)
return
}
} else {
stmt := nodedb.Stmts["finish-conjoined"]
_, err = stmt.Exec(
timestamp,
conjoined.CollectionID,
conjoined.Key,
conjoined.UnifiedID)
if err != nil {
request.resultChan <- fmt.Errorf("finish-conjoined %s", err)
return
}
}
request.resultChan <- nil
}
// DestroyKey makes a key inaccessible
func handleDestroyKey(state *writerState, request requestDestroyKey) {
destroyKey := request.DestroyKey
var err error
var ok bool
var timestamp time.Time
var sourceNodeID uint32
var handoffNodeID uint32
fog.Debug("DestroyKey (%d)", destroyKey.UnifiedIDToDestroy)
if sourceNodeID, ok = state.NodeIDMap[destroyKey.SourceNodeName]; !ok {
request.resultChan <- fmt.Errorf("unknown source node %s", destroyKey.SourceNodeName)
return
}
if timestamp, err = tools.ParseTimestampRepr(destroyKey.TimestampRepr); err != nil {
request.resultChan <- fmt.Errorf("unable to parse timestamp %s", err)
return
}
if destroyKey.UnifiedIDToDestroy > 0 {
if destroyKey.HandoffNodeName != "" {
if handoffNodeID, ok = state.NodeIDMap[destroyKey.HandoffNodeName]; !ok {
request.resultChan <- fmt.Errorf("unknown handoff node %s", destroyKey.HandoffNodeName)
return
}
stmt := nodedb.Stmts["new-tombstone-for-unified-id-for-handoff"]
_, err = stmt.Exec(
destroyKey.CollectionID,
destroyKey.Key,
destroyKey.UnifiedID,
timestamp,
destroyKey.SegmentNum,
destroyKey.UnifiedIDToDestroy,
sourceNodeID,
handoffNodeID)
if err != nil {
request.resultChan <- fmt.Errorf("new-tombstone-for-unified-id-for-handoff %d %s",
destroyKey.UnifiedIDToDestroy, err)
return
}
} else {
stmt := nodedb.Stmts["new-tombstone-for-unified-id"]
_, err = stmt.Exec(
destroyKey.CollectionID,
destroyKey.Key,
destroyKey.UnifiedID,
timestamp,
destroyKey.SegmentNum,
destroyKey.UnifiedIDToDestroy,
sourceNodeID)
if err != nil {
request.resultChan <- fmt.Errorf("new-tombstone-for-unified-id %d %s",
destroyKey.UnifiedIDToDestroy, err)
return
}
}
stmt := nodedb.Stmts["delete-conjoined-for-unified-id"]
_, err = stmt.Exec(
timestamp,
destroyKey.CollectionID,
destroyKey.Key,
destroyKey.UnifiedIDToDestroy)
if err != nil {
request.resultChan <- fmt.Errorf("delete-conjoined-for-unified-id %d %s",
destroyKey.UnifiedIDToDestroy, err)
return
}
} else {
if destroyKey.HandoffNodeName != "" {
if handoffNodeID, ok = state.NodeIDMap[destroyKey.HandoffNodeName]; !ok {
request.resultChan <- fmt.Errorf("unknown handoff node %s", destroyKey.HandoffNodeName)
return
}
stmt := nodedb.Stmts["new-tombstone-for-handoff"]
_, err = stmt.Exec(
destroyKey.CollectionID,
destroyKey.Key,
destroyKey.UnifiedID,
timestamp,
destroyKey.SegmentNum,
sourceNodeID,
handoffNodeID)
if err != nil {
request.resultChan <- fmt.Errorf("new-tombstone-for-handoff %s", err)
return
}
} else {
stmt := nodedb.Stmts["new-tombstone"]
_, err = stmt.Exec(
destroyKey.CollectionID,
destroyKey.Key,
destroyKey.UnifiedID,
timestamp,
destroyKey.SegmentNum,
sourceNodeID)
if err != nil {
request.resultChan <- fmt.Errorf("new-tombstone %s", err)
return
}
}
stmt := nodedb.Stmts["delete-conjoined"]
_, err = stmt.Exec(
timestamp,
destroyKey.CollectionID,
destroyKey.Key,
destroyKey.UnifiedID)
if err != nil {
request.resultChan <- fmt.Errorf("delete-conjoined %s", err)
return
}
}
request.resultChan <- nil
}
// FinishSegment finishes storing the segment
func handleFinishSegment(state *writerState, request requestFinishSegment) {
userRequestID := request.UserRequestID
segment := request.Segment
file := request.File
metaData := request.MetaData
var err error
var md5Digest []byte
var timestamp time.Time
fog.Debug("%s FinishSegment", userRequestID)
key := segmentKey{segment.UnifiedID, segment.ConjoinedPart,
segment.SegmentNum}
entry, ok := state.SegmentMap[key]
if !ok {
request.resultChan <- fmt.Errorf("FinishSegment unknown segment %s", key)
return
}
delete(state.SegmentMap, key)
md5Digest, err = base64.StdEncoding.DecodeString(file.EncodedFileMD5Digest)
if err != nil {
request.resultChan <- err
return
}
if timestamp, err = tools.ParseTimestampRepr(segment.TimestampRepr); err != nil {
request.resultChan <- fmt.Errorf("unable to parse timestamp %s", err)
return
}
stmt := nodedb.Stmts["finish-segment"]
_, err = stmt.Exec(
file.FileSize,
file.FileAdler32,
md5Digest,
entry.SegmentID)
if err != nil {
request.resultChan <- fmt.Errorf("finish-segment %s", err)
return
}
for _, metaEntry := range metaData {
stmt := nodedb.Stmts["new-meta-data"]
_, err = stmt.Exec(
segment.CollectionID,
entry.SegmentID,
metaEntry.Key,
metaEntry.Value,
timestamp)
if err != nil {
request.resultChan <- fmt.Errorf("new-meta-data %s", err)
return
}
}
request.resultChan <- nil
}
func handleStartSegment(state *writerState, request requestStartSegment) {
userRequestID := request.UserRequestID
segment := request.Segment
nodeNames := request.NodeNames
var entry segmentMapEntry
var err error
var sourceNodeID uint32
var handoffNodeID uint32
var ok bool
var timestamp time.Time
fog.Debug("%s StartSegment", userRequestID)
if sourceNodeID, ok = state.NodeIDMap[nodeNames.SourceNodeName]; !ok {
request.resultChan <- fmt.Errorf("unknown source node %s", nodeNames.SourceNodeName)
return
}
if timestamp, err = tools.ParseTimestampRepr(segment.TimestampRepr); err != nil {
request.resultChan <- fmt.Errorf("unable to parse timestamp %s", err)
return
}
if nodeNames.HandoffNodeName != "" {
if handoffNodeID, ok = state.NodeIDMap[nodeNames.HandoffNodeName]; !ok {
request.resultChan <- fmt.Errorf("unknown handoff node %s", nodeNames.HandoffNodeName)
return
}
stmt := nodedb.Stmts["new-segment-for-handoff"]
row := stmt.QueryRow(
segment.CollectionID,
segment.Key,
segment.UnifiedID,
timestamp,
segment.SegmentNum,
segment.ConjoinedPart,
sourceNodeID,
handoffNodeID)
if err = row.Scan(&entry.SegmentID); err != nil {
request.resultChan <- err
return
}
} else {
stmt := nodedb.Stmts["new-segment"]
row := stmt.QueryRow(
segment.CollectionID,
segment.Key,
segment.UnifiedID,
timestamp,
segment.SegmentNum,
segment.ConjoinedPart,
sourceNodeID)
if err = row.Scan(&entry.SegmentID); err != nil {
request.resultChan <- err
return
}
}
entry.LastActionTime = tools.Timestamp()
key := segmentKey{segment.UnifiedID, segment.ConjoinedPart,
segment.SegmentNum}
state.SegmentMap[key] = entry
request.resultChan <- nil
}