func AutoGOMAXPROCS() {
if os.Getenv("GOMAXPROCS") == "" {
cpus := runtime.NumCPU()
log.Infof("GOMAXPROCS environment variable not set. Defaulting to system CPU count: %d", cpus)
runtime.GOMAXPROCS(cpus)
} else {
log.Infof("GOMAXPROCS environment variable set, continuing")
}
}
func main() {
accessKey = os.Getenv("ACCESS_KEY")
secretKey = os.Getenv("SECRET_KEY")
port := os.Getenv("GITW_PORT")
hostNum := os.Getenv("HOST_NUM")
if hostNum == "" {
log.Fatalln("HOST_NUM must be set!")
}
infoLog = os.Getenv("INFO_LOG") != "off"
queueName = "gitw_" + hostNum
AutoGOMAXPROCS()
// go processLog(locations)
s := &http.Server{
Addr: ":" + port,
Handler: &Handler{},
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
MaxHeaderBytes: 1 << 20,
}
log.Infof("Listening on port %s...\n", port)
log.Fatalln(s.ListenAndServe())
}
// handle read directory request
func ReadDirectoryHandler(w http.ResponseWriter, req *http.Request) {
rhost := req.Header.Get("X-Forwarded-For")
log.Infof("Read request from %s", rhost)
DLock.RLock()
defer DLock.RUnlock()
w.Header().Add("Content-Type", "text/plain")
// make our own kdirectory
oAdj := GetAdjacentNodes(rhost)
for i := 0; i < len(KDirectory); i++ {
if strings.Split(KDirectory[i].Address, ":")[0] == strings.Split(rhost, ":")[0] {
oAdj = KDirectory[i].Adjacents
}
}
fDir := make([]KNode, len(KDirectory))
for i := 0; i < len(fDir); i++ {
fDir[i] = KDirectory[i]
contains := false
for j := 0; j < len(oAdj); j++ {
contains = contains || i == oAdj[j]
}
if !contains && !fDir[i].ExitNode {
fDir[i].Address = "(hidden)"
}
}
b, _ := json.MarshalIndent(fDir, "", " ")
w.Write(b)
}
// Add a node to the database
func AddNode(addr string, pkey string, pvers int, isexit bool) {
log.Infof("Adding %s with pkey %s and pvers %d", addr, pkey, pvers)
DLock.Lock()
defer func() {
DLock.Unlock()
revnum++
}()
for _, ele := range KDirectory {
if ele.Address == addr {
return
}
}
adj := GetAdjacentNodes(addr)
toadd := KNode{pkey, addr, pvers, adj, isexit}
KDirectory = append(KDirectory, toadd)
/*for i := 0; i < len(KDirectory); i++ {
nd := &KDirectory[i]
nd.Adjacents = GetAdjacentNodes(nd.Address)
}*/
// Now add the other direction
for i := 0; i < len(KDirectory); i++ {
lst := KDirectory[i].Adjacents
for j := 0; j < len(lst); j++ {
nd := &KDirectory[lst[j]]
nd.Adjacents = append(nd.Adjacents, i)
}
}
for i := 0; i < len(KDirectory); i++ {
nd := &KDirectory[i]
nd.Adjacents = FixDuplicates(nd.Adjacents)
}
}
func (mux *Handler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
guidRequest := &GUIDRequest{}
d := json.NewDecoder(request.Body)
if err := d.Decode(&guidRequest); err != nil {
log.Error("Error decoding JSON: ", err)
http.Error(writer, err.Error(), 500)
}
dynamo := dynamodb.Server{
Auth: awsAuth,
Region: aws.USEast,
}
locationsTable := dynamo.NewTable(dynamoTable, pk)
// lls, err := locationsTable.Scan([]dynamodb.AttributeComparison{
// *dynamodb.NewStringAttributeComparison("hash", dynamodb.COMPARISON_NOT_EQUAL, "asdf"),
// })
// for _, ll := range lls {
// log.Info(ll["hash"].Value)
// }
digest := sha256.Sum256([]byte(guidRequest.Guid))
hashKey := hex.EncodeToString(digest[:])
// item, err := locationsTable.GetItem(&dynamodb.Key{HashKey: hashKey})
item, err := locationsTable.GetItem(&dynamodb.Key{HashKey: guidRequest.Guid})
if err != nil {
log.Error("Error getting item from table: ", err)
}
location := &LocationResponse{
Hash: hashKey,
Bucket: item["bucket"].Value,
RandLine: item["randLine"].Value,
}
if infoLog {
log.Infof("S3 key: %s", location.Bucket)
}
response := GUIDResponse{
GuidHash: location.Hash,
BucketName: location.Bucket,
}
jsonResponse, err := json.Marshal(response)
if err != nil {
log.Error("Error marshalling JSON: ", err)
}
// locations <- location
go processLog(location)
writer.Write(jsonResponse)
}
func processLog(location *LocationResponse) {
// log.Info("Line number: ", location.RandLine)
lineNumber, err := strconv.Atoi(location.RandLine)
if err != nil {
log.Error("Error converting randLine to int: ", err)
return
}
reader, err := s3.New(awsAuth, awsRegion).Bucket(s3Bucket).GetReader(location.Bucket)
bufReader := bufio.NewReader(reader)
defer reader.Close()
s3KeyParts := strings.SplitN(location.Bucket, "_", 7)
filenameTimestamp, err := time.Parse(filenameTimeLayout, s3KeyParts[5])
if err != nil {
log.Error("Error parsing filename timestamp: ", err)
return
}
if infoLog {
log.Info("Filename timestamp: ", filenameTimestamp.String())
}
var sumT int = 0
sha := sha256.New()
for i := 0; i < lineNumber; i++ {
line, _, err := bufReader.ReadLine()
if err != nil {
log.Error("Error reading from S3 buffered: ", err)
return
}
splitLine := bytes.SplitN(line, logSeparator, 14)
sha.Write(splitLine[12])
logTimestamp, err := time.Parse(logTimeLayout, string(splitLine[0]))
if err != nil {
log.Error("Error parsing log line timestamp: ", err)
return
}
if infoLog {
log.Info("Log timestamp: ", logTimestamp.String())
}
timeDifference := int(filenameTimestamp.Sub(logTimestamp).Seconds())
if timeDifference < 0 {
timeDifference = 0
}
if infoLog {
log.Infof("Time diff: %d", timeDifference)
}
sumT += timeDifference
}
if infoLog {
log.Infof("Time difference sum: %d", sumT)
}
cksum := sha.Sum(nil)
if infoLog {
log.Info("cksum1: ", hex.EncodeToString(cksum))
}
for i := 0; i < sumT+1; i++ {
sha := sha256.New()
sha.Write(cksum)
cksum = sha.Sum(nil)
}
message, err := json.Marshal(SQSMessage{
GuidHash: location.Hash,
ResultHash: hex.EncodeToString(cksum),
})
if err != nil {
log.Error("Error marshalling SQS message JSON: ", err)
}
if infoLog {
log.Info("SQS message: ", string(message))
}
queue, err := sqs.New(awsAuth, awsRegion).GetQueue(queueName)
if err != nil {
log.Error("Error getting queue: ", err)
}
queue.SendMessage(string(message))
}