Ejemplo n.º 1
0
func AutoGOMAXPROCS() {
	if os.Getenv("GOMAXPROCS") == "" {
		cpus := runtime.NumCPU()
		log.Infof("GOMAXPROCS environment variable not set. Defaulting to system CPU count: %d", cpus)

		runtime.GOMAXPROCS(cpus)
	} else {
		log.Infof("GOMAXPROCS environment variable set, continuing")
	}
}
Ejemplo n.º 2
0
func main() {
	accessKey = os.Getenv("ACCESS_KEY")
	secretKey = os.Getenv("SECRET_KEY")
	port := os.Getenv("GITW_PORT")
	hostNum := os.Getenv("HOST_NUM")
	if hostNum == "" {
		log.Fatalln("HOST_NUM must be set!")
	}
	infoLog = os.Getenv("INFO_LOG") != "off"

	queueName = "gitw_" + hostNum
	AutoGOMAXPROCS()

	// go processLog(locations)

	s := &http.Server{
		Addr:           ":" + port,
		Handler:        &Handler{},
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}

	log.Infof("Listening on port %s...\n", port)
	log.Fatalln(s.ListenAndServe())
}
// handle read directory request
func ReadDirectoryHandler(w http.ResponseWriter, req *http.Request) {
	rhost := req.Header.Get("X-Forwarded-For")
	log.Infof("Read request from %s", rhost)
	DLock.RLock()
	defer DLock.RUnlock()
	w.Header().Add("Content-Type", "text/plain")
	// make our own kdirectory
	oAdj := GetAdjacentNodes(rhost)
	for i := 0; i < len(KDirectory); i++ {
		if strings.Split(KDirectory[i].Address, ":")[0] == strings.Split(rhost, ":")[0] {
			oAdj = KDirectory[i].Adjacents
		}
	}
	fDir := make([]KNode, len(KDirectory))
	for i := 0; i < len(fDir); i++ {
		fDir[i] = KDirectory[i]
		contains := false
		for j := 0; j < len(oAdj); j++ {
			contains = contains || i == oAdj[j]
		}
		if !contains && !fDir[i].ExitNode {
			fDir[i].Address = "(hidden)"
		}
	}
	b, _ := json.MarshalIndent(fDir, "", "  ")
	w.Write(b)
}
Ejemplo n.º 4
0
// Add a node to the database
func AddNode(addr string, pkey string, pvers int, isexit bool) {
	log.Infof("Adding %s with pkey %s and pvers %d", addr, pkey, pvers)
	DLock.Lock()
	defer func() {
		DLock.Unlock()
		revnum++
	}()
	for _, ele := range KDirectory {
		if ele.Address == addr {
			return
		}
	}

	adj := GetAdjacentNodes(addr)
	toadd := KNode{pkey, addr, pvers, adj, isexit}
	KDirectory = append(KDirectory, toadd)
	/*for i := 0; i < len(KDirectory); i++ {
		nd := &KDirectory[i]
		nd.Adjacents = GetAdjacentNodes(nd.Address)
	}*/
	// Now add the other direction
	for i := 0; i < len(KDirectory); i++ {
		lst := KDirectory[i].Adjacents
		for j := 0; j < len(lst); j++ {
			nd := &KDirectory[lst[j]]
			nd.Adjacents = append(nd.Adjacents, i)
		}
	}
	for i := 0; i < len(KDirectory); i++ {
		nd := &KDirectory[i]
		nd.Adjacents = FixDuplicates(nd.Adjacents)
	}
}
Ejemplo n.º 5
0
func (mux *Handler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
	guidRequest := &GUIDRequest{}
	d := json.NewDecoder(request.Body)
	if err := d.Decode(&guidRequest); err != nil {
		log.Error("Error decoding JSON: ", err)
		http.Error(writer, err.Error(), 500)
	}

	dynamo := dynamodb.Server{
		Auth:   awsAuth,
		Region: aws.USEast,
	}

	locationsTable := dynamo.NewTable(dynamoTable, pk)

	// lls, err := locationsTable.Scan([]dynamodb.AttributeComparison{
	// 	*dynamodb.NewStringAttributeComparison("hash", dynamodb.COMPARISON_NOT_EQUAL, "asdf"),
	// })
	// for _, ll := range lls {
	// 	log.Info(ll["hash"].Value)
	// }

	digest := sha256.Sum256([]byte(guidRequest.Guid))
	hashKey := hex.EncodeToString(digest[:])
	// item, err := locationsTable.GetItem(&dynamodb.Key{HashKey: hashKey})
	item, err := locationsTable.GetItem(&dynamodb.Key{HashKey: guidRequest.Guid})
	if err != nil {
		log.Error("Error getting item from table: ", err)
	}

	location := &LocationResponse{
		Hash:     hashKey,
		Bucket:   item["bucket"].Value,
		RandLine: item["randLine"].Value,
	}

	if infoLog {
		log.Infof("S3 key: %s", location.Bucket)
	}

	response := GUIDResponse{
		GuidHash:   location.Hash,
		BucketName: location.Bucket,
	}

	jsonResponse, err := json.Marshal(response)
	if err != nil {
		log.Error("Error marshalling JSON: ", err)
	}

	// locations <- location
	go processLog(location)

	writer.Write(jsonResponse)
}
Ejemplo n.º 6
0
func processLog(location *LocationResponse) {
	// log.Info("Line number: ", location.RandLine)
	lineNumber, err := strconv.Atoi(location.RandLine)
	if err != nil {
		log.Error("Error converting randLine to int: ", err)
		return
	}

	reader, err := s3.New(awsAuth, awsRegion).Bucket(s3Bucket).GetReader(location.Bucket)
	bufReader := bufio.NewReader(reader)
	defer reader.Close()

	s3KeyParts := strings.SplitN(location.Bucket, "_", 7)
	filenameTimestamp, err := time.Parse(filenameTimeLayout, s3KeyParts[5])
	if err != nil {
		log.Error("Error parsing filename timestamp: ", err)
		return
	}
	if infoLog {
		log.Info("Filename timestamp: ", filenameTimestamp.String())
	}

	var sumT int = 0
	sha := sha256.New()
	for i := 0; i < lineNumber; i++ {
		line, _, err := bufReader.ReadLine()
		if err != nil {
			log.Error("Error reading from S3 buffered: ", err)
			return
		}

		splitLine := bytes.SplitN(line, logSeparator, 14)
		sha.Write(splitLine[12])

		logTimestamp, err := time.Parse(logTimeLayout, string(splitLine[0]))
		if err != nil {
			log.Error("Error parsing log line timestamp: ", err)
			return
		}
		if infoLog {
			log.Info("Log timestamp: ", logTimestamp.String())
		}

		timeDifference := int(filenameTimestamp.Sub(logTimestamp).Seconds())
		if timeDifference < 0 {
			timeDifference = 0
		}

		if infoLog {
			log.Infof("Time diff: %d", timeDifference)
		}
		sumT += timeDifference
	}

	if infoLog {
		log.Infof("Time difference sum: %d", sumT)
	}

	cksum := sha.Sum(nil)

	if infoLog {
		log.Info("cksum1: ", hex.EncodeToString(cksum))
	}

	for i := 0; i < sumT+1; i++ {
		sha := sha256.New()
		sha.Write(cksum)
		cksum = sha.Sum(nil)
	}

	message, err := json.Marshal(SQSMessage{
		GuidHash:   location.Hash,
		ResultHash: hex.EncodeToString(cksum),
	})
	if err != nil {
		log.Error("Error marshalling SQS message JSON: ", err)
	}
	if infoLog {
		log.Info("SQS message: ", string(message))
	}

	queue, err := sqs.New(awsAuth, awsRegion).GetQueue(queueName)
	if err != nil {
		log.Error("Error getting queue: ", err)
	}
	queue.SendMessage(string(message))
}