コード例 #1
0
ファイル: output.go プロジェクト: jtwarren/Hurricane
func main() {

	file, err := os.Create("output.txt")
	if err != nil {
		panic(err)
	}
	defer file.Close()

	count := make(map[string]int)

	worker.ReadTupleStream(os.Stdin, func(tuple worker.Tuple, index int) {
		switch index {
		case 0:
			num, _ := strconv.Atoi(tuple.Slice[1])
			count[tuple.Slice[0]] += num
		case 1:
			num, _ := strconv.Atoi(tuple.Slice[1])
			count[tuple.Slice[0]] += num
		case 2:
			num, _ := strconv.Atoi(tuple.Slice[1])
			count[tuple.Slice[0]] -= num
		}
	})

	stdout := os.NewFile(uintptr(syscall.Stdout), "/dev/stdout")

	for key, value := range count {
		file.Write([]byte(key + ", " + strconv.Itoa(value) + "\n"))
		outTuple := worker.Tuple{[]string{key, strconv.Itoa(value)}}
		stdout.Write(outTuple.SerializeTuple(0))
		stdout.Write([]byte{'\n'})
	}
}
コード例 #2
0
func main() {

	// read from standard input to get the input tuples
	inputTuples := make([]worker.Tuple, 0)
	worker.ReadTupleStream(os.Stdin, func(tuple worker.Tuple, index int) {
		inputTuples = append(inputTuples, tuple)
	})

	stdout := os.NewFile(uintptr(syscall.Stdout), "/dev/stdout")

	counts := make(map[string]int)
	for _, tuple := range inputTuples {
		word := tuple.Slice[0]

		if count, ok := counts[word]; ok {
			counts[word] = count + 1
		} else {
			counts[word] = 1
		}
	}

	for word, count := range counts {
		outTuple := worker.Tuple{[]string{word, strconv.Itoa(count)}}
		stdout.Write(outTuple.SerializeTuple(0))
		stdout.Write([]byte{'\n'})
	}
}
コード例 #3
0
ファイル: input.go プロジェクト: jtwarren/Hurricane
// go run input.go start_time duration number_of_log random_seed
// go run input.go now 60 2000 5
func main() {
	d, _ := strconv.Atoi(os.Args[2])
	duration := time.Second * time.Duration(d)

	startTime := time.Now() // os.Args[1]

	num, _ := strconv.Atoi(os.Args[3])

	seed, _ := strconv.ParseInt(os.Args[4], 10, 64)

	stdout := os.NewFile(uintptr(syscall.Stdout), "/dev/stdout")

	r := rand.New(rand.NewSource(seed))

	step := duration / time.Duration(num)

	for st := startTime; st.Before(startTime.Add(duration)); st = st.Add(step) {
		s := []string{"[" + st.String() + "]"}

		if r.Float32() < 0.1 {
			s = append(s, "error")
		} else {
			s = append(s, "pass")
		}

		tuple := worker.Tuple{s}
		stdout.Write(tuple.SerializeTuple(0))
		stdout.Write([]byte{'\n'})
	}
}
コード例 #4
0
ファイル: combine.go プロジェクト: jtwarren/Hurricane
func main() {

	// read from standard input to get the input tuples
	newTuples := make([]worker.Tuple, 0)
	oldTuples := make([]worker.Tuple, 0)
	totalTuples := make([]worker.Tuple, 0)

	worker.ReadTupleStream(os.Stdin, func(tuple worker.Tuple, index int) {
		switch index {
		case 0:
			newTuples = append(newTuples, tuple)
		case 1:
			oldTuples = append(oldTuples, tuple)
		case 2:
			totalTuples = append(totalTuples, tuple)
		}
	})

	stdout := os.NewFile(uintptr(syscall.Stdout), "/dev/stdout")

	counts := make(map[string]int)
	for _, tuple := range totalTuples {
		word := tuple.Slice[0]
		prev, _ := strconv.Atoi(tuple.Slice[1])

		if count, ok := counts[word]; ok {
			counts[word] = count + prev
		} else {
			counts[word] = prev
		}
	}

	for _, tuple := range newTuples {
		word := tuple.Slice[0]

		if count, ok := counts[word]; ok {
			counts[word] = count + 1
		} else {
			counts[word] = 1
		}
	}

	for _, tuple := range oldTuples {
		word := tuple.Slice[0]

		if count, ok := counts[word]; ok {
			counts[word] = count - 1
		}
	}

	for word, count := range counts {
		outTuple := worker.Tuple{[]string{word, strconv.Itoa(count)}}

		stdout.Write(outTuple.SerializeTuple(2))
		stdout.Write([]byte{'\n'})
	}
}
コード例 #5
0
ファイル: mock_log_reader.go プロジェクト: jtwarren/Hurricane
func main() {
	stdout := os.NewFile(uintptr(syscall.Stdout), "/dev/stdout")

	t := time.Now()
	strings := []string{"Zack", "Jeff", "Vedha", "Vikas", "Zack"}

	for i := 0; i < 1000; i++ {
		for _, str := range strings {
			tuple := worker.Tuple{[]string{str, t.String()}}
			stdout.Write(tuple.SerializeTuple(0))
			stdout.Write([]byte{'\n'})
		}
	}
}
コード例 #6
0
ファイル: reduce.go プロジェクト: jtwarren/Hurricane
func main() {

	count := make(map[string]int)

	worker.ReadTupleStream(os.Stdin, func(tuple worker.Tuple, index int) {
		count[tuple.Slice[0]] += 1
	})

	stdout := os.NewFile(uintptr(syscall.Stdout), "/dev/stdout")

	for key, value := range count {
		outTuple := worker.Tuple{[]string{key, strconv.Itoa(value)}}
		stdout.Write(outTuple.SerializeTuple(0))
		stdout.Write([]byte{'\n'})
	}
}
コード例 #7
0
ファイル: wordcount_input.go プロジェクト: jtwarren/Hurricane
func main() {
	gopath := os.Getenv("GOPATH")
	path := path.Join(gopath, "/src/demo/wordcount_go/input_file.txt")

	file, err := os.Open(path)
	if err != nil {
		log.Panic(err)
	}
	defer file.Close()

	stdout := os.NewFile(uintptr(syscall.Stdout), "/dev/stdout")

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		tuple := worker.Tuple{[]string{scanner.Text()}}

		stdout.Write(tuple.SerializeTuple(0))
		stdout.Write([]byte{'\n'})
	}
}
コード例 #8
0
ファイル: reduce.go プロジェクト: jtwarren/Hurricane
func main() {

	// read from standard input to get the input tuples
	inputTuples := make([]worker.Tuple, 0)
	worker.ReadTupleStream(os.Stdin, func(tuple worker.Tuple, index int) {
		inputTuples = append(inputTuples, tuple)
	})

	stdout := os.NewFile(uintptr(syscall.Stdout), "/dev/stdout")

	count := 0
	for _, _ = range inputTuples {
		// word := tuple.Slice[0]
		count += 1
	}

	outTuple := worker.Tuple{[]string{"error", strconv.Itoa(count)}}
	stdout.Write(outTuple.SerializeTuple(0))
	stdout.Write([]byte{'\n'})
}
コード例 #9
0
ファイル: wordcount_map.go プロジェクト: jtwarren/Hurricane
func main() {

	// read input from standard input and create tuples
	inputTuples := make([]worker.Tuple, 0)
	worker.ReadTupleStream(os.Stdin, func(tuple worker.Tuple, index int) {
		inputTuples = append(inputTuples, tuple)
	})

	// get standard output
	stdout := os.NewFile(uintptr(syscall.Stdout), "/dev/stdout")

	// iterate over input tuples
	for _, tuple := range inputTuples {
		words := strings.Fields(tuple.Slice[0])
		// iterate over words
		for _, word := range words {
			// emit each word in a new tuple
			outTuple := worker.Tuple{[]string{strings.ToLower(word), "1"}}
			stdout.Write(outTuple.SerializeTuple(0))
			stdout.Write([]byte{'\n'})
		}
	}
}
コード例 #10
0
ファイル: input.go プロジェクト: jtwarren/Hurricane
// go run input.go start_time duration number_of_log
// go run input.go now 60 2000 5
func main() {
	var value string

	d, _ := strconv.Atoi(os.Args[2])
	start, _ := strconv.Atoi(os.Args[1])
	num, _ := strconv.Atoi(os.Args[3])

	max := d * num

	stdout := os.NewFile(uintptr(syscall.Stdout), "/dev/stdout")

	for i := 0; i < max; i++ {
		if i%num == 0 {
			value = ERROR
		} else {
			value = PASS
		}
		s := []string{value, strconv.Itoa(i + start)}
		tuple := worker.Tuple{s}
		stdout.Write(tuple.SerializeTuple(0))
		stdout.Write([]byte{'\n'})
	}
}
コード例 #11
0
ファイル: map.go プロジェクト: jtwarren/Hurricane
func main() {

	// read from standard input to get the input tuples
	inputTuples := make([]worker.Tuple, 0)
	worker.ReadTupleStream(os.Stdin, func(tuple worker.Tuple, index int) {
		inputTuples = append(inputTuples, tuple)
	})

	stdout := os.NewFile(uintptr(syscall.Stdout), "/dev/stdout")

	for _, tuple := range inputTuples {
		words := strings.Fields(tuple.Slice[1])

		for _, word := range words {
			word = strings.ToLower(word)
			if word == "error" {
				outTuple := worker.Tuple{[]string{word, "1"}}
				stdout.Write(outTuple.SerializeTuple(0))
				stdout.Write([]byte{'\n'})
			}
		}
	}
}