Esempio n. 1
0
func parseDefaultConfig(l string) (uint32, error) {
	// If we have already seen a default config line, and n has a value then we
	// log.Fatal.
	if groupConfig.n != 0 {
		return 0, fmt.Errorf("Default config can only be defined once: %v", l)
	}
	l = strings.TrimSpace(l)
	conf := strings.Split(l, " ")
	// + in (fp % n + k) is optional.
	if !(len(conf) == 5 || len(conf) == 3) || conf[0] != "fp" || conf[1] != "%" {
		return 0, fmt.Errorf("Default config format should be like: %v", "default: fp % n + k")
	}

	var err error
	var n uint64
	n, err = strconv.ParseUint(conf[2], 10, 32)
	x.Check(err)
	groupConfig.n = uint32(n)
	x.AssertTrue(groupConfig.n != 0)
	if len(conf) == 5 {
		if conf[3] != "+" {
			return 0, fmt.Errorf("Default config format should be like: %v", "default: fp % n + k")
		}
		n, err = strconv.ParseUint(conf[4], 10, 32)
		groupConfig.k = uint32(n)
		x.Check(err)
	}
	return groupConfig.k, nil
}
Esempio n. 2
0
func toRDF(buf *bytes.Buffer, item kv) {
	pl := item.list
	for _, p := range pl.Postings {
		x.Check2(buf.WriteString(item.prefix))

		if p.Uid == math.MaxUint64 && !bytes.Equal(p.Value, nil) {
			// Value posting
			// Convert to appropriate type
			typ := stype.ValueForType(stype.TypeID(p.ValType))
			x.Check(typ.UnmarshalBinary(p.Value))
			str, err := typ.MarshalText()
			x.Check(err)

			x.Check2(buf.WriteString(fmt.Sprintf("\"%s\"", str)))
			if p.ValType == uint32(stype.GeoID) {
				x.Check2(buf.WriteString(fmt.Sprintf("^^<geo:geojson> ")))
			} else if p.ValType != uint32(stype.BytesID) {
				x.Check2(buf.WriteString(fmt.Sprintf("^^<xs:%s> ", typ.Type().Name)))
			}
			x.Check2(buf.WriteString(" .\n"))
			return
		}
		x.Check2(buf.WriteString(fmt.Sprintf("<_uid_:%#x> .\n", p.Uid)))
	}
}
Esempio n. 3
0
func main() {
	flag.Parse()
	f, err := os.Open(*geoData)
	x.Check(err)

	gr, err := gzip.NewReader(f)
	x.Check(err)

	//var strBuf bytes.Buffer
	//bufReader := bufio.NewReader(gr)
	dec := json.NewDecoder(gr)

	countryToGeo := make(map[string]string)
	findFeatureArray(dec)
	for dec.More() {
		var f geojson.Feature
		err := dec.Decode(&f)
		fmt.Println(f.Properties["NAME_LONG"])
		gg, err := geojson.Marshal(f.Geometry)
		ggg := strings.Replace(string(gg), "\"", "'", -1)
		country, ok := f.Properties["NAME_LONG"].(string)
		if ok {
			countryToGeo[country] = ggg
		}
		//fmt.Printf("\"%s\"", ggg)
		if err != nil {
			fmt.Println(err)
		}
	}
	gr.Close()
	f.Close()

	f, err = os.Open(*rdf)
	x.Check(err)

	gr, err = gzip.NewReader(f)
	x.Check(err)

	scanner := bufio.NewScanner(gr)

	out, err := os.Create("countryGeoData")
	x.Check(err)
	defer out.Close()
	count1, count2 := 0, 0
	for scanner.Scan() {
		line := scanner.Text()
		if strings.Contains(line, "@en") {
			items := strings.Split(line, "\t")
			country := strings.Trim(strings.Split(items[2], "@")[0], "\"")
			fmt.Println(country)
			if geoD, ok := countryToGeo[country]; ok {
				count1++
				out.WriteString(fmt.Sprintf("%s <loc> \"%s\"^^<geo:geojson> .\n", items[0], geoD))
			} else {
				count2++
			}
		}
	}
	fmt.Println(count1, count2)
}
Esempio n. 4
0
func makeRequest(mutation chan string, c *uint64, wg *sync.WaitGroup) {
	var counter uint64
	for m := range mutation {
		counter = atomic.AddUint64(c, 1)
		if counter%100 == 0 {
			fmt.Printf("Request: %v\n", counter)
		}
	RETRY:
		req, err := http.NewRequest("POST", *dgraph, strings.NewReader(body(m)))
		x.Check(err)
		res, err := hc.Do(req)
		if err != nil {
			fmt.Printf("Retrying req: %d. Error: %v\n", counter, err)
			time.Sleep(5 * time.Millisecond)
			goto RETRY
		}

		body, err := ioutil.ReadAll(res.Body)
		x.Check(err)
		x.Check(json.Unmarshal(body, &r))
		if r.Code != "ErrorOk" {
			log.Fatalf("Error while performing mutation: %v, err: %v", m, r.Message)
		}
	}
	wg.Done()
}
Esempio n. 5
0
func makeRequests(mutation chan string, wg *sync.WaitGroup) {
	for m := range mutation {
		counter := atomic.AddUint64(&s.mutations, 1)
		if counter%100 == 0 {
			num := atomic.LoadUint64(&s.rdfs)
			dur := time.Since(s.start)
			rate := float64(num) / dur.Seconds()
			fmt.Printf("[Request: %6d] Total RDFs done: %8d RDFs per second: %7.0f\r", counter, num, rate)
		}
	RETRY:
		req, err := http.NewRequest("POST", *dgraph, strings.NewReader(body(m)))
		x.Check(err)
		res, err := hc.Do(req)
		if err != nil {
			fmt.Printf("Retrying req: %d. Error: %v\n", counter, err)
			time.Sleep(5 * time.Millisecond)
			goto RETRY
		}

		body, err := ioutil.ReadAll(res.Body)
		x.Check(err)
		if err = json.Unmarshal(body, &r); err != nil {
			// Not doing x.Checkf(json.Unmarshal..., "Response..", string(body))
			// to ensure that we don't try to convert body from []byte to string
			// when there's no errors.
			x.Checkf(err, "HTTP Status: %s Response body: %s.",
				http.StatusText(res.StatusCode), string(body))
		}
		if r.Code != "ErrorOk" {
			log.Fatalf("Error while performing mutation: %v, err: %v", m, r.Message)
		}
	}
	wg.Done()
}
Esempio n. 6
0
func convertToEdges(ctx context.Context, nquads []rdf.NQuad) (mutationResult, error) {
	var edges []*task.DirectedEdge
	var mr mutationResult

	newUids := make(map[string]uint64)
	for _, nq := range nquads {
		if strings.HasPrefix(nq.Subject, "_new_:") {
			newUids[nq.Subject] = 0
		} else if !strings.HasPrefix(nq.Subject, "_uid_:") {
			uid, err := rdf.GetUid(nq.Subject)
			x.Check(err)
			newUids[nq.Subject] = uid
		}

		if len(nq.ObjectId) > 0 {
			if strings.HasPrefix(nq.ObjectId, "_new_:") {
				newUids[nq.ObjectId] = 0
			} else if !strings.HasPrefix(nq.ObjectId, "_uid_:") {
				uid, err := rdf.GetUid(nq.ObjectId)
				x.Check(err)
				newUids[nq.ObjectId] = uid
			}
		}
	}

	if len(newUids) > 0 {
		if err := worker.AssignUidsOverNetwork(ctx, newUids); err != nil {
			x.TraceError(ctx, x.Wrapf(err, "Error while GetOrAssignUidsOverNetwork"))
			return mr, err
		}
	}

	for _, nq := range nquads {
		// Get edges from nquad using newUids.
		edge, err := nq.ToEdgeUsing(newUids)
		if err != nil {
			x.TraceError(ctx, x.Wrapf(err, "Error while converting to edge: %v", nq))
			return mr, err
		}
		edges = append(edges, edge)
	}

	resultUids := make(map[string]uint64)
	// Strip out _new_: prefix from the keys.
	for k, v := range newUids {
		if strings.HasPrefix(k, "_new_:") {
			resultUids[k[6:]] = v
		}
	}

	mr = mutationResult{
		edges:   edges,
		newUids: resultUids,
	}
	return mr, nil
}
Esempio n. 7
0
// ParseConfig parses a group config provided by reader.
func ParseConfig(r io.Reader) error {
	groupConfig = config{}
	scanner := bufio.NewScanner(r)
	// To keep track of last groupId seen across lines. If we the groups ids are
	// not sequential, we log.Fatal.
	var curGroupId uint32
	// If after seeing line with default config, we see other lines, we log.Fatal.
	// Default config should be specified as the last line, so that we can check
	// accurately that k in (fp % N + k) generates consecutive groups.
	seenDefault := false
	for scanner.Scan() {
		l := scanner.Text()
		// Skip empty lines and comments.
		if l == "" || strings.HasPrefix(l, "//") {
			continue
		}
		c := strings.Split(l, ":")
		if len(c) < 2 {
			return fmt.Errorf("Incorrect format for config line: %v", l)
		}
		if c[0] == "default" {
			seenDefault = true
			k, err := parseDefaultConfig(c[1])
			if err != nil {
				return err
			}
			if k == 0 {
				continue
			}
			if k > curGroupId {
				return fmt.Errorf("k in (fp mod N + k) should be <= the last groupno %v.",
					curGroupId)
			}
		} else {
			// There shouldn't be a line after the default config line.
			if seenDefault {
				return fmt.Errorf("Default config should be specified as the last line. Found %v",
					l)
			}
			groupId, err := strconv.ParseUint(c[0], 10, 32)
			x.Check(err)
			if curGroupId != uint32(groupId) {
				return fmt.Errorf("Group ids should be sequential and should start from 0. "+
					"Found %v, should have been %v", groupId, curGroupId)
			}
			curGroupId++
			err = parsePredicates(uint32(groupId), c[1])
			if err != nil {
				return err
			}
		}
	}
	x.Check(scanner.Err())
	return nil
}
Esempio n. 8
0
func main() {
	flag.Parse()
	logrus.SetLevel(logrus.DebugLevel)
	var srcl, dstl []R
	f, bufReader := getReader(*src)
	var err error

	srcCount := 0
	var strBuf bytes.Buffer
	for {
		err = x.ReadLine(bufReader, &strBuf)
		if err != nil {
			break
		}
		srcCount++
		rnq, err := rdf.Parse(strBuf.String())
		x.Checkf(err, "Unable to parse line: [%v]", strBuf.String())
		srcl = append(srcl, convert(rnq))
	}
	if err != nil && err != io.EOF {
		err := x.Errorf("Error while reading file: %v", err)
		log.Fatalf("%+v", err)
	}
	x.Check(f.Close())
	fmt.Println("Source done")

	f, bufReader = getReader(*dst)
	dstCount := 0
	for {
		err = x.ReadLine(bufReader, &strBuf)
		if err != nil {
			break
		}
		dstCount++
		rnq, err := rdf.Parse(strBuf.String())
		x.Checkf(err, "Unable to parse line: [%v]", strBuf.String())
		dstl = append(dstl, convert(rnq))
	}
	if err != nil && err != io.EOF {
		err := x.Errorf("Error while reading file: %v", err)
		log.Fatalf("%+v", err)
	}
	x.Check(f.Close())

	fmt.Printf("Src: [%d] Dst: [%d]\n", srcCount, dstCount)
	sort.Sort(ByR(srcl))
	sort.Sort(ByR(dstl))
	fmt.Println("Comparing now")
	//for i := 0; i < 100; i++ {
	//fmt.Printf("[S,D] %v %v\n", srcl[i], dstl[i])
	//}
	compare(srcl, dstl)
}
Esempio n. 9
0
func checkFlagsAndInitDirs() {
	numCpus := *numcpu
	if len(*cpuprofile) > 0 {
		f, err := os.Create(*cpuprofile)
		x.Check(err)
		pprof.StartCPUProfile(f)
	}

	prev := runtime.GOMAXPROCS(numCpus)
	log.Printf("num_cpu: %v. prev_maxprocs: %v. Set max procs to num cpus",
		numCpus, prev)
	// Create parent directories for postings, uids and mutations
	x.Check(os.MkdirAll(*postingDir, 0700))
}
Esempio n. 10
0
func main() {
	flag.Parse()
	f, err := os.Open(*file)
	x.Check(err)
	defer f.Close()
	gr, err := gzip.NewReader(f)
	x.Check(err)

	hc = http.Client{Timeout: time.Minute}
	mutation := make(chan string, 3*(*concurrent))

	var count uint64 = 0
	var wg sync.WaitGroup
	for i := 0; i < *concurrent; i++ {
		wg.Add(1)
		go makeRequest(mutation, &count, &wg)
	}

	var buf bytes.Buffer
	bufReader := bufio.NewReader(gr)
	num := 0
	var rdfCount uint64 = 0
	for {
		err = readLine(bufReader, &buf)
		if err != nil {
			break
		}
		buf.WriteRune('\n')

		if num >= *numRdf {
			mutation <- buf.String()
			buf.Reset()
			num = 0
		}
		rdfCount++
		num++
	}
	if err != io.EOF {
		log.Fatalf("Error while reading file: %+v", err)
	}
	if buf.Len() > 0 {
		mutation <- buf.String()
	}
	close(mutation)

	wg.Wait()
	fmt.Println("Number of RDF's parsed: ", rdfCount)
	fmt.Println("Number of mutations run: ", count)
}
Esempio n. 11
0
// stringHelper does simple DFS to convert FilterTree to string.
func (t *FilterTree) stringHelper(buf *bytes.Buffer) {
	x.AssertTrue(t != nil)
	if t.Func != nil && len(t.Func.Name) > 0 {
		// Leaf node.
		_, err := buf.WriteRune('(')
		x.Check(err)
		_, err = buf.WriteString(t.Func.Name)
		x.Check(err)

		if len(t.Func.Attr) > 0 {
			args := make([]string, len(t.Func.Args)+1)
			args[0] = t.Func.Attr
			copy(args[1:], t.Func.Args)

			for _, arg := range args {
				_, err = buf.WriteString(" \"")
				x.Check(err)
				_, err = buf.WriteString(arg)
				x.Check(err)
				_, err := buf.WriteRune('"')
				x.Check(err)
			}
		}
		_, err = buf.WriteRune(')')
		x.Check(err)
		return
	}
	// Non-leaf node.
	_, err := buf.WriteRune('(')
	x.Check(err)
	switch t.Op {
	case "&":
		_, err = buf.WriteString("AND")
	case "|":
		_, err = buf.WriteString("OR")
	default:
		err = x.Errorf("Unknown operator: %q", t.Op)
	}
	x.Check(err)

	for _, c := range t.Child {
		_, err = buf.WriteRune(' ')
		x.Check(err)
		c.stringHelper(buf)
	}
	_, err = buf.WriteRune(')')
	x.Check(err)
}
Esempio n. 12
0
func (p *poolsi) connect(addr string) {
	if addr == *myAddr {
		return
	}
	p.RLock()
	_, has := p.all[addr]
	p.RUnlock()
	if has {
		return
	}

	pool := newPool(addr, 5)
	query := new(Payload)
	query.Data = make([]byte, 10)
	x.Check2(rand.Read(query.Data))

	conn, err := pool.Get()
	x.Checkf(err, "Unable to connect")

	c := NewWorkerClient(conn)
	resp, err := c.Echo(context.Background(), query)
	x.Checkf(err, "Unable to Echo")
	x.AssertTrue(bytes.Equal(resp.Data, query.Data))
	x.Check(pool.Put(conn))
	fmt.Printf("Connection with %q successful.\n", addr)

	p.Lock()
	defer p.Unlock()
	_, has = p.all[addr]
	if has {
		return
	}
	p.all[addr] = pool
}
Esempio n. 13
0
func main() {
	rand.Seed(time.Now().UnixNano())
	x.Init()
	checkFlagsAndInitDirs()

	ps, err := store.NewStore(*postingDir)
	x.Checkf(err, "Error initializing postings store")
	defer ps.Close()

	if len(*schemaFile) > 0 {
		err = schema.Parse(*schemaFile)
		x.Checkf(err, "Error while loading schema: %s", *schemaFile)
	}
	// Posting will initialize index which requires schema. Hence, initialize
	// schema before calling posting.Init().
	posting.Init(ps)
	worker.Init(ps)
	x.Check(group.ParseGroupConfig(*conf))

	// Setup external communication.
	che := make(chan error, 1)
	go setupServer(che)
	go worker.StartRaftNodes(*walDir)

	if err := <-che; !strings.Contains(err.Error(),
		"use of closed network connection") {
		log.Fatal(err)
	}
}
Esempio n. 14
0
// InitAndStartNode gets called after having at least one membership sync with the cluster.
func (n *node) InitAndStartNode(wal *raftwal.Wal) {
	restart, err := n.initFromWal(wal)
	x.Check(err)

	if restart {
		fmt.Printf("RESTARTING\n")
		n.raft = raft.RestartNode(n.cfg)

	} else {
		if groups().HasPeer(n.gid) {
			n.joinPeers()
			n.raft = raft.StartNode(n.cfg, nil)

		} else {
			peers := []raft.Peer{{ID: n.id}}
			n.raft = raft.StartNode(n.cfg, peers)
			// Trigger election, so this node can become the leader of this single-node cluster.
			n.canCampaign = true
		}
	}
	go n.processCommitCh()
	go n.Run()
	// TODO: Find a better way to snapshot, so we don't lose the membership
	// state information, which isn't persisted.
	// go n.snapshotPeriodically()
	go n.batchAndSendMessages()
}
Esempio n. 15
0
func (w *grpcWorker) RaftMessage(ctx context.Context, query *Payload) (*Payload, error) {
	if ctx.Err() != nil {
		return &Payload{}, ctx.Err()
	}

	for idx := 0; idx < len(query.Data); {
		sz := int(binary.LittleEndian.Uint32(query.Data[idx : idx+4]))
		idx += 4
		msg := raftpb.Message{}
		if idx+sz-1 > len(query.Data) {
			return &Payload{}, x.Errorf(
				"Invalid query. Size specified: %v. Size of array: %v\n", sz, len(query.Data))
		}
		if err := msg.Unmarshal(query.Data[idx : idx+sz]); err != nil {
			x.Check(err)
		}
		if msg.Type != raftpb.MsgHeartbeat && msg.Type != raftpb.MsgHeartbeatResp {
			fmt.Printf("RECEIVED: %v %v-->%v\n", msg.Type, msg.From, msg.To)
		}
		if err := w.applyMessage(ctx, msg); err != nil {
			return &Payload{}, err
		}
		idx += sz
	}
	// fmt.Printf("Got %d messages\n", count)
	return &Payload{}, nil
}
Esempio n. 16
0
func (n *node) doSendMessage(to uint64, data []byte) {
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()

	addr := n.peers.Get(to)
	if len(addr) == 0 {
		return
	}
	pool := pools().get(addr)
	conn, err := pool.Get()
	x.Check(err)
	defer pool.Put(conn)

	c := NewWorkerClient(conn)
	p := &Payload{Data: data}

	ch := make(chan error, 1)
	go func() {
		_, err = c.RaftMessage(ctx, p)
		ch <- err
	}()

	select {
	case <-ctx.Done():
		return
	case <-ch:
		// We don't need to do anything if we receive any error while sending message.
		// RAFT would automatically retry.
		return
	}
}
Esempio n. 17
0
func doFilterString() {
	r := expand([]uint64{15161013152876854722}, "film.director.film")
	r = expand(r, "film.film.directed_by")
	r = expand(r, "film.director.film")
	fmt.Printf("Without filter: %d\n", len(r))

	var numHits int
	for _, u := range r {
		name, found := gNames[u]
		if !found {
			continue
		}

		tmp, err := normalize([]byte(name))
		x.Check(err)
		name = string(tmp)
		tokens := strings.Split(name, " ")

		var found1, found2 bool
		for _, t := range tokens {
			if t == "the" {
				found1 = true
			} else if t == "a" {
				found2 = true
			}
		}
		if found1 || found2 {
			numHits++
		}
	}
	fmt.Printf("With filter: %d\n", numHits)
}
Esempio n. 18
0
func writeToFile(fpath string, ch chan []byte) error {
	f, err := os.Create(fpath)
	if err != nil {
		return err
	}

	defer f.Close()
	x.Check(err)
	w := bufio.NewWriterSize(f, 1000000)
	gw, err := gzip.NewWriterLevel(w, gzip.BestCompression)
	if err != nil {
		return err
	}

	for buf := range ch {
		if _, err := gw.Write(buf); err != nil {
			return err
		}
	}
	if err := gw.Flush(); err != nil {
		return err
	}
	if err := gw.Close(); err != nil {
		return err
	}
	return w.Flush()
}
Esempio n. 19
0
func compare(srcl, dstl []R) {
	var buf bytes.Buffer
	var ps, ds, loop, matches int
	for ps < len(srcl) && ds < len(dstl) {
		loop++
		if loop%100 == 0 {
			//fmt.Printf(".")
		}
		s := srcl[ps]
		d := dstl[ds]
		if equal(s, d) {
			//fmt.Printf("Found match: %v\n", s)
			matches++
			ps++
			ds++
		} else if less(s, d) {
			// s < d, advance s
			buf.WriteString(s.String())
			buf.WriteRune('\n')
			ps++
		} else {
			ds++
		}
	}
	fmt.Printf("Found matches: %v\n", matches)
	x.Check(ioutil.WriteFile(*out, buf.Bytes(), 0644))
}
Esempio n. 20
0
func generateGroup(groupId uint32) (*task.GroupKeys, error) {
	it := pstore.NewIterator()
	defer it.Close()

	g := &task.GroupKeys{
		GroupId: groupId,
	}

	for it.SeekToFirst(); it.Valid(); it.Next() {
		k, v := it.Key(), it.Value()
		pk := x.Parse(k.Data())

		if pk == nil {
			continue
		}
		if group.BelongsTo(pk.Attr) != g.GroupId {
			it.Seek(pk.SkipPredicate())
			it.Prev() // To tackle it.Next() called by default.
			continue
		}

		var pl types.PostingList
		x.Check(pl.Unmarshal(v.Data()))

		kdup := make([]byte, len(k.Data()))
		copy(kdup, k.Data())
		key := &task.KC{
			Key:      kdup,
			Checksum: pl.Checksum,
		}
		g.Keys = append(g.Keys, key)
	}
	return g, it.Err()
}
Esempio n. 21
0
func (n *node) processCommitCh() {
	pending := make(chan struct{}, numPendingMutations)

	for e := range n.commitCh {
		if e.Data == nil {
			continue
		}

		if e.Type == raftpb.EntryConfChange {
			var cc raftpb.ConfChange
			cc.Unmarshal(e.Data)

			if len(cc.Context) > 0 {
				var rc task.RaftContext
				x.Check(rc.Unmarshal(cc.Context))
				n.Connect(rc.Id, rc.Addr)
			}

			n.raft.ApplyConfChange(cc)

		} else {
			go n.process(e, pending)
		}
	}
}
Esempio n. 22
0
// processFile sends mutations for a given gz file.
func processFile(file string) {
	fmt.Printf("\nProcessing %s\n", file)
	f, err := os.Open(file)
	x.Check(err)
	defer f.Close()
	gr, err := gzip.NewReader(f)
	x.Check(err)

	hc = http.Client{Timeout: time.Minute}
	mutation := make(chan string, 3*(*concurrent))

	var wg sync.WaitGroup
	for i := 0; i < *concurrent; i++ {
		wg.Add(1)
		go makeRequests(mutation, &wg)
	}

	var buf bytes.Buffer
	bufReader := bufio.NewReader(gr)
	num := 0
	for {
		err = readLine(bufReader, &buf)
		if err != nil {
			break
		}
		buf.WriteRune('\n')
		atomic.AddUint64(&s.rdfs, 1)
		num++

		if num >= *numRdf {
			mutation <- buf.String()
			buf.Reset()
			num = 0
		}
	}
	if err != io.EOF {
		x.Checkf(err, "Error while reading file")
	}
	if buf.Len() > 0 {
		mutation <- buf.String()
	}
	close(mutation)

	wg.Wait()
}
Esempio n. 23
0
func (n *node) Run() {
	firstRun := true
	ticker := time.NewTicker(time.Second)
	for {
		select {
		case <-ticker.C:
			n.raft.Tick()

		case rd := <-n.raft.Ready():
			x.Check(n.wal.Store(n.gid, rd.Snapshot, rd.HardState, rd.Entries))
			n.saveToStorage(rd.Snapshot, rd.HardState, rd.Entries)
			rcBytes, err := n.raftContext.Marshal()
			for _, msg := range rd.Messages {
				// NOTE: We can do some optimizations here to drop messages.
				x.Check(err)
				msg.Context = rcBytes
				n.send(msg)
			}

			if !raft.IsEmptySnap(rd.Snapshot) {
				n.processSnapshot(rd.Snapshot)
			}
			if len(rd.CommittedEntries) > 0 {
				x.Trace(n.ctx, "Found %d committed entries", len(rd.CommittedEntries))
			}
			for _, entry := range rd.CommittedEntries {
				// Just queue up to be processed. Don't wait on them.
				n.commitCh <- entry
			}

			n.raft.Advance()
			if firstRun && n.canCampaign {
				go n.raft.Campaign(n.ctx)
				firstRun = false
			}

		case <-n.done:
			return
		}
	}
}
Esempio n. 24
0
func (n *node) send(m raftpb.Message) {
	x.AssertTruef(n.id != m.To, "Seding message to itself")
	data, err := m.Marshal()
	x.Check(err)
	if m.Type != raftpb.MsgHeartbeat && m.Type != raftpb.MsgHeartbeatResp {
		fmt.Printf("\t\tSENDING: %v %v-->%v\n", m.Type, m.From, m.To)
	}
	select {
	case n.messages <- sendmsg{to: m.To, data: data}:
		// pass
	default:
		log.Fatalf("Unable to push messages to channel in send")
	}
}
Esempio n. 25
0
func (w *grpcWorker) applyMessage(ctx context.Context, msg raftpb.Message) error {
	var rc task.RaftContext
	x.Check(rc.Unmarshal(msg.Context))
	node := groups().Node(rc.Group)
	node.Connect(msg.From, rc.Addr)

	c := make(chan error, 1)
	go func() { c <- node.Step(ctx, msg) }()

	select {
	case <-ctx.Done():
		return ctx.Err()
	case err := <-c:
		return err
	}
}
Esempio n. 26
0
// ParseGroupConfig parses the config file and stores the predicate <-> group map.
func ParseGroupConfig(file string) error {
	cf, err := os.Open(file)
	if os.IsNotExist(err) {
		groupConfig.n = 1
		groupConfig.k = 1
		return nil
	}
	x.Check(err)
	if err = ParseConfig(cf); err != nil {
		return err
	}
	if groupConfig.n == 0 {
		return fmt.Errorf("Cant take modulo 0.")
	}
	return nil
}
Esempio n. 27
0
func (n *node) AddToCluster(ctx context.Context, pid uint64) error {
	addr := n.peers.Get(pid)
	x.AssertTruef(len(addr) > 0, "Unable to find conn pool for peer: %d", pid)
	rc := &task.RaftContext{
		Addr:  addr,
		Group: n.raftContext.Group,
		Id:    pid,
	}
	rcBytes, err := rc.Marshal()
	x.Check(err)
	return n.raft.ProposeConfChange(ctx, raftpb.ConfChange{
		ID:      pid,
		Type:    raftpb.ConfChangeAddNode,
		NodeID:  pid,
		Context: rcBytes,
	})
}
Esempio n. 28
0
func (n *node) process(e raftpb.Entry, pending chan struct{}) {
	if e.Type != raftpb.EntryNormal {
		return
	}

	pending <- struct{}{} // This will block until we can write to it.
	var proposal task.Proposal
	x.Check(proposal.Unmarshal(e.Data))

	var err error
	if proposal.Mutations != nil {
		err = n.processMutation(e, proposal.Mutations)
	} else if proposal.Membership != nil {
		err = n.processMembership(e, proposal.Membership)
	}
	n.props.Done(proposal.Id, err)
	<-pending // Release one.
}
Esempio n. 29
0
func doGen() {
	var numHits int
	for _, name := range gNames {
		tmp, err := normalize([]byte(name))
		x.Check(err)
		name = string(tmp)
		tokens := strings.Split(name, " ")

		var found1, found2 bool
		for _, t := range tokens {
			if t == "good" {
				found1 = true
			} else if t == "bad" {
				found2 = true
			}
		}
		if found1 || found2 {
			numHits++
		}
	}
	fmt.Printf("num gen hits %d\n", numHits)
}
Esempio n. 30
0
func (n *node) joinPeers() {
	// Get leader information for MY group.
	pid, paddr := groups().Leader(n.gid)
	n.Connect(pid, paddr)
	fmt.Printf("Connected with: %v\n", paddr)

	addr := n.peers.Get(pid)
	pool := pools().get(addr)
	x.AssertTruef(pool != nil, "Unable to find addr for peer: %d", pid)

	// Bring the instance up to speed first.
	_, err := populateShard(n.ctx, pool, 0)
	x.Checkf(err, "Error while populating shard")

	conn, err := pool.Get()
	x.Check(err)
	defer pool.Put(conn)

	c := NewWorkerClient(conn)
	x.Printf("Calling JoinCluster")
	_, err = c.JoinCluster(n.ctx, n.raftContext)
	x.Checkf(err, "Error while joining cluster")
	x.Printf("Done with JoinCluster call\n")
}