Beispiel #1
1
func retrieveFortune(clientConn net.UDPConn, fortuneInfoMsg clientServerUtils.FortuneInfoMessage) clientServerUtils.FortuneMessage {
	//Send FortuneReqMessage to fserver
	fortuneReqMsg := clientServerUtils.FortuneReqMessage{fortuneInfoMsg.FortuneNonce}
	fortuneReq, err := json.Marshal(fortuneReqMsg)
	if err != nil {
		fmt.Println("Error marshalling fortuneReqMsg: ", err)
		os.Exit(-1)
	}

	fmt.Println("Retrieving fortune from fserver")
	fserverUDPAddr := resolveUDPAddr(fortuneInfoMsg.FortuneServer)
	_, err = clientConn.WriteToUDP(fortuneReq, &fserverUDPAddr)
	if err != nil {
		fmt.Println("Error writing to fserver: ", err)
		os.Exit(-1)
	}

	//Receive FortuneMessage reply from fserver
	var buf [1024]byte
	msgLen, err := clientConn.Read(buf[:])
	if err != nil {
		fmt.Println("Error on read: ", err)
		os.Exit(-1)
	}
	fortuneReplyStr := string(buf[0:msgLen])
	fortuneBytes := []byte(fortuneReplyStr)

	var fortune clientServerUtils.FortuneMessage
	json.Unmarshal(fortuneBytes, &fortune)
	return fortune
}
func assertServerMatchesExpected(t *testing.T, server *net.UDPConn, buf []byte, expected string) {
	n, _ := server.Read(buf)
	msg := buf[:n]
	if string(msg) != expected {
		t.Fatalf("Line %s does not match expected: %s", string(msg), expected)
	}
}
Beispiel #3
0
// Returns a string of the socketreading
func UDPreadFromSocket(listenSock *net.UDPConn) (listenstring string, err error) {
	listenbuffer := make([]byte, 1024)
	antall, err := listenSock.Read(listenbuffer)

	listenstring = Trim(string(listenbuffer[0:antall]), "\x00")
	return listenstring, err
}
// Loops indefinitely to read packets and update connection status.
func ListenForDsPackets(listener *net.UDPConn) {
	var data [50]byte
	for {
		listener.Read(data[:])
		dsStatus := decodeStatusPacket(data)

		// Update the status and last packet times for this alliance/team in the global struct.
		dsConn := mainArena.AllianceStations[dsStatus.AllianceStation].DsConn
		if dsConn != nil && dsConn.TeamId == dsStatus.TeamId {
			dsConn.DriverStationStatus = dsStatus
			dsConn.LastPacketTime = time.Now()
			if dsStatus.RobotLinked {
				dsConn.LastRobotLinkedTime = time.Now()
			}
			dsConn.SecondsSinceLastRobotLink = time.Since(dsConn.LastRobotLinkedTime).Seconds()
			dsConn.DriverStationStatus.MissedPacketCount -= dsConn.missedPacketOffset

			// Log the packet if the match is in progress.
			matchTimeSec := mainArena.MatchTimeSec()
			if matchTimeSec > 0 && dsConn.log != nil {
				dsConn.log.LogDsStatus(matchTimeSec, dsStatus)
			}
		}
	}
}
Beispiel #5
0
// recv is used to receive until we get a shutdown
func (c *client) recv(l *net.UDPConn, msgCh chan *dns.Msg) {
	if l == nil {
		return
	}
	buf := make([]byte, 65536)

	var closed bool
	for {
		c.closeLock.RLock()
		closed = c.closed
		c.closeLock.RUnlock()
		if closed {
			break
		}
		n, err := l.Read(buf)
		if err != nil {
			Log.Printf("[ERR] mdns: Failed to read packet: %v", err)
			continue
		}
		msg := new(dns.Msg)
		if err := msg.Unpack(buf[:n]); err != nil {
			Log.Printf("[ERR] mdns: Failed to unpack packet: %v", err)
			continue
		}
		select {
		case msgCh <- msg:
		case <-c.closedCh:
			return
		}
	}
}
Beispiel #6
0
func PutRecv(conn *net.UDPConn, context *UdpContext) int {
	file := context.file
	buffer := make([]byte, 1024)
	read, err := conn.Read(buffer)

	if err == io.EOF {
		break
	}

	if err != nil {
		fmt.Printf("Error reading from network: %s. Aborting transfer\n", err)
		return
	}

	_, err = file.Write(buffer[:read])
	if err != nil {
		fmt.Println("Error writing to file; aborting transfer")
		return
	}

	_, err = conn.Write([]byte("put-ack"))
	if err != nil {
		fmt.Println("Error writing ack; aborting transfer")
		return
	}

	return read
}
Beispiel #7
0
func read(s *net.UDPConn) {
	b := make([]byte, 256)
	for {
		n, e := s.Read(b)
		if e != nil {
			log.Fatalf("Error reading from socket:  %s", e)
		}
		parts := strings.Split(string(b[0:n]), "\t")
		f, ferr := strconv.ParseFloat(parts[2], 64)
		if ferr != nil {
			log.Printf("Error parsing float:  %s", ferr)
			continue
		}
		t, terr := time.Parse(timeInFormat, strings.Split(parts[0], ".")[0])
		if terr != nil {
			log.Printf("Error parsing time: %s", terr)
			continue
		}

		// Do this to convert to UTC
		// t = time.SecondsToUTC(t.Seconds() - int64(time.LocalTime().ZoneOffset))

		record := reading{
			when:    t,
			sensor:  parts[1],
			reading: f,
		}
		readingsSingleton.input <- record
	}
}
Beispiel #8
0
func (proxy *UDPProxy) replyLoop(proxyConn *net.UDPConn, clientAddr *net.UDPAddr, clientKey *connTrackKey) {
	defer func() {
		proxy.connTrackLock.Lock()
		delete(proxy.connTrackTable, *clientKey)
		proxy.connTrackLock.Unlock()
		proxyConn.Close()
	}()

	readBuf := make([]byte, UDPBufSize)
	for {
		proxyConn.SetReadDeadline(time.Now().Add(UDPConnTrackTimeout))
	again:
		read, err := proxyConn.Read(readBuf)
		if err != nil {
			if err, ok := err.(*net.OpError); ok && err.Err == syscall.ECONNREFUSED {
				// This will happen if the last write failed
				// (e.g: nothing is actually listening on the
				// proxied port on the container), ignore it
				// and continue until UDPConnTrackTimeout
				// expires:
				goto again
			}
			return
		}
		for i := 0; i != read; {
			written, err := proxy.listener.WriteToUDP(readBuf[i:read], clientAddr)
			if err != nil {
				return
			}
			i += written
		}
	}
}
Beispiel #9
0
func receiveUDP(udpConn *net.UDPConn, ch chan []byte, queueSize *int32) {
	bytes := make([]byte, maxMessageSize)
	read, err := udpConn.Read(bytes)
	for err == nil {
		atomic.AddInt32(queueSize, 1)
		ch <- bytes[:read]
		bytes = make([]byte, maxMessageSize)
		read, err = udpConn.Read(bytes)
	}
}
Beispiel #10
0
func (l *Listener) start(conn *net.UDPConn) {
	for {
		buff := make([]byte, 2048)
		n, err := conn.Read(buff)
		if err != nil {
			log.Println(err)
		}

		secret, Lpos, err := getSecret(buff[0:n])
		if err != nil {
			continue
		}

		if l.print {
			log.Println(string(buff[0 : n-1]))
		}

		l.mapMu.RLock()
		source, ok := l.sources[secret]
		l.mapMu.RUnlock()

		if !ok {
			continue
		}

		go func() {
			if atomic.LoadInt32(source.closed) == 1 {
				return
			}

			handler := source.handler

			if source.test {
				l.mapMu.Lock()
				delete(l.sources, source.Secret)
				l.mapMu.Unlock()

				atomic.StoreInt32(source.closed, 1)

				handler.success <- struct{}{}

				source.rcon.StopLogRedirection(l.redirectAddr)
				source.rcon.Close()
				return
			}

			source.logsMu.Lock()
			source.logs.Write(buff[Lpos : n-1])
			source.logsMu.Unlock()

			m := ParseLogEntry(string(buff[Lpos : n-2]))
			m.Parsed.CallHandler(handler)
		}()
	}
}
Beispiel #11
0
// opusReceiver listens on the UDP socket for incoming packets
// and sends them across the given channel
// NOTE :: This function may change names later.
func (v *VoiceConnection) opusReceiver(udpConn *net.UDPConn, close <-chan struct{}, c chan *Packet) {

	if udpConn == nil || close == nil {
		return
	}

	p := Packet{}
	recvbuf := make([]byte, 1024)
	var nonce [24]byte

	for {
		rlen, err := udpConn.Read(recvbuf)
		if err != nil {
			// Detect if we have been closed manually. If a Close() has already
			// happened, the udp connection we are listening on will be different
			// to the current session.
			v.RLock()
			sameConnection := v.udpConn == udpConn
			v.RUnlock()
			if sameConnection {

				v.log(LogError, "udp read error, %s, %s", v.endpoint, err)
				v.log(LogDebug, "voice struct: %#v\n", v)

				go v.reconnect()
			}
			return
		}

		select {
		case <-close:
			return
		default:
			// continue loop
		}

		// For now, skip anything except audio.
		if rlen < 12 || recvbuf[0] != 0x80 {
			continue
		}

		// build a audio packet struct
		p.Type = recvbuf[0:2]
		p.Sequence = binary.BigEndian.Uint16(recvbuf[2:4])
		p.Timestamp = binary.BigEndian.Uint32(recvbuf[4:8])
		p.SSRC = binary.BigEndian.Uint32(recvbuf[8:12])
		// decrypt opus data
		copy(nonce[:], recvbuf[0:12])
		p.Opus, _ = secretbox.Open(nil, recvbuf[12:rlen], &nonce, &v.op4.SecretKey)

		if c != nil {
			c <- &p
		}
	}
}
Beispiel #12
0
// readInputUdp parses the buffer for UDP sockets.
func readInputUdp(conn net.UDPConn, parseChannel chan []byte, logger Logger, config *ConfigValues) {
	// config.Connection.Udp.Maxpacket is our max read

	// Large buffer to handle high UDP traffic, and manage GC pressure
	bufSize := 1 << 20
	buf := make([]byte, bufSize)
	offset := 0

	flush := func() []byte {
		parseChannel <- buf[0:offset]
		offset = 0
		return make([]byte, bufSize)
	}

	// Set initial deadline: 500ms
	sockErr := conn.SetDeadline(time.Now().Add(time.Millisecond * 500))
	if sockErr != nil {
		logger.Error.Printf("Error seting socket deadline: %s", sockErr)
		panic(sockErr)
	}

	for {
		length, err := conn.Read(buf[offset : offset+config.Connection.Udp.Maxpacket])
		if err == nil {
			// Always delimit our metrics
			buf[offset+length] = '\n'
			offset = offset + length + 1
		} else if terr, ok := err.(net.Error); ok && terr.Timeout() {
			if offset > 0 {
				buf = flush()
			}
			sockErr = conn.SetDeadline(time.Now().Add(time.Millisecond * 500))
			if sockErr != nil {
				panic(sockErr)
			}
		} else if strings.HasSuffix(err.Error(), "use of closed network connection") {
			// Go, it would be great if there was a better way to detect
			// this error...an enum?
			// Connection closed, lets wrap up and finish
			logger.Info.Printf("Stopping UDP read goroutine.")
			return
		} else {
			logger.Error.Println("UDP read error:", err)
		}

		// Full Buffer?
		if bufSize-offset <= config.Connection.Udp.Maxpacket {
			buf = flush()
		}

		// Track the number of UDP packets we read
		UdpPackets++
	}
}
Beispiel #13
0
// ListenAndWrite listens on the provided UDP address, parses the received
// packets and writes them to the provided api.Writer.
func (srv *Server) ListenAndWrite() error {
	addr := srv.Addr
	if addr == "" {
		addr = ":" + DefaultService
	}

	laddr, err := net.ResolveUDPAddr("udp", srv.Addr)
	if err != nil {
		return err
	}

	var sock *net.UDPConn
	if laddr.IP != nil && laddr.IP.IsMulticast() {
		var ifi *net.Interface
		if srv.Interface != "" {
			if ifi, err = net.InterfaceByName(srv.Interface); err != nil {
				return err
			}
		}
		sock, err = net.ListenMulticastUDP("udp", ifi, laddr)
	} else {
		sock, err = net.ListenUDP("udp", laddr)
	}
	if err != nil {
		return err
	}
	defer sock.Close()

	if srv.BufferSize <= 0 {
		srv.BufferSize = DefaultBufferSize
	}
	buf := make([]byte, srv.BufferSize)

	popts := ParseOpts{
		PasswordLookup: srv.PasswordLookup,
		SecurityLevel:  srv.SecurityLevel,
	}

	for {
		n, err := sock.Read(buf)
		if err != nil {
			return err
		}

		valueLists, err := Parse(buf[:n], popts)
		if err != nil {
			log.Printf("error while parsing: %v", err)
			continue
		}

		go dispatch(valueLists, srv.Writer)
	}
}
Beispiel #14
0
func (us *UDPSource) Run(l *net.UDPConn) {
	// Why doesn't this get called on ctrl + c
	var buf [1024]byte

	for {
		n, err := l.Read(buf[0:])
		if err != nil || n == 0 {
			break
		}
		us.data <- pipe.NewSimpleChunk(buf[0:n])
	}
}
Beispiel #15
0
// FIXME:  注意 conn 对象被多个goroutine 持有了,
func (client *UdpClient) readUDP(conn *net.UDPConn) {
	var err error

	defer func() {
		if err := recover(); nil != err {
			client.ERROR.Print("[panic]", client.logCtx, " read udp failed,", err)
		}
		conn.Close()
		atomic.StoreInt32(&client.conn_ok, 0)
	}()

	for 0 == atomic.LoadInt32(&client.is_closed) {
		var length int
		var bs []byte

		client.cached_rlock.Lock()
		bs = client.cached_readBytes
		client.cached_readBytes = nil
		client.cached_rlock.Unlock()
		if nil == bs {
			bs = newCachedBytes()
		}

		if client.DEBUG.IsEnabled() {
			client.DEBUG.Print(client.logCtx, "begin read pdu - ", len(bs))
		}
		length, err = conn.Read(bs)
		if 0 != atomic.LoadInt32(&client.is_closed) {
			break
		}
		if 10 > length {
			continue
		}

		if nil != err {
			client.cached_rlock.Lock()
			client.conn_error = err
			client.cached_rlock.Unlock()
			client.ERROR.Print(client.logCtx, "read udp from conn failed", err)
			break
		}

		if client.DEBUG.IsEnabled() {
			client.DEBUG.Print(client.logCtx, "read pdu ok - ", hex.EncodeToString(bs[:length]))
		}

		client.bytes_c <- bytesRequest{cached: bs, length: length}
	}

	client.ERROR.Print(client.logCtx, "read udp is exited.")
}
Beispiel #16
0
func eventLoop(backend appchilada.Backend, socket *net.UDPConn) {
	eventChan := make(chan appchilada.Event)
	// This is where all the aggregation is done
	go appchilada.Aggregator(eventChan, backend, *interval)

	buffer := make([]byte, 4096)
	for {
		if n, err := socket.Read(buffer); err != nil {
			log.Printf("Socket read error: %v", err)
		} else {
			handleMessage(eventChan, buffer[:n])
		}
	}
}
Beispiel #17
0
func sendRequest(sock *net.UDPConn, request, response []byte) (int, error) {
	if _, err := sock.Write(request); err != nil {
		return 0, err
	}

	sock.SetReadDeadline(time.Now().Add(time.Second))
	n, err := sock.Read(response)

	if err != nil {
		if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
			return 0, fmt.Errorf("no response from tracker: %s", err)
		}
	}

	return n, err
}
Beispiel #18
0
func proxyUdpToTun(conn *net.UDPConn, tun *os.File, tunMTU uint) {
	pkt := make([]byte, tunMTU)
	for {
		nrecv, err := conn.Read(pkt)
		if err != nil {
			log.V(1).Info("Error reading from socket: ", err)
		} else {
			nsent, err := tun.Write(pkt[:nrecv])
			switch {
			case err != nil:
				log.V(1).Info("Error writing to TUN device: ", err)
			case nsent != nrecv:
				log.V(1).Infof("Was only able to write %d out of %d bytes to TUN device: ", nsent, nrecv)
			}
		}
	}
}
Beispiel #19
0
func Listener(l *net.UDPConn, ctrl chan int, c chan []byte) {
	for {
		select {
		case _, ok := <-ctrl:
			if !ok {
				close(c)
				return
			}
		default:
			buffer := make([]byte, 8193)

			_, err := l.Read(buffer)

			if err != nil {
				continue
			}

			c <- bytes.Trim(buffer, "\x00")
		}
	}
}
Beispiel #20
0
// recv is used to receive until we get a shutdown
func (c *client) recv(l *net.UDPConn, msgCh chan *dns.Msg) {
	if l == nil {
		return
	}
	buf := make([]byte, 65536)
	for !c.closed {
		n, err := l.Read(buf)
		if err != nil {
			continue
		}
		msg := new(dns.Msg)
		if err := msg.Unpack(buf[:n]); err != nil {
			log.Printf("[ERR] mdns: Failed to unpack packet: %v", err)
			continue
		}
		select {
		case msgCh <- msg:
		case <-c.closedCh:
			return
		}
	}
}
Beispiel #21
0
func (d *discoverer) receiveBroadcasts(conn *net.UDPConn) {
	buf := make([]byte, 65536)
	for {

		n, err := conn.Read(buf)
		if err != nil {
			return
		}

		// 44 is ','
		elems := bytes.Split(buf[:n], []byte{44})
		if !bytes.Equal(elems[0], d.token) {
			continue
		}

		// pull the token off the front
		elems = elems[1:]

		for _, elem := range elems {
			// 64 is '@'
			parts := bytes.SplitN(elem, []byte{64}, 2)
			if len(parts) != 2 {
				continue
			}

			packet := idip{
				id: string(parts[0]),
				ip: string(parts[1]),
			}

			select {
			case d.aliveMsg <- packet:
			case <-d.done:
				return
			}
		}
	}
}
Beispiel #22
0
func retrieveNonce(clientConn net.UDPConn, aserverUDPAddr *net.UDPAddr) clientServerUtils.NonceMessage {
	//Send arbitrary UDP message to aserver to get nonce
	nonceReq := []byte("Hello aserver!  I'd like a nonce!")

	fmt.Println("Sending initial request to aserver") //XXX
	_, err := clientConn.WriteToUDP(nonceReq, aserverUDPAddr)

	//Receive NonceMessage reply
	var buf [1024]byte
	msgLen, err := clientConn.Read(buf[:])
	if err != nil {
		fmt.Println("Error on read: ", err)
		os.Exit(-1)
	}
	nonceReplyStr := string(buf[0:msgLen])
	bufBytes := []byte(nonceReplyStr)

	fmt.Println("Received reply from aserver:", nonceReplyStr) //XXX

	var nonce clientServerUtils.NonceMessage
	json.Unmarshal(bufBytes, &nonce)
	return nonce
}
Beispiel #23
0
func RetrieveNonce(clientConn net.UDPConn, aserverUDPAddr *net.UDPAddr) common.NonceMessage {
	//Send arbitrary UDP message to aserver to get nonce
	nonceReq := []byte("Hello aserver!  I'd like a nonce!")
	_, err := clientConn.WriteToUDP(nonceReq, aserverUDPAddr)
	if err != nil {
		fmt.Println("Error writing to aserver: ", err)
		os.Exit(-1)
	}

	//Receive NonceMessage reply
	var buf [1024]byte
	msgLen, err := clientConn.Read(buf[:])
	if err != nil {
		fmt.Println("Error on read: ", err)
		os.Exit(-1)
	}
	nonceReplyStr := string(buf[0:msgLen])
	bufBytes := []byte(nonceReplyStr)

	var nonce common.NonceMessage
	json.Unmarshal(bufBytes, &nonce)
	return nonce
}
Beispiel #24
0
//Retrieves GoalMessage from aserver
func RetrieveGoalMsg(clientConn net.UDPConn, aserverUDPAddr *net.UDPAddr, hashMsg common.HashMessage) common.GoalMessage {
	//Send HashMessage to aserver to get GoalMessage
	req, err := json.Marshal(hashMsg)
	if err != nil {
		fmt.Println("Error marshalling hashMsg: ", err)
		os.Exit(-1)
	}
	_, err = clientConn.WriteToUDP(req, aserverUDPAddr)
	if err != nil {
		fmt.Println("Error writing hashMsg to aserver: ", err)
		os.Exit(-1)
	}

	//Receive GoalMessage reply
	var buf [1024]byte
	msgLen, err := clientConn.Read(buf[:])
	if err != nil {
		fmt.Println("Error on read: ", err)
		os.Exit(-1)
	}
	var goalMsg common.GoalMessage
	json.Unmarshal(buf[0:msgLen], &goalMsg)
	return goalMsg
}
Beispiel #25
0
// NetIO shares GitChanges on toNet with the network via a multicast group. It
// will pass on GitChanges from the network via fromNet. It uniques the daemon
// instance by changing the .Name member to be name@<host IP>/<original .Name)
func NetIO(l log.Logger, repo Repo, addr *net.UDPAddr, fromNet, toNet chan GitChange) {
	var (
		err                error
		recvConn, sendConn *net.UDPConn // UDP connections to allow us to send and	receive change updates
	)

	l.Info("Joining %v multicast(%t) group", addr, addr.IP.IsMulticast())
	if recvConn, sendConn, err = establishConnPair(addr); err != nil {
		l.Critical("Error joining listening: %s\n", addr, err)
		return
	}

	l.Info("Successfully joined %v multicast(%t) group", addr, addr.IP.IsMulticast())
	defer recvConn.Close()
	defer sendConn.Close()
	hostIp := sendConn.LocalAddr().(*net.UDPAddr).IP.String()

	term := false
	defer func() { term = true }()
	rawFromNet := make(chan []byte, 128)
	go func() {
		for !term {
			b := make([]byte, 1024)

			if n, err := recvConn.Read(b); err != nil {
				l.Critical("Cannot read socket: %s", err)
				continue
			} else {
				rawFromNet <- b[:n]
			}
		}
	}()

	for {
		select {
		case req, ok := <-toNet:
			if !ok {
				return
			}

			req.User = repo.User()
			req.HostIp = hostIp

			l.Info("Sending %+v", req)
			buf := &bytes.Buffer{}
			enc := gob.NewEncoder(buf)

			if err := enc.Encode(req); err != nil {
				l.Critical("%s", err)
				continue
			}

			l.Fine("Sending %+v", buf.Bytes())
			if _, err := sendConn.Write(buf.Bytes()); err != nil {
				l.Critical("%s", err)
				continue
			}

		case resp := <-rawFromNet:
			var change GitChange
			dec := gob.NewDecoder(bytes.NewReader(resp))

			if err := dec.Decode(&change); err != nil {
				l.Critical("%s", err)
				continue
			} else {
				l.Debug("received %+v", change)
			}

			if rootCommit, err := repo.RootCommit(); err != nil {
				log.Critical("Error getting root commit")
			} else {
				if (repo.User() != change.User) && (rootCommit == change.RootCommit) {
					fromNet <- change
				}
			}
		}
	}
}
Beispiel #26
0
func processor(id int, conn *net.UDPConn, tick <-chan compaction_tick, quit, done chan struct{}) {
	buf := make([]byte, 65536, 65536)
	index := make(map[string]*deque)
	data := &Data{}
	total := 0

	t := (<-tick).t
	log.Printf("new processor %d", id)

	var file *os.File
	getFile := func(t time.Time) (*os.File, int64) {
		if file == nil {
			epoch := t.Unix()
			fileName := fmt.Sprintf("%s%d-%d", filePath, epoch, id)
			log.Printf("[%d] open new file %s for epoch %d", id, fileName, epoch)

			var err error
			file, err = os.OpenFile(fileName, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0664)
			if err != nil {
				log.Println("Failed to open file "+fileName, err)
				return nil, 0
			}

			return file, 0
		}

		offset, _ := file.Seek(0, 0)
		return file, offset
	}

loop:
	for {
		select {
		case ctick := <-tick:
			log.Printf("[%d] send job to compactor\n", id)
			ctick.ch <- compaction_job{id, total, index}

			total = 0
			t = ctick.t
			index = make(map[string]*deque)
			if file != nil {
				file.Close()
				file = nil
			}

			select {
			case <-quit:
				break loop
			default:
			}

		default:
			conn.SetReadDeadline(time.Now().Add(time.Millisecond))
			length, err := conn.Read(buf)
			if err != nil {
				if err.(net.Error).Timeout() == false {
					log.Println("UDP read error", err)
				}
				continue
			}

			if err := proto.Unmarshal(buf[:length], data); err != nil {
				log.Println("Failed to decode", err)
				continue
			}

			file, offset := getFile(t)
			if file == nil {
				log.Println("Failed to get file")
				continue
			}

			if _, err := file.Write(buf); err != nil {
				log.Println("Failed to write to file", err)
				continue
			}

			// file.Sync()

			tags := data.GetHeader().GetTags()
			if len(tags) > max_tags_in_message {
				log.Println("Too many tags in message")
				continue
			}

			for _, tag := range tags {
				if len(tag) > max_tag_length {
					log.Println("Too long tag")
					continue
				}

				deque, ok := index[tag]
				if !ok {
					deque = newDeque(id)
					index[tag] = deque
				}

				deque.Append(uint32(offset))
			}

			total++
		}
	}

	done <- struct{}{}
}
Beispiel #27
0
func processor(partition int, conn *net.UDPConn, tick <-chan compactionTick, quit chan struct{}, wg *sync.WaitGroup) {
	wg.Add(1)
	defer wg.Done()

	shouldQuit := false
	buf := make([]byte, 65536+4, 65536+4)
	index := make(map[string]*deque)
	data := pb.Data{}
	totalMsg := 0

	log.Printf("new processor %d", partition)

	var file *os.File
	defer file.Close()
	getFile := func(t time.Time) (*os.File, int64) {
		if file == nil {
			_, fileName, err := makePath(t)
			if err != nil {
				log.Println("Failed to create path", fileName, err)
				return nil, 0
			}

			fileName += fmt.Sprintf("%d.%d.data", t.Unix(), partition)
			log.Printf("[%d] open new file %s for epoch %d", partition, fileName, t.Unix())

			if file, err = createFile(fileName); err != nil {
				log.Println("Failed to open file", fileName, err)
				return nil, 0
			}

			if _, err = file.WriteString(dataFileHeader); err != nil {
				log.Println("Failed to write to file", err)
				return nil, 0
			}
		}

		offset, _ := file.Seek(0, 1)
		return file, offset
	}

	ctick := <-tick

loop:
	for {
		select {
		case <-quit:
			shouldQuit = true
			log.Printf("[%d] will quit", partition)

		case ct := <-tick:
			log.Printf("[%d] send job to compactor %d\n", partition, ctick.t.Unix())
			ctick.ch <- compactionJob{partition, totalMsg, index}
			index = make(map[string]*deque)
			file.Close()
			file = nil
			totalMsg = 0
			ctick = ct

			if shouldQuit {
				log.Printf("[%d] quiting", partition)
				break loop
			}

		default:
			conn.SetReadDeadline(time.Now().Add(time.Millisecond))
			length, err := conn.Read(buf[4:])
			if err != nil {
				if nerr, ok := err.(net.Error); ok && nerr.Timeout() == false {
					log.Println("UDP read error", err)
				}
				continue
			}

			data.Reset()
			if err := data.Unmarshal(buf[4 : length+4]); err != nil {
				log.Println("Failed to decode", err)
				continue
			}

			file, offset := getFile(ctick.t)
			if file == nil {
				log.Println("Failed to get file")
				continue
			}

			intToByteArray(uint32(length), buf[0:4])
			if _, err := file.Write(buf[:length+4]); err != nil {
				log.Println("Failed to write to file", err)
				continue
			}

			tags := data.GetHeader().GetTags()
			if len(tags) > maxTagsInMessage {
				log.Println("Too many tags in message ", len(tags))
				continue
			}

			for _, tag := range tags {
				if len(tag) > maxTagLength {
					log.Println("Too long tag")
					continue
				}

				deque, ok := index[tag]
				if !ok {
					deque = newDeque(partition)
					index[tag] = deque
				}

				deque.Append(uint32(offset))
			}

			totalMsg++
		}
	}
}
func getScrapeFromUDPTracker(con *net.UDPConn, connectionID uint64, infoHashes []string) (tr []InfoHashDetails, err error) {
	transactionID := rand.Uint32()

	announcementRequest := new(bytes.Buffer)
	err = binary.Write(announcementRequest, binary.BigEndian, connectionID)
	if err != nil {
		return
	}
	var action uint32 = 2
	err = binary.Write(announcementRequest, binary.BigEndian, action)
	if err != nil {
		return
	}
	err = binary.Write(announcementRequest, binary.BigEndian, transactionID)
	if err != nil {
		return
	}

	for _, infoHash := range infoHashes {
		var binaryInfoHash []byte
		binaryInfoHash, err = hex.DecodeString(infoHash)
		err = binary.Write(announcementRequest, binary.BigEndian, binaryInfoHash)
		if err != nil {
			return
		}
	}

	_, err = con.Write(announcementRequest.Bytes())
	if err != nil {
		return
	}

	torrentRequestCount := len(infoHashes)

	const minimumResponseLen = 8
	const torrentsDataSize = 12
	expectedResponseLen := minimumResponseLen + torrentsDataSize*torrentRequestCount
	responseBytes := make([]byte, expectedResponseLen)

	var responseLen int
	responseLen, err = con.Read(responseBytes)
	if err != nil {
		return
	}
	if responseLen < minimumResponseLen {
		err = fmt.Errorf("Unexpected response size %d", responseLen)
		return
	}
	response := bytes.NewBuffer(responseBytes)
	var responseAction uint32
	err = binary.Read(response, binary.BigEndian, &responseAction)
	if err != nil {
		return
	}
	if responseAction != action {
		err = fmt.Errorf("Unexpected response action %d", action)
		return
	}
	var responseTransactionID uint32
	err = binary.Read(response, binary.BigEndian, &responseTransactionID)
	if err != nil {
		return
	}
	if transactionID != responseTransactionID {
		err = fmt.Errorf("Unexpected response transactionID %x", responseTransactionID)
		return
	}
	tr = make([]InfoHashDetails, torrentRequestCount)
	for i, infoHash := range infoHashes {
		var seeders uint32
		err = binary.Read(response, binary.BigEndian, &seeders)
		if err != nil {
			return
		}
		var completed uint32
		err = binary.Read(response, binary.BigEndian, &completed)
		if err != nil {
			return
		}
		var leechers uint32
		err = binary.Read(response, binary.BigEndian, &leechers)
		if err != nil {
			return
		}
		tr[i] = InfoHashDetails{
			InfoHash:  infoHash,
			Seeders:   uint(seeders),
			Completed: uint(completed),
			Leechers:  uint(leechers),
		}
	}
	return
}
func connectToUDPTracker(con *net.UDPConn) (connectionID uint64, err error) {
	var connectionRequest_connectionID uint64 = 0x41727101980
	var action uint32 = 0
	transactionID := rand.Uint32()

	connectionRequest := new(bytes.Buffer)
	err = binary.Write(connectionRequest, binary.BigEndian, connectionRequest_connectionID)
	if err != nil {
		return
	}
	err = binary.Write(connectionRequest, binary.BigEndian, action)
	if err != nil {
		return
	}
	err = binary.Write(connectionRequest, binary.BigEndian, transactionID)
	if err != nil {
		return
	}

	_, err = con.Write(connectionRequest.Bytes())
	if err != nil {
		return
	}

	connectionResponseBytes := make([]byte, 16)

	var connectionResponseLen int
	connectionResponseLen, err = con.Read(connectionResponseBytes)
	if err != nil {
		return
	}
	if connectionResponseLen != 16 {
		err = fmt.Errorf("Unexpected response size %d", connectionResponseLen)
		return
	}
	connectionResponse := bytes.NewBuffer(connectionResponseBytes)
	var connectionResponseAction uint32
	err = binary.Read(connectionResponse, binary.BigEndian, &connectionResponseAction)
	if err != nil {
		return
	}
	if connectionResponseAction != 0 {
		err = fmt.Errorf("Unexpected response action %d", connectionResponseAction)
		return
	}
	var connectionResponseTransactionID uint32
	err = binary.Read(connectionResponse, binary.BigEndian, &connectionResponseTransactionID)
	if err != nil {
		return
	}
	if connectionResponseTransactionID != transactionID {
		err = fmt.Errorf("Unexpected response transactionID %x != %x",
			connectionResponseTransactionID, transactionID)
		return
	}

	err = binary.Read(connectionResponse, binary.BigEndian, &connectionID)
	if err != nil {
		return
	}
	return
}
Beispiel #30
0
func getAnnouncementFromUDPTracker(con *net.UDPConn, connectionID uint64, report ClientStatusReport) (tr *TrackerResponse, err error) {
	transactionID := rand.Uint32()

	announcementRequest := new(bytes.Buffer)
	err = binary.Write(announcementRequest, binary.BigEndian, connectionID)
	if err != nil {
		return
	}
	var action uint32 = 1
	err = binary.Write(announcementRequest, binary.BigEndian, action)
	if err != nil {
		return
	}
	err = binary.Write(announcementRequest, binary.BigEndian, transactionID)
	if err != nil {
		return
	}
	err = binary.Write(announcementRequest, binary.BigEndian, []byte(report.InfoHash))
	if err != nil {
		return
	}
	err = binary.Write(announcementRequest, binary.BigEndian, []byte(report.PeerId))
	if err != nil {
		return
	}
	err = binary.Write(announcementRequest, binary.BigEndian, report.Downloaded)
	if err != nil {
		return
	}
	err = binary.Write(announcementRequest, binary.BigEndian, report.Left)
	if err != nil {
		return
	}
	err = binary.Write(announcementRequest, binary.BigEndian, report.Uploaded)
	if err != nil {
		return
	}
	var event uint32 = 0
	switch report.Event {
	case "":
		event = 0
	case "completed":
		event = 1
	case "started":
		event = 2
	case "stopped":
		event = 3
	default:
		err = fmt.Errorf("Unknown event string %v", report.Event)
		return
	}
	err = binary.Write(announcementRequest, binary.BigEndian, event)
	if err != nil {
		return
	}
	var ipAddress uint32 = 0
	err = binary.Write(announcementRequest, binary.BigEndian, ipAddress)
	if err != nil {
		return
	}
	var key uint32 = 0
	err = binary.Write(announcementRequest, binary.BigEndian, key)
	if err != nil {
		return
	}

	const peerRequestCount = 10
	var numWant uint32 = peerRequestCount
	err = binary.Write(announcementRequest, binary.BigEndian, numWant)
	if err != nil {
		return
	}
	err = binary.Write(announcementRequest, binary.BigEndian, report.Port)
	if err != nil {
		return
	}

	_, err = con.Write(announcementRequest.Bytes())
	if err != nil {
		return
	}

	const minimumResponseLen = 20
	const peerDataSize = 6
	expectedResponseLen := minimumResponseLen + peerDataSize*peerRequestCount
	responseBytes := make([]byte, expectedResponseLen)

	var responseLen int
	responseLen, err = con.Read(responseBytes)
	if err != nil {
		return
	}
	if responseLen < minimumResponseLen {
		err = fmt.Errorf("Unexpected response size %d", responseLen)
		return
	}
	response := bytes.NewBuffer(responseBytes)
	var responseAction uint32
	err = binary.Read(response, binary.BigEndian, &responseAction)
	if err != nil {
		return
	}
	if action != 1 {
		err = fmt.Errorf("Unexpected response action %d", action)
		return
	}
	var responseTransactionID uint32
	err = binary.Read(response, binary.BigEndian, &responseTransactionID)
	if err != nil {
		return
	}
	if transactionID != responseTransactionID {
		err = fmt.Errorf("Unexpected response transactionID %x", responseTransactionID)
		return
	}
	var interval uint32
	err = binary.Read(response, binary.BigEndian, &interval)
	if err != nil {
		return
	}
	var leechers uint32
	err = binary.Read(response, binary.BigEndian, &leechers)
	if err != nil {
		return
	}
	var seeders uint32
	err = binary.Read(response, binary.BigEndian, &seeders)
	if err != nil {
		return
	}

	peerCount := (responseLen - minimumResponseLen) / peerDataSize
	peerDataBytes := make([]byte, peerDataSize*peerCount)
	err = binary.Read(response, binary.BigEndian, &peerDataBytes)
	if err != nil {
		return
	}

	tr = &TrackerResponse{
		Interval:   uint(interval),
		Complete:   uint(seeders),
		Incomplete: uint(leechers),
		Peers:      string(peerDataBytes)}
	return
}