Ejemplo n.º 1
0
// deriveKey fills out the Key field.
func (sk *SecretKey) deriveKey(password *[]byte) error {
	key, err := scrypt.Key(*password, sk.Parameters.Salt[:],
		sk.Parameters.N,
		sk.Parameters.R,
		sk.Parameters.P,
		len(sk.Key))
	if err != nil {
		return err
	}
	copy(sk.Key[:], key)
	zero.Bytes(key)

	// I'm not a fan of forced garbage collections, but scrypt allocates a
	// ton of memory and calling it back to back without a GC cycle in
	// between means you end up needing twice the amount of memory.  For
	// example, if your scrypt parameters are such that you require 1GB and
	// you call it twice in a row, without this you end up allocating 2GB
	// since the first GB probably hasn't been released yet.
	debug.FreeOSMemory()

	// I'm not a fan of forced garbage collections, but scrypt allocates a
	// ton of memory and calling it back to back without a GC cycle in
	// between means you end up needing twice the amount of memory.  For
	// example, if your scrypt parameters are such that you require 1GB and
	// you call it twice in a row, without this you end up allocating 2GB
	// since the first GB probably hasn't been released yet.
	debug.FreeOSMemory()

	return nil
}
Ejemplo n.º 2
0
// Starts another thread to perform OS memory freeing
func FreeMemory(durationMinutes int) {
	go func() {
		for {
			debug.FreeOSMemory()
			debug.FreeOSMemory()
			time.Sleep(time.Duration(durationMinutes) * time.Minute)
		}
	}()
}
Ejemplo n.º 3
0
func write(wr writeRequest, id string) {
	cmds, idArr, bufTypeArr := createCommands(wr, id)

	qr := qReport{
		Cmd: "Queued",
		//Type: bufTypeArr,
		Ids:  idArr,
		D:    cmds,
		QCnt: wr.p.itemsInBuffer,
		Port: wr.p.portConf.Name,
	}
	json, _ := json.Marshal(qr)
	h.broadcastSys <- json

	// now send off the commands to the appropriate channel
	for index, cmdToSendToChannel := range cmds {
		//cmdIdCtr++
		//cmdId := "fakeid-" + strconv.Itoa(cmdIdCtr)
		cmdId := idArr[index]
		if bufTypeArr[index] == "Buf" {
			log.Println("Send was normal send, so sending to wr.p.sendBuffered")
			wr.p.sendBuffered <- Cmd{cmdToSendToChannel, cmdId, false, false}
		} else {
			log.Println("Send was sendnobuf, so sending to wr.p.sendNoBuf")
			wr.p.sendNoBuf <- Cmd{cmdToSendToChannel, cmdId, true, false}
		}
	}

	// garbage collect
	if *gcType == "max" {
		debug.FreeOSMemory()
	}

}
Ejemplo n.º 4
0
func fillseq() {
	dbname := os.Args[0] + ".db"
	f, err := os.OpenFile(dbname, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666)
	if err != nil {
		log.Fatal(err)
	}

	defer func() {
		f.Close()
		os.Remove(f.Name())
	}()

	filer := lldb.NewSimpleFileFiler(f)
	a, err := lldb.NewAllocator(filer, &lldb.Options{})
	if err != nil {
		log.Println(err)
		return
	}

	a.Compress = true
	b, _, err := lldb.CreateBTree(a, nil)
	if err != nil {
		log.Println(err)
		return
	}

	var keys [N][16]byte
	for i := range keys {
		binary.BigEndian.PutUint32(keys[i][:], uint32(i))
	}

	debug.FreeOSMemory()
	t0 := time.Now()
	for _, key := range keys {
		if err = b.Set(key[:], value100); err != nil {
			log.Println(err)
			return
		}
	}
	if err := filer.Sync(); err != nil {
		log.Println(err)
		return
	}
	var ms runtime.MemStats
	runtime.ReadMemStats(&ms)

	d := time.Since(t0)
	fi, err := f.Stat()
	if err != nil {
		log.Println(err)
		return
	}

	secs := float64(d/time.Nanosecond) / float64(time.Second)
	sz := fi.Size()
	fmt.Printf("fillseq      :%19v/op;%7.1f MB/s (%g secs, %d bytes)\n", d/N, float64(sz)/secs/1e6, secs, sz)
	nn, bytes := bufs.GCache.Stats()
	fmt.Printf("%d %d\n", nn, bytes)
	fmt.Printf("%+v\n", ms)
}
Ejemplo n.º 5
0
func (s *MultipartTestSuite) TestPartGC() {
	part := NewPart()
	wrapper := NewDataWrapper()
	part.SetContentObject(wrapper)
	s.part.AddPart(part)

	allParts := s.part.GetPart(0)
	allParts2 := s.part.GetPart(0)

	_, ok := allParts.(Part)
	_, ok = allParts2.(Part)

	allParts = nil
	part = nil
	wrapper = nil
	debug.FreeOSMemory()

	// Try to access allParts2.
	// This should not fail under valgrind
	typeString := "multipart/mixed"
	boundary := "--foo++bar==foo--"
	contentType := NewContentTypeFromString(typeString)
	contentType.SetParameter("boundary", boundary)
	allParts2.SetContentType(contentType)

	assert.True(s.T(), ok)
}
Ejemplo n.º 6
0
func benchmarkPrev(b *testing.B, n int) {
	t := TreeNew(cmp)
	for i := 0; i < n; i++ {
		t.Set(i, 0)
	}
	debug.FreeOSMemory()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		en, err := t.SeekLast()
		if err != nil {
			b.Fatal(err)
		}

		m := 0
		for {
			if _, _, err = en.Prev(); err != nil {
				break
			}
			m++
		}
		if m != n {
			b.Fatal(m)
		}
	}
}
Ejemplo n.º 7
0
//
// for test only, run GC for profile GC in huge map
func (eng *Engine) gcRunner(interval int) {
	tk := time.NewTicker(time.Duration(interval) * time.Second)
	defer tk.Stop()
	var pre, cnt int64
	var iopre, iocnt int64
	var rwpre, rwcnt int64
	var norepeatgc bool
	for {
		//
		cnt = atomic.LoadInt64(eng.aliveCount)
		iocnt = atomic.LoadInt64(eng.newCount)
		rwcnt = atomic.LoadInt64(eng.rwCount)
		if cnt > 0 {
			if eng.conf.gctest && (pre != cnt || iopre != iocnt || rwpre != rwcnt) {
				fmt.Printf("GC with %d connections.\n", cnt)
				runtime.GC()
				//fmt.Printf("GC done.\n")
				pre = cnt
			}
			norepeatgc = false
		} else if norepeatgc == false {
			// even if eng.conf.gctest == false, still call FreeOSMemory when connections == 0
			norepeatgc = true
			fmt.Printf("FreeOSMemory with %d connections.\n", cnt)
			// free memory
			debug.FreeOSMemory()
			//fmt.Printf("FreeOSMemory done.\n")
			pre = cnt
		}
		<-tk.C
	}
}
Ejemplo n.º 8
0
func benchmarkNext(b *testing.B, n int) {
	t := TreeNew(cmp)
	for i := int64(0); i < int64(n); i++ {
		t.Set(i, struct{}{})
	}
	debug.FreeOSMemory()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		en, err := t.SeekFirst()
		if err != nil {
			b.Fatal(err)
		}

		m := 0
		for {
			if _, _, err = en.Next(); err != nil {
				break
			}
			m++
		}
		if m != n {
			b.Fatal(m)
		}
	}
	b.StopTimer()
	t.Close()
}
Ejemplo n.º 9
0
func monitor() {
	c := time.Tick(1 * time.Second)
	mem := new(runtime.MemStats)
	origPct := debug.SetGCPercent(100)
	debug.SetGCPercent(origPct)
	for _ = range c {
		runtime.ReadMemStats(mem)
		mu.Lock()
		defer mu.Unlock()
		if tSize < 0 {
			continue
		}
		// Occupancy fraction: 70%. Don't GC before hitting this.
		softLimit := float64(tSize) * 0.7
		pct := softLimit / float64(mem.Alloc) * 100
		fmt.Printf("gctune: pct: %0.5f, target: %d, softLimit: %0.2f, Alloc: %d, Sys: %d\n", pct, tSize, softLimit, mem.Alloc, mem.Sys)
		if pct < 50 {
			// If this is too low, GC frequency increases too much.
			pct = 50
		}
		debug.SetGCPercent(int(pct))
		if mem.Sys > uint64(tSize*70/100) {
			fmt.Println("freeing")
			debug.FreeOSMemory()
		}
	}
}
Ejemplo n.º 10
0
func receiveDecrypt(conn net.Conn) (Message, error) {
	// Our work is:
	// (receive) -> [de-GOB] -> [DECRYPT] -> [de-GOB] -> msg

	// Receive data and de-serialize to get the encrypted message
	encMsg := new([]byte)
	receive := gob.NewDecoder(conn)
	if err := receive.Decode(encMsg); err != nil {
		return Message{}, err
	}

	// Create decrypter and pass it the encrypted message
	r := bytes.NewReader(*encMsg)
	decrypter, err := saltsecret.NewReader(r, conf.Key, saltsecret.DECRYPT, false)
	if err != nil {
		return Message{}, err
	}

	// Read unencrypted serialized message and de-serialize it
	msg := new(Message)
	dec := gob.NewDecoder(decrypter)
	if err = dec.Decode(msg); err != nil {
		return Message{}, err
	}
	debug.FreeOSMemory()
	return *msg, nil
}
Ejemplo n.º 11
0
func encryptDispatch(conn net.Conn, m Message) error {
	// We want to sent encrypted data.
	// In order to encrypt, we need to first serialize the message.
	// In order to sent/receive hassle free, we need to serialize the encrypted message
	// So: msg -> [GOB] -> [ENCRYPT] -> [GOB] -> (dispatch)

	// Create encrypter
	var encMsg bytes.Buffer
	encrypter, err := saltsecret.NewWriter(&encMsg, conf.Key, saltsecret.ENCRYPT, true)
	if err != nil {
		return err
	}

	// Serialize message
	enc := gob.NewEncoder(encrypter)
	if err = enc.Encode(m); err != nil {
		return err
	}

	// Flush encrypter to actuall encrypt the message
	if err = encrypter.Flush(); err != nil {
		return err
	}

	// Serialize encrypted message and dispatch it
	dispatch := gob.NewEncoder(conn)
	if err = dispatch.Encode(encMsg.Bytes()); err != nil {
		return err
	}
	debug.FreeOSMemory()
	return nil
}
Ejemplo n.º 12
0
func (cmc *FileMonitor) cmpStat(f_old map[string]os.FileInfo, f_new map[string]os.FileInfo) {
	var fo FileOp
	for i, v := range f_old {
		if in, ok := f_new[i]; ok {
			if !v.ModTime().Equal(in.ModTime()) {
				// update
				f_old[i] = in
				delete(f_new, i)
				fo.fname = i
				fo.op = OP_MOD
				cmc.f <- fo
			}
			delete(f_new, i)
			// no changed
		} else {
			delete(f_old, i)
			fo.fname = i
			fo.op = OP_DEL
			cmc.f <- fo
			// delete
		}
	}

	for i, v := range f_new {
		// add
		f_old[i] = v
		delete(f_new, i)
		fo.fname = i
		fo.op = OP_ADD
		cmc.f <- fo
	}
	debug.FreeOSMemory()
}
Ejemplo n.º 13
0
func (chat *Chat) removeConnection(toRemove *Connection) {
	for i, el := range chat.connections {
		if *toRemove == el {
			//chat.connections = append(chat.connections[:i], chat.connections[i+1:]...)

			// THE FOLLOWING IS UNNECESSARILY COMPLEX AND ONLY TO RULE OUT A MemLeak
			copy(chat.connections[i:], chat.connections[i+1:])
			chat.connections[len(chat.connections)-1] = Connection{} //nil // or the zero value of T
			chat.connections = chat.connections[:len(chat.connections)-1]

			// THE FOLLOWING IS UNNECESSARILY COMPLEX AND ONLY TO RULE OUT A MemLeak
			if cap(chat.connections) > 2*len(chat.connections) {
				fmt.Println("SHRINK")
				shrink := make([]Connection, len(chat.connections))
				copy(shrink, chat.connections)
				chat.connections = shrink
				debug.FreeOSMemory()
			}

			break
		}
	}
	// DEBUG::
	fmt.Println("Num GoRoutines", runtime.NumGoroutine())
	fmt.Println("Len(connections)", len(chat.connections))
	fmt.Println("Cap(connections)", cap(chat.connections))
	GoRuntimeStats()
}
Ejemplo n.º 14
0
func TestDispositionIsAttachment(t *testing.T) {
	loop := 1
	for i := 0; i < loop; i++ {
		// Case 1: not an attachment
		contentString := ""
		cd := NewContentDispositionFromString(contentString)
		assert.False(t, cd.IsAttachment())

		// Case 2: is an attachment
		contentString = "attachment"
		cd = NewContentDispositionFromString(contentString)
		assert.True(t, cd.IsAttachment())

		// Case 3: is an inline attachment
		contentString = "inline"
		cd = NewContentDispositionFromString(contentString)
		assert.True(t, cd.IsAttachment())

		// Case 4: anything is an attachment for now
		contentString = "bogus"
		cd = NewContentDispositionFromString(contentString)
		assert.True(t, cd.IsAttachment())
	}
	debug.FreeOSMemory()
}
Ejemplo n.º 15
0
//10亿设备量的过滤器,占用1.8G*1 = 1.8G 内存   存在碰撞的数量: 236*1000 (cityhash) [大致值]
//10亿设备量的过滤器,占用1.8G*2 = 3.6G 内存   存在碰撞的数量: 236 (cityhash+md5)
//10亿设备量的过滤器,占用1.8G*3 = 5.4G 内存   存在碰撞的数量: 0 (cityhash+md5+fnv.New64())
func generateGarbage() {
	var capSize uint = 1000000000

	//
	//	fvv := []string{}
	//	fvv = append(fvv,"hello")

	filter := NewBloomWrap(capSize)
	v := []byte("Love")
	b := filter.Add(v).Check(v)
	log.Println("check @v:", b)
	bad := 0
	for i := 0; i < 1000000000; i++ {
		if i%1000000 == 0 {
			log.Println(i)
			debug.FreeOSMemory()
		}
		d := []byte(fmt.Sprint("data", i))
		if filter.Check(d) == true {
			bad++
			//			panic(fmt.Sprint("should not exist @d:",string(d)))
		}
		if flag := filter.Add(d).Check(d); flag == false {
			panic(d)
		}
	}
	log.Println("====>>That is all @bad:", bad)
}
Ejemplo n.º 16
0
// RebalanceGC rebalances the tree and runs the garbage collector.
func (st *SimpleTree) RebalanceGC() *SimpleTree {
	st.Rebalance()
	runtime.GC()
	debug.FreeOSMemory()

	return st
}
Ejemplo n.º 17
0
func bash(bash, content string) (out string, err error) {
	var buf bytes.Buffer

	cmd := exec.Command("/bin/sh", "-c", bash)
	cmd.Stdin = strings.NewReader(content)
	cmd.Stderr = &buf
	cmd.Stdout = &buf

	err = cmd.Run()
	if err != nil {
		printStackAndError(err)
		cmd.Process.Release()
		buf.Reset()
		return
	}

	out = buf.String()

	// Clean up resource
	cmd.Process.Kill()
	buf.Reset()
	debug.FreeOSMemory()

	return
}
Ejemplo n.º 18
0
// DeleteGC sets all the pointers of the tree to nil and runs the garbage
// collector.
func (st *SimpleTree) DeleteGC() *SimpleTree {
	st.Delete()
	runtime.GC()
	debug.FreeOSMemory()

	return nil
}
Ejemplo n.º 19
0
func main() {
	var m runtime.MemStats

	/*If that is not working, or it is too much time, you can add a periodic call to
	FreeOSMemory (no need to call runtime.GC() before, it is done by debug.FreeOSMemory() )

		Something like this: http://play.golang.org/p/mP7_sMpX4F

		package main

	import (
	    "runtime/debug"
	    "time"
	)

	func main() {
	    go periodicFree(1 * time.Minute)

	    // Your program goes here

	}

	func periodicFree(d time.Duration) {
	    tick := time.Tick(d)
	    for _ = range tick {
	        debug.FreeOSMemory()
	    }
	}
	Take into account that every call to FreeOSMemory will take some time (not much)
	and it can be partly run in parallel if GOMAXPROCS>1 since Go1.3.*/

	debug.FreeOSMemory()

	/*Then you can either render it to a dot file with graphical
	representation of the heap or convert it to hprof format. To render it to a dot file:

	$ go get github.com/randall77/hprof/dumptodot

	$ dumptodot heapdump mybinary > heap.dot

	and open heap.dot with Graphviz.*/

	f, err := os.Create("heapdump")
	if err != nil {
		panic(err)
	}

	debug.WriteHeapDump(f.Fd())

	fmt.Println(runtime.GOOS)
	fmt.Println(runtime.NumCPU())
	fmt.Println(runtime.NumGoroutine())
	fmt.Println(runtime.GOARCH)
	runtime.ReadMemStats(&m)

	fmt.Println(m.TotalAlloc)
	fmt.Println(m.Alloc)
	fmt.Println(m.Sys)
}
Ejemplo n.º 20
0
func StartGC() {
	for {
		time.Sleep(10 * time.Second)
		runtime.GC()
		debug.FreeOSMemory()
		log.Println("Current Routines", runtime.NumGoroutine())
	}
}
Ejemplo n.º 21
0
func startGcLoop(period time.Duration) {
	go func(period time.Duration) {
		for {
			debug.FreeOSMemory()
			time.Sleep(period * time.Second)
		}
	}(period)
}
Ejemplo n.º 22
0
func (memory *memoryDriver) expiredPart(a ...interface{}) {
	key := a[0].(string)
	// loop through all buckets
	for _, storedBucket := range memory.storedBuckets {
		delete(storedBucket.partMetadata, key)
	}
	debug.FreeOSMemory()
}
Ejemplo n.º 23
0
func hello(w http.ResponseWriter, r *http.Request) {
	a := bigBytes()
	fmt.Printf("memory type:%T address: %p  size: %d\n", a, &a, unsafe.Sizeof(a))
	b := bigBytes()
	fmt.Printf("memory type:%T address: %p  size: %d\n", b, &b, unsafe.Sizeof(b))
	time.Sleep(3000 * time.Millisecond)
	io.WriteString(w, "Hello world!")
	debug.FreeOSMemory()
}
Ejemplo n.º 24
0
Archivo: main.go Proyecto: devhq-io/ax
func startGcLoop(period time.Duration) {
	go func(period time.Duration) {
		for {
			log.Printf("gc\n")
			debug.FreeOSMemory()
			time.Sleep(period * time.Second)
		}
	}(period)
}
Ejemplo n.º 25
0
func TestNewContentDispositionFromString(t *testing.T) {
	loop := 1
	for i := 0; i < loop; i++ {
		contentString := "hola!"
		cd := NewContentDispositionFromString(contentString)
		assert.Equal(t, cd.Disposition(), contentString)
	}
	debug.FreeOSMemory()
}
Ejemplo n.º 26
0
func main() {
	DoSomeThing()
	for {
		println("idle")
		//runtime.GC()
		debug.FreeOSMemory()
		time.Sleep(2 * time.Second)
	}
}
Ejemplo n.º 27
0
// Periodic release of unused memory back to the OS.
// Long running network daemons, especially that see numerous TCP connections
// open and close end up consuming a lot of memory which doesn't seem to be
// freed in an efficient manner.
func periodMemoryRelease(interval int) {
	ticker := time.NewTicker(time.Duration(interval) * time.Second)
	go func() {
		for _ = range ticker.C {
			Logger.Println("FreeOSMemory()")
			debug.FreeOSMemory()
		}
	}()
}
Ejemplo n.º 28
0
// evictedPart - call back function called by caching module during individual cache evictions
func (donut API) evictedPart(a ...interface{}) {
	// loop through all buckets
	buckets := donut.storedBuckets.GetAll()
	for bucketName, bucket := range buckets {
		b := bucket.(storedBucket)
		donut.storedBuckets.Set(bucketName, b)
	}
	debug.FreeOSMemory()
}
Ejemplo n.º 29
0
// CreateObjectPart - create a part in a multipart session
func (donut API) CreateObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader, signature *Signature) (string, *probe.Error) {
	donut.lock.Lock()
	etag, err := donut.createObjectPart(bucket, key, uploadID, partID, "", expectedMD5Sum, size, data, signature)
	donut.lock.Unlock()
	// possible free
	debug.FreeOSMemory()

	return etag, err.Trace()
}
Ejemplo n.º 30
0
func memoryRelease(interval int) {
	ticker := time.NewTicker(time.Duration(interval) * time.Second)
	go func() {
		for _ = range ticker.C {
			debug("FreeOSMemory()")
			d.FreeOSMemory()
		}
	}()
}