Exemplo n.º 1
0
// Write write needles, if key exists append to super block, then update
// needle cache offset to new offset.
func (v *Volume) Write(ns *needle.Needles) (err error) {
	var (
		i               int
		ok              bool
		nc              int64
		ncs             []int64
		offset, ooffset uint32
		n               *needle.Needle
		now             = time.Now().UnixNano()
	)
	v.lock.Lock()
	offset = v.Block.Offset
	if err = v.Block.Write(ns.Buffer()); err == nil {
		for i = 0; i < ns.Num; i++ {
			n = ns.Needle(i)
			if err = v.Indexer.Add(n.Key, offset, n.TotalSize); err != nil {
				break
			}
			if nc, ok = v.needles[n.Key]; ok {
				ncs = append(ncs, nc)
			}
			v.needles[n.Key] = needle.NewCache(offset, n.TotalSize)
			v.addCheck(n.Key, n.Cookie)
			offset += n.IncrOffset
			if log.V(1) {
				log.Infof("add needle, offset: %d, size: %d", offset, n.TotalSize)
				log.Info(n)
			}
		}
	}
	v.lock.Unlock()
	if err != nil {
		return
	}
	for _, nc = range ncs {
		ooffset, _ = needle.Cache(nc)
		err = v.asyncDel(ooffset)
		log.Warningf("same key: %d, old offset: %d, new offset: %d", n.Key, ooffset, offset)
	}
	atomic.AddUint64(&v.Stats.TotalWriteProcessed, 1)
	atomic.AddUint64(&v.Stats.TotalWriteBytes, uint64(n.TotalSize))
	atomic.AddUint64(&v.Stats.TotalWriteDelay, uint64(time.Now().UnixNano()-now))
	return
}
Exemplo n.º 2
0
Arquivo: store.go Projeto: 1d7500/bfs
// FreeNeedles free the needles to pool.
func (s *Store) FreeNeedles(i int, n *needle.Needles) {
	n.TotalSize = 0
	s.nsp[i].Put(n)
}
Exemplo n.º 3
0
func (h httpUploadsHandler) ServeHTTP(wr http.ResponseWriter, r *http.Request) {
	var (
		i, nn   int
		ok      bool
		err     error
		vid     int64
		key     int64
		cookie  int64
		size    int64
		str     string
		keys    []string
		cookies []string
		sr      sizer
		fr      *os.File
		fi      os.FileInfo
		v       *Volume
		n       *needle.Needle
		ns      *needle.Needles
		uerr    errors.Error
		file    multipart.File
		fh      *multipart.FileHeader
		fhs     []*multipart.FileHeader
		res     = map[string]interface{}{"ret": errors.RetOK}
	)
	if r.Method != "POST" {
		http.Error(wr, "method not allowed", http.StatusMethodNotAllowed)
		return
	}
	defer HttpPostWriter(r, wr, time.Now(), res)
	// check total content-length
	if size, err = strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64); err != nil {
		res["ret"] = errors.RetInternalErr
		return
	}
	if size > int64(h.c.NeedleMaxSize*h.c.BatchMaxNum) {
		res["ret"] = errors.RetNeedleTooLarge
		return
	}
	str = r.FormValue("vid")
	if vid, err = strconv.ParseInt(str, 10, 32); err != nil {
		log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
		res["ret"] = errors.RetParamErr
		return
	}
	keys = r.MultipartForm.Value["keys"]
	cookies = r.MultipartForm.Value["cookies"]
	if len(keys) != len(cookies) {
		log.Errorf("param length not match, keys: %d, cookies: %d", len(keys), len(cookies))
		res["ret"] = errors.RetParamErr
		return
	}
	fhs = r.MultipartForm.File["file"]
	nn = len(fhs)
	if len(keys) != nn {
		log.Errorf("param length not match, keys: %d, cookies: %d, files: %d", len(keys), len(cookies), len(fhs))
		res["ret"] = errors.RetParamErr
		return
	}
	ns = h.s.Needles(nn)
	for i, fh = range fhs {
		if key, err = strconv.ParseInt(keys[i], 10, 64); err != nil {
			log.Errorf("strconv.ParseInt(\"%s\") error(%v)", keys[i], err)
			err = errors.ErrParam
			break
		}
		if cookie, err = strconv.ParseInt(cookies[i], 10, 32); err != nil {
			log.Errorf("strconv.ParseInt(\"%s\") error(%v)", cookies[i], err)
			err = errors.ErrParam
			break
		}
		if file, err = fh.Open(); err != nil {
			log.Errorf("fh.Open() error(%v)", err)
			break
		}
		// check size
		if sr, ok = file.(sizer); ok {
			size = sr.Size()
		} else if fr, ok = file.(*os.File); ok {
			if fi, err = fr.Stat(); err != nil {
				goto failed
			}
			size = fi.Size()
		}
		if size > int64(h.c.NeedleMaxSize) {
			err = errors.ErrNeedleTooLarge
			goto failed
		}
		n = &ns.Items[i]
		n.InitSize(key, int32(cookie), int32(size))
		if err = ns.WriteFrom(n, file); err != nil {
			goto failed
		}
		continue
	failed:
		file.Close()
		break
	}
	if err == nil {
		if v = h.s.Volumes[int32(vid)]; v != nil {
			err = v.Write(ns)
		} else {
			err = errors.ErrVolumeNotExist
		}
	}
	h.s.FreeNeedles(nn, ns)
	if err != nil {
		if uerr, ok = err.(errors.Error); ok {
			res["ret"] = int(uerr)
		} else {
			res["ret"] = errors.RetInternalErr
		}
	}
	return
}
Exemplo n.º 4
0
// FreeNeedles free the needles to pool.
func (s *Store) FreeNeedles(i int, ns *needle.Needles) {
	ns.Reset()
	s.nsp[i].Put(ns)
}
Exemplo n.º 5
0
func (h httpUploadsHandler) ServeHTTP(wr http.ResponseWriter, r *http.Request) {
	var (
		i, nn   int
		err     error
		vid     int64
		key     int64
		cookie  int64
		size    int64
		str     string
		keys    []string
		cookies []string
		v       *volume.Volume
		ns      *needle.Needles
		file    multipart.File
		fh      *multipart.FileHeader
		fhs     []*multipart.FileHeader
		res     = map[string]interface{}{}
	)
	if r.Method != "POST" {
		http.Error(wr, "method not allowed", http.StatusMethodNotAllowed)
		return
	}
	defer HttpPostWriter(r, wr, time.Now(), &err, res)
	// check total content-length
	if size, err = strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64); err != nil {
		err = errors.ErrInternal
		return
	}
	if size > int64(h.c.NeedleMaxSize*h.c.BatchMaxNum) {
		err = errors.ErrInternal
		return
	}
	str = r.FormValue("vid")
	if vid, err = strconv.ParseInt(str, 10, 32); err != nil {
		log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
		err = errors.ErrParam
		return
	}
	keys = r.MultipartForm.Value["keys"]
	cookies = r.MultipartForm.Value["cookies"]
	if len(keys) != len(cookies) {
		log.Errorf("param length not match, keys: %d, cookies: %d", len(keys), len(cookies))
		err = errors.ErrParam
		return
	}
	fhs = r.MultipartForm.File["file"]
	nn = len(fhs)
	if len(keys) != nn {
		log.Errorf("param length not match, keys: %d, cookies: %d, files: %d", len(keys), len(cookies), len(fhs))
		err = errors.ErrParam
		return
	}
	ns = h.s.Needles(nn)
	for i, fh = range fhs {
		if key, err = strconv.ParseInt(keys[i], 10, 64); err != nil {
			log.Errorf("strconv.ParseInt(\"%s\") error(%v)", keys[i], err)
			err = errors.ErrParam
			break
		}
		if cookie, err = strconv.ParseInt(cookies[i], 10, 32); err != nil {
			log.Errorf("strconv.ParseInt(\"%s\") error(%v)", cookies[i], err)
			err = errors.ErrParam
			break
		}
		if file, err = fh.Open(); err != nil {
			log.Errorf("fh.Open() error(%v)", err)
			break
		}
		defer file.Close()
		// check size
		if size, err = fileSize(file); err != nil {
			break
		}
		if size > int64(h.c.NeedleMaxSize) {
			err = errors.ErrNeedleTooLarge
			break
		}
		if err = ns.WriteFrom(key, int32(cookie), int32(size), file); err != nil {
			break
		}
	}
	if err == nil {
		if v = h.s.Volumes[int32(vid)]; v != nil {
			err = v.Write(ns)
		} else {
			err = errors.ErrVolumeNotExist
		}
	}
	h.s.FreeNeedles(nn, ns)
	return
}