Пример #1
0
func (f *FileWriter) startNewBlock() error {
	// TODO: we don't need to wait for previous blocks to ack before continuing

	if f.blockWriter != nil {
		err := f.blockWriter.Close()
		if err != nil {
			return err
		}
	}
	var previous *hdfs.ExtendedBlockProto
	if f.block != nil {
		previous = f.block.GetB()
	}

	addBlockReq := &hdfs.AddBlockRequestProto{
		Src:        proto.String(f.name),
		ClientName: proto.String(f.client.namenode.ClientName()),
		Previous:   previous,
	}
	addBlockResp := &hdfs.AddBlockResponseProto{}

	err := f.client.namenode.Execute("addBlock", addBlockReq, addBlockResp)
	if err != nil {
		if nnErr, ok := err.(*rpc.NamenodeError); ok {
			err = interpretException(nnErr.Exception, err)
		}

		return &os.PathError{"create", f.name, err}
	}

	f.block = addBlockResp.GetBlock()
	f.blockWriter = rpc.NewBlockWriter(f.block, f.client.namenode, f.blockSize)
	return nil
}
Пример #2
0
// Append opens an existing file in HDFS and returns an io.WriteCloser for
// writing to it. Because of the way that HDFS writes are buffered and
// acknowledged asynchronously, it is very important that Close is called after
// all data has been written.
func (c *Client) Append(name string) (*FileWriter, error) {
	info, err := c.getFileInfo(name)
	if err != nil {
		return nil, &os.PathError{"append", name, err}
	}

	appendReq := &hdfs.AppendRequestProto{
		Src:        proto.String(name),
		ClientName: proto.String(c.namenode.ClientName()),
	}
	appendResp := &hdfs.AppendResponseProto{}

	err = c.namenode.Execute("append", appendReq, appendResp)
	if err != nil {
		if nnErr, ok := err.(*rpc.NamenodeError); ok {
			err = interpretException(nnErr.Exception, err)
		}

		return nil, &os.PathError{"append", name, err}
	}

	req := &hdfs.GetBlockLocationsRequestProto{
		Src:    proto.String(name),
		Offset: proto.Uint64(0),
		Length: proto.Uint64(uint64(info.Size())),
	}
	resp := &hdfs.GetBlockLocationsResponseProto{}

	err = c.namenode.Execute("getBlockLocations", req, resp)
	if err != nil {
		return nil, err
	}

	blocks := resp.GetLocations().GetBlocks()
	f := &FileWriter{
		client:      c,
		name:        name,
		replication: int(appendResp.Stat.GetBlockReplication()),
		blockSize:   int64(appendResp.Stat.GetBlocksize()),
	}
	if len(blocks) == 0 {
		return f, nil
	}
	f.block = blocks[len(blocks)-1]
	f.blockWriter = rpc.NewBlockWriter(f.block, c.namenode, f.blockSize)
	return f, nil
}