Example #1
0
func (l *Layer0) Remove(id string) error {
	if !l.isLayer0(id) {
		return l.Driver.Remove(l.realID(id))
	}
	l.Lock()
	defer l.Unlock()
	var err error
	v, ok := l.volumes[id]

	if ok {
		atomic.AddInt32(&v.ref, -1)
		if v.ref == 0 {
			// Save the upper dir and blow away the rest.
			upperDir := path.Join(path.Join(l.home, l.realID(id)), "upper")
			err := os.Rename(upperDir, path.Join(v.path, "upper"))
			if err != nil {
				dlog.Warnf("Failed in rename(%v): %v", id, err)
			}
			l.Driver.Remove(l.realID(id))
			err = l.volDriver.Unmount(v.volumeID, v.path)
			if l.volDriver.Type() == api.DriverType_DRIVER_TYPE_BLOCK {
				_ = l.volDriver.Detach(v.volumeID)
			}
			err = os.RemoveAll(v.path)
			delete(l.volumes, v.id)
		}
	} else {
		dlog.Warnf("Failed to find layer0 vol for id %v", id)
	}
	return err
}
Example #2
0
func (d *driver) Create(
	locator *api.VolumeLocator,
	source *api.Source,
	spec *api.VolumeSpec) (string, error) {

	volumeID := uuid.New()
	volumeID = strings.TrimSuffix(volumeID, "\n")

	// Create a directory on the NFS server with this UUID.
	volPath := path.Join(nfsMountPath, volumeID)
	err := os.MkdirAll(volPath, 0744)
	if err != nil {
		dlog.Println(err)
		return "", err
	}
	if source != nil {
		if len(source.Seed) != 0 {
			seed, err := seed.New(source.Seed, spec.VolumeLabels)
			if err != nil {
				dlog.Warnf("Failed to initailize seed from %q : %v",
					source.Seed, err)
				return "", err
			}
			err = seed.Load(path.Join(volPath, config.DataDir))
			if err != nil {
				dlog.Warnf("Failed to  seed from %q to %q: %v",
					source.Seed, nfsMountPath, err)
				return "", err
			}
		}
	}

	f, err := os.Create(path.Join(nfsMountPath, volumeID+nfsBlockFile))
	if err != nil {
		dlog.Println(err)
		return "", err
	}
	defer f.Close()

	if err := f.Truncate(int64(spec.Size)); err != nil {
		dlog.Println(err)
		return "", err
	}

	v := common.NewVolume(
		volumeID,
		api.FSType_FS_TYPE_NFS,
		locator,
		source,
		spec,
	)
	v.DevicePath = path.Join(nfsMountPath, volumeID+nfsBlockFile)

	if err := d.CreateVol(v); err != nil {
		return "", err
	}
	return v.Id, err
}
Example #3
0
// Initialize node and alert listeners that we are joining the cluster.
func (c *ClusterManager) joinCluster(db *Database, self *api.Node, exist bool) error {
	var err error

	// If I am already in the cluster map, don't add me again.
	if exist {
		goto found
	}

	// Alert all listeners that we are a new node joining an existing cluster.
	for e := c.listeners.Front(); e != nil; e = e.Next() {
		err = e.Value.(ClusterListener).Init(self, db)
		if err != nil {
			self.Status = api.Status_STATUS_ERROR
			dlog.Warnf("Failed to initialize Init %s: %v",
				e.Value.(ClusterListener).String(), err)
			c.cleanupInit(db, self)
			goto done
		}
	}

found:
	// Alert all listeners that we are joining the cluster.
	for e := c.listeners.Front(); e != nil; e = e.Next() {
		err = e.Value.(ClusterListener).Join(self, db)
		if err != nil {
			self.Status = api.Status_STATUS_ERROR
			dlog.Warnf("Failed to initialize Join %s: %v",
				e.Value.(ClusterListener).String(), err)

			if exist == false {
				c.cleanupInit(db, self)
			}
			goto done
		}
	}

	for id, n := range db.NodeEntries {
		if id != c.config.NodeId {
			// Check to see if the IP is the same.  If it is, then we have a stale entry.
			if n.MgmtIp == self.MgmtIp {
				dlog.Warnf("Warning, Detected node %s with the same IP %s in the database.  Will not connect to this node.",
					id, n.MgmtIp)
			} else {
				// Gossip with this node.
				dlog.Infof("Connecting to node %s with IP %s.", id, n.MgmtIp)
				c.gossip.AddNode(n.MgmtIp+":9002", types.NodeId(id))
			}
		}
	}

done:
	return err
}
Example #4
0
// Unmount device at mountpoint or decrement refcnt. If device has no
// mountpoints left after this operation, it is removed from the matrix.
// ErrEnoent is returned if the device or mountpoint for the device is not found.
func (m *Mounter) Unmount(device, path string) error {
	m.Lock()

	info, ok := m.mounts[device]
	if !ok {
		m.Unlock()
		return ErrEnoent
	}
	m.Unlock()
	info.Lock()
	defer info.Unlock()
	for i, p := range info.Mountpoint {
		if p.Path == path {
			p.ref--
			// Unmount only if refcnt is 0
			if p.ref == 0 {
				err := syscall.Unmount(path, 0)
				if err != nil {
					return err
				}
				if _, pathExists := m.paths[path]; pathExists {
					delete(m.paths, path)
				} else {
					dlog.Warnf("Path %q for device %q does not exist in pathMap", path, device)
				}
				// Blow away this mountpoint.
				info.Mountpoint[i] = info.Mountpoint[len(info.Mountpoint)-1]
				info.Mountpoint = info.Mountpoint[0 : len(info.Mountpoint)-1]
				m.maybeRemoveDevice(device)
			}
			return nil
		}
	}
	return ErrEnoent
}
Example #5
0
func writeDatabase(db *Database) error {
	kvdb := kvdb.Instance()
	b, err := json.Marshal(db)
	if err != nil {
		dlog.Warnf("Fatal, Could not marshal cluster database to JSON: %v", err)
		return err
	}

	if _, err := kvdb.Put(ClusterDBKey, b, 0); err != nil {
		dlog.Warnf("Fatal, Could not marshal cluster database to JSON: %v", err)
		return err
	}

	dlog.Infoln("Cluster database updated.")
	return nil
}
Example #6
0
// Mount new mountpoint for specified device.
func (m *Mounter) Mount(minor int, device, path, fs string, flags uintptr, data string) error {
	m.Lock()

	dev, ok := m.paths[path]
	if ok && dev != device {
		dlog.Warnf("cannot mount %q,  device %q is mounted at %q", device, dev, path)
		m.Unlock()
		return ErrExist
	}
	info, ok := m.mounts[device]
	if !ok {
		info = &Info{
			Device:     device,
			Mountpoint: make([]*PathInfo, 0),
			Minor:      minor,
			Fs:         fs,
		}
	}
	m.mounts[device] = info
	m.Unlock()
	info.Lock()
	defer info.Unlock()

	// Validate input params
	if fs != info.Fs {
		dlog.Warnf("%s Existing mountpoint has fs %q cannot change to %q",
			device, info.Fs, fs)
		return ErrEinval
	}

	// Try to find the mountpoint. If it already exists, then increment refcnt
	for _, p := range info.Mountpoint {
		if p.Path == path {
			p.ref++
			return nil
		}
	}
	// The device is not mounted at path, mount it and add to its mountpoints.
	err := syscall.Mount(device, path, fs, flags, data)
	if err != nil {
		return err
	}
	info.Mountpoint = append(info.Mountpoint, &PathInfo{Path: path, ref: 1})
	m.paths[path] = device
	return nil
}
Example #7
0
// Shutdown can be called when THIS node is gracefully shutting down.
func (c *ClusterManager) Shutdown() error {
	db, err := readDatabase()
	if err != nil {
		dlog.Warnf("Could not read cluster database (%v).", err)
		return err
	}

	// Alert all listeners that we are shutting this node down.
	for e := c.listeners.Front(); e != nil; e = e.Next() {
		dlog.Infof("Shutting down %s", e.Value.(ClusterListener).String())
		if err := e.Value.(ClusterListener).Halt(&c.selfNode, &db); err != nil {
			dlog.Warnf("Failed to shutdown %s",
				e.Value.(ClusterListener).String())
		}
	}
	return nil
}
Example #8
0
// ApplyDiff extracts the changeset from the given diff into the
// layer with the specified id and parent, returning the size of the
// new layer in bytes.
// The archive.Reader must be an uncompressed stream.
func (d *Driver) ApplyDiff(id string, parent string, diff archive.Reader) (size int64, err error) {
	dir := path.Join(virtPath, id)
	// dir := path.Join("/tmp/chainfs/", id)

	dlog.Infof("Applying diff at path %s\n", dir)

	if err := chrootarchive.UntarUncompressed(diff, dir, nil); err != nil {
		dlog.Warnf("Error while applying diff to %s: %v", id, err)
		return 0, err
	}

	// show invalid whiteouts warning.
	files, err := ioutil.ReadDir(path.Join(dir, archive.WhiteoutLinkDir))
	if err == nil && len(files) > 0 {
		dlog.Warnf("Archive contains aufs hardlink references that are not supported.")
	}

	return d.DiffSize(id, parent)
}
Example #9
0
// Remove attempts to remove the filesystem layer with this id.
func (d *Driver) Remove(id string) error {
	dlog.Infof("Removing layer %s", id)

	cID := C.CString(id)
	ret, err := C.remove_layer(cID)
	if int(ret) != 0 {
		dlog.Warnf("Error while removing layer %s", id)
		return err
	}

	return nil
}
Example #10
0
func (c *ClusterManager) cleanupInit(db *Database, self *api.Node) error {
	var resErr error
	var err error

	dlog.Infof("Cleanup Init services")

	for e := c.listeners.Front(); e != nil; e = e.Next() {
		dlog.Warnf("Cleanup Init for service %s.",
			e.Value.(ClusterListener).String())

		err = e.Value.(ClusterListener).CleanupInit(self, db)
		if err != nil {
			dlog.Warnf("Failed to Cleanup Init %s: %v",
				e.Value.(ClusterListener).String(), err)
			resErr = err
		}

	}

	return resErr
}
Example #11
0
// Get returns the mountpoint for the layered filesystem referred
// to by this id. You can optionally specify a mountLabel or "".
// Returns the absolute path to the mounted layered filesystem.
func (d *Driver) Get(id, mountLabel string) (string, error) {
	cID := C.CString(id)

	ret, err := C.alloc_chainfs(cID)
	if int(ret) != 0 {
		dlog.Warnf("Error while creating a chain FS for %s", id)
		return "", err
	} else {
		dlog.Debugf("Created a chain FS for %s", id)
		chainPath := path.Join(virtPath, id)
		return chainPath, err
	}
}
Example #12
0
func (d *Driver) volumeState(ec2VolState *string) api.VolumeState {
	if ec2VolState == nil {
		return api.VolumeState_VOLUME_STATE_DETACHED
	}
	switch *ec2VolState {
	case ec2.VolumeAttachmentStateAttached:
		return api.VolumeState_VOLUME_STATE_ATTACHED
	case ec2.VolumeAttachmentStateDetached:
		return api.VolumeState_VOLUME_STATE_DETACHED
	case ec2.VolumeAttachmentStateAttaching, ec2.VolumeAttachmentStateDetaching:
		return api.VolumeState_VOLUME_STATE_PENDING
	default:
		dlog.Warnf("Failed to translate EC2 volume status %v", ec2VolState)
	}
	return api.VolumeState_VOLUME_STATE_ERROR
}
Example #13
0
// Create creates a new, empty, filesystem layer with the
// specified id and parent and mountLabel. Parent and mountLabel may be "".
func (d *Driver) Create(id string, parent string, ml string) error {
	if parent != "" {
		dlog.Infof("Creating layer %s with parent %s", id, parent)
	} else {
		dlog.Infof("Creating parent layer %s", id)
	}

	cID := C.CString(id)
	cParent := C.CString(parent)

	ret, err := C.create_layer(cID, cParent)
	if int(ret) != 0 {
		dlog.Warnf("Error while creating layer %s", id)
		return err
	}

	return nil
}
Example #14
0
// Create aws volume from spec.
func (d *Driver) Create(
	locator *api.VolumeLocator,
	source *api.Source,
	spec *api.VolumeSpec,
) (string, error) {
	var snapID *string
	// Spec size is in bytes, translate to GiB.
	sz := int64(spec.Size / (1024 * 1024 * 1024))
	iops, volType := mapCos(spec.Cos)
	if source != nil && string(source.Parent) != "" {
		id := string(source.Parent)
		snapID = &id
	}
	dryRun := false
	encrypted := false
	req := &ec2.CreateVolumeInput{
		AvailabilityZone: &d.md.zone,
		DryRun:           &dryRun,
		Encrypted:        &encrypted,
		Size:             &sz,
		Iops:             iops,
		VolumeType:       volType,
		SnapshotId:       snapID,
	}
	vol, err := d.ec2.CreateVolume(req)
	if err != nil {
		dlog.Warnf("Failed in CreateVolumeRequest :%v", err)
		return "", err
	}
	volume := common.NewVolume(
		*vol.VolumeId,
		api.FSType_FS_TYPE_NONE,
		locator,
		source,
		spec,
	)
	err = d.UpdateVol(volume)
	if err != nil {
		return "", err
	}
	err = d.waitStatus(volume.Id, ec2.VolumeStateAvailable)
	return volume.Id, err
}
Example #15
0
func (d *Driver) Format(volumeID string) error {
	v, err := d.GetVol(volumeID)
	if err != nil {
		return fmt.Errorf("Failed to locate volume %q", volumeID)
	}

	// XXX: determine mount state
	devicePath, err := d.devicePath(volumeID)
	if err != nil {
		return err
	}
	cmd := "/sbin/mkfs." + string(v.Spec.Format)
	o, err := exec.Command(cmd, devicePath).Output()
	if err != nil {
		dlog.Warnf("Failed to run command %v %v: %v", cmd, devicePath, o)
		return err
	}
	v.Format = v.Spec.Format
	err = d.UpdateVol(v)
	return err
}
Example #16
0
func Init(params map[string]string) (volume.VolumeDriver, error) {
	host, ok := params["url"]
	if !ok {
		return nil, ErrApiUrlRequired
	}

	token, ok := params["token"]
	if !ok {
		return nil, ErrApiAuthTokenRequired
	}

	// create a coprhd api client instance
	client := coprhd.NewClient(host, token)

	d := &driver{
		DefaultEnumerator: volume.NewDefaultEnumerator(Name, kvdb.Instance()),
		client:            client,
	}

	if projectName, ok := params["project"]; ok {
		if project, err := client.Project().Name(projectName).Query(); err != nil {
			return nil, err
		} else {
			d.project = project
		}
	} else {
		dlog.Warnln("Default coprhd 'project' not set")
	}

	if varrayName, ok := params["varray"]; ok {
		if varray, err := client.VArray().Name(varrayName).Query(); err != nil {
			return nil, err
		} else {
			d.varray = varray
		}
	} else {
		dlog.Warnf("Default coprhd 'varray' not set")
	}

	if vpoolName, ok := params["vpool"]; ok {
		if vpool, err := client.VPool().Name(vpoolName).Query(); err != nil {
			return nil, err
		} else {
			d.vpool = vpool
		}
	} else {
		dlog.Warnf("Default coprhd 'vpool' not set")
	}

	if port, ok := params["port"]; ok {
		if initiator, err := client.Initiator().Port(port).Query(); err != nil {
			return nil, err
		} else {
			d.initiator = initiator
		}
	} else {
		return nil, ErrPortRequired
	}

	return d, nil
}
Example #17
0
func start(c *cli.Context) {

	if !osdcli.DaemonMode(c) {
		cli.ShowAppHelp(c)
		return
	}

	datastores := []string{mem.Name, etcd.Name, consul.Name}

	// We are in daemon mode.
	file := c.String("file")
	if file == "" {
		dlog.Warnln("OSD configuration file not specified.  Visit openstorage.org for an example.")
		return
	}

	cfg, err := config.Parse(file)
	if err != nil {
		dlog.Errorln(err)
		return
	}
	kvdbURL := c.String("kvdb")
	u, err := url.Parse(kvdbURL)
	scheme := u.Scheme
	u.Scheme = "http"

	kv, err := kvdb.New(scheme, "openstorage", []string{u.String()}, nil)
	if err != nil {
		dlog.Warnf("Failed to initialize KVDB: %v (%v)", scheme, err)
		dlog.Warnf("Supported datastores: %v", datastores)
		return
	}
	err = kvdb.SetInstance(kv)
	if err != nil {
		dlog.Warnf("Failed to initialize KVDB: %v", err)
		return
	}

	// Start the cluster state machine, if enabled.
	clusterInit := false
	if cfg.Osd.ClusterConfig.NodeId != "" && cfg.Osd.ClusterConfig.ClusterId != "" {
		dlog.Infof("OSD enabling cluster mode.")

		if err := cluster.Init(cfg.Osd.ClusterConfig); err != nil {
			dlog.Errorln("Unable to init cluster server: %v", err)
			return
		}
		clusterInit = true

		if err := server.StartClusterAPI(config.ClusterAPIBase); err != nil {
			dlog.Warnf("Unable to start cluster API server: %v", err)
			return
		}
	}

	// Start the volume drivers.
	for d, v := range cfg.Osd.Drivers {
		dlog.Infof("Starting volume driver: %v", d)
		if _, err := volume.New(d, v); err != nil {
			dlog.Warnf("Unable to start volume driver: %v, %v", d, err)
			return
		}

		if err := server.StartPluginAPI(d, config.DriverAPIBase, config.PluginAPIBase); err != nil {
			dlog.Warnf("Unable to start volume plugin: %v", err)
			return
		}
	}

	if err := server.StartFlexVolumeAPI(config.FlexVolumePort); err != nil {
		dlog.Warnf("Unable to start flexvolume API: %v", err)
		return
	}

	// Start the graph drivers.
	for d, _ := range cfg.Osd.GraphDrivers {
		dlog.Infof("Starting graph driver: %v", d)
		if err := server.StartGraphAPI(d, config.PluginAPIBase); err != nil {
			dlog.Warnf("Unable to start graph plugin: %v", err)
			return
		}
	}

	if clusterInit {
		cm, err := cluster.Inst()
		if err != nil {
			dlog.Warnf("Unable to find cluster instance: %v", err)
			return
		}
		if err := cm.Start(); err != nil {
			dlog.Warnf("Unable to start cluster manager: %v", err)
			return
		}
	}

	// Daemon does not exit.
	select {}
}
Example #18
0
func (l *Layer0) create(id, parent string) (string, *Layer0Vol, error) {
	l.Lock()
	defer l.Unlock()

	// If this is the parent of the Layer0, add an entry for it.
	baseID, l0 := l.isLayer0Parent(id)
	if l0 {
		l.volumes[baseID] = &Layer0Vol{id: baseID, parent: parent}
		return id, nil, nil
	}

	// Don't do anything if this is not layer 0
	if !l.isLayer0(id) {
		return id, nil, nil
	}

	vol, ok := l.volumes[id]
	if !ok {
		dlog.Warnf("Failed to find layer0 volume for id %v", id)
		return id, nil, nil
	}

	// Query volume for Layer 0
	vols, err := l.volDriver.Enumerate(&api.VolumeLocator{Name: vol.parent}, nil)

	// If we don't find a volume configured for this image,
	// then don't track layer0
	if err != nil || vols == nil {
		dlog.Infof("Failed to find configured volume for id %v", vol.parent)
		delete(l.volumes, id)
		return id, nil, nil
	}

	// Find a volume that is available.
	index := -1
	for i, v := range vols {
		if len(v.AttachPath) == 0 {
			index = i
			break
		}
	}
	if index == -1 {
		dlog.Infof("Failed to find free volume for id %v", vol.parent)
		delete(l.volumes, id)
		return id, nil, nil
	}

	mountPath := path.Join(l.home, l.loID(id))
	os.MkdirAll(mountPath, 0755)

	// If this is a block driver, first attach the volume.
	if l.volDriver.Type() == api.DriverType_DRIVER_TYPE_BLOCK {
		_, err := l.volDriver.Attach(vols[index].Id)
		if err != nil {
			dlog.Errorf("Failed to attach volume %v", vols[index].Id)
			delete(l.volumes, id)
			return id, nil, nil
		}
	}
	err = l.volDriver.Mount(vols[index].Id, mountPath)
	if err != nil {
		dlog.Errorf("Failed to mount volume %v at path %v",
			vols[index].Id, mountPath)
		delete(l.volumes, id)
		return id, nil, nil
	}
	vol.path = mountPath
	vol.volumeID = vols[index].Id
	vol.ref = 1

	return l.realID(id), vol, nil
}
Example #19
0
func Init(params volume.DriverParams) (volume.VolumeDriver, error) {
	path, ok := params["path"]
	if !ok {
		return nil, errors.New("No NFS path provided")
	}
	server, ok := params["server"]
	if !ok {
		dlog.Printf("No NFS server provided, will attempt to bind mount %s", path)
	} else {
		dlog.Printf("NFS driver initializing with %s:%s ", server, path)
	}
	// Create a mount manager for this NFS server. Blank sever is OK.
	mounter, err := mount.New(mount.NFSMount, server)
	if err != nil {
		dlog.Warnf("Failed to create mount manager for server: %v (%v)", server, err)
		return nil, err
	}
	inst := &driver{
		IoNotSupported:    &volume.IoNotSupported{},
		DefaultEnumerator: volume.NewDefaultEnumerator(Name, kvdb.Instance()),
		nfsServer:         server,
		nfsPath:           path,
		mounter:           mounter,
	}
	if err := os.MkdirAll(nfsMountPath, 0744); err != nil {
		return nil, err
	}
	src := inst.nfsPath
	if server != "" {
		src = ":" + inst.nfsPath
	}
	// If src is already mounted at dest, leave it be.
	mountExists, err := mounter.Exists(src, nfsMountPath)
	if !mountExists {
		// Mount the nfs server locally on a unique path.
		syscall.Unmount(nfsMountPath, 0)
		if server != "" {
			err = syscall.Mount(src, nfsMountPath, "nfs", 0, "nolock,addr="+inst.nfsServer)
		} else {
			err = syscall.Mount(src, nfsMountPath, "", syscall.MS_BIND, "")
		}
		if err != nil {
			dlog.Printf("Unable to mount %s:%s at %s (%+v)", inst.nfsServer, inst.nfsPath, nfsMountPath, err)
			return nil, err
		}
	}
	volumeInfo, err := inst.DefaultEnumerator.Enumerate(&api.VolumeLocator{}, nil)
	if err == nil {
		for _, info := range volumeInfo {
			if info.Status == api.VolumeStatus_VOLUME_STATUS_NONE {
				info.Status = api.VolumeStatus_VOLUME_STATUS_UP
				inst.UpdateVol(info)
			}
		}
	} else {
		dlog.Println("Could not enumerate Volumes, ", err)
	}

	dlog.Println("NFS initialized and driver mounted at: ", nfsMountPath)
	return inst, nil
}
Example #20
0
func (d *driver) Create(locator *api.VolumeLocator, source *api.Source, spec *api.VolumeSpec) (string, error) {
	volumeID := uuid.New()
	volumeID = strings.TrimSuffix(volumeID, "\n")
	if spec.Size == 0 {
		return "", fmt.Errorf("Volume size cannot be zero", "buse")
	}
	if spec.Format == api.FSType_FS_TYPE_NONE {
		return "", fmt.Errorf("Missing volume format", "buse")
	}
	// Create a file on the local buse path with this UUID.
	buseFile := path.Join(BuseMountPath, volumeID)
	f, err := os.Create(buseFile)
	if err != nil {
		dlog.Println(err)
		return "", err
	}

	if err := f.Truncate(int64(spec.Size)); err != nil {
		dlog.Println(err)
		return "", err
	}

	bd := &buseDev{
		file: buseFile,
		f:    f,
	}
	nbd := Create(bd, int64(spec.Size))
	bd.nbd = nbd

	dlog.Infof("Connecting to NBD...")
	dev, err := bd.nbd.Connect()
	if err != nil {
		dlog.Println(err)
		return "", err
	}

	dlog.Infof("Formatting %s with %v", dev, spec.Format)
	cmd := "/sbin/mkfs." + spec.Format.SimpleString()
	o, err := exec.Command(cmd, dev).Output()
	if err != nil {
		dlog.Warnf("Failed to run command %v %v: %v", cmd, dev, o)
		return "", err
	}

	dlog.Infof("BUSE mapped NBD device %s (size=%v) to block file %s", dev, spec.Size, buseFile)

	v := common.NewVolume(
		volumeID,
		spec.Format,
		locator,
		source,
		spec,
	)
	v.DevicePath = dev

	d.buseDevices[dev] = bd

	err = d.CreateVol(v)
	if err != nil {
		return "", err
	}
	return v.Id, err
}
Example #21
0
func notFound(w http.ResponseWriter, r *http.Request) {
	dlog.Warnf("Not found: %+v ", r.URL)
	http.NotFound(w, r)
}