From b6428ac1730fa742ef6ae4472c1af5bf1bea3387 Mon Sep 17 00:00:00 2001 From: Jinshan Xiong Date: Mon, 19 Mar 2018 22:35:24 +0000 Subject: [PATCH 1/4] Revert "LU-8844 llite: delete lloop" This reverts commit 6298341e3d1f6b7dc1a859cf65b3252a9ae13447. --- lnet/utils/debug.c | 1 + lustre/autoconf/lustre-core.m4 | 21 + lustre/doc/lctl.8 | 12 + lustre/llite/Makefile.in | 4 + lustre/llite/autoMakefile.am | 9 +- lustre/llite/lloop.c | 905 +++++++++++++++++++++++++++++++++++++++++ lustre/scripts/lustre_rmmod | 2 +- lustre/tests/sanity.sh | 72 ++++ lustre/tests/test-framework.sh | 19 + lustre/utils/lctl.c | 12 + lustre/utils/obd.c | 202 +++++++++ lustre/utils/obdctl.h | 4 + 12 files changed, 1260 insertions(+), 3 deletions(-) create mode 100644 lustre/llite/lloop.c diff --git a/lnet/utils/debug.c b/lnet/utils/debug.c index 15e6046..1954a31 100644 --- a/lnet/utils/debug.c +++ b/lnet/utils/debug.c @@ -792,6 +792,7 @@ static struct mod_paths { { "mds", "lustre/mds" }, { "mdc", "lustre/mdc" }, { "lustre", "lustre/llite" }, + { "llite_lloop", "lustre/llite" }, { "ldiskfs", "ldiskfs" }, { "obdecho", "lustre/obdecho" }, { "ldlm", "lustre/ldlm" }, diff --git a/lustre/autoconf/lustre-core.m4 b/lustre/autoconf/lustre-core.m4 index 9ffa0ce..6de2bad 100644 --- a/lustre/autoconf/lustre-core.m4 +++ b/lustre/autoconf/lustre-core.m4 @@ -2261,6 +2261,7 @@ AC_DEFUN([LC_PROG_LINUX], [ LC_CONFIG_CHECKSUM LC_CONFIG_HEALTH_CHECK_WRITE LC_CONFIG_LRU_RESIZE + LC_LLITE_LLOOP_MODULE LC_GLIBC_SUPPORT_FHANDLES LC_CAPA_CRYPTO @@ -2546,6 +2547,25 @@ AS_IF([test "x$enable_nodemap_proc_debug" != xno], ]) # LC_NODEMAP_PROC_DEBUG # +# LC_LLITE_LLOOP_MODULE +# +# lloop_llite.ko does not currently work with page sizes +# of 64k or larger. +# +AC_DEFUN([LC_LLITE_LLOOP_MODULE], [ +LB_CHECK_COMPILE([whether to enable 'llite_lloop' module], +enable_llite_lloop_module, [ + #include +],[ + #if PAGE_SIZE >= 65536 + #error "PAGE_SIZE >= 65536" + #endif +], + [enable_llite_lloop_module="yes"], + [enable_llite_lloop_module="no"]) +]) # LC_LLITE_LLOOP_MODULE + +# # LC_OSD_ADDON # # configure support for optional OSD implementation @@ -2706,6 +2726,7 @@ AM_CONDITIONAL(GSS, test x$enable_gss = xyes) AM_CONDITIONAL(GSS_KEYRING, test x$enable_gss_keyring = xyes) AM_CONDITIONAL(GSS_PIPEFS, test x$enable_gss_pipefs = xyes) AM_CONDITIONAL(LIBPTHREAD, test x$enable_libpthread = xyes) +AM_CONDITIONAL(LLITE_LLOOP, test x$enable_llite_lloop_module = xyes) ]) # LC_CONDITIONALS # diff --git a/lustre/doc/lctl.8 b/lustre/doc/lctl.8 index 8353a04..879687c 100644 --- a/lustre/doc/lctl.8 +++ b/lustre/doc/lctl.8 @@ -272,6 +272,18 @@ communication with the failed OST. .BI abort_recovery Abort the recovery process on a restarting MDT or OST device .PP +.SS Virtual Block Device Operation +Lustre is able to emulate a virtual block device upon regular file. It is necessary to be used when you are trying to setup a swap space via file. +.TP +.BI blockdev_attach " " +Attach the lustre regular file to a block device. If the device node is not existent, lctl will create it \- it is recommended to create it by lctl since the emulator uses a dynamical major number. +.TP +.BI blockdev_detach " " +Detach the virtual block device. +.TP +.BI blockdev_info " " +Acquire which lustre file was attached to the device node. +.PP .SS Changelogs .TP .BI changelog_register diff --git a/lustre/llite/Makefile.in b/lustre/llite/Makefile.in index c6a3fdc..126f58e 100644 --- a/lustre/llite/Makefile.in +++ b/lustre/llite/Makefile.in @@ -1,4 +1,5 @@ MODULES := lustre +@LLITE_LLOOP_TRUE@MODULES += llite_lloop lustre-objs := dcache.o dir.o file.o llite_close.o llite_lib.o llite_nfs.o lustre-objs += rw.o lproc_llite.o namei.o symlink.o llite_mmap.o lustre-objs += xattr.o xattr_cache.o remote_perm.o llite_rmtacl.o llite_capa.o @@ -9,7 +10,10 @@ lustre-objs += lcommon_misc.o lustre-objs += vvp_dev.o vvp_page.o vvp_lock.o vvp_io.o vvp_object.o vvp_req.o lustre-objs += range_lock.o +llite_lloop-objs := lloop.o + EXTRA_DIST := $(lustre-objs:.o=.c) llite_internal.h rw26.c super25.c +EXTRA_DIST += $(llite_lloop-objs:.o=.c) EXTRA_DIST += vvp_internal.h range_lock.h @INCLUDE_RULES@ diff --git a/lustre/llite/autoMakefile.am b/lustre/llite/autoMakefile.am index 7cb48f6..4f68d57 100644 --- a/lustre/llite/autoMakefile.am +++ b/lustre/llite/autoMakefile.am @@ -35,7 +35,12 @@ # if MODULES -modulefs_DATA = lustre$(KMODEXT) +if LLITE_LLOOP + LLOOP_MOD = llite_lloop$(KMODEXT) +else + LLOOP_MOD = +endif +modulefs_DATA = lustre$(KMODEXT) $(LLOOP_MOD) endif -MOSTLYCLEANFILES := @MOSTLYCLEANFILES@ +MOSTLYCLEANFILES := @MOSTLYCLEANFILES@ diff --git a/lustre/llite/lloop.c b/lustre/llite/lloop.c new file mode 100644 index 0000000..b7b1db6 --- /dev/null +++ b/lustre/llite/lloop.c @@ -0,0 +1,905 @@ +/* + * GPL HEADER START + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 only, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 for more details (a copy is included + * in the LICENSE file that accompanied this code). + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; If not, see + * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf + * + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, + * CA 95054 USA or visit www.sun.com if you need additional information or + * have any questions. + * + * GPL HEADER END + */ +/* + * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. + * Use is subject to license terms. + * + * Copyright (c) 2011, 2014, Intel Corporation. + */ +/* + * This file is part of Lustre, http://www.lustre.org/ + * Lustre is a trademark of Sun Microsystems, Inc. + */ + +/* + * linux/drivers/block/loop.c + * + * Written by Theodore Ts'o, 3/29/93 + * + * Copyright 1993 by Theodore Ts'o. Redistribution of this file is + * permitted under the GNU General Public License. + * + * Modularized and updated for 1.1.16 kernel - Mitch Dsouza 28th May 1994 + * Adapted for 1.3.59 kernel - Andries Brouwer, 1 Feb 1996 + * + * Fixed do_loop_request() re-entrancy - Vincent.Renardias@waw.com Mar 20, 1997 + * + * Added devfs support - Richard Gooch 16-Jan-1998 + * + * Handle sparse backing files correctly - Kenn Humborg, Jun 28, 1998 + * + * Loadable modules and other fixes by AK, 1998 + * + * Maximum number of loop devices now dynamic via max_loop module parameter. + * Russell Kroll 19990701 + * + * Maximum number of loop devices when compiled-in now selectable by passing + * max_loop=<1-255> to the kernel on boot. + * Erik I. Bols?, , Oct 31, 1999 + * + * Completely rewrite request handling to be make_request_fn style and + * non blocking, pushing work to a helper thread. Lots of fixes from + * Al Viro too. + * Jens Axboe , Nov 2000 + * + * Support up to 256 loop devices + * Heinz Mauelshagen , Feb 2002 + * + * Support for falling back on the write file operation when the address space + * operations prepare_write and/or commit_write are not available on the + * backing filesystem. + * Anton Altaparmakov, 16 Feb 2005 + * + * Still To Fix: + * - Advisory locking is ignored here. + * - Should use an own CAP_* category instead of CAP_SYS_ADMIN + * + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* for invalidate_bdev() */ +#include +#include +#include +#include +#include + +#include + +#include +#include "llite_internal.h" + +#define LLOOP_MAX_SEGMENTS LNET_MAX_IOV + +/* Possible states of device */ +enum { + LLOOP_UNBOUND, + LLOOP_BOUND, + LLOOP_RUNDOWN, +}; + +struct lloop_device { + int lo_number; + int lo_refcnt; + loff_t lo_offset; + loff_t lo_sizelimit; + int lo_flags; + struct file *lo_backing_file; + struct block_device *lo_device; + unsigned lo_blocksize; + + gfp_t old_gfp_mask; + + spinlock_t lo_lock; + struct bio *lo_bio; + struct bio *lo_biotail; + int lo_state; + struct semaphore lo_sem; + struct mutex lo_ctl_mutex; + atomic_t lo_pending; + wait_queue_head_t lo_bh_wait; + + struct request_queue *lo_queue; + + const struct lu_env *lo_env; + struct cl_io lo_io; + struct ll_dio_pages lo_pvec; + + /* data to handle bio for lustre. */ + struct lo_request_data { + struct page *lrd_pages[LLOOP_MAX_SEGMENTS]; + loff_t lrd_offsets[LLOOP_MAX_SEGMENTS]; + } lo_requests[1]; +}; + +/* + * Loop flags + */ +enum { + LO_FLAGS_READ_ONLY = 1, +}; + +static int lloop_major; +#define MAX_LOOP_DEFAULT 16 +static int max_loop = MAX_LOOP_DEFAULT; +static struct lloop_device *loop_dev; +static struct gendisk **disks; +static struct mutex lloop_mutex; +static void *ll_iocontrol_magic = NULL; + +static loff_t get_loop_size(struct lloop_device *lo, struct file *file) +{ + loff_t size, offset, loopsize; + + /* Compute loopsize in bytes */ + size = i_size_read(file->f_mapping->host); + offset = lo->lo_offset; + loopsize = size - offset; + if (lo->lo_sizelimit > 0 && lo->lo_sizelimit < loopsize) + loopsize = lo->lo_sizelimit; + + /* + * Unfortunately, if we want to do I/O on the device, + * the number of 512-byte sectors has to fit into a sector_t. + */ + return loopsize >> 9; +} + +static int do_bio_lustrebacked(struct lloop_device *lo, struct bio *head) +{ + const struct lu_env *env = lo->lo_env; + struct cl_io *io = &lo->lo_io; + struct dentry *de = lo->lo_backing_file->f_path.dentry; + struct inode *inode = de->d_inode; + struct cl_object *obj = ll_i2info(inode)->lli_clob; + pgoff_t offset; + int ret; + int iter; + struct bio_vec *bvec; + int rw; + size_t page_count = 0; + struct bio *bio; + ssize_t bytes; + + struct ll_dio_pages *pvec = &lo->lo_pvec; + struct page **pages = pvec->ldp_pages; + loff_t *offsets = pvec->ldp_offsets; + + truncate_inode_pages(inode->i_mapping, 0); + + /* initialize the IO */ + memset(io, 0, sizeof(*io)); + io->ci_obj = obj; + ret = cl_io_init(env, io, CIT_MISC, obj); + if (ret) + return io->ci_result; + io->ci_lockreq = CILR_NEVER; + + LASSERT(head != NULL); + rw = head->bi_rw; + for (bio = head; bio != NULL; bio = bio->bi_next) { + LASSERT(rw == bio->bi_rw); + +#ifdef HAVE_BVEC_ITER + offset = (pgoff_t)(bio->bi_iter.bi_sector << 9) + lo->lo_offset; +#else + offset = (pgoff_t)(bio->bi_sector << 9) + lo->lo_offset; +#endif + bio_for_each_segment_all(bvec, bio, iter) { + BUG_ON(bvec->bv_offset != 0); + BUG_ON(bvec->bv_len != PAGE_CACHE_SIZE); + + pages[page_count] = bvec->bv_page; + offsets[page_count] = offset; + page_count++; + offset += bvec->bv_len; + } + LASSERT(page_count <= LLOOP_MAX_SEGMENTS); + } + + ll_stats_ops_tally(ll_i2sbi(inode), + (rw == WRITE) ? LPROC_LL_BRW_WRITE : LPROC_LL_BRW_READ, + page_count); + + pvec->ldp_size = page_count << PAGE_CACHE_SHIFT; + pvec->ldp_nr = page_count; + + /* FIXME: in ll_direct_rw_pages, it has to allocate many cl_page{}s to + * write those pages into OST. Even worse case is that more pages + * would be asked to write out to swap space, and then finally get here + * again. + * Unfortunately this is NOT easy to fix. + * Thoughts on solution: + * 0. Define a reserved pool for cl_pages, which could be a list of + * pre-allocated cl_pages; + * 1. Define a new operation in cl_object_operations{}, says clo_depth, + * which measures how many layers for this lustre object. Generally + * speaking, the depth would be 2, one for llite, and one for lovsub. + * However, for SNS, there will be more since we need additional page + * to store parity; + * 2. Reserve the # of (page_count * depth) cl_pages from the reserved + * pool. Afterwards, the clio would allocate the pages from reserved + * pool, this guarantees we neeedn't allocate the cl_pages from + * generic cl_page slab cache. + * Of course, if there is NOT enough pages in the pool, we might + * be asked to write less pages once, this purely depends on + * implementation. Anyway, we should be careful to avoid deadlocking. + */ + mutex_lock(&inode->i_mutex); + bytes = ll_direct_rw_pages(env, io, rw, inode, pvec); + mutex_unlock(&inode->i_mutex); + cl_io_fini(env, io); + return (bytes == pvec->ldp_size) ? 0 : (int)bytes; +} + +/* + * Add bio to back of pending list + */ +static void loop_add_bio(struct lloop_device *lo, struct bio *bio) +{ + unsigned long flags; + + spin_lock_irqsave(&lo->lo_lock, flags); + if (lo->lo_biotail) { + lo->lo_biotail->bi_next = bio; + lo->lo_biotail = bio; + } else + lo->lo_bio = lo->lo_biotail = bio; + spin_unlock_irqrestore(&lo->lo_lock, flags); + + atomic_inc(&lo->lo_pending); + if (waitqueue_active(&lo->lo_bh_wait)) + wake_up(&lo->lo_bh_wait); +} + +/* + * Grab first pending buffer + */ +static unsigned int loop_get_bio(struct lloop_device *lo, struct bio **req) +{ + struct bio *first; + struct bio **bio; + unsigned int count = 0; + unsigned int page_count = 0; + int rw; + + spin_lock_irq(&lo->lo_lock); + first = lo->lo_bio; + if (unlikely(first == NULL)) { + spin_unlock_irq(&lo->lo_lock); + return 0; + } + + /* TODO: need to split the bio, too bad. */ + LASSERT(first->bi_vcnt <= LLOOP_MAX_SEGMENTS); + + rw = first->bi_rw; + bio = &lo->lo_bio; + while (*bio && (*bio)->bi_rw == rw) { +#ifdef HAVE_BVEC_ITER + CDEBUG(D_INFO, "bio sector %llu size %u count %u vcnt%u \n", + (unsigned long long)(*bio)->bi_iter.bi_sector, + (*bio)->bi_iter.bi_size, page_count, (*bio)->bi_vcnt); +#else + CDEBUG(D_INFO, "bio sector %llu size %u count %u vcnt%u \n", + (unsigned long long)(*bio)->bi_sector, (*bio)->bi_size, + page_count, (*bio)->bi_vcnt); +#endif + if (page_count + (*bio)->bi_vcnt > LLOOP_MAX_SEGMENTS) + break; + + page_count += (*bio)->bi_vcnt; + count++; + bio = &(*bio)->bi_next; + } + if (*bio) { + /* Some of bios can't be mergable. */ + lo->lo_bio = *bio; + *bio = NULL; + } else { + /* Hit the end of queue */ + lo->lo_biotail = NULL; + lo->lo_bio = NULL; + } + *req = first; + spin_unlock_irq(&lo->lo_lock); + return count; +} + +static ll_mrf_ret +loop_make_request(struct request_queue *q, struct bio *old_bio) +{ + struct lloop_device *lo = q->queuedata; + int rw = bio_rw(old_bio); + int inactive; + + if (!lo) + goto err; + +#ifdef HAVE_BVEC_ITER + CDEBUG(D_INFO, "submit bio sector %llu size %u\n", + (unsigned long long)old_bio->bi_iter.bi_sector, + old_bio->bi_iter.bi_size); +#else + CDEBUG(D_INFO, "submit bio sector %llu size %u\n", + (unsigned long long)old_bio->bi_sector, old_bio->bi_size); +#endif + + spin_lock_irq(&lo->lo_lock); + inactive = (lo->lo_state != LLOOP_BOUND); + spin_unlock_irq(&lo->lo_lock); + if (inactive) + goto err; + + if (rw == WRITE) { + if (lo->lo_flags & LO_FLAGS_READ_ONLY) + goto err; + } else if (rw == READA) { + rw = READ; + } else if (rw != READ) { + CERROR("lloop: unknown command (%x)\n", rw); + goto err; + } + loop_add_bio(lo, old_bio); + LL_MRF_RETURN(0); +err: + bio_io_error(old_bio); + LL_MRF_RETURN(0); +} + +#ifdef HAVE_REQUEST_QUEUE_UNPLUG_FN +/* + * kick off io on the underlying address space + */ +static void loop_unplug(struct request_queue *q) +{ + struct lloop_device *lo = q->queuedata; + + clear_bit(QUEUE_FLAG_PLUGGED, &q->queue_flags); + blk_run_address_space(lo->lo_backing_file->f_mapping); +} +#endif + +static inline void loop_handle_bio(struct lloop_device *lo, struct bio *bio) +{ + int ret; + + ret = do_bio_lustrebacked(lo, bio); + while (bio) { + struct bio *tmp = bio->bi_next; + + bio->bi_next = NULL; +#ifdef HAVE_BIO_ENDIO_USES_ONE_ARG + bio->bi_error = ret; + bio_endio(bio); +#else + bio_endio(bio, ret); +#endif + bio = tmp; + } +} + +static inline int loop_active(struct lloop_device *lo) +{ + return atomic_read(&lo->lo_pending) || + (lo->lo_state == LLOOP_RUNDOWN); +} + +/* + * worker thread that handles reads/writes to file backed loop devices, + * to avoid blocking in our make_request_fn. + */ +static int loop_thread(void *data) +{ + struct lloop_device *lo = data; + struct bio *bio; + unsigned int count; + unsigned long times = 0; + unsigned long total_count = 0; + + struct lu_env *env; + __u16 refcheck; + int ret = 0; + + set_user_nice(current, -20); + + lo->lo_state = LLOOP_BOUND; + + env = cl_env_get(&refcheck); + if (IS_ERR(env)) + GOTO(out, ret = PTR_ERR(env)); + + lo->lo_env = env; + memset(&lo->lo_pvec, 0, sizeof(lo->lo_pvec)); + lo->lo_pvec.ldp_pages = lo->lo_requests[0].lrd_pages; + lo->lo_pvec.ldp_offsets = lo->lo_requests[0].lrd_offsets; + + /* + * up sem, we are running + */ + up(&lo->lo_sem); + + for (;;) { + wait_event(lo->lo_bh_wait, loop_active(lo)); + if (!atomic_read(&lo->lo_pending)) { + int exiting = 0; + spin_lock_irq(&lo->lo_lock); + exiting = (lo->lo_state == LLOOP_RUNDOWN); + spin_unlock_irq(&lo->lo_lock); + if (exiting) + break; + } + + bio = NULL; + count = loop_get_bio(lo, &bio); + if (!count) { + CWARN("lloop(minor: %d): missing bio\n", lo->lo_number); + continue; + } + + total_count += count; + if (total_count < count) { /* overflow */ + total_count = count; + times = 1; + } else { + times++; + } + if ((times & 127) == 0) { + CDEBUG(D_INFO, "total: %lu, count: %lu, avg: %lu\n", + total_count, times, total_count / times); + } + + LASSERT(bio != NULL); + LASSERT(count <= atomic_read(&lo->lo_pending)); + loop_handle_bio(lo, bio); + atomic_sub(count, &lo->lo_pending); + } + cl_env_put(env, &refcheck); + +out: + up(&lo->lo_sem); + return ret; +} + +static int loop_set_fd(struct lloop_device *lo, struct file *unused, + struct block_device *bdev, struct file *file) +{ + struct inode *inode; + struct address_space *mapping; + int lo_flags = 0; + int error; + loff_t size; + + if (!try_module_get(THIS_MODULE)) + return -ENODEV; + + error = -EBUSY; + if (lo->lo_state != LLOOP_UNBOUND) + goto out; + + mapping = file->f_mapping; + inode = mapping->host; + + error = -EINVAL; + if (!S_ISREG(inode->i_mode) || inode->i_sb->s_magic != LL_SUPER_MAGIC) + goto out; + + if (!(file->f_mode & FMODE_WRITE)) + lo_flags |= LO_FLAGS_READ_ONLY; + + size = get_loop_size(lo, file); + + if ((loff_t)(sector_t)size != size) { + error = -EFBIG; + goto out; + } + + /* remove all pages in cache so as dirty pages not to be existent. */ + truncate_inode_pages(mapping, 0); + + set_device_ro(bdev, (lo_flags & LO_FLAGS_READ_ONLY) != 0); + + lo->lo_blocksize = PAGE_CACHE_SIZE; + lo->lo_device = bdev; + lo->lo_flags = lo_flags; + lo->lo_backing_file = file; + lo->lo_sizelimit = 0; + lo->old_gfp_mask = mapping_gfp_mask(mapping); + mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS)); + + lo->lo_bio = lo->lo_biotail = NULL; + + /* + * set queue make_request_fn, and add limits based on lower level + * device + */ + blk_queue_make_request(lo->lo_queue, loop_make_request); + lo->lo_queue->queuedata = lo; +#ifdef HAVE_REQUEST_QUEUE_UNPLUG_FN + lo->lo_queue->unplug_fn = loop_unplug; +#endif + + /* queue parameters */ + blk_queue_max_hw_sectors(lo->lo_queue, + LLOOP_MAX_SEGMENTS << (PAGE_CACHE_SHIFT - 9)); + blk_queue_max_segments(lo->lo_queue, LLOOP_MAX_SEGMENTS); + + set_capacity(disks[lo->lo_number], size); + bd_set_size(bdev, size << 9); + + set_blocksize(bdev, lo->lo_blocksize); + + kthread_run(loop_thread, lo, "lloop%d", lo->lo_number); + down(&lo->lo_sem); + return 0; + +out: + /* This is safe: open() is still holding a reference. */ + module_put(THIS_MODULE); + return error; +} + +static int loop_clr_fd(struct lloop_device *lo, struct block_device *bdev, + int count) +{ + struct file *filp = lo->lo_backing_file; + gfp_t gfp = lo->old_gfp_mask; + + if (lo->lo_state != LLOOP_BOUND) + return -ENXIO; + + if (lo->lo_refcnt > count) /* we needed one fd for the ioctl */ + return -EBUSY; + + if (filp == NULL) + return -EINVAL; + + spin_lock_irq(&lo->lo_lock); + lo->lo_state = LLOOP_RUNDOWN; + spin_unlock_irq(&lo->lo_lock); + wake_up(&lo->lo_bh_wait); + + down(&lo->lo_sem); + lo->lo_backing_file = NULL; + lo->lo_device = NULL; + lo->lo_offset = 0; + lo->lo_sizelimit = 0; + lo->lo_flags = 0; + invalidate_bdev(bdev); + set_capacity(disks[lo->lo_number], 0); + bd_set_size(bdev, 0); + mapping_set_gfp_mask(filp->f_mapping, gfp); + lo->lo_state = LLOOP_UNBOUND; + fput(filp); + /* This is safe: open() is still holding a reference. */ + module_put(THIS_MODULE); + return 0; +} + +static int lo_open(struct block_device *bdev, fmode_t mode) +{ + struct lloop_device *lo = bdev->bd_disk->private_data; + + mutex_lock(&lo->lo_ctl_mutex); + lo->lo_refcnt++; + mutex_unlock(&lo->lo_ctl_mutex); + + return 0; +} + +#ifdef HAVE_BLKDEV_RELEASE_RETURN_INT +static int +#else +static void +#endif +lo_release(struct gendisk *disk, fmode_t mode) +{ + struct lloop_device *lo = disk->private_data; + + mutex_lock(&lo->lo_ctl_mutex); + --lo->lo_refcnt; + mutex_unlock(&lo->lo_ctl_mutex); +#ifdef HAVE_BLKDEV_RELEASE_RETURN_INT + return 0; +#endif +} + +/* lloop device node's ioctl function. */ +static int lo_ioctl(struct block_device *bdev, fmode_t mode, + unsigned int cmd, unsigned long arg) +{ + struct lloop_device *lo = bdev->bd_disk->private_data; + int err = 0; + + mutex_lock(&lloop_mutex); + switch (cmd) { + case LL_IOC_LLOOP_DETACH: { + err = loop_clr_fd(lo, bdev, 2); + if (err == 0) + blkdev_put(bdev, 0); /* grabbed in LLOOP_ATTACH */ + break; + } + + case LL_IOC_LLOOP_INFO: { + struct inode *inode; + struct lu_fid fid; + + if (lo->lo_backing_file == NULL) { + err = -ENOENT; + break; + } + inode = lo->lo_backing_file->f_path.dentry->d_inode; + if (inode != NULL && lo->lo_state == LLOOP_BOUND) + fid = ll_i2info(inode)->lli_fid; + else + fid_zero(&fid); + + if (copy_to_user((struct lu_fid __user *)arg, + &fid, sizeof(fid))) + err = -EFAULT; + break; + } + + default: + err = -EINVAL; + break; + } + mutex_unlock(&lloop_mutex); + + return err; +} + +static struct block_device_operations lo_fops = { + .owner = THIS_MODULE, + .open = lo_open, + .release = lo_release, + .ioctl = lo_ioctl, +}; + +/* dynamic iocontrol callback. + * This callback is registered in lloop_init and will be called by + * ll_iocontrol_call. + * + * This is a llite regular file ioctl function. It takes the responsibility + * of attaching or detaching a file by a lloop's device numner. + */ +static enum llioc_iter lloop_ioctl(struct inode *unused, struct file *file, + unsigned int cmd, unsigned long arg, + void *magic, int *rcp) +{ + struct lloop_device *lo = NULL; + struct block_device *bdev = NULL; + int err = 0; + dev_t dev; + + if (magic != ll_iocontrol_magic) + return LLIOC_CONT; + + if (disks == NULL) + GOTO(out1, err = -ENODEV); + + CWARN("Enter llop_ioctl\n"); + + mutex_lock(&lloop_mutex); + switch (cmd) { + case LL_IOC_LLOOP_ATTACH: { + struct inode *inode = file->f_path.dentry->d_inode; + struct lloop_device *lo_free = NULL; + int i; + + for (i = 0; i < max_loop; i++, lo = NULL) { + lo = &loop_dev[i]; + if (lo->lo_state == LLOOP_UNBOUND) { + if (!lo_free) + lo_free = lo; + continue; + } + if (lo->lo_backing_file->f_path.dentry->d_inode == + inode) + break; + } + if (lo || !lo_free) + GOTO(out, err = -EBUSY); + + lo = lo_free; + dev = MKDEV(lloop_major, lo->lo_number); + + /* quit if the used pointer is writable */ + if (put_user((long)old_encode_dev(dev), (long __user *)arg)) + GOTO(out, err = -EFAULT); + + bdev = blkdev_get_by_dev(dev, file->f_mode, NULL); + if (IS_ERR(bdev)) + GOTO(out, err = PTR_ERR(bdev)); + + get_file(file); + err = loop_set_fd(lo, NULL, bdev, file); + if (err) { + fput(file); + blkdev_put(bdev, 0); + } + + break; + } + + case LL_IOC_LLOOP_DETACH_BYDEV: { + int minor; + + dev = old_decode_dev(arg); + if (MAJOR(dev) != lloop_major) + GOTO(out, err = -EINVAL); + + minor = MINOR(dev); + if (minor > max_loop - 1) + GOTO(out, err = -EINVAL); + + lo = &loop_dev[minor]; + if (lo->lo_state != LLOOP_BOUND) + GOTO(out, err = -EINVAL); + + bdev = lo->lo_device; + err = loop_clr_fd(lo, bdev, 1); + if (err == 0) + blkdev_put(bdev, 0); /* grabbed in LLOOP_ATTACH */ + + break; + } + + default: + err = -EINVAL; + break; + } + +out: + mutex_unlock(&lloop_mutex); +out1: + if (rcp) + *rcp = err; + return LLIOC_STOP; +} + +static int __init lloop_init(void) +{ + int i; + unsigned int cmdlist[] = { + LL_IOC_LLOOP_ATTACH, + LL_IOC_LLOOP_DETACH_BYDEV, + }; + + if (max_loop < 1 || max_loop > 256) { + max_loop = MAX_LOOP_DEFAULT; + CWARN("lloop: invalid max_loop (must be between" + " 1 and 256), using default (%u)\n", max_loop); + } + + lloop_major = register_blkdev(0, "lloop"); + if (lloop_major < 0) + return -EIO; + + CDEBUG(D_CONFIG, "registered lloop major %d with %u minors\n", + lloop_major, max_loop); + + ll_iocontrol_magic = ll_iocontrol_register(lloop_ioctl, 2, cmdlist); + if (ll_iocontrol_magic == NULL) + goto out_mem1; + + OBD_ALLOC_WAIT(loop_dev, max_loop * sizeof(*loop_dev)); + if (!loop_dev) + goto out_mem1; + + OBD_ALLOC_WAIT(disks, max_loop * sizeof(*disks)); + if (!disks) + goto out_mem2; + + for (i = 0; i < max_loop; i++) { + disks[i] = alloc_disk(1); + if (!disks[i]) + goto out_mem3; + } + + mutex_init(&lloop_mutex); + + for (i = 0; i < max_loop; i++) { + struct lloop_device *lo = &loop_dev[i]; + struct gendisk *disk = disks[i]; + + lo->lo_queue = blk_alloc_queue(GFP_KERNEL); + if (!lo->lo_queue) + goto out_mem4; + + mutex_init(&lo->lo_ctl_mutex); + sema_init(&lo->lo_sem, 0); + init_waitqueue_head(&lo->lo_bh_wait); + lo->lo_number = i; + spin_lock_init(&lo->lo_lock); + disk->major = lloop_major; + disk->first_minor = i; + disk->fops = &lo_fops; + sprintf(disk->disk_name, "lloop%d", i); + disk->private_data = lo; + disk->queue = lo->lo_queue; + } + + /* We cannot fail after we call this, so another loop!*/ + for (i = 0; i < max_loop; i++) + add_disk(disks[i]); + return 0; + +out_mem4: + while (i--) + blk_cleanup_queue(loop_dev[i].lo_queue); + i = max_loop; +out_mem3: + while (i--) + put_disk(disks[i]); + OBD_FREE(disks, max_loop * sizeof(*disks)); +out_mem2: + OBD_FREE(loop_dev, max_loop * sizeof(*loop_dev)); +out_mem1: + unregister_blkdev(lloop_major, "lloop"); + ll_iocontrol_unregister(ll_iocontrol_magic); + CERROR("lloop: ran out of memory\n"); + return -ENOMEM; +} + +static void lloop_exit(void) +{ + int i; + + ll_iocontrol_unregister(ll_iocontrol_magic); + for (i = 0; i < max_loop; i++) { + del_gendisk(disks[i]); + blk_cleanup_queue(loop_dev[i].lo_queue); + put_disk(disks[i]); + } + unregister_blkdev(lloop_major, "lloop"); + + OBD_FREE(disks, max_loop * sizeof(*disks)); + OBD_FREE(loop_dev, max_loop * sizeof(*loop_dev)); +} + +module_init(lloop_init); +module_exit(lloop_exit); + +CFS_MODULE_PARM(max_loop, "i", int, 0444, "maximum of lloop_device"); +MODULE_AUTHOR("Sun Microsystems, Inc. "); +MODULE_DESCRIPTION("Lustre virtual block device"); +MODULE_LICENSE("GPL"); diff --git a/lustre/scripts/lustre_rmmod b/lustre/scripts/lustre_rmmod index 76a9d11..237ed1c 100755 --- a/lustre/scripts/lustre_rmmod +++ b/lustre/scripts/lustre_rmmod @@ -12,7 +12,7 @@ LCTL=${LCTL:-"$LUSTRE/utils/lctl"} [ ! -f "$LCTL" ] && export LCTL=$(which lctl 2> /dev/null) unload_dep_module() { - # libcfs 107852 17 lustre,obdfilter,ost,... + # libcfs 107852 17 llite_lloop,lustre,obdfilter,ost,... local MODULE=$1 local DEPS="$(lsmod | awk '($1 == "'$MODULE'") { print $4 }' | tr ',' ' ')" for SUBMOD in $DEPS; do diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 6beb18b..9d65f50 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -5567,6 +5567,10 @@ cleanup_68() { rm -f $LLOOP unset LLOOP fi + if [ ! -z "$LLITELOOPLOAD" ]; then + rmmod llite_lloop + unset LLITELOOPLOAD + fi rm -f $DIR/f68* } @@ -5578,6 +5582,74 @@ swap_used() { swapon -s | awk '($1 == "'$1'") { print $4 }' } +# test case for lloop driver, basic function +test_68a() { + [ $PARALLEL == "yes" ] && skip "skip parallel run" && return + [ "$UID" != 0 ] && skip_env "must run as root" && return + llite_lloop_enabled || \ + { skip_env "llite_lloop module disabled" && return; } + + trap cleanup_68 EXIT + + if ! module_loaded llite_lloop; then + if load_module llite/llite_lloop; then + LLITELOOPLOAD=yes + else + skip_env "can't find module llite_lloop" + return + fi + fi + + LLOOP=$TMP/lloop.`date +%s`.`date +%N` + dd if=/dev/zero of=$DIR/f68a bs=4k count=1024 + $LCTL blockdev_attach $DIR/f68a $LLOOP || error "attach failed" + + directio rdwr $LLOOP 0 1024 4096 || error "direct write failed" + directio rdwr $LLOOP 0 1025 4096 && error "direct write should fail" + + cleanup_68 +} +run_test 68a "lloop driver - basic test ========================" + +# excercise swapping to lustre by adding a high priority swapfile entry +# and then consuming memory until it is used. +test_68b() { # was test_68 + [ $PARALLEL == "yes" ] && skip "skip parallel run" && return + [ "$UID" != 0 ] && skip_env "must run as root" && return + lctl get_param -n devices | grep -q obdfilter && \ + skip "local OST" && return + + grep -q llite_lloop /proc/modules + [ $? -ne 0 ] && skip "can't find module llite_lloop" && return + + [ -z "`$LCTL list_nids | grep -v tcp`" ] && \ + skip "can't reliably test swap with TCP" && return + + MEMTOTAL=`meminfo MemTotal` + NR_BLOCKS=$((MEMTOTAL>>8)) + [[ $NR_BLOCKS -le 2048 ]] && NR_BLOCKS=2048 + + LLOOP=$TMP/lloop.`date +%s`.`date +%N` + dd if=/dev/zero of=$DIR/f68b bs=64k seek=$NR_BLOCKS count=1 + mkswap $DIR/f68b + + $LCTL blockdev_attach $DIR/f68b $LLOOP || error "attach failed" + + trap cleanup_68 EXIT + + swapon -p 32767 $LLOOP || error "swapon $LLOOP failed" + + echo "before: `swapon -s | grep $LLOOP`" + $MEMHOG $MEMTOTAL || error "error allocating $MEMTOTAL kB" + echo "after: `swapon -s | grep $LLOOP`" + SWAPUSED=`swap_used $LLOOP` + + cleanup_68 + + [ $SWAPUSED -eq 0 ] && echo "no swap used???" || true +} +run_test 68b "support swapping to Lustre ========================" + # bug5265, obdfilter oa2dentry return -ENOENT # #define OBD_FAIL_SRV_ENOENT 0x217 test_69() { diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index fe0681c..202dc96 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -18,6 +18,10 @@ export QUOTA_AUTO=1 # specify environment variable containing batch job name for server statistics export JOBID_VAR=${JOBID_VAR:-"procname_uid"} # or "existing" or "disable" +# LOAD_LLOOP: LU-409: only load llite_lloop module if kernel < 2.6.32 or +# LOAD_LLOOP is true. LOAD_LLOOP is false by default. +export LOAD_LLOOP=${LOAD_LLOOP:-false} + #export PDSH="pdsh -S -Rssh -w" export MOUNT_CMD=${MOUNT_CMD:-"mount -t lustre"} export UMOUNT=${UMOUNT:-"umount -d"} @@ -482,6 +486,20 @@ load_module() { fi } +llite_lloop_enabled() { + local n1=$(uname -r | cut -d. -f1) + local n2=$(uname -r | cut -d. -f2) + local n3=$(uname -r | cut -d- -f1 | cut -d. -f3) + + # load the llite_lloop module for < 2.6.32 kernels + if [[ $n1 -lt 2 ]] || [[ $n1 -eq 2 && $n2 -lt 6 ]] || \ + [[ $n1 -eq 2 && $n2 -eq 6 && $n3 -lt 32 ]] || \ + $LOAD_LLOOP; then + return 0 + fi + return 1 +} + load_modules_local() { if [ -n "$MODPROBE" ]; then # use modprobe @@ -565,6 +583,7 @@ load_modules_local() { fi load_module llite/lustre + llite_lloop_enabled && load_module llite/llite_lloop [ -d /r ] && OGDB=${OGDB:-"/r/tmp"} OGDB=${OGDB:-$TMP} rm -f $OGDB/ogdb-$HOSTNAME diff --git a/lustre/utils/lctl.c b/lustre/utils/lctl.c index 24cdd44..153e84d 100644 --- a/lustre/utils/lctl.c +++ b/lustre/utils/lctl.c @@ -226,6 +226,18 @@ command_t cmdlist[] = { "provide gdb-friendly module information\n" "usage: modules "}, + /* virtual block operations */ + {"==== virtual block device ====", jt_noop, 0, "virtual block device"}, + {"blockdev_attach", jt_blockdev_attach, 0, + "attach a lustre regular file to a virtual block device\n" + "usage: blockdev_attach "}, + {"blockdev_detach", jt_blockdev_detach, 0, + "detach a lustre regular file from a virtual block device\n" + "usage: blockdev_detach "}, + {"blockdev_info", jt_blockdev_info, 0, + "get the device info of a attached file\n" + "usage: blockdev_info "}, + /* Pool commands */ {"=== Pools ==", jt_noop, 0, "pool management"}, {"pool_new", jt_pool_cmd, 0, diff --git a/lustre/utils/obd.c b/lustre/utils/obd.c index 6e1bbe3..54f55d2 100644 --- a/lustre/utils/obd.c +++ b/lustre/utils/obd.c @@ -2804,6 +2804,208 @@ int jt_llog_remove(int argc, char **argv) return rc; } +/* attach a regular file to virtual block device. + * return vaule: + * -1: fatal error + * 1: error, it always means the command run failed + * 0: success + */ +static int jt_blockdev_run_process(const char *file, char *argv[]) +{ + pid_t pid; + int rc; + + pid = vfork(); + if (pid == 0) { /* child process */ + /* don't print error messages */ + close(1), close(2); + (void)execvp(file, argv); + exit(-1); + } else if (pid > 0) { + int status; + + rc = waitpid(pid, &status, 0); + if (rc < 0 || !WIFEXITED(status)) + return -1; + + return WEXITSTATUS(status); + } + + return -1; +} + +static int jt_blockdev_find_module(const char *module) +{ + FILE *fp; + int found = 0; + char buf[1024]; + char *ptr; + + fp = fopen("/proc/modules", "r"); + if (fp == NULL) + return -1; + + while (fgets(buf, 1024, fp) != NULL) { + ptr = strchr(buf, ' '); + if (ptr != NULL) + *ptr = 0; + if (strcmp(module, buf) == 0) { + found = 1; + break; + } + } + fclose(fp); + + return found; +} + +static int jt_blockdev_probe_module(const char *module) +{ + char buf[1024]; + char *argv[10]; + int c, rc; + + if (jt_blockdev_find_module(module) == 1) + return 0; + + /* run modprobe first */ + c = 0; + argv[c++] = "/sbin/modprobe"; + argv[c++] = "-q"; + argv[c++] = (char *)module; + argv[c++] = NULL; + rc = jt_blockdev_run_process("modprobe", argv); + if (rc != 1) + return rc; + + /* cannot find the module in default directory ... */ + sprintf(buf, "../llite/%s.ko", module); + c = 0; + argv[c++] = "/sbin/insmod"; + argv[c++] = buf; + argv[c++] = NULL; + rc = jt_blockdev_run_process("insmod", argv); + return rc ? -1 : 0; +} + +int jt_blockdev_attach(int argc, char **argv) +{ + int rc, fd; + struct stat st; + char *filename, *devname; + unsigned long dev; + + if (argc != 3) + return CMD_HELP; + + if (jt_blockdev_probe_module("llite_lloop") < 0) { + fprintf(stderr, "error: cannot find module llite_lloop.(k)o\n"); + return ENOENT; + } + + filename = argv[1]; + devname = argv[2]; + + fd = open(filename, O_RDWR); + if (fd < 0) { + fprintf(stderr, "file %s can't be opened(%s)\n\n", + filename, strerror(errno)); + return CMD_HELP; + } + + rc = ioctl(fd, LL_IOC_LLOOP_ATTACH, &dev); + if (rc < 0) { + rc = errno; + fprintf(stderr, "attach error(%s)\n", strerror(rc)); + goto out; + } + + rc = stat(devname, &st); + if (rc == 0 && (!S_ISBLK(st.st_mode) || st.st_rdev != dev)) { + rc = EEXIST; + } else if (rc < 0) { + if (errno == ENOENT && + !mknod(devname, S_IFBLK|S_IRUSR|S_IWUSR, dev)) + rc = 0; + else + rc = errno; + } + + if (rc) { + fprintf(stderr, "error: the file %s could be attached to block " + "device %X but creating %s failed: %s\n" + "now detaching the block device..", + filename, (int)dev, devname, strerror(rc)); + + (void)ioctl(fd, LL_IOC_LLOOP_DETACH_BYDEV, dev); + fprintf(stderr, "%s\n", strerror(errno)); + } +out: + close(fd); + return -rc; +} + +int jt_blockdev_detach(int argc, char **argv) +{ + char *filename; + int rc, fd; + + if (argc != 2) + return CMD_HELP; + + filename = argv[1]; + fd = open(filename, O_RDONLY); + if (fd < 0) { + fprintf(stderr, "cannot open file %s error %s\n", + filename, strerror(errno)); + return CMD_HELP; + } + + rc = ioctl(fd, LL_IOC_LLOOP_DETACH, 0); + if (rc < 0) { + rc = errno; + fprintf(stderr, "detach error(%s)\n", strerror(rc)); + } else { + (void)unlink(filename); + } + + close(fd); + return -rc; +} + +int jt_blockdev_info(int argc, char **argv) +{ + char *filename; + int rc, fd; + struct lu_fid fid; + + if (argc != 2) + return CMD_HELP; + + filename = argv[1]; + fd = open(filename, O_RDONLY); + if (fd < 0) { + fprintf(stderr, "cannot open file %s error: %s\n", + filename, strerror(errno)); + return CMD_HELP; + } + + rc = ioctl(fd, LL_IOC_LLOOP_INFO, &fid); + if (rc < 0) { + rc = errno; + fprintf(stderr, "error: %s\n", strerror(errno)); + goto out; + } + fprintf(stdout, "lloop device info: "); + if (fid_is_zero(&fid)) + fprintf(stdout, "Not attached\n"); + else + fprintf(stdout, "attached to inode "DFID"\n", PFID(&fid)); +out: + close(fd); + return -rc; +} + static void signal_server(int sig) { if (sig == SIGINT) { diff --git a/lustre/utils/obdctl.h b/lustre/utils/obdctl.h index f595107..2849a58 100644 --- a/lustre/utils/obdctl.h +++ b/lustre/utils/obdctl.h @@ -122,6 +122,10 @@ int jt_lcfg_getparam(int argc, char **argv); int jt_lcfg_setparam(int argc, char **argv); int jt_lcfg_listparam(int argc, char **argv); +int jt_blockdev_attach(int argc, char **argv); +int jt_blockdev_detach(int argc, char **argv); +int jt_blockdev_info(int argc, char **argv); + int jt_pool_cmd(int argc, char **argv); int jt_barrier_freeze(int argc, char **argv); int jt_barrier_thaw(int argc, char **argv); -- 1.9.1