diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 467093acadbb89..7229ca31be6671 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -411,6 +411,18 @@ static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff, if (likely(fi)) { spin_lock(&fi->lock); list_del(&ff->write_entry); + /* + * Leave forced direct IO mode once the last writer is gone: with + * no local writer left there is no cached-write contention with + * the remote modifier that triggered the switch. Restore + * FUSE_I_CACHE_IO_MODE for any frozen cached opens. + */ + if (test_bit(FUSE_I_FORCE_DIO, &fi->state) && + list_empty(&fi->write_files)) { + clear_bit(FUSE_I_FORCE_DIO, &fi->state); + if (fi->iocachectr > 0) + set_bit(FUSE_I_CACHE_IO_MODE, &fi->state); + } spin_unlock(&fi->lock); } spin_lock(&fc->lock); @@ -449,9 +461,24 @@ void fuse_file_release(struct inode *inode, struct fuse_file *ff, struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_release_args *ra = &ff->args->release_args; int opcode = isdir ? FUSE_RELEASEDIR : FUSE_RELEASE; + bool was_force_dio = test_bit(FUSE_I_FORCE_DIO, &fi->state); fuse_prepare_release(fi, ff, open_flags, opcode, false); + /* + * If this release dropped the last writer, fuse_prepare_release() + * cleared the forced-direct-IO latch (under fi->lock). Drop any clean + * folios a read racing the latch may have repopulated so they cannot be + * served stale once caching mode resumes. No inode lock or + * wb_inval_rwsem: release may run on the fuse server thread (async fput + * from aio completion), where blocking on a contended inode lock could + * stall the connection. Writes were routed direct while latched, so + * only clean folios exist and this invalidate is server-free; the last + * writer is gone, so no forced-dio writer can race the drop. + */ + if (was_force_dio && !test_bit(FUSE_I_FORCE_DIO, &fi->state)) + invalidate_inode_pages2(inode->i_mapping); + if (ra && ff->flock) { ra->inarg.release_flags |= FUSE_RELEASE_FLOCK_UNLOCK; ra->inarg.lock_owner = fuse_lock_owner_id(ff->fm->fc, id); @@ -1497,13 +1524,6 @@ static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii) return res; } -static bool fuse_io_past_eof(struct kiocb *iocb, struct iov_iter *iter) -{ - struct inode *inode = file_inode(iocb->ki_filp); - - return iocb->ki_pos + iov_iter_count(iter) > i_size_read(inode); -} - /* * @return true if an exclusive lock for direct IO writes is needed */ @@ -1513,9 +1533,15 @@ static bool fuse_dio_wr_exclusive_lock(struct kiocb *iocb, struct iov_iter *from struct fuse_file *ff = file->private_data; struct inode *inode = file_inode(iocb->ki_filp); struct fuse_inode *fi = get_fuse_inode(inode); + bool force_dio = test_bit(FUSE_I_FORCE_DIO, &fi->state); - /* Server side has to advise that it supports parallel dio writes. */ - if (!(ff->open_flags & FOPEN_PARALLEL_DIRECT_WRITES)) + /* + * Server side has to advise that it supports parallel dio writes. + * When the inode is latched into forced direct IO, parallel writes are + * used unconditionally: the page cache has been flushed and is bypassed + * for this inode. + */ + if (!force_dio && !(ff->open_flags & FOPEN_PARALLEL_DIRECT_WRITES)) return true; /* @@ -1526,18 +1552,14 @@ static bool fuse_dio_wr_exclusive_lock(struct kiocb *iocb, struct iov_iter *from return true; /* shared locks are not allowed with parallel page cache IO */ - if (test_bit(FUSE_I_CACHE_IO_MODE, &fi->state)) - return true; - - /* Parallel dio beyond EOF is not supported, at least for now. */ - if (fuse_io_past_eof(iocb, from)) + if (!force_dio && test_bit(FUSE_I_CACHE_IO_MODE, &fi->state)) return true; return false; } static void fuse_dio_lock(struct kiocb *iocb, struct iov_iter *from, - bool *exclusive) + bool *exclusive, bool *uncached) { struct inode *inode = file_inode(iocb->ki_filp); struct fuse_inode *fi = get_fuse_inode(inode); @@ -1551,19 +1573,20 @@ static void fuse_dio_lock(struct kiocb *iocb, struct iov_iter *from, * New parallal dio allowed only if inode is not in caching * mode and denies new opens in caching mode. This check * should be performed only after taking shared inode lock. - * Previous past eof check was without inode lock and might - * have raced, so check it again. */ - if (fuse_io_past_eof(iocb, from) || - fuse_inode_uncached_io_start(fi, NULL) != 0) { - inode_unlock_shared(inode); - inode_lock(inode); - *exclusive = true; + if (!test_bit(FUSE_I_FORCE_DIO, &fi->state)) { + if (fuse_inode_uncached_io_start(fi, NULL) != 0) { + inode_unlock_shared(inode); + inode_lock(inode); + *exclusive = true; + } else { + *uncached = true; + } } } } -static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive) +static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive, bool uncached) { struct inode *inode = file_inode(iocb->ki_filp); struct fuse_inode *fi = get_fuse_inode(inode); @@ -1571,8 +1594,8 @@ static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive) if (exclusive) { inode_unlock(inode); } else { - /* Allow opens in caching mode after last parallel dio end */ - fuse_inode_uncached_io_end(fi); + if (uncached) + fuse_inode_uncached_io_end(fi); inode_unlock_shared(inode); } } @@ -1666,6 +1689,47 @@ static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, return written < 0 ? written : total_written; } +static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from); + +/* + * @return true if an exclusive inode lock is needed for a cached (buffered) + * write. + * + * Buffered writes normally hold the inode rwsem exclusively, serialising all + * writers even on disjoint ranges. The DLM-serialised iomap writeback path is + * the exception: the DLM already excludes cluster-wide, and i_size is committed + * under fi->lock rather than the inode rwsem (see fuse_cache_write_iter()), so + * disjoint writers (MPI-IO / IOR) may share the lock. Mirrors + * fuse_dio_wr_exclusive_lock() for the direct path. + */ +static bool fuse_cache_wr_exclusive_lock(struct kiocb *iocb, bool writeback) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct fuse_conn *fc = get_fuse_conn(inode); + + /* Only the DLM-serialised iomap writeback path relaxes the lock. */ + if (!fc->dlm || !writeback) + return true; + + /* O_DIRECT writes fall back to generic_file_direct_write(). */ + if (iocb->ki_flags & IOCB_DIRECT) + return true; + + /* Append needs the eventual EOF - always needs an exclusive lock. */ + if (iocb->ki_flags & IOCB_APPEND) + return true; + + return false; +} + +static void fuse_cache_wr_unlock(struct inode *inode, bool exclusive) +{ + if (exclusive) + inode_unlock(inode); + else + inode_unlock_shared(inode); +} + static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) { struct file *file = iocb->ki_filp; @@ -1675,7 +1739,20 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) struct inode *inode = mapping->host; ssize_t err, count; struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); bool writeback = false; + bool wb_guard = false; + bool exclusive = true; + + /* + * The inode may have been latched into forced direct IO -- by a + * NOTIFY_INVAL_INODE arriving while this inode is open for writing here + * -- after this write was routed to the cached path but before it took + * any lock. Re-route to the direct path (before taking a DLM lock) so + * we do not repopulate the page cache the latch just dropped. + */ + if (fuse_inode_force_dio(inode)) + return fuse_direct_write_iter(iocb, from); if (fc->writeback_cache) { /* Update size (EOF optimization) and mode (SUID clearing) */ @@ -1713,7 +1790,32 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) } } - inode_lock(inode); + exclusive = fuse_cache_wr_exclusive_lock(iocb, writeback); + if (exclusive) + inode_lock(inode); + else + inode_lock_shared(inode); + + /* + * The forced-direct-IO latch feature is active under writeback+dlm; + * hold wb_inval_rwsem for read across the page-cache dirtying so a + * concurrent NOTIFY_INVAL_INODE -- which latches the inode under the + * write side of this lock via a non-blocking trylock -- cannot strand + * the folios we are about to write. Re-check the latch under it (it may + * have been set while we blocked on the inode lock) and re-route to the + * direct path if set. Taken before task_io_account_write() so a + * re-route is not double-counted; the DLM write lock taken above is + * harmless as the direct path does its own server coordination. + */ + wb_guard = fc->writeback_cache && fc->dlm; + if (wb_guard) { + down_read(&fi->wb_inval_rwsem); + if (fuse_inode_force_dio(inode)) { + up_read(&fi->wb_inval_rwsem); + fuse_cache_wr_unlock(inode, exclusive); + return fuse_direct_write_iter(iocb, from); + } + } err = count = generic_write_checks(iocb, from); if (err <= 0) @@ -1732,7 +1834,66 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) written = direct_write_fallback(iocb, from, written, fuse_perform_write(iocb, from)); } else if (writeback) { + loff_t pos = iocb->ki_pos; + loff_t end = pos + count; + loff_t orig_size = 0; + bool extended = false; + + /* + * i_size is not protected by the shared lock in inode->i_rwsem. + * So if iomap_write_iter() grew EOF past i_size via its normal + * unlocked read-modify-write, two concurrent writers could race + * and one's update would get lost. + * To avoid this, claim the extension up front under fi->lock, + * so iomap sees pos + written <= i_size and never touches i_size + * itself. The update can then safely happen here, the same way + * fuse_write_update_attr() commits size on the direct io path. + * + * The lockless pre-check below avoids needlessly locking fi->lock + * if writes fall within the existing i_size. + * Operations that grow the file size take fi->lock, whereas a + * truncate holds the inode->i_rwsem exclusive. A stale read + * may over trigger this slow path, but it won’t miss an extension + * beyond i_size. + * + * The exclusive path keeps the classic behavior + * iomap owns the i_size update, serialized by the inode lock. + */ + if (!exclusive && end > i_size_read(inode)) { + spin_lock(&fi->lock); + orig_size = i_size_read(inode); + if (end > orig_size) { + i_size_write(inode, end); + extended = true; + } + spin_unlock(&fi->lock); + + /* Zero the tail of the folio straddling the old EOF. */ + if (extended && orig_size < pos) + pagecache_isize_extended(inode, orig_size, pos); + } + written = fuse_writeback_write_iter(iocb, from, file); + + /* + * Reconcile the speculative extension with what was actually + * written (short write, error, or nothing written all retract + * to the reached position). Only retract if no concurrent + * extender has pushed i_size past our claim; otherwise + * [reached, end) is a legitimate hole inside their extension and + * must remain. + */ + if (extended) { + loff_t reached = written > 0 ? pos + written : orig_size; + + if (reached < end) { + spin_lock(&fi->lock); + if (i_size_read(inode) == end) + i_size_write(inode, reached); + spin_unlock(&fi->lock); + } + } + if (written < 0) { err = written; goto out; @@ -1741,7 +1902,9 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) written = fuse_perform_write(iocb, from); } out: - inode_unlock(inode); + if (wb_guard) + up_read(&fi->wb_inval_rwsem); + fuse_cache_wr_unlock(inode, exclusive); if (written > 0) written = generic_write_sync(iocb, written); @@ -1975,14 +2138,16 @@ static ssize_t __fuse_direct_read(struct fuse_io_priv *io, return res; } -static ssize_t fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter); +static ssize_t __fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter, + bool exclusive); static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to) { ssize_t res; if (!is_sync_kiocb(iocb)) { - res = fuse_direct_IO(iocb, to); + /* exclusive is unused on reads; rollback is write-only */ + res = __fuse_direct_IO(iocb, to, true); } else { struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); @@ -1997,15 +2162,16 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) struct inode *inode = file_inode(iocb->ki_filp); struct address_space *mapping = inode->i_mapping; loff_t pos = iocb->ki_pos; + bool exclusive = false; + bool uncached = false; ssize_t res; - bool exclusive; - fuse_dio_lock(iocb, from, &exclusive); + fuse_dio_lock(iocb, from, &exclusive, &uncached); res = generic_write_checks(iocb, from); if (res > 0) { task_io_account_write(res); if (!is_sync_kiocb(iocb)) { - res = fuse_direct_IO(iocb, from); + res = __fuse_direct_IO(iocb, from, exclusive); } else { struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); @@ -2024,7 +2190,7 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) (pos + res - 1) >> PAGE_SHIFT); } } - fuse_dio_unlock(iocb, exclusive); + fuse_dio_unlock(iocb, exclusive, uncached); return res; } @@ -2042,7 +2208,7 @@ static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to) return fuse_dax_read_iter(iocb, to); /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ - if (ff->open_flags & FOPEN_DIRECT_IO) + if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_inode_force_dio(inode)) return fuse_direct_read_iter(iocb, to); else if (fuse_file_passthrough(ff)) return fuse_passthrough_read_iter(iocb, to); @@ -2063,7 +2229,7 @@ static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from) return fuse_dax_write_iter(iocb, from); /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ - if (ff->open_flags & FOPEN_DIRECT_IO) + if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_inode_force_dio(inode)) return fuse_direct_write_iter(iocb, from); else if (fuse_file_passthrough(ff)) return fuse_passthrough_write_iter(iocb, from); @@ -2114,19 +2280,15 @@ static void fuse_writepage_finish(struct fuse_writepage_args *wpa) struct fuse_args_pages *ap = &wpa->ia.ap; struct inode *inode = wpa->inode; struct fuse_inode *fi = get_fuse_inode(inode); - struct backing_dev_info *bdi = inode_to_bdi(inode); int i; - for (i = 0; i < ap->num_folios; i++) { + for (i = 0; i < ap->num_folios; i++) /* * Benchmarks showed that ending writeback within the * scope of the fi->lock alleviates xarray lock * contention and noticeably improves performance. */ iomap_finish_folio_write(inode, ap->folios[i], 1); - dec_wb_stat(&bdi->wb, WB_WRITEBACK); - wb_writeout_inc(&bdi->wb); - } wake_up(&fi->page_waitq); } @@ -2301,14 +2463,11 @@ static void fuse_writepage_add_to_bucket(struct fuse_conn *fc, static void fuse_writepage_args_page_fill(struct fuse_writepage_args *wpa, struct folio *folio, uint32_t folio_index, loff_t offset, unsigned len) { - struct inode *inode = folio->mapping->host; struct fuse_args_pages *ap = &wpa->ia.ap; ap->folios[folio_index] = folio; ap->descs[folio_index].offset = offset; ap->descs[folio_index].length = len; - - inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK); } static struct fuse_writepage_args *fuse_writepage_args_setup(struct folio *folio, @@ -2691,6 +2850,29 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) else if (fuse_inode_backing(get_fuse_inode(inode))) return -ENODEV; + /* + * If the inode was latched into forced direct IO after a remote-modify + * notification, a mapping needs the page cache, so revert to caching + * mode. Revert without the inode lock or wb_inval_rwsem: ->mmap runs + * under mmap_lock and the buffered write path holds both across a fault + * on the user buffer (which takes mmap_lock), so taking either here + * would invert lock order (ABBA). Clearing the latch and dropping the + * cache is sufficient -- writers re-check the latch and route to cached + * IO once it is clear, and in-flight parallel dio drains itself. Cached + * opens frozen while latched are still counted in iocachectr, so restore + * FUSE_I_CACHE_IO_MODE for them. + */ + if (fuse_inode_force_dio(inode)) { + struct fuse_inode *fi = get_fuse_inode(inode); + + spin_lock(&fi->lock); + clear_bit(FUSE_I_FORCE_DIO, &fi->state); + if (fi->iocachectr > 0) + set_bit(FUSE_I_CACHE_IO_MODE, &fi->state); + spin_unlock(&fi->lock); + invalidate_inode_pages2(file->f_mapping); + } + /* * FOPEN_DIRECT_IO handling is special compared to O_DIRECT, * as does not allow MAP_SHARED mmap without FUSE_DIRECT_IO_ALLOW_MMAP. @@ -3119,7 +3301,7 @@ static inline loff_t fuse_round_up(struct fuse_conn *fc, loff_t off) } static ssize_t -fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +__fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter, bool exclusive) { DECLARE_COMPLETION_ONSTACK(wait); ssize_t ret = 0; @@ -3213,14 +3395,27 @@ fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter) if (iov_iter_rw(iter) == WRITE) { fuse_write_update_attr(inode, pos, ret); - /* For extending writes we already hold exclusive lock */ - if (ret < 0 && offset + count > i_size) + /* + * Whole-file rollback is only safe under an exclusive lock. + * Parallel writers commit i_size only on success (nothing to + * undo); the server owns failed-extend cleanup. + */ + if (exclusive && ret < 0 && offset + count > i_size) fuse_do_truncate(file); } return ret; } +static ssize_t fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +{ + /* + * Only reached via generic_file_direct_write/read() + * (caching-mode O_DIRECT), which holds the inode lock exclusively. + */ + return __fuse_direct_IO(iocb, iter, true); +} + static int fuse_writeback_range(struct inode *inode, loff_t start, loff_t end) { int err = filemap_write_and_wait_range(inode->i_mapping, start, LLONG_MAX); @@ -3502,10 +3697,27 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) fi->iocachectr = 0; init_waitqueue_head(&fi->page_waitq); init_waitqueue_head(&fi->direct_io_waitq); + init_rwsem(&fi->wb_inval_rwsem); + fi->notify_stamp = jiffies; + fi->notify_interval_ewma = FUSE_NOTIFY_EWMA_SEED << FUSE_NOTIFY_EWMA_SHIFT; if (IS_ENABLED(CONFIG_FUSE_DAX)) fuse_dax_inode_init(inode, flags); - if (enable_large_folios) - mapping_set_large_folios(inode->i_mapping); + if (enable_large_folios) { + /* + * Readahead and writeback batch whole folios into a single + * request, capped at min(fc->max_pages, fc->max_read/PAGE_SIZE) + * pages. The page cache must therefore never build a folio + * larger than that, or fuse_readahead() trips WARN_ON(!pages) + * and then dereferences a NULL ap->folios[0] in + * fuse_send_readpages(). Bound the folio order to the request + * limit instead of MAX_PAGECACHE_ORDER. + */ + unsigned int max_pages = min(fc->max_pages, + fc->max_read >> PAGE_SHIFT); + + mapping_set_folio_order_range(inode->i_mapping, 0, + ilog2(max_pages ?: 1)); + } } diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 365c28cb282146..dfa11e62cc00bd 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -119,6 +119,20 @@ struct dlm_locked_area size_t size; }; +/* + * Force-DIO switch trigger: an exponentially weighted moving average of the + * interval (in jiffies) between FUSE_NOTIFY_INVAL_INODE data invalidations for + * a file. When the average spacing falls below FUSE_NOTIFY_DIO_INTERVAL -- a + * remote writer streaming invalidations -- and the file is open for writing + * here, it is latched into direct IO. These are the source-level (not + * externally tunable) parameters of the heuristic: EWMA weight 1/2^SHIFT, + * seeded and capped at SEED so it takes a short burst rather than a single + * notify to trip. + */ +#define FUSE_NOTIFY_DIO_INTERVAL max_t(unsigned long, HZ / 10, 1) +#define FUSE_NOTIFY_EWMA_SHIFT 2 +#define FUSE_NOTIFY_EWMA_SEED (2 * FUSE_NOTIFY_DIO_INTERVAL) + /** FUSE inode */ struct fuse_inode { /** Inode data */ @@ -186,6 +200,34 @@ struct fuse_inode { /* dlm locked areas we have sent lock requests for */ struct fuse_dlm_cache dlm_locked_areas; + + /* + * Serializes buffered-write page-cache dirtying against + * the forced-direct-IO latch transition driven by + * NOTIFY_INVAL_INODE (fuse_reverse_inval_inode()), which + * may be delivered by the same server thread that still + * owes a reply to an in-flight write holding the inode + * lock. The buffered writer holds this for read around + * the dirtying and re-checks the latch under it; the + * NOTIFY latch site takes it for write (trylock, never + * blocking) around its page-cache invalidate + latch set. + * Only regular files initialise it -- it shares storage + * with the readdir-cache union arm. + */ + struct rw_semaphore wb_inval_rwsem; + + /* + * Rate of FUSE_NOTIFY_INVAL_INODE data invalidations + * for this whole file: notify_stamp is the jiffies of + * the last one, notify_interval_ewma the EWMA of the + * inter-arrival interval (jiffies, scaled by + * 2^FUSE_NOTIFY_EWMA_SHIFT). A rapid stream (short + * average interval) with a local writer latches the + * inode into direct IO. Protected by fi->lock; regular + * files only (shares the readdir-cache union arm). + */ + unsigned long notify_stamp; + unsigned int notify_interval_ewma; }; /* readdir cache (directory only) */ @@ -257,6 +299,14 @@ enum { FUSE_I_BTIME, /* Wants or already has page cache IO */ FUSE_I_CACHE_IO_MODE, + /* + * Latched into direct IO: a NOTIFY_INVAL_INODE arrived while the file + * was open for writing here, so another (remote) entity is modifying it + * concurrently. Reads and writes are routed direct (shared-lock + * parallel dio) until the last writer closes or the inode is mmapped. + * See fuse_reverse_inval_inode()/fuse_file_io_open(). + */ + FUSE_I_FORCE_DIO, }; struct fuse_conn; @@ -1600,6 +1650,12 @@ void fuse_inode_uncached_io_end(struct fuse_inode *fi); int fuse_file_io_open(struct file *file, struct inode *inode); void fuse_file_io_release(struct fuse_file *ff, struct inode *inode); +/* Inode latched into forced direct IO after a remote-modify notification */ +static inline bool fuse_inode_force_dio(struct inode *inode) +{ + return test_bit(FUSE_I_FORCE_DIO, &get_fuse_inode(inode)->state); +} + /* file.c */ struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid, struct inode *inode, diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 6a94de9528210f..e3703f35079676 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -757,6 +757,35 @@ static void fuse_invalidate_inode_entry(struct inode *inode) } } +/* + * Fold one FUSE_NOTIFY_INVAL_INODE data invalidation into the per-inode + * moving average of the notification inter-arrival interval and report whether + * the file is now "hot" -- notifications are arriving fast enough (short + * average interval) that a remote writer is repeatedly invalidating it. The + * average is an EWMA (weight 1/2^FUSE_NOTIFY_EWMA_SHIFT); the sample is clamped + * to FUSE_NOTIFY_EWMA_SEED so a notify after a long idle only cools the average + * and cannot overflow the accumulator. Must be called under fi->lock; called + * for every data invalidation so the average stays current even while no local + * writer is open. + */ +static bool fuse_notify_inval_hot(struct fuse_inode *fi) +{ + unsigned long now = jiffies; + unsigned long sample; + unsigned int avg; + + sample = min_t(unsigned long, now - fi->notify_stamp, + FUSE_NOTIFY_EWMA_SEED); + fi->notify_stamp = now; + + /* E += sample - (E >> SHIFT); avg = E >> SHIFT */ + fi->notify_interval_ewma += sample - + (fi->notify_interval_ewma >> FUSE_NOTIFY_EWMA_SHIFT); + avg = fi->notify_interval_ewma >> FUSE_NOTIFY_EWMA_SHIFT; + + return avg < FUSE_NOTIFY_DIO_INTERVAL; +} + int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, loff_t offset, loff_t len) { @@ -806,8 +835,73 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, pg_end == -1 ? 0 : (offset + len - 1)); - invalidate_inode_pages2_range(inode->i_mapping, - pg_start, pg_end); + /* + * A data invalidation means another (remote) entity is modifying + * the file. Keep a moving average of how fast these notifications + * arrive for the whole inode; when they come in a rapid stream -- + * a remote writer repeatedly invalidating the file -- and it is + * also open for writing here, latch the inode into direct IO: + * reads and writes are served direct (from the server) until the + * last writer closes or the inode is mmapped. A lone or occasional + * notify keeps the average high and does not trip the switch. Only + * under writeback+dlm, where the buffered write's RMW read is + * skipped so its wb_inval_rwsem read-side section is free of server + * round-trips. + * + * fuse_notify_inval_hot() updates the average under fi->lock and is + * called for every data invalidation so it stays current even while + * no local writer is open. When it trips, take wb_inval_rwsem for + * write so the buffered write path -- which holds it for read across + * its dirtying and re-checks the latch under it -- cannot strand + * dirty folios after the cache is dropped. Use a trylock and never + * block: this may run on the server thread that still owes an + * in-flight write (holding the inode lock) its reply, so blocking on + * the rwsem or the inode lock would deadlock. If the writer has + * gone, skip the latch this round (best effort); the invalidate + * still runs. Only regular files initialise the average and the + * rwsem (they share storage with the readdir-cache union arm), so + * gate on S_ISREG. When latched, drop the whole mapping rather than + * just the notified range, or dirty folios outside it would be + * invisible to the forced direct reads (stale read / lost write). + */ + if (S_ISREG(inode->i_mode) && fc->writeback_cache && fc->dlm && + !FUSE_IS_DAX(inode) && !fuse_inode_backing(fi) && + !mapping_mapped(inode->i_mapping)) { + bool hot, has_writer; + + spin_lock(&fi->lock); + hot = fuse_notify_inval_hot(fi); + has_writer = !list_empty(&fi->write_files); + spin_unlock(&fi->lock); + + if (hot && has_writer && !fuse_inode_force_dio(inode) && + down_write_trylock(&fi->wb_inval_rwsem)) { + bool latched = false; + + spin_lock(&fi->lock); + if (!list_empty(&fi->write_files)) { + set_bit(FUSE_I_FORCE_DIO, &fi->state); + latched = true; + } + spin_unlock(&fi->lock); + + if (latched) { + pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", + nodeid); + invalidate_inode_pages2(inode->i_mapping); + } else { + invalidate_inode_pages2_range(inode->i_mapping, + pg_start, pg_end); + } + up_write(&fi->wb_inval_rwsem); + } else { + invalidate_inode_pages2_range(inode->i_mapping, + pg_start, pg_end); + } + } else { + invalidate_inode_pages2_range(inode->i_mapping, + pg_start, pg_end); + } } iput(inode); return 0; @@ -1834,7 +1928,7 @@ static int fuse_bdi_init(struct fuse_conn *fc, struct super_block *sb) /* fuse does it's own writeback accounting */ sb->s_bdi->capabilities &= ~BDI_CAP_WRITEBACK_ACCT; - sb->s_bdi->capabilities |= BDI_CAP_STRICTLIMIT; + sb->s_bdi->capabilities &= ~BDI_CAP_STRICTLIMIT; /* * For a single fuse filesystem use max 1% of dirty + diff --git a/fs/fuse/iomode.c b/fs/fuse/iomode.c index c99e285f3183ef..301294c8fb25f0 100644 --- a/fs/fuse/iomode.c +++ b/fs/fuse/iomode.c @@ -233,6 +233,16 @@ int fuse_file_io_open(struct file *file, struct inode *inode) !(ff->open_flags & FOPEN_PASSTHROUGH)) return 0; + /* + * The inode was latched into direct IO after a remote-modify + * notification arrived while it was open for writing here. Open this + * file uncached as well so its IO is routed direct and it does not + * re-enter caching mode. + */ + if (test_bit(FUSE_I_FORCE_DIO, &fi->state) && + !(ff->open_flags & FOPEN_PASSTHROUGH)) + return 0; + if (ff->open_flags & FOPEN_PASSTHROUGH) err = fuse_file_passthrough_open(inode, file); else diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h index e721148c95d07d..9a1e895dd5df1b 100644 --- a/include/linux/backing-dev.h +++ b/include/linux/backing-dev.h @@ -66,16 +66,6 @@ static inline void wb_stat_mod(struct bdi_writeback *wb, percpu_counter_add_batch(&wb->stat[item], amount, WB_STAT_BATCH); } -static inline void inc_wb_stat(struct bdi_writeback *wb, enum wb_stat_item item) -{ - wb_stat_mod(wb, item, 1); -} - -static inline void dec_wb_stat(struct bdi_writeback *wb, enum wb_stat_item item) -{ - wb_stat_mod(wb, item, -1); -} - static inline s64 wb_stat(struct bdi_writeback *wb, enum wb_stat_item item) { return percpu_counter_read_positive(&wb->stat[item]);