blob: 9a0d32e4b422ad09518a6c6143638d0c68fb8b84 [file] [log] [blame]
// SPDX-License-Identifier: GPL-2.0-only
/* Network filesystem high-level write support.
*
* Copyright (C) 2023 Red Hat, Inc. All Rights Reserved.
* Written by David Howells (dhowells@redhat.com)
*/
#include <linux/export.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/pagemap.h>
#include <linux/slab.h>
#include <linux/pagevec.h>
#include "internal.h"
/*
* Determined write method. Adjust netfs_folio_traces if this is changed.
*/
enum netfs_how_to_modify {
NETFS_FOLIO_IS_UPTODATE, /* Folio is uptodate already */
NETFS_JUST_PREFETCH, /* We have to read the folio anyway */
NETFS_WHOLE_FOLIO_MODIFY, /* We're going to overwrite the whole folio */
NETFS_MODIFY_AND_CLEAR, /* We can assume there is no data to be downloaded. */
NETFS_STREAMING_WRITE, /* Store incomplete data in non-uptodate page. */
NETFS_STREAMING_WRITE_CONT, /* Continue streaming write. */
NETFS_FLUSH_CONTENT, /* Flush incompatible content. */
};
static void netfs_cleanup_buffered_write(struct netfs_io_request *wreq);
static void netfs_set_group(struct folio *folio, struct netfs_group *netfs_group)
{
if (netfs_group && !folio_get_private(folio))
folio_attach_private(folio, netfs_get_group(netfs_group));
}
#if IS_ENABLED(CONFIG_FSCACHE)
static void netfs_folio_start_fscache(bool caching, struct folio *folio)
{
if (caching)
folio_start_fscache(folio);
}
#else
static void netfs_folio_start_fscache(bool caching, struct folio *folio)
{
}
#endif
/*
* Decide how we should modify a folio. We might be attempting to do
* write-streaming, in which case we don't want to a local RMW cycle if we can
* avoid it. If we're doing local caching or content crypto, we award that
* priority over avoiding RMW. If the file is open readably, then we also
* assume that we may want to read what we wrote.
*/
static enum netfs_how_to_modify netfs_how_to_modify(struct netfs_inode *ctx,
struct file *file,
struct folio *folio,
void *netfs_group,
size_t flen,
size_t offset,
size_t len,
bool maybe_trouble)
{
struct netfs_folio *finfo = netfs_folio_info(folio);
loff_t pos = folio_file_pos(folio);
_enter("");
if (netfs_folio_group(folio) != netfs_group)
return NETFS_FLUSH_CONTENT;
if (folio_test_uptodate(folio))
return NETFS_FOLIO_IS_UPTODATE;
if (pos >= ctx->zero_point)
return NETFS_MODIFY_AND_CLEAR;
if (!maybe_trouble && offset == 0 && len >= flen)
return NETFS_WHOLE_FOLIO_MODIFY;
if (file->f_mode & FMODE_READ)
goto no_write_streaming;
if (test_bit(NETFS_ICTX_NO_WRITE_STREAMING, &ctx->flags))
goto no_write_streaming;
if (netfs_is_cache_enabled(ctx)) {
/* We don't want to get a streaming write on a file that loses
* caching service temporarily because the backing store got
* culled.
*/
if (!test_bit(NETFS_ICTX_NO_WRITE_STREAMING, &ctx->flags))
set_bit(NETFS_ICTX_NO_WRITE_STREAMING, &ctx->flags);
goto no_write_streaming;
}
if (!finfo)
return NETFS_STREAMING_WRITE;
/* We can continue a streaming write only if it continues on from the
* previous. If it overlaps, we must flush lest we suffer a partial
* copy and disjoint dirty regions.
*/
if (offset == finfo->dirty_offset + finfo->dirty_len)
return NETFS_STREAMING_WRITE_CONT;
return NETFS_FLUSH_CONTENT;
no_write_streaming:
if (finfo) {
netfs_stat(&netfs_n_wh_wstream_conflict);
return NETFS_FLUSH_CONTENT;
}
return NETFS_JUST_PREFETCH;
}
/*
* Grab a folio for writing and lock it. Attempt to allocate as large a folio
* as possible to hold as much of the remaining length as possible in one go.
*/
static struct folio *netfs_grab_folio_for_write(struct address_space *mapping,
loff_t pos, size_t part)
{
pgoff_t index = pos / PAGE_SIZE;
fgf_t fgp_flags = FGP_WRITEBEGIN;
if (mapping_large_folio_support(mapping))
fgp_flags |= fgf_set_order(pos % PAGE_SIZE + part);
return __filemap_get_folio(mapping, index, fgp_flags,
mapping_gfp_mask(mapping));
}
/**
* netfs_perform_write - Copy data into the pagecache.
* @iocb: The operation parameters
* @iter: The source buffer
* @netfs_group: Grouping for dirty pages (eg. ceph snaps).
*
* Copy data into pagecache pages attached to the inode specified by @iocb.
* The caller must hold appropriate inode locks.
*
* Dirty pages are tagged with a netfs_folio struct if they're not up to date
* to indicate the range modified. Dirty pages may also be tagged with a
* netfs-specific grouping such that data from an old group gets flushed before
* a new one is started.
*/
ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
struct netfs_group *netfs_group)
{
struct file *file = iocb->ki_filp;
struct inode *inode = file_inode(file);
struct address_space *mapping = inode->i_mapping;
struct netfs_inode *ctx = netfs_inode(inode);
struct writeback_control wbc = {
.sync_mode = WB_SYNC_NONE,
.for_sync = true,
.nr_to_write = LONG_MAX,
.range_start = iocb->ki_pos,
.range_end = iocb->ki_pos + iter->count,
};
struct netfs_io_request *wreq = NULL;
struct netfs_folio *finfo;
struct folio *folio;
enum netfs_how_to_modify howto;
enum netfs_folio_trace trace;
unsigned int bdp_flags = (iocb->ki_flags & IOCB_SYNC) ? 0: BDP_ASYNC;
ssize_t written = 0, ret;
loff_t i_size, pos = iocb->ki_pos, from, to;
size_t max_chunk = PAGE_SIZE << MAX_PAGECACHE_ORDER;
bool maybe_trouble = false;
if (unlikely(test_bit(NETFS_ICTX_WRITETHROUGH, &ctx->flags) ||
iocb->ki_flags & (IOCB_DSYNC | IOCB_SYNC))
) {
if (pos < i_size_read(inode)) {
ret = filemap_write_and_wait_range(mapping, pos, pos + iter->count);
if (ret < 0) {
goto out;
}
}
wbc_attach_fdatawrite_inode(&wbc, mapping->host);
wreq = netfs_begin_writethrough(iocb, iter->count);
if (IS_ERR(wreq)) {
wbc_detach_inode(&wbc);
ret = PTR_ERR(wreq);
wreq = NULL;
goto out;
}
if (!is_sync_kiocb(iocb))
wreq->iocb = iocb;
wreq->cleanup = netfs_cleanup_buffered_write;
}
do {
size_t flen;
size_t offset; /* Offset into pagecache folio */
size_t part; /* Bytes to write to folio */
size_t copied; /* Bytes copied from user */
ret = balance_dirty_pages_ratelimited_flags(mapping, bdp_flags);
if (unlikely(ret < 0))
break;
offset = pos & (max_chunk - 1);
part = min(max_chunk - offset, iov_iter_count(iter));
/* Bring in the user pages that we will copy from _first_ lest
* we hit a nasty deadlock on copying from the same page as
* we're writing to, without it being marked uptodate.
*
* Not only is this an optimisation, but it is also required to
* check that the address is actually valid, when atomic
* usercopies are used below.
*
* We rely on the page being held onto long enough by the LRU
* that we can grab it below if this causes it to be read.
*/
ret = -EFAULT;
if (unlikely(fault_in_iov_iter_readable(iter, part) == part))
break;
folio = netfs_grab_folio_for_write(mapping, pos, part);
if (IS_ERR(folio)) {
ret = PTR_ERR(folio);
break;
}
flen = folio_size(folio);
offset = pos & (flen - 1);
part = min_t(size_t, flen - offset, part);
if (signal_pending(current)) {
ret = written ? -EINTR : -ERESTARTSYS;
goto error_folio_unlock;
}
/* See if we need to prefetch the area we're going to modify.
* We need to do this before we get a lock on the folio in case
* there's more than one writer competing for the same cache
* block.
*/
howto = netfs_how_to_modify(ctx, file, folio, netfs_group,
flen, offset, part, maybe_trouble);
_debug("howto %u", howto);
switch (howto) {
case NETFS_JUST_PREFETCH:
ret = netfs_prefetch_for_write(file, folio, offset, part);
if (ret < 0) {
_debug("prefetch = %zd", ret);
goto error_folio_unlock;
}
break;
case NETFS_FOLIO_IS_UPTODATE:
case NETFS_WHOLE_FOLIO_MODIFY:
case NETFS_STREAMING_WRITE_CONT:
break;
case NETFS_MODIFY_AND_CLEAR:
zero_user_segment(&folio->page, 0, offset);
break;
case NETFS_STREAMING_WRITE:
ret = -EIO;
if (WARN_ON(folio_get_private(folio)))
goto error_folio_unlock;
break;
case NETFS_FLUSH_CONTENT:
trace_netfs_folio(folio, netfs_flush_content);
from = folio_pos(folio);
to = from + folio_size(folio) - 1;
folio_unlock(folio);
folio_put(folio);
ret = filemap_write_and_wait_range(mapping, from, to);
if (ret < 0)
goto error_folio_unlock;
continue;
}
if (mapping_writably_mapped(mapping))
flush_dcache_folio(folio);
copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
flush_dcache_folio(folio);
/* Deal with a (partially) failed copy */
if (copied == 0) {
ret = -EFAULT;
goto error_folio_unlock;
}
trace = (enum netfs_folio_trace)howto;
switch (howto) {
case NETFS_FOLIO_IS_UPTODATE:
case NETFS_JUST_PREFETCH:
netfs_set_group(folio, netfs_group);
break;
case NETFS_MODIFY_AND_CLEAR:
zero_user_segment(&folio->page, offset + copied, flen);
netfs_set_group(folio, netfs_group);
folio_mark_uptodate(folio);
break;
case NETFS_WHOLE_FOLIO_MODIFY:
if (unlikely(copied < part)) {
maybe_trouble = true;
iov_iter_revert(iter, copied);
copied = 0;
goto retry;
}
netfs_set_group(folio, netfs_group);
folio_mark_uptodate(folio);
break;
case NETFS_STREAMING_WRITE:
if (offset == 0 && copied == flen) {
netfs_set_group(folio, netfs_group);
folio_mark_uptodate(folio);
trace = netfs_streaming_filled_page;
break;
}
finfo = kzalloc(sizeof(*finfo), GFP_KERNEL);
if (!finfo) {
iov_iter_revert(iter, copied);
ret = -ENOMEM;
goto error_folio_unlock;
}
finfo->netfs_group = netfs_get_group(netfs_group);
finfo->dirty_offset = offset;
finfo->dirty_len = copied;
folio_attach_private(folio, (void *)((unsigned long)finfo |
NETFS_FOLIO_INFO));
break;
case NETFS_STREAMING_WRITE_CONT:
finfo = netfs_folio_info(folio);
finfo->dirty_len += copied;
if (finfo->dirty_offset == 0 && finfo->dirty_len == flen) {
if (finfo->netfs_group)
folio_change_private(folio, finfo->netfs_group);
else
folio_detach_private(folio);
folio_mark_uptodate(folio);
kfree(finfo);
trace = netfs_streaming_cont_filled_page;
}
break;
default:
WARN(true, "Unexpected modify type %u ix=%lx\n",
howto, folio->index);
ret = -EIO;
goto error_folio_unlock;
}
trace_netfs_folio(folio, trace);
/* Update the inode size if we moved the EOF marker */
i_size = i_size_read(inode);
pos += copied;
if (pos > i_size) {
if (ctx->ops->update_i_size) {
ctx->ops->update_i_size(inode, pos);
} else {
i_size_write(inode, pos);
#if IS_ENABLED(CONFIG_FSCACHE)
fscache_update_cookie(ctx->cache, NULL, &pos);
#endif
}
}
written += copied;
if (likely(!wreq)) {
folio_mark_dirty(folio);
} else {
if (folio_test_dirty(folio))
/* Sigh. mmap. */
folio_clear_dirty_for_io(folio);
/* We make multiple writes to the folio... */
if (!folio_test_writeback(folio)) {
folio_wait_fscache(folio);
folio_start_writeback(folio);
folio_start_fscache(folio);
if (wreq->iter.count == 0)
trace_netfs_folio(folio, netfs_folio_trace_wthru);
else
trace_netfs_folio(folio, netfs_folio_trace_wthru_plus);
}
netfs_advance_writethrough(wreq, copied,
offset + copied == flen);
}
retry:
folio_unlock(folio);
folio_put(folio);
folio = NULL;
cond_resched();
} while (iov_iter_count(iter));
out:
if (unlikely(wreq)) {
ret = netfs_end_writethrough(wreq, iocb);
wbc_detach_inode(&wbc);
if (ret == -EIOCBQUEUED)
return ret;
}
iocb->ki_pos += written;
_leave(" = %zd [%zd]", written, ret);
return written ? written : ret;
error_folio_unlock:
folio_unlock(folio);
folio_put(folio);
goto out;
}
EXPORT_SYMBOL(netfs_perform_write);
/**
* netfs_buffered_write_iter_locked - write data to a file
* @iocb: IO state structure (file, offset, etc.)
* @from: iov_iter with data to write
* @netfs_group: Grouping for dirty pages (eg. ceph snaps).
*
* This function does all the work needed for actually writing data to a
* file. It does all basic checks, removes SUID from the file, updates
* modification times and calls proper subroutines depending on whether we
* do direct IO or a standard buffered write.
*
* The caller must hold appropriate locks around this function and have called
* generic_write_checks() already. The caller is also responsible for doing
* any necessary syncing afterwards.
*
* This function does *not* take care of syncing data in case of O_SYNC write.
* A caller has to handle it. This is mainly due to the fact that we want to
* avoid syncing under i_rwsem.
*
* Return:
* * number of bytes written, even for truncated writes
* * negative error code if no data has been written at all
*/
ssize_t netfs_buffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *from,
struct netfs_group *netfs_group)
{
struct file *file = iocb->ki_filp;
ssize_t ret;
trace_netfs_write_iter(iocb, from);
ret = file_remove_privs(file);
if (ret)
return ret;
ret = file_update_time(file);
if (ret)
return ret;
return netfs_perform_write(iocb, from, netfs_group);
}
EXPORT_SYMBOL(netfs_buffered_write_iter_locked);
/**
* netfs_file_write_iter - write data to a file
* @iocb: IO state structure
* @from: iov_iter with data to write
*
* Perform a write to a file, writing into the pagecache if possible and doing
* an unbuffered write instead if not.
*
* Return:
* * Negative error code if no data has been written at all of
* vfs_fsync_range() failed for a synchronous write
* * Number of bytes written, even for truncated writes
*/
ssize_t netfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
{
struct file *file = iocb->ki_filp;
struct inode *inode = file->f_mapping->host;
struct netfs_inode *ictx = netfs_inode(inode);
ssize_t ret;
_enter("%llx,%zx,%llx", iocb->ki_pos, iov_iter_count(from), i_size_read(inode));
if (!iov_iter_count(from))
return 0;
if ((iocb->ki_flags & IOCB_DIRECT) ||
test_bit(NETFS_ICTX_UNBUFFERED, &ictx->flags))
return netfs_unbuffered_write_iter(iocb, from);
ret = netfs_start_io_write(inode);
if (ret < 0)
return ret;
ret = generic_write_checks(iocb, from);
if (ret > 0)
ret = netfs_buffered_write_iter_locked(iocb, from, NULL);
netfs_end_io_write(inode);
if (ret > 0)
ret = generic_write_sync(iocb, ret);
return ret;
}
EXPORT_SYMBOL(netfs_file_write_iter);
/*
* Notification that a previously read-only page is about to become writable.
* Note that the caller indicates a single page of a multipage folio.
*/
vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_group)
{
struct folio *folio = page_folio(vmf->page);
struct file *file = vmf->vma->vm_file;
struct inode *inode = file_inode(file);
vm_fault_t ret = VM_FAULT_RETRY;
int err;
_enter("%lx", folio->index);
sb_start_pagefault(inode->i_sb);
if (folio_wait_writeback_killable(folio))
goto out;
if (folio_lock_killable(folio) < 0)
goto out;
/* Can we see a streaming write here? */
if (WARN_ON(!folio_test_uptodate(folio))) {
ret = VM_FAULT_SIGBUS | VM_FAULT_LOCKED;
goto out;
}
if (netfs_folio_group(folio) != netfs_group) {
folio_unlock(folio);
err = filemap_fdatawait_range(inode->i_mapping,
folio_pos(folio),
folio_pos(folio) + folio_size(folio));
switch (err) {
case 0:
ret = VM_FAULT_RETRY;
goto out;
case -ENOMEM:
ret = VM_FAULT_OOM;
goto out;
default:
ret = VM_FAULT_SIGBUS;
goto out;
}
}
if (folio_test_dirty(folio))
trace_netfs_folio(folio, netfs_folio_trace_mkwrite_plus);
else
trace_netfs_folio(folio, netfs_folio_trace_mkwrite);
netfs_set_group(folio, netfs_group);
file_update_time(file);
ret = VM_FAULT_LOCKED;
out:
sb_end_pagefault(inode->i_sb);
return ret;
}
EXPORT_SYMBOL(netfs_page_mkwrite);
/*
* Kill all the pages in the given range
*/
static void netfs_kill_pages(struct address_space *mapping,
loff_t start, loff_t len)
{
struct folio *folio;
pgoff_t index = start / PAGE_SIZE;
pgoff_t last = (start + len - 1) / PAGE_SIZE, next;
_enter("%llx-%llx", start, start + len - 1);
do {
_debug("kill %lx (to %lx)", index, last);
folio = filemap_get_folio(mapping, index);
if (IS_ERR(folio)) {
next = index + 1;
continue;
}
next = folio_next_index(folio);
trace_netfs_folio(folio, netfs_folio_trace_kill);
folio_clear_uptodate(folio);
if (folio_test_fscache(folio))
folio_end_fscache(folio);
folio_end_writeback(folio);
folio_lock(folio);
generic_error_remove_folio(mapping, folio);
folio_unlock(folio);
folio_put(folio);
} while (index = next, index <= last);
_leave("");
}
/*
* Redirty all the pages in a given range.
*/
static void netfs_redirty_pages(struct address_space *mapping,
loff_t start, loff_t len)
{
struct folio *folio;
pgoff_t index = start / PAGE_SIZE;
pgoff_t last = (start + len - 1) / PAGE_SIZE, next;
_enter("%llx-%llx", start, start + len - 1);
do {
_debug("redirty %llx @%llx", len, start);
folio = filemap_get_folio(mapping, index);
if (IS_ERR(folio)) {
next = index + 1;
continue;
}
next = folio_next_index(folio);
trace_netfs_folio(folio, netfs_folio_trace_redirty);
filemap_dirty_folio(mapping, folio);
if (folio_test_fscache(folio))
folio_end_fscache(folio);
folio_end_writeback(folio);
folio_put(folio);
} while (index = next, index <= last);
balance_dirty_pages_ratelimited(mapping);
_leave("");
}
/*
* Completion of write to server
*/
static void netfs_pages_written_back(struct netfs_io_request *wreq)
{
struct address_space *mapping = wreq->mapping;
struct netfs_folio *finfo;
struct netfs_group *group = NULL;
struct folio *folio;
pgoff_t last;
int gcount = 0;
XA_STATE(xas, &mapping->i_pages, wreq->start / PAGE_SIZE);
_enter("%llx-%llx", wreq->start, wreq->start + wreq->len);
rcu_read_lock();
last = (wreq->start + wreq->len - 1) / PAGE_SIZE;
xas_for_each(&xas, folio, last) {
WARN(!folio_test_writeback(folio),
"bad %zx @%llx page %lx %lx\n",
wreq->len, wreq->start, folio->index, last);
if ((finfo = netfs_folio_info(folio))) {
/* Streaming writes cannot be redirtied whilst under
* writeback, so discard the streaming record.
*/
folio_detach_private(folio);
group = finfo->netfs_group;
gcount++;
trace_netfs_folio(folio, netfs_folio_trace_clear_s);
kfree(finfo);
} else if ((group = netfs_folio_group(folio))) {
/* Need to detach the group pointer if the page didn't
* get redirtied. If it has been redirtied, then it
* must be within the same group.
*/
if (folio_test_dirty(folio)) {
trace_netfs_folio(folio, netfs_folio_trace_redirtied);
goto end_wb;
}
if (folio_trylock(folio)) {
if (!folio_test_dirty(folio)) {
folio_detach_private(folio);
gcount++;
trace_netfs_folio(folio, netfs_folio_trace_clear_g);
} else {
trace_netfs_folio(folio, netfs_folio_trace_redirtied);
}
folio_unlock(folio);
goto end_wb;
}
xas_pause(&xas);
rcu_read_unlock();
folio_lock(folio);
if (!folio_test_dirty(folio)) {
folio_detach_private(folio);
gcount++;
trace_netfs_folio(folio, netfs_folio_trace_clear_g);
} else {
trace_netfs_folio(folio, netfs_folio_trace_redirtied);
}
folio_unlock(folio);
rcu_read_lock();
} else {
trace_netfs_folio(folio, netfs_folio_trace_clear);
}
end_wb:
if (folio_test_fscache(folio))
folio_end_fscache(folio);
xas_advance(&xas, folio_next_index(folio) - 1);
folio_end_writeback(folio);
}
rcu_read_unlock();
netfs_put_group_many(group, gcount);
_leave("");
}
/*
* Deal with the disposition of the folios that are under writeback to close
* out the operation.
*/
static void netfs_cleanup_buffered_write(struct netfs_io_request *wreq)
{
struct address_space *mapping = wreq->mapping;
_enter("");
switch (wreq->error) {
case 0:
netfs_pages_written_back(wreq);
break;
default:
pr_notice("R=%08x Unexpected error %d\n", wreq->debug_id, wreq->error);
fallthrough;
case -EACCES:
case -EPERM:
case -ENOKEY:
case -EKEYEXPIRED:
case -EKEYREJECTED:
case -EKEYREVOKED:
case -ENETRESET:
case -EDQUOT:
case -ENOSPC:
netfs_redirty_pages(mapping, wreq->start, wreq->len);
break;
case -EROFS:
case -EIO:
case -EREMOTEIO:
case -EFBIG:
case -ENOENT:
case -ENOMEDIUM:
case -ENXIO:
netfs_kill_pages(mapping, wreq->start, wreq->len);
break;
}
if (wreq->error)
mapping_set_error(mapping, wreq->error);
if (wreq->netfs_ops->done)
wreq->netfs_ops->done(wreq);
}
/*
* Extend the region to be written back to include subsequent contiguously
* dirty pages if possible, but don't sleep while doing so.
*
* If this page holds new content, then we can include filler zeros in the
* writeback.
*/
static void netfs_extend_writeback(struct address_space *mapping,
struct netfs_group *group,
struct xa_state *xas,
long *_count,
loff_t start,
loff_t max_len,
bool caching,
size_t *_len,
size_t *_top)
{
struct netfs_folio *finfo;
struct folio_batch fbatch;
struct folio *folio;
unsigned int i;
pgoff_t index = (start + *_len) / PAGE_SIZE;
size_t len;
void *priv;
bool stop = true;
folio_batch_init(&fbatch);
do {
/* Firstly, we gather up a batch of contiguous dirty pages
* under the RCU read lock - but we can't clear the dirty flags
* there if any of those pages are mapped.
*/
rcu_read_lock();
xas_for_each(xas, folio, ULONG_MAX) {
stop = true;
if (xas_retry(xas, folio))
continue;
if (xa_is_value(folio))
break;
if (folio->index != index) {
xas_reset(xas);
break;
}
if (!folio_try_get_rcu(folio)) {
xas_reset(xas);
continue;
}
/* Has the folio moved or been split? */
if (unlikely(folio != xas_reload(xas))) {
folio_put(folio);
xas_reset(xas);
break;
}
if (!folio_trylock(folio)) {
folio_put(folio);
xas_reset(xas);
break;
}
if (!folio_test_dirty(folio) ||
folio_test_writeback(folio) ||
folio_test_fscache(folio)) {
folio_unlock(folio);
folio_put(folio);
xas_reset(xas);
break;
}
stop = false;
len = folio_size(folio);
priv = folio_get_private(folio);
if ((const struct netfs_group *)priv != group) {
stop = true;
finfo = netfs_folio_info(folio);
if (finfo->netfs_group != group ||
finfo->dirty_offset > 0) {
folio_unlock(folio);
folio_put(folio);
xas_reset(xas);
break;
}
len = finfo->dirty_len;
}
*_top += folio_size(folio);
index += folio_nr_pages(folio);
*_count -= folio_nr_pages(folio);
*_len += len;
if (*_len >= max_len || *_count <= 0)
stop = true;
if (!folio_batch_add(&fbatch, folio))
break;
if (stop)
break;
}
xas_pause(xas);
rcu_read_unlock();
/* Now, if we obtained any folios, we can shift them to being
* writable and mark them for caching.
*/
if (!folio_batch_count(&fbatch))
break;
for (i = 0; i < folio_batch_count(&fbatch); i++) {
folio = fbatch.folios[i];
trace_netfs_folio(folio, netfs_folio_trace_store_plus);
if (!folio_clear_dirty_for_io(folio))
BUG();
folio_start_writeback(folio);
netfs_folio_start_fscache(caching, folio);
folio_unlock(folio);
}
folio_batch_release(&fbatch);
cond_resched();
} while (!stop);
}
/*
* Synchronously write back the locked page and any subsequent non-locked dirty
* pages.
*/
static ssize_t netfs_write_back_from_locked_folio(struct address_space *mapping,
struct writeback_control *wbc,
struct netfs_group *group,
struct xa_state *xas,
struct folio *folio,
unsigned long long start,
unsigned long long end)
{
struct netfs_io_request *wreq;
struct netfs_folio *finfo;
struct netfs_inode *ctx = netfs_inode(mapping->host);
unsigned long long i_size = i_size_read(&ctx->inode);
size_t len, max_len;
bool caching = netfs_is_cache_enabled(ctx);
long count = wbc->nr_to_write;
int ret;
_enter(",%lx,%llx-%llx,%u", folio->index, start, end, caching);
wreq = netfs_alloc_request(mapping, NULL, start, folio_size(folio),
NETFS_WRITEBACK);
if (IS_ERR(wreq)) {
folio_unlock(folio);
return PTR_ERR(wreq);
}
if (!folio_clear_dirty_for_io(folio))
BUG();
folio_start_writeback(folio);
netfs_folio_start_fscache(caching, folio);
count -= folio_nr_pages(folio);
/* Find all consecutive lockable dirty pages that have contiguous
* written regions, stopping when we find a page that is not
* immediately lockable, is not dirty or is missing, or we reach the
* end of the range.
*/
trace_netfs_folio(folio, netfs_folio_trace_store);
len = wreq->len;
finfo = netfs_folio_info(folio);
if (finfo) {
start += finfo->dirty_offset;
if (finfo->dirty_offset + finfo->dirty_len != len) {
len = finfo->dirty_len;
goto cant_expand;
}
len = finfo->dirty_len;
}
if (start < i_size) {
/* Trim the write to the EOF; the extra data is ignored. Also
* put an upper limit on the size of a single storedata op.
*/
max_len = 65536 * 4096;
max_len = min_t(unsigned long long, max_len, end - start + 1);
max_len = min_t(unsigned long long, max_len, i_size - start);
if (len < max_len)
netfs_extend_writeback(mapping, group, xas, &count, start,
max_len, caching, &len, &wreq->upper_len);
}
cant_expand:
len = min_t(unsigned long long, len, i_size - start);
/* We now have a contiguous set of dirty pages, each with writeback
* set; the first page is still locked at this point, but all the rest
* have been unlocked.
*/
folio_unlock(folio);
wreq->start = start;
wreq->len = len;
if (start < i_size) {
_debug("write back %zx @%llx [%llx]", len, start, i_size);
/* Speculatively write to the cache. We have to fix this up
* later if the store fails.
*/
wreq->cleanup = netfs_cleanup_buffered_write;
iov_iter_xarray(&wreq->iter, ITER_SOURCE, &mapping->i_pages, start,
wreq->upper_len);
__set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags);
ret = netfs_begin_write(wreq, true, netfs_write_trace_writeback);
if (ret == 0 || ret == -EIOCBQUEUED)
wbc->nr_to_write -= len / PAGE_SIZE;
} else {
_debug("write discard %zx @%llx [%llx]", len, start, i_size);
/* The dirty region was entirely beyond the EOF. */
fscache_clear_page_bits(mapping, start, len, caching);
netfs_pages_written_back(wreq);
ret = 0;
}
netfs_put_request(wreq, false, netfs_rreq_trace_put_return);
_leave(" = 1");
return 1;
}
/*
* Write a region of pages back to the server
*/
static ssize_t netfs_writepages_begin(struct address_space *mapping,
struct writeback_control *wbc,
struct netfs_group *group,
struct xa_state *xas,
unsigned long long *_start,
unsigned long long end)
{
const struct netfs_folio *finfo;
struct folio *folio;
unsigned long long start = *_start;
ssize_t ret;
void *priv;
int skips = 0;
_enter("%llx,%llx,", start, end);
search_again:
/* Find the first dirty page in the group. */
rcu_read_lock();
for (;;) {
folio = xas_find_marked(xas, end / PAGE_SIZE, PAGECACHE_TAG_DIRTY);
if (xas_retry(xas, folio) || xa_is_value(folio))
continue;
if (!folio)
break;
if (!folio_try_get_rcu(folio)) {
xas_reset(xas);
continue;
}
if (unlikely(folio != xas_reload(xas))) {
folio_put(folio);
xas_reset(xas);
continue;
}
/* Skip any dirty folio that's not in the group of interest. */
priv = folio_get_private(folio);
if ((const struct netfs_group *)priv != group) {
finfo = netfs_folio_info(folio);
if (finfo->netfs_group != group) {
folio_put(folio);
continue;
}
}
xas_pause(xas);
break;
}
rcu_read_unlock();
if (!folio)
return 0;
start = folio_pos(folio); /* May regress with THPs */
_debug("wback %lx", folio->index);
/* At this point we hold neither the i_pages lock nor the page lock:
* the page may be truncated or invalidated (changing page->mapping to
* NULL), or even swizzled back from swapper_space to tmpfs file
* mapping
*/
lock_again:
if (wbc->sync_mode != WB_SYNC_NONE) {
ret = folio_lock_killable(folio);
if (ret < 0)
return ret;
} else {
if (!folio_trylock(folio))
goto search_again;
}
if (folio->mapping != mapping ||
!folio_test_dirty(folio)) {
start += folio_size(folio);
folio_unlock(folio);
goto search_again;
}
if (folio_test_writeback(folio) ||
folio_test_fscache(folio)) {
folio_unlock(folio);
if (wbc->sync_mode != WB_SYNC_NONE) {
folio_wait_writeback(folio);
#ifdef CONFIG_FSCACHE
folio_wait_fscache(folio);
#endif
goto lock_again;
}
start += folio_size(folio);
if (wbc->sync_mode == WB_SYNC_NONE) {
if (skips >= 5 || need_resched()) {
ret = 0;
goto out;
}
skips++;
}
goto search_again;
}
ret = netfs_write_back_from_locked_folio(mapping, wbc, group, xas,
folio, start, end);
out:
if (ret > 0)
*_start = start + ret;
_leave(" = %zd [%llx]", ret, *_start);
return ret;
}
/*
* Write a region of pages back to the server
*/
static int netfs_writepages_region(struct address_space *mapping,
struct writeback_control *wbc,
struct netfs_group *group,
unsigned long long *_start,
unsigned long long end)
{
ssize_t ret;
XA_STATE(xas, &mapping->i_pages, *_start / PAGE_SIZE);
do {
ret = netfs_writepages_begin(mapping, wbc, group, &xas,
_start, end);
if (ret > 0 && wbc->nr_to_write > 0)
cond_resched();
} while (ret > 0 && wbc->nr_to_write > 0);
return ret > 0 ? 0 : ret;
}
/*
* write some of the pending data back to the server
*/
int netfs_writepages(struct address_space *mapping,
struct writeback_control *wbc)
{
struct netfs_group *group = NULL;
loff_t start, end;
int ret;
_enter("");
/* We have to be careful as we can end up racing with setattr()
* truncating the pagecache since the caller doesn't take a lock here
* to prevent it.
*/
if (wbc->range_cyclic && mapping->writeback_index) {
start = mapping->writeback_index * PAGE_SIZE;
ret = netfs_writepages_region(mapping, wbc, group,
&start, LLONG_MAX);
if (ret < 0)
goto out;
if (wbc->nr_to_write <= 0) {
mapping->writeback_index = start / PAGE_SIZE;
goto out;
}
start = 0;
end = mapping->writeback_index * PAGE_SIZE;
mapping->writeback_index = 0;
ret = netfs_writepages_region(mapping, wbc, group, &start, end);
if (ret == 0)
mapping->writeback_index = start / PAGE_SIZE;
} else if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) {
start = 0;
ret = netfs_writepages_region(mapping, wbc, group,
&start, LLONG_MAX);
if (wbc->nr_to_write > 0 && ret == 0)
mapping->writeback_index = start / PAGE_SIZE;
} else {
start = wbc->range_start;
ret = netfs_writepages_region(mapping, wbc, group,
&start, wbc->range_end);
}
out:
_leave(" = %d", ret);
return ret;
}
EXPORT_SYMBOL(netfs_writepages);
/*
* Deal with the disposition of a laundered folio.
*/
static void netfs_cleanup_launder_folio(struct netfs_io_request *wreq)
{
if (wreq->error) {
pr_notice("R=%08x Laundering error %d\n", wreq->debug_id, wreq->error);
mapping_set_error(wreq->mapping, wreq->error);
}
}
/**
* netfs_launder_folio - Clean up a dirty folio that's being invalidated
* @folio: The folio to clean
*
* This is called to write back a folio that's being invalidated when an inode
* is getting torn down. Ideally, writepages would be used instead.
*/
int netfs_launder_folio(struct folio *folio)
{
struct netfs_io_request *wreq;
struct address_space *mapping = folio->mapping;
struct netfs_folio *finfo = netfs_folio_info(folio);
struct netfs_group *group = netfs_folio_group(folio);
struct bio_vec bvec;
unsigned long long i_size = i_size_read(mapping->host);
unsigned long long start = folio_pos(folio);
size_t offset = 0, len;
int ret = 0;
if (finfo) {
offset = finfo->dirty_offset;
start += offset;
len = finfo->dirty_len;
} else {
len = folio_size(folio);
}
len = min_t(unsigned long long, len, i_size - start);
wreq = netfs_alloc_request(mapping, NULL, start, len, NETFS_LAUNDER_WRITE);
if (IS_ERR(wreq)) {
ret = PTR_ERR(wreq);
goto out;
}
if (!folio_clear_dirty_for_io(folio))
goto out_put;
trace_netfs_folio(folio, netfs_folio_trace_launder);
_debug("launder %llx-%llx", start, start + len - 1);
/* Speculatively write to the cache. We have to fix this up later if
* the store fails.
*/
wreq->cleanup = netfs_cleanup_launder_folio;
bvec_set_folio(&bvec, folio, len, offset);
iov_iter_bvec(&wreq->iter, ITER_SOURCE, &bvec, 1, len);
__set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags);
ret = netfs_begin_write(wreq, true, netfs_write_trace_launder);
out_put:
folio_detach_private(folio);
netfs_put_group(group);
kfree(finfo);
netfs_put_request(wreq, false, netfs_rreq_trace_put_return);
out:
folio_wait_fscache(folio);
_leave(" = %d", ret);
return ret;
}
EXPORT_SYMBOL(netfs_launder_folio);