From 65f27f38446e1976cc98fd3004b110fedcddd189 Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 22 Nov 2006 14:55:48 +0000 Subject: WorkStruct: Pass the work_struct pointer instead of context data Pass the work_struct pointer to the work function rather than context data. The work function can use container_of() to work out the data. For the cases where the container of the work_struct may go away the moment the pending bit is cleared, it is made possible to defer the release of the structure by deferring the clearing of the pending bit. To make this work, an extra flag is introduced into the management side of the work_struct. This governs auto-release of the structure upon execution. Ordinarily, the work queue executor would release the work_struct for further scheduling or deallocation by clearing the pending bit prior to jumping to the work function. This means that, unless the driver makes some guarantee itself that the work_struct won't go away, the work function may not access anything else in the work_struct or its container lest they be deallocated.. This is a problem if the auxiliary data is taken away (as done by the last patch). However, if the pending bit is *not* cleared before jumping to the work function, then the work function *may* access the work_struct and its container with no problems. But then the work function must itself release the work_struct by calling work_release(). In most cases, automatic release is fine, so this is the default. Special initiators exist for the non-auto-release case (ending in _NAR). Signed-Off-By: David Howells --- net/sunrpc/sched.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'net/sunrpc/sched.c') diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c index a1ab4eed41f4..eff44bcdc95a 100644 --- a/net/sunrpc/sched.c +++ b/net/sunrpc/sched.c @@ -41,7 +41,7 @@ static mempool_t *rpc_buffer_mempool __read_mostly; static void __rpc_default_timer(struct rpc_task *task); static void rpciod_killall(void); -static void rpc_async_schedule(void *); +static void rpc_async_schedule(struct work_struct *); /* * RPC tasks sit here while waiting for conditions to improve. @@ -305,7 +305,7 @@ static void rpc_make_runnable(struct rpc_task *task) if (RPC_IS_ASYNC(task)) { int status; - INIT_WORK(&task->u.tk_work, rpc_async_schedule, (void *)task); + INIT_WORK(&task->u.tk_work, rpc_async_schedule); status = queue_work(task->tk_workqueue, &task->u.tk_work); if (status < 0) { printk(KERN_WARNING "RPC: failed to add task to queue: error: %d!\n", status); @@ -695,9 +695,9 @@ rpc_execute(struct rpc_task *task) return __rpc_execute(task); } -static void rpc_async_schedule(void *arg) +static void rpc_async_schedule(struct work_struct *work) { - __rpc_execute((struct rpc_task *)arg); + __rpc_execute(container_of(work, struct rpc_task, u.tk_work)); } /** -- cgit v1.2.3 From cc4dc59e5580d6c0de1685a25b74d32175f43434 Mon Sep 17 00:00:00 2001 From: Christophe Saout Date: Sun, 5 Nov 2006 18:42:48 +0100 Subject: Subject: Re: [PATCH] Fix SUNRPC wakeup/execute race condition The sunrpc scheduler contains a race condition that can let an RPC task end up being neither running nor on any wait queue. The race takes place between rpc_make_runnable (called from rpc_wake_up_task) and __rpc_execute under the following condition: First __rpc_execute calls tk_action which puts the task on some wait queue. The task is dequeued by another process before __rpc_execute continues its execution. While executing rpc_make_runnable exactly after setting the task `running' bit and before clearing the `queued' bit __rpc_execute picks up execution, clears `running' and subsequently both functions fall through, both under the false assumption somebody else took the job. Swapping rpc_test_and_set_running with rpc_clear_queued in rpc_make_runnable fixes that hole. This introduces another possible race condition that can be handled by checking for `queued' after setting the `running' bit. Bug noticed on a 4-way x86_64 system under XEN with an NFSv4 server on the same physical machine, apparently one of the few ways to hit this race condition at all. Cc: Trond Myklebust Cc: J. Bruce Fields Signed-off-by: Christophe Saout Signed-off-by: Trond Myklebust --- net/sunrpc/sched.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'net/sunrpc/sched.c') diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c index a1ab4eed41f4..b57d4062d429 100644 --- a/net/sunrpc/sched.c +++ b/net/sunrpc/sched.c @@ -295,13 +295,15 @@ EXPORT_SYMBOL(__rpc_wait_for_completion_task); */ static void rpc_make_runnable(struct rpc_task *task) { - int do_ret; - BUG_ON(task->tk_timeout_fn); - do_ret = rpc_test_and_set_running(task); rpc_clear_queued(task); - if (do_ret) + if (rpc_test_and_set_running(task)) + return; + /* We might have raced */ + if (RPC_IS_QUEUED(task)) { + rpc_clear_running(task); return; + } if (RPC_IS_ASYNC(task)) { int status; -- cgit v1.2.3 From e6b3c4db6fbcd0d33720696f37790d6b8be12313 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sat, 11 Nov 2006 22:18:03 -0500 Subject: Fix a second potential rpc_wakeup race... Signed-off-by: Trond Myklebust --- net/sunrpc/sched.c | 80 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 33 deletions(-) (limited to 'net/sunrpc/sched.c') diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c index b57d4062d429..66d01365f3a5 100644 --- a/net/sunrpc/sched.c +++ b/net/sunrpc/sched.c @@ -266,12 +266,28 @@ static int rpc_wait_bit_interruptible(void *word) return 0; } +static void rpc_set_active(struct rpc_task *task) +{ + if (test_and_set_bit(RPC_TASK_ACTIVE, &task->tk_runstate) != 0) + return; + spin_lock(&rpc_sched_lock); +#ifdef RPC_DEBUG + task->tk_magic = RPC_TASK_MAGIC_ID; + task->tk_pid = rpc_task_id++; +#endif + /* Add to global list of all tasks */ + list_add_tail(&task->tk_task, &all_tasks); + spin_unlock(&rpc_sched_lock); +} + /* * Mark an RPC call as having completed by clearing the 'active' bit */ -static inline void rpc_mark_complete_task(struct rpc_task *task) +static void rpc_mark_complete_task(struct rpc_task *task) { - rpc_clear_active(task); + smp_mb__before_clear_bit(); + clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate); + smp_mb__after_clear_bit(); wake_up_bit(&task->tk_runstate, RPC_TASK_ACTIVE); } @@ -335,9 +351,6 @@ static void __rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task, return; } - /* Mark the task as being activated if so needed */ - rpc_set_active(task); - __rpc_add_wait_queue(q, task); BUG_ON(task->tk_callback != NULL); @@ -348,6 +361,9 @@ static void __rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task, void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task, rpc_action action, rpc_action timer) { + /* Mark the task as being activated if so needed */ + rpc_set_active(task); + /* * Protect the queue operations. */ @@ -673,8 +689,6 @@ static int __rpc_execute(struct rpc_task *task) } dprintk("RPC: %4d, return %d, status %d\n", task->tk_pid, status, task->tk_status); - /* Wake up anyone who is waiting for task completion */ - rpc_mark_complete_task(task); /* Release all resources associated with the task */ rpc_release_task(task); return status; @@ -788,15 +802,6 @@ void rpc_init_task(struct rpc_task *task, struct rpc_clnt *clnt, int flags, cons task->tk_flags |= RPC_TASK_NOINTR; } -#ifdef RPC_DEBUG - task->tk_magic = RPC_TASK_MAGIC_ID; - task->tk_pid = rpc_task_id++; -#endif - /* Add to global list of all tasks */ - spin_lock(&rpc_sched_lock); - list_add_tail(&task->tk_task, &all_tasks); - spin_unlock(&rpc_sched_lock); - BUG_ON(task->tk_ops == NULL); /* starting timestamp */ @@ -849,16 +854,35 @@ cleanup: goto out; } -void rpc_release_task(struct rpc_task *task) + +void rpc_put_task(struct rpc_task *task) { const struct rpc_call_ops *tk_ops = task->tk_ops; void *calldata = task->tk_calldata; + if (!atomic_dec_and_test(&task->tk_count)) + return; + /* Release resources */ + if (task->tk_rqstp) + xprt_release(task); + if (task->tk_msg.rpc_cred) + rpcauth_unbindcred(task); + if (task->tk_client) { + rpc_release_client(task->tk_client); + task->tk_client = NULL; + } + if (task->tk_flags & RPC_TASK_DYNAMIC) + rpc_free_task(task); + if (tk_ops->rpc_release) + tk_ops->rpc_release(calldata); +} +EXPORT_SYMBOL(rpc_put_task); + +void rpc_release_task(struct rpc_task *task) +{ #ifdef RPC_DEBUG BUG_ON(task->tk_magic != RPC_TASK_MAGIC_ID); #endif - if (!atomic_dec_and_test(&task->tk_count)) - return; dprintk("RPC: %4d release task\n", task->tk_pid); /* Remove from global task list */ @@ -871,23 +895,13 @@ void rpc_release_task(struct rpc_task *task) /* Synchronously delete any running timer */ rpc_delete_timer(task); - /* Release resources */ - if (task->tk_rqstp) - xprt_release(task); - if (task->tk_msg.rpc_cred) - rpcauth_unbindcred(task); - if (task->tk_client) { - rpc_release_client(task->tk_client); - task->tk_client = NULL; - } - #ifdef RPC_DEBUG task->tk_magic = 0; #endif - if (task->tk_flags & RPC_TASK_DYNAMIC) - rpc_free_task(task); - if (tk_ops->rpc_release) - tk_ops->rpc_release(calldata); + /* Wake up anyone who is waiting for task completion */ + rpc_mark_complete_task(task); + + rpc_put_task(task); } /** -- cgit v1.2.3 From 8aca67f0ae2d8811165c22326825a645cc8e1b48 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 13 Nov 2006 16:23:44 -0500 Subject: SUNRPC: Fix a potential race in rpc_wake_up_task() Use RCU to ensure that we can safely call rpc_finish_wakeup after we've called __rpc_do_wake_up_task. If not, there is a theoretical race, in which the rpc_task finishes executing, and gets freed first. Signed-off-by: Trond Myklebust --- net/sunrpc/sched.c | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) (limited to 'net/sunrpc/sched.c') diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c index 66d01365f3a5..6b808c03fb72 100644 --- a/net/sunrpc/sched.c +++ b/net/sunrpc/sched.c @@ -427,16 +427,19 @@ __rpc_default_timer(struct rpc_task *task) */ void rpc_wake_up_task(struct rpc_task *task) { + rcu_read_lock_bh(); if (rpc_start_wakeup(task)) { if (RPC_IS_QUEUED(task)) { struct rpc_wait_queue *queue = task->u.tk_wait.rpc_waitq; - spin_lock_bh(&queue->lock); + /* Note: we're already in a bh-safe context */ + spin_lock(&queue->lock); __rpc_do_wake_up_task(task); - spin_unlock_bh(&queue->lock); + spin_unlock(&queue->lock); } rpc_finish_wakeup(task); } + rcu_read_unlock_bh(); } /* @@ -499,14 +502,16 @@ struct rpc_task * rpc_wake_up_next(struct rpc_wait_queue *queue) struct rpc_task *task = NULL; dprintk("RPC: wake_up_next(%p \"%s\")\n", queue, rpc_qname(queue)); - spin_lock_bh(&queue->lock); + rcu_read_lock_bh(); + spin_lock(&queue->lock); if (RPC_IS_PRIORITY(queue)) task = __rpc_wake_up_next_priority(queue); else { task_for_first(task, &queue->tasks[0]) __rpc_wake_up_task(task); } - spin_unlock_bh(&queue->lock); + spin_unlock(&queue->lock); + rcu_read_unlock_bh(); return task; } @@ -522,7 +527,8 @@ void rpc_wake_up(struct rpc_wait_queue *queue) struct rpc_task *task, *next; struct list_head *head; - spin_lock_bh(&queue->lock); + rcu_read_lock_bh(); + spin_lock(&queue->lock); head = &queue->tasks[queue->maxpriority]; for (;;) { list_for_each_entry_safe(task, next, head, u.tk_wait.list) @@ -531,7 +537,8 @@ void rpc_wake_up(struct rpc_wait_queue *queue) break; head--; } - spin_unlock_bh(&queue->lock); + spin_unlock(&queue->lock); + rcu_read_unlock_bh(); } /** @@ -546,7 +553,8 @@ void rpc_wake_up_status(struct rpc_wait_queue *queue, int status) struct rpc_task *task, *next; struct list_head *head; - spin_lock_bh(&queue->lock); + rcu_read_lock_bh(); + spin_lock(&queue->lock); head = &queue->tasks[queue->maxpriority]; for (;;) { list_for_each_entry_safe(task, next, head, u.tk_wait.list) { @@ -557,7 +565,8 @@ void rpc_wake_up_status(struct rpc_wait_queue *queue, int status) break; head--; } - spin_unlock_bh(&queue->lock); + spin_unlock(&queue->lock); + rcu_read_unlock_bh(); } static void __rpc_atrun(struct rpc_task *task) @@ -817,8 +826,9 @@ rpc_alloc_task(void) return (struct rpc_task *)mempool_alloc(rpc_task_mempool, GFP_NOFS); } -static void rpc_free_task(struct rpc_task *task) +static void rpc_free_task(struct rcu_head *rcu) { + struct rpc_task *task = container_of(rcu, struct rpc_task, u.tk_rcu); dprintk("RPC: %4d freeing task\n", task->tk_pid); mempool_free(task, rpc_task_mempool); } @@ -872,7 +882,7 @@ void rpc_put_task(struct rpc_task *task) task->tk_client = NULL; } if (task->tk_flags & RPC_TASK_DYNAMIC) - rpc_free_task(task); + call_rcu_bh(&task->u.tk_rcu, rpc_free_task); if (tk_ops->rpc_release) tk_ops->rpc_release(calldata); } -- cgit v1.2.3 From bbd5a1f9fc9fad0f8725812d91c51b052e847de8 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Wed, 18 Oct 2006 16:01:05 -0400 Subject: SUNRPC: Fix up missing BKL in asynchronous RPC callback functions Signed-off-by: Trond Myklebust --- net/sunrpc/sched.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'net/sunrpc/sched.c') diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c index 6b808c03fb72..9c13050d23eb 100644 --- a/net/sunrpc/sched.c +++ b/net/sunrpc/sched.c @@ -608,6 +608,15 @@ void rpc_exit_task(struct rpc_task *task) } EXPORT_SYMBOL(rpc_exit_task); +void rpc_release_calldata(const struct rpc_call_ops *ops, void *calldata) +{ + if (ops->rpc_release != NULL) { + lock_kernel(); + ops->rpc_release(calldata); + unlock_kernel(); + } +} + /* * This is the RPC `scheduler' (or rather, the finite state machine). */ @@ -883,8 +892,7 @@ void rpc_put_task(struct rpc_task *task) } if (task->tk_flags & RPC_TASK_DYNAMIC) call_rcu_bh(&task->u.tk_rcu, rpc_free_task); - if (tk_ops->rpc_release) - tk_ops->rpc_release(calldata); + rpc_release_calldata(tk_ops, calldata); } EXPORT_SYMBOL(rpc_put_task); @@ -928,8 +936,7 @@ struct rpc_task *rpc_run_task(struct rpc_clnt *clnt, int flags, struct rpc_task *task; task = rpc_new_task(clnt, flags, ops, data); if (task == NULL) { - if (ops->rpc_release != NULL) - ops->rpc_release(data); + rpc_release_calldata(ops, data); return ERR_PTR(-ENOMEM); } atomic_inc(&task->tk_count); -- cgit v1.2.3 From 6d5fcb5a52bfd00eab3ba2c7ca890823388436ae Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Wed, 18 Oct 2006 16:01:06 -0400 Subject: SUNRPC: Remove BKL around the RPC socket operations etc. All internal RPC client operations should no longer depend on the BKL, however lockd and NFS callbacks may still require it. Signed-off-by: Trond Myklebust --- net/sunrpc/sched.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'net/sunrpc/sched.c') diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c index 9c13050d23eb..f9fd66b1d48b 100644 --- a/net/sunrpc/sched.c +++ b/net/sunrpc/sched.c @@ -588,7 +588,9 @@ void rpc_delay(struct rpc_task *task, unsigned long delay) */ static void rpc_prepare_task(struct rpc_task *task) { + lock_kernel(); task->tk_ops->rpc_call_prepare(task, task->tk_calldata); + unlock_kernel(); } /* @@ -598,7 +600,9 @@ void rpc_exit_task(struct rpc_task *task) { task->tk_action = NULL; if (task->tk_ops->rpc_call_done != NULL) { + lock_kernel(); task->tk_ops->rpc_call_done(task, task->tk_calldata); + unlock_kernel(); if (task->tk_action != NULL) { WARN_ON(RPC_ASSASSINATED(task)); /* Always release the RPC slot and buffer memory */ @@ -651,9 +655,7 @@ static int __rpc_execute(struct rpc_task *task) */ save_callback=task->tk_callback; task->tk_callback=NULL; - lock_kernel(); save_callback(task); - unlock_kernel(); } /* @@ -664,9 +666,7 @@ static int __rpc_execute(struct rpc_task *task) if (!RPC_IS_QUEUED(task)) { if (task->tk_action == NULL) break; - lock_kernel(); task->tk_action(task); - unlock_kernel(); } /* -- cgit v1.2.3 From e18b890bb0881bbab6f4f1a6cd20d9c60d66b003 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 6 Dec 2006 20:33:20 -0800 Subject: [PATCH] slab: remove kmem_cache_t Replace all uses of kmem_cache_t with struct kmem_cache. The patch was generated using the following script: #!/bin/sh # # Replace one string by another in all the kernel sources. # set -e for file in `find * -name "*.c" -o -name "*.h"|xargs grep -l $1`; do quilt add $file sed -e "1,\$s/$1/$2/g" $file >/tmp/$$ mv /tmp/$$ $file quilt refresh done The script was run like this sh replace kmem_cache_t "struct kmem_cache" Signed-off-by: Christoph Lameter Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- net/sunrpc/sched.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'net/sunrpc/sched.c') diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c index eff44bcdc95a..225e6510b523 100644 --- a/net/sunrpc/sched.c +++ b/net/sunrpc/sched.c @@ -34,8 +34,8 @@ static int rpc_task_id; #define RPC_BUFFER_MAXSIZE (2048) #define RPC_BUFFER_POOLSIZE (8) #define RPC_TASK_POOLSIZE (8) -static kmem_cache_t *rpc_task_slabp __read_mostly; -static kmem_cache_t *rpc_buffer_slabp __read_mostly; +static struct kmem_cache *rpc_task_slabp __read_mostly; +static struct kmem_cache *rpc_buffer_slabp __read_mostly; static mempool_t *rpc_task_mempool __read_mostly; static mempool_t *rpc_buffer_mempool __read_mostly; -- cgit v1.2.3