Skip to content

Commit

Permalink
[PATCH] rcu file: use atomic primitives
Browse files Browse the repository at this point in the history 
Use atomic_inc_not_zero for rcu files instead of special case rcuref.

Signed-off-by: Nick Piggin <npiggin@suse.de>
Cc: "Paul E. McKenney" <paulmck@us.ibm.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
  • Loading branch information
Nick Piggin authored and Linus Torvalds committed Jan 9, 2006
1 parent a57004e commit 095975d
Show file tree
Hide file tree
Showing 9 changed files with 48 additions and 291 deletions.
87 changes: 40 additions & 47 deletions Documentation/RCU/rcuref.txt
View file
Original file line number Diff line number Diff line change
@@ -1,74 +1,67 @@
Refcounter framework for elements of lists/arrays protected by
RCU.
Refcounter design for elements of lists/arrays protected by RCU.

Refcounting on elements of lists which are protected by traditional
reader/writer spinlocks or semaphores are straight forward as in:

1. 2.
add() search_and_reference()
{ {
alloc_object read_lock(&list_lock);
... search_for_element
atomic_set(&el->rc, 1); atomic_inc(&el->rc);
write_lock(&list_lock); ...
add_element read_unlock(&list_lock);
... ...
write_unlock(&list_lock); }
1. 2.
add() search_and_reference()
{ {
alloc_object read_lock(&list_lock);
... search_for_element
atomic_set(&el->rc, 1); atomic_inc(&el->rc);
write_lock(&list_lock); ...
add_element read_unlock(&list_lock);
... ...
write_unlock(&list_lock); }
}

3. 4.
release_referenced() delete()
{ {
... write_lock(&list_lock);
atomic_dec(&el->rc, relfunc) ...
... delete_element
} write_unlock(&list_lock);
...
if (atomic_dec_and_test(&el->rc))
kfree(el);
...
... write_lock(&list_lock);
atomic_dec(&el->rc, relfunc) ...
... delete_element
} write_unlock(&list_lock);
...
if (atomic_dec_and_test(&el->rc))
kfree(el);
...
}

If this list/array is made lock free using rcu as in changing the
write_lock in add() and delete() to spin_lock and changing read_lock
in search_and_reference to rcu_read_lock(), the rcuref_get in
in search_and_reference to rcu_read_lock(), the atomic_get in
search_and_reference could potentially hold reference to an element which
has already been deleted from the list/array. rcuref_lf_get_rcu takes
has already been deleted from the list/array. atomic_inc_not_zero takes
care of this scenario. search_and_reference should look as;

1. 2.
add() search_and_reference()
{ {
alloc_object rcu_read_lock();
... search_for_element
atomic_set(&el->rc, 1); if (rcuref_inc_lf(&el->rc)) {
write_lock(&list_lock); rcu_read_unlock();
return FAIL;
add_element }
... ...
write_unlock(&list_lock); rcu_read_unlock();
alloc_object rcu_read_lock();
... search_for_element
atomic_set(&el->rc, 1); if (atomic_inc_not_zero(&el->rc)) {
write_lock(&list_lock); rcu_read_unlock();
return FAIL;
add_element }
... ...
write_unlock(&list_lock); rcu_read_unlock();
} }
3. 4.
release_referenced() delete()
{ {
... write_lock(&list_lock);
rcuref_dec(&el->rc, relfunc) ...
... delete_element
} write_unlock(&list_lock);
...
if (rcuref_dec_and_test(&el->rc))
call_rcu(&el->head, el_free);
...
... write_lock(&list_lock);
atomic_dec(&el->rc, relfunc) ...
... delete_element
} write_unlock(&list_lock);
...
if (atomic_dec_and_test(&el->rc))
call_rcu(&el->head, el_free);
...
}

Sometimes, reference to the element need to be obtained in the
update (write) stream. In such cases, rcuref_inc_lf might be an overkill
since the spinlock serialising list updates are held. rcuref_inc
update (write) stream. In such cases, atomic_inc_not_zero might be an
overkill since the spinlock serialising list updates are held. atomic_inc
is to be used in such cases.
For arches which do not have cmpxchg rcuref_inc_lf
api uses a hashed spinlock implementation and the same hashed spinlock
is acquired in all rcuref_xxx primitives to preserve atomicity.
Note: Use rcuref_inc api only if you need to use rcuref_inc_lf on the
refcounter atleast at one place. Mixing rcuref_inc and atomic_xxx api
might lead to races. rcuref_inc_lf() must be used in lockfree
RCU critical sections only.

3 changes: 1 addition & 2 deletions fs/aio.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
#include <linux/highmem.h>
#include <linux/workqueue.h>
#include <linux/security.h>
#include <linux/rcuref.h>

#include <asm/kmap_types.h>
#include <asm/uaccess.h>
Expand Down Expand Up @@ -514,7 +513,7 @@ static int __aio_put_req(struct kioctx *ctx, struct kiocb *req)
/* Must be done under the lock to serialise against cancellation.
* Call this aio_fput as it duplicates fput via the fput_work.
*/
if (unlikely(rcuref_dec_and_test(&req->ki_filp->f_count))) {
if (unlikely(atomic_dec_and_test(&req->ki_filp->f_count))) {
get_ioctx(ctx);
spin_lock(&fput_lock);
list_add(&req->ki_list, &fput_head);
Expand Down
8 changes: 4 additions & 4 deletions fs/file_table.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ EXPORT_SYMBOL(get_empty_filp);

void fastcall fput(struct file *file)
{
if (rcuref_dec_and_test(&file->f_count))
if (atomic_dec_and_test(&file->f_count))
__fput(file);
}

Expand Down Expand Up @@ -166,7 +166,7 @@ struct file fastcall *fget(unsigned int fd)
rcu_read_lock();
file = fcheck_files(files, fd);
if (file) {
if (!rcuref_inc_lf(&file->f_count)) {
if (!atomic_inc_not_zero(&file->f_count)) {
/* File object ref couldn't be taken */
rcu_read_unlock();
return NULL;
Expand Down Expand Up @@ -198,7 +198,7 @@ struct file fastcall *fget_light(unsigned int fd, int *fput_needed)
rcu_read_lock();
file = fcheck_files(files, fd);
if (file) {
if (rcuref_inc_lf(&file->f_count))
if (atomic_inc_not_zero(&file->f_count))
*fput_needed = 1;
else
/* Didn't get the reference, someone's freed */
Expand All @@ -213,7 +213,7 @@ struct file fastcall *fget_light(unsigned int fd, int *fput_needed)

void put_filp(struct file *file)
{
if (rcuref_dec_and_test(&file->f_count)) {
if (atomic_dec_and_test(&file->f_count)) {
security_file_free(file);
file_kill(file);
file_free(file);
Expand Down
3 changes: 1 addition & 2 deletions include/linux/fs.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
#include <linux/config.h>
#include <linux/limits.h>
#include <linux/ioctl.h>
#include <linux/rcuref.h>

/*
* It's silly to have NR_OPEN bigger than NR_FILE, but you can change
Expand Down Expand Up @@ -653,7 +652,7 @@ extern spinlock_t files_lock;
#define file_list_lock() spin_lock(&files_lock);
#define file_list_unlock() spin_unlock(&files_lock);

#define get_file(x) rcuref_inc(&(x)->f_count)
#define get_file(x) atomic_inc(&(x)->f_count)
#define file_count(x) atomic_read(&(x)->f_count)

#define MAX_NON_LFS ((1UL<<31) - 1)
Expand Down
1 change: 1 addition & 0 deletions include/linux/radix-tree.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
#ifndef _LINUX_RADIX_TREE_H
#define _LINUX_RADIX_TREE_H

#include <linux/sched.h>
#include <linux/preempt.h>
#include <linux/types.h>

Expand Down
Loading

0 comments on commit 095975d

Please sign in to comment.