Skip to content

Commit

Permalink
[NETFILTER]: nf_conntrack: use hlists for conntrack hash
Browse files Browse the repository at this point in the history
Convert conntrack hash to hlists to reduce its size and cache
footprint. Since the default hashsize to max. entries ratio
sucks (1:16), this patch doesn't reduce the amount of memory
used for the hash by default, but instead uses a better ratio
of 1:8, which results in the same max. entries value.

One thing worth noting is early_drop. It really should use LRU,
so it now has to iterate over the entire chain to find the last
unconfirmed entry. Since chains shouldn't be very long and the
entire operation is very rare this shouldn't be a problem.

Signed-off-by: Patrick McHardy <kaber@trash.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
kaber authored and David S. Miller committed Jul 11, 2007
1 parent 8e5105a commit f205c5e
Show file tree
Hide file tree
Showing 7 changed files with 83 additions and 69 deletions.
4 changes: 2 additions & 2 deletions include/net/netfilter/nf_conntrack_core.h
Original file line number Diff line number Diff line change
Expand Up @@ -84,9 +84,9 @@ print_tuple(struct seq_file *s, const struct nf_conntrack_tuple *tuple,
struct nf_conntrack_l3proto *l3proto,
struct nf_conntrack_l4proto *proto);

extern struct list_head *nf_conntrack_hash;
extern struct hlist_head *nf_conntrack_hash;
extern struct list_head nf_conntrack_expect_list;
extern rwlock_t nf_conntrack_lock ;
extern struct list_head unconfirmed;
extern struct hlist_head unconfirmed;

#endif /* _NF_CONNTRACK_CORE_H */
3 changes: 1 addition & 2 deletions include/net/netfilter/nf_conntrack_tuple.h
Original file line number Diff line number Diff line change
Expand Up @@ -125,8 +125,7 @@ DEBUGP("tuple %p: %u %u " NIP6_FMT " %hu -> " NIP6_FMT " %hu\n", \
/* Connections have two entries in the hash table: one for each way */
struct nf_conntrack_tuple_hash
{
struct list_head list;

struct hlist_node hnode;
struct nf_conntrack_tuple tuple;
};

Expand Down
17 changes: 9 additions & 8 deletions net/ipv4/netfilter/nf_conntrack_l3proto_ipv4_compat.c
Original file line number Diff line number Diff line change
Expand Up @@ -41,35 +41,36 @@ struct ct_iter_state {
unsigned int bucket;
};

static struct list_head *ct_get_first(struct seq_file *seq)
static struct hlist_node *ct_get_first(struct seq_file *seq)
{
struct ct_iter_state *st = seq->private;

for (st->bucket = 0;
st->bucket < nf_conntrack_htable_size;
st->bucket++) {
if (!list_empty(&nf_conntrack_hash[st->bucket]))
return nf_conntrack_hash[st->bucket].next;
if (!hlist_empty(&nf_conntrack_hash[st->bucket]))
return nf_conntrack_hash[st->bucket].first;
}
return NULL;
}

static struct list_head *ct_get_next(struct seq_file *seq, struct list_head *head)
static struct hlist_node *ct_get_next(struct seq_file *seq,
struct hlist_node *head)
{
struct ct_iter_state *st = seq->private;

head = head->next;
while (head == &nf_conntrack_hash[st->bucket]) {
while (head == NULL) {
if (++st->bucket >= nf_conntrack_htable_size)
return NULL;
head = nf_conntrack_hash[st->bucket].next;
head = nf_conntrack_hash[st->bucket].first;
}
return head;
}

static struct list_head *ct_get_idx(struct seq_file *seq, loff_t pos)
static struct hlist_node *ct_get_idx(struct seq_file *seq, loff_t pos)
{
struct list_head *head = ct_get_first(seq);
struct hlist_node *head = ct_get_first(seq);

if (head)
while (pos && (head = ct_get_next(seq, head)))
Expand Down
100 changes: 56 additions & 44 deletions net/netfilter/nf_conntrack_core.c
Original file line number Diff line number Diff line change
Expand Up @@ -59,14 +59,14 @@ EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);
int nf_conntrack_max __read_mostly;
EXPORT_SYMBOL_GPL(nf_conntrack_max);

struct list_head *nf_conntrack_hash __read_mostly;
struct hlist_head *nf_conntrack_hash __read_mostly;
EXPORT_SYMBOL_GPL(nf_conntrack_hash);

struct nf_conn nf_conntrack_untracked __read_mostly;
EXPORT_SYMBOL_GPL(nf_conntrack_untracked);

unsigned int nf_ct_log_invalid __read_mostly;
LIST_HEAD(unconfirmed);
HLIST_HEAD(unconfirmed);
static int nf_conntrack_vmalloc __read_mostly;
static struct kmem_cache *nf_conntrack_cachep __read_mostly;
static unsigned int nf_conntrack_next_id;
Expand Down Expand Up @@ -142,8 +142,8 @@ static void
clean_from_lists(struct nf_conn *ct)
{
DEBUGP("clean_from_lists(%p)\n", ct);
list_del(&ct->tuplehash[IP_CT_DIR_ORIGINAL].list);
list_del(&ct->tuplehash[IP_CT_DIR_REPLY].list);
hlist_del(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnode);
hlist_del(&ct->tuplehash[IP_CT_DIR_REPLY].hnode);

/* Destroy all pending expectations */
nf_ct_remove_expectations(ct);
Expand Down Expand Up @@ -184,8 +184,8 @@ destroy_conntrack(struct nf_conntrack *nfct)

/* We overload first tuple to link into unconfirmed list. */
if (!nf_ct_is_confirmed(ct)) {
BUG_ON(list_empty(&ct->tuplehash[IP_CT_DIR_ORIGINAL].list));
list_del(&ct->tuplehash[IP_CT_DIR_ORIGINAL].list);
BUG_ON(hlist_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnode));
hlist_del(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnode);
}

NF_CT_STAT_INC(delete);
Expand Down Expand Up @@ -226,9 +226,10 @@ __nf_conntrack_find(const struct nf_conntrack_tuple *tuple,
const struct nf_conn *ignored_conntrack)
{
struct nf_conntrack_tuple_hash *h;
struct hlist_node *n;
unsigned int hash = hash_conntrack(tuple);

list_for_each_entry(h, &nf_conntrack_hash[hash], list) {
hlist_for_each_entry(h, n, &nf_conntrack_hash[hash], hnode) {
if (nf_ct_tuplehash_to_ctrack(h) != ignored_conntrack &&
nf_ct_tuple_equal(tuple, &h->tuple)) {
NF_CT_STAT_INC(found);
Expand Down Expand Up @@ -263,10 +264,10 @@ static void __nf_conntrack_hash_insert(struct nf_conn *ct,
unsigned int repl_hash)
{
ct->id = ++nf_conntrack_next_id;
list_add(&ct->tuplehash[IP_CT_DIR_ORIGINAL].list,
&nf_conntrack_hash[hash]);
list_add(&ct->tuplehash[IP_CT_DIR_REPLY].list,
&nf_conntrack_hash[repl_hash]);
hlist_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnode,
&nf_conntrack_hash[hash]);
hlist_add_head(&ct->tuplehash[IP_CT_DIR_REPLY].hnode,
&nf_conntrack_hash[repl_hash]);
}

void nf_conntrack_hash_insert(struct nf_conn *ct)
Expand All @@ -290,6 +291,7 @@ __nf_conntrack_confirm(struct sk_buff **pskb)
struct nf_conntrack_tuple_hash *h;
struct nf_conn *ct;
struct nf_conn_help *help;
struct hlist_node *n;
enum ip_conntrack_info ctinfo;

ct = nf_ct_get(*pskb, &ctinfo);
Expand Down Expand Up @@ -319,17 +321,17 @@ __nf_conntrack_confirm(struct sk_buff **pskb)
/* See if there's one in the list already, including reverse:
NAT could have grabbed it without realizing, since we're
not in the hash. If there is, we lost race. */
list_for_each_entry(h, &nf_conntrack_hash[hash], list)
hlist_for_each_entry(h, n, &nf_conntrack_hash[hash], hnode)
if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
&h->tuple))
goto out;
list_for_each_entry(h, &nf_conntrack_hash[repl_hash], list)
hlist_for_each_entry(h, n, &nf_conntrack_hash[repl_hash], hnode)
if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,
&h->tuple))
goto out;

/* Remove from unconfirmed list */
list_del(&ct->tuplehash[IP_CT_DIR_ORIGINAL].list);
hlist_del(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnode);

__nf_conntrack_hash_insert(ct, hash, repl_hash);
/* Timer relative to confirmation time, not original
Expand Down Expand Up @@ -378,22 +380,22 @@ EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken);

/* There's a small race here where we may free a just-assured
connection. Too bad: we're in trouble anyway. */
static int early_drop(struct list_head *chain)
static int early_drop(struct hlist_head *chain)
{
/* Traverse backwards: gives us oldest, which is roughly LRU */
/* Use oldest entry, which is roughly LRU */
struct nf_conntrack_tuple_hash *h;
struct nf_conn *ct = NULL, *tmp;
struct hlist_node *n;
int dropped = 0;

read_lock_bh(&nf_conntrack_lock);
list_for_each_entry_reverse(h, chain, list) {
hlist_for_each_entry(h, n, chain, hnode) {
tmp = nf_ct_tuplehash_to_ctrack(h);
if (!test_bit(IPS_ASSURED_BIT, &tmp->status)) {
if (!test_bit(IPS_ASSURED_BIT, &tmp->status))
ct = tmp;
atomic_inc(&ct->ct_general.use);
break;
}
}
if (ct)
atomic_inc(&ct->ct_general.use);
read_unlock_bh(&nf_conntrack_lock);

if (!ct)
Expand Down Expand Up @@ -535,7 +537,8 @@ init_conntrack(const struct nf_conntrack_tuple *tuple,
}

/* Overload tuple linked list to put us in unconfirmed list. */
list_add(&conntrack->tuplehash[IP_CT_DIR_ORIGINAL].list, &unconfirmed);
hlist_add_head(&conntrack->tuplehash[IP_CT_DIR_ORIGINAL].hnode,
&unconfirmed);

write_unlock_bh(&nf_conntrack_lock);

Expand Down Expand Up @@ -873,16 +876,17 @@ get_next_corpse(int (*iter)(struct nf_conn *i, void *data),
{
struct nf_conntrack_tuple_hash *h;
struct nf_conn *ct;
struct hlist_node *n;

write_lock_bh(&nf_conntrack_lock);
for (; *bucket < nf_conntrack_htable_size; (*bucket)++) {
list_for_each_entry(h, &nf_conntrack_hash[*bucket], list) {
hlist_for_each_entry(h, n, &nf_conntrack_hash[*bucket], hnode) {
ct = nf_ct_tuplehash_to_ctrack(h);
if (iter(ct, data))
goto found;
}
}
list_for_each_entry(h, &unconfirmed, list) {
hlist_for_each_entry(h, n, &unconfirmed, hnode) {
ct = nf_ct_tuplehash_to_ctrack(h);
if (iter(ct, data))
set_bit(IPS_DYING_BIT, &ct->status);
Expand Down Expand Up @@ -917,13 +921,14 @@ static int kill_all(struct nf_conn *i, void *data)
return 1;
}

static void free_conntrack_hash(struct list_head *hash, int vmalloced, int size)
static void free_conntrack_hash(struct hlist_head *hash, int vmalloced,
int size)
{
if (vmalloced)
vfree(hash);
else
free_pages((unsigned long)hash,
get_order(sizeof(struct list_head) * size));
get_order(sizeof(struct hlist_head) * size));
}

void nf_conntrack_flush(void)
Expand Down Expand Up @@ -965,26 +970,26 @@ void nf_conntrack_cleanup(void)
nf_conntrack_helper_fini();
}

static struct list_head *alloc_hashtable(int *sizep, int *vmalloced)
static struct hlist_head *alloc_hashtable(int *sizep, int *vmalloced)
{
struct list_head *hash;
struct hlist_head *hash;
unsigned int size, i;

*vmalloced = 0;

size = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct list_head));
size = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_head));
hash = (void*)__get_free_pages(GFP_KERNEL,
get_order(sizeof(struct list_head)
get_order(sizeof(struct hlist_head)
* size));
if (!hash) {
*vmalloced = 1;
printk(KERN_WARNING "nf_conntrack: falling back to vmalloc.\n");
hash = vmalloc(sizeof(struct list_head) * size);
hash = vmalloc(sizeof(struct hlist_head) * size);
}

if (hash)
for (i = 0; i < size; i++)
INIT_LIST_HEAD(&hash[i]);
INIT_HLIST_HEAD(&hash[i]);

return hash;
}
Expand All @@ -994,7 +999,7 @@ int set_hashsize(const char *val, struct kernel_param *kp)
int i, bucket, hashsize, vmalloced;
int old_vmalloced, old_size;
int rnd;
struct list_head *hash, *old_hash;
struct hlist_head *hash, *old_hash;
struct nf_conntrack_tuple_hash *h;

/* On boot, we can set this without any fancy locking. */
Expand All @@ -1015,12 +1020,12 @@ int set_hashsize(const char *val, struct kernel_param *kp)

write_lock_bh(&nf_conntrack_lock);
for (i = 0; i < nf_conntrack_htable_size; i++) {
while (!list_empty(&nf_conntrack_hash[i])) {
h = list_entry(nf_conntrack_hash[i].next,
struct nf_conntrack_tuple_hash, list);
list_del(&h->list);
while (!hlist_empty(&nf_conntrack_hash[i])) {
h = hlist_entry(nf_conntrack_hash[i].first,
struct nf_conntrack_tuple_hash, hnode);
hlist_del(&h->hnode);
bucket = __hash_conntrack(&h->tuple, hashsize, rnd);
list_add_tail(&h->list, &hash[bucket]);
hlist_add_head(&h->hnode, &hash[bucket]);
}
}
old_size = nf_conntrack_htable_size;
Expand All @@ -1042,18 +1047,25 @@ module_param_call(hashsize, set_hashsize, param_get_uint,

int __init nf_conntrack_init(void)
{
int max_factor = 8;
int ret;

/* Idea from tcp.c: use 1/16384 of memory. On i386: 32MB
* machine has 256 buckets. >= 1GB machines have 8192 buckets. */
* machine has 512 buckets. >= 1GB machines have 16384 buckets. */
if (!nf_conntrack_htable_size) {
nf_conntrack_htable_size
= (((num_physpages << PAGE_SHIFT) / 16384)
/ sizeof(struct list_head));
/ sizeof(struct hlist_head));
if (num_physpages > (1024 * 1024 * 1024 / PAGE_SIZE))
nf_conntrack_htable_size = 8192;
if (nf_conntrack_htable_size < 16)
nf_conntrack_htable_size = 16;
nf_conntrack_htable_size = 16384;
if (nf_conntrack_htable_size < 32)
nf_conntrack_htable_size = 32;

/* Use a max. factor of four by default to get the same max as
* with the old struct list_heads. When a table size is given
* we use the old value of 8 to avoid reducing the max.
* entries. */
max_factor = 4;
}
nf_conntrack_hash = alloc_hashtable(&nf_conntrack_htable_size,
&nf_conntrack_vmalloc);
Expand All @@ -1062,7 +1074,7 @@ int __init nf_conntrack_init(void)
goto err_out;
}

nf_conntrack_max = 8 * nf_conntrack_htable_size;
nf_conntrack_max = max_factor * nf_conntrack_htable_size;

printk("nf_conntrack version %s (%u buckets, %d max)\n",
NF_CONNTRACK_VERSION, nf_conntrack_htable_size,
Expand Down
5 changes: 3 additions & 2 deletions net/netfilter/nf_conntrack_helper.c
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ void nf_conntrack_helper_unregister(struct nf_conntrack_helper *me)
unsigned int i;
struct nf_conntrack_tuple_hash *h;
struct nf_conntrack_expect *exp, *tmp;
struct hlist_node *n;

/* Need write lock here, to delete helper. */
write_lock_bh(&nf_conntrack_lock);
Expand All @@ -132,10 +133,10 @@ void nf_conntrack_helper_unregister(struct nf_conntrack_helper *me)
}

/* Get rid of expecteds, set helpers to NULL. */
list_for_each_entry(h, &unconfirmed, list)
hlist_for_each_entry(h, n, &unconfirmed, hnode)
unhelp(h, me);
for (i = 0; i < nf_conntrack_htable_size; i++) {
list_for_each_entry(h, &nf_conntrack_hash[i], list)
hlist_for_each_entry(h, n, &nf_conntrack_hash[i], hnode)
unhelp(h, me);
}
write_unlock_bh(&nf_conntrack_lock);
Expand Down
6 changes: 3 additions & 3 deletions net/netfilter/nf_conntrack_netlink.c
Original file line number Diff line number Diff line change
Expand Up @@ -428,16 +428,16 @@ ctnetlink_dump_table(struct sk_buff *skb, struct netlink_callback *cb)
{
struct nf_conn *ct, *last;
struct nf_conntrack_tuple_hash *h;
struct list_head *i;
struct hlist_node *n;
struct nfgenmsg *nfmsg = NLMSG_DATA(cb->nlh);
u_int8_t l3proto = nfmsg->nfgen_family;

read_lock_bh(&nf_conntrack_lock);
last = (struct nf_conn *)cb->args[1];
for (; cb->args[0] < nf_conntrack_htable_size; cb->args[0]++) {
restart:
list_for_each_prev(i, &nf_conntrack_hash[cb->args[0]]) {
h = (struct nf_conntrack_tuple_hash *) i;
hlist_for_each_entry(h, n, &nf_conntrack_hash[cb->args[0]],
hnode) {
if (NF_CT_DIRECTION(h) != IP_CT_DIR_ORIGINAL)
continue;
ct = nf_ct_tuplehash_to_ctrack(h);
Expand Down
Loading

0 comments on commit f205c5e

Please sign in to comment.