[BRIDGE]: forwarding table RCU

Convert the bridge forwarding database over to using RCU.
This avoids a read_lock and atomic_inc/dec in the fast path
of output.

Signed-off-by: Stephen Hemminger <shemminger@osdl.org>
Signed-off-by: David S. Miller <davem@redhat.com>

diff --git a/include/linux/list.h b/include/linux/list.h
index 23e287fe425957ecc4b5df9ad81ae430f8f34a05..978876d258a964d6edef0291e6098afae6683345 100644 (file)
--- a/include/linux/list.h
+++ b/include/linux/list.h
@@ -678,6 +678,24 @@ static inline void hlist_add_after(struct hlist_node *n,
             pos && ({ n = pos->next; 1; }) &&                           \
                ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
             pos = n)
+
+/**
+ * hlist_for_each_entry_rcu - iterate over rcu list of given type
+ * @pos:       the type * to use as a loop counter.
+ * @pos:       the &struct hlist_node to use as a loop counter.
+ * @head:      the head for your list.
+ * @member:    the name of the hlist_node within the struct.
+ *
+ * This list-traversal primitive may safely run concurrently with
+ * the _rcu list-mutation primitives such as hlist_add_rcu()
+ * as long as the traversal is guarded by rcu_read_lock().
+ */
+#define hlist_for_each_entry_rcu(tpos, pos, head, member)               \
+       for (pos = (head)->first;                                        \
+            pos && ({ prefetch(pos->next); 1;}) &&                      \
+               ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
+            pos = pos->next, ({ smp_read_barrier_depends(); 0; }) )
+
 #else
 #warning "don't include kernel headers in userspace"
 #endif /* __KERNEL__ */

diff --git a/net/bridge/br_device.c b/net/bridge/br_device.c
index 2d0838456dc46a1fa1d81161477f0fd0404eaa39..44a976221318f46c4afc69c82ef65022d259b5d2 100644 (file)
--- a/net/bridge/br_device.c
+++ b/net/bridge/br_device.c
@@ -43,10 +43,9 @@ int br_dev_xmit(struct sk_buff *skb, struct net_device *dev)
        rcu_read_lock();
        if (dest[0] & 1) 
                br_flood_deliver(br, skb, 0);
-       else if ((dst = br_fdb_get(br, dest)) != NULL) {
+       else if ((dst = __br_fdb_get(br, dest)) != NULL)
                br_deliver(dst->dst, skb);
-               br_fdb_put(dst);
-       } else
+       else
                br_flood_deliver(br, skb, 0);
 
        rcu_read_unlock();

diff --git a/net/bridge/br_fdb.c b/net/bridge/br_fdb.c
index 324489dda89a5ce9e367e91d5cc3f80118084c9d..60b9697d18614d2d4f79c61d7921a399fd13f095 100644 (file)
--- a/net/bridge/br_fdb.c
+++ b/net/bridge/br_fdb.c
@@ -73,7 +73,7 @@ static __inline__ int br_mac_hash(const unsigned char *mac)
 
 static __inline__ void fdb_delete(struct net_bridge_fdb_entry *f)
 {
-       hlist_del(&f->hlist);
+       hlist_del_rcu(&f->hlist);
        if (!f->is_static)
                list_del(&f->age_list);
 
@@ -85,7 +85,7 @@ void br_fdb_changeaddr(struct net_bridge_port *p, const unsigned char *newaddr)
        struct net_bridge *br = p->br;
        int i;
        
-       write_lock_bh(&br->hash_lock);
+       spin_lock_bh(&br->hash_lock);
 
        /* Search all chains since old address/hash is unknown */
        for (i = 0; i < BR_HASH_SIZE; i++) {
@@ -117,7 +117,7 @@ void br_fdb_changeaddr(struct net_bridge_port *p, const unsigned char *newaddr)
        fdb_insert(br, p, newaddr, 1);
 
 
-       write_unlock_bh(&br->hash_lock);
+       spin_unlock_bh(&br->hash_lock);
 }
 
 void br_fdb_cleanup(unsigned long _data)
@@ -126,7 +126,7 @@ void br_fdb_cleanup(unsigned long _data)
        struct list_head *l, *n;
        unsigned long delay;
 
-       write_lock_bh(&br->hash_lock);
+       spin_lock_bh(&br->hash_lock);
        delay = hold_time(br);
 
        list_for_each_safe(l, n, &br->age_list) {
@@ -144,14 +144,14 @@ void br_fdb_cleanup(unsigned long _data)
                        break;
                }
        }
-       write_unlock_bh(&br->hash_lock);
+       spin_unlock_bh(&br->hash_lock);
 }
 
 void br_fdb_delete_by_port(struct net_bridge *br, struct net_bridge_port *p)
 {
        int i;
 
-       write_lock_bh(&br->hash_lock);
+       spin_lock_bh(&br->hash_lock);
        for (i = 0; i < BR_HASH_SIZE; i++) {
                struct hlist_node *h, *g;
                
@@ -182,33 +182,42 @@ void br_fdb_delete_by_port(struct net_bridge *br, struct net_bridge_port *p)
                skip_delete: ;
                }
        }
-       write_unlock_bh(&br->hash_lock);
+       spin_unlock_bh(&br->hash_lock);
 }
 
-struct net_bridge_fdb_entry *br_fdb_get(struct net_bridge *br, unsigned char *addr)
+/* No locking or refcounting, assumes caller has no preempt (rcu_read_lock) */
+struct net_bridge_fdb_entry *__br_fdb_get(struct net_bridge *br,
+                                         const unsigned char *addr)
 {
        struct hlist_node *h;
+       struct net_bridge_fdb_entry *fdb;
 
-       read_lock_bh(&br->hash_lock);
-               
-       hlist_for_each(h, &br->hash[br_mac_hash(addr)]) {
-               struct net_bridge_fdb_entry *fdb
-                       = hlist_entry(h, struct net_bridge_fdb_entry, hlist);
-
+       hlist_for_each_entry_rcu(fdb, h, &br->hash[br_mac_hash(addr)], hlist) {
                if (!memcmp(fdb->addr.addr, addr, ETH_ALEN)) {
-                       if (has_expired(br, fdb))
-                               goto ret_null;
-
-                       atomic_inc(&fdb->use_count);
-                       read_unlock_bh(&br->hash_lock);
+                       if (unlikely(has_expired(br, fdb)))
+                               break;
                        return fdb;
                }
        }
- ret_null:
-       read_unlock_bh(&br->hash_lock);
+
        return NULL;
 }
 
+/* Interface used by ATM hook that keeps a ref count */
+struct net_bridge_fdb_entry *br_fdb_get(struct net_bridge *br, 
+                                       unsigned char *addr)
+{
+       struct net_bridge_fdb_entry *fdb;
+
+       rcu_read_lock();
+       fdb = __br_fdb_get(br, addr);
+       if (fdb) 
+               atomic_inc(&fdb->use_count);
+       rcu_read_unlock();
+       return fdb;
+}
+
+
 void br_fdb_put(struct net_bridge_fdb_entry *ent)
 {
        if (atomic_dec_and_test(&ent->use_count))
@@ -229,9 +238,9 @@ int br_fdb_fillbuf(struct net_bridge *br, void *buf,
 
        memset(buf, 0, maxnum*sizeof(struct __fdb_entry));
 
-       read_lock_bh(&br->hash_lock);
+       rcu_read_lock();
        for (i = 0; i < BR_HASH_SIZE; i++) {
-               hlist_for_each_entry(f, h, &br->hash[i], hlist) {
+               hlist_for_each_entry_rcu(f, h, &br->hash[i], hlist) {
                        if (num >= maxnum)
                                goto out;
 
@@ -255,7 +264,7 @@ int br_fdb_fillbuf(struct net_bridge *br, void *buf,
        }
 
  out:
-       read_unlock_bh(&br->hash_lock);
+       rcu_read_unlock();
 
        return num;
 }
@@ -309,7 +318,7 @@ static int fdb_insert(struct net_bridge *br, struct net_bridge_port *source,
 
        memcpy(fdb->addr.addr, addr, ETH_ALEN);
        atomic_set(&fdb->use_count, 1);
-       hlist_add_head(&fdb->hlist, &br->hash[hash]);
+       hlist_add_head_rcu(&fdb->hlist, &br->hash[hash]);
 
        if (!timer_pending(&br->gc_timer)) {
                br->gc_timer.expires = jiffies + hold_time(br);
@@ -332,8 +341,8 @@ int br_fdb_insert(struct net_bridge *br, struct net_bridge_port *source,
 {
        int ret;
 
-       write_lock_bh(&br->hash_lock);
+       spin_lock_bh(&br->hash_lock);
        ret = fdb_insert(br, source, addr, is_local);
-       write_unlock_bh(&br->hash_lock);
+       spin_unlock_bh(&br->hash_lock);
        return ret;
 }

diff --git a/net/bridge/br_if.c b/net/bridge/br_if.c
index 588ead0731b0fd5eeba3806cc4baf62d7bc24158..43d03734e48b2b116d980e8dec6cc31708d4e85b 100644 (file)
--- a/net/bridge/br_if.c
+++ b/net/bridge/br_if.c
@@ -149,7 +149,7 @@ static struct net_device *new_bridge_dev(const char *name)
 
        br->lock = SPIN_LOCK_UNLOCKED;
        INIT_LIST_HEAD(&br->port_list);
-       br->hash_lock = RW_LOCK_UNLOCKED;
+       br->hash_lock = SPIN_LOCK_UNLOCKED;
 
        br->bridge_id.prio[0] = 0x80;
        br->bridge_id.prio[1] = 0x00;

diff --git a/net/bridge/br_input.c b/net/bridge/br_input.c
index 1c714c6860e759f8af8e4a09416dfded27ededb9..fdf8f9eb6554e0e3e5f3dc55be3f89579c7230e8 100644 (file)
--- a/net/bridge/br_input.c
+++ b/net/bridge/br_input.c
@@ -83,19 +83,17 @@ int br_handle_frame_finish(struct sk_buff *skb)
                goto out;
        }
 
-       dst = br_fdb_get(br, dest);
+       dst = __br_fdb_get(br, dest);
        if (dst != NULL && dst->is_local) {
                if (!passedup)
                        br_pass_frame_up(br, skb);
                else
                        kfree_skb(skb);
-               br_fdb_put(dst);
                goto out;
        }
 
        if (dst != NULL) {
                br_forward(dst->dst, skb);
-               br_fdb_put(dst);
                goto out;
        }
 

diff --git a/net/bridge/br_private.h b/net/bridge/br_private.h
index 1064d4c68c24121396a78e2f88d50563e50050e0..b9ce6a40a30153c99828b70bd49b2d2971ca7447 100644 (file)
--- a/net/bridge/br_private.h
+++ b/net/bridge/br_private.h
@@ -86,7 +86,7 @@ struct net_bridge
        struct list_head                port_list;
        struct net_device               *dev;
        struct net_device_stats         statistics;
-       rwlock_t                        hash_lock;
+       spinlock_t                      hash_lock;
        struct hlist_head               hash[BR_HASH_SIZE];
        struct list_head                age_list;
 
@@ -136,8 +136,10 @@ extern void br_fdb_changeaddr(struct net_bridge_port *p,
 extern void br_fdb_cleanup(unsigned long arg);
 extern void br_fdb_delete_by_port(struct net_bridge *br,
                           struct net_bridge_port *p);
+extern struct net_bridge_fdb_entry *__br_fdb_get(struct net_bridge *br,
+                                                const unsigned char *addr);
 extern struct net_bridge_fdb_entry *br_fdb_get(struct net_bridge *br,
-                                       unsigned char *addr);
+                                              unsigned char *addr);
 extern void br_fdb_put(struct net_bridge_fdb_entry *ent);
 extern int br_fdb_fillbuf(struct net_bridge *br, void *buf, 
                          unsigned long count, unsigned long off);