[openib-general] [PATCH] use wait_list for MADs waiting for a response

Thu Oct 7 15:21:28 PDT 2004

This patch uses a wait_list to store MADs waiting for a response.

- Sean

-- Index: access/ib_mad_priv.h
===================================================================

--- access/ib_mad_priv.h        (revision 958)
+++ access/ib_mad_priv.h        (working copy)
@@ -108,6 +108,7 @@
 
        spinlock_t lock;
        struct list_head send_list;
+       struct list_head wait_list;
 
        atomic_t refcount;
        wait_queue_head_t wait;
Index: access/ib_mad.c
===================================================================
--- access/ib_mad.c     (revision 958)
+++ access/ib_mad.c     (working copy)
@@ -232,6 +232,7 @@
 
        spin_lock_init(&mad_agent_priv->lock);
        INIT_LIST_HEAD(&mad_agent_priv->send_list);
+       INIT_LIST_HEAD(&mad_agent_priv->wait_list);
        atomic_set(&mad_agent_priv->refcount, 1);
        init_waitqueue_head(&mad_agent_priv->wait);
        mad_agent_priv->port_priv = port_priv;
@@ -259,7 +260,12 @@
        mad_agent_priv = container_of(mad_agent, struct ib_mad_agent_private,
                                      agent);
 
-       /* Cleanup pending receives for this agent !!! */
+       /* Note that we could still be handling received MADs. */
+
+       /*
+        * Canceling all sends results in dropping received response MADs,
+        * preventing us from queuing additional work.
+        */
        cancel_mads(mad_agent_priv);
 
        spin_lock_irqsave(&mad_agent_priv->port_priv->reg_lock, flags);
@@ -267,6 +273,8 @@
        list_del(&mad_agent_priv->agent_list);
        spin_unlock_irqrestore(&mad_agent_priv->port_priv->reg_lock, flags);
 
+       /* Cleanup pending RMPP receives for this agent !!! */
+
        atomic_dec(&mad_agent_priv->refcount);
        wait_event(mad_agent_priv->wait,
                   !atomic_read(&mad_agent_priv->refcount));
@@ -784,6 +792,16 @@
 {
        struct ib_mad_send_wr_private *mad_send_wr;
 
+       list_for_each_entry(mad_send_wr, &mad_agent_priv->wait_list,
+                           agent_list) {
+               if (mad_send_wr->tid == tid)
+                       return mad_send_wr;
+       }
+
+       /*
+        * It's possible to receive the response before we've been notified
+        * that the send has completed.
+        */
        list_for_each_entry(mad_send_wr, &mad_agent_priv->send_list,
                            agent_list) {
                if (mad_send_wr->tid == tid && mad_send_wr->timeout) {
@@ -932,6 +950,28 @@
        return;
 }
 
+static void wait_for_response(struct ib_mad_agent_private *mad_agent_priv,
+                             struct ib_mad_send_wr_private *mad_send_wr )
+{
+       struct ib_mad_send_wr_private *temp_mad_send_wr;
+       struct list_head *list_item;
+       unsigned long delay;
+
+       list_del(&mad_send_wr->agent_list);
+
+       delay = mad_send_wr->timeout;
+       mad_send_wr->timeout += jiffies;
+
+       list_for_each_prev(list_item, &mad_agent_priv->wait_list) {
+               temp_mad_send_wr = list_entry(list_item,
+                                             struct ib_mad_send_wr_private,
+                                             agent_list);
+               if (time_after(mad_send_wr->timeout, temp_mad_send_wr->timeout))
+                       break;
+       }
+       list_add(&mad_send_wr->agent_list, list_item);
+}
+
 /*
  * Process a send work completion.
  */
@@ -951,12 +991,11 @@
                mad_send_wr->refcount -= (mad_send_wr->timeout > 0);
        }
 
-       /*
-        * Leave sends with timeouts on the send list
-        * until either matching response is received
-        * or timeout occurs
-        */
        if (--mad_send_wr->refcount > 0) {
+               if (mad_send_wr->refcount == 1 && mad_send_wr->timeout &&
+                   mad_send_wr->status == IB_WC_SUCCESS) {
+                       wait_for_response(mad_agent_priv, mad_send_wr);
+               }
                spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
                return;
        }
@@ -1067,18 +1106,14 @@
        spin_lock_irqsave(&mad_agent_priv->lock, flags);
        list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,
                                 &mad_agent_priv->send_list, agent_list) {
-
-               if (mad_send_wr->status == IB_WC_SUCCESS)
+               if (mad_send_wr->status == IB_WC_SUCCESS) {
+                       mad_send_wr->status = IB_WC_WR_FLUSH_ERR;
                        mad_send_wr->refcount -= (mad_send_wr->timeout > 0);
-
-               if (mad_send_wr->refcount == 0) {
-                       list_del(&mad_send_wr->agent_list);
-                       list_add_tail(&mad_send_wr->agent_list,
-                                     &cancel_list);
-               } else {
-                       mad_send_wr->status = IB_WC_WR_FLUSH_ERR;
                }
        }
+
+       /* Empty wait list to prevent receives from finding a request. */
+       list_splice_init(&mad_agent_priv->wait_list, &cancel_list);
        spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
 
        /* Report all cancelled requests */
@@ -1087,14 +1122,12 @@
 
        list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,
                                 &cancel_list, agent_list) {
-
                mad_send_wc.wr_id = mad_send_wr->wr_id;
                mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
                                                   &mad_send_wc);
 
                list_del(&mad_send_wr->agent_list);
                kfree(mad_send_wr);
-
                atomic_dec(&mad_agent_priv->refcount);
        }
 }
@@ -1105,6 +1138,12 @@
 {
        struct ib_mad_send_wr_private *mad_send_wr;
 
+       list_for_each_entry(mad_send_wr, &mad_agent_priv->wait_list,
+                           agent_list) {
+               if (mad_send_wr->wr_id == wr_id)
+                       return mad_send_wr;
+       }
+
        list_for_each_entry(mad_send_wr, &mad_agent_priv->send_list,
                            agent_list) {
                if (mad_send_wr->wr_id == wr_id)

