VPP TCP定时器

对于多线程VPP，为主线程和worker线程分别分配timer_wheel，处理函数为tcp_expired_timers_dispatch。

static clib_error_t *
tcp_main_enable (vlib_main_t * vm)
{
  vlib_thread_main_t *vtm = vlib_get_thread_main ();

  num_threads = 1 /* main thread */  + vtm->n_threads;
  vec_validate (tm->wrk_ctx, num_threads - 1);
  n_workers = num_threads == 1 ? 1 : vtm->n_threads;

  for (thread = 0; thread < num_threads; thread++)
    { 
      wrk = &tm->wrk_ctx[thread];
 
      tcp_timer_initialize_wheel (&wrk->timer_wheel,
                  tcp_expired_timers_dispatch,
                  vlib_time_now (vm));
    }

超时处理函数中，首先根据线程号获得对应的tcp_worker_ctx_t上下文结构，并且获取当前待处理的定时器数量。

static void
tcp_expired_timers_dispatch (u32 * expired_timers)
{
  u32 thread_index = vlib_get_thread_index (), n_left, max_per_loop;
  u32 connection_index, timer_id, n_expired, max_loops;
  tcp_worker_ctx_t *wrk;
  tcp_connection_t *tc;
  int i;

  wrk = tcp_get_worker (thread_index);
  n_expired = vec_len (expired_timers);
  tcp_worker_stats_inc (wrk, timer_expirations, n_expired);
  n_left = clib_fifo_elts (wrk->pending_timers);

遍历超时的定时器，参数expired_timers的每个元素，最高的4位保存了定时器的ID，后28位保存了TCP连接的索引值。如果定时器ID为重传的SYN报文定时器，由函数tcp_half_open_connection_get获取TCP连接结构（实际上是根据连接索引，在线程号为0的TCP worker结构中获取TCP连接）；否则，由函数tcp_connection_get根据当前的线程号和连接索引获取TCP连接结构。

此处将TCP连接的定时器处理句柄设置为无效，设置挂起定时器ID。

  /*
   * Invalidate all timer handles before dispatching. This avoids dangling
   * index references to timer wheel pool entries that have been freed.
   */
  for (i = 0; i < n_expired; i++)
    {
      connection_index = expired_timers[i] & 0x0FFFFFFF;
      timer_id = expired_timers[i] >> 28;

      if (timer_id != TCP_TIMER_RETRANSMIT_SYN)
    tc = tcp_connection_get (connection_index, thread_index);
      else
    tc = tcp_half_open_connection_get (connection_index);

      TCP_EVT (TCP_EVT_TIMER_POP, connection_index, timer_id);

      tc->timers[timer_id] = TCP_TIMER_HANDLE_INVALID;
      tc->pending_timers |= (1 << timer_id);
    }

将到期的定时器添加到worker结构中的挂起定时器。之后，计算每次循环最多处理的定时器数量。首先确定最大的循环次数max_loops，即最大时长0.5 * TCP_TIMER_TICK，乘以每一秒的循环数量，即最大的循环次数。其次，所有待处理的定时器（n_left+n_expired）除以max_loops即为每次循环的最大数量。最后，最大值不能大于每次最多可处理的最大向量长度VLIB_FRAME_SIZE（256），

  clib_fifo_add (wrk->pending_timers, expired_timers, n_expired);

  max_loops =
    clib_max ((u32) 0.5 * TCP_TIMER_TICK * wrk->vm->loops_per_second, 1);
  max_per_loop = clib_max ((n_left + n_expired) / max_loops, 10);
  max_per_loop = clib_min (max_per_loop, VLIB_FRAME_SIZE);
  wrk->max_timers_per_loop = clib_max (n_left ? wrk->max_timers_per_loop : 0,
                       max_per_loop);

  if (thread_index == 0)
    session_queue_run_on_main_thread (wrk->vm);

挂起定时器处理函数tcp_dispatch_pending_timers如下，如果TCP线程结构中没有挂起的定时器，结束处理。

static void
tcp_dispatch_pending_timers (tcp_worker_ctx_t * wrk)
{
  u32 n_timers, connection_index, timer_id, thread_index, timer_handle;
  tcp_connection_t *tc;
  int i;

  if (!(n_timers = clib_fifo_elts (wrk->pending_timers)))
    return;

每次遍历处理的定时器数量不能超过以上计算得到到max_timers_per_loop的值，如果挂起定时器已经被重置，即TCP连接的pending_timers已经不再置位，不进行处理。

  thread_index = wrk->vm->thread_index;
  for (i = 0; i < clib_min (n_timers, wrk->max_timers_per_loop); i++)
    {
      clib_fifo_sub1 (wrk->pending_timers, timer_handle);
      connection_index = timer_handle & 0x0FFFFFFF;
      timer_id = timer_handle >> 28;

      if (PREDICT_TRUE (timer_id != TCP_TIMER_RETRANSMIT_SYN))
    tc = tcp_connection_get (connection_index, thread_index);
      else
    tc = tcp_half_open_connection_get (connection_index);

      if (PREDICT_FALSE (!tc))
    continue;

      /* Skip if the timer is not pending. Probably it was reset while
       * waiting for dispatch */
      if (PREDICT_FALSE (!(tc->pending_timers & (1 << timer_id))))
    continue;

      tc->pending_timers &= ~(1 << timer_id);

      /* Skip timer if it was rearmed while pending dispatch */
      if (PREDICT_FALSE (tc->timers[timer_id] != TCP_TIMER_HANDLE_INVALID))
    continue;

否则，调用定时器ID注册的超时处理函数，处理超时的TCP连接。

      (*timer_expiration_handlers[timer_id]) (tc);
    }

  if (thread_index == 0 && clib_fifo_elts (wrk->pending_timers))
    session_queue_run_on_main_thread (wrk->vm);
}

TCP连接定义了如下的四种定时器。

#define foreach_tcp_timer               \
  _(RETRANSMIT, "RETRANSMIT")           \
  _(PERSIST, "PERSIST")                 \
  _(WAITCLOSE, "WAIT CLOSE")            \
  _(RETRANSMIT_SYN, "RETRANSMIT SYN")   \

typedef enum _tcp_timers
{
#define _(sym, str) TCP_TIMER_##sym,
  foreach_tcp_timer
#undef _
  TCP_N_TIMERS
} __clib_packed tcp_timers_e;

以上四种定时器对应如下四个超时处理函数。

static timer_expiration_handler *timer_expiration_handlers[TCP_N_TIMERS] =
{
    tcp_timer_retransmit_handler,
    tcp_timer_persist_handler,
    tcp_timer_waitclose_handler,
    tcp_timer_retransmit_syn_handler,
};