内核发生了hard LOCKUP然后panic了，代码版本是linux-3.10.0-514.16.1.el7.x86_64
[4474426.249700] NMI watchdog: Watchdog detected hard LOCKUP on cpu 50
crash下的bt信息如下：
    [exception RIP: tg_unthrottle_up+24]
    RIP: ffffffff810c9658 RSP: ffff882f7fc83dc8 RFLAGS: 00000046
    RAX: ffff885d4767d800 RBX: ffff885f7e4d6c40 RCX: ffff8830767f2930
    RDX: 000000000000005b RSI: ffff885f7e4d6c40 RDI: ffff8830767f2800
    RBP: ffff882f7fc83dc8 R8: ffff885f697c7900 R9: 0000000000000001
    R10: 0000000000000000 R11: 0000000000000000 R12: ffff8830764e5400
    R13: ffffffff810c9640 R14: 0000000000000000 R15: ffff8830767f2800
    ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#12 [ffff882f7fc83dc8] tg_unthrottle_up at ffffffff810c9658
#13 [ffff882f7fc83dd0] walk_tg_tree_from at ffffffff810c17db
#14 [ffff882f7fc83e20] unthrottle_cfs_rq at ffffffff810d1675
#15 [ffff882f7fc83e58] distribute_cfs_runtime at ffffffff810d18e2
#16 [ffff882f7fc83ea0] sched_cfs_period_timer at ffffffff810d1a7f
#17 [ffff882f7fc83ed8] __hrtimer_run_queues at ffffffff810b4d72
#18 [ffff882f7fc83f30] hrtimer_interrupt at ffffffff810b5310
#19 [ffff882f7fc83f80] local_apic_timer_interrupt at ffffffff81050fd7
#20 [ffff882f7fc83f98] smp_apic_timer_interrupt at ffffffff8169978f
#21 [ffff882f7fc83fb0] apic_timer_interrupt at ffffffff81697cdd
--- <IRQ stack> ---
hard LOCKUP原理比较简单，就是在关中断的情况下栈里面的函数执行时间过长，时间和是否panic由以下开关决定
```shell
[oen@han]# cat /proc/sys/kernel/hardlockup_panic
1
[oen@han]# cat /proc/sys/kernel/watchdog_thresh
10
```
从栈上可以看出，整个函数栈都是cfs_bandwidth的东西，cfs_bandwidth就是控制cfs调度带宽的，先详细看一下cfs_bandwidth

struct cfs_bandwidth {
#ifdef CONFIG_CFS_BANDWIDTH
 raw_spinlock_t lock;
//cfs_b->period = ns_to_ktime(default_cfs_period()) = 0.1s
// 控制周期为 0.1s ,在/sys/fs/cgroup/cpu/cpu.cfs_period_us可以读到
 time_t period;
 u64 quota，// 周期的时间配额
      runtime; // 周期内剩余可运行时间
 s64 hierarchal_quota;
 u64 runtime_expires;

 int idle, timer_active;
 //周期性定时器
 struct hrtimer period_timer, slack_timer;
 struct list_head throttled_cfs_rq;

 /* statistics */
 int nr_periods, nr_throttled;
 u64 throttled_time;
#endif
};
先说一下大致上的运行逻辑
  1. cfs_rq有自己的运行时间，当运行时间为负数时，cfs_rq被标记为throttled，cfs_rq上运行的当前进程就被标记为重新调度。
  2. cfs_rq的运行时间都是从对应的cfs_bandwidth中获取的，当cfs_bandwidth时间总和是quota，会逐渐消耗完，cfs_rq得不到运行时间就被标记为throttled。
  3. cfs_bandwidth是每隔period时间补充一次quota。
看第一点，cfs_rq运行时间被限制是throttle_cfs_rq完成的，它的调用关系是：
                                                           +-> unthrottle_cfs_rq
                                                           |
                                                           |
               +-> check_enqueue_throttle+-> enqueue_entity|
               |                                           +-> enqueue_task_fair
throttle_cfs_rq|
               |
               +-> check_cfs_rq_runtime+---> put_prev_entity+> put_prev_task_fair
在throttle_cfs_rq和check_enqueue_throttle中可以看到，当cfs_rq->runtime_remaining不大于0时，cfs_rq就会被限制。
在throttle_cfs_rq中，干了如下一些事情，注意cfs_rq将自己添加到cfs_b->throttled_cfs_rq。
    cfs_rq->throttled = 1;
    cfs_rq->throttled_clock = rq_clock(rq);
    raw_spin_lock(&cfs_b->lock);
    list_add_tail_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq);
    if (!cfs_b->timer_active)
        __start_cfs_bandwidth(cfs_b, false);
    raw_spin_unlock(&cfs_b->lock);
再看第二步：scheduler_tick负责更新调度时间，具体调用路径如下：
scheduler_tick -> task_tick_fair -> entity_tick -> update_curr -> account_cfs_rq_runtime -> __account_cfs_rq_runtime
在__account_cfs_rq_runtime下，
static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
{
    /* 更新CFS_rq的运行时间 */
    cfs_rq->runtime_remaining -= delta_exec;
    expire_cfs_rq_runtime(cfs_rq);
        //如果cfs_rq没超时，则不需要处理
    if (likely(cfs_rq->runtime_remaining > 0))
        return;

    /*
     * 从cfs_bandwidth借时间，如果失败，则意味当前cfs_rq时间受到限制，当前运行的进程也被调度走
     */
    if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr))
        resched_task(rq_of(cfs_rq)->curr);
}

assign_cfs_rq_runtime负责cfs_rq从cfs_bandwith里面借时间，sched_cfs_bandwidth_slice的值默认是5ms

static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
{
    struct task_group *tg = cfs_rq->tg;
    struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);
    u64 amount = 0, min_amount, expires;

        /* cfs_rq->runtime_remaining此时肯定为负值， min_amount即需要借取的时间 */
    /* note: this is a positive sum as runtime_remaining <= 0 */
    min_amount = sched_cfs_bandwidth_slice() - cfs_rq->runtime_remaining;

    raw_spin_lock(&cfs_b->lock);
    if (cfs_b->quota == RUNTIME_INF)
                // quota没限制自然随便借
        amount = min_amount;
    else {
        /*
         * If the bandwidth pool has become inactive, then at least one
         * period must have elapsed since the last consumption.
         * Refresh the global state and ensure bandwidth timer becomes
         * active.
         */
        if (!cfs_b->timer_active) {
            __refill_cfs_bandwidth_runtime(cfs_b);
            __start_cfs_bandwidth(cfs_b, false);
        }

        if (cfs_b->runtime > 0) {
                        // 当quota有限制时最多借走全部的runtime
            amount = min(cfs_b->runtime, min_amount);
                        // 更新cfs_bandwith
            cfs_b->runtime -= amount;
            cfs_b->idle = 0;
        }
    }
    expires = cfs_b->runtime_expires;
    raw_spin_unlock(&cfs_b->lock);
        // cfs_rq时间更新
    cfs_rq->runtime_remaining += amount;
    /*
     * we may have advanced our local expiration to account for allowed
     * spread between our sched_clock and the one on which runtime was
     * issued.
     */
    if ((s64)(expires - cfs_rq->runtime_expires) > 0)
        cfs_rq->runtime_expires = expires;

    return cfs_rq->runtime_remaining > 0;
}
sched_cfs_bandwidth_slice值是白送的5ms，因为计算时间不够时需要一个负值的判断。
再看第三步, init_cfs_bandwidth搞了两个定时器，sched_cfs_period_timer和sched_cfs_slack_timer
void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
{
    raw_spin_lock_init(&cfs_b->lock);
    cfs_b->runtime = 0;
        // 默认不限制
    cfs_b->quota = RUNTIME_INF;
        // period默认是100ms
    cfs_b->period = ns_to_ktime(default_cfs_period());

    INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq);
    hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
    cfs_b->period_timer.function = sched_cfs_period_timer;
    hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
    cfs_b->slack_timer.function = sched_cfs_slack_timer;
}

do_sched_cfs_period_timer具体内容如下：

static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)
{
    u64 runtime, runtime_expires;
    int throttled;

    /* no need to continue the timer with no bandwidth constraint */
    if (cfs_b->quota == RUNTIME_INF)
        goto out_deactivate;

    // 获取cfs_b有没有被throttled的cfs_rq
    throttled = !list_empty(&cfs_b->throttled_cfs_rq);
    // period计数器
    cfs_b->nr_periods += overrun;

    /*
     * idle depends on !throttled (for the case of a large deficit), and if
     * we're going inactive then everything else can be deferred
     */
    if (cfs_b->idle && !throttled)
        goto out_deactivate;

    /*
     * if we have relooped after returning idle once, we need to update our
     * status as actually running, so that other cpus doing
     * __start_cfs_bandwidth will stop trying to cancel us.
     */
    cfs_b->timer_active = 1;

/*  __refill_cfs_bandwidth_runtime中给cfs_bandwith重新赋值，第三步的本质内容在这里完成
*   cfs_b->runtime = cfs_b->quota;
*   cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period);
*/
    __refill_cfs_bandwidth_runtime(cfs_b);

    if (!throttled) {
        /* mark as potentially idle for the upcoming period */
        cfs_b->idle = 1;
        return 0;
    }

    /* account preceding periods in which throttling occurred */
    cfs_b->nr_throttled += overrun;

    /*
     * There are throttled entities so we must first use the new bandwidth
     * to unthrottle them before making it generally available.  This
     * ensures that all existing debts will be paid before a new cfs_rq is
     * allowed to run.
     */
    runtime = cfs_b->runtime;
    runtime_expires = cfs_b->runtime_expires;
    cfs_b->runtime = 0;

    /*
     * 主动给各个rq发送时间，让各个rq尽快复活
     */
    while (throttled && runtime > 0) {
        raw_spin_unlock(&cfs_b->lock);
        /* we can't nest cfs_b->lock while distributing bandwidth */
        runtime = distribute_cfs_runtime(cfs_b, runtime,
                         runtime_expires);
        raw_spin_lock(&cfs_b->lock);

        throttled = !list_empty(&cfs_b->throttled_cfs_rq);
    }

    /* return (any) remaining runtime */
    cfs_b->runtime = runtime;
    /*
     * While we are ensured activity in the period following an
     * unthrottle, this also covers the case in which the new bandwidth is
     * insufficient to cover the existing bandwidth deficit.  (Forcing the
     * timer to remain active while there are any throttled entities.)
     */
    cfs_b->idle = 0;

    return 0;

out_deactivate:
    cfs_b->timer_active = 0;
    return 1;
}
回来接着看问题，问题就出在do_sched_cfs_period_timer函数下，来自zhoujian的结论，内核patch如下：
看代码就可以分析出来，内核并没有真正的发送死锁问题，只是单纯的在关中断环境下代码执行时间太长，关键是执行时间长在哪里？
do_sched_cfs_period_timer下有一个while循环 ：while (throttled && runtime > 0){}，在distribute_cfs_runtime下，第16行，唤醒cfs_rq时cfs_rq只被给了多余的1ns，不能再吝啬了，而在外层的do_sched_cfs_period_timer，runtime则是一个从全局赋值的本地变量，并发代码对runtime的消耗便体现不出来了，导致while循环次数非常多。
static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b,
        u64 remaining, u64 expires)
{
    struct cfs_rq *cfs_rq;
    u64 runtime = remaining;

    rcu_read_lock();
    list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,
                throttled_list) {
        struct rq *rq = rq_of(cfs_rq);

        raw_spin_lock(&rq->lock);
        if (!cfs_rq_throttled(cfs_rq))
            goto next;

        runtime = -cfs_rq->runtime_remaining + 1;
        if (runtime > remaining)
            runtime = remaining;
        remaining -= runtime;

        cfs_rq->runtime_remaining += runtime;
        cfs_rq->runtime_expires = expires;

        /* we check whether we're throttled above */
        if (cfs_rq->runtime_remaining > 0)
            unthrottle_cfs_rq(cfs_rq);

next:
        raw_spin_unlock(&rq->lock);

        if (!remaining)
            break;
    }
    rcu_read_unlock();

    return remaining;
}
而在另外一个循环中，distribute_cfs_runtime下有一个for循环： list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, throttled_list) {}，原理比较简单，即throttled_cfs_rq上处理的cfs_rq没有进栈的速度快。
bsegall的patch实际上两个方面都考虑了，那么哪个才可能性大呢，第一个循环需要苛刻的偶然性，而第二个则不需要，最重要的是通过稳定的重现方法，证明了第二个概率大。
最终结论：
在CentOS内核3.10.0-585.el7版本之后该问题解决：
[kernel] sched: Fix potential near-infinite distribute_cfs_runtime() loop (Lauro Ramos Venancio) [1399391]
kpatch热补丁：不要用原版patch，做出的热补丁不解决问题，需要魔改。

cfs_bandwidth下的distribute_cfs_runtime hard lockup来自于OENHAN

链接为:https://oenhan.com/cfs-bandwidth/

发表回复