Higher priority task failed to preempt current task after ISR done in 2.6.34 with RT patch

Question

1. The arch is powerpc.
2. I changed the ISR from threaded to no-threaded.
3. The ISR wake-up another FIFO task with prio==19
4. Sometimes at the end of 'ret_from_except' when do 'resume_kernel', the preempt schedule condition is not fulfilled: the 'preempt-able' is yes, the 'need_resched' flag not set, but,
5. In C code, the 'try_to_wake_up' has set the 'need_resched' flag to yes.
6. I tried cached sync, better but didn't eliminate the problem.
7. code snippet related:

The '5555555' and 'eeeeeeee' are printed but the 'ffffffff' isn't.

kernel/sched.c:

184 +static void my_resched_task( struct task_struct *new, struct task_struct *cur ) {
185 +    int cpu;
186 +
187 +    assert_raw_spin_locked(&task_rq(cur)->lock);
188 +
189 +    cpu = task_cpu(cur);
190 +
191 +    if (test_tsk_need_resched(cur)) {
192 +        if( new->prio <= 20 && in_irq() ) {
193 +            set_tsk_need_resched(cur);
194 +            __asm__ volatile("sync;isync": : : "memory");
195 +            PRMPT_SCHD_IRQ_TS_PRINT(0x55555555);
196 +            dcbf(&(task_thread_info(cur)->flags));
197 +            in_be32(&(task_thread_info(cur)->flags));
198 +            __asm__ volatile ("msync");
199 +        }
200 +        return;
201 +    }
202 +
203 +    set_tsk_need_resched(cur);
204 +    if( new->prio <= 20 && in_irq() ) {
205 +        __asm__ volatile("sync;isync": : : "memory");
206 +        PRMPT_SCHD_IRQ_TS_PRINT(0x66666666);
207 +        dcbf(&(task_thread_info(cur)->flags));
208 +        in_be32(&(task_thread_info(cur)->flags));
209 +        __asm__ volatile ("msync");
210 +    }

arch/powerpc/kernel/entry_32.S

 66 +    /*wqc add >>*/
 67 +    WQC_PRMT_TAG_IMT(0xeeeeeeee)
 68 +    /*would r0 has been changed in the above flow ? reload it*/
 69 +    rlwinm  r9,r1,0,0,(31-THREAD_SHIFT)
 70 +    lwz r0,TI_FLAGS(r9)
 71 +    /*wqc add <<*/
 72 +
 73     andi.   r0,r0,_TIF_NEED_RESCHED
 74     beq+    restore
 75 +    WQC_PRMT_TAG_IMT(0xffffffff)
 76     lwz     r3,_MSR(r1)
 77     andi.   r0,r3,MSR_EE    /* interrupts off? */
 78     beq restore     /* don't schedule if so */
 79 @@ -938,6 +970,7 @@ resume_kernel:
 80  //wqc   */
 81  //wqc  bl  trace_hardirqs_off
 82  #endif
 83 +    WQC_PRMT_TAG_IMT(0xa0a0a0a0)
 84  1: bl  preempt_schedule_irq

Answer 1

We got the problem fixed. It is because of accessing tainted register in checking preemption just before return from interrupt. Post here for other's reference.

diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index 88b2e4a..bd0a283 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -891,8 +891,8 @@ restore_user:
 resume_kernel:
    /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
    rlwinm  r9,r1,0,0,(31-THREAD_SHIFT)
-   lwz r0,TI_FLAGS(r9)
-   andis.  r0,r0,_TIF_EMULATE_STACK_STORE@h
+   lwz r8,TI_FLAGS(r9)
+   andis.  r0,r8,_TIF_EMULATE_STACK_STORE@h
    beq+    1f

    addi    r8,r1,INT_FRAME_SIZE    /* Get the kprobed function entry */
@@ -900,16 +900,22 @@ resume_kernel:
    lwz     r3,GPR1(r1)
    subi    r3,r3,INT_FRAME_SIZE    /* dst: Allocate a trampoline exception frame */
    mr      r4,r1                   /* src:  current exception frame */
-   li      r5,INT_FRAME_SIZE       /* size: INT_FRAME_SIZE */
    mr      r1,r3                   /* Reroute the trampoline frame to r1 */
-   bl      memcpy                  /* Copy from the original to the trampoline */
+
+   /* Copy from the original to the trampoline. */
+   li  r5,INT_FRAME_SIZE/4 /* size: INT_FRAME_SIZE */
+   li  r6,0            /* start offset: 0 */
+   mtctr   r5
+2: lwzx    r0,r6,r4
+   stwx    r0,r6,r3
+   addi    r6,r6,4
+   bdnz    2b

    /* Do real store operation to complete stwu */
    lwz     r5,GPR1(r1)
    stw     r8,0(r5)

    /* Clear _TIF_EMULATE_STACK_STORE flag */
-   rlwinm  r9,r1,0,0,(31-THREAD_SHIFT)
    lis     r11,_TIF_EMULATE_STACK_STORE@h
    addi    r5,r9,TI_FLAGS
 0: lwarx   r8,0,r5
@@ -923,10 +929,10 @@ resume_kernel:

 #ifdef CONFIG_PREEMPT
    /* check current_thread_info->preempt_count */
-   lwz     r8,TI_PREEMPT(r9)
-   cmpwi   0,r8,0          /* if non-zero, just restore regs and return */
+   lwz     r0,TI_PREEMPT(r9)
+   cmpwi   0,r0,0          /* if non-zero, just restore regs and return */
    bne     restore
-   andi.   r0,r0,_TIF_NEED_RESCHED
+   andi.   r8,r8,_TIF_NEED_RESCHED
    beq+    restore
    lwz     r3,_MSR(r1)
    andi.   r0,r3,MSR_EE    /* interrupts off? */