Troubleshooting ORA-600 [kghstack_alloc] & ORA-600 [kponPurgeUnreachLoc-3]

env: 11.2.0.3.7 2nodes RAC on HPux IA,  你不用这个环境，不知道这个环境BUG有多多 ()”!

#现象

adrci> show incident
*************************************************************************
INCIDENT_ID          PROBLEM_KEY                                                 CREATE_TIME                              
-------------------- ----------------------------------------------------------- ---------------------------------------- 
... 
...  
2070973              ORA 600 [kponPurgeUnreachLoc-3]                             2016-03-27 11:28:11.986000 +08:00       
2068044              ORA 600 [kghstack_alloc]                                    2016-03-27 14:28:13.754000 +08:00       
2068045              ORA 600 [kponPurgeUnreachLoc-3]                             2016-03-27 14:28:15.381000 +08:00       
2070180              ORA 600 [kghstack_alloc]                                    2016-03-27 17:28:16.905000 +08:00       
2070181              ORA 600 [kponPurgeUnreachLoc-3]                             2016-03-27 17:28:18.631000 +08:00       
2068396              ORA 600 [kghstack_alloc]                                    2016-03-27 20:28:17.894000 +08:00       
2068397              ORA 600 [kponPurgeUnreachLoc-3]                             2016-03-27 20:28:19.515000 +08:00   
...

SQL> @p cursor

NAME                                     VALUE
---------------------------------------- ----------------------------------------
_row_cache_cursors                       20_cursor_features_enabled                 2                 <<<<<<<<<<<<
_cursor_runtimeheap_memlimit             5242880
_cursor_stats_enabled                    TRUE
_fast_cursor_reexecute                   FALSE
cursor_space_for_time                    FALSE
session_cached_cursors                   200
_cursor_obsolete_threshold               100               
_disable_cursor_sharing                  FALSE
...

# 分析

 Trace file /oracle/app/oracle/diag/rdbms/anbob/anbob1/trace/anbob1_q003_17743.trc
      Oracle Database 11g Enterprise Edition Release 11.2.0.3.0 - 64bit Production
      With the Partitioning, Real Application Clusters, OLAP, Data Mining
      and Real Application Testing options
      ORACLE_HOME = /oracle/app/oracle/product/11.2.0.3/dbhome_1
      System name:      HP-UX
      Node name:        qdtza1
      Release:  B.11.31
      Version:  U
      Machine:  ia64
      Instance name: anbob1
      Redo thread mounted by this instance: 1
      Oracle process number: 549
      Unix process pid: 17743, image: oracle@qdtza1 (Q003)


      *** 2016-03-27 20:28:17.905
      *** SESSION ID:(1427.19279) 2016-03-27 20:28:17.905
      *** CLIENT ID:() 2016-03-27 20:28:17.905
      *** SERVICE NAME:(SYS$BACKGROUND) 2016-03-27 20:28:17.905
      *** MODULE NAME:(Streams) 2016-03-27 20:28:17.905
      *** ACTION NAME:(QMON Slave) 2016-03-27 20:28:17.9051>     ***** Incident 2068396 created, dump file:  *****
       /oracle/app/oracle/diag/rdbms/anbob/anbob1/incident/incdir_2068396/anbob1_q003_17743_i2068396.trc
1< ***** incident_file ***** 1>     ***** Error Stack *****       ORA-00600: internal error code, arguments: [kghstack_alloc], [define handles], [], [], [], [], [], [], [], [], [], []1< ***** Error Stack ***** 1>     ***** Incident 2068397 created, dump file:  *****
       /oracle/app/oracle/diag/rdbms/anbob/anbob1/incident/incdir_2068397/anbob1_q003_17743_i2068397.trc
1< ***** incident_file ***** Dump continued from file: /oracle/app/oracle/diag/rdbms/anbob/anbob1/trace/anbob1_q003_17743.trc 1>     ***** Error Stack *****
       ORA-00600: internal error code, arguments: [kghstack_alloc], [define handles], [], [], [], [], [], [], [], [], [], []
1< ***** Error Stack ***** 1>     ***** Dump for incident 2068396 (ORA 600 [kghstack_alloc]) *****

       *** 2016-03-27 20:28:17.907
       dbkedDefDump(): Starting incident default dumps (flags=0x2, level=3, mask=0x0)
2>      ***** Current SQL Statement for this session (sql_id=5udxh0ykgshkb) *****        select location_name, max(r.reg_id) from reg$ r left outer join gv$subscr_registration_stats v on r.reg_id = v.reg_id where location_name like
         'net8://(%' group by location_name having min(current_timestamp-nvl(last_ntfn_sent_time, reg_time)) > :12< ***** current_sql_statement ***** 2>      ***** Call Stack Trace *****
        calling              call     entry                
        location             type     point                
        -------------------- -------- -------------------- 
        skdstdst()+64        call     kgdsdst()  000000002 ?
        ksedst()+432         call     skdstdst()           
        dbkedDefDump()+1440  call     ksedst()             
        ksedmp()+64          call     dbkedDefDump()       
        ksfdmp()+96          call     ksedmp()             
        $cold_dbgexPhaseII(  call     ksfdmp()             
        )+576                                              
        dbgexProcessError()  call     $cold_dbgexPhaseII(  
        +2096                         )                    
        dbgeExecuteForError  call     dbgexProcessError()  
        ()+288                                             
        dbgePostErrorKGE()+  call     dbgeExecuteForError  
        2368                          ()                   
        dbkePostKGE_kgsf()+  call     dbgePostErrorKGE()   
        128                                                
        kgeade()+496         call     dbkePostKGE_kgsf()   
        kgeriv_int()+176     call     kgeade()             
        kgeriv()+48          call     kgeriv_int()         
        kgesiv()+192         call     kgeriv()             
        ksesic2()+176        call     kgesiv()             
        $cold_ksmapg()+736   call     ksesic2()            
        kghgex()+2016        call     $cold_ksmapg()       
        kghalf()+592         call     kghgex()             
        kghstack_overflow_i  call     kghalf()             
        nternal()+672                                      
        kghstack_alloc()+40  call     kghstack_overflow_i  
        0                             nternal()            
        kpoopq()+832         call     kghstack_alloc()     
        kpoal8()+7856        call     kpoopq()             
        opiodr()+2416        call     kpoal8()             
        kpoodr()+1776        call     opiodr()      
...

# 解决方案
Bug 14601231 ORA-7445 [kpughndlarr] / assorted ORA-600
The call stack includes kpoopq()

Set _cursor_features_enabled = 514 (I have see 1026 in A product DB)

NOTE:  you need change the parameter then restart the instance before changes take effect.