Redhat7.9环境11g rac安装记录及问题处理

1. 基础环境配置

1.1.关闭防火墙

# systemctl stop firewalld
# systemctl disable firewalld

1.2.关闭selinux

# sed -i 's/SELINUX\=enforcing/SELINUX\=disabled/g' /etc/selinux/config

1.3.安装必要包

# yum -y install binutils compat-libstdc++-33 gcc gcc-c++ glibc glibc-common  glibc-devel ksh libaio libaio-devel libgcc libstdc++ libstdc++-devel make sysstat openssh-clients compat-libcap1   xorg-x11-utils xorg-x11-xauth xorg-x11-apps xorg-x11-server-utils elfutils unixODBC unixODBC-devel libXp elfutils-libelf elfutils-libelf-devel smartmontools glibc-headers
rpm -ivhcompat-libstdc-33-3.2.3-72.el7.x86_64.rpm

1.4.内核参数调整

# cat /etc/sysctl.conf
fs.file-max = 6815744
kernel.sem = 256 102400000 1024 1024
kernel.shmmni = 4096
kernel.shmall = 1073741824
kernel.shmmax = 4398046511104
net.ipv4.ip_local_port_range = 9000 65500
net.core.rmem_default = 262144
net.core.wmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_max = 1048576
fs.aio-max-nr = 40960000
vm.dirty_ratio=20
vm.dirty_background_ratio=3
vm.dirty_writeback_centisecs=100
vm.dirty_expire_centisecs=500
vm.swappiness=10
vm.min_free_kbytes=524288
#rp_filter，这里假设ens2f0和ens4f0都是私有网卡
net.ipv4.conf.bond0.rp_filter = 1
net.ipv4.conf.ens2f0.rp_filter = 2
net.ipv4.conf.ens4f0.rp_filter = 2
net.ipv4.conf.bond1.rp_filter = 2
#解决packet reassembles failure问题
net.ipv4.ipfrag_high_thresh=8388608
net.ipv4.ipfrag_low_thresh=6291456
net.ipv4.ipfrag_time=60


##需要根据SGA来计算，可略大于计算出来的值.
vm.nr_hugepages = 16408  # MEM=64;SGA=32;expr ${SGA} \* 1024 \* 1024 \* 1024 / 2048 / 1024 + 24;
vm.nr_hugepages = 32792  # MEM=128;SGA=64; expr ${SGA} \* 1024 \* 1024 \* 1024 / 2048 / 1024 + 24;
vm.nr_hugepages = 65560  # MEM=256;SGA=128; expr ${SGA} \* 1024 \* 1024 \* 1024 / 2048 / 1024 + 24;
vm.nr_hugepages = 81944  # MEM=328;SGA=160; expr ${SGA} \* 1024 \* 1024 \* 1024 / 2048 / 1024 + 24;
vm.nr_hugepages = 131096 # MEM=512;SGA=256; expr ${SGA} \* 1024 \* 1024 \* 1024 / 2048 / 1024 + 24;

1.5.关闭操作系统Transparent Huge Pages\关闭NUMA

--需要重启主机生效
# cat /etc/default/grub
GRUB_TIMEOUT=5
GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
GRUB_DEFAULT=saved
GRUB_DISABLE_SUBMENU=true
GRUB_TERMINAL_OUTPUT="console"
GRUB_CMDLINE_LINUX="crashkernel=auto spectre_v2=retpoline rd.lvm.lv=centos/root rd.lvm.lv=centos/swap rhgb quiet transparent_hugepage=never numa=off"
GRUB_DISABLE_RECOVERY="true"
# grub2-mkconfig -o /boot/grub2/grub.cfg
# grub2-mkconfig -o /boot/efi/EFI/centos/grub.cfg

1.6.配置/etc/hosts

# cat /etc/hosts
192.168.135.22 rac1
192.168.135.23 rac2
192.168.135.24 rac1-vip
192.168.135.25 rac2-vip
1.1.10.22 rac1-pri
1.1.10.23 rac1-pri
192.168.135.26 racdb-scan

hostnamectl set-hostname rac1
hostnamectl set-hostname rac2

1.7.检查共享磁盘 ID_SERIAL

# udevadm info -q all -n /dev/qbd0
# udevadm info -q all -n /dev/qbd1
# udevadm info -q all -n /dev/qbd2
# udevadm info -q all -n /dev/qbd3

1.8.配置udev

# cat /etc/udev/rules.d/99-oracle-rules
KERNEL=="qbd[0-0]", ATTRS{serial}=="?*",ENV{ID_SERIAL}="$attr{serial}",SYMLINK+="asm/$env{ID_SERIAL}",OWNER="grid",GROUP="asmadmin",MODE="0660"
# udevadm control --reload-rules
# udevadm trigger

1.9.创建用户及目录

/usr/sbin/groupadd -g 501 dba
/usr/sbin/groupadd -g 500 oinstall 
/usr/sbin/groupadd -g 502 asmadmin

/usr/sbin/useradd -m -u 501 -g oinstall -G dba,asmadmin -d /home/grid -s /bin/bash -c "Grid Infrastructure Owner" grid

/usr/sbin/useradd -m -u 500 -g oinstall -G dba,asmadmin -d /home/oracle -s /bin/bash -c "Oracle Software Owner" oracle
#为安装方便，设置密码和用户名相同
echo "Gabc@123"|passwd oracle --stdin
echo "Gabc@123"|passwd grid –stdin
#检查用户ID和所属组
id grid
id oracle

#Grid Infrastructure BASE:
mkdir -p /u01/app/grid/gbase
#Grid Infrastructure HOME:
mkdir -p /u01/app/grid/ghome
#Database BASE:
mkdir -p /u01/app/oracle
chown -R root:oinstall /u01
chown -R grid:oinstall /u01/app/grid
chown -R oracle:oinstall /u01/app/oracle
chmod -R 775 /u01/app/grid
chmod -R 775 /u01/app/oracle

1.10.用户限制设置

# cat /etc/security/limits.conf
*   soft   nofile     1048576
*   hard   nofile     1048576
*   soft   nproc      131072
*   hard   nproc      131072
*   soft   stack      65536
*   hard   stack      65536
*   soft   core       6291456
*   hard   core       6291456
*     soft   memlock    unlimited
*     hard   memlock    unlimited

# cat /etc/security/limits.d/20-nproc.conf

*          soft    nproc     131072
root       soft    nproc     unlimited

1.11.配置环境变量

-- rac1节点grid用户,rac2节点自行修改配置
$ .bash_profile

# Get the aliases and functions
if [ -f ~/.bashrc ]; then
        . ~/.bashrc
fi

# User specific environment and startup programs

PATH=$PATH:$HOME/bin

export PATH

umask 022
export ORACLE_BASE=/u01/app/grid/gbase
export ORACLE_HOME=/u01/app/grid/ghome
export PATH=$ORACLE_HOME/bin:$ORACLE_HOME/OPatch:$PATH
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib:/usr/local/lib
export LANG=en_US
export NLS_LANG=american_america.ZHS16GBK
export NLS_DATE_FORMAT="yyyy-mm-dd hh24:mi:ss"
export ORACLE_SID=+ASM1

-- rac1节点oracle用户,rac2节点自行修改配置
$ .bash_profile

# Get the aliases and functions
if [ -f ~/.bashrc ]; then
        . ~/.bashrc
fi

# User specific environment and startup programs

PATH=$PATH:$HOME/bin

export PATH

umask 022
export ORACLE_BASE=/u01/app/oracle
export ORACLE_HOME=$ORACLE_BASE/dbhome
export PATH=$ORACLE_HOME/bin:$ORACLE_HOME/OPatch:$PATH
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib:/usr/local/lib
export LANG=en_US
export NLS_LANG=american_america.ZHS16GBK
export NLS_DATE_FORMAT="yyyy-mm-dd hh24:mi:ss"
export ORACLE_SID=rac1

2. 集群安装

2.1.配置x server启动图形

1. 修改D:\WindTerm_2.6.0\vendors\vcxsrv\X0.hosts 添加Windows主机ip

2. 打开x server按钮
   

3. grid和oracle用户需要设置DISPLAY

# echo $DISPLAY
localhost:10.0
# xhost +
# su - grid
$ export DISPLAY=localhost:10.0
或
# su - oracle
$ export DISPLAY=localhost:10.0

2.2.启动安装

# CVUQDISK_GRP=oinstall; export CVUQDISK_GRP
# cd /soft/grid/rpm
# rpm -iv cvuqdisk-1.0.9-1.rpm
# su - grid
$ cd /soft/grid/
$ ./runInstaller -ignoresysprereqs

2.3.安装过程

1-16 步安装步骤省略
执行到17步当出现需要执行 orainstRoot.sh和root.sh 脚本弹出框时，需要打补丁

-- rac1、rac2节点都要执行
$ unzip p6880880_112000_Linux-x86-64.zip -d /u01/app/grid/ghome
$ unzip p18370031_112040_Linux-x86-64.zip -d /tmp/
$ opatch napply -oh $ORACLE_HOME -local /tmp/18370031 -silent

执行 orainstRoot.sh

-- rac1、rac2节点使用root用户交替执行
# /u01/app/grid/11.2.0.3/oraInventory/orainstRoot.sh

执行root.sh

-- rac1、rac2节点使用root用户交替执行
# /u01/app/grid/11.2.0.3/ghome/root.sh

2.4.数据库软件安装

$
$ cd /soft/database
$ ./runInstaller
1-12 步安装省略

2.5.创建data磁盘组

# su - grid
$ asmca

3. 更新全量补丁

--rac1、rac2节点都需要执行
--grid用户解压
$ unzip p31718723_112040_Linux-x86-64.zip -d /tmp/
--root用户执行
# cd /tmp/31718723
# /u01/app/grid/ghome/OPatch/opatch auto /soft/31718723 -oh /u01/app/grid/ghome

--oracle用户需要更新opatch版本
$ unzip p6880880_112000_Linux-x86-64.zip -d /u01/app/oracle/dbhome
# cd /tmp/31718723
# /u01/app/grid/ghome/OPatch/opatch auto /soft/31718723 -oh /u01/app/oracle/dbhome

4.创建数据库

$ dbca 步骤省略
修改数据库参数及集群参数

--oracle用户
$ sqlplus -S / as sysdba<<eof
alter system set audit_trail=none sid='*' scope=spfile;
alter system set sec_case_sensitive_logon=false;
"alter profile ""DEFAULT"" limit PASSWORD_GRACE_TIME UNLIMITED;
alter profile ""DEFAULT"" limit PASSWORD_LIFE_TIME UNLIMITED;
alter profile ""DEFAULT"" limit PASSWORD_LOCK_TIME UNLIMITED;
alter profile ""DEFAULT"" limit FAILED_LOGIN_ATTEMPTS UNLIMITED;"
alter system set event='28401 trace name context forever,level 1'  sid='*' scope=spfile;
alter system set "_px_use_large_pool"=true  sid ='*' scope=spfile;
alter system set parallel_force_local=true;
"alter system set ""_gc_policy_time""=0 sid='*' scope=spfile;
alter system set ""_gc_undo_affinity""=false scope=spfile;
alter system set ""_lm_drm_disable""=5 scope=both sid='*'; 
    level 4   disable read mostly
    level 5   disable DRM for all but undo
    level 7   disable drm for all including undo"
"alter system set ""_optimizer_adaptive_cursor_sharing""=false;
alter system set ""_optimizer_extended_cursor_sharing""=none;
alter system set ""_optimizer_extended_cursor_sharing_rel""=none;"
alter system set "_optimizer_use_feedback"=false;
alter system set deferred_segment_creation=false;
alter system set "_serial_direct_read"=never;
"alter system set resource_limit=true;
alter system set resource_manager_plan='force:';"
alter system set "_use_adaptive_log_file_sync"=false;
"exec dbms_scheduler.disable( 'ORACLE_OCM.MGMT_CONFIG_JOB' );
exec dbms_scheduler.disable( 'ORACLE_OCM.MGMT_STATS_CONFIG_JOB' );
exec dbms_auto_task_admin.disable('auto space advisor',null,null);
exec dbms_auto_task_admin.disable('sql tuning advisor',null,null);"
"alter system set ""_undo_autotune""=false;
alter system set undo_retention=10800;"
alter system set "_partition_large_extents"=false;
alter system set "_gby_hash_aggregation_enabled"=false;
alter system set processes=5000 sid='*' scope=spfile;
alter system set pga_aggregate_target = 20g sid='*' scope=spfile;
"alter system set memory_max_target=0 sid='*' scope=spfile;
alter system set memory_target=0 sid='*' scope=spfile;"
alter system set db_files=2000 sid='*' scope=spfile;
alter system set open_cursors=600;
eof

--grid用户
$ sqlplus -S / as sysasm<<eof
alter system set memory_max_target=4096m scope=spfile;
alter system set memory_target=4096m scope=spfile;
alter system set processes=4000 scope=spfile;
eof

--rac1、rac2节点重启集群
# /u01/app/grid/ghome/bin/crsctl stop crs -f
# /u01/app/grid/ghome/bin/crsctl start crs

问题处理

更新 31718723 补丁重启后集群votedisk 盘异常如下：

处理如下：