Home > Databases, IT Architecture, Oracle DB > oracle rac on linux installation

oracle rac on linux installation

August 6th, 2013

1,2 -> database; 3-> grid; 4 -> client; 5 -> gateways; 6 -> examples; 7 -> deinstall

---private IP, eth1
192.168.13.86/24
192.168.13.87/24

---public ip, eth0
10.240.149.109/21
10.240.149.110/21

---vip #vip alias will auto install, should in same subnet with public ip/scan ip
10.240.149.107/21
10.240.149.108/21

---scan IP #scan ip should be defined in dns. multiple IPs can map to one scan-name(round-robin)
10.240.149.106

---/etc/hosts
127.0.0.1 localhost.localdomain localhost
::1 localhost6.localdomain6 localhost6

10.240.149.109 node1.example.com node1

10.240.149.110 node2.example.com node2
192.168.13.86 node1-priv.localdomain node1-priv
192.168.13.87 node2-priv.localdomain node2-priv

PS:
hostname should return node1.us.test.com

---DNS configuration

yum install bind bind-chroot caching-nameserver
cd /var/named/chroot/etc
cp -p named.caching-nameserver.conf named.conf
options {
listen-on port 53 { any; };
listen-on-v6 port 53 { ::1; };
directory "/var/named";
dump-file "/var/named/data/cache_dump.db";
statistics-file "/var/named/data/named_stats.txt";
memstatistics-file "/var/named/data/named_mem_stats.txt";

allow-query { any; };
allow-query-cache { any; };
};
logging {
channel default_debug {
file "data/named.run";
severity dynamic;
};
};
view localhost_resolver {
match-clients { any; };
match-destinations { any; };
recursion no;
include "/etc/named.rfc1912.zones";
};
vi named.rfc1912.zones #in the end

zone "us.test.com" IN {
type master;
file "us.test.com.zone" #dns -> ip
allow-update { none; };
};

zone "149.240.10.in-addr.arpa." IN {
type master;
file "149.240.10.in-addr.arpa"; #ip -> dns, reverse dns
allow-update { none; };
};
cd /var/named/chroot/var/named/
vi us.test.com.zone #in the end

scan-cluster IN A 10.240.149.106
node1 IN A 10.240.149.109
node1-vip IN A 10.240.149.107
node2 IN A 10.240.149.110
node2-vip IN A 10.240.149.108

 

cp -p named.local 149.240.10.in-addr.arpa
vi 149.240.10.in-addr.arpa #in the end

106 IN PTR scan-cluster.us.test.com. #if using scan, then multiple IPs should added here
107 IN PTR node1-vip.us.test.com.
108 IN PTR node2-vip.us.test.com.
109 IN PTR node1.us.test.com.
110 IN PTR node2.us.test.com.
/etc/init.d/named start
chkconfig named on

vi /etc/resolv.conf #on node1, node2
search us.test.com #on first line
nameserver 10.245.28.40

To get the scan VIP name and IPs, run /u01/app/grid/11.2.0.3/bin/srvctl config scan. To get the VIP name of each node, run /u01/app/grid/11.2.0.3/bin/srvctl config vip -n <node name>.

PS:

If you're using Oracle Linux, get yum repo from http://public-yum.oracle.com/ and do the following to automatically install any additional packages needed for installing Oracle Grid Infrastructure and Oracle Database, and configure your server operating system automatically, including setting kernel parameters and other basic operating system requirements for installation.

#Oracle Linux 6
#log /var/log/oracle-rdbms-server-11gR2-preinstall/results/orakernel.log

yum install oracle-rdbms-server-11gR2-preinstall -y

yum install oracle-rdbms-server-12cR1-preinstall -y

#Oracle Linux 5
#log /var/log/oracle-validated/results/orakernel.log

yum install oracle-validated -y

#When install OS with Oracle Linux

Customize now - Servers - System administration tools - select Oracle Preinstallation or Oracle Validated

---#1_preuser.sh,

---2_predir.sh

---3_prelimits.sh

---4_prelogin.sh

---5_preprofile.sh

---6_presysctl.sh

---stop ntpd
service ntpd stop
chkconfig ntpd off
57 18 * * * /usr/sbin/ntpdate 10.245.72.1

---ssh config for oracle/grid(passwordless even forthe host  itself)

ssh-keygen -t rsa
cat .ssh/id_rsa.pub | tr -d '\r\n'
vi .ssh/authorized_keys #for both peer host and itself

chmod 755 ~oracle

---config disks
allocate iscsi LUN
fdisk LUN(on one disk, no need to create filesystem)

for i in /etc/yum.repos.d/*.repo;do sed -i 's/gpgcheck=1/gpgcheck=0/g' $i;done

for i in /etc/yum.repos.d/*.repo;do sed -i 's/enabled=0/enabled=1/g' $i;done
yum install oracleasm-support oracleasm oracleasmlib kmod-oracleasm #on two nodes

#Configuring Storage Device Path Persistence Using Oracle ASMLIB, more info here

/usr/sbin/oracleasm status
/usr/sbin/oracleasm configure -i #grid/asmadmin/y/y
/usr/sbin/oracleasm init #more info is here about oracleasm commands with steps
/usr/sbin/oracleasm status
/usr/sbin/oracleasm configure
/usr/sbin/oracleasm listdisks #on one node
/usr/sbin/oracleasm createdisk VOL1 /dev/sda1
/usr/sbin/oracleasm createdisk VOL2 /dev/sdb1
/usr/sbin/oracleasm createdisk VOL3 /dev/sdc1
/usr/sbin/oracleasm createdisk VOL4 /dev/sdd1
/usr/sbin/oracleasm listdisks
/usr/sbin/oracleasm scandisks #on the other node
/usr/sbin/oracleasm listdisks
/usr/sbin/oracleasm querydisk /dev/sd*

#Configuring Oracle ASMLIB for Multipath Disks is here

#Configuring Disk Devices Manually for Oracle ASM(udev) is here

#get scsi id for LUN and set persistent name

[root@testohf u01]# /sbin/scsi_id -g -u -d /dev/sdg
3600144f0979678fa000057f53d770015

[root@testohf u01]# /sbin/scsi_id -g -u -d /dev/sdh
3600144f0979678fa000057f53db30016

[root@testohf u01]# /sbin/scsi_id -g -u -d /dev/sdf
3600144f0979678fa000057f53cd70014

[root@testohf u01]# /sbin/scsi_id -g -u -d /dev/sdi
3600144f0979678fa000057f53de90017

[root@testohf ~]# cat /etc/udev/rules.d/99-oracle-asmdevices.rules
KERNEL=="sd?1", BUS=="scsi", PROGRAM=="/sbin/scsi_id -g -u -d /dev/$parent", RESULT=="3600144f0979678fa000057f53d770015", NAME="asm-data", OWNER="oracle", GROUP="dba", MODE="0660"
KERNEL=="sd?1", BUS=="scsi", PROGRAM=="/sbin/scsi_id -g -u -d /dev/$parent", RESULT=="3600144f0979678fa000057f53db30016", NAME="asm-reco", OWNER="oracle", GROUP="dba", MODE="0660"
KERNEL=="sd?1", BUS=="scsi", PROGRAM=="/sbin/scsi_id -g -u -d /dev/$parent", RESULT=="3600144f0979678fa000057f53cd70014", NAME="asm-CRS", OWNER="oracle", GROUP="dba", MODE="0660"
KERNEL=="sd?1", BUS=="scsi", PROGRAM=="/sbin/scsi_id -g -u -d /dev/$parent", RESULT=="3600144f0979678fa000057f53de90017", NAME="oh-disk", OWNER="oracle", GROUP="dba", MODE="0660"

#configure SCSI devices as trusted(white listed)

# vi /etc/scsi_id.config
options=-g

[root@testohg ~]# cat /etc/udev/rules.d/99-oracle-asmdevices.rules
KERNEL=="sd?1", BUS=="scsi", PROGRAM=="/sbin/scsi_id -g -u -d /dev/$parent", RESULT=="3600144f0979678fa000057f53d770015", NAME="asm-data", OWNER="oracle", GROUP="dba", MODE="0660"
KERNEL=="sd?1", BUS=="scsi", PROGRAM=="/sbin/scsi_id -g -u -d /dev/$parent", RESULT=="3600144f0979678fa000057f53db30016", NAME="asm-reco", OWNER="oracle", GROUP="dba", MODE="0660"
KERNEL=="sd?1", BUS=="scsi", PROGRAM=="/sbin/scsi_id -g -u -d /dev/$parent", RESULT=="3600144f0979678fa000057f53cd70014", NAME="asm-CRS", OWNER="oracle", GROUP="dba", MODE="0660"
KERNEL=="sd?1", BUS=="scsi", PROGRAM=="/sbin/scsi_id -g -u -d /dev/$parent", RESULT=="3600144f0979678fa000057f53b610013", NAME="oh-disk", OWNER="oracle", GROUP="dba", MODE="0660"

# partprobe <devicename>
# udevtest
# udevadm control --reload-rules #or udevcontrol reload_rules. When do add, run "udevadm trigger --action=add"
# /sbin/start_udev

[root@testohg ~]# /sbin/scsi_id -g -u -d /dev/sdh
3600144f0979678fa000057f53d770015

[root@testohg ~]# /sbin/scsi_id -g -u -d /dev/sdi
3600144f0979678fa000057f53db30016

[root@testohg ~]# /sbin/scsi_id -g -u -d /dev/sdg
3600144f0979678fa000057f53cd70014

[root@testohg ~]# /sbin/scsi_id -g -u -d /dev/sdf
3600144f0979678fa000057f53b610013

---runcluvfy.sh
su - grid
cd grid
/u03/upgrade/11.2.0.4/software/grid/runcluvfy.sh stage -pre crsinst -n node1,node2 -verbose #run -fixup with root

cvu_stages

---install cvuqdisk
rpm -Uvh /u03/upgrade/11.2.0.4/software/grid/rpm/cvuqdisk-1.0.9-1.rpm

---install grid
vncviewer #su - grid
su - grid
/u03/upgrade/11.2.0.4/software/grid/runInstaller #Install and Configure Oracle Grid Infrastructure for a Cluster/Advanced Installation/scan-cluster(.localdomain), uncheck configure GNS<grid naming service>

Change Discovery Path -> /dev/oracleasm/disks

#GRIDDG -> VOL1_VOL2/first node1,then node2

run as root first in local node, then on other node(s):

/u01/app/oraInventory/orainstRoot.sh

/u01/app/12.1.0.2/grid/root.sh #Configure Oracle Grid Infrastructure for a Cluster

#For user - grep GROUP $OLD_GRID_HOME/crs/install/crsconfig_params

SILENT=false
ORACLE_OWNER=oracle
ORA_DBA_GROUP=oinstall
ORA_ASM_GROUP=oinstall
LANGUAGE_ID=AMERICAN_AMERICA.AL32UTF8
TZ=UTC
ISROLLING=true
REUSEDG=false
ASM_AU_SIZE=1
USER_IGNORED_PREREQ=true
INSTALL_NODE=node1.example.com

#LISTENER_USERNAME=
LISTENER_USERNAME=oracle
MGMT_DB=true
BIG_CLUSTER=false
HUB_SIZE=-1
HUB_NODE_LIST=
RIM_NODE_LIST=
HUB_NODE_VIPS=
PING_TARGETS=

ORACLE_HOME=/u01/app/12.1.0.2/grid
ORACLE_BASE=/u01/app/oracle
OLD_CRS_HOME=

JREDIR=/u01/app/12.1.0.2/grid/jdk/jre/
JLIBDIR=/u01/app/12.1.0.2/grid/jlib

VNDR_CLUSTER=false
OCR_LOCATIONS=NO_VAL
CLUSTER_NAME=ecradb
HOST_NAME_LIST=node1,node2
NODE_NAME_LIST=node1,node2
PRIVATE_NAME_LIST=
VOTING_DISKS=NO_VAL
#VF_DISCOVERY_STRING=%s_vfdiscoverystring%

# ASM consts
ASM_UPGRADE=false
ASM_SPFILE=
ASM_DISK_GROUP=DATA
ASM_DISCOVERY_STRING=/dev/oracleasm/disks
ASM_DISKS=/dev/oracleasm/disks/DATA01
ASM_REDUNDANCY=EXTERNAL
ASM_CONFIG=local
ASM_CREDENTIALS=

CRS_STORAGE_OPTION=1
CSS_LEASEDURATION=400
CRS_NODEVIPS='node1-vip.us.oracle.com/255.255.248.0/eth0,node2-vip.us.oracle.com/255.255.248.0/eth0'
NODELIST=node1,node2
NETWORKS="eth0"/10.240.216.0:public,"eth2"/192.168.1.0:cluster_interconnect
SCAN_NAME=node12-r.example.com
SCAN_PORT=1521
GPNP_PA=

# GNS consts
GNS_CONF=false
GNS_TYPE=
GNS_ADDR_LIST=
GNS_DOMAIN_LIST=
GNS_ALLOW_NET_LIST=
GNS_DENY_NET_LIST=
GNS_DENY_ITF_LIST=
GNS_CREDENTIALS=

#### Required by OUI add node
NEW_HOST_NAME_LIST=
NEW_NODE_NAME_LIST=
NEW_PRIVATE_NAME_LIST=
NEW_NODEVIPS='node1-vip.us.oracle.com/255.255.248.0/eth0,node2-vip.us.oracle.com/255.255.248.0/eth0'

############### OCR constants
# GPNPCONFIGDIR is handled differently in dev (T_HAS_WORK for all)
# GPNPGCONFIGDIR in dev expands to T_HAS_WORK_GLOBAL
GPNPCONFIGDIR=$ORACLE_HOME
GPNPGCONFIGDIR=$ORACLE_HOME
OCRLOC=
OLRLOC=
OCRID=
CLUSTER_GUID=

CLSCFG_MISSCOUNT=

#### IPD/OS
CRFHOME="/u01/app/12.1.0.2/grid"

 

---install RAC(aka RDBMS software)
vncviewer #su - oracle
/u03/upgrade/11.2.0.4/software/database/runInstaller #install database software only

run as root first in local node, then on other node(s):

$ORACLE_HOME/root.sh

---create DATA/FLASH disk groups
vncviewer to grid user
su - grid
asmca #create DATA/VOL3, FLASH/VOL4

---create db
vncviewer to oracle user
su - oracle
dbca #Admin-Managed/devdb/select node1,node2/+DATA/+FLASH/AL32UTF8_AL16UTF16, database template in $ORACLE_HOME/assistants/dbca/templates/. DBCA scripts in $ORACLE_BASE/admin/devdb/scripts/(will be removed when removing DB using dbca).

Scripts

1_preuser.sh

#!/bin/bash
#Purpose:Create 6 groups named 'oinstall','dba','asmadmin','asmdba','asmoper','oper', plus 2 users named 'oracle','grid'.
#Also setting the Environment
#variable for oracle user.
#variable for grid user.
#Usage:Log on as the superuser('root'),and then execute the command:#./1preusers.sh
#Author:Asher Huang
echo "Now create 6 groups named 'oinstall','dba','asmadmin','asmdba','asmoper','oper'"
echo "Plus 2 users named 'oracle','grid',Also setting the Environment"
groupadd -g 54321 oinstall
groupadd -g 54324 asmadmin
groupadd -g 54325 asmdba
groupadd -g 54326 asmoper
groupadd -g 54327 dba
groupadd -g 54328 oper
useradd -u 54329 -g oinstall -G asmadmin,asmdba,asmoper -d /home/grid -s /bin/bash -c "grid Infrastructure Owner" grid
echo "grid" | passwd --stdin grid
echo 'export PS1="`/bin/hostname`-> "'>> /home/grid/.bash_profile
echo "export TMP=/tmp">> /home/grid/.bash_profile
echo 'export TMPDIR=$TMP'>>/home/grid/.bash_profile
echo "export ORACLE_BASE=/u01/app/grid">> /home/grid/.bash_profile
echo "export ORACLE_HOME=/u01/app/11.2.0/grid">> /home/grid/.bash_profile
echo "export ORACLE_TERM=xterm">> /home/grid/.bash_profile
echo "export NLS_DATE_FORMAT='yyyy/mm/dd hh24:mi:ss'" >> /home/grid/.bash_profile
echo 'export TNS_ADMIN=$ORACLE_HOME/network/admin' >> /home/grid/.bash_profile
echo 'export PATH=/usr/sbin:$PATH'>> /home/grid/.bash_profile
echo 'export PATH=$ORACLE_HOME/bin:$PATH'>> /home/grid/.bash_profile
echo 'export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib'>> /home/grid/.bash_profile
echo 'export CLASSPATH=$ORACLE_HOME/JRE:$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib'>> /home/grid/.bash_profile
echo "export EDITOR=vi" >> /home/grid/.bash_profile
echo "export LANG=en_US" >> /home/grid/.bash_profile
echo "export NLS_LANG=american_america.AL32UTF8" >> /home/grid/.bash_profile
echo "umask 022">> /home/grid/.bash_profile

useradd -u 54330 -g oinstall -G dba,oper,asmdba -d /home/oracle -s /bin/bash -c "Oracle Software Owner" oracle
echo "oracle" | passwd --stdin oracle
echo 'export PS1="`/bin/hostname`-> "'>> /home/oracle/.bash_profile
echo "export TMP=/tmp">> /home/oracle/.bash_profile
echo 'export TMPDIR=$TMP'>>/home/oracle/.bash_profile
echo "export ORACLE_BASE=/u01/app/oracle">> /home/oracle/.bash_profile
echo 'export ORACLE_HOME=$ORACLE_BASE/product/11.2.0/db_1'>> /home/oracle/.bash_profile
echo "export ORACLE_UNQNAME=devdb">> /home/oracle/.bash_profile
echo 'export TNS_ADMIN=$ORACLE_HOME/network/admin' >> /home/oracle/.bash_profile
echo "export ORACLE_TERM=xterm">> /home/oracle/.bash_profile
echo 'export PATH=/usr/sbin:$PATH'>> /home/oracle/.bash_profile
echo 'export PATH=$ORACLE_HOME/bin:$PATH'>> /home/oracle/.bash_profile
echo 'export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib'>> /home/oracle/.bash_profile
echo 'export CLASSPATH=$ORACLE_HOME/JRE:$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib'>> /home/oracle/.bash_profile
echo "export EDITOR=vi" >> /home/oracle/.bash_profile
echo "export LANG=en_US" >> /home/oracle/.bash_profile
echo "export NLS_LANG=american_america.AL32UTF8" >> /home/oracle/.bash_profile
echo "export NLS_DATE_FORMAT='yyyy/mm/dd hh24:mi:ss'" >> /home/oracle/.bash_profile
echo "umask 022">> /home/oracle/.bash_profile
#should be changed on your env
#echo "export ORACLE_SID=+ASM1">> /home/grid/.bash_profile
#echo "export ORACLE_SID=+ASM2">> /home/grid/.bash_profile
#echo "export ORACLE_HOSTNAME=node1.us.test.com">> /home/oracle/.bash_profile
#echo "export ORACLE_HOSTNAME=node2.us.test.com">> /home/oracle/.bash_profile
#echo "export ORACLE_SID=devdb1">> /home/oracle/.bash_profile
#echo "export ORACLE_SID=devdb2">> /home/oracle/.bash_profile
chown -R oracle:oinstall /home/oracle/
chmod -R 755 /home/oracle/
chown -R grid:oinstall /home/grid
chmod -R 755 /home/grid
echo "The Groups and users has been created"
echo "The Environment for grid,oracle also has been set successfully"

2_predir.sh

#!/bin/bash
#Purpose:Create the necessary directory for oracle,grid users and change the authention to oracle,grid users.
#Usage:Log on as the superuser('root'),and then execute the command:#./2predir.sh
#Author:Asher Huang
echo "Now create the necessary directory for oracle,grid users and change the authention to oracle,grid users..."
mkdir -p /u01/app/grid
mkdir -p /u01/app/11.2.0/grid
mkdir -p /u01/app/oracle
chown -R oracle:oinstall /u01
chown -R grid:oinstall /u01/app/grid
chown -R grid:oinstall /u01/app/11.2.0
chmod -R 775 /u01
echo "The necessary directory for oracle,grid users and change the authention to oracle,grid users has been finished"

3_prelimits.sh

#!/bin/bash
#Purpose:Change the /etc/security/limits.conf.
#Usage:Log on as the superuser('root'),and then execute the command:#./3prelimits.sh
#Author:Asher Huang
echo "Now modify the /etc/security/limits.conf,but backup it named /etc/security/limits.conf.bak before"
cp /etc/security/limits.conf /etc/security/limits.conf.bak
echo "oracle soft nproc 2047" >>/etc/security/limits.conf
echo "oracle hard nproc 16384" >>/etc/security/limits.conf
echo "oracle soft nofile 1024" >>/etc/security/limits.conf
echo "oracle hard nofile 65536" >>/etc/security/limits.conf
echo "grid soft nproc 2047" >>/etc/security/limits.conf
echo "grid hard nproc 16384" >>/etc/security/limits.conf
echo "grid soft nofile 1024" >>/etc/security/limits.conf
echo "grid hard nofile 65536" >>/etc/security/limits.conf
echo "Modifing the /etc/security/limits.conf has been succeed."

4_prelogin.sh

#!/bin/bash
#Purpose:Modify the /etc/pam.d/login.
#Usage:Log on as the superuser('root'),and then execute the command:#./4prelimits.sh
#Author:Asher Huang
echo "Now modify the /etc/pam.d/login,but with a backup named /etc/pam.d/login.bak"
cp /etc/pam.d/login /etc/pam.d/login.bak
echo "session required /lib64/security/pam_limits.so" >>/etc/pam.d/login
echo "session required pam_limits.so" >>/etc/pam.d/login
echo "Modifing the /etc/pam.d/login has been succeed."

5_preprofile.sh

#!/bin/bash
#Purpose:Modify the /etc/profile.
#Usage:Log on as the superuser('root'),and then execute the command:#./5preprofile.sh
#Author:Asher Huang
echo "Now modify the /etc/profile,but with a backup named /etc/profile.bak"
cp /etc/profile /etc/profile.bak
echo 'if [ $USER = "oracle" ]||[ $USER = "grid" ]; then' >> /etc/profile
echo 'if [ $SHELL = "/bin/ksh" ]; then' >> /etc/profile
echo 'ulimit -p 16384' >> /etc/profile
echo 'ulimit -n 65536' >> /etc/profile
echo 'else' >> /etc/profile
echo 'ulimit -u 16384 -n 65536' >> /etc/profile
echo 'fi' >> /etc/profile
echo "Modifing the /etc/profile has been succeed."

6_presysctl.sh

#!/bin/bash
#Purpose:Modify the /etc/sysctl.conf.
#Usage:Log on as the superuser('root'),and then execute the command:#./6presysctl.sh
#Author:Asher Huang
echo "Now modify the /etc/sysctl.conf,but with a backup named /etc/sysctl.bak"
cp /etc/sysctl.conf /etc/sysctl.conf.bak
echo "fs.aio-max-nr = 1048576" >> /etc/sysctl.conf
echo "fs.file-max = 6815744" >> /etc/sysctl.conf
echo "kernel.shmall = 2097152" >> /etc/sysctl.conf
echo "kernel.shmmax = 2100549632" >> /etc/sysctl.conf
echo "kernel.shmmni = 4096" >> /etc/sysctl.conf
echo "kernel.sem = 500 144000 2048 8192" >> /etc/sysctl.conf
echo "net.ipv4.ip_local_port_range = 9000 65500" >> /etc/sysctl.conf
echo "net.core.rmem_default = 262144" >> /etc/sysctl.conf
echo "net.core.rmem_max = 4194304" >> /etc/sysctl.conf
echo "net.core.wmem_default = 262144" >> /etc/sysctl.conf
echo "net.core.wmem_max = 1048586" >> /etc/sysctl.conf
echo "net.ipv4.tcp_wmem = 262144 262144 262144" >> /etc/sysctl.conf
echo "net.ipv4.tcp_rmem = 4194304 4194304 4194304" >> /etc/sysctl.conf
echo "Modifing the /etc/sysctl.conf has been succeed."
echo "Now make the changes take effect....."
sysctl -p

Good Luck!


Categories: Databases, IT Architecture, Oracle DB Tags:
Comments are closed.