登录测试环境发现一个节点监听异常
root@xxxxdbb:/ #crsctl status res -t
--------------------------------------------------------------------------------
NAME TARGET STATE SERVER STATE_DETAILS
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.xxxx_DATA.dg
ONLINE ONLINE xxxxdba
ONLINE ONLINE xxxxdbb
ora.LISTENER.lsnr
ONLINE ONLINE xxxxdba
ONLINE OFFLINE xxxxdbb
ora.asm
ONLINE ONLINE xxxxdba Started
ONLINE ONLINE xxxxdbb Started
ora.gsd
OFFLINE OFFLINE xxxxdba
OFFLINE OFFLINE xxxxdbb
ora.net1.network
ONLINE ONLINE xxxxdba
ONLINE ONLINE xxxxdbb
ora.ons
ONLINE ONLINE xxxxdba
ONLINE ONLINE xxxxdbb
ora.registry.acfs
ONLINE ONLINE xxxxdba
ONLINE ONLINE xxxxdbb
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE xxxxdba
ora.cvu
1 ONLINE ONLINE xxxxdba
ora.xxxxdb.db
1 ONLINE ONLINE xxxxdba Open
2 ONLINE ONLINE xxxxdbb Open
ora.xxxxdb.xxxx.svc
1 ONLINE ONLINE xxxxdba
ora.xxxxdb.sjcj.svc
1 ONLINE ONLINE xxxxdba
ora.xxxxdb.zfbf.svc
1 ONLINE ONLINE xxxxdba
ora.xxxxdba.vip
1 ONLINE ONLINE xxxxdba
ora.xxxxdbb.vip
1 ONLINE ONLINE xxxxdbb
ora.oc4j
1 ONLINE ONLINE xxxxdba
ora.scan1.vip
1 ONLINE ONLINE xxxxdba
root@xxxxdbb:/ #su - grid |
尝试手工启动，一样报错
grid@xxxxdbb:/home/grid $lsnrctl statuss LSNRCTL for IBM/AIX RISC System/6000: Version 11.2.0.4.0 - Production on 04-MAR-2016 01:23:34 Copyright (c) 1991, 2013, Oracle. All rights reserved. Connecting to (DESCRIPTION=(ADDRESS=(PROTOCOL=IPC)(KEY=LISTENER))) TNS-12541: TNS:no listener TNS-12560: TNS:protocol adapter error TNS-00511: No listener IBM/AIX RISC System/6000 Error: 79: Connection refused grid@xxxxdbb:/home/grid $ grid@xxxxdbb:/home/grid $cat /etc/hosts |
第一怀疑就是hosts文件被改了，查看host没发现修改过。直接truss
从上面看出是使用ipc方式启动简单，尝试truss看看能不能有信息
truss lsnrctl start
execve("/usr/bin/lsnrctl", 0x2FF22BF8, 0x200138A8) Err#2 ENOENT
execve("/etc/lsnrctl", 0x2FF22BF8, 0x200138A8) Err#2 ENOENT
execve("/usr/sbin/lsnrctl", 0x2FF22BF8, 0x200138A8) Err#2 ENOENT
execve("/usr/ucb/lsnrctl", 0x2FF22BF8, 0x200138A8) Err#2 ENOENT
execve("/home/grid/bin/lsnrctl", 0x2FF22BF8, 0x200138A8) Err#2 ENOENT
execve("/usr/bin/X11/lsnrctl", 0x2FF22BF8, 0x200138A8) Err#2 ENOENT
execve("/sbin/lsnrctl", 0x2FF22BF8, 0x200138A8) Err#2 ENOENT
execve("./lsnrctl", 0x2FF22BF8, 0x200138A8) Err#2 ENOENT
execve("/u01/app/11.2.0.4/grid/bin/lsnrctl", 0x2FF22BF8, 0x200138A8) argc: 2
kusla(2, 0x09FFFFFFF0001170) Err#1 EPERM
read_sysconfig(0x09001000A07D1550, 0x0000000000000010, 0x0000000000000000, 0x00000000B0000010, 0x0000000110000E38, 0xBADC0FFEE0DDF00D, 0x0000000006010000, 0x09001000A0806E68) = 0x0000000000000000
sbrk(0x0000000000000000) = 0x00000001101D54D8
vmgetinfo(0x0FFFFFFFFFFFF140, 7, 16) = 0
sbrk(0x0000000000000000) = 0x00000001101D54D8
sbrk(0x0000000000000008) = 0x00000001101D54D8
__libc_sbrk(0x0000000000010020) = 0x00000001101D54E0
thread_init(0x0900000000520760, 0x09001000A0888470) =
sbrk(0x0000000000000000) = 0x00000001101E5500
vmgetinfo(0x0FFFFFFFFFFFF780, 7, 16) = 0
smcr_procattr(0, 1, 0x0FFFFFFFFFFFF778) Err#109 ENOSYS
getrpid(-1, -1, 648535941220675944) = 8912948
_getpid() = 8912948
getprocs64(0x00000001101D7BF0, 5024, 0x0000000000000000, 0, 0x09001000A0883098, 1) = 1
appulimit(1005, 0) = 0x0FFFFFFFFE000000
_thread_self() = 66453675
thread_setmystate(0x0000000000000000, 0x0FFFFFFFFFFFF2A0) = 0
thread_setmystate(0x0FFFFFFFFFFFEF00, 0x0FFFFFFFFFFFF288) = 0
_sigaction(3, 0x0FFFFFFFFFFFF670, 0x0FFFFFFFFFFFF6A0) = 0
_sigaction(4, 0x0FFFFFFFFFFFF670, 0x0FFFFFFFFFFFF6A0) = 0
_sigaction(5, 0x0FFFFFFFFFFFF670, 0x0FFFFFFFFFFFF6A0) = 0
_sigaction(6, 0x0FFFFFFFFFFFF670, 0x0FFFFFFFFFFFF6A0) = 0
_sigaction(7, 0x0FFFFFFFFFFFF670, 0x0FFFFFFFFFFFF6A0) = 0
_sigaction(8, 0x0FFFFFFFFFFFF670, 0x0FFFFFFFFFFFF6A0) = 0
_sigaction(10, 0x0FFFFFFFFFFFF670, 0x0FFFFFFFFFFFF6A0) = 0
_sigaction(11, 0x0FFFFFFFFFFFF670, 0x0FFFFFFFFFFFF6A0) = 0
_sigaction(12, 0x0FFFFFFFFFFFF670, 0x0FFFFFFFFFFFF6A0) = 0
_sigaction(36, 0x0FFFFFFFFFFFF670, 0x0FFFFFFFFFFFF6A0) = 0
.....省略....
_getpid() = 8912948
accessx("/etc/secvars.cfg", 04, 0) = 0
statx("/etc/secvars.cfg", 0x0FFFFFFFFFFF6FB0, 176, 0) = 0
_getpid() = 8912948
accessx("/etc/passwd", 04, 0) = 0
statx("/etc/passwd", 0x0FFFFFFFFFFF6E20, 176, 0) = 0
accessx("/etc/security/passwd", 04, 0) Err#13 EACCES
accessx("/etc/passwd", 04, 0) = 0
statx("/etc/passwd", 0x0FFFFFFFFFFF6B50, 176, 0) = 0
accessx("/etc/passwd", 04, 0) = 0
statx("/etc/passwd", 0x0FFFFFFFFFFF6C20, 176, 0) = 0
_getpid() = 8912948
accessx("/etc/passwd", 04, 0) = 0
statx("/etc/passwd", 0x0FFFFFFFFFFF6E20, 176, 0) = 0
accessx("/etc/security/passwd", 04, 0) Err#13 EACCES
accessx("/etc/passwd", 04, 0) = 0
statx("/etc/passwd", 0x0FFFFFFFFFFF6B50, 176, 0) = 0
accessx("/etc/passwd", 04, 0) = 0
statx("/etc/passwd", 0x0FFFFFFFFFFF6C20, 176, 0) = 0
accessx("/etc/security/passwd", 04, 0) Err#13 EACCES
close(5) = 0
gethostname(0x0FFFFFFFFFFF9C00, 512) = 0
access("/tmp/.oracle", 0) = 0
chmod("/tmp/.oracle", 01777) Err#1 EPERM
socket(1, 1, 0) = 5
access("/tmp/.oracle/sLISTENER", 0) = 0
connext(5, 0x0FFFFFFFFFFF5BA8, 1025) Err#79 ECONNREFUSED -->注意这里，发现要读取/tmp/.oracle/sLISTENER
access("/tmp/.oracle/sLISTENER", 0) = 0
_nsleep(0x0FFFFFFFFFFF5580, 0x0FFFFFFFFFFF5650) = 0
close(5) = 0
socket(1, 1, 0) = 5
connext(5, 0x0FFFFFFFFFFF5BA8, 1025) Err#79 ECONNREFUSED
access("/tmp/.oracle/sLISTENER", 0) = 0
_nsleep(0x0FFFFFFFFFFF5580, 0x0FFFFFFFFFFF5650) = 0
close(5) = 0
socket(1, 1, 0) = 5
connext(5, 0x0FFFFFFFFFFF5BA8, 1025) Err#79 ECONNREFUSED
access("/tmp/.oracle/sLISTENER", 0) = 0
_nsleep(0x0FFFFFFFFFFF5580, 0x0FFFFFFFFFFF5650) = 0
close(5) = 0
socket(1, 1, 0) = 5
connext(5, 0x0FFFFFFFFFFF5BA8, 1025) Err#79 ECONNREFUSED
access("/tmp/.oracle/sLISTENER", 0) = 0
_nsleep(0x0FFFFFFFFFFF5580, 0x0FFFFFFFFFFF5650) = 0
close(5) = 0
socket(1, 1, 0) = 5
connext(5, 0x0FFFFFFFFFFF5BA8, 1025) Err#79 ECONNREFUSED
access("/tmp/.oracle/sLISTENER", 0) = 0
_nsleep(0x0FFFFFFFFFFF5580, 0x0FFFFFFFFFFF5650) = 0
close(5) = 0
socket(1, 1, 0) = 5
connext(5, 0x0FFFFFFFFFFF5BA8, 1025) Err#79 ECONNREFUSED
access("/tmp/.oracle/sLISTENER", 0) = 0
_nsleep(0x0FFFFFFFFFFF5580, 0x0FFFFFFFFFFF5650) = 0
close(5) = 0
socket(1, 1, 0) = 5
connext(5, 0x0FFFFFFFFFFF5BA8, 1025) Err#79 ECONNREFUSED
access("/tmp/.oracle/sLISTENER", 0) = 0
_nsleep(0x0FFFFFFFFFFF5580, 0x0FFFFFFFFFFF5650) = 0
close(5) = 0
socket(1, 1, 0) = 5
connext(5, 0x0FFFFFFFFFFF5BA8, 1025) Err#79 ECONNREFUSED
access("/tmp/.oracle/sLISTENER", 0) = 0
_nsleep(0x0FFFFFFFFFFF5580, 0x0FFFFFFFFFFF5650) = 0
close(5) = 0
socket(1, 1, 0) = 5
connext(5, 0x0FFFFFFFFFFF5BA8, 1025) Err#79 ECONNREFUSED
access("/tmp/.oracle/sLISTENER", 0) = 0
_nsleep(0x0FFFFFFFFFFF5580, 0x0FFFFFFFFFFF5650) = 0
close(5) = 0
socket(1, 1, 0) = 5
connext(5, 0x0FFFFFFFFFFF5BA8, 1025) Err#79 ECONNREFUSED
access("/tmp/.oracle/sLISTENER", 0) = 0
_nsleep(0x0FFFFFFFFFFF5580, 0x0FFFFFFFFFFF5650) = 0
close(5) = 0
socket(1, 1, 0) = 5
connext(5, 0x0FFFFFFFFFFF5BA8, 1025) Err#79 ECONNREFUSED
access("/tmp/.oracle/sLISTENER", 0) = 0
_nsleep(0x0FFFFFFFFFFF5580, 0x0FFFFFFFFFFF5650) = 0
close(5) = 0
socket(1, 1, 0) = 5
connext(5, 0x0FFFFFFFFFFF5BA8, 1025) Err#79 ECONNREFUSED
access("/tmp/.oracle/sLISTENER", 0) = 0
_nsleep(0x0FFFFFFFFFFF5580, 0x0FFFFFFFFFFF5650) = 0
close(5) = 0
socket(1, 1, 0) = 5
connext(5, 0x0FFFFFFFFFFF5BA8, 1025) Err#79 ECONNREFUSED
access("/tmp/.oracle/sLISTENER", 0) = 0
_nsleep(0x0FFFFFFFFFFF5580, 0x0FFFFFFFFFFF5650) = 0
close(5) = 0
socket(1, 1, 0) = 5
connext(5, 0x0FFFFFFFFFFF5BA8, 1025) Err#79 ECONNREFUSED
access("/tmp/.oracle/sLISTENER", 0) = 0
_nsleep(0x0FFFFFFFFFFF5580, 0x0FFFFFFFFFFF5650) = 0
close(5) = 0
socket(1, 1, 0) = 5
connext(5, 0x0FFFFFFFFFFF5BA8, 1025) Err#79 ECONNREFUSED
access("/tmp/.oracle/sLISTENER", 0) = 0
_nsleep(0x0FFFFFFFFFFF5580, 0x0FFFFFFFFFFF5650) = 0
close(5) = 0
socket(1, 1, 0) = 5
connext(5, 0x0FFFFFFFFFFF5BA8, 1025) Err#79 ECONNREFUSED
access("/tmp/.oracle/sLISTENER", 0) = 0
_nsleep(0x0FFFFFFFFFFF5580, 0x0FFFFFFFFFFF5650) = 0
close(5) = 0
socket(1, 1, 0) = 5
connext(5, 0x0FFFFFFFFFFF5BA8, 1025) Err#79 ECONNREFUSED
access("/tmp/.oracle/sLISTENER", 0) = 0
_nsleep(0x0FFFFFFFFFFF5580, 0x0FFFFFFFFFFF5650) = 0
close(5) = 0
socket(1, 1, 0) = 5
connext(5, 0x0FFFFFFFFFFF5BA8, 1025) Err#79 ECONNREFUSED
access("/tmp/.oracle/sLISTENER", 0) = 0
close(5) = 0
kopen("/u01/app/11.2.0.4/grid/network/mesg/tnsus.msb", O_RDONLY) = 5
kfcntl(5, F_SETFD, 0x0000000000000001) = 0
lseek(5, 0, 0) = 0
kread(5, "1513 "011303\t\t\0\0\0\0".., 256) = 256
lseek(5, 512, 0) = 512
kread(5, " 19B\0\0\0\0\0\0\0\0\0\0".., 512) = 512
lseek(5, 1024, 0) = 1024
kread(5, "\0\t\012\01A\0 &\0 -\0 F".., 172) = 172
lseek(5, 19456, 0) = 19456
kread(5, "\00F04 $\0\0\0 b04 %\0\0".., 512) = 512
Starting /u01/app/11.2.0.4/grid/bin/tnslsnr: please wait...
kwrite(1, " S t a r t i n g / u 0".., 61) = 61
kfcntl(1, F_GETFL, 0x0000000000000008) = 67110914
pipe(0x0FFFFFFFFFFF80E0) = 0
pipe(0x0FFFFFFFFFFF80D8) = 0
sigprocmask(0, 0x09001000A090DB70, 0x09001000A090DB90) = 0
kfork() = 19988596
thread_setmystate(0x0FFFFFFFFFFF7A70, 0x0000000000000000) = 0
Received signal #20, SIGCHLD [default]
sigprocmask(0, 0x0FFFFFFFFFFF7E48, 0x0000000000000000) = 0
sigprocmask(2, 0x09001000A090DB70, 0x0FFFFFFFFFFF7BF0) = 0
_sigaction(13, 0x0FFFFFFFFFFF7D20, 0x0FFFFFFFFFFF7D78) = 0
thread_setmystate(0x0FFFFFFFFFFF77F0, 0x0000000000000000) = 0
sigprocmask(1, 0x0FFFFFFFFFFF7EC8, 0x0000000000000000) = 0
kwaitpid(0x0FFFFFFFFFFF8090, 19988596, 4, 0x0000000000000000, 0x0000000000000000) = 19988596
close(6) = 0
close(9) = 0
kfcntl(8, F_SETFD, 0x0000000000000001) = 0
kfcntl(7, F_SETFD, 0x0000000000000001) = 0
kread(8, " N T P 0 1 4 4 8 3 5 7".., 64) = 14
_getpid() = 8912948
kfcntl(8, F_SETFD, 0x0000000000000001) = 0
kwrite(7, "\0 \001\0\0\001 :01 ,".., 179) = 179
kread(8, "\0\f\0\004\0\0\0 "\001 ., 8208) = 492
close(7) = 0
close(8) = 0
TNSLSNR for IBM/AIX RISC System/6000: Version 11.2.0.4.0 - Production
System parameter file is /u01/app/11.2.0.4/grid/network/admin/listener.ora
Log messages written to /u01/app/grid/diag/tnslsnr/xxxxdbb/listener/alert/log.xml
Error listening on: (DESCRIPTION=(ADDRESS=(PROTOCOL=IPC)(KEY=LISTENER)))
TNS-12555: TNS:permission denied
TNS-12560: TNS:protocol adapter error
TNS-00525: Insufficient privilege for operation
IBM/AIX RISC System/6000 Error: 1: Not owner
kwrite(1, " T N S L S N R f o r ".., 470) = 470
kwrite(1, "\n", 1) = 1
lseek(5, 19968, 0) = 19968
kread(5, "\0\r04 5\0\0\0 V04 6\0\0".., 512) = 512
Listener failed to start. See the error message(s) above...
kwrite(1, " L i s t e n e r f a i".., 61) = 61
kfcntl(1, F_GETFL, 0x0000000000000008) = 67110914
close(4) = 0
kfcntl(1, F_GETFL, 0x0000000000000008) = 67110914
close(5) = 0
kfcntl(1, F_GETFL, 0x0000000013F600AB) = 67110914
kfcntl(2, F_GETFL, 0x0000000013F600AB) = 67110914
_exit(1) |
尝试删除/tmp/.oracle/sLISTENER后监听正常启动
