CDH6.1.1+CM6.1.0版本安装过程及问题记录

准备工作
1.安装包准备

2.服务器初始化

所有节点--
##修改hostname
hostnamectl set-hostname jxq-100-73-13-30
##修改网卡
vi /etc/sysconfig/network
NETWORKWAIT=1
NOZEROCONF=no
HOSTNAME=jxq-100-73-13-30
##重启网关
systemctl  restart  network
##修改hosts文件
[root@jxq-100-73-13-30 ~]# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
100.73.13.30 jxq-100-73-13-30
...
...
##禁用ipv6
grubby --args="ipv6.disable=1" --update-kernel=ALL
##免密操作(root/hadoop)
ssh-keygen
ssh-copy-id -i root@${ip}或
yum install -y sshpass
sshpass -p "***" ssh -o StrictHostKeyChecking=no root@${ip} "ssh-keygen"
sshpass -p "***" ssh-copy-id -i root@${ip}##关闭防火墙
systemctl stop firewalld.service
systemctl disable firewalld.service
systemctl status firewalld.service##修改内核参数
#修改transparent_hugepage参数(透明大页面压缩)
#修改系统参数，这一参数默认值可能会导致CDH性能下降。
#执行如下命令：
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
#关闭告警：
echo 'echo never > /sys/kernel/mm/transparent_hugepage/defrag' >> /etc/rc.d/rc.local
echo 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' >> /etc/rc.d/rc.local
echo "alias ipv6 off" >> /etc/modprobe.d/dist.conf  
echo "alias net-pf-10 off" >> /etc/modprobe.d/dist.conf
##修改swap空间的swappiness=0
##该配置的目的为CDH最大限度的使用环境内存，执行如下命名：
cat  /proc/sys/vm/swappiness 
sysctl vm.swappiness=0
echo 0 > /proc/sys/vm/swappiness
#关闭告警：
echo never > /sys/kernel/mm/transparent_hugepage/defrag##关闭selinux
vi /etc/sysconfig/selinux
SELINUX=disabled##修改NTP设置
yum -y install ntp
vi /etc/ntp.conf
...
systemctl start ntpd
systemctl enable ntpd.service
##查看ntp服务器与上层ntp的状态
ntpq -p
ntpstat
systemctl ntpd restart // 重启
systemctl  ntpd  enable // 开机启动##JDK配置，下载安装包
http://download.oracle.com/otn-pub/java/jdk/8u181-b13/96a7b8442fe848ef90c96a2fad6ed6d1/jdk-8u181-linux-x64.tar.gzexport JAVA_HOME=/usr/java/jdk1.8.0_141-cloudera
export JRE_HOME=${JAVA_HOME}/jre
export MYSQL_HOME=/data/mysql
export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
export PATH=${JAVA_HOME}/bin:${MYSQL_HOME}/bin:$PATH

3.主节点配置

##配置yum源
wget https://archive.cloudera.com/cm6/6.1.0/redhat7/yum/cloudera-manager.repo -P /etc/yum.repos.d/
rpm --import https://archive.cloudera.com/cm6/6.0.0/redhat7/yum/RPM-GPG-KEY-cloudera
##提前下载相关的RPM安装包以及封装的组件的Parcel包
mkdir -p /opt/cloudera-manager
yum -y install oracle-j2sdk1.8-1.8.0+update141-1.x86_64.rpm 
yum -y install cloudera-manager-daemons-6.1.0-769885.el7.x86_64.rpm  
yum -y install cloudera-manager-server-6.1.0-769885.el7.x86_64.rpm 
yum -y install cloudera-manager-agent-6.1.0-769885.el7.x86_64.rpm
##配置httpd
yum -y install httpd createrepo
systemctl start httpd && systemctl enable httpd##将安装包拷贝至/var/www/html/cm6.1.0/yum目录下
ll /var/www/html/cloudera-repos/
cloudera-manager-agent-6.1.0-769885.el7.x86_64.rpm  
cloudera-manager-daemons-6.1.0-769885.el7.x86_64.rpm    
cloudera-manager-server-6.1.0-769885.el7.x86_64.rpm 
cloudera-manager-server-db-2-6.1.0-769885.el7.x86_64.rpm    
oracle-j2sdk1.8-1.8.0+update141-1.x86_64.rpm
repodata##Centos7版本下的CDH6.1.0二进制Parcel包下载地址（下载与系统匹配的包）：https://archive.cloudera.com/cdh6/6.1.0/parcels/
CDH-6.1.0-1.cdh6.1.0.p0.770702-el7.parcel
CDH-6.1.0-1.cdh6.1.0.p0.770702-el7.parcel.sha256
manifest.json

4.下载安装MySQL

可参考：https://blog.csdn.net/weixin_44154134/article/details/106380755
##相关组件元数据库配置
grant all privileges on *.* to 'root'@'%' identified by '**';
grant all on scm.* to 'scm'@'%' identified by '**';
grant all on amon.* to 'amon'@'%' identified by 'amon';
grant all on rman.* to 'rman'@'%' identified by 'rman';
grant all on hue.* to 'hue'@'%' identified by 'hue';
grant all on metastore.* to 'hive'@'%' identified by 'metastore';
grant all on sentry.* to 'sentry'@'%' identified by 'sentry';
grant all on nav.* to 'nav'@'%' identified by 'nav';
grant all on oozie.* to 'oozie'@'%' identified by 'oozie';
flush privileges;mysql驱动：
[root@jxq-100-73-13-31 java]# pwd
/usr/share/java
[root@jxq-100-73-13-31 java]# ll
total 7156
-rw-r--r-- 1 root root 3930793 May  7 20:22 mysql-connector-java-8.0.20.tar.gz
-rw-r--r-- 1 root root 2385601 Jun  7 18:05 mysql-connector-java.jar

5.使用cm初始化mysql

/opt/cloudera/cm/schema/scm_prepare_database.sh mysql cm -hlocalhost -uroot -p** --scm-host localhost scm scm scm

6.所有节点安装cloudera-manager-agent

##配置yum源
[root@jxq-100-73-13-31 repos.d]# pwd
/etc/yum.d/repos.d
[root@jxq-100-73-13-31 repos.d]# cat cloudera-manager.repo 
[cloudera-manager]
name=Cloudera Manager 6.1.0
baseurl=http://100.73.13.30/cm6.1.0/yum/
#gpgkey=https://archive.cloudera.com/cm6/6.1.0/redhat7/yum/RPM-GPG-KEY-cloudera
gpgcheck=0

7.agent配置

systemctl enable cloudera-scm-agent.service
sed  -i '/hadoop config/,$d' /etc/security/limits.conf
cat >>/etc/security/limits.conf <<EOF ##### hadoop config ##### * soft nsoft 65536 * hard nsoft 65536 * soft nproc 65536 * hard nproc 65536 hadoop soft nofile 131072 hadoop hard nofile 131072 yarn soft nofile 131072 yarn hard nofile 131072 hdfs soft nofile 131072 hdfs hard nofile 131072 hbase soft nofile 131072 hbase hard nofile 131072 EOF##server_host修改为主节点
vim /etc/cloudera-scm-agent/config.ini
[root@jxq-100-73-13-31 java]# cat /etc/cloudera-scm-agent/config.ini
[General]
# Hostname of the CM server.
server_host=100.73.13.30

8.启动服务

##主节点，启动cloudera-scm-server
systemctl start cloudera-scm-server##子节点，启动cloudera-scm-agent
systemctl start cloudera-scm-agent使用初始化密码(admin/admin)访问，进行web端安装：
http://$IP:7180/cmf/login

安装参考链接：

https://www.cnblogs.com/yyy-blog/p/10697234.html

问题汇总：

##1、CDH entropy was available 问题解决-
https://www.jianshu.com/p/037f9dda5e7d?utm_campaign=maleskine&utm_content=note&utm_medium=seo_notes&utm_source=recommendation##2、CM+CDH 主机运行状况不良
1）cm 安装时，hosts显示受管，可通过删除scm库，适用于集群首次安装场景
2）https://blog.csdn.net/u010766519/article/details/90633887##3、cloudera-scm-agent start 启动失败
https://blog.csdn.net/qq_16162763/article/details/81359680##4、hive之权限问题（org.apache.hadoop.security.AccessControlException Permission denied: user=root, access=WR）-
hadoop fs -setfacl -m -R user:root:rwx /user
hadoop fs -getfacl /user##5、hdfs数据块丢失处理
https://blog.csdn.net/zgc625238677/article/details/105194439