当前位置: 代码迷 >> 综合 >> CDH6.3.0集群在Centos7.9上的安装
  详细解决方案

CDH6.3.0集群在Centos7.9上的安装

热度:51   发布时间:2023-11-27 22:41:09.0

一、概览

主机名 ip 配置
master 192.168.159.160 Centos7.9 8G 100G 2core
slave1 192.168.159.161 Centos7.9 8G 100G 2core
slave2 192.168.159.162 Centos7.9 8G 100G 2core

二、虚拟机配置

1、修改主机名和hosts

  • 分别执行以下命令修改主机名
[root@master ~]# hostnamectl set-hostname master
[root@slave1 ~]# hostnamectl set-hostname slave1
[root@slave2 ~]# hostnamectl set-hostname slave2
  • 修改hosts文件
[root@master ~]# vim /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.159.160 master
192.168.159.161 slave1
192.168.159.162 slave2
  • 修改后ping各个主机,验证是否连通
[root@master ~]# ping slave1
PING slave1 (192.168.159.161) 56(84) bytes of data.
64 bytes from slave1 (192.168.159.161): icmp_seq=1 ttl=64 time=0.395 ms
64 bytes from slave1 (192.168.159.161): icmp_seq=2 ttl=64 time=0.879 ms
64 bytes from slave1 (192.168.159.161): icmp_seq=3 ttl=64 time=0.837 ms
64 bytes from slave1 (192.168.159.161): icmp_seq=4 ttl=64 time=1.02 ms

2、修改静态ip

  • 依次修改三台主机的ip地址,如下:
[root@master ~]# vim /etc/sysconfig/network-scripts/ifcfg-ens33TYPE="Ethernet"
PROXY_METHOD="none"
BROWSER_ONLY="no"
BOOTPROTO="static"
DEFROUTE="yes"
IPV4_FAILURE_FATAL="no"
IPV6INIT="yes"
IPV6_AUTOCONF="yes"
IPV6_DEFROUTE="yes"
IPV6_FAILURE_FATAL="no"
IPV6_ADDR_GEN_MODE="stable-privacy"
NAME="ens33"
DEVICE="ens33"
ONBOOT="yes"
IPADDR=192.168.159.160
NETMASK=255.255.255.0
GATEWAY=192.168.159.2
DNS1=192.168.159.2
  • 修改完毕后重启网络
[root@master ~]# systemctl restart network

3、关闭防火墙

  • 关闭、禁用、查看当前防火墙状态
[root@master ~]# systemctl stop firewalld && systemctl disable firewalld && systemctl status firewalldRemoved symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
● firewalld.service - firewalld - dynamic firewall daemonLoaded: loaded (/usr/lib/systemd/system/firewalld.service; disabled; vendor preset: enabled)Active: inactive (dead)Docs: man:firewalld

4、禁用SELinux

  • 检查selinux的状态
[root@master ~]# getenforceEnforcing   //输出若为Permissive 或者 Disabled,跳过本步骤
  • 打开/etc/selinux/config文件,修改SELINUX=Enforcing为SELINUX=permissive
[root@master ~]# vim /etc/selinux/config# This file controls the state of SELinux on the system.
# SELINUX= can take one of these three values:
# enforcing - SELinux security policy is enforced.
# permissive - SELinux prints warnings instead of enforcing.
# disabled - No SELinux policy is loaded.
SELINUX=permissive
# SELINUXTYPE= can take one of three values:
# targeted - Targeted processes are protected,
# minimum - Modification of targeted policy. Only selected processes are protected. 
# mls - Multi Level Security protection.
SELINUXTYPE=targeted
  • 保存修改,执行以下命令立即禁用SELinux(安装完CDH后可以修改回enforcing模式)
[root@master ~]# setenforce 0

5、配置root用户之间免密登录

[root@master ~]# cd .ssh
[root@master .ssh]# ssh-keygen -t rsa
#一路回车..........
[root@master .ssh]# cat id_rsa.pub >> authorized_keys
[root@master .ssh]# ls
authorized_keys  id_rsa  id_rsa.pub  known_hosts
[root@master .ssh]# ssh master
Last login: Mon Dec 28 11:45:39 2020 from 192.168.159.1#将master节点的公钥复制到slave的authorized_keys中即可
[root@slave1 .ssh]# ssh-copy-id -i slave1
#输入yes和密码
[root@slave1 .ssh]# ssh slave1
Last login: Mon Dec 28 11:56:42 2020 from fe80::d7d4:6826:4c17:9459%ens33
[root@master ~]# 

6、安装ntp

  • 禁用chronyd服务
[root@master ~]# systemctl stop chronyd && systemctl disable chronyd && systemctl status chronyd
  • 安装和启用ntp服务
[root@slave2 ~]# yum install ntp
#主机master配置
[root@master ~]# vim /etc/ntp.conf 
#server 0.centos.pool.ntp.org iburst
#server 1.centos.pool.ntp.org iburst
#server 2.centos.pool.ntp.org iburst
#server 3.centos.pool.ntp.org iburst#新增
server  127.127.1.0
fudge   127.127.1.0 stratum 10#从机slave设置
[root@slave2 ~]# vim /etc/ntp.conf
#server 0.centos.pool.ntp.org iburst
#server 1.centos.pool.ntp.org iburst
#server 2.centos.pool.ntp.org iburst
#server 3.centos.pool.ntp.org iburst#新增
server master#启动ntp服务
[root@master ~]# systemctl start ntpd
#设置开机启动
[root@master ~]# systemctl enable ntpd
Created symlink from /etc/systemd/system/multi-user.target.wants/ntpd.service to /usr/lib/systemd/system/ntpd.service.#将系统时钟同步到NTP服务器:
[root@master ~]# ntpdate -u <ntp服务器>
#将硬件时钟与系统时钟同步:
[root@master ~]#hwclock --systohc

7、安装mysql

  • 在master节点上安装mysql,MySQL下载地址,下载什么版本自己根据情况决定,我这里下载的是mysql-5.7.27-1.el7.x86_64.rpm-bundle.tar,里面包含所有安装MySQL需要的RPM包。
[root@master download]# tar -xvf mysql-5.7.27-1.el7.x86_64.rpm-bundle.tar -C .
mysql-community-libs-5.7.27-1.el7.x86_64.rpm
mysql-community-embedded-devel-5.7.27-1.el7.x86_64.rpm
mysql-community-libs-compat-5.7.27-1.el7.x86_64.rpm
mysql-community-devel-5.7.27-1.el7.x86_64.rpm
mysql-community-embedded-compat-5.7.27-1.el7.x86_64.rpm
mysql-community-common-5.7.27-1.el7.x86_64.rpm
mysql-community-client-5.7.27-1.el7.x86_64.rpm
mysql-community-server-5.7.27-1.el7.x86_64.rpm
mysql-community-test-5.7.27-1.el7.x86_64.rpm
mysql-community-embedded-5.7.27-1.el7.x86_64.rpm
[root@master download]# rpm -i mysql-community-common-5.7.27-1.el7.x86_64.rpm 
[root@master download]# rpm -i mysql-community-libs-5.7.27-1.el7.x86_64.rpm 
[root@master download]# rpm -i mysql-community-client-5.7.27-1.el7.x86_64.rpm 
[root@master download]# rpm -i mysql-community-server-5.7.27-1.el7.x86_64.rpm
#查看msyql服务状态
[root@master download]# systemctl status mysqld
● mysqld.service - MySQL ServerLoaded: loaded (/usr/lib/systemd/system/mysqld.service; enabled; vendor preset: disabled)Active: inactive (dead)Docs: man:mysqld(8)http://dev.mysql.com/doc/refman/en/using-systemd.html
#启动mysql
[root@master download]# systemctl start mysqld
[root@master download]# systemctl status mysqld
● mysqld.service - MySQL ServerLoaded: loaded (/usr/lib/systemd/system/mysqld.service; enabled; vendor preset: disabled)Active: active (running) since 一 2020-12-28 14:00:03 CST; 2s agoDocs: man:mysqld(8)http://dev.mysql.com/doc/refman/en/using-systemd.htmlProcess: 50627 ExecStart=/usr/sbin/mysqld --daemonize --pid-file=/var/run/mysqld/mysqld.pid $MYSQLD_OPTS (code=exited, status=0/SUCCESS)Process: 50553 ExecStartPre=/usr/bin/mysqld_pre_systemd (code=exited, status=0/SUCCESS)Main PID: 50631 (mysqld)CGroup: /system.slice/mysqld.service└─50631 /usr/sbin/mysqld --daemonize --pid-file=/var/run/mysqld/mysqld.pid
#查看启动mysql生成的临时密码
[root@master download]# cat /var/log/mysqld.log | grep 'password'
2020-12-28T05:59:55.350376Z 1 [Note] A temporary password is generated for root@localhost: tZF%khh1_8ha
#修改密码
[root@master download]# mysql -uroot -p
mysql>ALTER USER 'root'@'localhost' IDENTIFIED BY 'Password@123';
#若出现ERROR 1819 (HY000): Your password does not satisfy the current policy requirements
#则先设置一个不少于8位数的复杂密码,然后执行以下sql
mysql> set global validate_password_policy=LOW;
mysql> set global validate_password_length=4;
mysql> show variables like 'validate_password%';
+--------------------------------------+-------+
| Variable_name                        | Value |
+--------------------------------------+-------+
| validate_password_check_user_name    | OFF   |
| validate_password_dictionary_file    |       |
| validate_password_length             | 4     |
| validate_password_mixed_case_count   | 1     |
| validate_password_number_count       | 1     |
| validate_password_policy             | LOW   |
| validate_password_special_char_count | 1     |
+--------------------------------------+-------+
#然后再修改简单的密码
#启用远程连接
mysql> use mysql;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -ADatabase changed
mysql> select user,host from user;
+---------------+-----------+
| user          | host      |
+---------------+-----------+
| mysql.session | localhost |
| mysql.sys     | localhost |
| root          | localhost |
+---------------+-----------+
3 rows in set (0.00 sec)mysql> update user set host='%' where user='root';
Query OK, 1 row affected (0.00 sec)
Rows matched: 1  Changed: 1  Warnings: 0mysql> flush privileges;
Query OK, 0 rows affected (0.00 sec)
#配置msyql数据库和权限
create database scm default character set utf8 default collate utf8_general_ci;
grant all on scm.* to 'scm'@'%' identified by 'cdh_scm';
create database amon default character set utf8 default collate utf8_general_ci;
grant all on amon.* to 'amon'@'%' identified by 'cdh_amon';
create database rman default character set utf8 default collate utf8_general_ci;
grant all on rman.* to 'rman'@'%' identified by 'cdh_rman';
create database hue default character set utf8 default collate utf8_general_ci;
grant all on hue.* to 'hue'@'%' identified by 'cdh__hue';
create database metastore default character set utf8 default collate utf8_general_ci;
grant all on metastore.* to 'hive'@'%' identified by 'cdh_hive';
create database sentry default character set utf8 default collate utf8_general_ci;
grant all on sentry.* to 'sentry'@'%' identified by 'cdh_sentry';
create database nav default character set utf8 default collate utf8_general_ci;
grant all on nav.* to 'nav'@'%' identified by 'cdh_nav';
create database oozie default character set utf8 default collate utf8_general_ci;
grant all on oozie.* to 'oozie'@'%' identified by 'cdh_oozie';
flush privileges;

8、安装http服务器和createrepo

[root@master download]# yum install httpd createrepo
[root@master download]# systemctl start httpd
[root@master download]# systemctl status httpd
● httpd.service - The Apache HTTP ServerLoaded: loaded (/usr/lib/systemd/system/httpd.service; disabled; vendor preset: disabled)Active: active (running) since 一 2020-12-28 15:39:51 CST; 4s agoDocs: man:httpd(8)man:apachectl(8)

9、安装MySQL的jdbc驱动

  • 下载MySQL的jdbc驱动,将驱动改名为mysql-connector-java.jar,然后将驱动复制到/usr/share/java目录下(没有该目录则自己创建,所有机器上都要装驱动)。

10、修改swappiness参数

  • 修改Linux swappiness参数(所有节点)
    为了避免服务器使用swap功能而影响服务器性能,一般都会把vm.swappiness修改为0(cloudera建议10以下)
[root@master download]# cd /usr/lib/tuned/
[root@master tuned]# grep "vm.swappiness" * -R
latency-performance/tuned.conf:vm.swappiness=10
throughput-performance/tuned.conf:vm.swappiness=10
virtual-guest/tuned.conf:vm.swappiness = 30
[root@master tuned]# vim latency-performance/tuned.conf 
[root@master tuned]# vim throughput-performance/tuned.conf 
[root@master tuned]# vim virtual-guest/tuned.conf 
[root@master tuned]# grep "vm.swappiness" * -R
latency-performance/tuned.conf:vm.swappiness=0
throughput-performance/tuned.conf:vm.swappiness=0
virtual-guest/tuned.conf:vm.swappiness = 0
#修改完成后同步到其它节点

11、禁用透明页(所有节点)

[root@master ~]# vim /etc/rc.local
在文件中添加如下内容:
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
#然后将该文件同步其他机器上,然后启动所有服务器

12、配置本地Parcel存储库

  • 创建/opt/cloudera/parcel-repo目录,将下载好的parcel和manifest.json文件放置到该目录,然后执行下面的命令:
[root@cdh01 parcel-repo]# sha1sum CDH-6.3.0-1.cdh6.3.0.p0.1279813-el7.parcel | awk '{ print $1 }' > CDH-6.3.0-1.cdh6.3.0.p0.1279813-el7.parcel.sha
[root@cdh01 parcel-repo]# ll
total 2036856
-rwxr-xr-x 1 root    root         2085690155 Oct  9 15:29 CDH-6.3.0-1.cdh6.3.0.p0.1279813-el7.parcel
-rw-r--r-- 1 root    root            41 Oct 11 14:36 CDH-6.3.0-1.cdh6.3.0.p0.1279813-el7.parcel.sha
-rwxr-xr-x 1 root    root         64 Oct  9 14:18 CDH-6.3.0-1.cdh6.3.0.p0.1279813-el7.parcel.sha256
-rwxr-xr-x 1 root    root         33887 Oct 11 14:35 manifest.json
[root@master yum.repos.d]# yum clean all && yum makecache
#安装CDH需要的依赖包
[root@master cloudera-repo]# yum install edhat-lsb portmap mod_ssl openssl-devel python-psycopg2 MySQL-python chkconfig bind-utils libxslt zlib sqlite cyrus-sasl-plain cyrus-sasl-gssapi fuse fuse-libs
#如果安装报下面的错
2:postfix-2.10.1-6.el7.x86_64 has missing requires of libmysqlclient.so.18()(64bit)
2:postfix-2.10.1-6.el7.x86_64 has missing requires of libmysqlclient.so.18(libmysqlclient_18)(64bit)
则是缺少依赖,需要安装相关的包,执行下面的操作
[root@master cloudera-repo]# wget http://www.percona.com/redir/downloads/Percona-XtraDB-Cluster/5.5.37-25.10/RPM/rhel6/x86_64/Percona-XtraDB-Cluster-shared-55-5.5.37-25.10.756.el6.x86_64.rpm
[root@master cloudera-repo]# rpm -ivh Percona-XtraDB-Cluster-shared-55-5.5.37-25.10.756.el6.x86_64.rpm
  • 安装CDH,执行以下命令
[root@master cloudera-repo]# yum install cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server
#设置CDH数据库
#Cloudera Manager Server有一个配置数据库的脚本,执行该脚本就好
#如果MySQL数据库与CDH Server在同一台主机上,执行如下命令
[root@master cloudera-repo]# /opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm
Enter SCM password: 
......
INFO  Successfully connected to database.
All done, your SCM database is configured correctly!如果MySQL数据库与CDH Server不在同一台主机上,执行如下命令
[root@master cloudera-repo]# /opt/cloudera/cm/schema/scm_prepare_database.sh mysql -h <mysql-host-ip> --scm-host <cm-server-ip> scm scm
#启动CDH Server
[root@master cloudera-repo]# systemctl start cloudera-scm-server
[root@master cloudera-repo]# systemctl start cloudera-scm-agent
[root@master cloudera-repo]# systemctl status cloudera-scm-agent