openEuler-22.03-LTS-SP3 编译安装 Greenplum-db 6.20.0

1、配置 yum 华为源

cd /etc/yum.repos.d/
mkdir -p repobak
mv -f *.repo repobak
vi /etc/yum.repos.d/openEuler.repo

openEuler.repo

#generic-repos is licensed under the Mulan PSL v2.
#You can use this software according to the terms and conditions of the Mulan PSL v2.
#You may obtain a copy of Mulan PSL v2 at:
#    http://license.coscl.org.cn/MulanPSL2
#THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
#IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
#PURPOSE.
#See the Mulan PSL v2 for more details.[OS]
name=OS
baseurl=https://repo.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP3/OS/$basearch/
enabled=1
gpgcheck=1
gpgkey=https://repo.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP3/OS/$basearch/RPM-GPG-KEY-openEuler[everything]
name=everything
baseurl=https://repo.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP3/everything/$basearch/
enabled=1
gpgcheck=1
gpgkey=https://repo.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP3/everything/$basearch/RPM-GPG-KEY-openEuler[EPOL]
name=EPOL
baseurl=https://repo.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP3/EPOL/main/$basearch/
enabled=1
gpgcheck=1
gpgkey=https://repo.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP3/OS/$basearch/RPM-GPG-KEY-openEuler[debuginfo]
name=debuginfo
baseurl=https://repo.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP3/debuginfo/$basearch/
enabled=1
gpgcheck=1
gpgkey=https://repo.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP3/debuginfo/$basearch/RPM-GPG-KEY-openEuler[source]
name=source
baseurl=https://repo.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP3/source/
enabled=1
gpgcheck=1
gpgkey=https://repo.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP3/source/RPM-GPG-KEY-openEuler[update]
name=update
baseurl=https://repo.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP3/update/$basearch/
enabled=1
gpgcheck=1
gpgkey=https://repo.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP3/OS/$basearch/RPM-GPG-KEY-openEuler

2、安装依赖

yum install -y apr-devel bison bzip2-devel cmake3 flex gcc gcc-c++ krb5-devel libcurl-devel libevent-devel libkadm5 libyaml-devel libxml2-devel libzstd-devel openssl-devel perl-ExtUtils-Embed readline-devel xerces-c-devel zlib-devel ncurses-devel sqlite-devel tk-devel gdbm-devel libpcap-devel xz-devel libffi-devel python-setuptools make git python3-devel python-pip tar pam pam-devel

cmake >= 3.1 ！！！

3、源码安装 openssl 1.0.1u

3.1、openssl 1.1.1 降级到 openssl 1.0.1

# 备份文件
mv /usr/bin/openssl /usr/bin/openssl.old
mv /etc/pki/ca-trust/extracted/openssl /etc/pki/ca-trust/extracted/openssl.old
# 下载源码
cd /usr/local/
wget -O openssl-1.0.1u.tar.gz https://www.openssl.org/source/old/1.0.1/openssl-1.0.1u.tar.gz
tar -xzf openssl-1.0.1u.tar.gz
cd openssl-1.0.1u
mkdir -p /usr/local/openssl /usr/local/ssl# 编译安装 
CFLAGS="-fPIC" ./config shared zlib-dynamic --prefix=/usr/local/openssl --openssldir=/usr/local/ssl && make -j4 && make -j4 install# 注意，部分电脑会编译生成 /usr/local/openssl/lib64，复制前看一下目录名称！ln -sf /usr/local/openssl/bin/openssl /usr/bin/openssl
cp -f /usr/local/openssl/lib/libssl.so.1.0.0 /usr/lib64/
cp -f /usr/local/openssl/lib/libcrypto.so.1.0.0 /usr/lib64/
# 生成软链接
ln -s /usr/lib64/libssl.so.1.0.0 /usr/lib64/libssl.so.10
ln -s /usr/lib64/libcrypto.so.1.0.0 /usr/lib64/libcrypto.so.10
openssl versioncd /usr/local
rm -rf /usr/local/openssl-1.0.1u

4、源码安装 python 2.7

# 下载python安装包
cd /usr/local/
wget -O Python-2.7.18.tar.xz https://www.python.org/ftp/python/2.7.18/Python-2.7.18.tar.xz
# 解压安装包
tar -xf Python-2.7.18.tar.xz
cd Python-2.7.18

优化编译命令

# ./configure --enable-shared --prefix=/usr/local/python2
# --prefix=*** ：指定了python的安装目录
# --enable-shared ：配置构建选项的命令，构建过程中启用共享库支持（默认带上，否则后续无法使用打包工具）
# --enable-optimizations：启用优化，对Python进行一些优化以提高性能。
# 执行configure文件，指定安装位置/usr/local/python2
# 注意，部分电脑会编译生成 /usr/local/openssl/lib64，复制前看一下目录名称！
./configure --enable-shared --prefix=/usr/local/python2 --enable-optimizations LDFLAGS="-L/usr/local/openssl/lib" CPPFLAGS="-I/usr/local/openssl/include" PKG_CONFIG_PATH="/usr/local/openssl/lib/pkgconfig"

不进行优化编译命令

#注意，部分电脑会编译生成 /usr/local/openssl/lib64，复制前看一下目录名称！
./configure --enable-shared --prefix=/usr/local/python2 LDFLAGS="-L/usr/local/openssl/lib" CPPFLAGS="-I/usr/local/openssl/include" PKG_CONFIG_PATH="/usr/local/openssl/lib/pkgconfig"

# 编译安装
make -j4 && make -j4 install# 将默认/usr/local/lib/路径添加到动态链接器的配置文件，并通过ldconfig更新缓存
# 路径根据实际路径替换，例如:/usr/local/python3/lib/
echo "/usr/local/python2/lib/" >> /etc/ld.so.conf
ldconfig# 删除源代码
cd /usr/local
rm -rf /usr/local/Python-2.7.18# 生成软链接
ln -s /usr/local/python2/bin/python2.7 /usr/bin/python# 安装pip
wget -O get_pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py --no-check-certificate
python get_pip.py -i https://pypi.tuna.tsinghua.edu.cn/simple/# 更新pip，可选
/usr/local/python2/bin/pip2.7 install -i https://pypi.tuna.tsinghua.edu.cn/simple/ --upgrade pip# 安装wheel
/usr/local/python2/bin/pip2.7 install -i https://pypi.tuna.tsinghua.edu.cn/simple/ wheel

安装 psutil

# 安装psutil，必装，不然数据库会初始化失败
/usr/local/python2/bin/pip2.7 install psutil  -i https://pypi.tuna.tsinghua.edu.cn/simple/

5、使用 pip3 安装 Python 相关依赖

openEuler 22.03 自带Python 3.9.9，使用自带的pip安装依赖

cd /usr/local
wget -O 6.20.0.tar.gz https://github.com/greenplum-db/gpdb/archive/refs/tags/6.20.0.tar.gz
tar -zxvf 6.20.0.tar.gz
cd gpdb-6.20.0
# 使用清华源安装conan
pip install -i https://pypi.tuna.tsinghua.edu.cn/simple/ conan

修改文件 python-dependencies.txt，调整依赖为以下版本

argparse==1.2.1
behave==1.2.6
epydoc==3.0.1
logilab-astng==0.24.3
logilab-common==1.9.7
MarkupSafe==2.0.1
mock==1.0.1
parse==1.8.2
psutil==4.0.0
setuptools==42.0.0
unittest2==1.1.0
pyyaml==6.0.1

安装依赖

pip install -i https://pypi.tuna.tsinghua.edu.cn/simple/ -r python-dependencies.txt

修改文件 python-developer-dependencies.txt，调整依赖为以下版本

enum34==1.1.6
Jinja2==3.0.3
parse-type==0.4.2
pexpect==4.4.0
PSI==0.3b2
PyYAML==6.0.1
ptyprocess==0.5.2
six==1.11.0

安装依赖

pip install -i https://pypi.tuna.tsinghua.edu.cn/simple/ -r python-developer-dependencies.txt

6、编译安装 Greenplum-db 6.20.0

6.1、修改配置

ln -sf /usr/bin/cmake3 /usr/local/bin/cmake
echo "/usr/local/lib" >> /etc/ld.so.conf
echo "/usr/local/lib64" >> /etc/ld.so.confldconfig

修改配置

cat >> /etc/sysctl.conf <<-EOF
kernel.shmmax = 500000000
kernel.shmmni = 4096
kernel.shmall = 4000000000
kernel.sem = 500 1024000 200 4096
kernel.sysrq = 1
kernel.core_uses_pid = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.msgmni = 2048
net.ipv4.tcp_syncookies = 1
net.ipv4.ip_forward = 0
net.ipv4.conf.default.accept_source_route = 0
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_max_syn_backlog = 4096
net.ipv4.conf.all.arp_filter = 1
net.ipv4.ip_local_port_range = 1025 65535
net.core.netdev_max_backlog = 10000
net.core.rmem_max = 2097152
net.core.wmem_max = 2097152
vm.overcommit_memory = 2
EOF

使生效

sysctl -p

修改 limits

cat >> /etc/security/limits.conf <<-EOF
* soft nofile 65536
* hard nofile 65536
* soft nproc 131072
* hard nproc 131072
EOF

6.2、基于 GPORCA 构建

下载 gp-xerces 编译

cd /usr/local
git clone https://github.com/greenplum-db/gp-xerces.git
cd gp-xerces
mkdir build
cd build
../configure --prefix=/usr/local/build-gp-xerces
make -j4 && make -j4 install
echo "/usr/local/build-gp-xerces/lib" >> /etc/ld.so.conf
ldconfig
cd /usr/local
rm -rf /usr/local/gp-xerces

编译 gpdb

cd /usr/local/gpdb-6.20.0
PYTHON=/usr/bin/python3 ./configure --with-perl --with-python --with-libxml --prefix=/usr/local/gpdb --with-includes=/usr/local/build-gp-xerces/include --with-libraries=/usr/local/build-gp-xerces/lib --with-pam --enable-cassert --enable-depend

安装

make -j8 && make -j8 install

6.3、不基于 GPORCA 构建

cd /usr/local/gpdb-6.20.0
# make distclean# 安装到 /usr/local/gpdb
PYTHON=/usr/bin/python3 ./configure --disable-orca --with-perl --with-python --with-libxml --prefix=/usr/local/gpdb  --with-pam --enable-cassert --enable-dependmake -j8 && make -j8 install

7、Greenplum-db 6.20.0 使用

编译出来的可执行文件在 /usr/local/gpdb 目录中，需要拷贝到部署的机器上
注意，有符号连接，不能 scp 直接拷贝目录，需要 tar 压缩后再拷贝、解压

7.1、安装 greenplum 单机版（1 master，2 segment）

7.1.1、关闭 SELINUX

vi /etc/selinux/config

SELINUX=disabled

7.1.2、关闭防火墙

systemctl stop firewalld
systemctl disable firewalld

7.1.3、安装单机版GP

1 master，2 segment
gp安装准备

7.1.3.1、改机器名

#查机器名
uname -a
#改名为master
hostnamectl set-hostname master

7.1.3.2、创建 gpaadmin 用户和用户组

groupadd -g 3030 gpadmin
useradd -u 3030 gpadmin -g gpadmin -d /home/gpdata
mkdir -p /home/gpdata
passwd gpadmin
# (密码123456）
#补充知识点：删除 groupdel gpadmin userdel gpadmin

7.1.3.3、配置内核参数

vim /etc/sysctl.conf

以下面内容覆盖原有内容：

kernel.shmmax = 500000000
kernel.shmmni = 4096
kernel.shmall = 4000000000
kernel.sem = 250 512000 100 2048
kernel.sysrq = 1
kernel.core_uses_pid = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.msgmni = 2048
net.ipv4.tcp_syncookies = 1
net.ipv4.conf.default.accept_source_route = 0
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_max_syn_backlog = 4096
net.ipv4.conf.all.arp_filter = 1
net.ipv4.ip_local_port_range = 10000 65535
net.core.netdev_max_backlog = 10000
net.core.rmem_max = 2097152
net.core.wmem_max = 2097152
vm.overcommit_memory = 2

使生效

sysctl -p

7.1.3.4、修改 limits

vim /etc/security/limits.conf

以下面内容覆盖：

soft nofile 65536
hard nofile 65536
soft nproc 131072
hard nproc 131072

7.1.4、gp 安装

7.1.4.1、设置 host

vi /etc/hosts

添加：192.168.100.161 master

192.168.100.161 这里更改成你自己的ip地址

vi /etc/sysconfig/network

添加：hostname=master

7.1.4.2、设置 gp6 环境变量

su gpadmin
cd /usr/local/gpdb
source greenplum_path.sh

7.1.4.3、创建 instance 需要的目录

mkdir -p /home/gpdata
mkdir -p /home/gpdata/master
mkdir -p /home/gpdata/gp3
mkdir -p /home/gpdata/gp4#修改目录属主
chown -R gpadmin:gpadmin /home/gpdata
chown -R gpadmin:gpadmin /home/gpdata/master
chown -R gpadmin:gpadmin /home/gpdata/gp*

7.1.4.4、切换用户 gpadmin

chown -R gpadmin:gpadmin /usr/local/gpdb
su gpadmin

进入$开头的bash环境

cd ~
vi ~/.bash_profile

添加以下

source /usr/local/gpdb/greenplum_path.sh
export MASTER_DATA_DIRECTORY=/home/gpdata/master/gpseg-1

vi ~/.bashrc

添加以下

source /usr/local/gpdb/greenplum_path.sh
export MASTER_DATA_DIRECTORY=/home/gpdata/master/gpseg-1

7.1.4.5、权限互通

su gpadmin
cd /usr/local/gpdb/
vi all_host_list

添加一行
master

su gpadmin
python /usr/local/gpdb/bin/gpssh-exkeys -f /usr/local/gpdb/all_host_list

在 ~/.ssh 目录下生成一对密匙

su gpadmin
rm -rf ~/.ssh
ssh-keygen -t rsa

输入该命令后会有提示，一直回车即可

将公共密匙保存到authorized_keys文件中，实现对自身免密码登录

cd ~/.ssh
cat id_rsa.pub >> authorized_keys

修改权限，需要修改 ~/.ssh 文件夹和 authorized_keys 文件的，不然会报不允许的错误或者无法出现 know_hosts 文件

chmod 700 ~/.ssh
chmod 644 ~/.ssh/authorized_keyssystemctl restart sshd

7.1.4.6、初始化 GP

vi /usr/local/gpdb/seg_hosts_file

写入：
master
本例单机，只有master 这一台

vi /usr/local/gpdb/initgp_config

添加以下内容

ARRAY_NAME="GPDB SINGLENODE"
declare -a DATA_DIRECTORY=(/home/gpdata/gp3 /home/gpdata/gp4)
MASTER_DIRECTORY=/home/gpdata/master
MASTER_HOSTNAME=master
SEG_PREFIX=gpseg
PORT_BASE=33000
MASTER_PORT=5432
DATABASE_NAME=gpdb
MACHINE_LIST_FILE=/usr/local/gpdb/seg_hosts_file
ENCODING=UNICODE
TRUSTED_SHELL=ssh
CHECK_POINT_SEGMENTS=8

初始化GP

/usr/local/gpdb/bin/gpinitsystem -c /usr/local/gpdb/initgp_config -h /usr/local/gpdb/all_host_list

7.1.5、连接GP

基本使用：https://www.cnblogs.com/kingle-study/p/10550785.html

/usr/local/gpdb/bin/gpstate -c

/usr/local/gpdb/bin/gpstate -s

登录数据库之前得先将程序运行，执行

/usr/local/gpdb/bin/gpstart

查询所有数据库

/usr/local/gpdb/bin/psql -l 

附录
其他启停命令

• gpstate #查看服务状态
• gpstart #正常启动
• gpstop #正常关闭
• gpstop -M fast #快速关闭
• gpstop r #重启

服务器重启后需要在 gpadmin 用户下用 gpstart 启动 gp 服务！！！

7.2、安装 greenplum 单机集群（1 master，2 slave，2 mirror）

7.2.1、创建用户组

groupadd gpadmin
useradd gpadmin -g gpadmin
passwd gpadmin

7.2.2、设置 host

su root
# ip要改成本机ip
cat >> /etc/hosts << EOF
192.168.102.30 mdw sdw1 sdw2
EOF

mkdir -p /home/gpadmin
chown -R gpadmin:gpadmin /home/gpadmin
su - gpadmin
cat >>  /home/gpadmin/hostlist << EOF
mdw
sdw1
sdw2
EOF

cat >>  /home/gpadmin/seg_hosts << EOF
sdw1
sdw2
EOF

su gpadmin
cd ~
vi ~/.bash_profile

添加以下

source /usr/local/gpdb/greenplum_path.sh
export MASTER_DATA_DIRECTORY=/data/gpadmin/mdw/gpseg-1

vi ~/.bashrc

添加以下

source /usr/local/gpdb/greenplum_path.sh
export MASTER_DATA_DIRECTORY=/data/gpadmin/mdw/gpseg-1

7.2.3、创建 instance 需要的目录

su root
mkdir -p /data/gpadmin/mdw
mkdir -p /data/gpadmin/sdw1
mkdir -p /data/gpadmin/sdw1_mirror
mkdir -p /data/gpadmin/sdw2
mkdir -p /data/gpadmin/sdw2_mirror
chown gpadmin:gpadmin /data/gpadmin -R

7.2.4、初始化 GP

su gpadmin
cat >> /home/gpadmin/gpinitsystem_config << EOF
ARRAY_NAME="EMC Greenplum DW"
SEG_PREFIX=gpseg
PORT_BASE=33000
declare -a DATA_DIRECTORY=(/data/gpadmin/sdw1 /data/gpadmin/sdw2)
MASTER_HOSTNAME=mdw
MASTER_DIRECTORY=/data/gpadmin/mdw
MASTER_PORT=5432
MACHINE_LIST_FILE=/home/gpadmin/seg_hosts
TRUSTED_SHELL=ssh
CHECK_POINT_SEGMENTS=8
ENCODING=UNICODE
MIRROR_PORT_BASE=43000
REPLICATION_PORT_BASE=34000
MIRROR_REPLICATION_PORT_BASE=44000
declare -a MIRROR_DATA_DIRECTORY=(/data/gpadmin/sdw1_mirror /data/gpadmin/sdw2_mirror)
EOF

gpinitsystem -c /home/gpadmin/gpinitsystem_config

gpstate -c

psql -l