ElasticSearch 集群部署
标签（空格分隔）： ElasticSearch系列
[toc]
一：ElasticSearch集群部署
1.1 系统环境准备
系统：rhel8.3x64
cat /etc/hosts
----
192.168.100.41 rhel01.flyfish.cn
192.168.100.42 rhel02.flyfish.cn
192.168.100.43 rhel03.flyfish.cn
192.168.100.44 rhel04.flyfish.cn
192.168.100.45 rhel05.flyfish.cn
192.168.100.46 rhel06.flyfish.cn
192.168.100.47 rhel07.flyfish.cn
192.168.100.48 rhel08.flyfish.cn
----
本次安装 只涉及到 前三台
系统语言：
echo "export LANG=en_US.UTF8" >> ~/.bash_profile
cat ~/.bash_profile
---
分区：
pvcreate /dev/sdb
vgcreate esdatavg /dev/sdb
lvcreate -n esdatalv -L 100000M esdatavg
mkfs.xfs /dev/esdatavg/esdatalv
---
vi /etc/fstab
---
/dev/esdatavg/esdatalv /esdb xfs defaults 0 0
---
mkdir /esdb
mount /esdbmkdir /esdb/soft
groupadd -g 60001 esadmin
useradd -u 61001 -g esadmin esadmin
chown -R esadmin:esadmin /esdb
chmod -R 775 /esdb
echo "esadmin" | passwd --stdin esadmin
systemctl set-default multi-user.target
cat >> /etc/security/limits.conf << EOF
root soft nofile 1048576
root hard nofile 1048576
esadmin soft nproc 1048576
esadmin hard nproc 1048576
esadmin soft nofile 1048576
esadmin hard nofile 1048576
esadmin soft stack 10240
esadmin hard stack 32768
esadmin hard memlock unlimited
esadmin soft memlock unlimited
EOF
---
---
cat >> /etc/sysctl.conf << EOF
fs.aio-max-nr = 1048576
fs.file-max = 6815744
net.ipv4.ip_local_port_range = 1024 65535
net.ipv4.tcp_mem = 786432 2097152 3145728
net.ipv4.tcp_rmem = 4096 4096 16777216
net.ipv4.tcp_wmem = 4096 4096 16777216
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_tw_recycle = 1
vm.swappiness=1
vm.min_free_kbytes=204800
vm.max_map_count=2048000
kernel.pid_max=819200
vm.zone_reclaim_mode=0
#vm.nr_hugepages = 0
EOF
sysctl -pln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
hwclock
echo "SELINUX=disabled" > /etc/selinux/config
echo "#SELINUXTYPE=targeted " >> /etc/selinux/config
cat /etc/selinux/config
setenforce 0
systemctl stop firewalld.service
systemctl disable firewalld.service
cat >> /home/esadmin/.vimrc << EOF
map <F10>:set paste<CR>
map <F11>:set nopaste<CR>
EOF
shutdown -r now做esadmin 的无密码登录
/esdb/soft/fgssh/fgssh -user esadmin -hosts "rhel01.flyfish.cn rhel02.flyfish.cn rhel03.flyfish.cn" -advanced -exverify -confirm
chmod 600 /home/esadmin/.ssh/configsu - esadmin
cd /esdb/
tar zxvf /esdb/soft/jdk-14.0.2_linux-x64_bin.tar.gz
scp -r /esdb/jdk-14.0.2 rhel02.flyfish.cn:/esdb/
scp -r /esdb/jdk-14.0.2 rhel03.flyfish.cn:/esdb/su - esadmin
cat >> ~/.bash_profile << EOF
export JAVA_HOME=/esdb/jdk-14.0.2
export PATH=/esdb/jdk-14.0.2/bin:$PATH
export LANG=en_US.UTF8
EOF
source ~/.bash_profile
java -version
全部主机：
mkdir -p /esdb/soft/
mkdir -p /esdb/esapp/esapp9200
mkdir -p /esdb/esdata/esdata9200
mkdir -p /esdb/eslog/eslog9200
chown esadmin:esadmin -R /esdb/rhel01.flyfish.cn 节点：
cd /esdb/soft
tar zxvf /esdb/soft/elasticsearch-7.9.0-linux-x86_64.tar.gz
cp -r /esdb/soft/elasticsearch-7.9.0/* /esdb/esapp/esapp9200/
vim /esdb/esapp/esapp9200/config/elasticsearch.yml
---
cluster.name: flyfishescluster1
node.name: 192.168.100.41
path.data: /esdb/esdata/esdata9200
path.logs: /esdb/eslog/eslog9200
node.master: true
node.data: true
bootstrap.memory_lock: true
bootstrap.system_call_filter: false
network.host: 192.168.100.41
http.port: 9200
transport.tcp.port: 9300
http.cors.enabled: true
http.cors.allow-origin: "*"
discovery.seed_hosts: ["192.168.100.41:9300","192.168.100.42:9300","192.168.100.43:9300"]
discovery.zen.fd.ping_timeout: 60s
discovery.zen.fd.ping_retries: 5
cluster.initial_master_nodes:
["192.168.100.41:9300","192.168.100.42:9300","192.168.100.43:9300"]
---
vi /esdb/esapp/esapp9200/bin/elasticsearch-env
---
export JAVA_HOME=/esdb/jdk-14.0.2
export PATH=$JAVA_HOME/bin:$PATH
---vi /esdb/esapp/esapp9200/config/jvm.options
---
-Xms256m
-Xmx256m
-Xmn100m
-Xss228K
-XX:+DisableExplicitGC
-XX:HeapDumpPath=/esdb/eslog/eslog9200
-XX:ErrorFile=/esdb/eslog/eslog9200/hs_err_pid%p.log
9-:-Xlog:gc*,gc+age=trace,safepoint:file=/esdb/eslog/eslog9200/gc.log:utctime,pid,ta
gs:filecount=32,filesize=64mcd /esdb/esapp/esapp9200
scp -r * esadmin@rhel02.flyfish.cn:/esdb/esapp/esapp9200/
scp -r * esadmin@rhel03.flyfish.cn:/esdb/esapp/esapp9200/rhel02.flyfish.cn 主机：
vi /esdb/esapp/esapp9200/config/elasticsearch.yml
---
cluster.name: flyfishescluster1
node.name: 192.168.100.42
path.data: /esdb/esdata/esdata9200
path.logs: /esdb/eslog/eslog9200
node.master: true
node.data: true
bootstrap.memory_lock: true
bootstrap.system_call_filter: false
network.host: 192.168.100.42
http.port: 9200
transport.tcp.port: 9300
http.cors.enabled: true
http.cors.allow-origin: "*"
discovery.seed_hosts: ["192.168.100.41:9300","192.168.100.42:9300","192.168.100.43:9300"]
discovery.zen.fd.ping_timeout: 60s
discovery.zen.fd.ping_retries: 5
cluster.initial_master_nodes: ["192.168.100.41:9300","192.168.100.42:9300","192.168.100.43:9300"]
---rhel03.flyfish.cn 主机：
vi /esdb/esapp/esapp9200/config/elasticsearch.yml
----
cluster.name: flyfishescluster1
node.name: 192.168.100.43
path.data: /esdb/esdata/esdata9200
path.logs: /esdb/eslog/eslog9200
node.master: true
node.data: true
bootstrap.memory_lock: true
bootstrap.system_call_filter: false
network.host: 192.168.100.43
http.port: 9200
transport.tcp.port: 9300
http.cors.enabled: true
http.cors.allow-origin: "*"
discovery.seed_hosts: ["192.168.100.41:9300","192.168.100.42:9300","192.168.100.43:9300"]
discovery.zen.fd.ping_timeout: 60s
discovery.zen.fd.ping_retries: 5
cluster.initial_master_nodes: ["192.168.100.41:9300","192.168.100.42:9300","192.168.100.43:9300"]
----启动es （全部机器执行）
/esdb/esapp/esapp9200/bin/elasticsearch
测试：
curl http://192.168.100.41:9200/_cat/nodes
curl http://192.168.100.41:9200/_cat/health?pretty -H 'Content-Type: application/json'
启动脚本：
mkdir -p /esdb/scripts/
vi /esdb/scripts/es_start.sh
----
#!/bin/bash
echo "es cluster all node start..."
for host in rhel01.flyfish.cn rhel02.flyfish.cn rhel03.flyfish.cn
#for host in `cat /esdb/scripts/es_host.sh | awk '{print $1}'`
do
echo "--------------------------------"
echo $host "node es start..."
ssh -n -T $host "/esdb/esapp/esapp9200/bin/elasticsearch -d >/dev/null 2>&1 &"
echo $host "node es start cancel..."
done
echo "es cluster all node start OK!"
----vi /esdb/scripts/es_kill.sh
#!/bin/bash
----
ps -ef|grep java|grep esapp|awk '{print $2}'|xargs kill -9
vi /esdb/scripts/es_stop.sh
#!/bin/bash
echo "es cluster all node stop..."
for host in fgesdb01 fgesdb02 fgesdb03
#for host in `cat /esdb/scripts/host.sh | awk '{print $1}'`
do
echo "--------------------------------"
echo $host "node es stop..."
ssh -n -T $host "/esdb/scripts/es_kill.sh"
echo $host "node es stop cancel..."
done
echo "es cluster all node stop OK!"
----chmod 755 /esdb/scripts/es_*.sh
scp -r /esdb/scripts rhel02.flyfish.cn:/esdb/
scp -r /esdb/scripts rhel03.flyfish.cn:/esdb/
ssh rhel02.flyfish.cn "chmod -R 755 /esdb/scripts"
ssh rhel03.flyfish.cn "chmod -R 755 /esdb/scripts"
/esdb/scripts/es_stop.sh
/esdb/scripts/es_start.sh设置集群分片与副本：
curl -XPUT 192.168.100.41:9200/_template/all_index_shards -H 'Content-Type:application/json' -d '{
"template": "*",
"settings": {
"number_of_shards": 6,
"number_of_replicas": "1"
}
}'
设置路由分片防止堆积在一台上面：
curl -XPUT '192.168.100.41:9200/_cluster/settings' -H 'Content-Type: application/json' -d '{
"transient": {
"cluster.routing.allocation.same_shard.host": true
}
}'
插入数据：
curl -XPOST http://192.168.100.41:9200/itpuxdb/_doc/?pretty -H 'Content-Type:application/json' -d '
{
"id" : "1",
"name" : "itpux001",
"city" : "上海",
"course" : "bigdata",
"teacher" : "flyfish",
"pxdate": "20200829"
}'
查看数据：
curl -XPOST 'http://192.168.100.41:9200/itpuxdb/_search?pretty' -H 'Content-Type:application/json'
curl http://192.168.100.41:9200/_cat/shards?v
