Kubernetes+EFK日志收集

Kubernetes离线部署

基础配置(所有主机都需要完成)
1、设置hosts解析
hostnamectl set-hostname k8s-init && bash
cat >>/etc/hosts <<EOF
192.168.180.210 k8s-init
192.168.180.200 k8s-master1
192.168.180.190 k8s-master2
192.168.180.180 k8s-master3
192.168.180.170 k8s-node1
192.168.180.160 k8s-node2
EOF
2、关闭防火墙
systemctl stop firewalld && systemctl disable firewalld
3、关闭SELinux
sed -ri ‘s#(SELINUX=).#\1disabled#’ /etc/selinux/config && setenforce 0
4、关闭swap
swapoff -a && sed -i '/swap/s/^(.)$/#\1/g’ /etc/fstab
5、修改内核参数
cat < /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF

modprobe br_netfilter
sysctl -p /etc/sysctl.d/k8s.conf
6、加载ipvs模块
cat > /etc/sysconfig/modules/ipvs.modules << EOF
#!/bin/bash
modprobe – ip_vs
modprobe – ip_vs_rr
modprobe – ip_vs_wrr
modprobe – ip_vs_sh
modprobe – nf_conntrack_ipv4
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules
bash /etc/sysconfig/modules/ipvs.modules

====================================================================================
6、拷贝安装包(只在init节点执行)
1)将安装拷贝到opt目录
yum -y install vim lrzsz unzip
cd /opt
unzip k8s-installer.zip
ls -l k8s-installer
2)部署依赖组件
部署YUM仓库
配置本地repo文件
cat < /etc/yum.repos.d/local.repo
[local]
name=local
baseurl=file:///opt/k8s-installer/docker-ce
gpgcheck=0
enabled=1
EOF
yum clean all && yum makecache
安装并配置httpd服务
yum install -y httpd --disablerepo=* --enablerepo=local
sed -i ‘s/Listen 80/Listen 60080/g’ /etc/httpd/conf/httpd.conf
cp -r /opt/k8s-installer/docker-ce/ /var/www/html/
systemctl enable httpd && systemctl start httpd

============================================================================================

安装并配置Docker(在所有节点均需执行)
配置yum repo
cat < /etc/yum.repos.d/local-http.repo
[local-http]
name=local-http
baseurl=http://192.168.180.210:60080/docker-ce
gpgcheck=0
enabled=1
EOF

yum clean all && yum makecache
配置docker daemon文件
mkdir /etc/docker
cat < /etc/docker/daemon.json
{
“insecure-registries”: [“192.168.180.210:65000”],
“storage-driver”: “overlay2”
}
EOF

yum remove -y policycoreutils
scp 192.168.180.210:/opt/k8s-installer/docker-ce/audit-libs-* ./
rpm -ivh audit-libs-2.8.4-4.el7.x86_64.rpm audit-libs-python-2.8.4-4.el7.x86_64.rpm --force
yum install -y docker-ce docker-ce-cli containerd.io --disablerepo=* --enablerepo=local-http
systemctl enable docker && systemctl start docker

==================================================================================================
配置镜像仓库（只在k8s-init节点执行）
加载镜像到本地
docker load -i /opt/k8s-installer/registry-image.tar
docker images
启动registry镜像仓库服务
docker run -d --restart=always --name pkg-registry -p 65000:5000 -v /opt/k8s-installer/registry/:/var/lib/registry index.alauda.cn/alaudaorg/distribution:latest
部署Haproxy
安装Haproxy
rpm -Uvh /var/www/html/docker-ce/openssl-*
yum install -y haproxy --disablerepo=* --enablerepo=local-http
配置haproxy
cp /etc/haproxy/haproxy.cfg{,.bak}
vim /etc/haproxy/haproxy.cfg

Global settings

global
log 127.0.0.1 local2
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
maxconn 4000
user haproxy
group haproxy
daemon
# turn on stats unix socket
stats socket /var/lib/haproxy/stats
defaults
log global
option dontlognull
option forwardfor except 127.0.0.0/8
option redispatch
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 1m
timeout server 1m
timeout http-keep-alive 10s
timeout check 10s
maxconn 3000
frontend kubernetes
bind *:7443
mode tcp
default_backend kubernetes-master
backend kubernetes-master
balance roundrobin
server master1 192.168.180.200:6443 check maxconn 2000
server master2 192.168.180.190:6443 check maxconn 2000
server master3 192.168.180.180:6443 check maxconn 2000

systemctl enable haproxy && systemctl start haproxy

========================================================================================================

部署Kubernetes(在Master和node节点上执行)
安装Kubeadm、kubelet和kubectl
rpm -Uvh http://192.168.180.210:60080/docker-ce/libnetfilter_conntrack-1.0.6-1.el7_3.x86_64.rpm
yum install -y kubeadm kubectl kubelet --disablerepo=* --enablerepo=local-http
配置Kubelet
systemctl enable kubelet
配置Kubelet Service
cat < /etc/systemd/system/kubelet.service
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=https://kubernetes.io/docs/
[Service]
Environment=“KUBELET_SYSTEM_PODS_ARGS=–pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true”
Environment=“KUBELET_INFRA_CONTAINER_IMAGE=–pod-infra-container-image=192.168.180.210:60080/k8s/pause:3.1”
ExecStart=/usr/bin/kubelet $KUBELET_SYSTEM_PODS_ARGS $KUBELET_INFRA_CONTAINER_IMAGE
Restart=always
StartLimitInterval=0
RestartSec=10
[Install]
WantedBy=multi-user.target
EOF

=============================================================================================================
配置Kubeadm初始化文件(只需要在第一台Master节点执行 k8s-master1)
advertiseAddress： 修改为 k8s-master1 的内网 IP 地址；
controlPlaneEndpoint： 修改为 k8s-init 的内网 IP 地址加端口（默认 7443 端口） ，如果使用第三方 lb 做负载均衡， 则配置对应的 lb 的地址+端口；
imageRepository： 修改为 k8s-init 的内网 IP 地址:65000/k8s。
vim /opt/kubeadm.conf
apiVersion: kubeadm.k8s.io/v1beta1
bootstrapTokens:

• groups:
  • system:bootstrappers:kubeadm:default-node-token
    token: abcdef.0123456789abcdef
    ttl: 24h0m0s
    usages:
  • signing
  • authentication
    kind: InitConfiguration
    localAPIEndpoint:
    advertiseAddress: 192.168.180.200
    bindPort: 6443

---

apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta1
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: “192.168.180.210:7443”
controllerManager: {}
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: 192.168.180.210:65000/k8s
kind: ClusterConfiguration
kubernetesVersion: v1.13.3
networking:
dnsDomain: cluster.local
podSubnet: “10.244.0.0/16”
serviceSubnet: 10.96.0.0/12
scheduler: {}
提前下载镜像
kubeadm config images list --config /opt/kubeadm.conf
kubeadm config images pull --config /opt/kubeadm.conf
执行初始化：
kubeadm init --config /opt/kubeadm.conf
配置kubectl认证（下面信息是初始化时产生）
#mkdir -p $HOME/.kube
#sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
#sudo chown $(id?u):$(id -g) $HOME/.kube/config
kubeadm join 192.168.180.210:7443 --token abcdef.0123456789abcdef --discovery-token-ca-cert-hash sha256:5de632ee3645adbcd830a1045d94f7ae184e1c1e7275f7812e19ce4463970168
添加其他Master节点到集群中（在K8S-Master2、K8S-Master3节点执行）
mkdir -p /etc/kubernetes/pki/etcd
K8S-Master1节点执行以下命令
ssh-keygen
ssh-copy-id root@k8s-master2
ssh-copy-id root@k8s-master3
scp /etc/kubernetes/pki/ca.crt root@k8s-master2:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/ca.key root@k8s-master2:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/sa.key root@k8s-master2:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/sa.pub root@k8s-master2:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/front-proxy-ca.crt root@k8s-master2:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/front-proxy-ca.key root@k8s-master2:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/etcd/ca.crt root@k8s-master2:/etc/kubernetes/pki/etcd/
scp /etc/kubernetes/pki/etcd/ca.key root@k8s-master2:/etc/kubernetes/pki/etcd/
scp /etc/kubernetes/admin.conf root@k8s-master2:/etc/kubernetes/admin.conf
scp /etc/kubernetes/pki/ca.crt root@k8s-master3:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/ca.key root@k8s-master3:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/sa.key root@k8s-master3:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/sa.pub root@k8s-master3:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/front-proxy-ca.crt root@k8s-master3:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/front-proxy-ca.key root@k8s-master3:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/etcd/ca.crt root@k8s-master3:/etc/kubernetes/pki/etcd/
scp /etc/kubernetes/pki/etcd/ca.key root@k8s-master3:/etc/kubernetes/pki/etcd/
scp /etc/kubernetes/admin.conf root@k8s-master3:/etc/kubernetes/admin.conf

kubeadm join 192.168.180.210:7443 --token abcdef.0123456789abcdef --discovery-token-ca-cert-hash sha256:5de632ee3645adbcd830a1045d94f7ae184e1c1e7275f7812e19ce4463970168 --experimental-control-plane
kubeadm join --experimental-control-plane 现在可用于将控制面实例连接到现有集群
创建对应目录

添加Node节点到集群
kubeadm join 192.168.180.210:7443 --token abcdef.0123456789abcdef --discovery-token-ca-cert-hash sha256:5de632ee3645adbcd830a1045d94f7ae184e1c1e7275f7812e19ce4463970168

安装Flannel插件（只在第一台 Master 节点（k8s-master1） 执行）
拷贝kube-flannel.yml
scp root@k8s-init:/opt/k8s-installer/kube-flannel.yml /opt
替换Flannel镜像地址
sed -i “s#quay.io/coreos#192.168.180.210:65000/k8s#g” /opt/kube-flannel.yml
创建Flannel相关资源
kubectl create -f /opt/kube-flannel.yml
设置Master节点是否可调度
kubectl taint node k8s-master1 node-role.kubernetes.io/master:NoSchedule-
kubectl taint node k8s-master2 node-role.kubernetes.io/master:NoSchedule-
kubectl taint node k8s-master3 node-role.kubernetes.io/master:NoSchedule-
验证集群功能
kubectl get nodes
创建Nginx测试服务
kubectl run --generator=run-pod/v1 test-nginx --image=192.168.180.210:65000/k8s/nginx
kubectl get pods -o wide |grep test-nginx

Kubernetes+EFK日志收集
1、部署Kubernetes集群
2、导入EFK镜像（在K8S-master1节点上执行）
上传压缩包
tar xvf efk.tar
cd efk
导入Elasticsearch镜像
docker load < elasticsearch-7.4.2.tar
docker tag b1179d41a7b4 192.168.180.210:65000/efk/elasticsearch:7.4.2
docker push 192.168.180.210:65000/efk/elasticsearch:7.4.2
导入Fluentd镜像
docker load < fluentd-es.tar
docker tag 636f3d 192.168.180.210:65000/efk/fluentd-es-root:v2.5.2
docker push 192.168.180.210:65000/efk/fluentd-es-root:v2.5.2
导入Kibana镜像
docker load < kibana-7.4.2.tar
docker tag 230d3d 192.168.180.210:65000/efk/kibana:7.4.2
docker push 192.168.180.210:65000/efk/kibana:7.4.2
导入alpine镜像
docker load < alpine-3.6.tar
docker tag 43773d 192.168.180.210:65000/efk/alpine:3.6
docker push 192.168.180.210:65000/efk/alpine:3.6
修改Yaml文件
修改elasticsearch.yaml
sed -i ‘s/192.168.8.10/192.168.180.210/g’ elasticsearch.yaml kibana.yaml fluentd.yaml test-pod.yaml
grep “image:” elasticsearch.yaml kibana.yaml fluentd.yaml test-pod.yaml
image: 192.168.180.210:65000/efk/elasticsearch:7.4.2
image: 192.168.180.210:65000/efk/alpine:3.6
image: 192.168.180.210:65000/efk/alpine:3.6

grep -A1 “nodeSelector” elasticsearch.yaml
nodeSelector:
kubernetes.io/hostname: k8s-node1

grep -A1 “nodeSelector” kibana.yaml
nodeSelector:
kubernetes.io/hostname: k8s-node2

部署Elasticsearch
创建命名空间
kubectl create -f namespace.yaml
kubectl get namespaces | grep logging

到K8S-node1节点上
创建es数据存储目录
mkdir /esdata
在k8s-master1上
部署es容器
kubectl create -f elasticsearch.yaml
kubectl -n logging get pods -o wide
kubectl -n logging get svc

curl 10.108.236.66:9200

部署kibana
kubectl create -f kibana.yaml
kubectl -n logging get pods
kubectl -n logging get svc |grep kibana
可以使用浏览器测试访问
http://192.168.180.200:端口

部署Flunentd
给集群节点打标签
kubectl label node k8s-node1 fluentd=true
kubectl label node k8s-node2 fluentd=true
kubectl create -f fluentd-es-config-main.yaml
kubectl create -f fluentd-configmap.yaml
kubectl create -f fluentd.yaml
kubectl -n logging get pods

验证日志的收集
kubectl create -f test-pod.yaml
kubectl get pods