k8s-etcd部署 (2)

etcd

分布式键值存储系统，用于保持集群状态数据，比如pod、server信息

etcd部署
master：192.168.175.148
node01：192.168.175.149
node02：192.168.175.157

master操作

///创建目录k8s
[root@promote ~]# mkdir k8s 
[root@promote ~]# cd k8s/
///创建两个脚本etcd-cert.sh（证书创建脚本） etcd.sh(服务脚本其中包含启动脚本和配置文件)
vim etcd-cert.sh
cat > ca-config.json <<EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"     ///设置时效
    },
    "profiles": {
      "www": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",    服务端验证
            "client auth"       客户端验证
        ]
      }
    }
  }
}
EOF
cat > ca-csr.json <<EOF
{
    "CN": "etcd CA",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing"
        }
    ]
}
EOF
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
cat > server-csr.json <<EOF
{
    "CN": "etcd",
    "hosts": [
    "10.206.240.188",
    "10.206.240.189",
    "10.206.240.111"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing"
        }
    ]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server

[root@promote k8s]# vim etcd.sh
#!/bin/bash
# example: ./etcd.sh etcd01 192.168.1.10 etcd02=https://192.168.1.11:2380,etcd03=https://192.168.1.12:2380
ETCD_NAME=$1
ETCD_IP=$2
ETCD_CLUSTER=$3
WORK_DIR=/opt/etcd
cat <<EOF >$WORK_DIR/cfg/etcd
#[Member]
ETCD_NAME="${ETCD_NAME}"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://${ETCD_IP}:2380"     ///内部端口
ETCD_LISTEN_CLIENT_URLS="https://${ETCD_IP}:2379"   //外部端口
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://${ETCD_IP}:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://${ETCD_IP}:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://${ETCD_IP}:2380,${ETCD_CLUSTER}"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF
cat <<EOF >/usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=${WORK_DIR}/cfg/etcd
ExecStart=${WORK_DIR}/bin/etcd \
--name=\${ETCD_NAME} \
--data-dir=\${ETCD_DATA_DIR} \
--listen-peer-urls=\${ETCD_LISTEN_PEER_URLS} \
--listen-client-urls=\${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
--advertise-client-urls=\${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-advertise-peer-urls=\${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster=\${ETCD_INITIAL_CLUSTER} \
--initial-cluster-token=\${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-cluster-state=new \
--cert-file=${WORK_DIR}/ssl/server.pem \
--key-file=${WORK_DIR}/ssl/server-key.pem \
--peer-cert-file=${WORK_DIR}/ssl/server.pem \
--peer-key-file=${WORK_DIR}/ssl/server-key.pem \
--trusted-ca-file=${WORK_DIR}/ssl/ca.pem \
--peer-trusted-ca-file=${WORK_DIR}/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable etcd
systemctl restart etcd

//创建目录存放证书及原材料
[root@promote k8s]# mkdir etcd-cert
[root@promote k8s]# ls
etcd-cert  etcd-cert.sh  etcd.sh
///将证书脚本移到目录下
[root@promote k8s]# mv etcd-cert.sh etcd-cert
//cd /usr/local/bin将cfssl官方包放入里面
[root@promote k8s]# cd /usr/local/bin
[root@promote bin]# ls
cfssl  cfssl-certinfo  cfssljson
//权限添加
[root@promote bin]# chmod +x cfssl-certinfo 
[root@promote bin]# chmod +x cfssljson
[root@promote bin]# chmod +x cfssl
[root@promote bin]# cd /root/k8s/etcd-cert/
[root@promote etcd-cert]# cat > ca-config.json <<EOF
> {
>   "signing": {
>     "default": {
>       "expiry": "87600h"
>     },
>     "profiles": {
>       "www": {
>          "expiry": "87600h",
>          "usages": [
>             "signing",
>             "key encipherment",
>             "server auth",
>             "client auth"
>         ]
>       }
>     }
>   }
> }
> EOF
[root@promote etcd-cert]# ls
ca-config.json  etcd-cert.sh
//实现证书签名
[root@promote etcd-cert]# cat > ca-csr.json <<EOF
> {
>     "CN": "etcd CA",
>     "key": {
>         "algo": "rsa",
>         "size": 2048
>     },
>     "names": [
>         {
>             "C": "CN",
>             "L": "Beijing",
>             "ST": "Beijing"
>         }
>     ]
> }
> EOF
//生成证书（ca-key.pem  ca.pem）
[root@promote etcd-cert]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
[root@promote etcd-cert]# ls
ca-config.json  ca.csr  ca-csr.json  ca-key.pem  ca.pem  etcd-cert.sh

///指定etcd三个节点之间的通信验证
[root@promote etcd-cert]# cat > server-csr.json <<EOF
> {
>     "CN": "etcd",
>     "hosts": [
>     "192.168.175.148",
>     "192.168.175.149",
>     "192.168.175.157"
>     ],
>     "key": {
>         "algo": "rsa",
>         "size": 2048
>     },
>     "names": [
>         {
>             "C": "CN",
>             "L": "BeiJing",
>             "ST": "BeiJing"
>         }
>     ]
> }
> EOF
//生成etcd证书server-key.pem server.pem
[root@promote etcd-cert]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
[root@promote etcd-cert]# ls
ca-config.json  ca-csr.json  ca.pem        server.csr       server-key.pem
ca.csr          ca-key.pem   etcd-cert.sh  server-csr.json  server.pem
///回到k8s目录
[root@promote etcd-cert]# cd ..
[root@promote k8s]# 
//将源码包挂载到目录下
[root@promote k8s]# ls
etcd-cert  etcd-v3.3.10-linux-amd64.tar.gz     kubernetes-server-linux-amd64.tar.gz
etcd.sh    flannel-v0.10.0-linux-amd64.tar.gz
//解压源码包
[root@promote k8s]# tar zxvf etcd-v3.3.10-linux-amd64.tar.gz
[root@promote k8s]# cd etcd-v3.3.10-linux-amd64/
[root@promote etcd-v3.3.10-linux-amd64]# ls
Documentation  etcd  etcdctl  README-etcdctl.md  README.md  READMEv2-etcdctl.md
//创建工作目录
[root@promote etcd-v3.3.10-linux-amd64]# mkdir /opt/etcd/{cfg,bin,ssl} -p   （配置文件，命令文件，证书）-p 递归创建
[root@promote etcd-v3.3.10-linux-amd64]# mv etcd etcdctl /opt/etcd/bin
//将所有证书拷贝到工作目录中
[root@promote etcd-cert]# cp /root/k8s/etcd-cert/*.pem /opt/etcd/ssl
[root@promote k8s]# pwd
/root/k8s
//进入卡住状态等待其他节点加入
[root@promote k8s]# bash etcd.sh etcd01 192.168.175.148 etcd02=https://192.168.175.149:2380,etcd03=https://192.168.175.157:2380
//将证书拷贝到其他节点（拷贝整个文件夹要加 -r）
[root@promote k8s]# scp -r /opt/etcd/ root@192.168.175.149:/opt
[root@promote k8s]# scp -r /opt/etcd/ root@192.168.175.157:/opt
//将启动脚本拷贝到其他节点
[root@promote cfg]# scp /usr/lib/systemd/system/etcd.service root@192.168.175.149://usr/lib/systemd/system/
[root@promote cfg]# scp /usr/lib/systemd/system/etcd.service root@192.168.175.157://usr/lib/systemd/system/

在node01上修改拷贝过去的配置文件

[root@promote cfg]# vim /opt/etcd/cfg/etcd
#[Member]
ETCD_NAME="etcd02"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.175.149:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.175.149:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.175.149:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.175.149:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.175.148:2380,etcd02=https://192.168.175.149:2380,etcd03=https://192.168.175.157:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

在node02上修改拷贝过去的配置文件

vim /opt/etcd/cfg/etcd
#[Member]
ETCD_NAME="etcd03"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.175.157:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.175.157:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.175.157:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.175.157:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.175.148:2380,etcd02=https://192.168.175.149:2380,etcd03=https://192.168.175.157:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

cd /root/k8s
[root@promote k8s]# bash etcd.sh etcd01 192.168.175.148 etcd02=https://192.168.175.149:2380,etcd03=https://192.168.175.157:2380
//同时开启node01、02节点服务
systemctl start etcd
systemctl start etcd

//查看集群状态
//进入证书目录
[root@promote k8s]# cd etcd-cert/
[root@promote etcd-cert]# /opt/etcd/bin/etcdctl --ca-file=ca.pem --cert-file=server.pem --key-file=server-key.pem --endpoints="https://192.168.175.148:2379,https://192.168.175.149,https://192.168.175.157:1379" cluster-health
member 7c47183c6509de5f is healthy: got healthy result from https://192.168.175.148:2379
member 7e8a91759f0063e0 is healthy: got healthy result from https://192.168.175.157:2379
member f310fe5aedbd89e1 is healthy: got healthy result from https://192.168.175.149:2379
cluster is healthy