Skip to content

Latest commit

 

History

History
636 lines (595 loc) · 17 KB

k8s二进制安装 未完待整理2.md

File metadata and controls

636 lines (595 loc) · 17 KB
Raw

MASTER_IP="192.168.47.120" SERVICE_CIDR="10.254.0.0/16" NODE_PORT_RANGE="8400-9000" ETCD_ENDPOINTS="http://192.168.47.120:2379" CLUSTER_CIDR=“172.30.0.0/16” BOOTSTRAP_TOKEN="ecf11198cb68fde328065f54563ca00b" KUBE_APISERVER="https://${MASTER_IP}:6443" CLUSTER_DNS_SVC_IP="10.254.0.2" CLUSTER_DNS_DOMAIN="cluster.local."

CFSSL是CloudFlare开源的一款PKI/TLS工具。 CFSSL 包含一个命令行工具 和一个用于 签名，验证并且捆绑TLS证书的 HTTP API 服务 下载

wget http://pkg.cfssl.org/R1.2/cfssl_linux-amd64
chmod +x cfssl_linux-amd64
cp cfssl_linux-amd64 /usr/bin/cfssl

wget http://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
chmod +x cfssl-certinfo_linux-amd64
cp cfssl-certinfo_linux-amd64 /usr/bin/cfssl-certinfo

wget http://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
chmod +x cfssljson_linux-amd64
cp cfssljson_linux-amd64 /usr/bin/cfssljson

生成CA证书

cat > ca-config.json <<EOF	
{
  "signing": {
    "default": {
      "expiry": "8760h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "8760h"
      }
    }
  }
}
EOF
cat > ca-csr.json <<EOF
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

cfssl gencert -initca ca-csr.json | cfssljson -bare ca
将CA证书分发到每台k8s机器的/etc/kubernetes/ssl
mkdir /etc/kubernetes/ssl
cp ca* /etc/kubernetes/ssl

etcd安装

yum install etcd -y

生成ETCD证书 三台master node1 node2

cat > etcd-csr.json<<EOF
{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "${NODE_IP}"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF
cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
      -ca-key=/etc/kubernetes/ssl/ca-key.pem \
      -config=/etc/kubernetes/ssl/ca-config.json \
-profile=kubernetes etcd-csr.json | cfssljson -bare etcd

[root@k8s-master etcd]# tree
.
├── etcd.csr
├── etcd-csr.json
├── etcd-key.pem
└── etcd.pem

mkdir /etc/etcd/ssl
cp ca*.pem /etc/etcd/ssl

# grep -v '^#' /etc/etcd/etcd.conf 
ETCD_DATA_DIR="/var/lib/etcd/"
ETCD_LISTEN_PEER_URLS="http://192.168.47.120:2380"
ETCD_LISTEN_CLIENT_URLS="http://192.168.47.120:2379,http://127.0.0.1:2379"
ETCD_NAME="k8s-master"
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.47.120:2380"
ETCD_ADVERTISE_CLIENT_URLS="http://192.168.47.120:2379"
ETCD_INITIAL_CLUSTER="k8s-master=http://192.168.47.120:2380,k8s-node1=http://192.168.47.121:2380,k8s-node2=http://192.168.47.122:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
ETCD_TRUSTED_CA_FILE="/etc/kubernetes/ssl/ca.pem"
ETCD_PEER_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_PEER_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
ETCD_PEER_TRUSTED_CA_FILE="/etc/kubernetes/ssl/ca.pem"

启动
systemctl enable etcd
systemctl start etcd

相关组件安装

解压kubernetes-server-linux-amd64.tar.gz

tar xf kubernetes-server-linux-amd64.tar.gz
cd kubernetes/server/bin

kubectl组件安装

cp kubectl /usr/bin/

#创建 admin 证书,kubectl 与 kube-apiserver 的安全端口通信，需要为安全通信提供 TLS 证书和秘钥。
cat > admin-csr.json <<EOF
{
  "CN": "admin",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}
EOF
cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
 -ca-key=/etc/kubernetes/ssl/ca-key.pem \
 -config=/etc/kubernetes/ssl/ca-config.json \
 -profile=kubernetes admin-csr.json | cfssljson -bare admin
 
#生成以下文件
# tree
.
├── admin.csr
├── admin-csr.json
├── admin-key.pem
└── admin.pem
cp admin*.pem /etc/kubernetes/ssl/

#创建 kubectl kubeconfig 文件   apiserver
     kubectl config set-cluster kubernetes \
        --certificate-authority=/etc/kubernetes/ssl/ca.pem \
        --embed-certs=true \
        --server="https://192.168.47.120:6443"
#设置客户端认证参数
     kubectl config set-credentials admin \
        --client-certificate=/etc/kubernetes/ssl/admin.pem \
        --embed-certs=true \
        --client-key=/etc/kubernetes/ssl/admin-key.pem
#设置上下文参数
    kubectl config set-context kubernetes \
        --cluster=kubernetes \
        --user=admin
#设置默认上下文
    kubectl config use-context kubernetes

#生成的 kubeconfig 被保存到 ~/.kube/config 文件；
#将 ~/.kube/config 文件拷贝到运行 kubelet 命令的机器的 ~/.kube/ 目录下。
scp  ~/.kube/config node1:/root/.kube/
scp  ~/.kube/config node2:/root/.kube/

flannel安装 三个节点都操作

wget https://github.com/coreos/flannel/releases/download/v0.11.0/flannel-v0.11.0-linux-amd64.tar.gz
tar xf flannel-v0.11.0-linux-amd64.tar.gz
cp {flanneld,mk-docker-opts.sh} /usr/bin

证书生成

cat > flanneld-csr.json <<EOF
{
  "CN": "flanneld",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF
cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
      -ca-key=/etc/kubernetes/ssl/ca-key.pem \
      -config=/etc/kubernetes/ssl/ca-config.json \
-profile=kubernetes flanneld-csr.json | cfssljson -bare flanneld

# tree
.
├── flanneld.csr
├── flanneld-csr.json
├── flanneld-key.pem
└── flanneld.pem

mkdir /etc/flanneld/ssl -pv
cp flanneld*.pem /etc/flanneld/ssl
scp -r /etc/flanneld/ node1:/etc
scp -r /etc/flanneld/ node2:/etc

    etcdctl \
        --endpoints=http://192.168.47.120:2379 \
      --ca-file=/etc/kubernetes/ssl/ca.pem \
      --cert-file=/etc/flanneld/ssl/flanneld.pem \
      --key-file=/etc/flanneld/ssl/flanneld-key.pem \
set /kubernetes/network/config '{"Network":"172.30.0.0/16","Backend": {"Type": "vxlan"}}'

# cat > /usr/lib/systemd/system/flanneld.service  << EOF
[Unit]
Description=Flanneld overlay address etcd agent
After=network.target
After=network-online.target
Wants=network-online.target
After=etcd.service
Before=docker.service
[Service]
Type=notify
ExecStart= /usr/bin/flanneld \
  -etcd-cafile=/etc/kubernetes/ssl/ca.pem \
  -etcd-certfile=/etc/flanneld/ssl/flanneld.pem \
  -etcd-keyfile=/etc/flanneld/ssl/flanneld-key.pem \
  -etcd-endpoints=http://192.168.47.120:2379 \
  -etcd-prefix=/kubernetes/network
ExecStartPost=/usr/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker
Restart=on-failure
[Install]
WantedBy=multi-user.target
RequiredBy=docker.service
EOF

vim /usr/lib/systemd/system/docker.service
[Unit]
Description=Docker Application Container Engine
Documentation=http://docs.docker.io
[Service]
Environment="PATH=/usr/bin:/bin:/sbin:/usr/bin:/usr/sbin"
EnvironmentFile=-/run/flannel/docker
ExecStart=/usr/bin/dockerd --log-level=error $DOCKER_NETWORK_OPTIONS
ExecReload=/bin/kill -s HUP $MAINPID
Restart=on-failure
RestartSec=5
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
Delegate=yes
KillMode=process
[Install]
WantedBy=multi-user.target

主节点 配置kube_apiserver

cp -r server/bin/{kube-apiserver,kube-controller-manager,kube-scheduler,kubectl,kube-proxy,kubelet} /usr/bin/

生成kubernetes证书和私钥

cat > kubernetes-csr.json <<EOF
{
  "CN": "kubernetes",
  "hosts": [
    "127.0.0.1",
    "192.168.47.120",
    "10.254.0.1",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
      -ca-key=/etc/kubernetes/ssl/ca-key.pem \
      -config=/etc/kubernetes/ssl/ca-config.json \
-profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes
# tree
.
├── kubernetes.csr
├── kubernetes-csr.json
├── kubernetes-key.pem
└── kubernetes.pem

cp kubernetes*.pem /etc/kubernetes/ssl/

#配置和启动kube-apiserver
#创建 kube-apiserver 使用的客户端 token 文件。
#kubelet 首次启动时向 kube-apiserver 发送 TLS Bootstrapping 请求，kube-apiserver 验证 kubelet 请求中的 token 是否与它配置的 token.csv 一致

#TLS Bootstrapping 使用的 Token，可以使用命令 head -c 16 /dev/urandom | od -An -t x | tr -d ' ' 生成
BOOTSTRAP_TOKEN="ecf11198cb68fde328065f54563ca00b"
cat > token.csv <<EOF
${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF
cp token.csv /etc/kubernetes/

#创建 kube-apiserver 的 systemd unit 文件

cat  > /usr/lib/systemd/system/kube-apiserver.service <<EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target
[Service]
ExecStart=/usr/bin/kube-apiserver \\
  --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \\
  --advertise-address=${MASTER_IP} \\
  --bind-address=${MASTER_IP} \\
  --insecure-bind-address=${MASTER_IP} \\
  --authorization-mode=RBAC \\
  --runtime-config=rbac.authorization.k8s.io/v1alpha1 \\
  --kubelet-https=true \\
  --enable-bootstrap-token-auth=true \\
  --token-auth-file=/etc/kubernetes/token.csv \\
  --service-cluster-ip-range=${SERVICE_CIDR} \\
  --service-node-port-range=${NODE_PORT_RANGE} \\
  --tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem \\
  --tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem \\
  --client-ca-file=/etc/kubernetes/ssl/ca.pem \\
  --service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \\
  --etcd-cafile=/etc/kubernetes/ssl/ca.pem \\
  --etcd-certfile=/etc/kubernetes/ssl/kubernetes.pem \\
  --etcd-keyfile=/etc/kubernetes/ssl/kubernetes-key.pem \\
  --etcd-servers=${ETCD_ENDPOINTS} \\
  --enable-swagger-ui=true \\
  --allow-privileged=true \\
  --apiserver-count=3 \\
  --audit-log-maxage=30 \\
  --audit-log-maxbackup=3 \\
  --audit-log-maxsize=100 \\
  --audit-log-path=/var/lib/audit.log \\
  --event-ttl=1h \\
  --v=2
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable kube-apiserver
systemctl start kube-apiserver

配置和启动 kube-controller-manager

#创建 kube-controller-manager 的 systemd unit 文件
cat > /usr/lib/systemd/system/kube-controller-manager.service <<EOF
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
[Service]
ExecStart=/usr/bin/kube-controller-manager \\
  --address=127.0.0.1 \\
  --master=http://${MASTER_IP}:8080 \\
  --allocate-node-cidrs=true \\
  --service-cluster-ip-range=${SERVICE_CIDR} \\
  --cluster-cidr=${CLUSTER_CIDR} \\
  --cluster-name=kubernetes \\
  --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \\
  --cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \\
  --service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \\
  --root-ca-file=/etc/kubernetes/ssl/ca.pem \\
  --leader-elect=true \\
  --v=2
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
#启动 kube-controller-manager
systemctl daemon-reload
systemctl enable kube-controller-manager
systemctl restart kube-controller-manager
ps aux|grep kube-controller-manager

配置和启动 kube-scheduler

#创建 kube-scheduler 的 systemd unit 文件
cat > /usr/lib/systemd/system/kube-scheduler.service <<EOF
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
[Service]
ExecStart=/usr/bin/kube-scheduler \\
  --address=127.0.0.1 \\
  --master=http://${MASTER_IP}:8080 \\
  --leader-elect=true \\
  --v=2
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable kube-scheduler
systemctl restart kube-scheduler
ps aux|grep kube-scheduler

主节点验证安装

[root@k8s-master k8s]# kubectl get componentstatus
NAME                 STATUS    MESSAGE             ERROR
scheduler            Healthy   ok                  
etcd-0               Healthy   {"health":"true"}   
controller-manager   Healthy   ok

node节点 配置kubelet、kube-proxy

[root@k8s-master bin]# scp {kube-proxy,kubelet} node1:/usr/bin/ kube-proxy 100% 33MB 10.4MB/s 00:03
kubelet 100% 108MB 11.6MB/s 00:09
[root@k8s-master bin]# scp {kube-proxy,kubelet} node2:/usr/bin/ kube-proxy 100% 33MB 39.4MB/s 00:00
kubelet

将kubelet-bootstrap用户绑定到系统集群角色 kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap

#创建 kubelet bootstrapping kubeconfig 文件 #设置集群参数 kubectl config set-cluster kubernetes
--certificate-authority=/etc/kubernetes/ssl/ca.pem
--embed-certs=true
--server=${KUBE_APISERVER}
--kubeconfig=bootstrap.kubeconfig #设置客户端认证参数 kubectl config set-credentials kubelet-bootstrap
--token=${BOOTSTRAP_TOKEN}
--kubeconfig=bootstrap.kubeconfig #设置上下文参数 kubectl config set-context default
--cluster=kubernetes
--user=kubelet-bootstrap
--kubeconfig=bootstrap.kubeconfig #设置默认上下文 kubectl config use-context default --kubeconfig=bootstrap.kubeconfig

mkdir -p /var/lib/kubelet cat > kubelet.service <<EOF [Unit] Description=Kubernetes Kubelet Documentation=https://github.com/GoogleCloudPlatform/kubernetes After=docker.service Requires=docker.service [Service] WorkingDirectory=/var/lib/kubelet ExecStart=/usr/bin/kubelet \ --address=${NODE_IP} \ --hostname-override=${NODE_IP} \ --pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest \ --experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \ --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \ --cert-dir=/etc/kubernetes/ssl \ --cluster_dns=${CLUSTER_DNS_SVC_IP} \ --cluster_domain=${CLUSTER_DNS_DOMAIN} \ --hairpin-mode promiscuous-bridge \ --allow-privileged=true \ --serialize-image-pulls=false \ --logtostderr=true \ --v=2 ExecStopPost=/sbin/iptables -A INPUT -s 10.0.0.0/8 -p tcp --dport 4194 -j ACCEPT ExecStopPost=/sbin/iptables -A INPUT -s 172.16.0.0/12 -p tcp --dport 4194 -j ACCEPT ExecStopPost=/sbin/iptables -A INPUT -s 192.168.0.0/16 -p tcp --dport 4194 -j ACCEPT ExecStopPost=/sbin/iptables -A INPUT -p tcp --dport 4194 -j DROP Restart=on-failure RestartSec=5 [Install] WantedBy=multi-user.target EOF #启动 kubelet cp kubelet.service /etc/systemd/system/kubelet.service rm /etc/systemd/system/kubelet.service.d/10-kubeadm.conf systemctl daemon-reload systemctl enable kubelet systemctl restart kubelet


*配置kube-proxy*
```shell
mkdir -p /var/lib/kube-proxy
cat > kube-proxy-csr.json <<EOF
{
  "CN": "system:kube-proxy",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF
cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
      -ca-key=/etc/kubernetes/ssl/ca-key.pem \
      -config=/etc/kubernetes/ssl/ca-config.json \
      -profile=kubernetes  kube-proxy-csr.json | cfssljson -bare kube-proxy

# tree
.
├── kube-proxy.csr
├── kube-proxy-csr.json
├── kube-proxy-key.pem
└── kube-proxy.pem
cp kube-proxy*.pem /etc/kubernetes/ssl/
scp -r /etc/kubernetes/ node1:/etc/
scp -r /etc/kubernetes/ node2:/etc/
#创建 kube-proxy kubeconfig 文件
kubectl config set-cluster kubernetes \
      --certificate-authority=/etc/kubernetes/ssl/ca.pem \
      --embed-certs=true \
      --server=${KUBE_APISERVER} \
      --kubeconfig=kube-proxy.kubeconfig
#设置客户端认证参数
kubectl config set-credentials kube-proxy \
      --client-certificate=/etc/kubernetes/ssl/kube-proxy.pem \
      --client-key=/etc/kubernetes/ssl/kube-proxy-key.pem \
      --embed-certs=true \
      --kubeconfig=kube-proxy.kubeconfig
#设置上下文参数
kubectl config set-context default \
      --cluster=kubernetes \
      --user=kube-proxy \
      --kubeconfig=kube-proxy.kubeconfig
#设置默认上下文
kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
cp kube-proxy.kubeconfig /etc/kubernetes/
scp -r /etc/kubernetes/ node1:/etc/
scp -r /etc/kubernetes/ node2:/etc/

cat > kube-proxy.service <<EOF
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target
[Service]
WorkingDirectory=/var/lib/kube-proxy
ExecStart=/usr/bin/kube-proxy \\
  --bind-address=${NODE_IP} \\
  --hostname-override=${NODE_IP} \\
  --cluster-cidr=${SERVICE_CIDR} \\
  --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig \\
  --logtostderr=true \\
  --v=2
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
#启动 kube-proxy
    cp kube-proxy.service /etc/systemd/system/
    systemctl daemon-reload
    systemctl enable kube-proxy
    systemctl start kube-proxy
    ps aux|grep kube-proxy
}