*: tiny cleanup

01.系统初始化和全局变量.md

@@ -130,9 +130,10 @@ $ sudo sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
 
 ``` bash
 $ sudo setenforce 0
-$ grep SELINUX /etc/selinux/config  # 修改配置文件，永久生效
+$ grep SELINUX /etc/selinux/config 
 SELINUX=disabled
 ```
++ 修改配置文件，永久生效；
 
 ## 关闭 dnsmasq
 
@@ -153,17 +154,12 @@ net.ipv4.ip_forward=1
 vm.swappiness=0
 vm.overcommit_memory=1
 vm.panic_on_oom=0
+fs.inotify.max_user_watches=89100
 EOF
 $ sudo cp kubernetes.conf  /etc/sysctl.d/kubernetes.conf
 $ sudo sysctl -p /etc/sysctl.d/kubernetes.conf
 ```
 
-切换到 root 账号执行：
-
-``` bash
-[root@kube-node1 ~]# for intf in /sys/devices/virtual/net/docker0/brif/*; do echo 1 > $intf/hairpin_mode; done
-```
-
 ## 加载内核模块
 
 ``` bash
@@ -279,5 +275,6 @@ for node_ip in ${NODE_IPS[@]}
   done
 ```
 
-## 参考：
+## 参考
+
 1. 系统内核相关参数参考：https://docs.openshift.com/enterprise/3.2/admin_guide/overcommit.html

03.部署kubectl命令行工具.md

@@ -27,7 +27,7 @@ tar -xzvf kubernetes-client-linux-amd64.tar.gz
 分发到所有使用 kubectl 的节点：
 
 ``` bash
-source /opt/k8s/bin/environment.sh # 导入 NODE_IPS 环境变量
+source /opt/k8s/bin/environment.sh
 for node_ip in ${NODE_IPS[@]}
   do
     echo ">>> ${node_ip}"
@@ -117,7 +117,7 @@ kubectl config use-context kubernetes --kubeconfig=kubectl.kubeconfig
 分发到所有使用 `kubelet` 命令的节点：
 
 ``` bash
-source /opt/k8s/bin/environment.sh # 导入 NODE_IPS 环境变量
+source /opt/k8s/bin/environment.sh
 for node_ip in ${NODE_IPS[@]}
   do
     echo ">>> ${node_ip}"

04.部署etcd集群.md

@@ -79,6 +79,7 @@ cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \
     -ca-key=/etc/kubernetes/cert/ca-key.pem \
     -config=/etc/kubernetes/cert/ca-config.json \
     -profile=kubernetes etcd-csr.json | cfssljson -bare etcd
+ls etcd*
 ```
 
 分发生成的证书和私钥到各 etcd 节点：

05.部署flannel网络.md

@@ -191,7 +191,7 @@ etcdctl \
 
 输出：
 
-  `{"Network":"172.30.0.0/16", "SubnetLen": 24, "Backend": {"Type": "vxlan"}}`
+`{"Network":"172.30.0.0/16", "SubnetLen": 24, "Backend": {"Type": "vxlan"}}`
 
 查看已分配的 Pod 子网段列表(/24):
 
@@ -227,7 +227,7 @@ etcdctl \
 
 输出：
 
-  `{"PublicIP":"172.27.129.105","BackendType":"vxlan","BackendData":{"VtepMAC":"12:21:93:9e:b1:eb"}}`
+`{"PublicIP":"172.27.129.105","BackendType":"vxlan","BackendData":{"VtepMAC":"12:21:93:9e:b1:eb"}}`
 
 ## 验证各节点能通过 Pod 网段互通
 

07-1.docker.md

@@ -122,7 +122,7 @@ for node_ip in ${NODE_IPS[@]}
     ssh root@${node_ip} "iptables -F && iptables -X && iptables -F -t nat && iptables -X -t nat"
     ssh root@${node_ip} "iptables -P FORWARD ACCEPT"
     ssh root@${node_ip} "systemctl daemon-reload && systemctl enable docker && systemctl restart docker"
-    ssh root@${node_ip} "for intf in /sys/devices/virtual/net/docker0/brif/*; do echo 1 > $intf/hairpin_mode; done"
+    ssh root@${node_ip} 'for intf in /sys/devices/virtual/net/docker0/brif/*; do echo 1 > $intf/hairpin_mode; done'
     ssh root@${node_ip} "sudo sysctl -p /etc/sysctl.d/kubernetes.conf"
   done
 ```

10.部署Docker-Registry.md

@@ -10,13 +10,13 @@ tags: registry, ceph
 
 示例两台机器 IP 如下：
 
-+ ceph rgw: 10.64.3.9
-+ docker registry: 10.64.3.7
++ ceph rgw: 172.27.132.66
++ docker registry: 172.27.132.67
 
 ## 部署 ceph RGW 节点
 
 ``` bash
-$ ceph-deploy rgw create 10.64.3.9 # rgw 默认监听7480端口
+$ ceph-deploy rgw create 172.27.132.66 # rgw 默认监听7480端口
 $
 ```
 
@@ -63,7 +63,7 @@ $ radosgw-admin key create --subuser=demo:swift --key-type=swift --gen-secret
     "swift_keys": [
         {
             "user": "demo:swift",
-            "secret_key": "aCgVTx3Gfz1dBiFS4NfjIRmvT0sgpHDP6aa0Yfrh"
+            "secret_key": "ttQcU1O17DFQ4I9xzKqwgUe7WIYYX99zhcIfU9vb"
         }
     ],
     "caps": [],
@@ -84,20 +84,20 @@ $ radosgw-admin key create --subuser=demo:swift --key-type=swift --gen-secret
 }
 ```
 
-+ `aCgVTx3Gfz1dBiFS4NfjIRmvT0sgpHDP6aa0Yfrh` 为子账号 demo:swift 的 secret key；
++ `ttQcU1O17DFQ4I9xzKqwgUe7WIYYX99zhcIfU9vb` 为子账号 demo:swift 的 secret key；
 
 ## 创建 docker registry
 
 创建 registry 使用的 x509 证书
 
 ``` bash
-$ mdir -p registry/{auth,certs}
-$ cat registry-csr.json
+$ mkdir -p registry/{auth,certs}
+$ cat > registry-csr.json <<EOF
 {
   "CN": "registry",
   "hosts": [
       "127.0.0.1",
-      "10.64.3.7"
+      "172.27.132.67"
   ],
   "key": {
     "algo": "rsa",
@@ -113,6 +113,7 @@ $ cat registry-csr.json
     }
   ]
 }
+EOF
 $ cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \
     -ca-key=/etc/kubernetes/cert/ca-key.pem \
     -config=/etc/kubernetes/cert/ca-config.json \
@@ -126,18 +127,18 @@ $
 创建 HTTP Baisc 认证文件
 
 ``` bash
-$ docker run --entrypoint htpasswd registry:2 -Bbn foo foo123  > auth/htpasswd
-$ cat auth/htpasswd
-foo:$2y$05$I60z69MdluAQ8i1Ka3x3Neb332yz1ioow2C4oroZSOE0fqPogAmZm
+$ docker run --entrypoint htpasswd registry:2 -Bbn foo foo123  > registry/auth/htpasswd
+$ cat  registry/auth/htpasswd
+foo:$2y$05$iZaM45Jxlcg0DJKXZMggLOibAsHLGybyU.CgU9AHqWcVDyBjiScN.
 ```
 
 配置 registry 参数
 
 ``` bash
-$ export RGW_AUTH_URL="http://10.64.3.9:7480/auth/v1"
-$ export RGW_USER="demo:swift"
-$ export RGW_SECRET_KEY="aCgVTx3Gfz1dBiFS4NfjIRmvT0sgpHDP6aa0Yfrh"
-$ cat > config.yml << EOF
+export RGW_AUTH_URL="http://172.27.132.66:7480/auth/v1"
+export RGW_USER="demo:swift"
+export RGW_SECRET_KEY="ttQcU1O17DFQ4I9xzKqwgUe7WIYYX99zhcIfU9vb"
+cat > config.yml << EOF
 # https://docs.docker.com/registry/configuration/#list-of-configuration-options
 version: 0.1
 log:
@@ -176,6 +177,8 @@ health:
     interval: 10s
     threshold: 3
 EOF
+[k8s@kube-node1 cert]$ cp config.yml registry
+[k8s@kube-node1 cert]$ scp -r registry 172.27.132.67:/opt/k8s
 ```
 
 + storage.swift 指定后端使用 swfit 接口协议的存储，这里配置的是 ceph rgw 存储参数；
@@ -185,29 +188,29 @@ EOF
 创建 docker registry
 
 ``` bash
+ssh k8s@172.27.132.67
 $ docker run -d -p 8000:8000 \
-    -v $(pwd)/registry/auth/:/auth \
-    -v $(pwd)/registry/certs:/certs \
-    -v $(pwd)/config.yml:/etc/docker/registry/config.yml \
+    -v /opt/k8s/registry/auth/:/auth \
+    -v /opt/k8s/registry/certs:/certs \
+    -v /opt/k8s/registry/config.yml:/etc/docker/registry/config.yml \
     --name registry registry:2
 ```
 
-+ 执行该 docker run 命令的机器 IP 为 10.64.3.7；
++ 执行该 docker run 命令的机器 IP 为 172.27.132.67；
 
 ## 向 registry push image
 
-将签署 registry 证书的 CA 证书拷贝到 `/etc/docker/certs.d/10.64.3.7:8000` 目录下
+将签署 registry 证书的 CA 证书拷贝到 `/etc/docker/certs.d/172.27.132.67:8000` 目录下
 
 ``` bash
-$ sudo mkdir -p /etc/docker/certs.d/10.64.3.7:8000
-$ sudo cp /etc/kubernetes/cert/ca.pem /etc/docker/certs.d/10.64.3.7:8000/ca.crt
-$
+[k8s@kube-node1 cert]$ sudo mkdir -p /etc/docker/certs.d/172.27.132.67:8000
+[k8s@kube-node1 cert]$ sudo cp /etc/kubernetes/cert/ca.pem /etc/docker/certs.d/172.27.132.67:8000/ca.crt
 ```
 
 登陆私有 registry
 
 ``` bash
-$ docker login 10.64.3.7:8000
+$ docker login 172.27.132.67:8000
 Username: foo
 Password:
 Login Succeeded
@@ -219,7 +222,7 @@ Login Succeeded
 $ cat ~/.docker/config.json
 {
         "auths": {
-                "10.64.3.7:8000": {
+                "172.27.132.67:8000": {
                         "auth": "Zm9vOmZvbzEyMw=="
                 }
         }
@@ -229,17 +232,17 @@ $ cat ~/.docker/config.json
 将本地的 image 打上私有 registry 的 tag
 
 ``` bash
-$ docker tag docker.io/kubernetes/pause 10.64.3.7:8000/zhangjun3/pause
+$ docker tag docker.io/kubernetes/pause 172.27.132.67:8000/zhangjun3/pause
 $ docker images |grep pause
 docker.io/kubernetes/pause                            latest              f9d5de079539        2 years ago         239.8 kB
-10.64.3.7:8000/zhangjun3/pause                        latest              f9d5de079539        2 years ago         239.8 kB
+172.27.132.67:8000/zhangjun3/pause                        latest              f9d5de079539        2 years ago         239.8 kB
 ```
 
 将 image push 到私有 registry
 
 ``` bash
-$ docker push 10.64.3.7:8000/zhangjun3/pause
-The push refers to a repository [10.64.3.7:8000/zhangjun3/pause]
+$ docker push 172.27.132.67:8000/zhangjun3/pause
+The push refers to a repository [172.27.132.67:8000/zhangjun3/pause]
 5f70bf18a086: Pushed
 e16a89738269: Pushed
 latest: digest: sha256:9a6b437e896acad3f5a2a8084625fdd4177b2e7124ee943af642259f2f283359 size: 916
@@ -275,14 +278,14 @@ $ rados --pool default.rgw.buckets.data ls|grep pause
 ### 查询私有镜像中的 images
 
 ``` bash
-$ curl  --user zhangjun3:xxx --cacert /etc/docker/certs.d/10.64.3.7\:8000/ca.crt https://10.64.3.7:8000/v2/_catalog
+$ curl  --user zhangjun3:xxx --cacert /etc/docker/certs.d/172.27.132.67\:8000/ca.crt https://172.27.132.67:8000/v2/_catalog
 {"repositories":["library/redis","zhangjun3/busybox","zhangjun3/pause","zhangjun3/pause2"]}
 ```
 
 ### 查询某个镜像的 tags 列表
 
 ``` bash
-$ curl  --user zhangjun3:xxx --cacert /etc/docker/certs.d/10.64.3.7\:8000/ca.crt https://10.64.3.7:8000/v2/zhangjun3/busybox/tags/list
+$ curl  --user zhangjun3:xxx --cacert /etc/docker/certs.d/172.27.132.67\:8000/ca.crt https://172.27.132.67:8000/v2/zhangjun3/busybox/tags/list
 {"name":"zhangjun3/busybox","tags":["latest"]}
 ```
 
@@ -293,11 +296,11 @@ $ curl  --user zhangjun3:xxx --cacert /etc/docker/certs.d/10.64.3.7\:8000/ca.crt
 注意，必须包含请求头：`Accept: application/vnd.docker.distribution.manifest.v2+json`：
 
 ``` bash
-$ curl -v -H "Accept: application/vnd.docker.distribution.manifest.v2+json" --user zhangjun3:xxx --cacert /etc/docker/certs.d/10.64.3.7\:8000/ca.crt https://10.64.3.7:8000/v2/zhangjun3/busybox/manifests/latest
+$ curl -v -H "Accept: application/vnd.docker.distribution.manifest.v2+json" --user zhangjun3:xxx --cacert /etc/docker/certs.d/172.27.132.67\:8000/ca.crt https://172.27.132.67:8000/v2/zhangjun3/busybox/manifests/latest
 
 > GET /v2/zhangjun3/busybox/manifests/latest HTTP/1.1
 > User-Agent: curl/7.29.0
-> Host: 10.64.3.7:8000
+> Host: 172.27.132.67:8000
 > Accept: application/vnd.docker.distribution.manifest.v2+json
 >
 < HTTP/1.1 200 OK
@@ -332,7 +335,7 @@ $ curl -v -H "Accept: application/vnd.docker.distribution.manifest.v2+json" --us
 向 `/v2/<name>/manifests/<reference>` 发送 DELETE 请求，reference 为上一步返回的 Docker-Content-Digest 字段内容：
 
 ``` bash
-$ curl -X DELETE  --user zhangjun3:xxx --cacert /etc/docker/certs.d/10.64.3.7\:8000/ca.crt https://10.64.3.7:8000/v2/zhangjun3/busybox/manifests/sha256:68effe31a4ae8312e47f54bec52d1fc925908009ce7e6f734e1b54a4169081c5
+$ curl -X DELETE  --user zhangjun3:xxx --cacert /etc/docker/certs.d/172.27.132.67\:8000/ca.crt https://172.27.132.67:8000/v2/zhangjun3/busybox/manifests/sha256:68effe31a4ae8312e47f54bec52d1fc925908009ce7e6f734e1b54a4169081c5
 $
 ```
 
@@ -341,7 +344,7 @@ $
 向 `/v2/<name>/blobs/<digest>`发送 DELETE 请求，其中 digest 是上一步返回的 `fsLayers.blobSum` 字段内容：
 
 ``` bash
-$ curl -X DELETE  --user zhangjun3:xxx --cacert /etc/docker/certs.d/10.64.3.7\:8000/ca.crt https://10.64.3.7:8000/v2/zhangjun3/busybox/blobs/sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4
-$ curl -X DELETE  --cacert /etc/docker/certs.d/10.64.3.7\:8000/ca.crt https://10.64.3.7:8000/v2/zhangjun3/busybox/blobs/sha256:04176c8b224aa0eb9942af765f66dae866f436e75acef028fe44b8a98e045515
+$ curl -X DELETE  --user zhangjun3:xxx --cacert /etc/docker/certs.d/172.27.132.67\:8000/ca.crt https://172.27.132.67:8000/v2/zhangjun3/busybox/blobs/sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4
+$ curl -X DELETE  --cacert /etc/docker/certs.d/172.27.132.67\:8000/ca.crt https://172.27.132.67:8000/v2/zhangjun3/busybox/blobs/sha256:04176c8b224aa0eb9942af765f66dae866f436e75acef028fe44b8a98e045515
 $
 ```