Linux - K8S - 安全機制之SA和UA
阿新 • • 發佈：2022-01-03
# 3.安全機制
## 3.1 SA
```sh
# 建立測試SA，我們發現一個SA有相對應的secrets
[19:25:06 root@master1 security]#kubectl create -f - <<EOF
> apiVersion: v1
> kind: ServiceAccount
> metadata:
> name: build-robot
> EOF
serviceaccount/build-robot created
[20:12:17 root@master1 security]#kubectl get sa,secrets
NAME SECRETS AGE
serviceaccount/build-robot 1 7m39s
serviceaccount/default 1 21d
NAME TYPE DATA AGE
secret/build-robot-token-rwfsb kubernetes.io/service-account-token 3 7m39s
secret/default-token-zjxfd kubernetes.io/service-account-token 3 21d
# 結合pod測試
[20:20:52 root@master1 security]#cat robot.yaml
apiVersion: v1
kind: Pod
metadata:
name: nginx
spec:
containers:
- image: 10.0.0.55:80/mykubernetes/nginx:1.21.3
name: nginx
volumeMounts:
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: build-robot-token-rwfsb
serviceAccountName: build-robot
volumes:
- name: build-robot-token-rwfsb
projected:
sources:
- serviceAccountToken:
path: build-robot-token-rwfsb
expirationSeconds: 7200
[20:20:44 root@master1 security]#kubectl describe pod nginx
Name: nginx
Namespace: default
Priority: 0
Node: node2.noisedu.cn/10.0.0.54
Start Time: Sun, 02 Jan 2022 20:20:44 +0800
Labels: <none>
Annotations: cni.projectcalico.org/containerID: 87d85747bcce0bce508455b4697e8ed2e6b387c3ef2511a7bd735fd288f5af13
cni.projectcalico.org/podIP: 10.244.4.4/32
cni.projectcalico.org/podIPs: 10.244.4.4/32
Status: Running
IP: 10.244.4.4
IPs:
IP: 10.244.4.4
Containers:
nginx:
Container ID: docker://66696744862f0e95efaf6eda484df98deb96967ccb2d7ef79c284c47fc09b09f
Image: 10.0.0.55:80/mykubernetes/nginx:1.21.3
Image ID: docker-pullable://10.0.0.55:80/mykubernetes/nginx@sha256:4424e31f2c366108433ecca7890ad527b243361577180dfd9a5bb36e828abf47
Port: <none>
Host Port: <none>
State: Running
Started: Sun, 02 Jan 2022 20:20:46 +0800
Ready: True
Restart Count: 0
Environment: <none>
Mounts:
/var/run/secrets/kubernetes.io/serviceaccount from build-robot-token-rwfsb (rw)
Conditions:
Type Status
Initialized True
Ready True
ContainersReady True
PodScheduled True
Volumes:
build-robot-token-rwfsb:
Type: Projected (a volume that contains injected data from multiple sources)
TokenExpirationSeconds: 7200
QoS Class: BestEffort
Node-Selectors: <none>
Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Scheduled 8s default-scheduler Successfully assigned default/nginx to node2.noisedu.cn
Normal Pulled 7s kubelet Container image "10.0.0.55:80/mykubernetes/nginx:1.21.3" already present on machine
Normal Created 6s kubelet Created container nginx
Normal Started 6s kubelet Started container nginx
```
## 3.2 UA
### user account 至少分為三個部分：
#### 使用者條目 - credentials設定具體的user account名稱
#### 叢集 - cluster 設定該user account所工作的區域
#### 上下文環境 - context 設定使用者和叢集的關係
```sh
[16:34:11 root@master1 ~]#kubectl config view
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: DATA+OMITTED
server: https://10.0.0.70:6443
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: kubernetes-admin
name: kubernetes-admin@kubernetes
current-context: kubernetes-admin@kubernetes
kind: Config
preferences: {}
users:
- name: kubernetes-admin
user:
client-certificate-data: REDACTED
client-key-data: REDACTED
```
建立UA的流程主要是分為7個步驟,最後的結果和上面類似.
1. 建立私鑰
```sh
[17:02:17 root@master1 security]#cd /etc/kubernetes/pki/
[17:03:35 root@master1 pki]#umask 077; openssl genrsa -out noiselys.key 2048
Generating RSA private key, 2048 bit long modulus (2 primes)
..................................................................................+++++
..+++++
e is 65537 (0x010001)
[17:04:14 root@master1 pki]#ll
total 72
drwxr-xr-x 3 root root 4096 Jan 3 17:04 ./
drwxr-xr-x 5 root root 4096 Dec 12 00:18 ../
-rw-r--r-- 1 root root 1294 Dec 12 00:18 apiserver.crt
-rw-r--r-- 1 root root 1155 Dec 12 00:18 apiserver-etcd-client.crt
-rw------- 1 root root 1675 Dec 12 00:18 apiserver-etcd-client.key
-rw------- 1 root root 1675 Dec 12 00:18 apiserver.key
-rw-r--r-- 1 root root 1164 Dec 12 00:18 apiserver-kubelet-client.crt
-rw------- 1 root root 1679 Dec 12 00:18 apiserver-kubelet-client.key
-rw-r--r-- 1 root root 1066 Dec 11 23:29 ca.crt
-rw------- 1 root root 1675 Dec 11 23:29 ca.key
drwxr-xr-x 2 root root 4096 Dec 11 23:29 etcd/
-rw-r--r-- 1 root root 1078 Dec 11 23:29 front-proxy-ca.crt
-rw------- 1 root root 1679 Dec 11 23:29 front-proxy-ca.key
-rw-r--r-- 1 root root 1119 Dec 12 00:18 front-proxy-client.crt
-rw------- 1 root root 1679 Dec 12 00:18 front-proxy-client.key
-rw------- 1 root root 1679 Jan 3 17:04 noiselys.key
-rw------- 1 root root 1675 Dec 11 23:29 sa.key
-rw------- 1 root root 451 Dec 11 23:29 sa.pub
```
2. 基於私鑰檔案建立證書籤名請求
```sh
[17:04:17 root@master1 pki]#openssl req -new -key noiselys.key -out noiselys.csr -subj "/CN=noiselys/O=noiselys"
Can't load /root/.rnd into RNG
140623717016000:error:2406F079:random number generator:RAND_load_file:Cannot open file:../crypto/rand/randfile.c:88:Filename=/root/.rnd
# 若是報這個error，可執行下面
[17:07:23 root@master1 pki]#cd /root
[17:08:00 root@master1 ~]#openssl rand -writerand .rnd
# 再次生成證書籤名檔案
[17:08:29 root@master1 pki]#rm -f noiselys.csr
[17:08:35 root@master1 pki]#openssl req -new -key noiselys.key -out noiselys.csr -subj "/CN=noiselys/O=noiselys"
[17:08:41 root@master1 pki]#ll
total 76
drwxr-xr-x 3 root root 4096 Jan 3 17:08 ./
drwxr-xr-x 5 root root 4096 Dec 12 00:18 ../
-rw-r--r-- 1 root root 1294 Dec 12 00:18 apiserver.crt
-rw-r--r-- 1 root root 1155 Dec 12 00:18 apiserver-etcd-client.crt
-rw------- 1 root root 1675 Dec 12 00:18 apiserver-etcd-client.key
-rw------- 1 root root 1675 Dec 12 00:18 apiserver.key
-rw-r--r-- 1 root root 1164 Dec 12 00:18 apiserver-kubelet-client.crt
-rw------- 1 root root 1679 Dec 12 00:18 apiserver-kubelet-client.key
-rw-r--r-- 1 root root 1066 Dec 11 23:29 ca.crt
-rw------- 1 root root 1675 Dec 11 23:29 ca.key
drwxr-xr-x 2 root root 4096 Dec 11 23:29 etcd/
-rw-r--r-- 1 root root 1078 Dec 11 23:29 front-proxy-ca.crt
-rw------- 1 root root 1679 Dec 11 23:29 front-proxy-ca.key
-rw-r--r-- 1 root root 1119 Dec 12 00:18 front-proxy-client.crt
-rw------- 1 root root 1679 Dec 12 00:18 front-proxy-client.key
-rw------- 1 root root 915 Jan 3 17:08 noiselys.csr
-rw------- 1 root root 1679 Jan 3 17:04 noiselys.key
-rw------- 1 root root 1675 Dec 11 23:29 sa.key
-rw------- 1 root root 451 Dec 11 23:29 sa.pub
```
3. 基於私鑰和簽名請求生產證書檔案
```sh
[17:09:32 root@master1 pki]#openssl x509 -req -in noiselys.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out noiselys.crt -days 365
Signature ok
subject=CN = noiselys, O = noiselys
Getting CA Private Key
[17:11:44 root@master1 pki]#ll noise*
-rw------- 1 root root 1017 Jan 3 17:11 noiselys.crt
-rw------- 1 root root 915 Jan 3 17:08 noiselys.csr
-rw------- 1 root root 1679 Jan 3 17:04 noiselys.key
[17:11:49 root@master1 pki]#openssl x509 -in noiselys.crt -text -noout
Certificate:
Data:
Version: 1 (0x0)
Serial Number:
45:60:a7:2a:61:6f:67:25:67:ed:2a:8f:6f:77:0b:f9:55:4c:b7:6c
Signature Algorithm: sha256WithRSAEncryption
Issuer: CN = kubernetes
Validity
Not Before: Jan 3 09:11:44 2022 GMT
Not After : Jan 3 09:11:44 2023 GMT
Subject: CN = noiselys, O = noiselys
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
RSA Public-Key: (2048 bit)
Modulus:
00:e4:d1:fd:14:84:5f:f4:e6:5c:87:c3:c0:8e:f0:
8c:dc:16:c7:b3:76:42:46:a6:5d:ab:a2:3f:7a:25:
3d:92:64:af:49:27:ab:49:97:c4:ae:89:79:f5:a8:
60:40:b6:b2:44:bc:87:fd:79:75:0f:7e:99:6f:e0:
bd:fe:59:e4:6d:52:82:64:bd:a5:f5:b2:27:39:72:
fb:57:25:8c:54:f1:ce:93:d4:59:18:02:ae:d0:31:
33:07:1e:70:ee:e2:db:0b:31:26:c6:a7:64:a0:99:
5f:a7:57:17:09:98:d9:e4:12:a4:89:e6:fe:33:61:
f7:14:7c:b6:be:9d:5c:0d:57:32:b4:c1:2b:48:db:
97:ec:7a:64:f0:17:63:b3:35:16:4d:b5:ff:0d:69:
09:55:3b:7f:ea:2b:a2:62:54:37:1c:43:94:e8:c6:
f2:cb:63:85:76:61:71:1d:5b:98:ab:46:c9:4a:b1:
68:7e:ea:a0:fb:bc:b1:ba:16:f1:65:82:43:bc:56:
27:51:03:f3:9b:bd:c3:7d:f6:6a:eb:65:41:88:51:
08:21:36:c7:c3:ec:91:fc:8a:5c:5e:ea:fe:c0:46:
7a:18:69:8a:9e:cb:5a:cf:de:cc:e6:16:22:dd:96:
ca:bb:4d:fb:ed:44:e8:59:14:b8:ab:59:a3:c6:8d:
52:57
Exponent: 65537 (0x10001)
Signature Algorithm: sha256WithRSAEncryption
bb:61:ef:ad:91:da:b7:ac:45:c2:f0:6e:f8:a0:f4:66:06:aa:
cd:e1:db:4c:a3:43:24:20:88:45:86:3e:1c:5f:88:9b:85:d1:
70:52:47:13:fa:ba:e1:87:2a:08:b1:a8:fe:5a:5f:7a:01:46:
b4:6d:9a:fd:2c:14:6b:eb:23:b1:a6:31:2f:7b:2d:da:9f:04:
fa:f5:19:f1:d9:41:5b:0a:7b:3e:b7:b9:3b:32:b2:12:cc:04:
38:4c:1f:63:df:c9:c5:d5:82:38:a2:7c:5f:27:9f:94:46:3c:
cb:c1:32:6c:ed:c5:02:b4:2f:bf:9f:b7:66:24:18:f0:a7:80:
71:68:5a:4f:30:02:f3:ef:fb:0b:9f:2b:ce:b1:59:8d:f7:d3:
8a:92:88:2e:9c:16:91:17:ec:b7:4c:2a:87:81:c5:f9:30:ed:
94:dd:b2:b2:29:fb:86:f1:db:7f:11:d3:ea:ac:46:a3:bb:65:
ac:84:cc:04:77:7b:3d:70:bf:34:f5:59:b0:4b:b4:fa:af:8d:
2b:d0:83:aa:cc:7f:0a:68:d0:a8:40:3a:5c:fe:e7:cc:45:5e:
0f:dd:01:42:69:3b:9d:40:d0:a0:95:ff:42:42:cf:6b:4e:8a:
00:d3:4c:bb:37:57:65:51:2a:10:0a:15:33:2d:58:2f:a9:d4:
50:d7:c3:b4
```
4. 基於tls檔案在k8s上建立使用者
```sh
[17:12:30 root@master1 pki]#kubectl config set-credentials noiselys --client-certificate=noiselys.crt --client-key=noiselys.key --embed-certs=true --kubeconfig=/data/noiselys.conf
User "noiselys" set.
[17:15:11 root@master1 pki]#kubectl config view --kubeconfig=/data/noiselys.conf
apiVersion: v1
clusters: null
contexts: null
current-context: ""
kind: Config
preferences: {}
users:
- name: noiselys
user:
client-certificate-data: REDACTED
client-key-data: REDACTED
```
5. 建立工作區域 - cluster
```shell
[17:16:54 root@master1 pki]#kubectl config set-cluster mycluster --server="https://10.0.0.70:6443" --certificate-authority=/etc/kubernetes/pki/ca.crt --embed-certs=true --kubeconfig=/data/noiselys.conf
Cluster "mycluster" set.
[17:20:23 root@master1 pki]#kubectl config view --kubeconfig=/data/noiselys.conf
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: DATA+OMITTED
server: https://10.0.0.70:6443
name: mycluster
contexts: null
current-context: ""
kind: Config
preferences: {}
users:
- name: noiselys
user:
client-certificate-data: REDACTED
client-key-data: REDACTED
```
6. 將cluster和user關聯起來 - context
```shell
[17:20:39 root@master1 pki]#kubectl config set-context noiselys@mycluster --cluster=mycluster --user=noiselys --kubeconfig=/data/noiselys.conf
Context "noiselys@mycluster" created.
[17:22:37 root@master1 pki]#kubectl config view --kubeconfig=/data/noiselys.conf
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: DATA+OMITTED
server: https://10.0.0.70:6443
name: mycluster
contexts:
- context:
cluster: mycluster
user: noiselys
name: noiselys@mycluster
current-context: ""
kind: Config
preferences: {}
users:
- name: noiselys
user:
client-certificate-data: REDACTED
client-key-data: REDACTED
```
7. 驗證結果
```sh
# 上面的current-context是空，我們需要切換一下使用者
[17:24:53 root@master1 pki]#kubectl config use-context noiselys@mycluster --kubeconfig=/data/noiselys.conf
Switched to context "noiselys@mycluster".
[17:25:01 root@master1 pki]#kubectl config view --kubeconfig=/data/noiselys.conf
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: DATA+OMITTED
server: https://10.0.0.70:6443
name: mycluster
contexts:
- context:
cluster: mycluster
user: noiselys
name: noiselys@mycluster
current-context: noiselys@mycluster
kind: Config
preferences: {}
users:
- name: noiselys
user:
client-certificate-data: REDACTED
client-key-data: REDACTED
# 但是關於許可權我們還未設定，所以無法得到資源資訊
[17:25:09 root@master1 pki]#kubectl get pod --kubeconfig=/data/noiselys.conf
Error from server (Forbidden): pods is forbidden: User "noiselys" cannot list resource "pods" in API group "" in the namespace "default"
# 關於檢視conf，我們有三種方法，從下面可以看出,推薦第一種
# 如果想多個配置檔案並行，可以加入到環境變數KUBECONFIG，用：隔開
[17:26:05 root@master1 pki]#kubectl config --help
Modify kubeconfig files using subcommands like "kubectl config set current-context my-context"
The loading order follows these rules:
1. If the --kubeconfig flag is set, then only that file is loaded. The flag may only be set once and no merging takes
place.
2. If $KUBECONFIG environment variable is set, then it is used as a list of paths (normal path delimiting rules for
your system). These paths are merged. When a value is modified, it is modified in the file that defines the stanza. When
a value is created, it is created in the first file that exists. If no files in the chain exist, then it creates the
last file in the list.
3. Otherwise, ${HOME}/.kube/config is used and no merging takes place.
```
##