docker資源隔離cgroup
阿新 • • 發佈：2020-08-15
linux的cgroup理論：https://www.cnblogs.com/menkeyi/p/10941843.html
容器的資源隔離就是基於linux的cgroup實現。
1.實踐linux cgroup
[crmop@localhost ~]$ mount -t cgroup ##檢視cgroup掛載，子目錄（cpuset即cpu核數,cpu,memory）表示可以被cgroup限制的資源種類
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd)
cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset)
cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb)
cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_cls,net_prio)
cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory)
cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event)
cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer)
cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids)
cgroup on /sys/fs/cgroup/rdma type cgroup (rw,nosuid,nodev,noexec,relatime,rdma)
[root@localhost ~]# cd /sys/fs/cgroup [root@localhost cgroup]# ls ##子目錄（cpuset即cpu核數,cpu,memory）表示可以被cgroup限制的資源種類 blkio cpuacct cpuset freezer memory net_cls,net_prio perf_event rdma cpu cpu,cpuacct devices hugetlb net_cls net_prio pids systemd [root@localhost cgroup]# cd cpu ##對cpu子系統來說，提供具體可以被限制的方法，如cfs_period_us與cfs_quota_us組合，
##表示在cfs_period_us時間長度內分得cfs_quota_us的cpu時間，即限制cpu的最高使用率為cfs_quota_us/cfs_period_us*100% [root@localhost cpu]# ls cgroup.clone_children cpuacct.usage cpuacct.usage_percpu_user cpu.cfs_quota_us cpu.stat cgroup.procs cpuacct.usage_all cpuacct.usage_sys cpu.rt_period_us notify_on_release cgroup.sane_behavior cpuacct.usage_percpu cpuacct.usage_user cpu.rt_runtime_us release_agent cpuacct.stat cpuacct.usage_percpu_sys cpu.cfs_period_us cpu.shares tasks [root@localhost cpu]# ll total 0 -rw-r--r-- 1 root root 0 Aug 15 00:55 cgroup.clone_children -rw-r--r-- 1 root root 0 Jun 4 10:18 cgroup.procs -r--r--r-- 1 root root 0 Aug 15 00:55 cgroup.sane_behavior -r--r--r-- 1 root root 0 Aug 15 00:55 cpuacct.stat -rw-r--r-- 1 root root 0 Aug 15 00:55 cpuacct.usage -r--r--r-- 1 root root 0 Aug 15 00:55 cpuacct.usage_all -r--r--r-- 1 root root 0 Aug 15 00:55 cpuacct.usage_percpu -r--r--r-- 1 root root 0 Aug 15 00:55 cpuacct.usage_percpu_sys -r--r--r-- 1 root root 0 Aug 15 00:55 cpuacct.usage_percpu_user -r--r--r-- 1 root root 0 Aug 15 00:55 cpuacct.usage_sys -r--r--r-- 1 root root 0 Aug 15 00:55 cpuacct.usage_user -rw-r--r-- 1 root root 0 Aug 15 00:55 cpu.cfs_period_us -rw-r--r-- 1 root root 0 Aug 15 00:55 cpu.cfs_quota_us -rw-r--r-- 1 root root 0 Aug 15 00:55 cpu.rt_period_us -rw-r--r-- 1 root root 0 Aug 15 00:55 cpu.rt_runtime_us -rw-r--r-- 1 root root 0 Aug 15 00:55 cpu.shares -r--r--r-- 1 root root 0 Aug 15 00:55 cpu.stat -rw-r--r-- 1 root root 0 Aug 15 00:55 notify_on_release -rw-r--r-- 1 root root 0 Aug 15 00:55 release_agent -rw-r--r-- 1 root root 0 Aug 15 00:55 tasks [root@localhost cpu]# mkdir container ##這個目錄稱為一個“控制組”，在該目錄下，自動生成該子系統對應的資源限制檔案 [root@localhost cpu]# ll total 0 -rw-r--r-- 1 root root 0 Aug 15 00:55 cgroup.clone_children -rw-r--r-- 1 root root 0 Jun 4 10:18 cgroup.procs -r--r--r-- 1 root root 0 Aug 15 00:55 cgroup.sane_behavior drwxr-xr-x 2 root root 0 Aug 15 00:56 container -r--r--r-- 1 root root 0 Aug 15 00:55 cpuacct.stat -rw-r--r-- 1 root root 0 Aug 15 00:55 cpuacct.usage -r--r--r-- 1 root root 0 Aug 15 00:55 cpuacct.usage_all -r--r--r-- 1 root root 0 Aug 15 00:55 cpuacct.usage_percpu -r--r--r-- 1 root root 0 Aug 15 00:55 cpuacct.usage_percpu_sys -r--r--r-- 1 root root 0 Aug 15 00:55 cpuacct.usage_percpu_user -r--r--r-- 1 root root 0 Aug 15 00:55 cpuacct.usage_sys -r--r--r-- 1 root root 0 Aug 15 00:55 cpuacct.usage_user -rw-r--r-- 1 root root 0 Aug 15 00:55 cpu.cfs_period_us -rw-r--r-- 1 root root 0 Aug 15 00:55 cpu.cfs_quota_us -rw-r--r-- 1 root root 0 Aug 15 00:55 cpu.rt_period_us -rw-r--r-- 1 root root 0 Aug 15 00:55 cpu.rt_runtime_us -rw-r--r-- 1 root root 0 Aug 15 00:55 cpu.shares -r--r--r-- 1 root root 0 Aug 15 00:55 cpu.stat -rw-r--r-- 1 root root 0 Aug 15 00:55 notify_on_release -rw-r--r-- 1 root root 0 Aug 15 00:55 release_agent -rw-r--r-- 1 root root 0 Aug 15 00:55 tasks [root@localhost cpu]# cd container/ [root@localhost container]# ls cgroup.clone_children cpuacct.usage cpuacct.usage_percpu_sys cpuacct.usage_user cpu.rt_period_us cpu.stat cgroup.procs cpuacct.usage_all cpuacct.usage_percpu_user cpu.cfs_period_us cpu.rt_runtime_us notify_on_release cpuacct.stat cpuacct.usage_percpu cpuacct.usage_sys cpu.cfs_quota_us cpu.shares tasks [root@localhost container]# ls cgroup.clone_children cpuacct.usage cpuacct.usage_percpu_sys cpuacct.usage_user cpu.rt_period_us cpu.stat cgroup.procs cpuacct.usage_all cpuacct.usage_percpu_user cpu.cfs_period_us cpu.rt_runtime_us notify_on_release cpuacct.stat cpuacct.usage_percpu cpuacct.usage_sys cpu.cfs_quota_us cpu.shares tasks [root@localhost container]# cat cpu.cfs_period_us 100000 [root@localhost container]# cat cpu.cfs_quota_us ## -1表示不作限制 -1 [root@localhost container]# cat tasks [root@localhost container]# top top - 00:58:19 up 2 days, 7:05, 2 users, load average: 0.01, 0.03, 0.00 Tasks: 187 total, 1 running, 133 sleeping, 0 stopped, 0 zombie %Cpu(s): 0.0 us, 0.3 sy, 0.0 ni, 99.7 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st KiB Mem : 977804 total, 149268 free, 608916 used, 219620 buff/cache KiB Swap: 1048572 total, 1031848 free, 16724 used. 178692 avail Mem
寫一個死迴圈且不作限制，cpu被打滿
[root@localhost container]# while : ; do : ; done & [1] 35789 [root@localhost container]# top top - 00:59:04 up 2 days, 7:05, 2 users, load average: 0.16, 0.06, 0.01 Tasks: 187 total, 2 running, 132 sleeping, 0 stopped, 0 zombie %Cpu(s):100.0 us, 0.0 sy, 0.0 ni, 0.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st KiB Mem : 977804 total, 151268 free, 606924 used, 219612 buff/cache KiB Swap: 1048572 total, 1031848 free, 16724 used. 180692 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
35789 root 20 0 116376 1328 0 R 99.3 0.1 0:13.47 bash
作資源隔離限制
[root@localhost container]# pwd /sys/fs/cgroup/cpu/container [root@localhost container]# cat cpu.cfs_period_us 100000 [root@localhost container]# cat cpu.cfs_quota_us -1 [root@localhost container]# echo 20000 > cpu.cfs_quota_us ##及限制cpu最高作用率為20% [root@localhost container]# cat cpu.cfs_quota_us 20000 [root@localhost container]# cat tasks [root@localhost container]# echo 35798 > tasks ## 將該程序對應的pid寫入對應控制組container的tasks檔案 [root@localhost container]# cat tasks 35798 [root@localhost container]# top top - 01:04:12 up 2 days, 7:11, 2 users, load average: 0.80, 0.46, 0.19 Tasks: 189 total, 2 running, 132 sleeping, 0 stopped, 0 zombie %Cpu(s): 12.9 us, 0.4 sy, 0.0 ni, 86.7 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st KiB Mem : 977804 total, 149876 free, 608284 used, 219644 buff/cache KiB Swap: 1048572 total, 1031848 free, 16724 used. 179316 avail Mem PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 35798 root 20 0 116376 1328 0 R 19.9 0.1 2:16.42 bash 1157 gdm 20 0 2975120 186892 26480 S 1.0 19.1 3:46.17 gnome-shell
2.docker實踐
啟動一個容器docker啟動busybox
-i表示輸入有效。
-t表示提供一個偽終端，用於我們輸入內容。
[root@localhost container]# docker run -it --cpu-period=100000 --cpu-quota=20000 --name my-busybox1 docker.io/busybox /bin/sh / # top Mem: 841296K used, 136508K free, 18500K shrd, 0K buff, 151804K cached CPU: 0.0% usr 10.0% sys 0.0% nic 90.0% idle 0.0% io 0.0% irq 0.0% sirq Load average: 0.00 0.00 0.02 2/365 6 PID PPID USER STAT VSZ %VSZ CPU %CPU COMMAND 1 0 root S 1308 0.1 0 0.0 /bin/sh 6 1 root R 1304 0.1 0 0.0 top Mem: 842244K used, 135560K free, 18500K shrd, 0K buff, 151892K cached CPU: 1.2% usr 0.6% sys 0.0% nic 97.5% idle 0.6% io 0.0% irq 0.0% sirq Load average: 0.00 0.00 0.02 3/365 6 PID PPID USER STAT VSZ %VSZ CPU %CPU COMMAND 1 0 root S 1308 0.1 0 0.0 /bin/sh 6 1 root R 1304 0.1 0 0.0 top Mem: 842276K used, 135528K free, 18500K shrd, 0K buff, 151892K cached CPU: 0.4% usr 0.4% sys 0.0% nic 99.1% idle 0.0% io 0.0% irq 0.0% sirq Load average: 0.00 0.00 0.02 2/365 6 PID PPID USER STAT VSZ %VSZ CPU %CPU COMMAND 1 0 root S 1308 0.1 0 0.0 /bin/sh 6 1 root R 1304 0.1 0 0.0 top / # ps -ef PID USER TIME COMMAND 1 root 0:00 /bin/sh 7 root 0:00 ps -ef
退出後重新啟動已暫停的容器
docker ps檢視正在執行的容器列表
[root@localhost container]# docker ps -a ##檢視已經暫停的容器 CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 9b8c4b4cce0d docker.io/busybox "/bin/sh" About an hour ago Exited (130) 6 minutes ago my-busybox1 [root@localhost container]# docker start 9b8c4b4cce0d ## 啟動 9b8c4b4cce0d [root@localhost container]#
進入容器，加入一個死迴圈，容器內檢視cpu使用率
[root@localhost cpu]# docker exec -it 9b8c4b4cce0d /bin/sh / # while : ; do : ; done & / # top Mem: 869356K used, 108448K free, 18536K shrd, 0K buff, 167576K cached CPU: 20.0% usr 0.0% sys 0.0% nic 80.0% idle 0.0% io 0.0% irq 0.0% sirq Load average: 0.03 0.02 0.00 3/374 12 PID PPID USER STAT VSZ %VSZ CPU %CPU COMMAND 11 6 root R 1308 0.1 0 20.0 /bin/sh 6 0 root S 1312 0.1 0 0.0 /bin/sh 1 0 root S 1308 0.1 0 0.0 /bin/sh 12 6 root R 1304 0.1 0 0.0 top
容器外檢視，說明容器內的top命令檢視的資源是宿主機上，驗證了容器只是掛載檔案目錄，核心還是用的宿主機作業系統核心
同時37732程序即為容器例項的程序，說明容器本質就是一個特殊程序
[root@localhost ~]# docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 9b8c4b4cce0d docker.io/busybox "/bin/sh" About an hour ago Up 8 minutes my-busybox1 [root@localhost ~]# top top - 02:56:52 up 2 days, 9:03, 2 users, load average: 0.04, 0.02, 0.00 Tasks: 197 total, 5 running, 141 sleeping, 0 stopped, 0 zombie %Cpu(s): 18.8 us, 0.3 sy, 0.0 ni, 80.9 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st KiB Mem : 977804 total, 84432 free, 683024 used, 210348 buff/cache KiB Swap: 1048572 total, 999976 free, 48596 used. 133088 avail Mem PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 37732 root 20 0 1308 60 4 R 19.9 0.0 0:36.01 sh 1164 mysql 20 0 1119840 156260 0 S 0.3 16.0 2:22.78 mysqld
在容器外的cgroup並未找到相關的控制組，但在容器內找到對應的限制pid 11
[root@localhost cpu]# docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 9b8c4b4cce0d docker.io/busybox "/bin/sh" About an hour ago Up 22 minutes my-busybox1 [root@localhost cpu]# top top - 03:17:02 up 2 days, 9:23, 2 users, load average: 0.41, 0.32, 0.21 Tasks: 198 total, 2 running, 143 sleeping, 0 stopped, 0 zombie %Cpu(s): 16.7 us, 5.6 sy, 0.0 ni, 72.2 id, 5.6 wa, 0.0 hi, 0.0 si, 0.0 st KiB Mem : 977804 total, 83188 free, 686168 used, 208448 buff/cache KiB Swap: 1048572 total, 972028 free, 76544 used. 106920 avail Mem PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 37732 root 20 0 1308 56 0 R 12.5 0.0 4:37.74 sh 38908 root 20 0 162028 4580 3864 R 6.2 0.5 0:00.02 top 1 root 20 0 191316 3152 1524 S 0.0 0.3 0:09.62 systemd 2 root 20 0 0 0 0 S 0.0 0.0 0:00.05 kthreadd 3 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 rcu_gp 4 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 rcu_par_gp 6 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 kworker/0:0H-kb 8 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 mm_percpu_wq 9 root 20 0 0 0 0 S 0.0 0.0 0:08.29 ksoftirqd/0 10 root 20 0 0 0 0 I 0.0 0.0 0:06.56 rcu_sched 11 root rt 0 0 0 0 S 0.0 0.0 0:01.99 migration/0 13 root 20 0 0 0 0 S 0.0 0.0 0:00.00 cpuhp/0 14 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kdevtmpfs 15 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 netns 16 root 20 0 0 0 0 S 0.0 0.0 0:00.03 kauditd 17 root 20 0 0 0 0 S 0.0 0.0 0:00.20 khungtaskd 18 root 20 0 0 0 0 S 0.0 0.0 0:00.00 oom_reaper 19 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 writeback 20 root 20 0 0 0 0 S 0.0 0.0 0:00.59 kcompactd0 21 root 25 5 0 0 0 S 0.0 0.0 0:00.00 ksmd 22 root 39 19 0 0 0 S 0.0 0.0 0:04.96 khugepaged 112 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 kintegrityd [root@localhost cpu]# cd /sys/fs/cgroup/ [root@localhost cgroup]# ls blkio cpuacct cpuset freezer memory net_cls,net_prio perf_event rdma cpu cpu,cpuacct devices hugetlb net_cls net_prio pids systemd [root@localhost cgroup]# cd cpu [root@localhost cpu]# ls cgroup.clone_children cpuacct.stat cpuacct.usage_percpu_sys cpu.cfs_period_us cpu.shares system.slice cgroup.procs cpuacct.usage cpuacct.usage_percpu_user cpu.cfs_quota_us cpu.stat tasks cgroup.sane_behavior cpuacct.usage_all cpuacct.usage_sys cpu.rt_period_us notify_on_release user.slice container cpuacct.usage_percpu cpuacct.usage_user cpu.rt_runtime_us release_agent [root@localhost cpu]# cd .. [root@localhost cgroup]# grep 37732 */tasks rdma/tasks:37732 [root@localhost cgroup]# cd cpu [root@localhost cpu]# grep 37732 */tasks [root@localhost cpu]# [root@localhost ~]# docker exec -it 9b8c4b4cce0d /bin/sh / # cd /sys/fs/cgroup/ /sys/fs/cgroup # ls blkio cpuacct freezer net_cls perf_event systemd cpu cpuset hugetlb net_cls,net_prio pids cpu,cpuacct devices memory net_prio rdma /sys/fs/cgroup # cd cpu /sys/fs/cgroup/cpu,cpuacct # ls cgroup.clone_children cpu.rt_runtime_us cpuacct.usage_all cpuacct.usage_user cgroup.procs cpu.shares cpuacct.usage_percpu notify_on_release cpu.cfs_period_us cpu.stat cpuacct.usage_percpu_sys tasks cpu.cfs_quota_us cpuacct.stat cpuacct.usage_percpu_user cpu.rt_period_us cpuacct.usage cpuacct.usage_sys /sys/fs/cgroup/cpu,cpuacct # cd .. /sys/fs/cgroup # ls blkio cpuacct freezer net_cls perf_event systemd cpu cpuset hugetlb net_cls,net_prio pids cpu,cpuacct devices memory net_prio rdma /sys/fs/cgroup # cd cpu /sys/fs/cgroup/cpu,cpuacct # ls cgroup.clone_children cpu.rt_runtime_us cpuacct.usage_all cpuacct.usage_user cgroup.procs cpu.shares cpuacct.usage_percpu notify_on_release cpu.cfs_period_us cpu.stat cpuacct.usage_percpu_sys tasks cpu.cfs_quota_us cpuacct.stat cpuacct.usage_percpu_user cpu.rt_period_us cpuacct.usage cpuacct.usage_sys /sys/fs/cgroup/cpu,cpuacct # ll /bin/sh: ll: not found /sys/fs/cgroup/cpu,cpuacct # ls cgroup.clone_children cpu.rt_runtime_us cpuacct.usage_all cpuacct.usage_user cgroup.procs cpu.shares cpuacct.usage_percpu notify_on_release cpu.cfs_period_us cpu.stat cpuacct.usage_percpu_sys tasks cpu.cfs_quota_us cpuacct.stat cpuacct.usage_percpu_user cpu.rt_period_us cpuacct.usage cpuacct.usage_sys /sys/fs/cgroup/cpu,cpuacct # ls -l total 0 -rw-r--r-- 1 root root 0 Aug 15 10:10 cgroup.clone_children -rw-r--r-- 1 root root 0 Aug 15 10:09 cgroup.procs -rw-r--r-- 1 root root 0 Aug 15 09:47 cpu.cfs_period_us -rw-r--r-- 1 root root 0 Aug 15 09:47 cpu.cfs_quota_us -rw-r--r-- 1 root root 0 Aug 15 10:10 cpu.rt_period_us -rw-r--r-- 1 root root 0 Aug 15 10:10 cpu.rt_runtime_us -rw-r--r-- 1 root root 0 Aug 15 09:47 cpu.shares -r--r--r-- 1 root root 0 Aug 15 10:10 cpu.stat -r--r--r-- 1 root root 0 Aug 15 10:10 cpuacct.stat -rw-r--r-- 1 root root 0 Aug 15 10:10 cpuacct.usage -r--r--r-- 1 root root 0 Aug 15 10:10 cpuacct.usage_all -r--r--r-- 1 root root 0 Aug 15 10:10 cpuacct.usage_percpu -r--r--r-- 1 root root 0 Aug 15 10:10 cpuacct.usage_percpu_sys -r--r--r-- 1 root root 0 Aug 15 10:10 cpuacct.usage_percpu_user -r--r--r-- 1 root root 0 Aug 15 10:10 cpuacct.usage_sys -r--r--r-- 1 root root 0 Aug 15 10:10 cpuacct.usage_user -rw-r--r-- 1 root root 0 Aug 15 10:10 notify_on_release -rw-r--r-- 1 root root 0 Aug 15 10:10 tasks /sys/fs/cgroup/cpu,cpuacct # cat tasks 1 11 13 24 /sys/fs/cgroup/cpu,cpuacct # top Mem: 897488K used, 80316K free, 18616K shrd, 0K buff, 181188K cached CPU: 27.2% usr 0.0% sys 0.0% nic 72.7% idle 0.0% io 0.0% irq 0.0% sirq Load average: 0.40 0.34 0.18 4/389 25 PID PPID USER STAT VSZ %VSZ CPU %CPU COMMAND 11 1 root R 1308 0.1 0 27.2 /bin/sh 13 0 root S 1316 0.1 0 0.0 /bin/sh 1 0 root S 1308 0.1 0 0.0 /bin/sh 25 13 root R 1304 0.1 0 0.0 top /sys/fs/cgroup/cpu,cpuacct # /sys/fs/cgroup/cpu,cpuacct # /sys/fs/cgroup/cpu,cpuacct # ll /bin/sh: ll: not found /sys/fs/cgroup/cpu,cpuacct # ls cgroup.clone_children cpu.rt_runtime_us cpuacct.usage_all cpuacct.usage_user cgroup.procs cpu.shares cpuacct.usage_percpu notify_on_release cpu.cfs_period_us cpu.stat cpuacct.usage_percpu_sys tasks cpu.cfs_quota_us cpuacct.stat cpuacct.usage_percpu_user cpu.rt_period_us cpuacct.usage cpuacct.usage_sys /sys/fs/cgroup/cpu,cpuacct # cat cpu.cfs_quota_us 20000 /sys/fs/cgroup/cpu,cpuacct # cat cpu.cfs_period_us 100000 /sys/fs/cgroup/cpu,cpuacct # cat tasks 1 11 13 29 /sys/fs/cgroup/cpu,cpuacct #