包转发率（k8s 学习笔记）

一、系统初始化

包转发率（k8s 学习笔记）

1、关闭firewalld

systemctl stop firewalld.service && systemctl disable firewalld #或者这样也行 systemctl disable/enable --now firewalld

2、关闭selinux

sed -i 's/enforcing/disabled/' /etc/selinux/config #永久

#临时

setenforce 0

3、关闭swap分区

备份fstab

cp /etc/fstab /etc/__fstab.bak

#临时重启失效

swapoff -a

#永久

sed -ri 's/.*swap.*/#&/' /etc/fstab

4.修改主机名配置静态ip及安装常用软件

[root@demo.localdomain ~]# hostnamectl set-hostname k8s-m1 [root@demo.localdomain ~]# bash [root@k8s-m1 ~]#

同理修改工作节点主机名

修改ip地址

[root@k8s-m1 ~]# cat /etc/sysconfig/network-scripts/ifcfg-ens33 TYPE="Ethernet" PROXY_METHOD="none" BROWSER_ONLY="no" BOOTPROTO="none" DEFROUTE="yes" IPv4_FAILURE_FATAL="no" IPV6INIT="yes" IPV6_AUTOCONF="yes" IPV6_DEFROUTE="yes" IPV6_FAILURE_FATAL="no" IPV6_ADDR_GEN_MODE="stable-privacy" NAME="ens33" DEVICE="ens33" ONBOOT="yes" IPADDR=192.168.128.120 NETMASK=255.255.255.0 GATEWAY=192.168.128.2 DNS=223.5.5.5 #DNS2=192.168.128.2

添加阿里云yum源

curl -s -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo curl -s -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo yum clean all && yum makecache

如果刷新yum缓存过程中报错 # Could not resolve host: mirrors.cloud.aliyuncs.com; Unknown error"

"http://mirrors.aliyun.com/centos/"，发现并无 7Server/目录。因此，应该是$releasever变量解析成了"7Server"，进而导致了错误的发生。

vim CentOS-Base.repo :%s/$releasever/7/g # 替换变量

安装常用软件

yum -y install gcc gcc-c++ net-tools nmap-ncat sysstat git ipset ipvsadm bash-completion wget unzip lrzsz lsof vim tree telnet unzip nc nmap tree htop iftop net-tools # 此方法也行 #yum install -y epel-release #sed -e 's!^metalink=!#metalink=!g' # -e 's!^#baseurl=!baseurl=!g' # -e 's!//download.fedoraproject.org/pub!//mirrors.tuna.tsinghua.edu.cn!g' # -e 's!http://mirrors!https://mirrors!g' # -i /etc/yum.repos.d/epel.repo /etc/yum.repos.d/epel-testing.repo

5、添加hosts解析及配置ssh免密登录

cat >>/etc/hosts<<EOF 192.168.128.20 k8s-m1 192.168.128.21 k8s-n1 192.168.128.22 k8s-n2 EOF

或者

tee /etc/hosts <<-EOF 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 192.168.128.120 k8s-m1 192.168.128.121 k8s-n1 192.168.128.122 k8s-n2 EOF

# 添加ssh 免密登录 ssh-keygen -t rsa -b 2048 -P '' -f ~/.ssh/id_rsa ssh-copy-id k8s-m1 ssh-copy-id k8s-n2 ssh-copy-id k8s-n3

6、将桥接的IPv4流量传递到iptables链

cat > /etc/sysctl.d/k8s.conf << EOF net.bridge.bridge-nf-call-ip6tables = 1 net.bridge.bridge-nf-call-iptables = 1 EOF #加载生效 sysctl --system

7、时间同步

# 设置时区 timedatectl set-timezone Asia/Shanghai

yum install -y ntpdate crontab -e # sync time * */1 * * * /usr/sbin/ntpdate cn.pool.ntp.org # 时间同步此方法也行 yum makecache fast yum -y install chrony systemctl enable --now chronyd

8、挂载数据盘

mkdir /data fdisk /dev/sdb mkfs.xfs /dev/sdb1 echo "UUID=$(blkid /dev/sdb1 |awk -F '[="]+' '{print $2}') /data xfs default 0 0" >>/etc/fstab mount -a

9、升级内核

安装 elrepo 源 # elrepo源官网http://elrepo.org/tiki/tiki-index.php

导入key

rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org

安装repo文件

rpm -Uvh https://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm

查看可用版本

yum makecache fast [root@k8s-m1 /etc/yum.repos.d]# yum --disablerepo="*" --enablerepo="elrepo-kernel" search all kernel-lt --show-duplicates Loaded plugins: fastestmirror Loading mirror speeds from cached hostFile * elrepo-kernel: mirrors.tuna.tsinghua.edu.cn elrepo-kernel | 3.0 kB 00:00:00 elrepo-kernel/primary_db | 2.1 MB 00:00:08 ===================================================================== Matched: kernel-lt ====================================================================== kernel-lt-5.4.206-1.el7.elrepo.x86_64 : The Linux kernel. (The core of any Linux-based operating system.) kernel-lt-5.4.207-1.el7.elrepo.x86_64 : The Linux kernel. (The core of any Linux-based operating system.) kernel-lt-devel-5.4.206-1.el7.elrepo.x86_64 : Development package for building kernel modules to match the kernel. kernel-lt-devel-5.4.207-1.el7.elrepo.x86_64 : Development package for building kernel modules to match the kernel. kernel-lt-doc-5.4.206-1.el7.elrepo.noarch : Various bits of documentation found in the kernel sources. kernel-lt-doc-5.4.207-1.el7.elrepo.noarch : Various bits of documentation found in the kernel sources. kernel-lt-headers-5.4.206-1.el7.elrepo.x86_64 : Header files of the kernel, for use by glibc. kernel-lt-headers-5.4.207-1.el7.elrepo.x86_64 : Header files of the kernel, for use by glibc. kernel-lt-tools-5.4.206-1.el7.elrepo.x86_64 : Assortment of tools for the kernel. kernel-lt-tools-5.4.207-1.el7.elrepo.x86_64 : Assortment of tools for the kernel. kernel-lt-tools-libs-5.4.206-1.el7.elrepo.x86_64 : Libraries for the kernel tools. kernel-lt-tools-libs-5.4.207-1.el7.elrepo.x86_64 : Libraries for the kernel tools. kernel-lt-tools-libs-devel-5.4.206-1.el7.elrepo.x86_64 : Development package for the kernel tools libraries. kernel-lt-tools-libs-devel-5.4.207-1.el7.elrepo.x86_64 : Development package for the kernel tools libraries. [root@k8s-m1 /etc/yum.repos.d]#

安装

[root@k8s-m1 /etc/yum.repos.d]# yum --enablerepo="elrepo-kernel" -y install kernel-lt-5.4.207-1.el7.elrepo.x86_64 Loaded plugins: fastestmirror Loading mirror speeds from cached hostfile 。。。

更改内核启动顺序

grep "menuentry " /boot/grub2/grub.cfg

[root@k8s-m1 /etc/yum.repos.d]# grep "menuentry " /boot/grub2/grub.cfg menuentry 'CentOS Linux (5.4.207-1.el7.elrepo.x86_64) 7 (Core)' --class centos --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-3.10.0-1127.el7.x86_64-advanced-2b95b4bc-4e81-4999-adf1-1639baf3cd9b' { menuentry 'CentOS Linux (3.10.0-1127.el7.x86_64) 7 (Core)' --class centos --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-3.10.0-1127.el7.x86_64-advanced-2b95b4bc-4e81-4999-adf1-1639baf3cd9b' { menuentry 'CentOS Linux (0-rescue-78bbcbc9e6d2407d91fa0b452f9949e7) 7 (Core)' --class centos --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-0-rescue-78bbcbc9e6d2407d91fa0b452f9949e7-advanced-2b95b4bc-4e81-4999-adf1-1639baf3cd9b' {

设置5.4位默认启动内核，然后重启系统

grub2-set-default 'CentOS Linux (5.4.173-1.el7.elrepo.x86_64) 7 (Core)'

[root@k8s-m1 /etc/yum.repos.d]# grub2-set-default 'CentOS Linux (5.4.207-1.el7.elrepo.x86_64) 7 (Core)' [root@k8s-m1 /etc/yum.repos.d]#

然后重启系统验证内核

reboot

[root@k8s-m1 ~]# uname -r

5.4.207-1.el7.elrepo.x86_64

10、开启IPvs模块，官方推荐使用ipvs，它的包转发率高于iptables(yum install -y iptables-services && systemctl disable iptables && systemctl stop iptables )

[root@k8s-m1 ~]# tee /etc/modules-load.d/k8s-ipvs.conf <<EOF > ip_vs > ip_vs_rr > ip_vs_wrr > ip_vs_sh > nf_conntrack > EOF _vs ip_vs_rr ip_vs_wrr ip_vs_sh nf_conntrack

手动加载模块

for m in ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh nf_conntrack;do modprobe $m;done [root@k8s-m1 ~]# lsmod |grep ip ip_vs_sh 16384 0 ip_vs_wrr 16384 0 ip_vs_rr 16384 0 ip_vs 155648 6 ip_vs_rr,ip_vs_sh,ip_vs_wrr nf_conntrack 147456 1 ip_vs

# 解决加载iptables不对bridge的数据进行处理时出现的问题 sysctl: cannot stat /proc/sys/net/bridge/bridge-nf-call-iptables: No such file or directory

[root@k8s-m1 ~]# modprobe br_netfilter [root@k8s-m1 ~]# lsmod |grep 'br_netfilter' br_netfilter 28672 0

11、系统参数调优

tee /etc/sysctl.d/k8s.conf <<EOF net.ipv6.conf.all.disable_ipv6 = 1 net.ipv6.conf.default.disable_ipv6 = 1 net.ipv6.conf.lo.disable_ipv6 = 1 net.ipv4.neigh.default.gc_stale_time = 120 net.ipv4.conf.all.rp_filter = 0 net.ipv4.conf.default.rp_filter = 0 net.ipv4.conf.default.arp_announce = 2 net.ipv4.conf.lo.arp_announce = 2 net.ipv4.conf.all.arp_announce = 2 net.ipv4.ip_forward = 1 net.ipv4.tcp_max_tw_buckets = 5000 net.ipv4.tcp_syncookies = 1 net.ipv4.tcp_max_syn_backlog = 1024 net.ipv4.tcp_synack_retries = 2 # 要求iptables不对bridge的数据进行处理 net.bridge.bridge-nf-call-ip6tables = 1 net.bridge.bridge-nf-call-iptables = 1 net.bridge.bridge-nf-call-arptables = 1 net.netfilter.nf_conntrack_max = 2310720 fs.inotify.max_user_watches=89100 fs.may_detach_mounts = 1 fs.file-max = 52706963 fs.nr_open = 52706963 vm.overcommit_memory=1 # 开启OOM vm.panic_on_oom=0 # 禁止使用 swap 空间，只有当系统 OOM 时才允许使用它 vm.swappiness=0 # ipvs优化 net.ipv4.tcp_keepalive_time = 600 net.ipv4.tcp_keepalive_intvl = 30 net.ipv4.tcp_keepalive_probes = 10 EOF

sysctl --system # 加载生效

调整系统进程打开的最大文件数

tee /etc/security/limits.d/k8s.conf <<EOF * soft nproc 102400 * hard nproc 102400 * soft nofile 102400 * hard nofile 102400 root soft nproc 102400 root hard nproc 102400 root soft nofile 102400 root hard nofile 102400 EOF

优化系统日志配置减少磁盘IO

sed -ri 's/^$ModLoad imjournal/#&/' /etc/rsyslog.conf sed -ri 's/^$IMJournalStateFile/#&/' /etc/rsyslog.conf sed -ri 's/^#(DefaultLimitCORE)=/1=100000/' /etc/systemd/system.conf sed -ri 's/^#(DefaultLimitNOFILE)=/1=100000/' /etc/systemd/system.conf

--系统初始化结束，漏的网友们补充---