程序笔记屋

程序笔记屋
程序笔记屋

跨厂商云服务器 k8s 搭建

2025/11/20
21
0

基于Ubuntu 24 本次尝试了5台低配活动云服分别为2c4g,2c4g,2c2g,2c2g,2c2g

k8s 1.28.15 flannel 使用的cri-docker

与网上大多数方案不同，这个是基于vpn组网，没什么厂商的限制

虚拟网卡试过了，腾讯云轻量服务器不支持，加虚拟网卡就寄，iptables转发没试

不知道是k8s源的问题还是什么，本地的虚拟机集群非常丝滑，跨公网这个问题百出

一会coredns NotReady，一会HealthyCheck error，flannel 插件缺失，pod内部DNS解析失败...

所以仅供参考

到最后集群是可以正常运行，但是总有几个厂商的云服一运行就卡死，无奈放弃

时间同步

timedatectl set-timezone Asia/Shanghai
sudo apt install -y ntpsec-ntpdate
ntpdate ntp.aliyun.com
crontab -e
0 0 * * * ntpdate ntp.aliyun.com

swap

# 确认 swap 是否启用
sudo swapon --show
​
# 暂时关闭 swap
sudo swapoff -a
​
# 永久关闭 swap
sed -i '/swap/d' /etc/fstab

内核 网桥

cat << EOF | tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF

加载配置

sudo modprobe bridge
sudo modprobe br_netfilter

配置源

k8s

apt-get update && apt-get install -y apt-transport-https
curl -fsSL https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.28/deb/Release.key |
    gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.28/deb/ /" |
    tee /etc/apt/sources.list.d/kubernetes.list
apt-get update

docker

sudo curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | sudo apt-key add -
sudo add-apt-repository "deb [arch=amd64] http://mirrors.aliyun.com/docker-ce/linux/ubuntu $(lsb_release -cs) stable"
​
sudo curl -fsSL https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | sudo apt-key add -
sudo add-apt-repository "deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main"
​
apt update

docker 安装

apt install docker-ce docker-ce-cli containerd.io

cri-docker.socket 错误

systemctl stop docker
systemctl stop cri-dockerd
​
sudo apt remove --purge docker-ce docker-ce-cli containerd.io
sudo rm -rf /var/lib/docker
sudo rm -rf /var/lib/containerd
sudo rm -rf /etc/docker
​
sudo apt update
sudo apt install docker-ce docker-ce-cli containerd.io

修改cgroup

sudo tee /etc/docker/daemon.json > /dev/null <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2"
}
EOF

cri 下载

wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.18/cri-dockerd-0.3.18.amd64.tgz
tar xzvf cri-dockerd-0.3.18.amd64.tgz
sudo cp cri-dockerd/cri-dockerd /usr/local/bin/
sudo chmod +x /usr/local/bin/cri-dockerd

cri.service 

cat > /etc/systemd/system/cri-dockerd.service<<-EOF
[Unit]
Description=CRI Interface for Docker Application Container Engine
Documentation=https://docs.mirantis.com
After=network-online.target firewalld.service docker.service
Wants=network-online.target
Requires=cri-docker.socket
 
[Service]
Type=notify
ExecStart=/usr/local/bin/cri-dockerd --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.10
 --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --cri-dockerd-root-directory=/var/lib/dockershim --docker-endpoint=unix:///var/run/docker.sock --cri-dockerd-root-directory=/var/lib/docker
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always
StartLimitBurst=3
StartLimitInterval=60s
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TasksMax=infinity
Delegate=yes
KillMode=process
[Install]
WantedBy=multi-user.target
EOF

cri.socket

cat > /etc/systemd/system/cri-docker.socket <<-EOF
[Unit]
Description=CRI Docker Socket for the API
PartOf=cri-docker.service
 
[Socket]
ListenStream=/var/run/cri-dockerd.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker
 
[Install]
WantedBy=sockets.target
EOF
systemctl daemon-reload
systemctl enable cri-dockerd.service
systemctl restart cri-dockerd.service

kubectl bukeadm

sudo apt update
sudo apt install -y kubelet=1.28.* kubeadm=1.28.* kubectl=1.28.*

sudo apt-mark hold kubelet kubeadm kubectl

创建集群

sudo kubeadm init \
  --apiserver-advertise-address=10.8.0.1 \
  --control-plane-endpoint=10.8.0.1 \
  --image-repository registry.cn-hangzhou.aliyuncs.com/google_containers \
  --kubernetes-version v1.28.15 \
  --service-cidr=10.96.0.0/12 \
  --pod-network-cidr=10.244.0.0/16 \
  --cri-socket unix:///var/run/cri-dockerd.sock \
  --upload-certs \
  --v=5

重置集群

sudo kubeadm reset -f --cri-socket unix:///var/run/cri-dockerd.sock
sudo rm -rf /etc/kubernetes/ /var/lib/etcd /$HOME/.kube
sudo iptables -F && sudo iptables -t nat -F && sudo iptables -t mangle -F && sudo iptables -X

特殊配置

cni插件安装

apt-get install -y containernetworking-plugins
# 部分依赖不一定能装全

VPN网络组建

安装

sudo apt update
sudo apt install wireguard wireguard-tools -y

# 加载内核模块
sudo modprobe wireguard

生成密钥

# 生成私钥
wg genkey | sudo tee /etc/wireguard/private.key
sudo chmod 600 /etc/wireguard/private.key

# 生成公钥
sudo cat /etc/wireguard/private.key | wg pubkey | sudo tee /etc/wireguard/public.key

vpn配置一键脚本

#!/bin/bash
## 配置文件记得加上每个node的nodeip，不然会出现master节点可以进入pod，但是ping不通pod

echo "=== WireGuard 一键配置 ==="

# ========== 配置区域 ==========
declare -A SERVERS=(
    ["cloud001"]="xxx"
    ["cloud002"]="xxx"
    ["cloud003"]="xxx"
    ["cloud004"]="xxx" 
    ["cloud005"]="xxx"
)

# SSH 密码
SSH_PASSWORD="xxx"

# WireGuard 内网IP分配
declare -A WG_IPS=(
    ["cloud001"]="10.8.0.1"
    ["cloud002"]="10.8.0.2"
    ["cloud003"]="10.8.0.3"
    ["cloud004"]="10.8.0.4" 
    ["cloud005"]="10.8.0.5"
)
# ========== 配置结束 ==========

# 安装 sshpass
echo "安装 sshpass..."
sudo apt update && sudo apt install -y sshpass

# 生成密钥对和配置文件
echo "生成 WireGuard 配置..."
mkdir -p temp-configs
cd temp-configs

# 生成所有密钥
for server in "${!SERVERS[@]}"; do
    echo "生成 $server 密钥..."
    wg genkey > ${server}.private
    cat ${server}.private | wg pubkey > ${server}.public
    chmod 600 ${server}.private
done

# 为每个服务器生成配置并上传
for server in "${!SERVERS[@]}"; do
    SERVER_IP=${SERVERS[$server]}
    SERVER_WG_IP=${WG_IPS[$server]}
    
    echo "配置 $server ($SERVER_IP)..."
    
    # 生成配置文件内容
    CONFIG_CONTENT="[Interface]
PrivateKey = $(cat ${server}.private)
Address = $SERVER_WG_IP/24
ListenPort = 51820
SaveConfig = true

"

    # 添加其他节点作为 Peer
    for peer in "${!SERVERS[@]}"; do
        if [ "$peer" != "$server" ]; then
            PEER_WG_IP=${WG_IPS[$peer]}
            PEER_PUBLIC_IP=${SERVERS[$peer]}
            PEER_PUBKEY=$(cat ${peer}.public)
            
            CONFIG_CONTENT+="# $peer
[Peer]
PublicKey = $PEER_PUBKEY
AllowedIPs = $PEER_WG_IP/32
Endpoint = $PEER_PUBLIC_IP:51820
"
            # 为非控制平面节点添加持久连接
            if [ "$server" != "cloud001" ]; then
                CONFIG_CONTENT+="PersistentKeepalive = 25
"
            fi
            CONFIG_CONTENT+="
"
        fi
    done

    # 在目标服务器上安装 WireGuard 并配置
    echo "在 $server 上安装和配置 WireGuard..."
    
    # 1. 安装 WireGuard
    sshpass -p "$SSH_PASSWORD" ssh -o StrictHostKeyChecking=no root@$SERVER_IP "
        apt update && apt install -y wireguard wireguard-tools
    "
    
    # 2. 上传配置文件
    echo "$CONFIG_CONTENT" | sshpass -p "$SSH_PASSWORD" ssh -o StrictHostKeyChecking=no root@$SERVER_IP "cat > /etc/wireguard/wg0.conf"
    
    # 3. 设置权限
    sshpass -p "$SSH_PASSWORD" ssh -o StrictHostKeyChecking=no root@$SERVER_IP "chmod 600 /etc/wireguard/wg0.conf"
    
    # 4. 启用并启动服务
    sshpass -p "$SSH_PASSWORD" ssh -o StrictHostKeyChecking=no root@$SERVER_IP "
        systemctl enable wg-quick@wg0
        systemctl start wg-quick@wg0
    "
    
    echo "✓ $server 配置完成"
done

# 验证连接
echo ""
echo "=== 验证 WireGuard 连接 ==="
for server in "${!SERVERS[@]}"; do
    SERVER_IP=${SERVERS[$server]}
    echo "检查 $server WireGuard 状态:"
    sshpass -p "$SSH_PASSWORD" ssh -o StrictHostKeyChecking=no root@$SERVER_IP "wg show"
    echo "---"
done

# 测试节点间连通性
echo "=== 测试节点间连通性 ==="
CONTROL_PLANE_IP=${SERVERS[cloud001]}
for server in "${!SERVERS[@]}"; do
    if [ "$server" != "cloud001" ]; then
        SERVER_IP=${SERVERS[$server]}
        echo "测试 $server 到 cloud001 的连通性:"
        sshpass -p "$SSH_PASSWORD" ssh -o StrictHostKeyChecking=no root@$SERVER_IP "ping -c 3 10.8.0.1"
        echo "---"
    fi
done

# 清理临时文件
cd ..
rm -rf temp-configs

echo "=== WireGuard 配置完成 ==="
echo "所有节点现在可以通过 10.8.0.x 网段互相访问"
echo "节点IP分配:"
for server in "${!SERVERS[@]}"; do
    echo "  $server: ${WG_IPS[$server]}"
done

k8s配置

# 编辑 kubelet 配置
sudo vi /var/lib/kubelet/kubeadm-flags.env

# cloud001: --node-ip=10.8.0.1
# cloud002: --node-ip=10.8.0.2  
# cloud003: --node-ip=10.8.0.3