k8s的openstack-cloud-provider测试安装
部署前准备
一套openstack的环境，支持octavia负载均衡模式
openstack的公网网络
root@liran:~/openstack# openstack network show ext-net
+---------------------------+--------------------------------------+
| Field | Value |
+---------------------------+--------------------------------------+
| admin_state_up | UP |
| availability_zone_hints | |
| availability_zones | |
| created_at | 2021-04-06T07:30:53Z |
| description | |
| dns_domain | None |
| id | 3b66487a-fe87-4dba-9c8f-7d6446ae24bf |
| ipv4_address_scope | None |
| ipv6_address_scope | None |
| is_default | False |
| is_vlan_transparent | None |
| mtu | 1500 |
| name | ext-net |
| port_security_enabled | True |
| project_id | 733cdcdf14574ef192fd30544eb2067b |
| provider:network_type | vlan |
| provider:physical_network | physnet1 |
| provider:segmentation_id | 1020 |
| qos_policy_id | None |
| revision_number | 2 |
| router:external | External |
| segments | None |
| shared | True |
| status | ACTIVE |
| subnets | 786044d3-f847-4154-b82d-71074901594e |
| tags | |
| updated_at | 2021-04-06T07:30:54Z |
+---------------------------+--------------------------------------+
root@liran:~/openstack# openstack subnet show 786044d3-f847-4154-b82d-71074901594e
+----------------------+--------------------------------------+
| Field | Value |
+----------------------+--------------------------------------+
| allocation_pools | 10.0.20.100-10.0.20.200 |
| cidr | 10.0.20.0/24 |
| created_at | 2021-04-06T07:30:54Z |
| description | |
| dns_nameservers | 114.114.114.114 |
| dns_publish_fixed_ip | None |
| enable_dhcp | True |
| gateway_ip | 10.0.20.1 |
| host_routes | |
| id | 786044d3-f847-4154-b82d-71074901594e |
| ip_version | 4 |
| ipv6_address_mode | None |
| ipv6_ra_mode | None |
| name | ext-subnet |
| network_id | 3b66487a-fe87-4dba-9c8f-7d6446ae24bf |
| prefix_length | None |
| project_id | 733cdcdf14574ef192fd30544eb2067b |
| revision_number | 0 |
| segment_id | None |
| service_types | |
| subnetpool_id | None |
| tags | |
| updated_at | 2021-04-06T07:30:54Z |openstack资源创建
由于k8s机器运行在openstack的vm里面，所以需要提前创建一些资源
- 创建project和tenant
root@liran:~/openstack# openstack project list
+----------------------------------+---------+
| ID | Name |
+----------------------------------+---------+
| 473868d80d6547b1a99c37500021d407 | service |
| 733cdcdf14574ef192fd30544eb2067b | admin |
| e2eaa2311fe948b78394b5a340c57604 | test |
+----------------------------------+---------+
这里我们采用test作为k8s的集群的项目名和租户- 创建私有网络
#openstack network create vm-net
#root@liran:~/openstack# openstack network show vm-net
+---------------------------+--------------------------------------+
| Field | Value |
+---------------------------+--------------------------------------+
| admin_state_up | UP |
| availability_zone_hints | |
| availability_zones | |
| created_at | 2021-04-07T07:55:17Z |
| description | |
| dns_domain | None |
| id | dcfcdeef-e25e-488b-b63e-354714a790ee |
| ipv4_address_scope | None |
| ipv6_address_scope | None |
| is_default | None |
| is_vlan_transparent | None |
| mtu | 1442 |
| name | vm-net |
| port_security_enabled | True |
| project_id | e2eaa2311fe948b78394b5a340c57604 |
| provider:network_type | geneve |
| provider:physical_network | None |
| provider:segmentation_id | 1003 |
| qos_policy_id | None |
| revision_number | 3 |
| router:external | Internal |
| segments | None |
| shared | False |
| status | ACTIVE |
| subnets | 341ffd9f-d2ee-4319-9724-58519253de41 |
| tags | |
| updated_at | 2021-04-07T09:11:04Z |
+---------------------------+--------------------------------------+- 创建子网
#openstack subnet create --network vm-net \
--gateway 172.98.10.254 \
--subnet-range 172.98.10.0/24 \
vm-subnet
#root@liran:~/openstack# openstack subnet show vm-subnet
+----------------------+--------------------------------------+
| Field | Value |
+----------------------+--------------------------------------+
| allocation_pools | 172.98.10.2-172.98.10.250 |
| cidr | 172.98.10.0/24 |
| created_at | 2021-04-07T07:55:18Z |
| description | |
| dns_nameservers | |
| dns_publish_fixed_ip | None |
| enable_dhcp | True |
| gateway_ip | 172.98.10.254 |
| host_routes | |
| id | 341ffd9f-d2ee-4319-9724-58519253de41 |
| ip_version | 4 |
| ipv6_address_mode | None |
| ipv6_ra_mode | None |
| name | vm-subnet |
| network_id | dcfcdeef-e25e-488b-b63e-354714a790ee |
| prefix_length | None |
| project_id | e2eaa2311fe948b78394b5a340c57604 |
| revision_number | 1 |
| segment_id | None |
| service_types | |
| subnetpool_id | None |
| tags | |
| updated_at | 2021-04-07T09:11:04Z |
+----------------------+--------------------------------------+- 创建路由
#openstack router create test-router
#root@liran:~/openstack# openstack router show test-router
+-------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Field | Value |
+-------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| admin_state_up | UP |
| availability_zone_hints | |
| availability_zones | |
| created_at | 2021-04-07T08:06:48Z |
| description | |
| external_gateway_info | {"network_id": "3b66487a-fe87-4dba-9c8f-7d6446ae24bf", "external_fixed_ips": [{"subnet_id": "786044d3-f847-4154-b82d-71074901594e", "ip_address": "10.0.20.190"}], "enable_snat": true} |
| flavor_id | None |
| id | 87fe1b5d-986b-4d00-97ec-ca54a782b538 |
| interfaces_info | [{"port_id": "5c0e1ea9-3d46-45ce-a892-dddcd840d533", "ip_address": "172.98.30.254", "subnet_id": "3347f74b-d2e2-4cf4-9b73-3c2a8963b986"}, {"port_id": "b8882e6c-95da-4274-b29f-475fcd001f03", "ip_address": "172.98.10.254", "subnet_id": "341ffd9f-d2ee-4319-9724-58519253de41"}, {"port_id": "d29fa9de-e7d8-414e-a7b0-057730e09b09", "ip_address": "172.98.20.254", "subnet_id": "d36a3b5e-8f92-4209-9603-e6b00da09550"}] |
| name | test-router |
| project_id | e2eaa2311fe948b78394b5a340c57604 |
| revision_number | 6 |
| routes | |
| status | ACTIVE |
| tags | |
| updated_at | 2021-04-07T08:09:58Z |
+-------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+- 创建和绑定路由端口
#openstack port create --network vm-net --fixed-ip ip-address=10.0.20.190 test_router
#openstack router add port <router-id> <port-id>- 创建虚拟机 通过web页面创建（略）
k8s的集群配置
规划 ip|role|hostname|version|os ---|---|---|---|--- 172.98.10.103|master|k8s4-master-1|v1.17.5|centos8 172.98.10.209|node|k8s4-node-1|v1.17.5|centos8 172.98.10.184|node|k8s4-node-2|v1.17.5|centos8
注意：这里的hostname需要和openstack中的实例名称相同。etcd部署在集群内部
k8s的机器初始化
- os初始化（略）
- selinux关闭
- chrony安装
- docker安装
- 相关k8s的包安装
配置kubeadmin的初始化配置文件
- 打印相关配置模板
kubeadm config print init-defaults > kubead-config.yaml- 修改模板
# cat kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 172.98.10.103
bindPort: 6443
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: k8s4-master-1
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master
kubeletExtraArgs:
cloud-provider: "external"
---
apiServer:
timeoutForControlPlane: 4m0s
extraArgs:
enable-admission-plugins: NodeRestriction
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager:
extraVolumes:
- name: "cloud-config"
hostPath: "/etc/kubernetes/cloud-config"
mountPath: "/etc/kubernetes/cloud-config"
readOnly: true
pathType: File
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: harbor-wh.sreblog.com/k8s
kind: ClusterConfiguration
kubernetesVersion: v1.17.5
networking:
dnsDomain: cluster.local
serviceSubnet: 172.99.30.0/24
podSubnet: "172.99.20.0/24"
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
注意：这里需要改动的地方
1，advertiseAddress需要修改地址
2，cloud-provider: "external"需要添加
3，name: "cloud-config"的配置需要添加
4，networking下面的需要添加
5，imageRepository修改为内网的仓库- 添加openstack的配置文件
添加相关openstack的配置文件，具体上面openstack上面添加的时候已经完成。
# cat cloud-config
[Global]
region=RegionOne
username=liran
password=dgfhgghg
auth-url=http://192.168.7.85:5000/v3
tenant-id=e2eaa2311fe948b78394b5a340c57604
domain-id=default
#ca-file=/etc/kubernetes/pki/ca.pem
[LoadBalancer]
use-octavia=yes
subnet-id=341ffd9f-d2ee-4319-9724-58519253de41
floating-network-id=3b66487a-fe87-4dba-9c8f-7d6446ae24bf
[BlockStorage]
bs-version=v2
[Networking]
public-network-name=ext-net
ipv6-support-disabled=false
创建相关目录
#mkdir /etc/kubernetes
#cp cloud-config /etc/kubernetes- 初始化
kubeadm init --config=kubeadm-config.yml- 配置用户的kubectl账号
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config- 检查集群状态
注意这里有一个cloudprovider的注解
# kubectl describe no k8s4-master-1
Name: master1
Roles: master
......
Taints: node-role.kubernetes.io/master:NoSchedule
node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule
node.kubernetes.io/not-ready:NoSchedule
......- 创建秘钥
注意：如果openstack有https的话，需要配置ca证书。需要提前放到相关目录
kubectl create secret -n kube-system generic cloud-config --from-literal=cloud.conf="$(cat /etc/kubernetes/cloud-config)" --dry-run -o yaml > cloud-config-secret.yaml
kubectl apply -f cloud-config-secret.yaml- 配置daemonset
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cloud-controller-manager
namespace: kube-system
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: openstack-cloud-controller-manager
namespace: kube-system
labels:
k8s-app: openstack-cloud-controller-manager
spec:
selector:
matchLabels:
k8s-app: openstack-cloud-controller-manager
updateStrategy:
type: RollingUpdate
template:
metadata:
labels:
k8s-app: openstack-cloud-controller-manager
spec:
nodeSelector:
node-role.kubernetes.io/master: ""
securityContext:
runAsUser: 1001
tolerations:
- key: node.cloudprovider.kubernetes.io/uninitialized
value: "true"
effect: NoSchedule
- key: node-role.kubernetes.io/master
effect: NoSchedule
- effect: NoSchedule
key: node.kubernetes.io/not-ready
serviceAccountName: cloud-controller-manager
containers:
- name: openstack-cloud-controller-manager
image: docker.io/k8scloudprovider/openstack-cloud-controller-manager:v1.15.0
args:
- /bin/openstack-cloud-controller-manager
- --v=1
- --cloud-config=$(CLOUD_CONFIG)
- --cloud-provider=openstack
- --use-service-account-credentials=true
- --address=127.0.0.1
volumeMounts:
- mountPath: /etc/kubernetes/pki
name: k8s-certs
readOnly: true
- mountPath: /etc/ssl/certs
name: ca-certs
readOnly: true
- mountPath: /etc/config
name: cloud-config-volume
readOnly: true
- mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
name: flexvolume-dir
- mountPath: /etc/kubernetes
name: ca-cert
readOnly: true
resources:
requests:
cpu: 200m
env:
- name: CLOUD_CONFIG
value: /etc/config/cloud.conf
hostNetwork: true
volumes:
- hostPath:
path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
type: DirectoryOrCreate
name: flexvolume-dir
- hostPath:
path: /etc/kubernetes/pki
type: DirectoryOrCreate
name: k8s-certs
- hostPath:
path: /etc/ssl/certs
type: DirectoryOrCreate
name: ca-certs
- name: cloud-config-volume
secret:
secretName: cloud-config
- name: ca-cert
secret:
secretName: openstack-ca-cert- 查看master的状态
可以看到ProviderID中有相关openstack中vm的实例id了。
# kubectl describe no k8s4-master-1
Name: master1
Roles: master
......
Taints: node-role.kubernetes.io/master:NoSchedule
node.kubernetes.io/not-ready:NoSchedule
......
sage:docker: network plugin is not ready: cni config uninitialized
......
PodCIDR: 10.224.0.0/24
ProviderID: openstack:///548e3c46-2477-4ce2-968b-3de1314560a5- 部署网络
这里我们采用calico部署。具体（略）
- 检查相关集群的配置
检查controller配置
## cat /etc/kubernetes/manifests/kube-controller-manager.yaml |grep cloud-provider
- --cloud-provider=external
检查kubelet配置
#Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --cloud-provider=external
如果没有cloud-provider配置，手动添加。
#systemctl daemon-reload
#systemctl restart kubeletnode节点的安装
- 创建配置文件,在node节点上面执行
apiVersion: kubeadm.k8s.io/v1beta2
discovery:
bootstrapToken:
apiServerEndpoint: 192.168.1.7:6443
token: 0c0z4p.dnafh6vnmouus569
caCertHashes: ["sha256:fcb3e956a6880c05fc9d09714424b827f57a6fdc8afc44497180905946527adf"]
kind: JoinConfiguration
nodeRegistration:
kubeletExtraArgs:
cloud-provider: "external"
如果不清楚的话，在master上面执行
kubeadm token list
kubeadm token create --print-join-command- 修改kubelet文件
检查kubelet配置
#Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --cloud-provider=external
如果没有cloud-provider配置，手动添加。
#systemctl daemon-reload
#systemctl restart kubelet- 检查集群,打label
在master上面执行
# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s4-master-1 Ready master 7h17m v1.17.5
k8s4-node-1 Ready None 6h57m v1.17.5
k8s4-node-2 Ready None 6h57m v1.17.5
#kubectl label node k8s4-node-1 node-role.kubernetes.io/node=
#kubectl label node k8s4-node-2 node-role.kubernetes.io/node=
#kubectl get pod -n kube-system
# kubectl get pod -n kube-system
NAME READY STATUS RESTARTS AGE
calico-kube-controllers-5cf689bf59-s9kkk 1/1 Running 0 7h4m
calico-node-g7vrp 1/1 Running 0 7h4m
calico-node-m7jp7 1/1 Running 0 6h58m
calico-node-vcv8w 1/1 Running 0 6h58m
coredns-7f5879b99f-682gv 1/1 Running 0 7h44m
coredns-7f5879b99f-sj4w6 1/1 Running 0 7h44m
etcd-k8s4-master-1 1/1 Running 0 7h17m
kube-apiserver-k8s4-master-1 1/1 Running 0 7h17m
kube-controller-manager-k8s4-master-1 1/1 Running 6 7h17m
kube-proxy-4m9w5 1/1 Running 0 7h17m
kube-proxy-g6cmm 1/1 Running 0 6h58m
kube-proxy-nfw6h 1/1 Running 0 6h58m
kube-scheduler-k8s4-master-1 1/1 Running 5 7h17m
openstack-cloud-controller-manager-lfhz5 1/1 Running 0 4h57m测试lb
内网测试
# cat k8s/test/internal-http-nginx.yaml
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: internal-http-nginx-deployment
spec:
replicas: 2
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx
ports:
- containerPort: 80
---
kind: Service
apiVersion: v1
metadata:
name: internal-http-nginx-service
annotations:
service.beta.kubernetes.io/openstack-internal-load-balancer: "true"
spec:
selector:
app: nginx
type: LoadBalancer
ports:
- name: http
port: 80
targetPort: 80这里通过注解service.beta.kubernetes.io/openstack-internal-load-balancer: "true"表示openstack中为一个内部的负载均衡器
外网测试
# cat k8s/test/external-http-nginx.yaml
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: external-http-nginx-deployment
spec:
replicas: 2
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx
ports:
- containerPort: 80
---
kind: Service
apiVersion: v1
metadata:
name: external-http-nginx-service
annotations:
service.beta.kubernetes.io/openstack-internal-load-balancer: "false"
spec:
selector:
app: nginx
type: LoadBalancer
ports:
- name: http
port: 80
targetPort: 80注意
- service.beta.kubernetes.io/openstack-internal-load-balancer: "false"
- service.beta.kubernetes.io/floating-network-id: "xxxxx" 如果定义了floating-network-id，则采用floating-network-id分配浮动ip，否则采用默认的分配
浮动ip
现在我们可以通过浮动ip访问pod中的网页了。
# curl 10.0.20.172
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
body {
width: 35em;
margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif;
}
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>
<p><em>Thank you for using nginx.</em></p>
</body>
</html>