k8s的ceph-rbd的volumes的源码阅读
ceph的rbd的扩容分析
改源码为kubernetes官方提供的ceph的rbd代码。
线上采用的是ceph.com官方的rbd部署。2种有些不同
kubernetes的rbd的Provision
func (r *rbdVolumeProvisioner) Provision(selectedNode *v1.Node, allowedTopologies []v1.TopologySelectorTerm) (*v1.PersistentVolume, error) {
...创建pv名称为kubernetes-dynamic-pvc-后面接一个随机的UUID
// create random image name
image := fmt.Sprintf("kubernetes-dynamic-pvc-%s", uuid.NewUUID())
r.rbdMounter.Image = image创建pv
rbd, sizeMB, err := r.manager.CreateImage(r)
if err != nil {
klog.Errorf("rbd: create volume failed, err: %v", err)
return nil, err
}
klog.Infof("successfully created rbd image %q", image)具体的pv创建，调用rbd的manage中的接口createimage来创建镜像
func (util *RBDUtil) CreateImage(p *rbdVolumeProvisioner) (r *v1.RBDPersistentVolumeSource, size int, err error) {
var output []byte
capacity := p.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
// Convert to MB that rbd defaults on.
sz, err := volumehelpers.RoundUpToMiBInt(capacity)
if err != nil {
return nil, 0, err
}
volSz := fmt.Sprintf("%d", sz)
mon := util.kernelRBDMonitorsOpt(p.Mon)
if p.rbdMounter.imageFormat == rbdImageFormat2 {
klog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", p.rbdMounter.Image, volSz, p.rbdMounter.imageFormat, p.rbdMounter.imageFeatures, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret)
} else {
klog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", p.rbdMounter.Image, volSz, p.rbdMounter.imageFormat, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret)
}
args := []string{"create", p.rbdMounter.Image, "--size", volSz, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key=" + p.rbdMounter.adminSecret, "--image-format", p.rbdMounter.imageFormat}
if p.rbdMounter.imageFormat == rbdImageFormat2 {
// If no image features is provided, it results in empty string
// which disable all RBD image format 2 features as expected.
features := strings.Join(p.rbdMounter.imageFeatures, ",")
args = append(args, "--image-feature", features)
}
output, err = p.exec.Run("rbd", args...)
if err != nil {
klog.Warningf("failed to create rbd image, output %v", string(output))
return nil, 0, fmt.Errorf("failed to create rbd image: %v, command output: %s", err, string(output))
}
return &v1.RBDPersistentVolumeSource{
CephMonitors: p.rbdMounter.Mon,
RBDImage: p.rbdMounter.Image,
RBDPool: p.rbdMounter.Pool,
}, sz, nil
}
在镜像中采用rbd命令:
rbd crete $imagename --size $size --pool $pool --id $id -m $mon --key=$secret --image-format $image-format添加注解
pv := new(v1.PersistentVolume)
metav1.SetMetaDataAnnotation(&pv.ObjectMeta, volutil.VolumeDynamicallyCreatedByKey, "rbd-dynamic-provisioner")给pv添加属性
if secretName != "" {
rbd.SecretRef = new(v1.SecretReference)
rbd.SecretRef.Name = secretName
rbd.SecretRef.Namespace = secretNamespace
} else {
var filePathRegex = regexp.MustCompile(`^(?:/[^/!;` + "`" + ` ]+)+$`)
if keyring != "" && !filePathRegex.MatchString(keyring) {
return nil, fmt.Errorf("keyring field must contain a path to a file")
}
rbd.Keyring = keyring
}
var volumeMode *v1.PersistentVolumeMode
if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) {
volumeMode = r.options.PVC.Spec.VolumeMode
if volumeMode != nil && *volumeMode == v1.PersistentVolumeBlock {
// Block volumes should not have any FSType
fstype = ""
}
}
rbd.RadosUser = r.Id
rbd.FSType = fstype
pv.Spec.PersistentVolumeSource.RBD = rbd
pv.Spec.PersistentVolumeReclaimPolicy = r.options.PersistentVolumeReclaimPolicy
pv.Spec.AccessModes = r.options.PVC.Spec.AccessModes
if len(pv.Spec.AccessModes) == 0 {
pv.Spec.AccessModes = r.plugin.GetAccessModes()
}
pv.Spec.Capacity = v1.ResourceList{
v1.ResourceName(v1.ResourceStorage): resource.MustParse(fmt.Sprintf("%dMi", sizeMB)),
}
pv.Spec.MountOptions = r.options.MountOptions
pv.Spec.VolumeMode = volumeMode
return pv, nil
}rbd扩容
rbd的扩容在rbd_util.go中
func (expander *rbdVolumeExpander) ResizeImage(oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) {
return expander.manager.ExpandImage(expander, oldSize, newSize)
}然后调用函数ExpandImage,查看info信息
// ExpandImage runs rbd resize command to resize the specified image.
func (util *RBDUtil) ExpandImage(rbdExpander *rbdVolumeExpander, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) {
var output []byte
var err error
// Convert to MB that rbd defaults on.
sz := int(volumehelpers.RoundUpToMiB(newSize))
newVolSz := fmt.Sprintf("%d", sz)
newSizeQuant := resource.MustParse(fmt.Sprintf("%dMi", sz))
// Check the current size of rbd image, if equals to or greater that the new request size, do nothing.
curSize, infoErr := util.rbdInfo(rbdExpander.rbdMounter)
if infoErr != nil {
return oldSize, fmt.Errorf("rbd info failed, error: %v", infoErr)
}
if curSize >= sz {
return newSizeQuant, nil
}
调用函数info进行消息查询，进行判断rbdInfo函数
// rbdInfo runs `rbd info` command to get the current image size in MB.
func (util *RBDUtil) rbdInfo(b *rbdMounter) (int, error) {
var err error
var output []byte
// If we don't have admin id/secret (e.g. attaching), fallback to user id/secret.
id := b.adminId
secret := b.adminSecret
if id == "" {
id = b.Id
secret = b.Secret
}
mon := util.kernelRBDMonitorsOpt(b.Mon)
klog.V(4).Infof("rbd: info %s using mon %s, pool %s id %s key %s", b.Image, mon, b.Pool, id, secret)
output, err = b.exec.Run("rbd",
"info", b.Image, "--pool", b.Pool, "-m", mon, "--id", id, "--key="+secret, "--format=json")
if err, ok := err.(*exec.Error); ok {
if err.Err == exec.ErrNotFound {
klog.Errorf("rbd cmd not found")
// fail fast if rbd command is not found.
return 0, err
}
}
// If command never succeed, returns its last error.
if err != nil {
return 0, err
}
if len(output) == 0 {
return 0, fmt.Errorf("can not get image size info %s: %s", b.Image, string(output))
}
return getRbdImageSize(output)
}
通过rbd命令获取相关信息
rbd info $image --pool $pool -m $mon --id $id --key=$secrte --format=json
然后格式化输出返回镜像大小在函数ExpandImage进行扩容
// rbd resize.
mon := util.kernelRBDMonitorsOpt(rbdExpander.rbdMounter.Mon)
klog.V(4).Infof("rbd: resize %s using mon %s, pool %s id %s key %s", rbdExpander.rbdMounter.Image, mon, rbdExpander.rbdMounter.Pool, rbdExpander.rbdMounter.adminId, rbdExpander.rbdMounter.adminSecret)
output, err = rbdExpander.exec.Run("rbd",
"resize", rbdExpander.rbdMounter.Image, "--size", newVolSz, "--pool", rbdExpander.rbdMounter.Pool, "--id", rbdExpander.rbdMounter.adminId, "-m", mon, "--key="+rbdExpander.rbdMounter.adminSecret)
if err == nil {
return newSizeQuant, nil
}
klog.Errorf("failed to resize rbd image: %v, command output: %s", err, string(output))
return oldSize, err
}
在改函数中先调用rbd resize $image --size $newsize --pool $pool --id $id --m $mon --key=$secret_key,完成扩容扩展: rbd的命令的使用
查看info信息
# rbd info kubernetes-dynamic-pvc-376331bd-ddb6-11e9-86f1-d2ff46c62203 -m 10.0.7.52 --key=AQBRRgNduPgTIxAAp9KLXRxjFYLLXwiENdGvFw== --pool kube --format=json {"name":"kubernetes-dynamic-pvc-376331bd-ddb6-11e9-86f1-d2ff46c62203","id":"ac726b8b4567","size":3221225472,"objects":768,"order":22,"object_size":4194304,"block_name_prefix":"rbd_data.ac726b8b4567","format":2,"features":["layering"],"op_features":[],"flags":[],"create_timestamp":"Mon Sep 23 11:57:12 2019"}删除image
[root@k8s-ceshi-01.novalocal 16:46 ~/k8s/ceph/rbd/deploy/rbac] # rbd remove kubernetes-dynamic-pvc-376331bd-ddb6-11e9-86f1-d2ff46c62203 -m 10.0.7.52 --key=AQBRRgNduPgTIxAAp9KLXRxjFYLLXwiENdGvFw== --pool kube Removing image: 100% complete...done.列出images
# rbd list -m 10.0.7.52 --key=AQBRRgNduPgTIxAAp9KLXRxjFYLLXwiENdGvFw== --pool kube kubernetes-dynamic-pvc-50444df4-d9f9-11e9-94f6-12f1cfd47f5e扩容images
# rbd resize --image kubernetes-dynamic-pvc-376331bd-ddb6-11e9-86f1-d2ff46c62203 --size 51200 -m 10.0.7.52 --key=AQBRRgNduPgTIxAAp9KLXRxjFYLLXwiENdGvFw== --pool kube Resizing image: 100% complete...done.