diff --git a/build.env b/build.env index 2ccc79324..05959a384 100644 --- a/build.env +++ b/build.env @@ -15,7 +15,7 @@ CSI_IMAGE_VERSION=canary CSI_UPGRADE_VERSION=v3.11.0 # Ceph version to use -BASE_IMAGE=quay.io/ceph/ceph:v19 +BASE_IMAGE=quay.ceph.io/ceph-ci/ceph:wip-pkalever-rbd-group-snap-mirror CEPH_VERSION=squid # standard Golang options diff --git a/go.mod b/go.mod index a3bc66939..1b2dc6549 100644 --- a/go.mod +++ b/go.mod @@ -190,6 +190,8 @@ replace ( code.cloudfoundry.org/gofileutils => github.com/cloudfoundry/gofileutils v0.0.0-20170111115228-4d0c80011a0f github.com/ceph/ceph-csi/api => ./api + github.com/ceph/go-ceph => github.com/sp98/go-ceph v0.0.0-20240718045739-2a7b9501c66c + // Required for kubernetes 1.26 github.com/portworx/sched-ops => github.com/portworx/sched-ops v0.20.4-openstorage-rc3 diff --git a/go.sum b/go.sum index 959f598ee..f068c0649 100644 --- a/go.sum +++ b/go.sum @@ -864,8 +864,6 @@ github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyY github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= -github.com/ceph/go-ceph v0.28.0 h1:ZjlDV9XiVmBQIe9bKbT5j2Ft/bse3Jm+Ui65yE/oFFU= -github.com/ceph/go-ceph v0.28.0/go.mod h1:EwEITEDpuFCMnFrPLbV+/Vyi59jUihgCxBKvlTWGot0= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -911,8 +909,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/csi-addons/spec v0.2.1-0.20240718113938-dc98b454ba65 h1:i9JGGQTEmRQXSpQQPR96+DV4D4o+V1+gjAWf+bpxQxk= -github.com/csi-addons/spec v0.2.1-0.20240718113938-dc98b454ba65/go.mod h1:Mwq4iLiUV4s+K1bszcWU6aMsR5KPsbIYzzszJ6+56vI= github.com/csi-addons/spec v0.2.1-0.20240730084235-3958a5b17d24 h1:tJETaYbnnzlCSaqDXQzbszYyuAtG/sFzm6DargeVzJA= github.com/csi-addons/spec v0.2.1-0.20240730084235-3958a5b17d24/go.mod h1:Mwq4iLiUV4s+K1bszcWU6aMsR5KPsbIYzzszJ6+56vI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -1549,6 +1545,8 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/sp98/go-ceph v0.0.0-20240718045739-2a7b9501c66c h1:qlO/70uYWLZSyBhLnRSsFOp8jQPT4tRxqomFEXoVJYE= +github.com/sp98/go-ceph v0.0.0-20240718045739-2a7b9501c66c/go.mod h1:EwEITEDpuFCMnFrPLbV+/Vyi59jUihgCxBKvlTWGot0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= diff --git a/scripts/Dockerfile.test b/scripts/Dockerfile.test index a05d7737e..a37530580 100644 --- a/scripts/Dockerfile.test +++ b/scripts/Dockerfile.test @@ -8,7 +8,7 @@ # little different. # -FROM registry.fedoraproject.org/fedora:latest +FROM quay.ceph.io/ceph-ci/ceph:wip-pkalever-rbd-group-snap-mirror ARG GOPATH=/go ARG GOROOT=/usr/local/go diff --git a/vendor/github.com/ceph/go-ceph/rbd/group.go b/vendor/github.com/ceph/go-ceph/rbd/group.go index 654d15e3e..629283ed1 100644 --- a/vendor/github.com/ceph/go-ceph/rbd/group.go +++ b/vendor/github.com/ceph/go-ceph/rbd/group.go @@ -110,7 +110,9 @@ func GroupImageAdd(groupIoctx *rados.IOContext, groupName string, cephIoctx(groupIoctx), cGroupName, cephIoctx(imageIoctx), - cImageName) + cImageName, + C.uint32_t(0), + ) return getError(ret) } @@ -135,7 +137,8 @@ func GroupImageRemove(groupIoctx *rados.IOContext, groupName string, cephIoctx(groupIoctx), cGroupName, cephIoctx(imageIoctx), - cImageName) + cImageName, + C.uint32_t(0)) return getError(ret) } @@ -160,7 +163,8 @@ func GroupImageRemoveByID(groupIoctx *rados.IOContext, groupName string, cephIoctx(groupIoctx), cGroupName, cephIoctx(imageIoctx), - cid) + cid, + C.uint32_t(0)) return getError(ret) } diff --git a/vendor/github.com/ceph/go-ceph/rbd/mirror_group.go b/vendor/github.com/ceph/go-ceph/rbd/mirror_group.go new file mode 100644 index 000000000..c29559793 --- /dev/null +++ b/vendor/github.com/ceph/go-ceph/rbd/mirror_group.go @@ -0,0 +1,332 @@ +//go:build ceph_preview + +package rbd + +// #cgo LDFLAGS: -lrbd +// #include +// #include +import "C" +import ( + "fmt" + "unsafe" + + "github.com/ceph/go-ceph/internal/cutil" + "github.com/ceph/go-ceph/rados" +) + +// MirrorGroupEnable will enable mirroring for a group using the specified mode. +// +// Implements: +// +// int rbd_mirror_group_enable(rados_ioctx_t p, const char *name, +// rbd_mirror_image_mode_t mirror_image_mode, +// uint32_t flags); +func MirrorGroupEnable(groupIoctx *rados.IOContext, groupName string, mode ImageMirrorMode) error { + cGroupName := C.CString(groupName) + defer C.free(unsafe.Pointer(cGroupName)) + ret := C.rbd_mirror_group_enable( + cephIoctx(groupIoctx), + cGroupName, + C.rbd_mirror_image_mode_t(mode), + (C.uint32_t)(2), + ) + return getError(ret) +} + +// MirrorGroupDisable will disabling mirroring for a group +// +// Implements: +// +// int rbd_mirror_group_disable(rados_ioctx_t p, const char *name, +// bool force) +func MirrorGroupDisable(groupIoctx *rados.IOContext, groupName string, force bool) error { + cGroupName := C.CString(groupName) + defer C.free(unsafe.Pointer(cGroupName)) + ret := C.rbd_mirror_group_disable( + cephIoctx(groupIoctx), + cGroupName, + C.bool(force)) + return getError(ret) +} + +// MirrorGroupPromote will promote the mirrored group to primary status +// +// Implements: +// +// int rbd_mirror_group_promote(rados_ioctx_t p, const char *name, +// uint32_t flags, bool force) +func MirrorGroupPromote(groupIoctx *rados.IOContext, groupName string, force bool) error { + cGroupName := C.CString(groupName) + defer C.free(unsafe.Pointer(cGroupName)) + ret := C.rbd_mirror_group_promote( + cephIoctx(groupIoctx), + cGroupName, + (C.uint32_t)(0), + C.bool(force)) + return getError(ret) +} + +// MirrorGroupDemote will demote the mirrored group to primary status +// +// Implements: +// +// int rbd_mirror_group_demote(rados_ioctx_t p, const char *name, +// uint32_t flags) +func MirrorGroupDemote(groupIoctx *rados.IOContext, groupName string) error { + cGroupName := C.CString(groupName) + defer C.free(unsafe.Pointer(cGroupName)) + ret := C.rbd_mirror_group_demote( + cephIoctx(groupIoctx), + cGroupName, + (C.uint32_t)(0)) + return getError(ret) +} + +// MirrorGroupResync is used to manually resolve split-brain status by triggering +// resynchronization +// +// Implements: +// +// int rbd_mirror_group_resync(rados_ioctx_t p, const char *name) +func MirrorGroupResync(groupIoctx *rados.IOContext, groupName string) error { + cGroupName := C.CString(groupName) + defer C.free(unsafe.Pointer(cGroupName)) + ret := C.rbd_mirror_group_resync( + cephIoctx(groupIoctx), + cGroupName) + return getError(ret) +} + +// MirrorGroupState represents the current state of the mirrored group +type MirrorGroupState C.rbd_mirror_group_state_t + +// String representation of MirrorGroupState. +func (mgs MirrorGroupState) String() string { + switch mgs { + case MirrorGroupEnabled: + return "enabled" + case MirrorGroupDisabled: + return "disabled" + case MirrorGroupEnabling: + return "enabling" + case MirrorGrpupDisabling: + return "disabled" + default: + return "" + } +} + +const ( + // MirrorGrpupDisabling is the representation of + // RBD_MIRROR_GROUP_DISABLING from librbd. + MirrorGrpupDisabling = MirrorGroupState(C.RBD_MIRROR_GROUP_DISABLING) + // MirrorGroupEnabling is the representation of + // RBD_MIRROR_GROUP_ENABLING from librbd + MirrorGroupEnabling = MirrorGroupState(C.RBD_MIRROR_GROUP_ENABLING) + // MirrorGroupEnabled is the representation of + // RBD_MIRROR_IMAGE_ENABLED from librbd. + MirrorGroupEnabled = MirrorGroupState(C.RBD_MIRROR_GROUP_ENABLED) + // MirrorGroupDisabled is the representation of + // RBD_MIRROR_GROUP_DISABLED from librbd. + MirrorGroupDisabled = MirrorGroupState(C.RBD_MIRROR_GROUP_DISABLED) +) + +// MirrorGroupInfo represents the mirroring status information of group. +type MirrorGroupInfo struct { + GlobalID string + State MirrorGroupState + MirrorImageMode ImageMirrorMode + Primary bool +} + +// GetMirrorGroupInfo returns the mirroring status information of the mirrored group +// +// Implements: +// +// int rbd_mirror_group_get_info(rados_ioctx_t p, const char *name, +// rbd_mirror_group_info_t *mirror_group_info, +// size_t info_size) +func GetMirrorGroupInfo(groupIoctx *rados.IOContext, groupName string) (*MirrorGroupInfo, error) { + var cgInfo C.rbd_mirror_group_info_t + cGroupName := C.CString(groupName) + defer C.free(unsafe.Pointer(cGroupName)) + + ret := C.rbd_mirror_group_get_info( + cephIoctx(groupIoctx), + cGroupName, + &cgInfo, + C.sizeof_rbd_mirror_group_info_t) + + if ret < 0 { + return nil, getError(ret) + } + + info := convertMirrorGroupInfo(&cgInfo) + + // free C memory allocated by C.rbd_mirror_group_get_info call + C.rbd_mirror_group_get_info_cleanup(&cgInfo) + return &info, nil + +} + +func convertMirrorGroupInfo(cgInfo *C.rbd_mirror_group_info_t) MirrorGroupInfo { + return MirrorGroupInfo{ + GlobalID: C.GoString(cgInfo.global_id), + MirrorImageMode: ImageMirrorMode(cgInfo.mirror_image_mode), + State: MirrorGroupState(cgInfo.state), + Primary: bool(cgInfo.primary), + } +} + +// MirrorGroupStatusState is used to indicate the state of a mirrored group +// within the site status info. +type MirrorGroupStatusState int64 + +const ( + // MirrorGroupStatusStateUnknown is equivalent to MIRROR_GROUP_STATUS_STATE_UNKNOWN + MirrorGroupStatusStateUnknown = MirrorGroupStatusState(C.MIRROR_GROUP_STATUS_STATE_UNKNOWN) + // MirrorGroupStatusStateError is equivalent to MIRROR_GROUP_STATUS_STATE_ERROR + MirrorGroupStatusStateError = MirrorGroupStatusState(C.MIRROR_GROUP_STATUS_STATE_ERROR) + // MirrorGroupStatusStateStartingReplay is equivalent to MIRROR_GROUP_STATUS_STATE_STARTING_REPLAY + MirrorGroupStatusStateStartingReplay = MirrorGroupStatusState(C.MIRROR_GROUP_STATUS_STATE_STARTING_REPLAY) + // MirrorGroupStatusStateReplaying is equivalent to MIRROR_GROUP_STATUS_STATE_REPLAYING + MirrorGroupStatusStateReplaying = MirrorGroupStatusState(C.MIRROR_GROUP_STATUS_STATE_REPLAYING) + // MirrorGroupStatusStateStoppingReplay is equivalent to MIRROR_GROUP_STATUS_STATE_STOPPING_REPLAY + MirrorGroupStatusStateStoppingReplay = MirrorGroupStatusState(C.MIRROR_GROUP_STATUS_STATE_STOPPING_REPLAY) + // MirrorGroupStatusStateStopped is equivalent to MIRROR_IMAGE_GROUP_STATUS_STATE_STOPPED + MirrorGroupStatusStateStopped = MirrorGroupStatusState(C.MIRROR_GROUP_STATUS_STATE_STOPPED) +) + +// String represents the MirrorImageStatusState as a short string. +func (state MirrorGroupStatusState) String() (s string) { + switch state { + case MirrorGroupStatusStateUnknown: + s = "unknown" + case MirrorGroupStatusStateError: + s = "error" + case MirrorGroupStatusStateStartingReplay: + s = "starting_replay" + case MirrorGroupStatusStateReplaying: + s = "replaying" + case MirrorGroupStatusStateStoppingReplay: + s = "stopping_replay" + case MirrorGroupStatusStateStopped: + s = "stopped" + default: + s = fmt.Sprintf("unknown(%d)", state) + } + return s +} + +// SiteMirrorGroupStatus contains information pertaining to the status of +// a mirrored group within a site. +type SiteMirrorGroupStatus struct { + MirrorUUID string + State MirrorGroupStatusState + MirrorImageCount int + MirrorImagePoolIDs int64 + MirrorImageGlobalIDs string + MirrorImages []SiteMirrorImageStatus + Description string + LastUpdate int64 + Up bool +} + +// GlobalMirrorGroupStatus contains information pertaining to the global +// status of a mirrored group. It contains general information as well +// as per-site information stored in the SiteStatuses slice. +type GlobalMirrorGroupStatus struct { + Name string + Info MirrorGroupInfo + SiteStatusesCount int + SiteStatuses []SiteMirrorGroupStatus +} + +// LocalStatus returns one SiteMirrorGroupStatus item from the SiteStatuses +// slice that corresponds to the local site's status. If the local status +// is not found than the error ErrNotExist will be returned. +func (gmis GlobalMirrorGroupStatus) LocalStatus() (SiteMirrorGroupStatus, error) { + var ( + ss SiteMirrorGroupStatus + err error = ErrNotExist + ) + for i := range gmis.SiteStatuses { + // I couldn't find it explicitly documented, but a site mirror uuid + // of an empty string indicates that this is the local site. + // This pattern occurs in both the pybind code and ceph c++. + if gmis.SiteStatuses[i].MirrorUUID == "" { + ss = gmis.SiteStatuses[i] + err = nil + break + } + } + return ss, err +} + +type groupSiteArray [cutil.MaxIdx]C.rbd_mirror_group_site_status_t + +// GetGlobalMirrorGroupStatus returns status information pertaining to the state +// of a groups's mirroring. +// +// Implements: +// +// int rbd_mirror_group_get_global_status( +// IoCtx& io_ctx, +// const char *group_name +// mirror_group_global_status_t *mirror_group_status, +// size_t status_size); +func GetGlobalMirrorGroupStatus(ioctx *rados.IOContext, groupName string) (GlobalMirrorGroupStatus, error) { + s := C.rbd_mirror_group_global_status_t{} + cGroupName := C.CString(groupName) + defer C.free(unsafe.Pointer(cGroupName)) + // ret := C.rbd_mirror_group_get_global_status( + // cephIoctx(ioctx), + // (*C.char)(cGroupName), + // &s, + // C.sizeof_rbd_mirror_group_global_status_t) + // if err := getError(ret); err != nil { + // return GlobalMirrorGroupStatus{}, err + // } + + status := newGlobalMirrorGroupStatus(&s) + return status, nil +} + +func newGlobalMirrorGroupStatus( + s *C.rbd_mirror_group_global_status_t) GlobalMirrorGroupStatus { + + status := GlobalMirrorGroupStatus{ + Name: C.GoString(s.name), + Info: convertMirrorGroupInfo(&s.info), + SiteStatusesCount: int(s.site_statuses_count), + SiteStatuses: make([]SiteMirrorGroupStatus, s.site_statuses_count), + } + gsscs := (*groupSiteArray)(unsafe.Pointer(s.site_statuses))[:s.site_statuses_count:s.site_statuses_count] + for i := C.uint32_t(0); i < s.site_statuses_count; i++ { + gss := gsscs[i] + status.SiteStatuses[i] = SiteMirrorGroupStatus{ + MirrorUUID: C.GoString(gss.mirror_uuid), + MirrorImageGlobalIDs: C.GoString(*gss.mirror_image_global_ids), + MirrorImagePoolIDs: int64(*gss.mirror_image_pool_ids), + State: MirrorGroupStatusState(gss.state), + Description: C.GoString(gss.description), + MirrorImageCount: int(gss.mirror_image_count), + LastUpdate: int64(gss.last_update), + MirrorImages: make([]SiteMirrorImageStatus, gss.mirror_image_count), + Up: bool(gss.up), + } + + sscs := (*siteArray)(unsafe.Pointer(gss.mirror_images))[:gss.mirror_image_count:gss.mirror_image_count] + for i := C.uint32_t(0); i < gss.mirror_image_count; i++ { + ss := sscs[i] + status.SiteStatuses[i].MirrorImages[i] = SiteMirrorImageStatus{ + MirrorUUID: C.GoString(ss.mirror_uuid), + State: MirrorImageStatusState(ss.state), + Description: C.GoString(ss.description), + LastUpdate: int64(ss.last_update), + Up: bool(ss.up), + } + } + } + return status +} diff --git a/vendor/modules.txt b/vendor/modules.txt index fa4e5cb92..32acbf0fb 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -206,7 +206,7 @@ github.com/ceph/ceph-csi/api/deploy/kubernetes/cephfs github.com/ceph/ceph-csi/api/deploy/kubernetes/nfs github.com/ceph/ceph-csi/api/deploy/kubernetes/rbd github.com/ceph/ceph-csi/api/deploy/ocp -# github.com/ceph/go-ceph v0.28.0 +# github.com/ceph/go-ceph v0.28.0 => github.com/sp98/go-ceph v0.0.0-20240718045739-2a7b9501c66c ## explicit; go 1.19 github.com/ceph/go-ceph/cephfs github.com/ceph/go-ceph/cephfs/admin @@ -1795,6 +1795,7 @@ sigs.k8s.io/yaml sigs.k8s.io/yaml/goyaml.v2 # code.cloudfoundry.org/gofileutils => github.com/cloudfoundry/gofileutils v0.0.0-20170111115228-4d0c80011a0f # github.com/ceph/ceph-csi/api => ./api +# github.com/ceph/go-ceph => github.com/sp98/go-ceph v0.0.0-20240718045739-2a7b9501c66c # github.com/portworx/sched-ops => github.com/portworx/sched-ops v0.20.4-openstorage-rc3 # github.com/prometheus/client_golang => github.com/prometheus/client_golang v1.18.0 # github.com/prometheus/common => github.com/prometheus/common v0.47.0