Skip to content

Commit

Permalink
backup-manager: try to directly get the backup size from backupmeta b…
Browse files Browse the repository at this point in the history 
…efore calculate the backup size (#5411) (#5530)

Signed-off-by: ti-chi-bot <ti-community-prow-bot@tidb.io>
Signed-off-by: Leavrth <jianjun.liao@outlook.com>
Co-authored-by: Jianjun Liao <36503113+Leavrth@users.noreply.github.com>
Co-authored-by: Leavrth <jianjun.liao@outlook.com>
Co-authored-by: Xuecheng Zhang <csuzhangxc@gmail.com>
  • Loading branch information
@ti-chi-bot @Leavrth @csuzhangxc
4 people authored Sep 2, 2024
1 parent 2764203 commit 34d8cdc
Show file tree
Hide file tree
Showing 7 changed files with 193 additions and 166 deletions.
6 changes: 5 additions & 1 deletion cmd/backup-manager/app/util/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import (
"github.com/Masterminds/semver"
"github.com/gogo/protobuf/proto"
"github.com/pingcap/errors"
kvbackup "github.com/pingcap/kvproto/pkg/backup"
kvbackup "github.com/pingcap/kvproto/pkg/brpb"
"github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants"
"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
"github.com/pingcap/tidb-operator/pkg/backup/util"
Expand Down Expand Up @@ -352,6 +352,10 @@ func GetCommitTsFromMetadata(backupPath string) (string, error) {

// GetBRArchiveSize returns the total size of the backup archive.
func GetBRArchiveSize(meta *kvbackup.BackupMeta) uint64 {
if meta.BackupSize != 0 {
return meta.BackupSize
}
// ASSERT: the version of meta must be v1
total := uint64(meta.Size())
for _, file := range meta.Files {
total += file.Size_
Expand Down
69 changes: 38 additions & 31 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ module github.com/pingcap/tidb-operator
go 1.21

require (
cloud.google.com/go/storage v1.6.0
cloud.google.com/go/storage v1.30.1
github.com/Azure/azure-storage-blob-go v0.8.0
github.com/Azure/go-autorest/autorest/azure/auth v0.5.2
github.com/Masterminds/semver v1.4.2
Expand All @@ -23,7 +23,7 @@ require (
github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
github.com/go-sql-driver/mysql v1.5.0
github.com/gogo/protobuf v1.3.2
github.com/google/go-cmp v0.5.8
github.com/google/go-cmp v0.5.9
github.com/google/gofuzz v1.1.0
github.com/mholt/archiver v3.1.1+incompatible
github.com/minio/minio-go/v6 v6.0.55
Expand All @@ -34,7 +34,7 @@ require (
github.com/pingcap/TiProxy/lib v0.0.0-20230201020701-df06ec482c69
github.com/pingcap/advanced-statefulset/client v1.17.1-0.20230403114412-d141a788a127
github.com/pingcap/errors v0.11.4
github.com/pingcap/kvproto v0.0.0-20200927054727-1290113160f0
github.com/pingcap/kvproto v0.0.0-20231122054644-fb0f5c2a0a10
github.com/pingcap/tidb-operator/pkg/apis v1.5.3
github.com/pingcap/tidb-operator/pkg/client v1.5.3
github.com/prometheus/client_golang v1.7.1
Expand All @@ -52,10 +52,10 @@ require (
github.com/yisaer/crd-validation v0.0.3
go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489
gocloud.dev v0.18.0
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e
golang.org/x/sync v0.1.0
golang.org/x/time v0.3.0
gomodules.xyz/jsonpatch/v2 v2.1.0
google.golang.org/grpc v1.27.1
google.golang.org/grpc v1.59.0
gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.20.15
k8s.io/apiextensions-apiserver v0.20.15
Expand All @@ -75,14 +75,19 @@ require (
)

require (
cloud.google.com/go/compute v1.23.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.1.2 // indirect
github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
go.uber.org/atomic v1.9.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect
k8s.io/controller-manager v0.20.15 // indirect
k8s.io/kubelet v0.0.0 // indirect
)

require (
cloud.google.com/go v0.54.0 // indirect
cloud.google.com/go v0.110.8 // indirect
github.com/Azure/azure-pipeline-go v0.2.1 // indirect
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
Expand All @@ -109,7 +114,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/sts v1.16.13 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/blang/semver v3.5.1+incompatible // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chai2010/gettext-go v0.0.0-20170215093142-bf70f2a70fb1 // indirect
github.com/containerd/containerd v1.4.1 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
Expand All @@ -132,24 +137,22 @@ require (
github.com/go-openapi/jsonreference v0.19.3 // indirect
github.com/go-openapi/spec v0.19.3 // indirect
github.com/go-openapi/swag v0.19.5 // indirect
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/golang/protobuf v1.4.3 // indirect
github.com/golang/snappy v0.0.1 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.0.0 // indirect
github.com/google/uuid v1.1.2 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/google/wire v0.3.0 // indirect
github.com/googleapis/gax-go v2.0.2+incompatible // indirect
github.com/googleapis/gax-go/v2 v2.0.5 // indirect
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
github.com/googleapis/gnostic v0.5.1 // indirect
github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.13.0 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/imdario/mergo v0.3.10 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.11 // indirect
github.com/jstemmer/go-junit-report v0.9.1 // indirect
github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
github.com/mailru/easyjson v0.7.0 // indirect
Expand Down Expand Up @@ -180,31 +183,27 @@ require (
github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
go.opencensus.io v0.22.3 // indirect
go.opencensus.io v0.24.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.23.0 // indirect
golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b // indirect
golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
golang.org/x/net v0.7.0 // indirect
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
golang.org/x/sys v0.5.0 // indirect
golang.org/x/term v0.5.0 // indirect
golang.org/x/text v0.7.0 // indirect
golang.org/x/tools v0.1.12 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/api v0.20.0 // indirect
google.golang.org/appengine v1.6.6 // indirect
google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect
google.golang.org/protobuf v1.26.0-rc.1 // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/oauth2 v0.11.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/api v0.128.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/gcfg.v1 v1.2.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.51.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/warnings.v0 v0.1.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
honnef.co/go/tools v0.0.1-2020.1.3 // indirect
k8s.io/cloud-provider v0.20.15 // indirect
k8s.io/csi-translation-lib v0.20.15 // indirect
k8s.io/klog v1.0.0 // indirect
Expand Down Expand Up @@ -297,3 +296,11 @@ replace github.com/ulikunitz/xz => github.com/ulikunitz/xz v0.5.8
// workaround for github.com/advisories/GHSA-w73w-5m7g-f7qc
// TODO: remove it after upgrading k8s.io/client-go equal or greater than v0.20.0
replace github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt v3.2.1+incompatible

// workaround for "but does not contain package google.golang.org/grpc/naming"

replace google.golang.org/grpc => google.golang.org/grpc v1.27.1

replace cloud.google.com/go/storage => cloud.google.com/go/storage v1.6.0

replace google.golang.org/api => google.golang.org/api v0.46.0
Loading

0 comments on commit 34d8cdc

Please sign in to comment.