rate-limit: add stats; prometheus metrics

alex-aizman · alex-aizman · commit fcba62b77102 · 2025-03-18T11:35:23.000-04:00
* consolidate all stats in ais/rlbackend
  - remove from all backend/* impl-s
* do not count proactive delay    - only (reactive) retries
* do not count individual retries - only totals (see "increment" comment below)
* do not count delete retries     - only get and put

Signed-off-by: Alex Aizman &lt;alex.aizman@gmail.com&gt;
diff --git a/ais/backend/aws.go b/ais/backend/aws.go
@@ -923,7 +923,6 @@ func awsErrorToAISError(awsError error, bck *cmn.Bck, objName string) (int, erro
 				debug.Assert(false, "empty error code in ", awsError.Error()) // (unlikely)
 				code = strconv.Itoa(status)
 			}
-			core.T.StatsUpdater().Inc(stats.ErrRateRetryCount)
 			e := fmt.Errorf("%s[%s: %s]", aiss3.ErrPrefix, code, bck.Cname(objName))
 			return status, cmn.NewErrTooManyRequests(e, status)
 		default:
diff --git a/ais/backend/azure.go b/ais/backend/azure.go
@@ -174,7 +174,6 @@ func azureErrorToAISError(azureError error, bck *cmn.Bck, objName string) (int,
 
 	status, err := _azureErr(azureError, stgErr)
 	if status == http.StatusTooManyRequests || status == http.StatusServiceUnavailable {
-		core.T.StatsUpdater().Inc(stats.ErrRateRetryCount)
 		return status, cmn.NewErrTooManyRequests(err, status)
 	}
 	return status, err
diff --git a/ais/backend/gcp.go b/ais/backend/gcp.go
@@ -491,7 +491,6 @@ func gcpErrorToAISError(gcpError error, bck *cmn.Bck) (int, error) {
 		}
 		return http.StatusNotFound, err
 	case apiErr.Code == http.StatusTooManyRequests || apiErr.Code == http.StatusServiceUnavailable:
-		core.T.StatsUpdater().Inc(stats.ErrRateRetryCount)
 		return apiErr.Code, cmn.NewErrTooManyRequests(err, apiErr.Code)
 	default:
 		return apiErr.Code, err
diff --git a/ais/plstcx.go b/ais/plstcx.go
@@ -1,6 +1,6 @@
 // Package ais provides AIStore's proxy and target nodes.
 /*
- * Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2023-2025, NVIDIA CORPORATION. All rights reserved.
  */
 package ais
 
diff --git a/ais/rlbackend.go b/ais/rlbackend.go
@@ -20,10 +20,15 @@ import (
 )
 
 // rate-limit 4 backend APIs:
-// - PutObj()
-// - DeleteObj()
 // - GetObj()
 // - GetObjReader()
+// - PutObj()
+// - DeleteObj()
+
+// stats:
+// - not counting proactive delay    - only (reactive) retries
+// - not counting individual retries - only totals (see "increment" comment below)
+// - not counting delete retries     - only get and put
 
 type (
 	rlbackend struct {
@@ -43,9 +48,10 @@ func (bp *rlbackend) GetObj(ctx context.Context, lom *core.LOM, owt cmn.OWT, ori
 	cb := func() (int, error) {
 		return bp.Backend.GetObj(ctx, lom, owt, origReq)
 	}
-	tot, code, e := bp.retry(ctx, arl, cb)
+	total, code, e := bp.retry(ctx, arl, cb)
 
-	bp.stats(ctx, lom.Bck(), stats.GetRateRetryLatencyTotal, tot)
+	// increment retry count by 1, retry latency by `total`
+	bp.stats(ctx, lom.Bck(), stats.RatelimGetRetryCount, stats.RatelimGetRetryLatencyTotal, total)
 	return code, e
 }
 
@@ -61,10 +67,11 @@ func (bp *rlbackend) GetObjReader(ctx context.Context, lom *core.LOM, offset, le
 		res = bp.Backend.GetObjReader(ctx, lom, offset, length)
 		return res.ErrCode, res.Err
 	}
-	tot, code, e := bp.retry(ctx, arl, cb)
+	total, code, e := bp.retry(ctx, arl, cb)
 	debug.Assertf(res.ErrCode == code && res.Err == e, "(%d, %v) vs (%d, %v)", res.ErrCode, res.Err, code, e)
 
-	bp.stats(ctx, lom.Bck(), stats.GetRateRetryLatencyTotal, tot)
+	// ditto
+	bp.stats(ctx, lom.Bck(), stats.RatelimGetRetryCount, stats.RatelimGetRetryLatencyTotal, total)
 	return res
 }
 
@@ -79,12 +86,13 @@ func (bp *rlbackend) PutObj(ctx context.Context, r io.ReadCloser, lom *core.LOM,
 	cb := func() (int, error) {
 		return bp.Backend.PutObj(ctx, r, lom, origReq)
 	}
-	tot, code, e := bp.retry(ctx, arl, cb)
+	total, code, e := bp.retry(ctx, arl, cb)
 
-	bp.stats(ctx, lom.Bck(), stats.PutRateRetryLatencyTotal, tot)
+	bp.stats(ctx, lom.Bck(), stats.RatelimPutRetryCount, stats.RatelimPutRetryLatencyTotal, total)
 	return code, e
 }
 
+// note: not counting stats
 func (bp *rlbackend) DeleteObj(ctx context.Context, lom *core.LOM) (int, error) {
 	// proactive
 	arl := bp.acquire(lom.Bck(), http.MethodDelete)
@@ -96,9 +104,7 @@ func (bp *rlbackend) DeleteObj(ctx context.Context, lom *core.LOM) (int, error)
 	cb := func() (int, error) {
 		return bp.Backend.DeleteObj(ctx, lom)
 	}
-	tot, code, e := bp.retry(ctx, arl, cb)
-
-	bp.stats(ctx, lom.Bck(), stats.DeleteRateRetryLatencyTotal, tot)
+	_, code, e := bp.retry(ctx, arl, cb)
 	return code, e
 }
 
@@ -126,11 +132,11 @@ func (bp *rlbackend) acquire(bck *meta.Bck, verb string) (arl *cos.AdaptRateLim)
 	return arl
 }
 
-func (*rlbackend) retry(ctx context.Context, arl *cos.AdaptRateLim, cb func() (int, error)) (tot time.Duration, ecode int, err error) {
-	for tot < cos.DfltRateMaxWait {
+func (*rlbackend) retry(ctx context.Context, arl *cos.AdaptRateLim, cb func() (int, error)) (total time.Duration, ecode int, err error) {
+	for total < cos.DfltRateMaxWait {
 		// reactive
 		sleep := arl.OnErr()
-		tot += sleep
+		total += sleep
 		if err = ctx.Err(); err != nil {
 			break
 		}
@@ -139,18 +145,19 @@ func (*rlbackend) retry(ctx context.Context, arl *cos.AdaptRateLim, cb func() (i
 			break
 		}
 	}
-	return tot, ecode, err
+	return total, ecode, err
 }
 
-func (bp *rlbackend) stats(ctx context.Context, bck *meta.Bck, metric string, total time.Duration) {
+func (bp *rlbackend) stats(ctx context.Context, bck *meta.Bck, count, latency string, total time.Duration) {
 	if total == 0 {
 		return
 	}
 	vlabs := xact.GetCtxVlabs(ctx) // fast path
 	if vlabs == nil {
 		vlabs = map[string]string{stats.VlabBucket: bck.Cname(""), stats.VlabXkind: ""}
 	}
+	bp.t.statsT.IncWith(count, vlabs)
 	bp.t.statsT.AddWith(
-		cos.NamedVal64{Name: metric, Value: int64(total), VarLabs: vlabs},
+		cos.NamedVal64{Name: latency, Value: int64(total), VarLabs: vlabs},
 	)
 }
diff --git a/ais/s3/xattr.go b/ais/s3/xattr.go
@@ -1,6 +1,6 @@
 // Package s3 provides Amazon S3 compatibility layer
 /*
- * Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved.
  */
 package s3
 
diff --git a/cmd/cli/teb/performance.go b/cmd/cli/teb/performance.go
@@ -1,6 +1,6 @@
 // Package teb contains templates and (templated) tables to format CLI output.
 /*
- * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
  */
 package teb
 
@@ -126,6 +126,10 @@ func (c *PerfTabCtx) MakeTab(st StstMap) (*Table, int, error) {
 	// 10. construct an empty table
 	table := newTable(printedColumns...)
 
+	if len(cols) < 2 {
+		return table, numNZ, nil
+	}
+
 	// 11. finally, add rows
 	for _, tid := range tids {
 		ds := st[tid]
diff --git a/core/backend.go b/core/backend.go
@@ -40,21 +40,25 @@ type (
 		MetricName(string) string
 
 		CreateBucket(bck *meta.Bck) (ecode int, err error)
-		ListObjects(bck *meta.Bck, msg *apc.LsoMsg, lst *cmn.LsoRes) (ecode int, err error)
 		ListBuckets(qbck cmn.QueryBcks) (bcks cmn.Bcks, ecode int, err error)
+
+		// list-objects
+		ListObjects(bck *meta.Bck, msg *apc.LsoMsg, lst *cmn.LsoRes) (ecode int, err error)
+		ListObjectsInv(bck *meta.Bck, msg *apc.LsoMsg, lst *cmn.LsoRes, ctx *LsoInvCtx) error
+
 		PutObj(ctx context.Context, r io.ReadCloser, lom *LOM, origReq *http.Request) (ecode int, err error)
 		DeleteObj(ctx context.Context, lom *LOM) (ecode int, err error)
 
 		// head
 		HeadBucket(ctx context.Context, bck *meta.Bck) (bckProps cos.StrKVs, ecode int, err error)
 		HeadObj(ctx context.Context, lom *LOM, origReq *http.Request) (objAttrs *cmn.ObjAttrs, ecode int, err error)
 
-		// get
-		GetObj(ctx context.Context, lom *LOM, owt cmn.OWT, origReq *http.Request) (ecode int, err error) // calls GetObjReader
+		// get (exclusively via GetCold; calls GetObjReader)
+		GetObj(ctx context.Context, lom *LOM, owt cmn.OWT, origReq *http.Request) (ecode int, err error)
+		// get (jobs; REST)
 		GetObjReader(ctx context.Context, lom *LOM, offset, length int64) GetReaderResult
 
 		// bucket inventory
 		GetBucketInv(bck *meta.Bck, ctx *LsoInvCtx) (ecode int, err error)
-		ListObjectsInv(bck *meta.Bck, msg *apc.LsoMsg, lst *cmn.LsoRes, ctx *LsoInvCtx) error
 	}
 )
diff --git a/docs/metrics-reference.md b/docs/metrics-reference.md
@@ -188,14 +188,16 @@ For convenience, a table that summarizes target metrics follows below.
 | `append.ns` | `append_ms` | latency | APPEND(object): average time (milliseconds) over the last periodic.stats_time interval | default |
 | `get.redir.ns` | `get_redir_ms` | latency | GET: average gateway-to-target HTTP redirect latency (milliseconds) over the last periodic.stats_time interval | default |
 | `put.redir.ns` | `put_redir_ms` | latency | PUT: average gateway-to-target HTTP redirect latency (milliseconds) over the last periodic.stats_time interval | default |
-| `rate.retry.ns.total` | `rate_retry_ns_total` | total | total waiting time (nanoseconds) caused by remote backends returning 409 and 503 status codes | default |
+| `ratelim.retry.get.n` | `ratelim_retry_get_n` | counter | GET: number of rate-limited retries triggered by remote backends returning 409 and 503 status codes | default |
+| `ratelim.retry.get.ns.total` | `ratelim_retry_get_ns_total` | total | GET: total retrying time (nanoseconds) caused by remote backends returning 409 and 503 status codes | default |
+| `ratelim.retry.put.n` | `ratelim_retry_put_n` | counter | PUT: number of rate-limited retries triggered by remote backends returning 409 and 503 status codes | default |
+| `ratelim.retry.put.ns.total` | `ratelim_retry_put_ns_total` | total | PUT: total retrying time (nanoseconds) caused by remote backends returning 409 and 503 status codes | default |
 | `get.bps` | `get_mbps` | bandwidth | GET: average throughput (MB/s) over the last periodic.stats_time interval | default |
 | `put.bps` | `put_mbps` | bandwidth | PUT: average throughput (MB/s) over the last periodic.stats_time interval | default |
 | `get.size` | `get_bytes` | size | GET: total cumulative size (bytes) | default |
 | `put.size` | `put_bytes` | size | PUT: total cumulative size (bytes) | default |
 | `err.cksum.n` | `err_cksum_count` | counter | PUT: number of checksum errors | default |
 | `err.fshc.n` | `err_fshc_count` | counter | number of times filesystem health checker (FSHC) was triggered by an I/O error or errors | default |
-| `err.rate.retry.n` | `err_rate_retry_count` | counter | number of rate-limited retries triggered by remote backends returning 409 and 503 status codes | default |
 | `err.io.get.n` | `err_io_get_count` | counter | GET: number of I/O errors _not_ including remote backend and network errors | default |
 | `err.io.put.n` | `err_io_put_count` | counter | PUT: number of I/O errors _not_ including remote backend and network errors | default |
 | `err.io.del.n` | `err_io_del_count` | counter | DELETE(object): number of I/O errors _not_ including remote backend and network errors | default |
diff --git a/stats/api.go b/stats/api.go
@@ -131,14 +131,18 @@ func IsIOErrMetric(name string) bool {
 // see stats/common for "Naming conventions"
 //
 
-// compare with base.init() at ais/backend/common
+// see also base.init() in ais/backend/common
 func LatencyToCounter(latName string) string {
-	// 1. basics first
+	// 1. common
 	switch latName {
 	case GetLatency, GetRedirLatency, GetLatencyTotal:
 		return GetCount
 	case PutLatency, PutRedirLatency, PutLatencyTotal:
 		return PutCount
+	case RatelimGetRetryLatencyTotal:
+		return RatelimGetRetryCount
+	case RatelimPutRetryLatencyTotal:
+		return RatelimPutRetryCount
 	case HeadLatency:
 		return HeadCount
 	case ListLatency:
@@ -147,20 +151,21 @@ func LatencyToCounter(latName string) string {
 		return AppendCount
 	}
 	// 2. filter out
-	if !strings.Contains(latName, "get.") && !strings.Contains(latName, "put.") {
+	isget := strings.Contains(latName, "get.")
+	if !isget && !strings.Contains(latName, "put.") {
 		return ""
 	}
-	// backend first
+	// backend
 	if strings.HasSuffix(latName, ".ns.total") {
-		for prefix := range apc.Providers {
-			if prefix == apc.AIS {
-				prefix = apc.RemAIS
+		for p := range apc.Providers {
+			if p == apc.AIS {
+				p = apc.RemAIS
 			}
-			if strings.HasPrefix(latName, prefix) {
-				if strings.Contains(latName, ".get.") {
-					return prefix + "." + GetCount
+			if strings.HasPrefix(latName, p) {
+				if isget {
+					return p + "." + GetCount
 				}
-				return prefix + "." + PutCount
+				return p + "." + PutCount
 			}
 		}
 	}
diff --git a/stats/target_stats.go b/stats/target_stats.go
@@ -64,10 +64,10 @@ const (
 	PutE2ELatencyTotal = "e2e.put.ns.total" // end to end (e2e) PUT latency
 
 	// rate limit (409, 503)
-	ErrRateRetryCount           = errPrefix + "rate.retry.n" // http 409 and 503 from remote backends
-	GetRateRetryLatencyTotal    = "get.rate.retry.ns.total"
-	PutRateRetryLatencyTotal    = "put.rate.retry.ns.total"
-	DeleteRateRetryLatencyTotal = "del.rate.retry.ns.total"
+	RatelimGetRetryCount        = "ratelim.retry.get.n"
+	RatelimGetRetryLatencyTotal = "ratelim.retry.get.ns.total"
+	RatelimPutRetryCount        = "ratelim.retry.put.n"
+	RatelimPutRetryLatencyTotal = "ratelim.retry.put.ns.total"
 
 	AppendLatency     = "append.ns"
 	GetRedirLatency   = "get.redir.ns"
@@ -381,11 +381,6 @@ func (r *Trunner) RegMetrics(snode *meta.Snode) {
 			Help: "downloader: number of download errors",
 		},
 	)
-	r.reg(snode, ErrRateRetryCount, KindCounter,
-		&Extra{
-			Help: "number of rate-limited retries triggered by remote backends returning 409 and 503 status codes",
-		},
-	)
 
 	r.reg(snode, IOErrGetCount, KindCounter,
 		&Extra{
@@ -434,7 +429,6 @@ func (r *Trunner) RegMetrics(snode *meta.Snode) {
 		},
 	)
 
-	// download
 	r.reg(snode, DloadSize, KindSize,
 		&Extra{
 			Help:    "total downloaded size (bytes)",
@@ -447,21 +441,29 @@ func (r *Trunner) RegMetrics(snode *meta.Snode) {
 			VarLabs: BckVlabs,
 		},
 	)
-	r.reg(snode, GetRateRetryLatencyTotal, KindTotal,
+
+	// rate limit
+	r.reg(snode, RatelimGetRetryCount, KindCounter,
+		&Extra{
+			Help:    "GET: number of rate-limited retries triggered by remote backends returning 409 and 503 status codes",
+			VarLabs: BckXlabs,
+		},
+	)
+	r.reg(snode, RatelimGetRetryLatencyTotal, KindTotal,
 		&Extra{
-			Help:    "GET: total waiting time (nanoseconds) caused by remote backends returning 409 and 503 status codes",
+			Help:    "GET: total retrying time (nanoseconds) caused by remote backends returning 409 and 503 status codes",
 			VarLabs: BckXlabs,
 		},
 	)
-	r.reg(snode, PutRateRetryLatencyTotal, KindTotal,
+	r.reg(snode, RatelimPutRetryCount, KindCounter,
 		&Extra{
-			Help:    "PUT: total waiting time (nanoseconds) caused by remote backends returning 409 and 503 status codes",
+			Help:    "PUT: number of rate-limited retries triggered by remote backends returning 409 and 503 status codes",
 			VarLabs: BckXlabs,
 		},
 	)
-	r.reg(snode, DeleteRateRetryLatencyTotal, KindTotal,
+	r.reg(snode, RatelimPutRetryLatencyTotal, KindTotal,
 		&Extra{
-			Help:    "DELETE: total waiting time (nanoseconds) caused by remote backends returning 409 and 503 status codes",
+			Help:    "PUT: total retrying time (nanoseconds) caused by remote backends returning 409 and 503 status codes",
 			VarLabs: BckXlabs,
 		},
 	)

Original file line number	Diff line number	Diff line change
`@@ -923,7 +923,6 @@ func awsErrorToAISError(awsError error, bck *cmn.Bck, objName string) (int, erro`
`923`	`923`	`debug.Assert(false, "empty error code in ", awsError.Error()) // (unlikely)`
`924`	`924`	`code = strconv.Itoa(status)`
`925`	`925`	`}`
`926`		`- core.T.StatsUpdater().Inc(stats.ErrRateRetryCount)`
`927`	`926`	`e := fmt.Errorf("%s[%s: %s]", aiss3.ErrPrefix, code, bck.Cname(objName))`
`928`	`927`	`return status, cmn.NewErrTooManyRequests(e, status)`
`929`	`928`	`default:`
Original file line number	Diff line number	Diff line change
`@@ -174,7 +174,6 @@ func azureErrorToAISError(azureError error, bck *cmn.Bck, objName string) (int,`
`174`	`174`
`175`	`175`	`status, err := _azureErr(azureError, stgErr)`
`176`	`176`	`if status == http.StatusTooManyRequests \|\| status == http.StatusServiceUnavailable {`
`177`		`- core.T.StatsUpdater().Inc(stats.ErrRateRetryCount)`
`178`	`177`	`return status, cmn.NewErrTooManyRequests(err, status)`
`179`	`178`	`}`
`180`	`179`	`return status, err`
Original file line number	Diff line number	Diff line change
`@@ -491,7 +491,6 @@ func gcpErrorToAISError(gcpError error, bck *cmn.Bck) (int, error) {`
`491`	`491`	`}`
`492`	`492`	`return http.StatusNotFound, err`
`493`	`493`	`case apiErr.Code == http.StatusTooManyRequests \|\| apiErr.Code == http.StatusServiceUnavailable:`
`494`		`- core.T.StatsUpdater().Inc(stats.ErrRateRetryCount)`
`495`	`494`	`return apiErr.Code, cmn.NewErrTooManyRequests(err, apiErr.Code)`
`496`	`495`	`default:`
`497`	`496`	`return apiErr.Code, err`
Original file line number	Diff line number	Diff line change
`@@ -20,10 +20,15 @@ import (`
`20`	`20`	`)`
`21`	`21`
`22`	`22`	`// rate-limit 4 backend APIs:`
`23`		`-// - PutObj()`
`24`		`-// - DeleteObj()`
`25`	`23`	`// - GetObj()`
`26`	`24`	`// - GetObjReader()`
	`25`	`+// - PutObj()`
	`26`	`+// - DeleteObj()`
	`27`	`+`
	`28`	`+// stats:`
	`29`	`+// - not counting proactive delay - only (reactive) retries`
	`30`	`+// - not counting individual retries - only totals (see "increment" comment below)`
	`31`	`+// - not counting delete retries - only get and put`
`27`	`32`
`28`	`33`	`type (`
`29`	`34`	`rlbackend struct {`
`@@ -43,9 +48,10 @@ func (bp rlbackend) GetObj(ctx context.Context, lom core.LOM, owt cmn.OWT, ori`
`43`	`48`	`cb := func() (int, error) {`
`44`	`49`	`return bp.Backend.GetObj(ctx, lom, owt, origReq)`
`45`	`50`	`}`
`46`		`- tot, code, e := bp.retry(ctx, arl, cb)`
	`51`	`+ total, code, e := bp.retry(ctx, arl, cb)`
`47`	`52`
`48`		`- bp.stats(ctx, lom.Bck(), stats.GetRateRetryLatencyTotal, tot)`
	`53`	+ // increment retry count by 1, retry latency by `total`
	`54`	`+ bp.stats(ctx, lom.Bck(), stats.RatelimGetRetryCount, stats.RatelimGetRetryLatencyTotal, total)`
`49`	`55`	`return code, e`
`50`	`56`	`}`
`51`	`57`
`@@ -61,10 +67,11 @@ func (bp rlbackend) GetObjReader(ctx context.Context, lom core.LOM, offset, le`
`61`	`67`	`res = bp.Backend.GetObjReader(ctx, lom, offset, length)`
`62`	`68`	`return res.ErrCode, res.Err`
`63`	`69`	`}`
`64`		`- tot, code, e := bp.retry(ctx, arl, cb)`
	`70`	`+ total, code, e := bp.retry(ctx, arl, cb)`
`65`	`71`	`debug.Assertf(res.ErrCode == code && res.Err == e, "(%d, %v) vs (%d, %v)", res.ErrCode, res.Err, code, e)`
`66`	`72`
`67`		`- bp.stats(ctx, lom.Bck(), stats.GetRateRetryLatencyTotal, tot)`
	`73`	`+ // ditto`
	`74`	`+ bp.stats(ctx, lom.Bck(), stats.RatelimGetRetryCount, stats.RatelimGetRetryLatencyTotal, total)`
`68`	`75`	`return res`
`69`	`76`	`}`
`70`	`77`
`@@ -79,12 +86,13 @@ func (bp rlbackend) PutObj(ctx context.Context, r io.ReadCloser, lom core.LOM,`
`79`	`86`	`cb := func() (int, error) {`
`80`	`87`	`return bp.Backend.PutObj(ctx, r, lom, origReq)`
`81`	`88`	`}`
`82`		`- tot, code, e := bp.retry(ctx, arl, cb)`
	`89`	`+ total, code, e := bp.retry(ctx, arl, cb)`
`83`	`90`
`84`		`- bp.stats(ctx, lom.Bck(), stats.PutRateRetryLatencyTotal, tot)`
	`91`	`+ bp.stats(ctx, lom.Bck(), stats.RatelimPutRetryCount, stats.RatelimPutRetryLatencyTotal, total)`
`85`	`92`	`return code, e`
`86`	`93`	`}`
`87`	`94`
	`95`	`+// note: not counting stats`
`88`	`96`	`func (bp rlbackend) DeleteObj(ctx context.Context, lom core.LOM) (int, error) {`
`89`	`97`	`// proactive`
`90`	`98`	`arl := bp.acquire(lom.Bck(), http.MethodDelete)`
`@@ -96,9 +104,7 @@ func (bp rlbackend) DeleteObj(ctx context.Context, lom core.LOM) (int, error)`
`96`	`104`	`cb := func() (int, error) {`
`97`	`105`	`return bp.Backend.DeleteObj(ctx, lom)`
`98`	`106`	`}`
`99`		`- tot, code, e := bp.retry(ctx, arl, cb)`
`100`		`-`
`101`		`- bp.stats(ctx, lom.Bck(), stats.DeleteRateRetryLatencyTotal, tot)`
	`107`	`+ _, code, e := bp.retry(ctx, arl, cb)`
`102`	`108`	`return code, e`
`103`	`109`	`}`
`104`	`110`
`@@ -126,11 +132,11 @@ func (bp rlbackend) acquire(bck meta.Bck, verb string) (arl *cos.AdaptRateLim)`
`126`	`132`	`return arl`
`127`	`133`	`}`
`128`	`134`
`129`		`-func (rlbackend) retry(ctx context.Context, arl cos.AdaptRateLim, cb func() (int, error)) (tot time.Duration, ecode int, err error) {`
`130`		`- for tot < cos.DfltRateMaxWait {`
	`135`	`+func (rlbackend) retry(ctx context.Context, arl cos.AdaptRateLim, cb func() (int, error)) (total time.Duration, ecode int, err error) {`
	`136`	`+ for total < cos.DfltRateMaxWait {`
`131`	`137`	`// reactive`
`132`	`138`	`sleep := arl.OnErr()`
`133`		`- tot += sleep`
	`139`	`+ total += sleep`
`134`	`140`	`if err = ctx.Err(); err != nil {`
`135`	`141`	`break`
`136`	`142`	`}`
`@@ -139,18 +145,19 @@ func (rlbackend) retry(ctx context.Context, arl cos.AdaptRateLim, cb func() (i`
`139`	`145`	`break`
`140`	`146`	`}`
`141`	`147`	`}`
`142`		`- return tot, ecode, err`
	`148`	`+ return total, ecode, err`
`143`	`149`	`}`
`144`	`150`
`145`		`-func (bp rlbackend) stats(ctx context.Context, bck meta.Bck, metric string, total time.Duration) {`
	`151`	`+func (bp rlbackend) stats(ctx context.Context, bck meta.Bck, count, latency string, total time.Duration) {`
`146`	`152`	`if total == 0 {`
`147`	`153`	`return`
`148`	`154`	`}`
`149`	`155`	`vlabs := xact.GetCtxVlabs(ctx) // fast path`
`150`	`156`	`if vlabs == nil {`
`151`	`157`	`vlabs = map[string]string{stats.VlabBucket: bck.Cname(""), stats.VlabXkind: ""}`
`152`	`158`	`}`
	`159`	`+ bp.t.statsT.IncWith(count, vlabs)`
`153`	`160`	`bp.t.statsT.AddWith(`
`154`		`- cos.NamedVal64{Name: metric, Value: int64(total), VarLabs: vlabs},`
	`161`	`+ cos.NamedVal64{Name: latency, Value: int64(total), VarLabs: vlabs},`
`155`	`162`	`)`
`156`	`163`	`}`