prefetch (job): now rate-limited as well; up modules

alex-aizman · alex-aizman · commit 9fa92650a4fb · 2025-02-26T11:41:09.000-05:00
Signed-off-by: Alex Aizman &lt;alex.aizman@gmail.com&gt;
diff --git a/cmd/cli/go.mod b/cmd/cli/go.mod
@@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/cli
 go 1.23.5
 
 require (
-	github.com/NVIDIA/aistore v1.3.28-0.20250226022703-1cb573b700b3
+	github.com/NVIDIA/aistore v1.3.28-0.20250226152703-6097a72679a2
 	github.com/fatih/color v1.18.0
 	github.com/json-iterator/go v1.1.12
 	github.com/onsi/ginkgo/v2 v2.22.2
diff --git a/cmd/cli/go.sum b/cmd/cli/go.sum
@@ -1,7 +1,7 @@
 code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
 github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
-github.com/NVIDIA/aistore v1.3.28-0.20250226022703-1cb573b700b3 h1:1uLAx+pTvFjVTP59jnMh0nmxg2AH7FIltmxuFRjmiRo=
-github.com/NVIDIA/aistore v1.3.28-0.20250226022703-1cb573b700b3/go.mod h1:iaKSdhrg0KejxzcxX/jEpRuMhsneu4EFdYAjQBTWx1I=
+github.com/NVIDIA/aistore v1.3.28-0.20250226152703-6097a72679a2 h1:cwnLSqoa6evWPeSaOQCYy06QaKIDQ+yuDwpJxIfZ7Mw=
+github.com/NVIDIA/aistore v1.3.28-0.20250226152703-6097a72679a2/go.mod h1:iaKSdhrg0KejxzcxX/jEpRuMhsneu4EFdYAjQBTWx1I=
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
diff --git a/cmd/ishard/go.mod b/cmd/ishard/go.mod
@@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/ishard
 go 1.23.5
 
 require (
-	github.com/NVIDIA/aistore v1.3.28-0.20250226022703-1cb573b700b3
+	github.com/NVIDIA/aistore v1.3.28-0.20250226152703-6097a72679a2
 	github.com/json-iterator/go v1.1.12
 	github.com/vbauerster/mpb/v4 v4.12.2
 )
diff --git a/cmd/ishard/go.sum b/cmd/ishard/go.sum
@@ -1,6 +1,6 @@
 code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
-github.com/NVIDIA/aistore v1.3.28-0.20250226022703-1cb573b700b3 h1:1uLAx+pTvFjVTP59jnMh0nmxg2AH7FIltmxuFRjmiRo=
-github.com/NVIDIA/aistore v1.3.28-0.20250226022703-1cb573b700b3/go.mod h1:iaKSdhrg0KejxzcxX/jEpRuMhsneu4EFdYAjQBTWx1I=
+github.com/NVIDIA/aistore v1.3.28-0.20250226152703-6097a72679a2 h1:cwnLSqoa6evWPeSaOQCYy06QaKIDQ+yuDwpJxIfZ7Mw=
+github.com/NVIDIA/aistore v1.3.28-0.20250226152703-6097a72679a2/go.mod h1:iaKSdhrg0KejxzcxX/jEpRuMhsneu4EFdYAjQBTWx1I=
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
diff --git a/xact/xs/prefetch.go b/xact/xs/prefetch.go
@@ -47,6 +47,10 @@ type (
 	prefetch struct {
 		config *cmn.Config
 		msg    *apc.PrefetchMsg
+		rate   struct {
+			arl   *cos.AdaptRateLim
+			sleep time.Duration
+		}
 		lrit
 		pebl pebl
 		xact.Base
@@ -96,6 +100,10 @@ func newPrefetch(xargs *xreg.Args, kind string, bck *meta.Bck, msg *apc.Prefetch
 	r.InitBase(xargs.UUID, kind, msg.Str(r.lrp == lrpPrefix), bck)
 	r.latestVer = bck.VersionConf().ValidateWarmGet || msg.LatestVer
 
+	smap := core.T.Sowner().Get()
+	nat := smap.CountActiveTs()
+	r.rate.arl, r.rate.sleep = bck.NewBackendRateLim(nat)
+
 	if r.msg.BlobThreshold > 0 {
 		r.pebl.init(r)
 	}
@@ -157,6 +165,7 @@ func (r *prefetch) do(lom *core.LOM, lrit *lrit) {
 	// NOTE ref 6735188: _not_ setting negative atime, flushing lom metadata
 	//
 
+retry:
 	if r.msg.BlobThreshold > 0 && size >= r.msg.BlobThreshold && r.pebl.num() < maxPebls {
 		err = r.blobdl(lom, oa)
 	} else {
@@ -182,12 +191,28 @@ func (r *prefetch) do(lom *core.LOM, lrit *lrit) {
 		}
 	}
 
-	if err == nil { // done
-		return
-	}
-	if cos.IsNotExist(err, ecode) && lrit.lrp != lrpList {
-		return // not found, prefix or range
+	switch {
+	case err == nil:
+		if cmn.Rom.FastV(5, cos.SmoduleXs) {
+			nlog.Infoln(r.Name(), lom.Cname())
+		}
+	case cos.IsNotExist(err, ecode):
+		if lrit.lrp == lrpList {
+			r.AddErr(err, 5, cos.SmoduleXs)
+		}
+	case cos.IsErrOOS(err):
+		r.Abort(err)
+	case cmn.IsErrTooManyRequests(err):
+		if r.rate.arl != nil {
+			r.rate.arl.OnErr()
+			goto retry
+		}
+		fallthrough
+	default:
+		r.AddErr(err, 5, cos.SmoduleXs)
 	}
+	return
+
 eret:
 	r.AddErr(err, 5, cos.SmoduleXs)
 }
diff --git a/xact/xs/xaction_test.go b/xact/xs/xaction_test.go

Original file line number	Diff line number	Diff line change
`@@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/ishard`
`3`	`3`	`go 1.23.5`
`4`	`4`
`5`	`5`	`require (`
`6`		`- github.com/NVIDIA/aistore v1.3.28-0.20250226022703-1cb573b700b3`
	`6`	`+ github.com/NVIDIA/aistore v1.3.28-0.20250226152703-6097a72679a2`
`7`	`7`	`github.com/json-iterator/go v1.1.12`
`8`	`8`	`github.com/vbauerster/mpb/v4 v4.12.2`
`9`	`9`	`)`