core: object chunks; chunk manifest

alex-aizman · alex-aizman · commit 9eaaba8db28f · 2025-08-12T19:18:58.000-04:00
* add StoreCompleted() and StorePartial() - the latter for checkpointing * add lom.CompleteUfest() to persist (manifest, LOM) atomically * add chunk #1 fixup logic; add rollback * micro-optimize GetChunk() for sequential access ------ * chunk manifest to support io.ReaderAt (with this, all LomReader interfaces) ------ * unit: fix CSM resolver registration typo * unit: add testing parameter to StoreCompleted() for non-existing chunks ------ * cleanup and refactor; unify error handling * part eight, prev. commit: 7c8a770 Signed-off-by: Alex Aizman <alex.aizman@gmail.com>
diff --git a/ais/tgts3mpt.go b/ais/tgts3mpt.go
@@ -475,8 +475,8 @@ func (t *target) completeMpt(w http.ResponseWriter, r *http.Request, items []str
 	ecode, errF := poi.finalize()
 	freePOI(poi)
 
-	// .6 write manifest
-	if err := manifest.Store(lom); err != nil {
+	// .6 write manifest // TODO -- FIXME: niy
+	if err := manifest.StoreCompleted(lom, true); err != nil {
 		s3.WriteMptErr(w, r, errF, ecode, lom, uploadID)
 		return
 	}
diff --git a/core/lchunk_test.go b/core/lchunk_test.go
@@ -42,6 +42,8 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
 		bucketLocal = "CHUNK_TEST_Local"
 	)
 
+	const unitTestingWithNoRealChunks = true
+
 	localBck := cmn.Bck{Name: bucketLocal, Provider: apc.AIS, Ns: cmn.NsGlobal}
 
 	fs.CSM.Reg(fs.ObjectType, &fs.ObjectContentResolver{}, true)
@@ -142,7 +144,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
 				manifest := createChunkManifest(testFileSize, 3, chunkSizes, "test-session-001", lom)
 
 				// Store manifest
-				err := manifest.Store(lom)
+				err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
 				Expect(err).NotTo(HaveOccurred())
 
 				// After store, manifest should be marked as completed
@@ -202,7 +204,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
 				chunkSizes := []int64{testFileSize}
 				manifest := createChunkManifest(testFileSize, 1, chunkSizes, "single-chunk-session", lom)
 
-				err := manifest.Store(lom)
+				err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
 				Expect(err).NotTo(HaveOccurred())
 
 				loadedManifest := &core.Ufest{}
@@ -231,7 +233,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
 
 				manifest := createChunkManifest(testFileSize, numChunks, chunkSizes, "many-chunks-session", lom)
 
-				err := manifest.Store(lom)
+				err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
 				Expect(err).NotTo(HaveOccurred())
 
 				loadedManifest := &core.Ufest{}
@@ -267,7 +269,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
 				testTime := time.Date(2025, 8, 2, 15, 30, 45, 0, time.UTC)
 				manifest.Created = testTime
 
-				err := manifest.Store(lom)
+				err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
 				Expect(err).NotTo(HaveOccurred())
 
 				loadedManifest := &core.Ufest{}
@@ -293,7 +295,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
 					{Siz: 524000, Num: 2, Path: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Cksum: cos.NewCksum(cos.ChecksumCesXxh, "def456")},
 				}
 
-				err := manifest.Store(lom)
+				err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
 				Expect(err).To(HaveOccurred())
 				Expect(err.Error()).To(ContainSubstring("invalid chunk-manifest"))
 			})
@@ -307,7 +309,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
 				manifest.Num = 0
 				manifest.Chunks = []core.Uchunk{}
 
-				err := manifest.Store(lom)
+				err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
 				Expect(err).To(HaveOccurred())
 				Expect(err.Error()).To(ContainSubstring("invalid chunk-manifest"))
 			})
@@ -335,7 +337,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
 					manifest.Chunks[i].Cksum = cos.NewCksum(cos.ChecksumCesXxh, longVal)
 				}
 
-				err := manifest.Store(lom)
+				err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
 				Expect(err).To(HaveOccurred())
 				Expect(err.Error()).To(ContainSubstring("too large"))
 			})
@@ -359,7 +361,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
 				// Store valid manifest first
 				chunkSizes := []int64{testFileSize}
 				manifest := createChunkManifest(testFileSize, 1, chunkSizes, "version-test", lom)
-				err := manifest.Store(lom)
+				err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
 				Expect(err).NotTo(HaveOccurred())
 
 				// Corrupt the version byte
@@ -382,7 +384,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
 				// Store valid manifest
 				chunkSizes := []int64{testFileSize}
 				manifest := createChunkManifest(testFileSize, 1, chunkSizes, "checksum-test", lom)
-				err := manifest.Store(lom)
+				err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
 				Expect(err).NotTo(HaveOccurred())
 
 				b, err := fs.GetXattr(localFQN, xattrChunk)
@@ -408,7 +410,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
 				// Store valid manifest first
 				chunkSizes := []int64{testFileSize}
 				manifest := createChunkManifest(testFileSize, 1, chunkSizes, "truncated-test", lom)
-				err := manifest.Store(lom)
+				err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
 				Expect(err).NotTo(HaveOccurred())
 
 				// Truncate the xattr data
@@ -437,7 +439,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
 					{Siz: 548576, Num: 2, Cksum: cos.NewCksum(cos.ChecksumCesXxh, "validchecksum"), MD5: "md5hash2"}, // total = 1048576
 				}
 
-				err := manifest.Store(lom)
+				err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
 				Expect(err).NotTo(HaveOccurred())
 
 				loadedManifest := &core.Ufest{}
@@ -462,7 +464,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
 					{Siz: 548576, Num: 2, Cksum: cos.NewCksum(cos.ChecksumCesXxh, "validchecksum"), MD5: "md5hash2"}, // total = 1048576
 				}
 
-				err := manifest.Store(lom)
+				err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
 				Expect(err).NotTo(HaveOccurred())
 
 				loadedManifest := &core.Ufest{}
@@ -489,7 +491,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
 					{Siz: 0, Num: 3, Cksum: cos.NewCksum(cos.ChecksumCesXxh, "empty2"), MD5: "md5_3"}, // another zero - total = 1048576
 				}
 
-				err := manifest.Store(lom)
+				err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
 				Expect(err).NotTo(HaveOccurred())
 
 				loadedManifest := &core.Ufest{}
@@ -513,7 +515,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
 					{Siz: testFileSize, Num: 1, Path: "test", Cksum: cos.NewCksum(cos.ChecksumCesXxh, "checksum"), MD5: "md5hash"},
 				}
 
-				err := manifest.Store(lom)
+				err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
 				Expect(err).NotTo(HaveOccurred())
 
 				loadedManifest := &core.Ufest{}
@@ -536,7 +538,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
 					{Siz: testFileSize, Num: 1, Path: "test", Cksum: cos.NewCksum(cos.ChecksumCesXxh, "checksum"), MD5: "md5hash"},
 				}
 
-				err := manifest.Store(lom)
+				err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
 				Expect(err).NotTo(HaveOccurred())
 
 				loadedManifest := &core.Ufest{}
@@ -569,7 +571,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
 					{Siz: testFileSize, Num: 1, Path: "chunk1", Cksum: cos.NewCksum(cos.ChecksumCesXxh, "abc123"), MD5: "md5hash"},
 				}
 
-				err := manifest.Store(lom)
+				err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
 				Expect(err).NotTo(HaveOccurred())
 
 				loadedManifest := &core.Ufest{}
@@ -605,7 +607,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
 				// Initially not completed
 				Expect(manifest.Completed()).To(BeFalse())
 
-				err := manifest.Store(lom)
+				err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
 				Expect(err).NotTo(HaveOccurred())
 
 				// After store, should be completed
diff --git a/core/lom_test.go b/core/lom_test.go
@@ -20,7 +20,6 @@ import (
 	"github.com/NVIDIA/aistore/core/mock"
 	"github.com/NVIDIA/aistore/fs"
 	"github.com/NVIDIA/aistore/tools/readers"
-	"github.com/NVIDIA/aistore/tools/trand"
 
 	onexxh "github.com/OneOfOne/xxhash"
 	. "github.com/onsi/ginkgo/v2"
@@ -73,7 +72,7 @@ var _ = Describe("LOM", func() {
 
 	fs.CSM.Reg(fs.ObjectType, &fs.ObjectContentResolver{}, true)
 	fs.CSM.Reg(fs.WorkfileType, &fs.WorkfileContentResolver{}, true)
-	fs.CSM.Reg(fs.ObjChunkType, &fs.ObjectContentResolver{}, true)
+	fs.CSM.Reg(fs.ObjChunkType, &fs.ObjChunkContentResolver{}, true)
 
 	bmd := mock.NewBaseBownerMock(
 		meta.NewBck(
@@ -1070,68 +1069,6 @@ var _ = Describe("LOM", func() {
 		})
 	})
 
-	Describe("Chunked Objects – round-trip store/load", func() {
-		const (
-			objName = "chunked/roundtrip.bin"
-			// total size 1 MiB split into three chunks
-			sz1       = int64(400_000)
-			sz2       = int64(400_000)
-			sz3       = int64(cos.MiB) - sz1 - sz2
-			totalSize = int64(cos.MiB)
-		)
-
-		var (
-			fqn = mis[0].MakePathFQN(&localBckB, fs.ObjectType, objName)
-			// helper to build a manifest with paths + fake checksums
-			newManifest = func() *core.Ufest {
-				return &core.Ufest{
-					Size: totalSize,
-					Num:  3,
-					Chunks: []core.Uchunk{
-						{Siz: sz1, Num: 3, Path: "ch/" + objName + ".00001", Cksum: cos.NewCksum(cos.ChecksumCesXxh, trand.String(16))},
-						{Siz: sz2, Num: 2, Path: "ch/" + objName + ".00002", Cksum: cos.NewCksum(cos.ChecksumCesXxh, trand.String(16))},
-						{Siz: sz3, Num: 1, Path: "ch/" + objName + ".00003", Cksum: cos.NewCksum(cos.ChecksumCesXxh, trand.String(16))},
-					},
-				}
-			}
-		)
-
-		It("should persist and reload chunk manifest & flag", func() {
-			// ── 1. create Chunk-#1 and initial LOM ────────────────────────────────
-			createTestFile(fqn, int(sz1))
-			lom := newBasicLom(fqn, totalSize)
-			manifest := newManifest()
-
-			// store manifest, then persist normal LOM metadata
-			Expect(manifest.Store(lom)).To(Succeed())
-			Expect(persist(lom)).To(Succeed())
-			lom.UncacheUnless()
-
-			// ── 2. reload the object (fresh LOM) ──────────────────────────────────
-			re := newBasicLom(fqn)
-			Expect(re.Load(false, false)).To(Succeed())
-
-			// NOTE: IsChunked() is still false at this point because the
-			// manifest hasn’t been loaded yet – load it now, which also
-			// re-sets the lmflChunk flag in memory.
-			var out core.Ufest
-			Expect(out.Load(re)).To(Succeed())
-
-			// flag should now be restored
-			// TODO -- FIXME Expect(re.IsChunked()).To(BeTrue())
-			Expect(re.Lsize()).To(Equal(totalSize))
-
-			// compare manifest contents
-			Expect(out.Num).To(Equal(manifest.Num))
-			Expect(out.Size).To(Equal(manifest.Size))
-			for i := range 3 {
-				Expect(out.Chunks[i].Siz).To(Equal(manifest.Chunks[i].Siz))
-				Expect(out.Chunks[i].Path).To(Equal(manifest.Chunks[i].Path))
-				Expect(out.Chunks[i].Cksum.Equal(manifest.Chunks[i].Cksum)).To(BeTrue())
-			}
-		})
-	})
-
 	Describe("UfestReader", func() {
 		const (
 			fileSize  = 1 * cos.GiB
@@ -1142,7 +1079,7 @@ var _ = Describe("LOM", func() {
 			testObject := "chunked/large-file.bin"
 			localFQN := mis[0].MakePathFQN(&localBckB, fs.ObjectType, testObject)
 
-			createDummyFile(localFQN)
+			createTestFile(localFQN, 0)
 			lom := newBasicLom(localFQN, fileSize)
 
 			Expect(cos.CreateDir(filepath.Dir(localFQN))).NotTo(HaveOccurred())
@@ -1180,7 +1117,7 @@ var _ = Describe("LOM", func() {
 			Expect(ufest.Size).To(Equal(int64(fileSize)))
 
 			// Store manifest (this will mark it as completed)
-			err := ufest.Store(lom)
+			err := ufest.StoreCompleted(lom)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(ufest.Completed()).To(BeTrue())
 
@@ -1225,7 +1162,7 @@ var _ = Describe("LOM", func() {
 			testObject := "chunked/empty-file.bin"
 			localFQN := mis[0].MakePathFQN(&localBckB, fs.ObjectType, testObject)
 
-			createDummyFile(localFQN)
+			createTestFile(localFQN, 0)
 			lom := newBasicLom(localFQN, 0)
 			ufest := core.NewUfest("empty-test-"+cos.GenTie(), lom)
 
@@ -1239,7 +1176,7 @@ var _ = Describe("LOM", func() {
 			err = ufest.Add(chunk, 0, 1)
 			Expect(err).NotTo(HaveOccurred())
 
-			err = ufest.Store(lom)
+			err = ufest.StoreCompleted(lom)
 			Expect(err).NotTo(HaveOccurred())
 
 			reader, err := ufest.NewReader()
@@ -1255,7 +1192,7 @@ var _ = Describe("LOM", func() {
 
 			By("testing incomplete manifest")
 			fqn2 := mis[0].MakePathFQN(&localBckB, fs.ObjectType, "incomplete.bin")
-			createDummyFile(fqn2)
+			createTestFile(fqn2, 0)
 			lom2 := newBasicLom(fqn2)
 			ufest2 := core.NewUfest("incomplete-test-"+cos.GenTie(), lom2)
 
@@ -1269,7 +1206,7 @@ var _ = Describe("LOM", func() {
 			testObject := "chunked/size-test.bin"
 			localFQN := mis[0].MakePathFQN(&localBckB, fs.ObjectType, testObject)
 
-			createDummyFile(localFQN)
+			createTestFile(localFQN, 0)
 			lom := newBasicLom(localFQN)
 			ufest := core.NewUfest("size-test-"+cos.GenTie(), lom)
 
@@ -1286,7 +1223,7 @@ var _ = Describe("LOM", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			lom.SetSize(200)
-			err = ufest.Store(lom)
+			err = ufest.StoreCompleted(lom)
 			Expect(err).NotTo(HaveOccurred())
 
 			reader, err := ufest.NewReader()
@@ -1311,7 +1248,7 @@ var _ = Describe("LOM", func() {
 			testObject := "chunked/many-small-chunks.bin"
 			localFQN := mis[0].MakePathFQN(&localBckB, fs.ObjectType, testObject)
 
-			createDummyFile(localFQN)
+			createTestFile(localFQN, 0)
 			lom := newBasicLom(localFQN, totalFileSize)
 			ufest := core.NewUfest("multi-chunk-test-"+cos.GenTie(), lom)
 
@@ -1330,7 +1267,7 @@ var _ = Describe("LOM", func() {
 				Expect(err).NotTo(HaveOccurred())
 			}
 
-			err := ufest.Store(lom)
+			err := ufest.StoreCompleted(lom)
 			Expect(err).NotTo(HaveOccurred())
 
 			reader, err := ufest.NewReader()
diff --git a/core/ufest.go b/core/ufest.go
