docs: unified s3 compatibility, and more

alex-aizman · alex-aizman · commit 0d03b71a30e0 · 2025-05-14T16:34:44.000-04:00
* consolidate all 3 docs as /docs/s3compat
* add introduction: the "3 fronts" of supporting AWS S3
* explains the pros and cons of using S3 compatibility API (as opposed
  to native)
* add Boto3 section
* reduce duplication
* clarify examples
* summarize S3 compatibility (table)

Signed-off-by: Alex Aizman &lt;alex.aizman@gmail.com&gt;
diff --git a/ais/test/scripted_cli_test.go b/ais/test/scripted_cli_test.go
@@ -155,7 +155,7 @@ func TestMultipartUploadUsingScript(t *testing.T) {
 	tempdir := t.TempDir()
 	bck := cmn.Bck{Name: trand.String(10), Provider: apc.AIS}
 
-	// 1. set MD5 to satisfy `s3cmd` (for details, see docs/s3cmd.md & docs/s3compat.md)
+	// 1. set MD5 to satisfy `s3cmd` (for details, see docs/s3compat.md)
 	bprops := &cmn.BpropsToSet{
 		Cksum: &cmn.CksumConfToSet{Type: apc.Ptr(cos.ChecksumMD5)},
 	}
diff --git a/cmd/cli/cli/const.go b/cmd/cli/cli/const.go
@@ -630,11 +630,11 @@ var (
 
 	useInventoryFlag = cli.BoolFlag{
 		Name: "inventory",
-		Usage: "List objects using _bucket inventory_ (docs/s3inventory.md); requires s3:// backend; will provide significant performance\n" +
+		Usage: "List objects using _bucket inventory_ (docs/s3compat.md); requires s3:// backend; will provide significant performance\n" +
 			indent4 + "\tboost when used with very large s3 buckets; e.g. usage:\n" +
 			indent4 + "\t  1) 'ais ls s3://abc --inventory'\n" +
 			indent4 + "\t  2) 'ais ls s3://abc --inventory --paged --prefix=subdir/'\n" +
-			indent4 + "\t(see also: docs/s3inventory.md)",
+			indent4 + "\t(see also: docs/s3compat.md)",
 	}
 	invNameFlag = cli.StringFlag{
 		Name:  "inv-name", // compare w/ HdrInvName
diff --git a/docs/bucket.md b/docs/bucket.md
@@ -267,7 +267,7 @@ AIS (`ais://`) buckets can be created using native HTTP-based ([Go](https://gith
 New ais buckets must be given a unique name that does not duplicate any existing ais bucket.
 
 If you are going to use an AIS bucket as an S3-compatible one, consider changing the bucket's checksum to `MD5`.
-For details, see [S3 compatibility](s3compat.md#s3-compatibility).
+For details, see [S3 compatibility](/docs/s3compat.md).
 
 ## CLI: create, rename and, destroy ais bucket
 
diff --git a/docs/cli/archive.md b/docs/cli/archive.md
@@ -567,11 +567,11 @@ OPTIONS:
                           use the job's UUID to query the results interactively
    --inv-id value         Bucket inventory ID (optional; by default, we use bucket name as the bucket's inventory ID)
    --inv-name value       Bucket inventory name (optional; system default name is '.inventory')
-   --inventory            List objects using _bucket inventory_ (docs/s3inventory.md); requires s3:// backend; will provide significant performance
+   --inventory            List objects using _bucket inventory_ (docs/s3compat.md); requires s3:// backend; will provide significant performance
                           boost when used with very large s3 buckets; e.g. usage:
                             1) 'ais ls s3://abc --inventory'
                             2) 'ais ls s3://abc --inventory --paged --prefix=subdir/'
-                          (see also: docs/s3inventory.md)
+                          (see also: docs/s3compat.md)
    --limit value          The maximum number of objects to list, get, or otherwise handle (0 - unlimited; see also '--max-pages'),
                           e.g.:
                           - 'ais ls gs://abc/dir --limit 1234 --cached --props size,custom,atime'  - list no more than 1234 objects
@@ -721,11 +721,11 @@ OPTIONS:
    extract,x       Extract all files from archive(s)
    inv-id          Bucket inventory ID (optional; by default, we use bucket name as the bucket's inventory ID)
    inv-name        Bucket inventory name (optional; system default name is '.inventory')
-   inventory       List objects using _bucket inventory_ (docs/s3inventory.md); requires s3:// backend; will provide significant performance
+   inventory       List objects using _bucket inventory_ (docs/s3compat.md); requires s3:// backend; will provide significant performance
                    boost when used with very large s3 buckets; e.g. usage:
                      1) 'ais ls s3://abc --inventory'
                      2) 'ais ls s3://abc --inventory --paged --prefix=subdir/'
-                   (see also: docs/s3inventory.md)
+                   (see also: docs/s3compat.md)
    latest          Check in-cluster metadata and, possibly, GET, download, prefetch, or otherwise copy the latest object version
                    from the associated remote bucket;
                    the option provides operation-level control over object versioning (and version synchronization)
diff --git a/docs/cli/bucket.md b/docs/cli/bucket.md
@@ -345,11 +345,11 @@ OPTIONS:
                           use the job's UUID to query the results interactively
    --inv-id value         Bucket inventory ID (optional; by default, we use bucket name as the bucket's inventory ID)
    --inv-name value       Bucket inventory name (optional; system default name is '.inventory')
-   --inventory            List objects using _bucket inventory_ (docs/s3inventory.md); requires s3:// backend; will provide significant performance
+   --inventory            List objects using _bucket inventory_ (docs/s3compat.md); requires s3:// backend; will provide significant performance
                           boost when used with very large s3 buckets; e.g. usage:
                             1) 'ais ls s3://abc --inventory'
                             2) 'ais ls s3://abc --inventory --paged --prefix=subdir/'
-                          (see also: docs/s3inventory.md)
+                          (see also: docs/s3compat.md)
    --limit value          The maximum number of objects to list, get, or otherwise handle (0 - unlimited; see also '--max-pages'),
                           e.g.:
                           - 'ais ls gs://abc/dir --limit 1234 --cached --props size,custom,atime'  - list no more than 1234 objects
diff --git a/docs/cli/object.md b/docs/cli/object.md
@@ -119,11 +119,11 @@ OPTIONS:
    --extract, -x        Extract all files from archive(s)
    --inv-id value       Bucket inventory ID (optional; by default, we use bucket name as the bucket's inventory ID)
    --inv-name value     Bucket inventory name (optional; system default name is '.inventory')
-   --inventory          List objects using _bucket inventory_ (docs/s3inventory.md); requires s3:// backend; will provide significant performance
+   --inventory          List objects using _bucket inventory_ (docs/s3compat.md); requires s3:// backend; will provide significant performance
                         boost when used with very large s3 buckets; e.g. usage:
                           1) 'ais ls s3://abc --inventory'
                           2) 'ais ls s3://abc --inventory --paged --prefix=subdir/'
-                        (see also: docs/s3inventory.md)
+                        (see also: docs/s3compat.md)
    --latest             Check in-cluster metadata and, possibly, GET, download, prefetch, or otherwise copy the latest object version
                         from the associated remote bucket;
                         the option provides operation-level control over object versioning (and version synchronization)
diff --git a/docs/docs.md b/docs/docs.md
@@ -26,7 +26,7 @@ redirect_from:
 - [REST API](/docs/http_api.md)
   - [Easy URL](https://github.com/NVIDIA/aistore/blob/main/docs/easy_url.md)
 - Amazon S3
-  - [`s3cmd` client](/docs/s3cmd.md)
+  - [`s3cmd` client](/docs/s3compat.md#quick-start-with-s3cmd)
   - [S3 compatibility](/docs/s3compat.md)
   - [Presigned S3 requests](/docs/s3compat.md#presigned-s3-requests)
   - [Boto3 support](https://github.com/NVIDIA/aistore/tree/main/python/aistore/botocore_patch)
diff --git a/docs/overview.md b/docs/overview.md
@@ -243,7 +243,7 @@ In addition to industry-standard [S3](/docs/s3compat.md), AIS provides its own (
 - [HTTP REST](/docs/http_api.md)
 
 For Amazon S3 compatibility and related topics, see also:
-  - [`s3cmd` client](/docs/s3cmd.md)
+  - [`s3cmd` client](/docs/s3compat.md#quick-start-with-s3cmd)
   - [S3 compatibility](/docs/s3compat.md)
   - [Presigned S3 requests](/docs/s3compat.md#presigned-s3-requests)
   - [Boto3 support](https://github.com/NVIDIA/aistore/tree/main/python/aistore/botocore_patch)
diff --git a/docs/s3cmd.md b/docs/s3cmd.md
diff --git a/docs/s3compat.md b/docs/s3compat.md
diff --git a/docs/s3inventory.md b/docs/s3inventory.md
diff --git a/scripts/s3/README.md b/scripts/s3/README.md

Original file line number	Diff line number	Diff line change
`@@ -155,7 +155,7 @@ func TestMultipartUploadUsingScript(t *testing.T) {`
`155`	`155`	`tempdir := t.TempDir()`
`156`	`156`	`bck := cmn.Bck{Name: trand.String(10), Provider: apc.AIS}`
`157`	`157`
`158`		- // 1. set MD5 to satisfy `s3cmd` (for details, see docs/s3cmd.md & docs/s3compat.md)
	`158`	+ // 1. set MD5 to satisfy `s3cmd` (for details, see docs/s3compat.md)
`159`	`159`	`bprops := &cmn.BpropsToSet{`
`160`	`160`	`Cksum: &cmn.CksumConfToSet{Type: apc.Ptr(cos.ChecksumMD5)},`
`161`	`161`	`}`