feat(objectarium): implement Snappy compression algorithm

ccamel · ccamel · commit 29587308b824 · 2023-05-02T19:41:20.000+02:00
diff --git a/contracts/okp4-objectarium/src/compress.rs b/contracts/okp4-objectarium/src/compress.rs
@@ -1,7 +1,9 @@
+use std::io;
+
 use enum_iterator::Sequence;
-use lz4_flex;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
+use snap;
 use thiserror::Error;
 
 /// CompressionAlgorithm is an enumeration that defines the different compression algorithms
@@ -10,16 +12,16 @@ use thiserror::Error;
 pub enum CompressionAlgorithm {
     /// Represents the "No compression" algorithm.
     Passthrough,
-    /// Represents the LZ4 algorithm.
-    Lz4,
+    /// Represents the Snappy algorithm.
+    Snappy,
 }
 
 impl CompressionAlgorithm {
     /// compress returns the compressed data using the given algorithm.
     pub fn compress(&self, data: &[u8]) -> Result<Vec<u8>, CompressionError> {
         let compressor = match self {
             CompressionAlgorithm::Passthrough => passthrough,
-            CompressionAlgorithm::Lz4 => lz4_compress,
+            CompressionAlgorithm::Snappy => snappy_compress,
         };
         compressor(data)
     }
@@ -29,7 +31,7 @@ impl CompressionAlgorithm {
     pub fn decompress(&self, data: &[u8]) -> Result<Vec<u8>, CompressionError> {
         let decompressor = match self {
             CompressionAlgorithm::Passthrough => passthrough,
-            CompressionAlgorithm::Lz4 => lz4_decompress,
+            CompressionAlgorithm::Snappy => snappy_decompress,
         };
         decompressor(data)
     }
@@ -41,8 +43,8 @@ pub enum CompressionError {
     Error(String),
 }
 
-impl From<lz4_flex::block::DecompressError> for CompressionError {
-    fn from(err: lz4_flex::block::DecompressError) -> Self {
+impl From<std::io::Error> for CompressionError {
+    fn from(err: std::io::Error) -> Self {
         CompressionError::Error(err.to_string())
     }
 }
@@ -53,37 +55,45 @@ fn passthrough(data: &[u8]) -> Result<Vec<u8>, CompressionError> {
     Ok(data.to_vec())
 }
 
-// lz4_compress returns the LZ4 compressed data.
+// snappy_compress returns the Snappy compressed data.
 #[inline]
-fn lz4_compress(data: &[u8]) -> Result<Vec<u8>, CompressionError> {
-    Ok(lz4_flex::compress_prepend_size(data))
+fn snappy_compress(data: &[u8]) -> Result<Vec<u8>, CompressionError> {
+    let mut reader = io::Cursor::new(data);
+    let mut writer = Vec::new();
+    {
+        let mut snappy_writer = snap::write::FrameEncoder::new(&mut writer);
+        io::copy(&mut reader, &mut snappy_writer)?;
+    }
+    Ok(writer)
 }
 
-// lz4_decompress returns the LZ4 decompressed data.
+// snappy_decompress returns the Snappy decompressed data.
 #[inline]
-fn lz4_decompress(data: &[u8]) -> Result<Vec<u8>, CompressionError> {
-    Ok(lz4_flex::decompress_size_prepended(data)?)
+fn snappy_decompress(data: &[u8]) -> Result<Vec<u8>, CompressionError> {
+    let reader = io::Cursor::new(data);
+    let mut snappy_reader = snap::read::FrameDecoder::new(reader);
+    let mut writer = Vec::new();
+    io::copy(&mut snappy_reader, &mut writer)?;
+    Ok(writer)
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
 
     #[test]
-    fn test_from_lz4_flex_decompress_error() {
-        // Define test cases as tuples of input DecompressError and expected CompressionError
-        let test_cases = vec![(
-            lz4_flex::block::DecompressError::UncompressedSizeDiffers {
-                expected: 1000,
-                actual: 998,
-            },
+    fn test_from_io_decompress_error() {
+        let cases = vec![(
+            std::io::Error::new(
+                std::io::ErrorKind::InvalidData,
+                "the expected decompressed size differs, actual 998, expected 1000",
+            ),
             CompressionError::Error(
                 "the expected decompressed size differs, actual 998, expected 1000".to_string(),
             ),
         )];
 
-        // Iterate over test cases using a for loop
-        for (error, expected_error) in test_cases {
+        for (error, expected_error) in cases {
             let compression_err = CompressionError::from(error);
 
             assert_eq!(compression_err, expected_error);
diff --git a/contracts/okp4-objectarium/src/contract.rs b/contracts/okp4-objectarium/src/contract.rs
@@ -448,7 +448,7 @@ mod tests {
             BucketLimits {
                 accepted_compression_algorithms: Some(vec![
                     CompressionAlgorithm::Passthrough,
-                    CompressionAlgorithm::Lz4,
+                    CompressionAlgorithm::Snappy,
                 ]),
                 ..Default::default()
             }
@@ -989,10 +989,10 @@ mod tests {
             },
             TC {
                 accepted_compression_algorithms: None,
-                compression_algorithm: Some(CompressionAlgorithm::Lz4),
+                compression_algorithm: Some(CompressionAlgorithm::Snappy),
                 expected_result: Either::Right(ExpectedCompressionResult {
-                    compression_algorithm: CompressionAlgorithm::Lz4,
-                    compressed_size: 415,
+                    compression_algorithm: CompressionAlgorithm::Snappy,
+                    compressed_size: 414,
                 }),
             },
             TC {
@@ -1006,21 +1006,21 @@ mod tests {
             TC {
                 accepted_compression_algorithms: Some(vec![
                     CompressionAlgorithm::Passthrough,
-                    CompressionAlgorithm::Lz4,
+                    CompressionAlgorithm::Snappy,
                 ]),
-                compression_algorithm: Some(CompressionAlgorithm::Lz4),
+                compression_algorithm: Some(CompressionAlgorithm::Snappy),
                 expected_result: Either::Right(ExpectedCompressionResult {
-                    compression_algorithm: CompressionAlgorithm::Lz4,
-                    compressed_size: 415,
+                    compression_algorithm: CompressionAlgorithm::Snappy,
+                    compressed_size: 414,
                 }),
             },
             TC {
-                accepted_compression_algorithms: Some(vec![CompressionAlgorithm::Lz4]),
+                accepted_compression_algorithms: Some(vec![CompressionAlgorithm::Snappy]),
                 compression_algorithm: Some(CompressionAlgorithm::Passthrough),
                 expected_result: Either::Left(ContractError::Bucket(
                     BucketError::CompressionAlgorithmNotAccepted(
                         CompressionAlgorithm::Passthrough,
-                        vec![CompressionAlgorithm::Lz4],
+                        vec![CompressionAlgorithm::Snappy],
                     ),
                 )),
             },
diff --git a/contracts/okp4-objectarium/src/error.rs b/contracts/okp4-objectarium/src/error.rs
@@ -88,10 +88,10 @@ fn test_bucket_error_messages() {
         ),
         (
             ContractError::Bucket(BucketError::CompressionAlgorithmNotAccepted(
-                CompressionAlgorithm::Lz4,
+                CompressionAlgorithm::Snappy,
                 vec![CompressionAlgorithm::Passthrough],
             )),
-            "Compression algorithm is not accepted: Lz4 (accepted: \"[Passthrough]\")",
+            "Compression algorithm is not accepted: Snappy (accepted: \"[Passthrough]\")",
         ),
         (ContractError::ObjectAlreadyPinned {}, "Object is already pinned"),
         (
diff --git a/contracts/okp4-objectarium/src/msg.rs b/contracts/okp4-objectarium/src/msg.rs
@@ -160,15 +160,13 @@ pub enum CompressionAlgorithm {
     /// Represents no compression algorithm.
     /// The object is stored as is without any compression.
     Passthrough,
-    /// # LZ4
-    /// Represents the LZ4 algorithm.
-    /// LZ4 is a lossless compression algorithm that is used to compress objects.
-    /// It is extremely fast to compress and decompress, and is suitable for real-time compression.
-    /// The compression ratio is relatively low, but the computational cost is also low, so it is a good choice
-    /// for compressing data that is already highly compressible.
+    /// # Snappy
+    /// Represents the Snappy algorithm.
+    /// Snappy is a compression/decompression algorithm that does not aim for maximum compression. Instead, it aims for very high speeds and reasonable
+    /// compression.
     ///
-    /// See [the LZ4 Wikipedia page](https://en.wikipedia.org/wiki/LZ4_(compression_algorithm)) for more information.
-    Lz4,
+    /// See [the snappy web page](https://google.github.io/snappy/) for more information.
+    Snappy,
 }
 
 /// HashAlgorithm is an enumeration that defines the different hash algorithms
diff --git a/contracts/okp4-objectarium/src/state.rs b/contracts/okp4-objectarium/src/state.rs
@@ -115,7 +115,7 @@ impl From<msg::CompressionAlgorithm> for CompressionAlgorithm {
     fn from(algorithm: msg::CompressionAlgorithm) -> Self {
         match algorithm {
             msg::CompressionAlgorithm::Passthrough => CompressionAlgorithm::Passthrough,
-            msg::CompressionAlgorithm::Lz4 => CompressionAlgorithm::Lz4,
+            msg::CompressionAlgorithm::Snappy => CompressionAlgorithm::Snappy,
         }
     }
 }
@@ -124,7 +124,7 @@ impl From<CompressionAlgorithm> for msg::CompressionAlgorithm {
     fn from(algorithm: CompressionAlgorithm) -> Self {
         match algorithm {
             CompressionAlgorithm::Passthrough => msg::CompressionAlgorithm::Passthrough,
-            CompressionAlgorithm::Lz4 => msg::CompressionAlgorithm::Lz4,
+            CompressionAlgorithm::Snappy => msg::CompressionAlgorithm::Snappy,
         }
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -88,10 +88,10 @@ fn test_bucket_error_messages() {`
`88`	`88`	`),`
`89`	`89`	`(`
`90`	`90`	`ContractError::Bucket(BucketError::CompressionAlgorithmNotAccepted(`
`91`		`- CompressionAlgorithm::Lz4,`
	`91`	`+ CompressionAlgorithm::Snappy,`
`92`	`92`	`vec![CompressionAlgorithm::Passthrough],`
`93`	`93`	`)),`
`94`		`- "Compression algorithm is not accepted: Lz4 (accepted: \"[Passthrough]\")",`
	`94`	`+ "Compression algorithm is not accepted: Snappy (accepted: \"[Passthrough]\")",`
`95`	`95`	`),`
`96`	`96`	`(ContractError::ObjectAlreadyPinned {}, "Object is already pinned"),`
`97`	`97`	`(`
Original file line number	Diff line number	Diff line change
`@@ -115,7 +115,7 @@ impl From<msg::CompressionAlgorithm> for CompressionAlgorithm {`
`115`	`115`	`fn from(algorithm: msg::CompressionAlgorithm) -> Self {`
`116`	`116`	`match algorithm {`
`117`	`117`	`msg::CompressionAlgorithm::Passthrough => CompressionAlgorithm::Passthrough,`
`118`		`- msg::CompressionAlgorithm::Lz4 => CompressionAlgorithm::Lz4,`
	`118`	`+ msg::CompressionAlgorithm::Snappy => CompressionAlgorithm::Snappy,`
`119`	`119`	`}`
`120`	`120`	`}`
`121`	`121`	`}`
`@@ -124,7 +124,7 @@ impl From<CompressionAlgorithm> for msg::CompressionAlgorithm {`
`124`	`124`	`fn from(algorithm: CompressionAlgorithm) -> Self {`
`125`	`125`	`match algorithm {`
`126`	`126`	`CompressionAlgorithm::Passthrough => msg::CompressionAlgorithm::Passthrough,`
`127`		`- CompressionAlgorithm::Lz4 => msg::CompressionAlgorithm::Lz4,`
	`127`	`+ CompressionAlgorithm::Snappy => msg::CompressionAlgorithm::Snappy,`
`128`	`128`	`}`
`129`	`129`	`}`
`130`	`130`	`}`