Prevent huge memory allocations from bad data

pedrocr · pedrocr · commit aaf584b4b10d · 2019-02-22T02:37:56.000Z
If we get bad width/height data for the image bail out instead of doing
a huge allocation that will crash the program.

This was caught using afl-fuzz
diff --git a/src/decoders/arw.rs b/src/decoders/arw.rs
@@ -141,7 +141,7 @@ impl<'a> ArwDecoder<'a> {
 
   fn decode_arw1(buf: &[u8], width: usize, height: usize) -> Vec<u16> {
     let mut pump = BitPumpMSB::new(buf);
-    let mut out: Vec<u16> = vec![0; width*height];
+    let mut out: Vec<u16> = alloc_image!(width, height);
 
     let mut sum: i32 = 0;
     for x in 0..width {
diff --git a/src/decoders/basics.rs b/src/decoders/basics.rs
@@ -83,7 +83,7 @@ pub static LITTLE_ENDIAN: Endian = Endian{big: false};
 pub fn decode_threaded<F>(width: usize, height: usize, closure: &F) -> Vec<u16>
   where F : Fn(&mut [u16], usize)+Sync {
 
-  let mut out: Vec<u16> = vec![0; width*height];
+  let mut out: Vec<u16> = alloc_image!(width, height);
   out.par_chunks_mut(width).enumerate().for_each(|(row, line)| {
     closure(line, row);
   });
@@ -93,7 +93,7 @@ pub fn decode_threaded<F>(width: usize, height: usize, closure: &F) -> Vec<u16>
 pub fn decode_threaded_multiline<F>(width: usize, height: usize, lines: usize, closure: &F) -> Vec<u16>
   where F : Fn(&mut [u16], usize)+Sync {
 
-  let mut out: Vec<u16> = vec![0; width*height];
+  let mut out: Vec<u16> = alloc_image!(width, height);
   out.par_chunks_mut(width*lines).enumerate().for_each(|(row, line)| {
     closure(line, row*lines);
   });
diff --git a/src/decoders/cr2.rs b/src/decoders/cr2.rs
@@ -44,7 +44,7 @@ impl<'a> Decoder for Cr2Decoder<'a> {
       let mut width = ljpegwidth;
       let mut height = decompressor.height();
       let cpp = if decompressor.super_h() == 2 {3} else {1};
-      let mut ljpegout = vec![0 as u16; width*height];
+      let mut ljpegout = alloc_image!(width, height);
       try!(decompressor.decode(&mut ljpegout, 0, width, width, height));
 
       // Linearize the output (applies only to D2000 as far as I can tell)
@@ -87,7 +87,7 @@ impl<'a> Decoder for Cr2Decoder<'a> {
         if canoncol.get_usize(0) == 0 {
           (width, height, cpp, ljpegout)
         } else {
-          let mut out = vec![0 as u16; width*height];
+          let mut out = alloc_image!(width, height);
           let mut fieldwidths = Vec::new();
           for _ in 0..canoncol.get_usize(0) {
             fieldwidths.push(canoncol.get_usize(1));
diff --git a/src/decoders/crw.rs b/src/decoders/crw.rs
@@ -189,7 +189,7 @@ impl<'a> CrwDecoder<'a> {
   }
 
   fn decode_compressed(&self, cam: &Camera, width: usize, height: usize) -> Result<Vec<u16>,String> {
-    let mut out = vec![0 as u16; width*height];
+    let mut out = alloc_image!(width, height);
 
     let dectable = fetch_tag!(self.ciff, CiffTag::DecoderTable).get_usize(0);
     if dectable > 2 {
diff --git a/src/decoders/dcr.rs b/src/decoders/dcr.rs
@@ -61,7 +61,7 @@ impl<'a> DcrDecoder<'a> {
   }
 
   fn decode_kodak65000(buf: &[u8], curve: &LookupTable, width: usize, height: usize) -> Vec<u16> {
-    let mut out: Vec<u16> = vec![0; width*height];
+    let mut out: Vec<u16> = alloc_image!(width, height);
     let mut input = ByteStream::new(buf, LITTLE_ENDIAN);
 
     let mut random: u32 = 0;
diff --git a/src/decoders/dng.rs b/src/decoders/dng.rs
@@ -200,7 +200,7 @@ impl<'a> DngDecoder<'a> {
       }
       let offset = offsets.get_usize(0);
       let src = &self.buffer[offset..];
-      let mut out = vec![0 as u16; width*height];
+      let mut out = alloc_image!(width, height);
       let decompressor = try!(LjpegDecompressor::new(src));
       try!(decompressor.decode(&mut out, 0, width, width, height));
       Ok(out)
diff --git a/src/decoders/kdc.rs b/src/decoders/kdc.rs
@@ -82,7 +82,7 @@ impl<'a> KdcDecoder<'a> {
   }
 
   fn decode_dc120(src: &[u8], width: usize, height: usize) -> Vec<u16> {
-    let mut out = vec![0u16; width*height];
+    let mut out = alloc_image!(width, height);
 
     let mul: [usize;4] = [162, 192, 187,  92];
     let add: [usize;4] = [  0, 636, 424, 212];
diff --git a/src/decoders/mod.rs b/src/decoders/mod.rs
@@ -25,6 +25,18 @@ macro_rules! fetch_ifd {
   );
 }
 
+macro_rules! alloc_image {
+  ($width:expr, $height:expr) => (
+    {
+      if $width * $height > 500000000 {
+        panic!("rawloader: surely there's no such thing as a >500MP image!");
+      }
+      let mut out: Vec<u16> = vec![0; $width * $height];
+      out
+    }
+  );
+}
+
 extern crate toml;
 use self::toml::Value;
 mod image;
diff --git a/src/decoders/mos.rs b/src/decoders/mos.rs
@@ -89,7 +89,7 @@ impl<'a> MosDecoder<'a> {
     let decompressor = try!(LjpegDecompressor::new_full(src, true, true));
     let ljpegout = try!(decompressor.decode_leaf(width, height));
     if cam.find_hint("interlaced") {
-      let mut out = vec![0 as u16; width*height];
+      let mut out = alloc_image!(width, height);
       for (row,line) in ljpegout.chunks_exact(width).enumerate() {
         let orow = if row & 1 == 1 {height-1-row/2} else {row/2};
         out[orow*width .. (orow+1)*width].copy_from_slice(line);
diff --git a/src/decoders/nef.rs b/src/decoders/nef.rs
@@ -285,7 +285,7 @@ impl<'a> NefDecoder<'a> {
     }
     let curve = LookupTable::new(&points[0..max]);
 
-    let mut out = vec![0 as u16; width * height];
+    let mut out = alloc_image!(width, height);
     let mut pump = BitPumpMSB::new(src);
     let mut random = pump.peek_bits(24);
 
diff --git a/src/decoders/orf.rs b/src/decoders/orf.rs
@@ -77,7 +77,7 @@ impl<'a> OrfDecoder<'a> {
    */
 
   pub fn decode_compressed(buf: &'a [u8], width: usize, height: usize) -> Vec<u16> {
-    let mut out: Vec<u16> = vec![0; width*height];
+    let mut out: Vec<u16> = alloc_image!(width, height);
 
     /* Build a table to quickly look up "high" value */
     let mut bittable: [u8; 4096] = [0; 4096];
diff --git a/src/decoders/packed.rs b/src/decoders/packed.rs
@@ -74,7 +74,7 @@ pub fn decode_12be(buf: &[u8], width: usize, height: usize) -> Vec<u16> {
 }
 
 pub fn decode_12be_msb16(buf: &[u8], width: usize, height: usize) -> Vec<u16> {
-  let mut out: Vec<u16> = vec![0; width*height];
+  let mut out: Vec<u16> = alloc_image!(width, height);
 
   for (o, i) in out.chunks_exact_mut(4).zip(buf.chunks_exact(6)) {
     let g1:  u16 = i[ 0] as u16;
@@ -109,7 +109,7 @@ pub fn decode_12le_16bitaligned(buf: &[u8], width: usize, height: usize) -> Vec<
 }
 
 pub fn decode_12be_msb32(buf: &[u8], width: usize, height: usize) -> Vec<u16> {
-  let mut out: Vec<u16> = vec![0; width*height];
+  let mut out: Vec<u16> = alloc_image!(width, height);
 
   for (o, i) in out.chunks_exact_mut(8).zip(buf.chunks_exact(12)) {
     let g1:  u16 = i[ 0] as u16;
diff --git a/src/decoders/pef.rs b/src/decoders/pef.rs
@@ -121,7 +121,7 @@ impl<'a> PefDecoder<'a> {
     let mut pred_left1: i32;
     let mut pred_left2: i32;
 
-    let mut out = vec![0 as u16; width*height];
+    let mut out = alloc_image!(width, height);
     for row in 0..height {
       pred_up1[row & 1] += try!(htable.huff_decode(&mut pump));
       pred_up2[row & 1] += try!(htable.huff_decode(&mut pump));
diff --git a/src/decoders/raf.rs b/src/decoders/raf.rs
@@ -110,7 +110,7 @@ impl<'a> RafDecoder<'a> {
       let rotatedwidth = cropheight + cropwidth/2;
       let rotatedheight = rotatedwidth-1;
 
-      let mut out: Vec<u16> = vec![0; rotatedwidth * rotatedheight];
+      let mut out: Vec<u16> = alloc_image!(rotatedwidth, rotatedheight);
       for row in 0..cropheight {
         let inb = &src[(row+y)*width+x..];
         for col in 0..cropwidth {
@@ -125,7 +125,7 @@ impl<'a> RafDecoder<'a> {
       let rotatedwidth = cropwidth + cropheight/2;
       let rotatedheight = rotatedwidth-1;
 
-      let mut out: Vec<u16> = vec![0; rotatedwidth * rotatedheight];
+      let mut out: Vec<u16> = alloc_image!(rotatedwidth, rotatedheight);
       for row in 0..cropheight {
         let inb = &src[(row+y)*width+x..];
         for col in 0..cropwidth {
diff --git a/src/decoders/srw.rs b/src/decoders/srw.rs
@@ -74,7 +74,7 @@ impl<'a> Decoder for SrwDecoder<'a> {
 
 impl<'a> SrwDecoder<'a> {
   pub fn decode_srw1(buf: &[u8], loffsets: &[u8], width: usize, height: usize) -> Vec<u16> {
-    let mut out: Vec<u16> = vec![0; width*height];
+    let mut out: Vec<u16> = alloc_image!(width, height);
 
     for row in 0..height {
       let mut len: [u32; 4] = [if row < 2 {7} else {4}; 4];
@@ -141,7 +141,7 @@ impl<'a> SrwDecoder<'a> {
   }
 
   pub fn decode_srw2(buf: &[u8], width: usize, height: usize) -> Vec<u16> {
-    let mut out: Vec<u16> = vec![0; width*height];
+    let mut out: Vec<u16> = alloc_image!(width, height);
 
     // This format has a variable length encoding of how many bits are needed
     // to encode the difference between pixels, we use a table to process it
@@ -213,7 +213,7 @@ impl<'a> SrwDecoder<'a> {
     // and Loring von Palleske (Samsung) for pointing to the open-source code of
     // Samsung's DNG converter at https://opensource.samsung.com/
 
-    let mut out: Vec<u16> = vec![0; width*height];
+    let mut out: Vec<u16> = alloc_image!(width, height);
     let mut pump = BitPumpMSB32::new(buf);
 
     // Process the initial metadata bits, we only really use initVal, width and
diff --git a/src/decoders/tfr.rs b/src/decoders/tfr.rs
@@ -47,7 +47,7 @@ impl<'a> TfrDecoder<'a> {
   }
 
   fn decode_compressed(&self, src: &[u8], width: usize, height: usize) -> Result<Vec<u16>,String> {
-    let mut out = vec![0 as u16; width*height];
+    let mut out = alloc_image!(width, height);
     let decompressor = try!(LjpegDecompressor::new_full(src, true, false));
     try!(decompressor.decode(&mut out, 0, width, width, height));
     Ok(out)

Original file line number	Diff line number	Diff line change
`@@ -189,7 +189,7 @@ impl<'a> CrwDecoder<'a> {`
`189`	`189`	`}`
`190`	`190`
`191`	`191`	`fn decode_compressed(&self, cam: &Camera, width: usize, height: usize) -> Result<Vec<u16>,String> {`
`192`		`- let mut out = vec![0 as u16; width*height];`
	`192`	`+ let mut out = alloc_image!(width, height);`
`193`	`193`
`194`	`194`	`let dectable = fetch_tag!(self.ciff, CiffTag::DecoderTable).get_usize(0);`
`195`	`195`	`if dectable > 2 {`
Original file line number	Diff line number	Diff line change
`@@ -61,7 +61,7 @@ impl<'a> DcrDecoder<'a> {`
`61`	`61`	`}`
`62`	`62`
`63`	`63`	`fn decode_kodak65000(buf: &[u8], curve: &LookupTable, width: usize, height: usize) -> Vec<u16> {`
`64`		`- let mut out: Vec<u16> = vec![0; width*height];`
	`64`	`+ let mut out: Vec<u16> = alloc_image!(width, height);`
`65`	`65`	`let mut input = ByteStream::new(buf, LITTLE_ENDIAN);`
`66`	`66`
`67`	`67`	`let mut random: u32 = 0;`
Original file line number	Diff line number	Diff line change
`@@ -200,7 +200,7 @@ impl<'a> DngDecoder<'a> {`
`200`	`200`	`}`
`201`	`201`	`let offset = offsets.get_usize(0);`
`202`	`202`	`let src = &self.buffer[offset..];`
`203`		`- let mut out = vec![0 as u16; width*height];`
	`203`	`+ let mut out = alloc_image!(width, height);`
`204`	`204`	`let decompressor = try!(LjpegDecompressor::new(src));`
`205`	`205`	`try!(decompressor.decode(&mut out, 0, width, width, height));`
`206`	`206`	`Ok(out)`
Original file line number	Diff line number	Diff line change
`@@ -82,7 +82,7 @@ impl<'a> KdcDecoder<'a> {`
`82`	`82`	`}`
`83`	`83`
`84`	`84`	`fn decode_dc120(src: &[u8], width: usize, height: usize) -> Vec<u16> {`
`85`		`- let mut out = vec![0u16; width*height];`
	`85`	`+ let mut out = alloc_image!(width, height);`
`86`	`86`
`87`	`87`	`let mul: [usize;4] = [162, 192, 187, 92];`
`88`	`88`	`let add: [usize;4] = [ 0, 636, 424, 212];`
Original file line number	Diff line number	Diff line change
`@@ -285,7 +285,7 @@ impl<'a> NefDecoder<'a> {`
`285`	`285`	`}`
`286`	`286`	`let curve = LookupTable::new(&points[0..max]);`
`287`	`287`
`288`		`- let mut out = vec![0 as u16; width * height];`
	`288`	`+ let mut out = alloc_image!(width, height);`
`289`	`289`	`let mut pump = BitPumpMSB::new(src);`
`290`	`290`	`let mut random = pump.peek_bits(24);`
`291`	`291`