flate2/gz/bufread.rs
1use crate::io;
2use crate::io::{BufRead, Read, Write};
3use alloc::vec::Vec;
4use core::cmp;
5use core::mem;
6
7use super::{corrupt, read_into, GzBuilder, GzHeader, GzHeaderParser};
8use crate::crc::CrcReader;
9use crate::deflate;
10use crate::Compression;
11
12fn copy(into: &mut [u8], from: &[u8], pos: &mut usize) -> usize {
13 let min = cmp::min(into.len(), from.len() - *pos);
14 into[..min].copy_from_slice(&from[*pos..*pos + min]);
15 *pos += min;
16 min
17}
18
19/// A gzip streaming encoder
20///
21/// This structure implements a [`Read`] interface. When read from, it reads
22/// uncompressed data from the underlying [`BufRead`] and provides the compressed data.
23///
24/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html
25/// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html
26///
27/// # Examples
28///
29/// ```
30/// use std::io::prelude::*;
31/// use std::io;
32/// use flate2::Compression;
33/// use flate2::bufread::GzEncoder;
34/// use std::fs::File;
35/// use std::io::BufReader;
36///
37/// // Opens sample file, compresses the contents and returns a Vector or error
38/// // File wrapped in a BufReader implements BufRead
39///
40/// fn open_hello_world() -> io::Result<Vec<u8>> {
41/// let f = File::open("examples/hello_world.txt")?;
42/// let b = BufReader::new(f);
43/// let mut gz = GzEncoder::new(b, Compression::fast());
44/// let mut buffer = Vec::new();
45/// gz.read_to_end(&mut buffer)?;
46/// Ok(buffer)
47/// }
48/// ```
49#[derive(Debug)]
50pub struct GzEncoder<R> {
51 inner: deflate::bufread::DeflateEncoder<CrcReader<R>>,
52 header: Vec<u8>,
53 pos: usize,
54 eof: bool,
55}
56
57pub fn gz_encoder<R: BufRead>(header: Vec<u8>, r: R, lvl: Compression) -> GzEncoder<R> {
58 let crc = CrcReader::new(r);
59 GzEncoder {
60 inner: deflate::bufread::DeflateEncoder::new(crc, lvl),
61 header,
62 pos: 0,
63 eof: false,
64 }
65}
66
67impl<R: BufRead> GzEncoder<R> {
68 /// Creates a new encoder which will use the given compression level.
69 ///
70 /// The encoder is not configured specially for the emitted header. For
71 /// header configuration, see the `GzBuilder` type.
72 ///
73 /// The data read from the stream `r` will be compressed and available
74 /// through the returned reader.
75 pub fn new(r: R, level: Compression) -> GzEncoder<R> {
76 GzBuilder::new().buf_read(r, level)
77 }
78
79 fn read_footer(&mut self, into: &mut [u8]) -> io::Result<usize> {
80 if self.pos == 8 {
81 return Ok(0);
82 }
83 let crc = self.inner.get_ref().crc();
84 let calced_crc_bytes = crc.sum().to_le_bytes();
85 let arr = [
86 calced_crc_bytes[0],
87 calced_crc_bytes[1],
88 calced_crc_bytes[2],
89 calced_crc_bytes[3],
90 crc.amount() as u8,
91 (crc.amount() >> 8) as u8,
92 (crc.amount() >> 16) as u8,
93 (crc.amount() >> 24) as u8,
94 ];
95 Ok(copy(into, &arr, &mut self.pos))
96 }
97}
98
99impl<R> GzEncoder<R> {
100 /// Acquires a reference to the underlying reader.
101 pub fn get_ref(&self) -> &R {
102 self.inner.get_ref().get_ref()
103 }
104
105 /// Acquires a mutable reference to the underlying reader.
106 ///
107 /// The underlying reader may be mutated as long as its unread input and
108 /// current position are preserved for subsequent reads by this encoder.
109 ///
110 /// To process a new stream, wait for this encoder to reach EOF and create a
111 /// new encoder; replacing the reader directly does not reset it.
112 pub fn get_mut(&mut self) -> &mut R {
113 self.inner.get_mut().get_mut()
114 }
115
116 /// Returns the underlying stream, consuming this encoder
117 pub fn into_inner(self) -> R {
118 self.inner.into_inner().into_inner()
119 }
120}
121
122#[inline]
123fn finish(buf: &[u8; 8]) -> (u32, u32) {
124 let crc = (buf[0] as u32)
125 | ((buf[1] as u32) << 8)
126 | ((buf[2] as u32) << 16)
127 | ((buf[3] as u32) << 24);
128 let amt = (buf[4] as u32)
129 | ((buf[5] as u32) << 8)
130 | ((buf[6] as u32) << 16)
131 | ((buf[7] as u32) << 24);
132 (crc, amt)
133}
134
135impl<R: BufRead> Read for GzEncoder<R> {
136 fn read(&mut self, mut into: &mut [u8]) -> io::Result<usize> {
137 let mut amt = 0;
138 if self.eof {
139 return self.read_footer(into);
140 } else if self.pos < self.header.len() {
141 amt += copy(into, &self.header, &mut self.pos);
142 if amt == into.len() {
143 return Ok(amt);
144 }
145 let tmp = into;
146 into = &mut tmp[amt..];
147 }
148 match self.inner.read(into)? {
149 0 => {
150 self.eof = true;
151 self.pos = 0;
152 self.read_footer(into)
153 }
154 n => Ok(amt + n),
155 }
156 }
157}
158
159impl<R: BufRead + Write> Write for GzEncoder<R> {
160 fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
161 self.get_mut().write(buf)
162 }
163
164 fn flush(&mut self) -> io::Result<()> {
165 self.get_mut().flush()
166 }
167}
168
169/// A decoder for a single member of a [gzip file].
170///
171/// This structure implements a [`Read`] interface. When read from, it reads
172/// compressed data from the underlying [`BufRead`] and provides the uncompressed data.
173///
174/// After reading a single member of the gzip data this reader will return
175/// Ok(0) even if there are more bytes available in the underlying reader.
176/// If you need the following bytes, call `into_inner()` after Ok(0) to
177/// recover the underlying reader.
178///
179/// To handle gzip files that may have multiple members, see [`MultiGzDecoder`]
180/// or read more
181/// [in the introduction](../index.html#about-multi-member-gzip-files).
182///
183/// [gzip file]: https://www.rfc-editor.org/rfc/rfc1952#page-5
184/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html
185/// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html
186///
187/// # Examples
188///
189/// ```
190/// use std::io::prelude::*;
191/// use std::io;
192/// # use flate2::Compression;
193/// # use flate2::write::GzEncoder;
194/// use flate2::bufread::GzDecoder;
195///
196/// # fn main() {
197/// # let mut e = GzEncoder::new(Vec::new(), Compression::default());
198/// # e.write_all(b"Hello World").unwrap();
199/// # let bytes = e.finish().unwrap();
200/// # println!("{}", decode_reader(bytes).unwrap());
201/// # }
202/// #
203/// // Uncompresses a Gz Encoded vector of bytes and returns a string or error
204/// // Here &[u8] implements BufRead
205///
206/// fn decode_reader(bytes: Vec<u8>) -> io::Result<String> {
207/// let mut gz = GzDecoder::new(&bytes[..]);
208/// let mut s = String::new();
209/// gz.read_to_string(&mut s)?;
210/// Ok(s)
211/// }
212/// ```
213#[derive(Debug)]
214pub struct GzDecoder<R> {
215 state: GzState,
216 reader: CrcReader<deflate::bufread::DeflateDecoder<R>>,
217 multi: bool,
218}
219
220#[derive(Debug)]
221enum GzState {
222 Header(GzHeaderParser),
223 Body(GzHeader),
224 Finished(GzHeader, usize, [u8; 8]),
225 Err(io::Error),
226 End(Option<GzHeader>),
227}
228
229pub fn reset_decoder_data<R>(decoder: &mut GzDecoder<R>) {
230 decoder.state = GzState::Header(GzHeaderParser::new());
231 decoder.reader.reset(); // reset CrcReader
232 decoder.reader.get_mut().reset_data(); // reset DeflateDecoder
233}
234
235impl<R: BufRead> GzDecoder<R> {
236 /// Creates a new decoder from the given reader, immediately parsing the
237 /// gzip header.
238 pub fn new(mut r: R) -> GzDecoder<R> {
239 let mut header_parser = GzHeaderParser::new();
240
241 let state = match header_parser.parse(&mut r) {
242 Ok(_) => GzState::Body(GzHeader::from(header_parser)),
243 Err(ref err) if io::ErrorKind::WouldBlock == err.kind() => {
244 GzState::Header(header_parser)
245 }
246 Err(err) => GzState::Err(err),
247 };
248
249 GzDecoder {
250 state,
251 reader: CrcReader::new(deflate::bufread::DeflateDecoder::new(r)),
252 multi: false,
253 }
254 }
255
256 fn multi(mut self, flag: bool) -> GzDecoder<R> {
257 self.multi = flag;
258 self
259 }
260}
261
262impl<R> GzDecoder<R> {
263 /// Returns the header associated with this stream, if it was valid
264 pub fn header(&self) -> Option<&GzHeader> {
265 match &self.state {
266 GzState::Body(header) | GzState::Finished(header, _, _) => Some(header),
267 GzState::End(header) => header.as_ref(),
268 _ => None,
269 }
270 }
271
272 /// Acquires a reference to the underlying reader.
273 pub fn get_ref(&self) -> &R {
274 self.reader.get_ref().get_ref()
275 }
276
277 /// Acquires a mutable reference to the underlying stream.
278 ///
279 /// The underlying reader may be mutated as long as its unread input and
280 /// current position are preserved for subsequent reads by this decoder.
281 ///
282 /// To process a new stream, wait for this decoder to reach EOF and use
283 /// [`reset`](Self::reset); replacing the reader directly does not reset it.
284 pub fn get_mut(&mut self) -> &mut R {
285 self.reader.get_mut().get_mut()
286 }
287
288 /// Consumes this decoder, returning the underlying reader.
289 pub fn into_inner(self) -> R {
290 self.reader.into_inner().into_inner()
291 }
292
293 /// Resets the state of this decoder entirely, swapping out the input
294 /// stream for another.
295 ///
296 /// This will reset the internal state of this decoder and replace the
297 /// input stream with the one provided, returning the previous input
298 /// stream. Future data read from this decoder will be the decompressed
299 /// version of `r`'s data.
300 pub fn reset(&mut self, r: R) -> R {
301 reset_decoder_data(self);
302 self.reader.get_mut().reset(r)
303 }
304}
305
306impl<R: BufRead> Read for GzDecoder<R> {
307 fn read(&mut self, into: &mut [u8]) -> io::Result<usize> {
308 loop {
309 match &mut self.state {
310 GzState::Header(parser) => {
311 parser.parse(self.reader.get_mut().get_mut())?;
312 self.state = GzState::Body(GzHeader::from(mem::take(parser)));
313 }
314 GzState::Body(header) => {
315 if into.is_empty() {
316 return Ok(0);
317 }
318 match self.reader.read(into)? {
319 0 => {
320 self.state = GzState::Finished(mem::take(header), 0, [0; 8]);
321 }
322 n => {
323 return Ok(n);
324 }
325 }
326 }
327 GzState::Finished(header, pos, buf) => {
328 if *pos < buf.len() {
329 *pos += read_into(self.reader.get_mut().get_mut(), &mut buf[*pos..])?;
330 } else {
331 let (crc, amt) = finish(buf);
332
333 if crc != self.reader.crc().sum() || amt != self.reader.crc().amount() {
334 self.state = GzState::End(Some(mem::take(header)));
335 return Err(corrupt());
336 } else if self.multi {
337 let is_eof = self
338 .reader
339 .get_mut()
340 .get_mut()
341 .fill_buf()
342 .map(|buf| buf.is_empty())?;
343
344 if is_eof {
345 self.state = GzState::End(Some(mem::take(header)));
346 } else {
347 self.reader.reset();
348 self.reader.get_mut().reset_data();
349 self.state = GzState::Header(GzHeaderParser::new())
350 }
351 } else {
352 self.state = GzState::End(Some(mem::take(header)));
353 }
354 }
355 }
356 GzState::Err(err) => {
357 let result = Err(mem::replace(err, io::ErrorKind::Other.into()));
358 self.state = GzState::End(None);
359 return result;
360 }
361 GzState::End(_) => return Ok(0),
362 }
363 }
364 }
365}
366
367impl<R: BufRead + Write> Write for GzDecoder<R> {
368 fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
369 self.get_mut().write(buf)
370 }
371
372 fn flush(&mut self) -> io::Result<()> {
373 self.get_mut().flush()
374 }
375}
376
377/// A gzip streaming decoder that decodes a [gzip file] that may have multiple members.
378///
379/// This structure implements a [`Read`] interface. When read from, it reads
380/// compressed data from the underlying [`BufRead`] and provides the uncompressed data.
381///
382/// A gzip file consists of a series of *members* concatenated one after another.
383/// MultiGzDecoder decodes all members from the data and only returns Ok(0) when the
384/// underlying reader does. For a file, this reads to the end of the file.
385///
386/// To handle members separately, see [GzDecoder] or read more
387/// [in the introduction](../index.html#about-multi-member-gzip-files).
388///
389/// [gzip file]: https://www.rfc-editor.org/rfc/rfc1952#page-5
390/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html
391/// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html
392///
393/// # Examples
394///
395/// ```
396/// use std::io::prelude::*;
397/// use std::io;
398/// # use flate2::Compression;
399/// # use flate2::write::GzEncoder;
400/// use flate2::bufread::MultiGzDecoder;
401///
402/// # fn main() {
403/// # let mut e = GzEncoder::new(Vec::new(), Compression::default());
404/// # e.write_all(b"Hello World").unwrap();
405/// # let bytes = e.finish().unwrap();
406/// # println!("{}", decode_reader(bytes).unwrap());
407/// # }
408/// #
409/// // Uncompresses a Gz Encoded vector of bytes and returns a string or error
410/// // Here &[u8] implements BufRead
411///
412/// fn decode_reader(bytes: Vec<u8>) -> io::Result<String> {
413/// let mut gz = MultiGzDecoder::new(&bytes[..]);
414/// let mut s = String::new();
415/// gz.read_to_string(&mut s)?;
416/// Ok(s)
417/// }
418/// ```
419#[derive(Debug)]
420pub struct MultiGzDecoder<R>(GzDecoder<R>);
421
422impl<R: BufRead> MultiGzDecoder<R> {
423 /// Creates a new decoder from the given reader, immediately parsing the
424 /// (first) gzip header. If the gzip stream contains multiple members all will
425 /// be decoded.
426 pub fn new(r: R) -> MultiGzDecoder<R> {
427 MultiGzDecoder(GzDecoder::new(r).multi(true))
428 }
429}
430
431impl<R> MultiGzDecoder<R> {
432 /// Returns the current header associated with this stream, if it's valid
433 pub fn header(&self) -> Option<&GzHeader> {
434 self.0.header()
435 }
436
437 /// Acquires a reference to the underlying reader.
438 pub fn get_ref(&self) -> &R {
439 self.0.get_ref()
440 }
441
442 /// Acquires a mutable reference to the underlying stream.
443 ///
444 /// The underlying reader may be mutated as long as its unread input and
445 /// current position are preserved for subsequent reads by this decoder.
446 ///
447 /// To process a new stream, wait for this decoder to reach EOF and create a
448 /// new decoder; replacing the reader directly does not reset it.
449 pub fn get_mut(&mut self) -> &mut R {
450 self.0.get_mut()
451 }
452
453 /// Consumes this decoder, returning the underlying reader.
454 pub fn into_inner(self) -> R {
455 self.0.into_inner()
456 }
457}
458
459impl<R: BufRead> Read for MultiGzDecoder<R> {
460 fn read(&mut self, into: &mut [u8]) -> io::Result<usize> {
461 self.0.read(into)
462 }
463}
464
465#[cfg(test)]
466mod test {
467 use crate::bufread::GzDecoder;
468 use crate::gz::write;
469 use crate::io::{Read, Write};
470 use crate::Compression;
471 use alloc::vec::Vec;
472
473 // GzDecoder consumes one gzip member and then returns 0 for subsequent reads, allowing any
474 // additional data to be consumed by the caller.
475 #[test]
476 fn decode_extra_data() {
477 let expected = "Hello World";
478
479 let compressed = {
480 let mut e = write::GzEncoder::new(Vec::new(), Compression::default());
481 e.write_all(expected.as_ref()).unwrap();
482 let mut b = e.finish().unwrap();
483 b.push(b'x');
484 b
485 };
486
487 let mut output = Vec::new();
488 let mut decoder = GzDecoder::new(compressed.as_slice());
489 let decoded_bytes = decoder.read_to_end(&mut output).unwrap();
490 assert_eq!(decoded_bytes, output.len());
491 let actual = core::str::from_utf8(&output).expect("String parsing error");
492 assert_eq!(
493 actual, expected,
494 "after decompression we obtain the original input"
495 );
496
497 output.clear();
498 assert_eq!(
499 decoder.read(&mut output).unwrap(),
500 0,
501 "subsequent read of decoder returns 0, but inner reader can return additional data"
502 );
503 let mut reader = decoder.into_inner();
504 assert_eq!(
505 reader.read_to_end(&mut output).unwrap(),
506 1,
507 "extra data is accessible in underlying buf-read"
508 );
509 assert_eq!(output, b"x");
510 }
511
512 fn compress_data(data: &[u8]) -> Vec<u8> {
513 use crate::write::GzEncoder;
514 use crate::Compression;
515
516 let mut e = GzEncoder::new(Vec::new(), Compression::default());
517 e.write_all(data).unwrap();
518 e.finish().unwrap()
519 }
520
521 #[test]
522 fn decode_with_reset() {
523 let data1 = b"Hello World";
524 let data2 = b"Goodbye World";
525
526 let compressed1 = compress_data(data1);
527 let compressed2 = compress_data(data2);
528
529 let mut output = Vec::new();
530 let mut decoder = GzDecoder::new(compressed1.as_slice());
531 decoder.read_to_end(&mut output).unwrap();
532 assert_eq!(output, data1);
533
534 output.clear();
535 decoder.reset(compressed2.as_slice());
536 decoder.read_to_end(&mut output).unwrap();
537 assert_eq!(output, data2);
538 }
539
540 #[test]
541 fn decode_with_reset_after_corruption() {
542 let valid_data = b"Hello World";
543 let valid_compressed = compress_data(valid_data);
544
545 // Create a corrupted payload (valid gzip header but corrupted body)
546 let mut corrupted = valid_compressed.clone();
547 assert!(corrupted.len() >= 14);
548 corrupted[12] ^= 0xFF;
549 corrupted[13] ^= 0xFF;
550
551 // Try to decode corrupted data
552 let mut decoder = GzDecoder::new(corrupted.as_slice());
553 let mut output = Vec::new();
554 let _ = decoder.read_to_end(&mut output).unwrap_err();
555
556 // Reset with valid payload and decode
557 decoder.reset(valid_compressed.as_slice());
558 output.clear();
559 decoder.read_to_end(&mut output).unwrap();
560 assert_eq!(output, valid_data);
561 }
562}