brotli/enc/
mod.rs

Help
1#[macro_use]
2pub mod vectorization;
3pub mod backward_references;
4pub mod bit_cost;
5pub mod block_split;
6pub mod block_splitter;
7pub mod brotli_bit_stream;
8pub mod cluster;
9pub mod combined_alloc;
10pub mod command;
11mod compat;
12pub mod compress_fragment;
13pub mod compress_fragment_two_pass;
14pub mod constants;
15pub mod context_map_entropy;
16pub mod dictionary_hash;
17pub mod encode;
18pub mod entropy_encode;
19pub mod find_stride;
20pub mod fixed_queue;
21pub mod histogram;
22pub mod input_pair;
23pub mod interface;
24pub mod ir_interpret;
25pub mod literal_cost;
26mod log_table_16;
27mod log_table_8;
28pub mod metablock;
29pub mod multithreading;
30mod parameters;
31pub mod pdf;
32pub mod prior_eval;
33pub mod reader;
34pub mod singlethreading;
35pub mod static_dict;
36pub mod static_dict_lut;
37pub mod stride_eval;
38mod test;
39pub mod threading;
40pub mod utf8_util;
41pub mod util;
42mod weights;
43pub mod worker_pool;
44pub mod writer;
45
46// FIXME: Remove this in 7.0
47#[deprecated(note = "Leaving this here to avoid breaking 6.0 compatibility.")]
48pub mod fast_log {}
49
50pub use alloc::{AllocatedStackMemory, Allocator, SliceWrapper, SliceWrapperMut, StackAllocator};
51#[cfg(feature = "std")]
52use std::io;
53#[cfg(feature = "std")]
54use std::io::{Error, ErrorKind, Read, Write};
55
56#[cfg(feature = "std")]
57pub use alloc_stdlib::StandardAlloc;
58use brotli_decompressor::{CustomRead, CustomWrite};
59#[cfg(feature = "std")]
60pub use brotli_decompressor::{IntoIoReader, IoReaderWrapper, IoWriterWrapper};
61pub use interface::{InputPair, InputReference, InputReferenceMut};
62
63pub use self::backward_references::{
64    hash_to_binary_tree, hq as backward_references_hq, BrotliEncoderParams, UnionHasher,
65};
66pub use self::combined_alloc::{BrotliAlloc, CombiningAllocator};
67use self::encode::{BrotliEncoderDestroyInstance, BrotliEncoderOperation};
68pub use self::encode::{
69    BrotliEncoderInitParams, BrotliEncoderMaxCompressedSize, BrotliEncoderMaxCompressedSizeMulti,
70};
71pub use self::hash_to_binary_tree::ZopfliNode;
72pub use self::interface::StaticCommand;
73pub use self::pdf::PDF;
74#[cfg(not(feature = "std"))]
75pub use self::singlethreading::{compress_worker_pool, new_work_pool, WorkerPool};
76pub use self::threading::{
77    BatchSpawnableLite, BrotliEncoderThreadError, CompressionThreadResult, Owned, SendAlloc,
78};
79pub use self::util::floatX;
80pub use self::vectorization::{v256, v256i, Mem256f};
81#[cfg(feature = "std")]
82pub use self::worker_pool::{compress_worker_pool, new_work_pool, WorkerPool};
83use crate::enc::encode::BrotliEncoderStateStruct;
84
85#[cfg(feature = "simd")]
86pub type s16 = core::simd::i16x16;
87#[cfg(feature = "simd")]
88pub type v8 = core::simd::f32x8;
89#[cfg(feature = "simd")]
90pub type s8 = core::simd::i32x8;
91#[cfg(not(feature = "simd"))]
92pub type s16 = compat::Compat16x16;
93#[cfg(not(feature = "simd"))]
94pub type v8 = compat::CompatF8;
95#[cfg(not(feature = "simd"))]
96pub type s8 = compat::Compat32x8;
97
98#[cfg(feature = "std")]
99pub fn compress_multi<
100    Alloc: BrotliAlloc + Send + 'static,
101    SliceW: SliceWrapper<u8> + Send + 'static + Sync,
102>(
103    params: &BrotliEncoderParams,
104    owned_input: &mut Owned<SliceW>,
105    output: &mut [u8],
106    alloc_per_thread: &mut [SendAlloc<
107        CompressionThreadResult<Alloc>,
108        backward_references::UnionHasher<Alloc>,
109        Alloc,
110        <WorkerPool<
111            CompressionThreadResult<Alloc>,
112            backward_references::UnionHasher<Alloc>,
113            Alloc,
114            (SliceW, BrotliEncoderParams),
115        > as BatchSpawnableLite<
116            CompressionThreadResult<Alloc>,
117            backward_references::UnionHasher<Alloc>,
118            Alloc,
119            (SliceW, BrotliEncoderParams),
120        >>::JoinHandle,
121    >],
122) -> Result<usize, BrotliEncoderThreadError>
123where
124    <Alloc as Allocator<u8>>::AllocatedMemory: Send,
125    <Alloc as Allocator<u16>>::AllocatedMemory: Send + Sync,
126    <Alloc as Allocator<u32>>::AllocatedMemory: Send + Sync,
127{
128    let mut work_pool = self::worker_pool::new_work_pool(alloc_per_thread.len() - 1);
129    compress_worker_pool(
130        params,
131        owned_input,
132        output,
133        alloc_per_thread,
134        &mut work_pool,
135    )
136}
137
138#[cfg(feature = "std")]
139pub use self::multithreading::compress_multi as compress_multi_no_threadpool;
140#[cfg(not(feature = "std"))]
141pub use self::singlethreading::compress_multi;
142#[cfg(not(feature = "std"))]
143pub use self::singlethreading::compress_multi as compress_multi_no_threadpool;
144
145#[cfg(feature = "std")]
146pub fn BrotliCompress<InputType, OutputType>(
147    r: &mut InputType,
148    w: &mut OutputType,
149    params: &BrotliEncoderParams,
150) -> Result<usize, io::Error>
151where
152    InputType: Read,
153    OutputType: Write,
154{
155    let mut input_buffer: [u8; 4096] = [0; 4096];
156    let mut output_buffer: [u8; 4096] = [0; 4096];
157    BrotliCompressCustomAlloc(
158        r,
159        w,
160        &mut input_buffer[..],
161        &mut output_buffer[..],
162        params,
163        StandardAlloc::default(),
164    )
165}
166
167#[cfg(feature = "std")]
168pub fn BrotliCompressCustomAlloc<InputType, OutputType, Alloc: BrotliAlloc>(
169    r: &mut InputType,
170    w: &mut OutputType,
171    input_buffer: &mut [u8],
172    output_buffer: &mut [u8],
173    params: &BrotliEncoderParams,
174    alloc: Alloc,
175) -> Result<usize, io::Error>
176where
177    InputType: Read,
178    OutputType: Write,
179{
180    let mut nop_callback = |_data: &mut interface::PredictionModeContextMap<InputReferenceMut>,
181                            _cmds: &mut [interface::StaticCommand],
182                            _mb: interface::InputPair,
183                            _m: &mut Alloc| ();
184    BrotliCompressCustomIo(
185        &mut IoReaderWrapper::<InputType>(r),
186        &mut IoWriterWrapper::<OutputType>(w),
187        input_buffer,
188        output_buffer,
189        params,
190        alloc,
191        &mut nop_callback,
192        Error::new(ErrorKind::UnexpectedEof, "Unexpected EOF"),
193    )
194}
195
196pub fn BrotliCompressCustomIo<
197    ErrType,
198    InputType,
199    OutputType,
200    Alloc: BrotliAlloc,
201    MetablockCallback: FnMut(
202        &mut interface::PredictionModeContextMap<InputReferenceMut>,
203        &mut [interface::StaticCommand],
204        interface::InputPair,
205        &mut Alloc,
206    ),
207>(
208    r: &mut InputType,
209    w: &mut OutputType,
210    input_buffer: &mut [u8],
211    output_buffer: &mut [u8],
212    params: &BrotliEncoderParams,
213    alloc: Alloc,
214    metablock_callback: &mut MetablockCallback,
215    unexpected_eof_error_constant: ErrType,
216) -> Result<usize, ErrType>
217where
218    InputType: CustomRead<ErrType>,
219    OutputType: CustomWrite<ErrType>,
220{
221    BrotliCompressCustomIoCustomDict(
222        r,
223        w,
224        input_buffer,
225        output_buffer,
226        params,
227        alloc,
228        metablock_callback,
229        &[],
230        unexpected_eof_error_constant,
231    )
232}
233pub fn BrotliCompressCustomIoCustomDict<
234    ErrType,
235    InputType,
236    OutputType,
237    Alloc: BrotliAlloc,
238    MetablockCallback: FnMut(
239        &mut interface::PredictionModeContextMap<InputReferenceMut>,
240        &mut [interface::StaticCommand],
241        interface::InputPair,
242        &mut Alloc,
243    ),
244>(
245    r: &mut InputType,
246    w: &mut OutputType,
247    input_buffer: &mut [u8],
248    output_buffer: &mut [u8],
249    params: &BrotliEncoderParams,
250    alloc: Alloc,
251    metablock_callback: &mut MetablockCallback,
252    dict: &[u8],
253    unexpected_eof_error_constant: ErrType,
254) -> Result<usize, ErrType>
255where
256    InputType: CustomRead<ErrType>,
257    OutputType: CustomWrite<ErrType>,
258{
259    assert!(!input_buffer.is_empty());
260    assert!(!output_buffer.is_empty());
261    let mut s_orig = BrotliEncoderStateStruct::new(alloc);
262    s_orig.params = params.clone();
263    if !dict.is_empty() {
264        s_orig.set_custom_dictionary(dict.len(), dict);
265    }
266    let mut next_in_offset: usize = 0;
267    let mut next_out_offset: usize = 0;
268    let mut total_out = Some(0);
269    let mut read_err: Result<(), ErrType> = Ok(());
270    {
271        let s = &mut s_orig;
272
273        //BrotliEncoderSetParameter(s, BrotliEncoderParameter::BROTLI_PARAM_MODE, 0 as u32); // gen, text, font
274        //BrotliEncoderSetParameter(s,
275        //                          BrotliEncoderParameter::BROTLI_PARAM_SIZE_HINT,
276        //                          input.len() as u32);
277        let mut available_in: usize = 0;
278        let mut available_out: usize = output_buffer.len();
279        let mut eof = false;
280        loop {
281            if available_in == 0 && !eof {
282                next_in_offset = 0;
283                match r.read(input_buffer) {
284                    Err(e) => {
285                        read_err = Err(e);
286                        available_in = 0;
287                        eof = true;
288                    }
289                    Ok(size) => {
290                        if size == 0 {
291                            eof = true;
292                        }
293                        available_in = size;
294                    }
295                }
296            }
297            let op: BrotliEncoderOperation;
298            if available_in == 0 {
299                op = BrotliEncoderOperation::BROTLI_OPERATION_FINISH;
300            } else {
301                op = BrotliEncoderOperation::BROTLI_OPERATION_PROCESS;
302            }
303            let result = s.compress_stream(
304                op,
305                &mut available_in,
306                input_buffer,
307                &mut next_in_offset,
308                &mut available_out,
309                output_buffer,
310                &mut next_out_offset,
311                &mut total_out,
312                metablock_callback,
313            );
314            let fin = s.is_finished();
315            if available_out == 0 || fin {
316                let lim = output_buffer.len() - available_out;
317                assert_eq!(next_out_offset, lim);
318                next_out_offset = 0;
319                while next_out_offset < lim {
320                    match w.write(&mut output_buffer[next_out_offset..lim]) {
321                        Err(e) => {
322                            BrotliEncoderDestroyInstance(s);
323                            read_err?;
324                            return Err(e);
325                        }
326                        Ok(size) => {
327                            next_out_offset += size;
328                        }
329                    }
330                }
331                available_out = output_buffer.len();
332                next_out_offset = 0;
333            }
334            if !result {
335                if read_err.is_ok() {
336                    read_err = Err(unexpected_eof_error_constant);
337                }
338                break;
339            }
340            if fin {
341                break;
342            }
343        }
344        BrotliEncoderDestroyInstance(s);
345    }
346    read_err?;
347    Ok(total_out.unwrap())
348}
brotli/enc/mod.rs

brotli/enc/
mod.rs
