/// Compression and decompression streaming algorithms.
///
/// Each compression or decompression algorithm is represented by a struct
/// that contains parameters for compression/decompression.
/// Besides the parameters they carry, algorithms have no state. Each
/// algorithm instance can be used for an unlimited number of parallel jobs.
///
/// The algorithms create a stream, which carry the state and allocated
/// resources of the ongoing compression.
///
/// The compression/decompression jobs are run by the `squiz` function,
/// or one of the related helpers built upon it (e.g. deflate, deflateGz, inflate, ...).
///
/// `squiz` and related functions take and InputRange of ubyte[] and return an InputRange of ubyte[].
/// This allows streaming in the most natural way for a D program and provide
/// the greatest versatility.
/// It is possible to read the data from any source (file, network, memory),
/// process the data, and write to any kind of destination.
/// This also allows to process gigabytes of data with little memory usage.
///
/// Compression often wraps the compressed data with header and trailer
/// that give the decompression algorithm useful information, especially
/// to check the integrity of the data after decompression.
/// This is called the format.
/// Some compressions algorithms offer different formats, and sometimes
/// the possibility to not wrap the data at all (raw format), in which
/// case integrity check is not performed. This is usually used when
/// an external integrity check is done, for example when archiving
/// compressed stream in Zip or 7z archives.
module squiz_box.squiz;

import squiz_box.c.bzip2;
import squiz_box.c.lzma;
import squiz_box.c.zlib;
import squiz_box.c.zstd;
import squiz_box.priv;

import std.datetime.systime;
import std.exception;
import std.range;
import std.typecons;

/// default chunk size for data exchanges and I/O operations
enum defaultChunkSize = 8192;

/// definition of a byte chunk, which is the unit of data
/// exchanged during I/O and data transformation operations
alias ByteChunk = const(ubyte)[];

/// A dynamic type of input range of chunks of bytes
alias ByteRange = InputRange!ByteChunk;

/// Static check that a type is a byte range.
template isByteRange(BR)
{
    import std.traits : isArray, Unqual;
    import std.range : ElementType, isInputRange;

    alias Arr = ElementType!BR;
    alias El = ElementType!Arr;

    enum isByteRange = isInputRange!BR && is(Unqual!El == ubyte);
}

static assert(isByteRange!ByteRange);

/// Exception thrown when inconsistent data is given to
/// a decompression algorithm.
/// I.e. the data was not compressed with the corresponding algorithm
/// or the wrapping format is not the one expected.
@safe class DataException : Exception
{
    mixin basicExceptionCtors!();
}

/// Check whether a type is a proper squiz algorithm.
template isSquizAlgo(A)
{
    enum isSquizAlgo = is(typeof((A algo) {
                auto stream = algo.initialize();
                Flag!"streamEnded" ended = algo.process(stream, Yes.lastChunk);
                algo.reset(stream);
                algo.end(stream);
                static assert(is(typeof(stream) : SquizStream));
            }));
}

/// Get the type of a SquizStream for the Squiz algorithm
template StreamType(A) if (isSquizAlgo!A)
{
    import std.traits : ReturnType;

    alias StreamType = ReturnType!(A.initialize);
}

/// A squiz algorithm whom type is erased behind an interface.
/// This helps to choose algorithm at run time.
interface SquizAlgo
{
    /// Initialize a new stream for processing data
    /// with this algorithm.
    SquizStream initialize() @safe;

    /// Processes the input stream data to produce output stream data.
    /// lastChunk indicates that the input chunk in stream is the last one.
    /// This is an indication to the algorithm that it can start to finish
    /// the work.
    /// Returned value indicates that there won't be more output generated
    /// than the one in stream.output
    Flag!"streamEnded" process(SquizStream stream, Flag!"lastChunk" lastChunk) @safe;

    /// Reset the state of this stream, yet reusing the same
    /// allocating resources, in order to start processing
    /// another data stream.
    void reset(SquizStream stream) @safe;

    /// Release the resources used by this stream.
    /// Most of the memory (if not all) used by algorithm
    /// is allocating with the garbage collector, so not
    /// calling this function has little consequence (if not none).
    void end(SquizStream stream) @safe;
}

static assert(isSquizAlgo!SquizAlgo);

/// Get a runtime type for the provided algorithm
SquizAlgo squizAlgo(A)(A algo) @safe if (isSquizAlgo!A)
{
    return new CSquizAlgo!A(algo);
}

///
@("squizAlgo")
unittest
{
    import test.util;
    import std.array : join;

    auto ctAlgo = Deflate.init;
    auto rtAlgo = squizAlgo(Deflate.init);

    const len = 10_000;
    const phrase = cast(const(ubyte)[]) "Some very repetitive phrase.\n";
    const input = generateRepetitiveData(len, phrase).join();

    const ctSquized = only(input).squiz(ctAlgo).join();
    const rtSquized = only(input).squiz(rtAlgo).join();

    assert(ctSquized == rtSquized);
}

private class CSquizAlgo(A) : SquizAlgo
{
    alias Stream = StreamType!A;

    A algo;

    private this(A algo) @safe
    {
        this.algo = algo;
    }

    private Stream checkStream(SquizStream stream)
    {
        auto s = cast(Stream) stream;
        assert(s, "provided stream is not produced by this algorithm");
        return s;
    }

    SquizStream initialize() @safe
    {
        return algo.initialize();
    }

    Flag!"streamEnded" process(SquizStream stream, Flag!"lastChunk" lastChunk) @safe
    {
        return algo.process(checkStream(stream), lastChunk);
    }

    void reset(SquizStream stream) @safe
    {
        return algo.reset(checkStream(stream));
    }

    void end(SquizStream stream) @safe
    {
        return algo.end(checkStream(stream));
    }
}

/// A state carrying, processing stream for squiz algorithms.
/// The stream does not carry any buffer, only slices to external buffer.
/// One may normally not use this directly as everything is handled
/// by the `squiz` function.
interface SquizStream
{
    /// Input data for the algorithm
    /// The slice is reduced by its begining as the processing moves on.
    /// Must be refilled when empty before calling the algorithm `process` method.
    @property const(ubyte)[] input() const @safe;
    /// Ditto
    @property void input(const(ubyte)[] inp) @safe;

    /// How many bytes read since the start of the stream processing.
    @property size_t totalInput() const @safe;

    /// Output buffer for the algorithm to write to.
    /// This is NOT the data ready after process, but where the
    /// algorithm must write next.
    /// after a call to process, the slice is reduced by its beginning,
    /// and the data written is therefore the one before the slice.
    @property inout(ubyte)[] output() inout @safe;
    @property void output(ubyte[] outp) @safe;

    /// How many bytes written since the start of the stream processing.
    @property size_t totalOutput() const @safe;
}

private template isZlibLikeStream(S)
{
    enum isZlibLikeStream = is(typeof((S stream) {
                stream.next_in = cast(const(ubyte)*) null;
                stream.avail_in = 0;
                stream.next_out = cast(ubyte*) null;
                stream.avail_out = 0;
            }));
}

private mixin template ZlibLikeStreamImpl(S) if (isZlibLikeStream!S)
{
    private S strm;

    @property const(ubyte)[] input() const @trusted
    {
        return strm.next_in[0 .. strm.avail_in];
    }

    @property void input(const(ubyte)[] inp) @trusted
    {
        strm.next_in = inp.ptr;
        strm.avail_in = cast(typeof(strm.avail_in)) inp.length;
    }

    @property inout(ubyte)[] output() inout @trusted
    {
        return strm.next_out[0 .. strm.avail_out];
    }

    @property void output(ubyte[] outp) @trusted
    {
        strm.next_out = outp.ptr;
        strm.avail_out = cast(typeof(strm.avail_out)) outp.length;
    }
}

mixin template ZlibLikeTotalInOutImpl()
{
    @property size_t totalInput() const
    {
        return cast(size_t) strm.total_in;
    }

    @property size_t totalOutput() const
    {
        return cast(size_t) strm.total_out;
    }
}

/// Returns an InputRange containing the input data processed through the supplied algorithm.
auto squiz(I, A)(I input, A algo, size_t chunkSize = defaultChunkSize)
        if (isByteRange!I && isSquizAlgo!A)
{
    return squiz(input, algo, new ubyte[chunkSize]);
}

/// ditto
auto squiz(I, A)(I input, A algo, ubyte[] chunkBuffer)
        if (isByteRange!I && isSquizAlgo!A)
{
    auto stream = algo.initialize();
    return Squiz!(I, A, Yes.endStream)(input, algo, stream, chunkBuffer, ulong.max);
}

/// Returns an InputRange containing the input data processed through the supplied algorithm.
/// To the difference of `squiz`, `squizReuse` will not manage the state (aka stream) of the algorithm,
/// which allows to reuse it (and its allocated resources) for several jobs.
/// squizReuse will drive the algorithm and move the stream forward until processing is over.
/// The stream must be either freshly initialized or freshly reset before being passed
/// to this function.
auto squizReuse(I, A, S)(I input, A algo, S stream, ubyte[] chunkBuffer)
        if (isByteRange!I && isSquizAlgo!A)
{
    static assert(is(StreamType!A == S), S.strinof ~ " is not the stream produced by " ~ A.stringof);
    return Squiz!(I, A, No.endStream)(input, algo, stream, chunkBuffer, ulong.max);
}

/// Same as squiz, but will stop encoding/decoding after len bytes has been written out
/// Useful to decode some raw encoded streams where the uncompressed size is known
/// and the algorithm not always report Yes.streamEnded.
auto squizMaxOut(I, A)(I input, A algo, ulong maxOut, size_t chunkSize = defaultChunkSize)
{
    import std.algorithm : min;

    const sz = cast(size_t) min(maxOut, chunkSize);
    auto chunkBuffer = new ubyte[sz];
    auto stream = algo.initialize();
    return Squiz!(I, A, Yes.endStream)(input, algo, stream, chunkBuffer, maxOut);
}

// Common transformation range for all compression/decompression functions.
// I is a byte input range
// A is a squiz algorithm
// if Yes.end, the stream is ended when data is done processing
private struct Squiz(I, A, Flag!"endStream" endStream)
{
    private alias Stream = StreamType!A;

    // Byte input range (by chunks)
    private I input;

    // The algorithm
    private A algo;

    // Processed stream stream
    private Stream stream;

    // Buffer used to store the front chunk
    private ubyte[] chunkBuffer;
    // Slice of the buffer that is valid for read out
    private ByteChunk chunk;

    // maximum number of bytes to write out
    private ulong maxLen;

    /// Whether the end of stream was reported by the Policy
    private bool ended;

    private this(I input, A algo, Stream stream, ubyte[] chunkBuffer, ulong maxLen)
    {
        this.input = input;
        this.algo = algo;
        this.stream = stream;
        this.chunkBuffer = chunkBuffer;
        this.maxLen = maxLen;
        prime();
    }

    @property bool empty()
    {
        return chunk.length == 0;
    }

    @property ByteChunk front()
    {
        return chunk;
    }

    void popFront()
    {
        chunk = null;
        if (!ended)
            prime();
    }

    private void prime()
    {
        import std.algorithm : min;

        while (chunk.length < chunkBuffer.length)
        {
            if (stream.input.length == 0 && !input.empty)
                stream.input = input.front;

            const len = min(chunkBuffer.length - chunk.length, maxLen);
            stream.output = chunkBuffer[chunk.length .. chunk.length + len];

            const streamEnded = algo.process(stream, cast(Flag!"lastChunk") input.empty);

            chunk = chunkBuffer[0 .. $ - stream.output.length];
            maxLen -= len;

            // popFront must be called at the end because it invalidates inChunk
            if (stream.input.length == 0 && !input.empty)
                input.popFront();

            if (streamEnded || maxLen == 0)
            {
                ended = true;
                static if (endStream)
                    algo.end(stream);
                break;
            }
        }
    }
}

@("squizMaxOut")
unittest
{
    // encoded header of test/data/archive.7z
    const(ubyte)[] dataIn = [
        0x00, 0x00, 0x81, 0x33, 0x07, 0xae, 0x0f, 0xd1, 0xf2, 0xfb, 0xfd, 0x40,
        0xc0, 0x90, 0xd2, 0xff, 0x7d, 0x69, 0x4d, 0x90, 0xd3, 0x2c, 0x42, 0x66,
        0xb0, 0xc6, 0xcc, 0xeb, 0xcf, 0x59, 0xcc, 0x96, 0x23, 0xf9, 0x91, 0xc8,
        0x75, 0x49, 0xe9, 0x9d, 0x1a, 0xa8, 0xa5, 0x9d, 0xf7, 0x75, 0x29, 0x1a,
        0x90, 0x78, 0x18, 0x8e, 0x42, 0x1a, 0x97, 0x0c, 0x40, 0xb7, 0xaa, 0xb6,
        0x16, 0xa9, 0x91, 0x0c, 0x58, 0xad, 0x75, 0xf7, 0x8f, 0xaf, 0x8f, 0x45,
        0xdb, 0x78, 0xd0, 0x8e, 0xc6, 0x1b, 0x72, 0xa5, 0xf4, 0xd2, 0x46, 0xf7,
        0xe1, 0xce, 0x01, 0x80, 0x7f, 0x3d, 0x66, 0xa5, 0x2d, 0x64, 0xd7, 0xb0,
        0x41, 0xdc, 0x92, 0x59, 0x88, 0xb0, 0x4c, 0x67, 0x34, 0xb6, 0x4e, 0xd3,
        0xd5, 0x01, 0x8d, 0x43, 0x13, 0x9c, 0x82, 0x78, 0x4d, 0xcf, 0x8c, 0x51,
        0x25, 0x0f, 0xd5, 0x1d, 0x80, 0x4b, 0x80, 0xea, 0x18, 0xc1, 0x29, 0x49,
        0xe4, 0x4d, 0x4d, 0x8b, 0xb9, 0xa1, 0xfc, 0x17, 0x2b, 0xb3, 0xe6, 0x00,
        0x00, 0x00
    ];
    // decoded header data of test/data/archive.7z
    const(ubyte)[] expectedDataOut = [
        0x01, 0x04, 0x06, 0x00, 0x01, 0x09, 0x40, 0x00, 0x07, 0x0b, 0x01, 0x00,
        0x01, 0x21, 0x21, 0x01, 0x00, 0x0c, 0x8d, 0xe2, 0x00, 0x08, 0x0d, 0x03,
        0x09, 0x8d, 0xc1, 0x07, 0x0a, 0x01, 0x84, 0x4d, 0x4d, 0xa8, 0x9e, 0xf4,
        0xb3, 0xdb, 0x12, 0xed, 0x64, 0x40, 0x00, 0x00, 0x05, 0x03, 0x19, 0x0d,
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
        0x00, 0x11, 0x55, 0x00, 0x66, 0x00, 0x69, 0x00, 0x6c, 0x00, 0x65, 0x00,
        0x20, 0x00, 0x32, 0x00, 0x2e, 0x00, 0x74, 0x00, 0x78, 0x00, 0x74, 0x00,
        0x00, 0x00, 0x66, 0x00, 0x69, 0x00, 0x6c, 0x00, 0x65, 0x00, 0x31, 0x00,
        0x2e, 0x00, 0x74, 0x00, 0x78, 0x00, 0x74, 0x00, 0x00, 0x00, 0x66, 0x00,
        0x6f, 0x00, 0x6c, 0x00, 0x64, 0x00, 0x65, 0x00, 0x72, 0x00, 0x2f, 0x00,
        0x63, 0x00, 0x68, 0x00, 0x6d, 0x00, 0x6f, 0x00, 0x64, 0x00, 0x20, 0x00,
        0x36, 0x00, 0x36, 0x00, 0x36, 0x00, 0x2e, 0x00, 0x74, 0x00, 0x78, 0x00,
        0x74, 0x00, 0x00, 0x00, 0x14, 0x1a, 0x01, 0x00, 0x80, 0x96, 0x9f, 0xd5,
        0xc8, 0x53, 0xd8, 0x01, 0x80, 0x50, 0x82, 0x4f, 0xc6, 0x53, 0xd8, 0x01,
        0x00, 0xff, 0x13, 0x13, 0xb7, 0x52, 0xd8, 0x01, 0x15, 0x0e, 0x01, 0x00,
        0x20, 0x80, 0xa4, 0x81, 0x20, 0x80, 0xa4, 0x81, 0x20, 0x80, 0xb6, 0x81,
        0x00, 0x00
    ];

    auto algo = DecompressLzma(LzmaFormat.rawLegacy);

    const dataOut = only(dataIn)
        .squizMaxOut(algo, expectedDataOut.length)
        .join();

    assert(dataOut == expectedDataOut);
}

/// Copy algorithm do not transform data at all
/// This is useful in cases of reading/writing data
/// that may or may not be compressed. Using Copy
/// allows that the same code handles both kind of streams.
final class CopyStream : SquizStream
{
    private const(ubyte)[] _inp;
    size_t _totalIn;
    private ubyte[] _outp;
    size_t _totalOut;

    @property const(ubyte)[] input() const @safe
    {
        return _inp;
    }

    @property void input(const(ubyte)[] inp) @safe
    {
        _inp = inp;
    }

    @property size_t totalInput() const @safe
    {
        return _totalIn;
    }

    @property inout(ubyte)[] output() inout @safe
    {
        return _outp;
    }

    @property void output(ubyte[] outp) @safe
    {
        _outp = outp;
    }

    @property size_t totalOutput() const @safe
    {
        return _totalOut;
    }
}

/// ditto
struct Copy
{
    static assert(isSquizAlgo!Copy);

    CopyStream initialize() @safe
    {
        return new CopyStream;
    }

    Flag!"streamEnded" process(CopyStream stream, Flag!"lastChunk" lastChunk) @safe
    {
        import std.algorithm : min;

        const len = min(stream._inp.length, stream._outp.length);

        stream._outp[0 .. len] = stream._inp[0 .. len];

        stream._inp = stream._inp[len .. $];
        stream._outp = stream._outp[len .. $];
        stream._totalIn += len;
        stream._totalOut += len;

        return cast(Flag!"streamEnded")(lastChunk && stream._inp.length == 0);
    }

    void reset(CopyStream stream) @safe
    {
        stream._inp = null;
        stream._outp = null;
        stream._totalIn = 0;
        stream._totalOut = 0;
    }

    void end(CopyStream) @safe
    {
    }
}

/// ditto
auto copy(I)(I input, size_t chunkSize = defaultChunkSize)
{
    return squiz(input, Copy.init, chunkSize);
}

///
@("Copy")
unittest
{
    import test.util : generateRepetitiveData;
    import std.array : join;

    const len = 10_000;
    const phrase = cast(const(ubyte)[]) "Some very repetitive phrase.\n";
    const input = generateRepetitiveData(len, phrase).join();

    /// copying with arbitrary chunk sizes on input and output
    const cop1 = generateRepetitiveData(len, phrase, 1231).copy(234).join();
    const cop2 = generateRepetitiveData(len, phrase, 296).copy(6712).join();

    assert(input == cop1);
    assert(input == cop2);
}

/// Describe what type of header and trailer are wrapping
/// a deflated stream.
enum ZlibFormat
{
    /// Zlib header and trailer
    zlib,
    /// Gzip header and trailer
    gz,
    /// Auto detection of Zlib or Gzip format (only used with Inflate)
    autoDetect,
    /// No header and trailer, therefore no integrity check included.
    /// This to be used in other formats such as Zip.
    /// When using raw, it is advised to use an external integrity check.
    raw,
}

private size_t strnlen(const(byte)* str, size_t maxlen) @system
{
    if (!str)
        return 0;

    size_t l;
    while (*str != 0 && l < maxlen)
    {
        str++;
        l++;
    }
    return l;
}

@("strnlen")
unittest
{
    assert(strnlen(null, 0) == 0);
    assert(strnlen(cast(const(byte)*)("abcdefghij\0klmn".ptr), 15) == 10);
    assert(strnlen(cast(const(byte)*)("abcdefghij\0klmn".ptr), 10) == 10);
    assert(strnlen(cast(const(byte)*)("abcdefghij\0klmn".ptr), 9) == 9);
    assert(strnlen(cast(const(byte)*)("abcdefghij\0klmn".ptr), 0) == 0);
    assert(strnlen(cast(const(byte)*)("\0bcdefghij\0klmn".ptr), 15) == 0);
}

/// Header data for the Gzip format.
/// Gzip includes metadata about the file which is compressed.
/// These can be specified here when compressing from a stream
/// rather than directly from a file.
struct GzHeader
{
    import core.stdc.config : c_ulong;

    /// operating system encoded in the Gz header
    /// Not all possible values are listed here, only
    /// the most useful ones
    enum Os
    {
        fatFs = 0,
        unix = 3,
        macintosh = 7,
        ntFs = 11,
        unknown = 255,
    }

    version (OSX)
        enum defaultOs = Os.macintosh;
    else version (iOS)
        enum defaultOs = Os.macintosh;
    else version (Posix)
        enum defaultOs = Os.unix;
    else version (Windows)
        enum defaultOs = Os.ntFs;

    /// Whether the content is believed to be text
    Flag!"text" text;

    // storing in unix format to avoid
    // negative numbers with SysTime.init
    private c_ulong _mtime;

    /// Modification time
    @property SysTime mtime() const @safe
    {
        return SysTime(unixTimeToStdTime(_mtime));
    }

    /// ditto
    @property void mtime(SysTime time) @safe
    {
        _mtime = stdTimeToUnixTime(time.stdTime);
    }

    /// Operating system that wrote the gz file
    Os os = defaultOs;

    /// Filename to be included in the header
    string filename;

    /// Comment to be included in the header
    string comment;

    private enum bufSize = 256;

    private string fromLatin1z(const(byte)* ptr) @system
    {
        // ptr points to a buffer of bufSize characters.
        // End of string is a null character or end of buffer.
        // Encoding is latin 1.
        import std.encoding : Latin1Char, transcode;

        const len = strnlen(ptr, bufSize);
        auto str = cast(const(Latin1Char)[]) ptr[0 .. len];

        string res;
        transcode(str, res);
        return res;
    }

    private byte* toLatin1z(string str) @trusted
    {
        import std.encoding : Latin1Char, transcode;

        Latin1Char[] l1;
        transcode(str, l1);
        auto res = (cast(byte[]) l1) ~ 0;
        return res.ptr;
    }

    private this(gz_headerp gzh) @system
    {
        text = gzh.text ? Yes.text : No.text;
        _mtime = gzh.time;
        os = cast(Os) gzh.os;
        if (gzh.name)
            filename = fromLatin1z(gzh.name);
        if (gzh.comment)
            comment = fromLatin1z(gzh.comment);
    }

    private gz_headerp toZlib() @safe
    {
        import core.stdc.config : c_long;

        auto gzh = new gz_header;
        gzh.text = text ? 1 : 0;
        gzh.time = _mtime;
        gzh.os = cast(int) os;
        if (filename)
            gzh.name = toLatin1z(filename);
        if (comment)
            gzh.comment = toLatin1z(comment);
        return gzh;
    }
}

/// Type of delegate to use as callback for Inflate.gzHeaderDg
alias GzHeaderDg = void delegate(GzHeader header);

/// Helper to set GzHeader.text
/// Will check if the data are all ascii characters
Flag!"text" isText(const(ubyte)[] data)
{
    import std.algorithm : all;

    return cast(Flag!"text") data.all!(
        c => c == 0x0a || c == 0x0d || (c >= 0x20 && c <= 0x7e)
    );
}

class ZlibStream : SquizStream
{
    mixin ZlibLikeStreamImpl!z_stream;
    mixin ZlibLikeTotalInOutImpl!();

    private this() @safe
    {
        strm.zalloc = &(gcAlloc!uint);
        strm.zfree = &gcFree;
    }
}

/// Returns an InputRange containing the input data processed through Zlib's deflate algorithm.
/// The produced stream of data is wrapped by Zlib header and trailer.
auto deflate(I)(I input, size_t chunkSize = defaultChunkSize) if (isByteRange!I)
{
    return squiz(input, Deflate.init, chunkSize);
}

/// Returns an InputRange containing the input data processed through Zlib's deflate algorithm.
/// The produced stream of data is wrapped by Gzip header and trailer.
/// Suppliying a header is entirely optional. Zlib produces a default header if not supplied.
/// The default header has text false, mtime zero, unknown os, and
/// no name or comment.
auto deflateGz(I)(I input, GzHeader header, size_t chunkSize = defaultChunkSize)
        if (isByteRange!I)
{
    auto algo = Deflate.init;
    algo.format = ZlibFormat.gz;
    algo.gzHeader = header;
    return squiz(input, algo, chunkSize);
}

/// ditto
auto deflateGz(I)(I input, size_t chunkSize = defaultChunkSize) if (isByteRange!I)
{
    auto algo = Deflate.init;
    algo.format = ZlibFormat.gz;
    return squiz(input, algo, chunkSize);
}

/// Returns an InputRange containing the input data processed through Zlib's deflate algorithm.
/// The produced stream of data isn't wrapped by any header or trailer.
auto deflateRaw(I)(I input, size_t chunkSize = defaultChunkSize) if (isByteRange!I)
{
    auto algo = Deflate.init;
    algo.format = ZlibFormat.raw;
    return squiz(input, algo, chunkSize);
}

/// Zlib's deflate algorithm
struct Deflate
{
    static assert(isSquizAlgo!Deflate);
    static assert(is(StreamType!Deflate == Stream));

    /// Which format to use for the deflated stream.
    /// In case ZlibFormat.gz, the gzHeader field will be used if supplied,
    /// other wise default values will be used.
    ZlibFormat format;

    /// Compression level from 1 (fastest) to 9 (best compression).
    int level = 6;

    /// The GzHeader to be used with ZlibFormat.gz.
    Nullable!GzHeader gzHeader;

    /// Advanced parameters
    /// See zlib's documentation of `deflateInit2`.
    /// windowBits must be between 9 and 15 included
    /// and is adjusted according chosen format.
    int windowBits = 15;
    /// ditto
    int memLevel = 8;
    /// ditto
    int strategy = Z_DEFAULT_STRATEGY;

    static final class Stream : ZlibStream
    {
    }

    Stream initialize() @safe
    {
        assert(
            9 <= windowBits && windowBits <= 15,
            "inconsistent windowBits"
        );
        int wb = windowBits;
        final switch (format)
        {
        case ZlibFormat.zlib:
            break;
        case ZlibFormat.gz:
            wb += 16;
            break;
        case ZlibFormat.autoDetect:
            throw new Exception("invalid ZlibFormat for Deflate");
        case ZlibFormat.raw:
            wb = -wb;
            break;
        }

        auto stream = new Stream();

        const res = (() @trusted => deflateInit2(
                &stream.strm, level, Z_DEFLATED,
                wb, memLevel, cast(int) strategy,
        ))();

        enforce(
            res == Z_OK,
            "Could not initialize Zlib deflate stream: " ~ zResultToString(res)
        );

        if (format == ZlibFormat.gz && !gzHeader.isNull)
        {
            auto head = gzHeader.get.toZlib();
            (() @trusted => deflateSetHeader(&stream.strm, head))();
        }

        return stream;
    }

    Flag!"streamEnded" process(Stream stream, Flag!"lastChunk" lastChunk) @safe
    {
        const flush = lastChunk ? Z_FINISH : Z_NO_FLUSH;
        const res = (() @trusted => squiz_box.c.zlib.deflate(&stream.strm, flush))();

        enforce(
            res == Z_OK || res == Z_STREAM_END,
            "Zlib deflate failed with code: " ~ zResultToString(res)
        );

        return cast(Flag!"streamEnded")(res == Z_STREAM_END);
    }

    void reset(Stream stream) @trusted
    {
        deflateReset(&stream.strm);
    }

    void end(Stream stream) @trusted
    {
        deflateEnd(&stream.strm);
    }
}

/// Returns an InputRange streaming over data inflated with Zlib.
/// The input data must be deflated with a zlib format.
auto inflate(I)(I input, size_t chunkSize = defaultChunkSize)
{
    return squiz(input, Inflate.init, chunkSize);
}

/// Returns an InputRange streaming over data inflated with Zlib.
/// The input data must be deflated with a gz format.
/// If headerDg is not null, it will be called
/// as soon as the header is read from the stream.
auto inflateGz(I)(I input, GzHeaderDg headerDg, size_t chunkSize = defaultChunkSize)
{
    auto algo = Inflate.init;
    algo.format = ZlibFormat.gz;
    algo.gzHeaderDg = headerDg;
    return squiz(input, algo, chunkSize);
}

/// ditto
auto inflateGz(I)(I input, size_t chunkSize = defaultChunkSize)
{
    return inflateGz(input, null, chunkSize);
}

/// Returns an InputRange streaming over data inflated with Zlib.
/// The input must be raw deflated data
auto inflateRaw(I)(I input, size_t chunkSize = defaultChunkSize)
{
    auto algo = Inflate.init;
    algo.format = ZlibFormat.raw;
    return squiz(input, algo, chunkSize);
}

/// Zlib's inflate algorithm
struct Inflate
{
    static assert(isSquizAlgo!Inflate);

    /// Which format to use for the deflated stream.
    /// In case ZlibFormat.gz, the gzHeader field will be written if set.
    ZlibFormat format;

    /// If set, will be assigned to the Gz header once it is known
    GzHeaderDg gzHeaderDg;

    /// Advanced parameters
    /// See zlib's documentation of `deflateInit2`.
    /// windowBits can be 0 if format is ZlibFormat.zlib.
    /// Otherwise it must be between 9 and 15 included.
    int windowBits = 15;

    private static final class Gzh
    {
        private gz_header gzh;
        private byte[GzHeader.bufSize] nameBuf;
        private byte[GzHeader.bufSize] commentBuf;

        private GzHeaderDg dg;
        private bool dgCalled;

        this(GzHeaderDg dg) @safe
        {
            gzh.name = &nameBuf[0];
            gzh.name_max = cast(uint) nameBuf.length;
            gzh.comment = &commentBuf[0];
            gzh.comm_max = cast(uint) commentBuf.length;

            this.dg = dg;
        }
    }

    static final class Stream : ZlibStream
    {
        Gzh gzh;
    }

    Stream initialize() @safe
    {
        assert(
            (windowBits == 0 && format == ZlibFormat.zlib) ||
                (9 <= windowBits && windowBits <= 15),
                "inconsistent windowBits"
        );
        int wb = windowBits;
        final switch (format)
        {
        case ZlibFormat.zlib:
            break;
        case ZlibFormat.gz:
            wb += 16;
            break;
        case ZlibFormat.autoDetect:
            wb += 32;
            break;
        case ZlibFormat.raw:
            wb = -wb;
            break;
        }

        auto stream = new Stream();

        const res = (() @trusted => inflateInit2(&stream.strm, wb))();

        enforce(
            res == Z_OK,
            "Could not initialize Zlib's inflate stream: " ~ zResultToString(res)
        );

        if (gzHeaderDg)
        {
            stream.gzh = new Gzh(gzHeaderDg);
            (() @trusted => inflateGetHeader(&stream.strm, &stream.gzh.gzh))();
        }

        return stream;
    }

    package Flag!"streamEnded" process(Stream stream, Flag!"lastChunk" /+ lastChunk +/ )
    {
        const res = (() @trusted => squiz_box.c.zlib.inflate(&stream.strm, Z_NO_FLUSH))();
        //
        if (res == Z_DATA_ERROR)
            throw new DataException("Improper data given to deflate");

        enforce(
            res == Z_OK || res == Z_STREAM_END,
            "Zlib inflate failed with code: " ~ zResultToString(res)
        );

        auto gzh = stream.gzh;
        if (gzh && !gzh.dgCalled && gzh.gzh.done)
        {
            auto head = (() @trusted => GzHeader(&gzh.gzh))();
            gzh.dg(head);
            gzh.dgCalled = true;
        }

        return cast(Flag!"streamEnded")(res == Z_STREAM_END);
    }

    package void reset(Stream stream) @trusted
    {
        inflateReset(&stream.strm);
    }

    package void end(Stream stream) @trusted
    {
        inflateEnd(&stream.strm);
    }
}

///
@("Deflate / Inflate")
unittest
{
    import test.util;
    import std.array : join;

    auto def = Deflate.init;
    auto inf = Inflate.init;

    const len = 100_000;
    const phrase = cast(const(ubyte)[]) "Some very repetitive phrase.\n";
    const input = generateRepetitiveData(len, phrase).join();

    // deflating
    const squized = only(input).squiz(def).join();

    // re-inflating
    const output = only(squized).squiz(inf).join();

    assert(squized.length < input.length);
    assert(output == input);

    // for such long and repetitive data, ratio is around 0.3%
    const ratio = cast(double) squized.length / cast(double) input.length;
    assert(ratio < 0.004);
}

///
@("Deflate / Inflate in Gz format and custom header")
unittest
{
    import test.util;
    import std.array : join;

    const len = 100_000;
    const phrase = cast(const(ubyte)[]) "Some very repetitive phrase.\n";
    const input = generateRepetitiveData(len, phrase).join();

    GzHeader inHead;
    inHead.mtime = Clock.currTime;
    inHead.os = GzHeader.Os.fatFs;
    inHead.text = Yes.text;
    inHead.filename = "boring.txt";
    inHead.comment = "A very boring file";

    // deflating
    const squized = only(input)
        .deflateGz(inHead)
        .join();

    // re-inflating
    GzHeader outHead;
    int numCalls;
    void setOutHead(GzHeader gzh)
    {
        outHead = gzh;
        numCalls++;
    }

    const output = only(squized)
        .inflateGz(&setOutHead)
        .join();

    assert(squized.length < input.length);
    assert(output == input);
    assert(inHead == outHead);
    assert(numCalls == 1);
}

///
@("Deflate / Inflate in raw format")
unittest
{
    import test.util;
    import std.array : join;

    const len = 100_000;
    const phrase = cast(const(ubyte)[]) "Some very repetitive phrase.\n";
    const input = generateRepetitiveData(len, phrase).join();

    // deflating
    const squized = only(input)
        .deflateRaw()
        .join();

    // re-inflating
    const output = only(squized)
        .inflateRaw()
        .join();

    assert(squized.length < input.length);
    assert(output == input);
}

package string zResultToString(int res) @safe pure nothrow @nogc
{
    switch (res)
    {
    case Z_OK:
        return "OK";
    case Z_STREAM_END:
        return "STREAM_END";
    case Z_NEED_DICT:
        return "NEED_DICT";
    case Z_ERRNO:
        return "ERRNO";
    case Z_STREAM_ERROR:
        return "STREAM_ERROR";
    case Z_DATA_ERROR:
        return "DATA_ERROR";
    case Z_MEM_ERROR:
        return "MEM_ERROR";
    case Z_BUF_ERROR:
        return "BUF_ERROR";
    case Z_VERSION_ERROR:
        return "VERSION_ERROR";
    default:
        return "(Unknown result)";
    }
}

package string zFlushToString(int flush) @safe pure nothrow @nogc
{
    switch (flush)
    {
    case Z_NO_FLUSH:
        return "NO_FLUSH";
    case Z_PARTIAL_FLUSH:
        return "PARTIAL_FLUSH";
    case Z_SYNC_FLUSH:
        return "SYNC_FLUSH";
    case Z_FULL_FLUSH:
        return "FULL_FLUSH";
    case Z_FINISH:
        return "FINISH";
    case Z_BLOCK:
        return "BLOCK";
    case Z_TREES:
        return "TREES";
    default:
        return "(Unknown flush)";
    }
}

/// Returns an InputRange containing the input data processed through Bzip2 compression.
auto compressBzip2(I)(I input, size_t chunkSize = defaultChunkSize)
        if (isByteRange!I)
{
    return squiz(input, CompressBzip2.init, chunkSize);
}

final class Bz2Stream : SquizStream
{
    mixin ZlibLikeStreamImpl!(bz_stream);

    @property size_t totalInput() const @safe
    {
        ulong hi = strm.total_in_hi32;
        return cast(size_t)(
            (hi << 32) | strm.total_in_lo32
        );
    }

    @property size_t totalOutput() const @safe
    {
        ulong hi = strm.total_out_hi32;
        return cast(size_t)(
            (hi << 32) | strm.total_out_lo32
        );
    }

    this() @safe
    {
        strm.bzalloc = &(gcAlloc!int);
        strm.bzfree = &gcFree;
    }
}

/// Compression with the Bzip2 algorithm.
///
/// Although having better compression capabilities than Zlib (deflate),
/// Bzip2 has poor latenty when it comes to streaming.
/// I.e. it can swallow several Mb of data before starting to produce output.
/// If streaming latenty is an important factor, deflate/inflate
/// should be the favorite algorithm.
///
/// This algorithm does not support resource reuse, so calling reset
/// is equivalent to a call to end followed by initialize.
/// (but the same instance of stream is kept).
struct CompressBzip2
{
    static assert(isSquizAlgo!CompressBzip2);

    /// Advanced Bzip2 parameters
    /// See Bzip2 documentation
    /// https://www.sourceware.org/bzip2/manual/manual.html#bzcompress-init
    int blockSize100k = 9;
    /// ditto
    int verbosity = 0;
    /// ditto
    int workFactor = 30;

    alias Stream = Bz2Stream;

    Stream initialize() @safe
    {
        auto stream = new Stream;

        const res = (() @trusted => BZ2_bzCompressInit(
                &stream.strm, blockSize100k, verbosity, workFactor,
        ))();
        enforce(
            res == BZ_OK,
            "Could not initialize Bzip2 compressor: " ~ bzResultToString(res)
        );
        return stream;
    }

    Flag!"streamEnded" process(Stream stream, Flag!"lastChunk" lastChunk) @safe
    {
        const action = lastChunk ? BZ_FINISH : BZ_RUN;
        const res = (() @trusted => BZ2_bzCompress(&stream.strm, action))();

        if (res == BZ_STREAM_END)
            return Yes.streamEnded;

        enforce(
            (action == BZ_RUN && res == BZ_RUN_OK) ||
                (action == BZ_FINISH && res == BZ_FINISH_OK),
                "Bzip2 compress failed with code: " ~ bzResultToString(res)
        );

        return No.streamEnded;
    }

    void reset(Stream stream) @safe
    {
        (() @trusted => BZ2_bzCompressEnd(&stream.strm))();

        stream.strm = bz_stream.init;
        stream.strm.bzalloc = &(gcAlloc!int);
        stream.strm.bzfree = &gcFree;

        const res = (() @trusted => BZ2_bzCompressInit(
                &stream.strm, blockSize100k, verbosity, workFactor,
        ))();
        enforce(
            res == BZ_OK,
            "Could not initialize Bzip2 compressor: " ~ bzResultToString(res)
        );
    }

    void end(Stream stream) @trusted
    {
        BZ2_bzCompressEnd(&stream.strm);
    }
}

/// Returns an InputRange streaming over data decompressed with Bzip2.
auto decompressBzip2(I)(I input, size_t chunkSize = defaultChunkSize)
        if (isByteRange!I)
{
    return squiz(input, DecompressBzip2.init, chunkSize);
}

/// Decompression of data encoded with Bzip2.
///
/// This algorithm does not support resource reuse, so calling reset
/// is equivalent to a call to end followed by initialize.
/// (but the same instance of stream is kept).
struct DecompressBzip2
{
    static assert(isSquizAlgo!DecompressBzip2);

    /// Advanced Bzip2 parameters
    /// See Bzip2 documentation
    /// https://www.sourceware.org/bzip2/manual/manual.html#bzDecompress-init
    int verbosity;
    /// ditto
    bool small;

    alias Stream = Bz2Stream;

    Stream initialize() @safe
    {
        auto stream = new Stream;

        const res = (() @trusted => BZ2_bzDecompressInit(
                &stream.strm, verbosity, small ? 1 : 0,
        ))();
        enforce(
            res == BZ_OK,
            "Could not initialize Bzip2 decompressor: " ~ bzResultToString(res)
        );
        return stream;
    }

    Flag!"streamEnded" process(Stream stream, Flag!"lastChunk") @safe
    {
        const res = (() @trusted => BZ2_bzDecompress(&stream.strm))();

        if (res == BZ_DATA_ERROR)
            throw new DataException("Input data was not compressed with Bzip2");

        enforce(
            res == BZ_OK || res == BZ_STREAM_END,
            "Bzip2 decompress failed with code: " ~ bzResultToString(res)
        );

        return cast(Flag!"streamEnded")(res == BZ_STREAM_END);
    }

    void reset(Stream stream) @safe
    {
        (() @trusted => BZ2_bzDecompressEnd(&stream.strm))();

        stream.strm = bz_stream.init;
        stream.strm.bzalloc = &(gcAlloc!int);
        stream.strm.bzfree = &gcFree;

        const res = (() @trusted => BZ2_bzDecompressInit(
                &stream.strm, verbosity, small ? 1 : 0,
        ))();
        enforce(
            res == BZ_OK,
            "Could not initialize Bzip2 decompressor: " ~ bzResultToString(res)
        );
    }

    void end(Stream stream) @trusted
    {
        BZ2_bzDecompressEnd(&stream.strm);
    }
}

///
@("Compress / Decompress Bzip2")
unittest
{
    import test.util;
    import std.array : join;

    const len = 100_000;
    const phrase = cast(const(ubyte)[]) "Some very repetitive phrase.\n";
    const input = generateRepetitiveData(len, phrase).join();

    const squized = only(input)
        .compressBzip2()
        .join();

    const output = only(squized)
        .decompressBzip2()
        .join();

    assert(squized.length < input.length);
    assert(output == input);

    // for such long and repetitive data, ratio is around 0.12%
    const ratio = cast(double) squized.length / cast(double) input.length;
    assert(ratio < 0.002);
}

private string bzActionToString(int action) @safe pure nothrow @nogc
{
    switch (action)
    {
    case BZ_RUN:
        return "RUN";
    case BZ_FLUSH:
        return "FLUSH";
    case BZ_FINISH:
        return "FINISH";
    default:
        return "(Unknown result)";
    }
}

private string bzResultToString(int res) @safe pure nothrow @nogc
{
    switch (res)
    {
    case BZ_OK:
        return "OK";
    case BZ_RUN_OK:
        return "RUN_OK";
    case BZ_FLUSH_OK:
        return "FLUSH_OK";
    case BZ_FINISH_OK:
        return "FINISH_OK";
    case BZ_STREAM_END:
        return "STREAM_END";
    case BZ_SEQUENCE_ERROR:
        return "SEQUENCE_ERROR";
    case BZ_PARAM_ERROR:
        return "PARAM_ERROR";
    case BZ_MEM_ERROR:
        return "MEM_ERROR";
    case BZ_DATA_ERROR:
        return "DATA_ERROR";
    case BZ_DATA_ERROR_MAGIC:
        return "DATA_ERROR_MAGIC";
    case BZ_IO_ERROR:
        return "IO_ERROR";
    case BZ_UNEXPECTED_EOF:
        return "UNEXPECTED_EOF";
    case BZ_OUTBUFF_FULL:
        return "OUTBUFF_FULL";
    case BZ_CONFIG_ERROR:
        return "CONFIG_ERROR";
    default:
        return "(Unknown result)";
    }
}

final class LzmaStream : SquizStream
{
    mixin ZlibLikeStreamImpl!(lzma_stream);
    mixin ZlibLikeTotalInOutImpl!();

    private lzma_allocator alloc;
    private lzma_options_delta optsDelta;
    private lzma_options_lzma optsLzma;
    private lzma_filter[] filterChain;

    this() @safe
    {
        alloc.alloc = &(gcAlloc!size_t);
        alloc.free = &gcFree;
        strm.allocator = &alloc;
    }

    private lzma_filter[] buildFilterChain(LzmaFormat format, LzmaFilter[] filters,
        uint preset, uint deltaDist) @safe
    {
        lzma_filter[] res;
        foreach (f; filters)
        {
            final switch (f)
            {
            case LzmaFilter.delta:
                optsDelta.dist = deltaDist;
                res ~= lzma_filter(LZMA_FILTER_DELTA, cast(void*)&optsDelta);
                break;
            case LzmaFilter.bcjX86:
                res ~= lzma_filter(LZMA_FILTER_X86, null);
                break;
            case LzmaFilter.bcjPowerPc:
                res ~= lzma_filter(LZMA_FILTER_POWERPC, null);
                break;
            case LzmaFilter.bcjIa64:
                res ~= lzma_filter(LZMA_FILTER_IA64, null);
                break;
            case LzmaFilter.bcjArm:
                res ~= lzma_filter(LZMA_FILTER_ARM, null);
                break;
            case LzmaFilter.bcjArmThumb:
                res ~= lzma_filter(LZMA_FILTER_ARMTHUMB, null);
                break;
            case LzmaFilter.bcjSparc:
                res ~= lzma_filter(LZMA_FILTER_SPARC, null);
                break;
            }
        }

        enforce(res.length <= 3, "Too many filters supplied");

        if (format != LzmaFormat.rawCopy)
        {
            (() @trusted => lzma_lzma_preset(&optsLzma, preset))();
            const compFilter = format.isLegacy ? LZMA_FILTER_LZMA1 : LZMA_FILTER_LZMA2;
            res ~= lzma_filter(compFilter, cast(void*)&optsLzma);
        }

        res ~= lzma_filter(LZMA_VLI_UNKNOWN, null); // end marker

        filterChain = res;
        return res;
    }

}

/// Header/trailer format for Lzma compression
enum LzmaFormat
{
    /// Lzma with Xz format, suitable to write *.xz files
    xz,
    /// LZMA1 encoding and format, suitable for legacy *.lzma files
    /// This format doesn't support filters.
    legacy,
    /// Raw LZMA2 compression, without header/trailer.
    /// Use this to include compressed LZMA data in
    /// a container defined externally (e.g. this is used
    /// for the *.7z archives)
    raw,
    /// Raw LZMA1 compression, without header/trailer.
    /// This one is still found in some *.7z files.
    rawLegacy,
    /// Just copy bytes out.
    /// You may use this in combination with a filter to observe its
    /// effect, but has otherwise no use.
    rawCopy,
}

/// Whether this is a legacy format
bool isLegacy(LzmaFormat format) @safe pure nothrow @nogc
{
    return format == LzmaFormat.legacy || format == LzmaFormat.rawLegacy;
}

/// Whether this is a raw format
bool isRaw(LzmaFormat format) @safe pure nothrow @nogc
{
    return cast(int) format >= cast(int) LzmaFormat.raw;
}

/// Filters to use with the LZMA compression.
///
/// Up to 3 filters can be used from this list.
/// These filters transform the input to increase
/// redundancy of the data supplied to the LZMA compression.
enum LzmaFilter
{
    /// Delta filter, which store differences between bytes
    /// to produce more repetitive data in some circumstances.
    /// Works with `deltaDist` parameter of `CompressLzma`.
    delta,

    /// BCJ (Branch/Call/Jump) filters aim optimize machine code
    /// compression by converting relative branches, calls and jumps
    /// to absolute addresses. This increases redundancy and can be
    /// exploited by the LZMA compression.
    ///
    /// BCJ filters are available for a set of CPU architectures.
    /// Use one (or two) of them when compressing compiled binaries.
    bcjX86,
    /// ditto
    bcjPowerPc,
    /// ditto
    bcjIa64,
    /// ditto
    bcjArm,
    /// ditto
    bcjArmThumb,
    /// ditto
    bcjSparc,
}

/// Integrity check to include in the compressed data
/// (only for the Xz format)
/// Default for xz is CRC-64.
enum LzmaCheck
{
    /// No integrity check included
    none,
    /// CRC-32 integrity check
    crc32,
    /// CRC-64 integrity check
    crc64,
    /// SHA-256 integrity check
    sha256,
}

private lzma_check toLzma(LzmaCheck check) @safe pure nothrow @nogc
{
    final switch (check)
    {
    case LzmaCheck.none:
        return lzma_check.NONE;
    case LzmaCheck.crc32:
        return lzma_check.CRC32;
    case LzmaCheck.crc64:
        return lzma_check.CRC64;
    case LzmaCheck.sha256:
        return lzma_check.SHA256;
    }
}

auto compressXz(I)(I input, size_t chunkSize = defaultChunkSize)
{
    return squiz(input, CompressLzma.init, chunkSize);
}

auto compressLzmaRaw(I)(I input, size_t chunkSize = defaultChunkSize)
{
    CompressLzma algo;
    algo.format = LzmaFormat.raw;
    return squiz(input, algo, chunkSize);
}

struct CompressLzma
{
    import std.conv : to;

    static assert(isSquizAlgo!CompressLzma);

    /// The format of the compressed stream
    LzmaFormat format;

    /// The integrity check to include in compressed stream.
    /// Only used with XZ format.
    LzmaCheck check = LzmaCheck.crc64;

    /// The compression preset between 0 (fast) to 9 (higher compression).
    /// The default is 6.
    uint preset = 6;

    /// Makes the encoding significantly slower for marginal compression
    /// improvement. Only useful if you don't mind about CPU time at all.
    Flag!"extreme" extreme;

    /// Filters to include in the encoding.
    /// Maximum three filters can be provided.
    /// For most input, no filtering is necessary.
    LzmaFilter[] filters;

    /// Number of bytes between 1 and 256 to use for the Delta filter.
    /// For example for 16bit PCM stero audio, you should use 4.
    /// For RGB data 8bit per channel, you should use 3.
    uint deltaDist;

    alias Stream = LzmaStream;

    private void initStream(Stream stream) @trusted
    {
        uint pres = preset;
        if (extreme)
            pres |= LZMA_PRESET_EXTREME;

        lzma_ret res;
        final switch (format)
        {
        case LzmaFormat.xz:
            const chain = stream.buildFilterChain(format, filters, pres, deltaDist);
            res = lzma_stream_encoder(&stream.strm, chain.ptr, check.toLzma());
            break;
        case LzmaFormat.legacy:
            enforce(filters.length == 0, "Filters are not supported with the legacy format");
            lzma_lzma_preset(&stream.optsLzma, preset);
            res = lzma_alone_encoder(&stream.strm, &stream.optsLzma);
            break;
        case LzmaFormat.raw:
        case LzmaFormat.rawLegacy:
        case LzmaFormat.rawCopy:
            const chain = stream.buildFilterChain(format, filters, pres, deltaDist);
            res = lzma_raw_encoder(&stream.strm, chain.ptr);
            break;
        }

        enforce(res == lzma_ret.OK, "Could not initialize LZMA encoder: ", res.to!string);
    }

    Stream initialize() @safe
    {
        auto stream = new LzmaStream;
        initStream(stream);
        return stream;
    }

    Flag!"streamEnded" process(Stream stream, Flag!"lastChunk" lastChunk) @safe
    {
        return lzmaCode(stream, lastChunk);
    }

    void reset(Stream stream) @safe
    {
        // Lzma supports reset out of the box by recalling initialization
        // function without calling lzma_end.

        initStream(stream);
    }

    void end(Stream stream) @trusted
    {
        lzma_end(&stream.strm);
    }
}

auto decompressXz(I)(I input, size_t chunkSize = defaultChunkSize)
{
    return squiz(input, DecompressLzma.init, chunkSize);
}

auto decompressLzmaRaw(I)(I input, size_t chunkSize = defaultChunkSize)
{
    DecompressLzma algo;
    algo.format = LzmaFormat.raw;
    return squiz(input, algo, chunkSize);
}

struct DecompressLzma
{
    import std.conv : to;

    static assert(isSquizAlgo!DecompressLzma);

    /// The format of the compressed stream
    LzmaFormat format;

    /// The memory usage limit in bytes.
    /// by default no limit is enforced
    size_t memLimit = size_t.max;

    /// Parameters for the raw decompression.
    /// They are the same than for the compression.
    /// As there is no header to tell Lzma what filters were used during
    /// compression, it is the responsibility of the programmer to
    /// correctly ensure that the same options are used for decompression.
    /// All these options are ignored when decompressing .xz stream.
    uint preset = 6;
    /// ditto
    Flag!"extreme" extreme;
    /// ditto
    LzmaFilter[] filters;
    /// ditto
    uint deltaDist;

    alias Stream = LzmaStream;

    this(LzmaFormat format) @safe
    {
        this.format = format;
    }

    /// convenience constructor to copy parameters of the compression
    /// for the decompression. Especially useful for the raw decompression,
    /// to ensure that the parameters fit the ones used for compression.
    this(CompressLzma compress) @safe
    {
        format = compress.format;
        preset = compress.preset;
        extreme = compress.extreme;
        filters = compress.filters;
        deltaDist = compress.deltaDist;
    }

    private void initStream(Stream stream) @trusted
    {
        ulong memlim = memLimit;
        if (memLimit == size_t.max)
            memlim = ulong.max;

        lzma_ret res;

        final switch (format)
        {
        case LzmaFormat.xz:
            res = lzma_stream_decoder(&stream.strm, memlim, 0);
            break;
        case LzmaFormat.legacy:
            res = lzma_alone_decoder(&stream.strm, memlim);
            break;
        case LzmaFormat.raw:
        case LzmaFormat.rawLegacy:
        case LzmaFormat.rawCopy:
            uint pres = preset;
            if (extreme)
                pres |= LZMA_PRESET_EXTREME;

            const chain = stream.buildFilterChain(format, filters, pres, deltaDist);

            res = lzma_raw_decoder(&stream.strm, chain.ptr);
        }
        enforce(res == lzma_ret.OK, "Could not initialize LZMA encoder: ", res.to!string);
    }

    Flag!"streamEnded" process(Stream stream, Flag!"lastChunk" lastChunk) @safe
    {
        return lzmaCode(stream, lastChunk);
    }

    Stream initialize() @safe
    {
        auto stream = new LzmaStream;
        initStream(stream);
        return stream;
    }

    void reset(Stream stream) @safe
    {
        // Lzma supports reset out of the box by recalling initialization
        // function without calling lzma_end.

        initStream(stream);
    }

    void end(Stream stream) @trusted
    {
        lzma_end(&stream.strm);
    }
}

private Flag!"streamEnded" lzmaCode(LzmaStream stream, Flag!"lastChunk" lastChunk) @safe
{
    import std.conv : to;

    const action = lastChunk ? lzma_action.FINISH : lzma_action.RUN;
    const res = (() @trusted => lzma_code(&stream.strm, action))();

    enforce(
        res == lzma_ret.OK || res == lzma_ret.STREAM_END,
        "LZMA encoding failed with code: " ~ res.to!string
    );

    return cast(Flag!"streamEnded")(res == lzma_ret.STREAM_END);
}

///
@("Compress / Decompress XZ")
unittest
{
    import test.util;
    import std.array : join;

    const len = 100_000;
    const phrase = cast(const(ubyte)[]) "Some very repetitive phrase.\n";
    const input = generateRepetitiveData(len, phrase).join();

    const squized = only(input)
        .compressXz()
        .join();

    const output = only(squized)
        .decompressXz()
        .join();

    assert(squized.length < input.length);
    assert(output == input);

    // for such long and repetitive data, ratio is around 0.2%
    const ratio = cast(double) squized.length / cast(double) input.length;
    assert(ratio < 0.003);
}

///
@("Integrity check XZ")
unittest
{
    import test.util;
    import std.array : join;

    const len = 100_000;
    const phrase = cast(const(ubyte)[]) "Some very repetitive phrase.\n";
    const input = generateRepetitiveData(len, phrase).join();

    auto squized = only(input)
        .compressXz()
        .join()
        .dup; // dup because const(ubyte)[] is returned

    squized[squized.length / 2] += 1;

    assertThrown(
        only(squized)
            .decompressXz()
            .join()
    );
}

///
@("Compress / Decompress XZ with filter")
unittest
{
    import test.util;
    import std.array : join;

    const len = 100_000;
    const input = generateSequentialData(len, 1245, 27).join();

    const reference = only(input)
        .compressXz()
        .join();

    CompressLzma comp;
    comp.filters ~= LzmaFilter.delta;
    comp.deltaDist = 8; // sequential data of 8 byte integers

    const withDelta = only(input)
        .squiz(comp)
        .join();

    const output = only(withDelta)
        .decompressXz()
        .join();

    assert(output == input);
    // < 20% compression without filter (sequential data is tough)
    // < 0.5% compression with delta (peace of cake)
    assert(input.length > reference.length * 5);
    assert(input.length > withDelta.length * 200);
}

///
@("Compress / Decompress Lzma Raw")
unittest
{
    import test.util;
    import std.array : join;

    const len = 100_000;
    const phrase = cast(const(ubyte)[]) "Some very repetitive phrase.\n";
    const input = generateRepetitiveData(len, phrase).join();

    const reference = only(input)
        .compressXz()
        .join();

    const squized = only(input)
        .compressLzmaRaw()
        .join();

    const output = only(squized)
        .decompressLzmaRaw()
        .join();

    assert(output == input);
    assert(squized.length < input.length);
    assert(squized.length < reference.length); // win header/trailer space

    // for such repetitive data, ratio is around 1.13%
    // also generally better than zlib, bzip2 struggles a lot for repetitive data
    const ratio = cast(double) squized.length / cast(double) input.length;
    assert(ratio < 0.003);
}

///
@("Compress / Decompress Lzma Raw with filter")
unittest
{
    import test.util;
    import std.array : join;

    const len = 100_000;
    const input = generateSequentialData(len, 1245, 27).join();

    const reference = only(input)
        .compressLzmaRaw()
        .join();

    CompressLzma comp;
    comp.format = LzmaFormat.raw;
    comp.filters ~= LzmaFilter.delta;
    comp.deltaDist = 8; // sequential data of 8 byte integers

    const withDelta = only(input)
        .squiz(comp)
        .join();

    const output = only(withDelta) // using compression parameters for decompression
    .squiz(DecompressLzma(comp))
        .join();

    assert(output == input);
    // < 20% compression without filter (sequential data is tough)
    // < 0.4% compression with delta (peace of cake)
    assert(input.length > reference.length * 5);
    assert(input.length > withDelta.length * 250);
}

///
@("Compress / Decompress Lzma Legacy")
unittest
{
    import test.util;
    import std.array : join;

    const len = 100_000;
    const phrase = cast(const(ubyte)[]) "Some very repetitive phrase.\n";
    const input = generateRepetitiveData(len, phrase).join();

    auto comp = CompressLzma(LzmaFormat.legacy);
    auto decomp = DecompressLzma(comp);

    const squized = only(input)
        .squiz(comp)
        .join();

    const output = only(squized)
        .squiz(decomp)
        .join();

    assert(squized.length < input.length);
    assert(output == input);

    // for such repetitive data, ratio is around 1.13%
    // also generally better than zlib, bzip2 struggles a lot for repetitive data
    const ratio = cast(double) squized.length / cast(double) input.length;
    assert(ratio < 0.003);
}

///
@("Compress / Decompress Lzma Raw Legacy")
unittest
{
    import test.util;
    import std.array : join;

    const len = 100_000;
    const phrase = cast(const(ubyte)[]) "Some very repetitive phrase.\n";
    const input = generateRepetitiveData(len, phrase).join();

    auto comp = CompressLzma(LzmaFormat.rawLegacy);
    auto decomp = DecompressLzma(comp);

    const squized = only(input)
        .squiz(comp)
        .join();

    const output = only(squized)
        .squiz(decomp)
        .join();

    assert(squized.length < input.length);
    assert(output == input);

    // for such repetitive data, ratio is around 1.13%
    // also generally better than zlib, bzip2 struggles a lot for repetitive data
    const ratio = cast(double) squized.length / cast(double) input.length;
    assert(ratio < 0.003);
}

///
@("Compress / Decompress Lzma rawLegacy with filter")
unittest
{
    import test.util;
    import std.array : join;

    const len = 100_000;
    const input = generateSequentialData(len, 1245, 27).join();

    const reference = only(input)
        .squiz(CompressLzma(LzmaFormat.legacy))
        .join();

    CompressLzma comp;
    comp.format = LzmaFormat.rawLegacy;
    comp.filters ~= LzmaFilter.delta;
    comp.deltaDist = 8; // sequential data of 8 byte integers

    auto decomp = DecompressLzma(comp);

    const withDelta = only(input)
        .squiz(comp)
        .join();

    const output = only(withDelta)
        .squiz(decomp)
        .join();

    assert(output == input);
    // < 20% compression without filter (sequential data is tough)
    // < 0.4% compression with delta (peace of cake)
    assert(input.length > reference.length * 5);
    assert(input.length > withDelta.length * 250);
}

auto compressZstd(I)(I input, size_t chunkSize = defaultChunkSize)
{
    return squiz(input, CompressZstd.init, chunkSize);
}

auto decompressZstd(I)(I input, size_t chunkSize = defaultChunkSize)
{
    return squiz(input, DecompressZstd.init, chunkSize);
}

class ZstdStream : SquizStream
{
    private ZSTD_inBuffer inBuf;
    private ZSTD_outBuffer outBuf;
    private size_t totalIn;
    private size_t totalOut;

    @property const(ubyte)[] input() const @trusted
    {
        auto ptr = cast(const(ubyte)*) inBuf.src;
        return ptr[inBuf.pos .. inBuf.size];
    }

    @property void input(const(ubyte)[] inp) @trusted
    {
        totalIn += inBuf.pos;
        inBuf.pos = 0;
        inBuf.src = cast(const(void)*) inp.ptr;
        inBuf.size = inp.length;
    }

    @property size_t totalInput() const @safe
    {
        return totalIn + inBuf.pos;
    }

    @property inout(ubyte)[] output() inout @trusted
    {
        auto ptr = cast(inout(ubyte)*) outBuf.dst;
        return ptr[outBuf.pos .. outBuf.size];
    }

    @property void output(ubyte[] outp) @trusted
    {
        totalOut += outBuf.pos;
        outBuf.pos = 0;
        outBuf.dst = cast(void*) outp.ptr;
        outBuf.size = outp.length;
    }

    @property size_t totalOutput() const @safe
    {
        return totalOut + outBuf.pos;
    }

    override string toString() const @safe
    {
        import std.format : format;

        string res;
        res ~= "ZstdStream:\n";
        res ~= "  Input:\n";
        res ~= format!"    start 0x%016x\n"(inBuf.src);
        res ~= format!"    pos %s\n"(inBuf.pos);
        res ~= format!"    size %s\n"(inBuf.size);
        res ~= format!"    total %s\n"(totalInput);
        res ~= "  Output:\n";
        res ~= format!"    start 0x%016x\n"(outBuf.dst);
        res ~= format!"    pos %s\n"(outBuf.pos);
        res ~= format!"    size %s\n"(outBuf.size);
        res ~= format!"    total %s"(totalOutput);

        return res;
    }
}

private string zstdSetCParam(string name)
{
    return "if (" ~ name ~ ") " ~
        "ZSTD_CCtx_setParameter(cctx, ZSTD_cParameter." ~ name ~ ", " ~ name ~ ");";
}

private void zstdError(size_t code, string desc) @trusted
{
    import std.string : fromStringz;

    if (ZSTD_isError(code))
    {
        const msg = fromStringz(ZSTD_getErrorName(code));
        throw new Exception((desc ~ ": " ~ msg).idup);
    }
}

/// Zstandard is a fast compression algorithm designed for streaming.
/// See zstd.h (enum ZSTD_cParameter) for details.
struct CompressZstd
{
    static assert(isSquizAlgo!CompressZstd);

    /// Common paramters.
    /// A value of zero indicates that the default should be used.
    int compressionLevel;
    /// ditto
    int windowLog;
    /// ditto
    int hashLog;
    /// ditto
    int chainLog;
    /// ditto
    int searchLog;
    /// ditto
    int minMatch;
    /// ditto
    int targetLength;
    /// ditto
    int strategy;

    /// Long distance matching parameters (LDM)
    /// Can be activated for large inputs to improve the compression ratio.
    /// Increases memory usage and the window size
    /// A value of zero indicate that the default should be used.
    bool enableLongDistanceMatching;
    /// ditto
    int ldmHashLog;
    /// ditto
    int ldmMinMatch;
    /// ditto
    int ldmBucketSizeLog;
    /// ditto
    int ldmHashRateLog;

    // frame parameters

    /// If input data content size is known, before
    /// start of streaming, set contentSize to its value.
    /// It will enable the size to be written in the header
    /// and checked after decompression.
    ulong contentSize = ulong.max;
    /// Include a checksum of the content in the trailer.
    bool checksumFlag = false;
    /// When applicable, dictionary's ID is written in the header
    bool dictIdFlag = true;

    /// Multi-threading parameters
    int nbWorkers;
    /// ditto
    int jobSize;
    /// ditto
    int overlapLog;

    static final class Stream : ZstdStream
    {
        private ZSTD_CStream* strm;
    }

    private void setParams(Stream stream) @trusted
    {
        auto cctx = cast(ZSTD_CCtx*) stream.strm;

        mixin(zstdSetCParam("compressionLevel"));
        mixin(zstdSetCParam("windowLog"));
        mixin(zstdSetCParam("hashLog"));
        mixin(zstdSetCParam("chainLog"));
        mixin(zstdSetCParam("searchLog"));
        mixin(zstdSetCParam("minMatch"));
        mixin(zstdSetCParam("targetLength"));
        mixin(zstdSetCParam("strategy"));

        if (enableLongDistanceMatching)
        {
            ZSTD_CCtx_setParameter(cctx,
                ZSTD_cParameter.enableLongDistanceMatching,
                1
            );

            mixin(zstdSetCParam("ldmHashLog"));
            mixin(zstdSetCParam("ldmMinMatch"));
            mixin(zstdSetCParam("ldmBucketSizeLog"));
            mixin(zstdSetCParam("ldmHashRateLog"));
        }

        if (contentSize != size_t.max)
            ZSTD_CCtx_setPledgedSrcSize(cctx, contentSize);
        if (checksumFlag)
            ZSTD_CCtx_setParameter(
                cctx,
                ZSTD_cParameter.checksumFlag,
                1
            );
        if (!dictIdFlag)
            ZSTD_CCtx_setParameter(
                cctx,
                ZSTD_cParameter.checksumFlag,
                0
            );

        mixin(zstdSetCParam("nbWorkers"));
        mixin(zstdSetCParam("jobSize"));
        mixin(zstdSetCParam("overlapLog"));
    }

    Stream initialize() @trusted
    {
        auto stream = new Stream;

        stream.strm = ZSTD_createCStream();

        setParams(stream);

        return stream;
    }

    Flag!"streamEnded" process(Stream stream, Flag!"lastChunk" lastChunk) @safe
    {
        auto cctx = cast(ZSTD_CCtx*) stream.strm;
        const directive = lastChunk ? ZSTD_EndDirective.end : ZSTD_EndDirective._continue;

        const res = (() @trusted => ZSTD_compressStream2(cctx, &stream.outBuf, &stream.inBuf, directive))();

        zstdError(res, "Could not compress data with Zstandard");
        return cast(Flag!"streamEnded")(lastChunk && res == 0);
    }

    void reset(Stream stream) @trusted
    {
        auto cctx = cast(ZSTD_CCtx*) stream.strm;
        ZSTD_CCtx_reset(cctx, ZSTD_ResetDirective.session_only);

        if (contentSize != size_t.max)
            ZSTD_CCtx_setPledgedSrcSize(cctx, contentSize);

        stream.inBuf = ZSTD_inBuffer.init;
        stream.outBuf = ZSTD_outBuffer.init;
        stream.totalIn = 0;
        stream.totalOut = 0;
    }

    void end(Stream stream) @trusted
    {
        ZSTD_freeCStream(stream.strm);
    }
}

struct DecompressZstd
{
    static assert(isSquizAlgo!DecompressZstd);

    int windowLogMax;

    static final class Stream : ZstdStream
    {
        private ZSTD_DStream* strm;
    }

    private void setParams(Stream stream) @trusted
    {
        auto dctx = cast(ZSTD_DCtx*) stream.strm;

        if (windowLogMax)
            ZSTD_DCtx_setParameter(dctx,
                ZSTD_dParameter.windowLogMax, windowLogMax);
    }

    Stream initialize() @trusted
    {
        auto stream = new Stream;

        stream.strm = ZSTD_createDStream();

        setParams(stream);

        return stream;
    }

    Flag!"streamEnded" process(Stream stream, Flag!"lastChunk") @safe
    {
        const res = (() @trusted => ZSTD_decompressStream(stream.strm, &stream.outBuf, &stream
                .inBuf))();

        zstdError(res, "Could not decompress data with Zstandard");
        return cast(Flag!"streamEnded")(res == 0);
    }

    void reset(Stream stream) @trusted
    {
        auto dctx = cast(ZSTD_DCtx*) stream.strm;
        ZSTD_DCtx_reset(dctx, ZSTD_ResetDirective.session_only);
    }

    void end(Stream stream) @trusted
    {
        ZSTD_freeDStream(stream.strm);
    }
}

///
@("Compress / Decompress Zstandard")
unittest
{
    import test.util;
    import std.array : join;

    const len = 100_000;
    const phrase = cast(const(ubyte)[]) "Some very repetitive phrase.\n";
    const input = generateRepetitiveData(len, phrase).join();

    const squized = only(input)
        .compressZstd()
        .join();

    const output = only(squized)
        .decompressZstd()
        .join();

    assert(squized.length < input.length);
    assert(output == input);

    // for such long and repetitive data, ratio is around 0.047%
    const ratio = cast(double) squized.length / cast(double) input.length;
    assert(ratio < 0.0005);
}