payloads/bayou/util/pbuilder/lzma/C/7zip/Compress/LZMA/LZMAEncoder.h

   1 // LZMA/Encoder.h
   2
   3 #ifndef __LZMA_ENCODER_H
   4 #define __LZMA_ENCODER_H
   5
   6 #include "../../../Common/MyCom.h"
   7 #include "../../../Common/Alloc.h"
   8 #include "../../ICoder.h"
   9 #include "../LZ/IMatchFinder.h"
  10 #include "../RangeCoder/RangeCoderBitTree.h"
  11
  12 #include "LZMA.h"
  13
  14 namespace NCompress {
  15 namespace NLZMA {
  16
  17 typedef NRangeCoder::CBitEncoder<kNumMoveBits> CMyBitEncoder;
  18
  19 class CBaseState
  20 {
  21 protected:
  22   CState _state;
  23   Byte _previousByte;
  24   UInt32 _repDistances[kNumRepDistances];
  25   void Init()
  26   {
  27     _state.Init();
  28     _previousByte = 0;
  29     for(UInt32 i = 0 ; i < kNumRepDistances; i++)
  30       _repDistances[i] = 0;
  31   }
  32 };
  33
  34 struct COptimal
  35 {
  36   CState State;
  37
  38   bool Prev1IsChar;
  39   bool Prev2;
  40
  41   UInt32 PosPrev2;
  42   UInt32 BackPrev2;
  43
  44   UInt32 Price;
  45   UInt32 PosPrev;         // posNext;
  46   UInt32 BackPrev;
  47   UInt32 Backs[kNumRepDistances];
  48   void MakeAsChar() { BackPrev = UInt32(-1); Prev1IsChar = false; }
  49   void MakeAsShortRep() { BackPrev = 0; ; Prev1IsChar = false; }
  50   bool IsShortRep() { return (BackPrev == 0); }
  51 };
  52
  53
  54 extern Byte g_FastPos[1 << 11];
  55 inline UInt32 GetPosSlot(UInt32 pos)
  56 {
  57   if (pos < (1 << 11))
  58     return g_FastPos[pos];
  59   if (pos < (1 << 21))
  60     return g_FastPos[pos >> 10] + 20;
  61   return g_FastPos[pos >> 20] + 40;
  62 }
  63
  64 inline UInt32 GetPosSlot2(UInt32 pos)
  65 {
  66   if (pos < (1 << 17))
  67     return g_FastPos[pos >> 6] + 12;
  68   if (pos < (1 << 27))
  69     return g_FastPos[pos >> 16] + 32;
  70   return g_FastPos[pos >> 26] + 52;
  71 }
  72
  73 const UInt32 kIfinityPrice = 0xFFFFFFF;
  74
  75 const UInt32 kNumOpts = 1 << 12;
  76
  77
  78 class CLiteralEncoder2
  79 {
  80   CMyBitEncoder _encoders[0x300];
  81 public:
  82   void Init()
  83   {
  84     for (int i = 0; i < 0x300; i++)
  85       _encoders[i].Init();
  86   }
  87   void Encode(NRangeCoder::CEncoder *rangeEncoder, Byte symbol);
  88   void EncodeMatched(NRangeCoder::CEncoder *rangeEncoder, Byte matchByte, Byte symbol);
  89   UInt32 GetPrice(bool matchMode, Byte matchByte, Byte symbol) const;
  90 };
  91
  92 class CLiteralEncoder
  93 {
  94   CLiteralEncoder2 *_coders;
  95   int _numPrevBits;
  96   int _numPosBits;
  97   UInt32 _posMask;
  98 public:
  99   CLiteralEncoder(): _coders(0) {}
 100   ~CLiteralEncoder()  { Free(); }
 101   void Free()
 102   {
 103     MyFree(_coders);
 104     _coders = 0;
 105   }
 106   bool Create(int numPosBits, int numPrevBits)
 107   {
 108     if (_coders == 0 || (numPosBits + numPrevBits) != (_numPrevBits + _numPosBits))
 109     {
 110       Free();
 111       UInt32 numStates = 1 << (numPosBits + numPrevBits);
 112       _coders = (CLiteralEncoder2 *)MyAlloc(numStates * sizeof(CLiteralEncoder2));
 113     }
 114     _numPosBits = numPosBits;
 115     _posMask = (1 << numPosBits) - 1;
 116     _numPrevBits = numPrevBits;
 117     return (_coders != 0);
 118   }
 119   void Init()
 120   {
 121     UInt32 numStates = 1 << (_numPrevBits + _numPosBits);
 122     for (UInt32 i = 0; i < numStates; i++)
 123       _coders[i].Init();
 124   }
 125   CLiteralEncoder2 *GetSubCoder(UInt32 pos, Byte prevByte)
 126     { return &_coders[((pos & _posMask) << _numPrevBits) + (prevByte >> (8 - _numPrevBits))]; }
 127 };
 128
 129 namespace NLength {
 130
 131 class CEncoder
 132 {
 133   CMyBitEncoder _choice;
 134   CMyBitEncoder _choice2;
 135   NRangeCoder::CBitTreeEncoder<kNumMoveBits, kNumLowBits> _lowCoder[kNumPosStatesEncodingMax];
 136   NRangeCoder::CBitTreeEncoder<kNumMoveBits, kNumMidBits> _midCoder[kNumPosStatesEncodingMax];
 137   NRangeCoder::CBitTreeEncoder<kNumMoveBits, kNumHighBits> _highCoder;
 138 public:
 139   void Init(UInt32 numPosStates);
 140   void Encode(NRangeCoder::CEncoder *rangeEncoder, UInt32 symbol, UInt32 posState);
 141   void SetPrices(UInt32 posState, UInt32 numSymbols, UInt32 *prices) const;
 142 };
 143
 144 const UInt32 kNumSpecSymbols = kNumLowSymbols + kNumMidSymbols;
 145
 146 class CPriceTableEncoder: public CEncoder
 147 {
 148   UInt32 _prices[kNumPosStatesEncodingMax][kNumSymbolsTotal];
 149   UInt32 _tableSize;
 150   UInt32 _counters[kNumPosStatesEncodingMax];
 151 public:
 152   void SetTableSize(UInt32 tableSize) { _tableSize = tableSize;  }
 153   UInt32 GetPrice(UInt32 symbol, UInt32 posState) const { return _prices[posState][symbol]; }
 154   void UpdateTable(UInt32 posState)
 155   {
 156     SetPrices(posState, _tableSize, _prices[posState]);
 157     _counters[posState] = _tableSize;
 158   }
 159   void UpdateTables(UInt32 numPosStates)
 160   {
 161     for (UInt32 posState = 0; posState < numPosStates; posState++)
 162       UpdateTable(posState);
 163   }
 164   void Encode(NRangeCoder::CEncoder *rangeEncoder, UInt32 symbol, UInt32 posState, bool updatePrice)
 165   {
 166     CEncoder::Encode(rangeEncoder, symbol, posState);
 167     if (updatePrice)
 168       if (--_counters[posState] == 0)
 169         UpdateTable(posState);
 170   }
 171 };
 172
 173 }
 174
 175 class CEncoder :
 176   public ICompressCoder,
 177   public ICompressSetOutStream,
 178   public ICompressSetCoderProperties,
 179   public ICompressWriteCoderProperties,
 180   public CBaseState,
 181   public CMyUnknownImp
 182 {
 183   COptimal _optimum[kNumOpts];
 184   CMyComPtr<IMatchFinder> _matchFinder; // test it
 185   NRangeCoder::CEncoder _rangeEncoder;
 186
 187   CMyBitEncoder _isMatch[kNumStates][NLength::kNumPosStatesEncodingMax];
 188   CMyBitEncoder _isRep[kNumStates];
 189   CMyBitEncoder _isRepG0[kNumStates];
 190   CMyBitEncoder _isRepG1[kNumStates];
 191   CMyBitEncoder _isRepG2[kNumStates];
 192   CMyBitEncoder _isRep0Long[kNumStates][NLength::kNumPosStatesEncodingMax];
 193
 194   NRangeCoder::CBitTreeEncoder<kNumMoveBits, kNumPosSlotBits> _posSlotEncoder[kNumLenToPosStates];
 195
 196   CMyBitEncoder _posEncoders[kNumFullDistances - kEndPosModelIndex];
 197   NRangeCoder::CBitTreeEncoder<kNumMoveBits, kNumAlignBits> _posAlignEncoder;
 198
 199   NLength::CPriceTableEncoder _lenEncoder;
 200   NLength::CPriceTableEncoder _repMatchLenEncoder;
 201
 202   CLiteralEncoder _literalEncoder;
 203
 204   UInt32 _matchDistances[kMatchMaxLen * 2 + 2 + 1];
 205
 206   bool _fastMode;
 207   // bool _maxMode;
 208   UInt32 _numFastBytes;
 209   UInt32 _longestMatchLength;
 210   UInt32 _numDistancePairs;
 211
 212   UInt32 _additionalOffset;
 213
 214   UInt32 _optimumEndIndex;
 215   UInt32 _optimumCurrentIndex;
 216
 217   bool _longestMatchWasFound;
 218
 219   UInt32 _posSlotPrices[kNumLenToPosStates][kDistTableSizeMax];
 220
 221   UInt32 _distancesPrices[kNumLenToPosStates][kNumFullDistances];
 222
 223   UInt32 _alignPrices[kAlignTableSize];
 224   UInt32 _alignPriceCount;
 225
 226   UInt32 _distTableSize;
 227
 228   UInt32 _posStateBits;
 229   UInt32 _posStateMask;
 230   UInt32 _numLiteralPosStateBits;
 231   UInt32 _numLiteralContextBits;
 232
 233   UInt32 _dictionarySize;
 234
 235   UInt32 _dictionarySizePrev;
 236   UInt32 _numFastBytesPrev;
 237
 238   UInt32 _matchPriceCount;
 239   UInt64 nowPos64;
 240   bool _finished;
 241   ISequentialInStream *_inStream;
 242
 243   UInt32 _matchFinderCycles;
 244   int _matchFinderIndex;
 245   #ifdef COMPRESS_MF_MT
 246   bool _multiThread;
 247   #endif
 248
 249   bool _writeEndMark;
 250
 251   bool _needReleaseMFStream;
 252
 253   IMatchFinderSetNumPasses *setMfPasses;
 254
 255   void ReleaseMatchFinder()
 256   {
 257     setMfPasses = 0;
 258     _matchFinder.Release();
 259   }
 260
 261   HRESULT ReadMatchDistances(UInt32 &len, UInt32 &numDistancePairs);
 262
 263   HRESULT MovePos(UInt32 num);
 264   UInt32 GetRepLen1Price(CState state, UInt32 posState) const
 265   {
 266     return _isRepG0[state.Index].GetPrice0() +
 267         _isRep0Long[state.Index][posState].GetPrice0();
 268   }
 269
 270   UInt32 GetPureRepPrice(UInt32 repIndex, CState state, UInt32 posState) const
 271   {
 272     UInt32 price;
 273     if(repIndex == 0)
 274     {
 275       price = _isRepG0[state.Index].GetPrice0();
 276       price += _isRep0Long[state.Index][posState].GetPrice1();
 277     }
 278     else
 279     {
 280       price = _isRepG0[state.Index].GetPrice1();
 281       if (repIndex == 1)
 282         price += _isRepG1[state.Index].GetPrice0();
 283       else
 284       {
 285         price += _isRepG1[state.Index].GetPrice1();
 286         price += _isRepG2[state.Index].GetPrice(repIndex - 2);
 287       }
 288     }
 289     return price;
 290   }
 291   UInt32 GetRepPrice(UInt32 repIndex, UInt32 len, CState state, UInt32 posState) const
 292   {
 293     return _repMatchLenEncoder.GetPrice(len - kMatchMinLen, posState) +
 294         GetPureRepPrice(repIndex, state, posState);
 295   }
 296   /*
 297   UInt32 GetPosLen2Price(UInt32 pos, UInt32 posState) const
 298   {
 299     if (pos >= kNumFullDistances)
 300       return kIfinityPrice;
 301     return _distancesPrices[0][pos] + _lenEncoder.GetPrice(0, posState);
 302   }
 303   UInt32 GetPosLen3Price(UInt32 pos, UInt32 len, UInt32 posState) const
 304   {
 305     UInt32 price;
 306     UInt32 lenToPosState = GetLenToPosState(len);
 307     if (pos < kNumFullDistances)
 308       price = _distancesPrices[lenToPosState][pos];
 309     else
 310       price = _posSlotPrices[lenToPosState][GetPosSlot2(pos)] +
 311           _alignPrices[pos & kAlignMask];
 312     return price + _lenEncoder.GetPrice(len - kMatchMinLen, posState);
 313   }
 314   */
 315   UInt32 GetPosLenPrice(UInt32 pos, UInt32 len, UInt32 posState) const
 316   {
 317     UInt32 price;
 318     UInt32 lenToPosState = GetLenToPosState(len);
 319     if (pos < kNumFullDistances)
 320       price = _distancesPrices[lenToPosState][pos];
 321     else
 322       price = _posSlotPrices[lenToPosState][GetPosSlot2(pos)] +
 323           _alignPrices[pos & kAlignMask];
 324     return price + _lenEncoder.GetPrice(len - kMatchMinLen, posState);
 325   }
 326
 327   UInt32 Backward(UInt32 &backRes, UInt32 cur);
 328   HRESULT GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes);
 329   HRESULT GetOptimumFast(UInt32 position, UInt32 &backRes, UInt32 &lenRes);
 330
 331   void FillDistancesPrices();
 332   void FillAlignPrices();
 333
 334   void ReleaseMFStream()
 335   {
 336     if (_matchFinder && _needReleaseMFStream)
 337     {
 338       _matchFinder->ReleaseStream();
 339       _needReleaseMFStream = false;
 340     }
 341   }
 342
 343   void ReleaseStreams()
 344   {
 345     ReleaseMFStream();
 346     ReleaseOutStream();
 347   }
 348
 349   HRESULT Flush(UInt32 nowPos);
 350   class CCoderReleaser
 351   {
 352     CEncoder *_coder;
 353   public:
 354     CCoderReleaser(CEncoder *coder): _coder(coder) {}
 355     ~CCoderReleaser()
 356     {
 357       _coder->ReleaseStreams();
 358     }
 359   };
 360   friend class CCoderReleaser;
 361
 362   void WriteEndMarker(UInt32 posState);
 363
 364 public:
 365   CEncoder();
 366   void SetWriteEndMarkerMode(bool writeEndMarker)
 367     { _writeEndMark= writeEndMarker; }
 368
 369   HRESULT Create();
 370
 371   MY_UNKNOWN_IMP3(
 372       ICompressSetOutStream,
 373       ICompressSetCoderProperties,
 374       ICompressWriteCoderProperties
 375       )
 376
 377   HRESULT Init();
 378
 379   // ICompressCoder interface
 380   HRESULT SetStreams(ISequentialInStream *inStream,
 381       ISequentialOutStream *outStream,
 382       const UInt64 *inSize, const UInt64 *outSize);
 383   HRESULT CodeOneBlock(UInt64 *inSize, UInt64 *outSize, Int32 *finished);
 384
 385   HRESULT CodeReal(ISequentialInStream *inStream,
 386       ISequentialOutStream *outStream,
 387       const UInt64 *inSize, const UInt64 *outSize,
 388       ICompressProgressInfo *progress);
 389
 390   // ICompressCoder interface
 391   STDMETHOD(Code)(ISequentialInStream *inStream,
 392       ISequentialOutStream *outStream,
 393       const UInt64 *inSize, const UInt64 *outSize,
 394       ICompressProgressInfo *progress);
 395
 396   // ICompressSetCoderProperties2
 397   STDMETHOD(SetCoderProperties)(const PROPID *propIDs,
 398       const PROPVARIANT *properties, UInt32 numProperties);
 399
 400   // ICompressWriteCoderProperties
 401   STDMETHOD(WriteCoderProperties)(ISequentialOutStream *outStream);
 402
 403   STDMETHOD(SetOutStream)(ISequentialOutStream *outStream);
 404   STDMETHOD(ReleaseOutStream)();
 405
 406   virtual ~CEncoder() {}
 407 };
 408
 409 }}
 410
 411 #endif