Free Compression Library - No Patent

[Jokaste]

I am looking for a free compression library with no patent and usuable in C under Windows.
For the instance I use the Bcl (Basic Compression Library), I use the hoffman compression but I don't compress more than 50%.
I download files from the NASA, some files have a zize of 23Mb, if I continue my hard disk will be too small. My program which create the SQLite databases compress the source files, but it is not enought. [size=78%]https://sourceforge.net/projects/bcl/files/?source=navbar[/size]
Who has a library for doing this? I don't want GZip nor Bzip.
Thanks to the ghosts

[Siekmanski]

You can do this with,

RtlCompressBuffer
RtlGetCompressionWorkSpaceSize
RtlDecompressBuffer

available via ntdll.dll

https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/content/ntifs/nf-ntifs-rtlcompressbuffer

[TimoVJL]

Link Maximum Compression gives some info what to expect.
What kind of data have to compress?

[Jokaste]

Itis only text. Ascii from 32 to 127

[Jokaste]

I will try Siekmanski's solution. Even if the algo is slow, but I hope it will compress better than 50%.
Thank You guys

[Jokaste]

When I call RtlCompressBuffer it always returns 0xC000000D

0xC000000D STATUS_INVALID_PARAMETERAn invalid parameter was passed to a service or function.

Code: [Select]

int CompressMax(LPSTR __lpszInputFileName,LPSTR __lpszOutputFileName)
{
   LPBYTE            _lpUnCompressedBuffer, _lpCompressedBuffer ;
   LPVOID            _lpWorkSpaceSize ;
   DWORD            _dwUnCompressedBufferSize, _dwFinalCompressedSize ;
   DWORD            _dwCompressBufferWorkSpaceSize, _dwCompressFragmentWorkSpaceSize ;
   DWORD            _dwCompressedBufferSize ;
   HANDLE            _hOutput ;
   int               _bRes ;
   COMPRESS         _Compress, *_lpCompress ;
   char            _szTmp[1024] ;
   LPSTR            _lpszTmp = _szTmp ;

   _bRes = FALSE ;
   _lpCompress = &_Compress ;

   __stosq((unsigned long long *)_lpCompress,0,sizeof(COMPRESS) / ;

   _lpUnCompressedBuffer = LoadThisFile(__lpszInputFileName,&_dwUnCompressedBufferSize) ;
   if(_lpUnCompressedBuffer)
   {
      _dwCompressedBufferSize = _dwUnCompressedBufferSize + 4096 ;
      _lpCompressedBuffer = MemoryAlloc(_dwCompressedBufferSize) ;
      if(_lpCompressedBuffer)
      {
         _dwCompressBufferWorkSpaceSize = _dwCompressedBufferSize ;
         _dwCompressFragmentWorkSpaceSize = 0 ;

         _bRes = RtlGetCompressionWorkSpaceSize(   COMPRESSION_ENGINE_MAXIMUM,
                                       &_dwCompressBufferWorkSpaceSize,
                                       &_dwCompressFragmentWorkSpaceSize) ;

         _lpWorkSpaceSize = (LPVOID) MemoryAlloc(_dwCompressBufferWorkSpaceSize) ;
         if(_lpWorkSpaceSize)
         {
            _lpCompress->dwAlgorithme = COMPRESS_NT ;
            _dwFinalCompressedSize = 0 ;

            _bRes = RtlCompressBuffer(   COMPRESSION_ENGINE_MAXIMUM,
                                 _lpUnCompressedBuffer,
                                 _dwUnCompressedBufferSize,
                                 _lpCompressedBuffer,
                                 _dwCompressedBufferSize,
                                 4096,
                                 &_dwFinalCompressedSize,
                                 _lpWorkSpaceSize) ;
            MemoryFree(_lpWorkSpaceSize) ;

            if(_bRes != STATUS_SUCCESS)

I would like some help.

[Jokaste]

I have found my error. I have added COMPRESSION_FORMAT_LZNT1 to the first parameter.
See the debug image joint to this post. The compression ratio is 46.22%. I expected better!

[TimoVJL]

https://www.maximumcompression.com/data/text.php

Give a link for your example to test.
LZ family could beat it?
PAQ family is just far too slow to be usable, don't except those to finish in same day.

Windows 8 -> use XPRESS+huffman as COMPRESSION_ENGINE_MAXIMUM ?

[Jokaste]

In my other algo I use HUFFMANN compression, this is poor, nearly 50%. I will try what you said.
What I am afraid using Huffman is its destructive algo. We know how it does when compressing jpeg.

[Jokaste]

Except COMPRESSION_FORMAT_LZNT1 all the others retunr 0C000025F : STATUS_UNSUPPORTED_COMPRESSION

[Jokaste]

Final compression function code :

Code: [Select]

#include <Windows.h>
#include "huffman.h"
#include "MaxStruct.h"
#include "MaxUtils.h"
#include "Log.h"

int Error(LPSTR __lpszMessage) ;
size_t   StringLength(const char * str) ;
LPVOID   MemorySetTo0(LPVOID __lpszBuffer,__int64 __qwSize) ;
LPVOID   MemoryAlloc(DWORD __dwSize) ;
void   MemoryFree(LPVOID __lpBuffer) ;

typedef int (*ptrRtlCompressBuffer)(USHORT,PUCHAR,ULONG,PUCHAR,ULONG,ULONG,PULONG,PVOID) ;
typedef int (*ptrRtlGetCompressionWorkSpaceSize)(USHORT,PULONG,PULONG) ;

ptrRtlGetCompressionWorkSpaceSize   RtlGetCompressionWorkSpaceSize ;
ptrRtlCompressBuffer            RtlCompressBuffer ;

HINSTANCE   hNtDll ;

int InitCompress(void)
{
   if(!hNtDll)
   {
      hNtDll = LoadLibrary("NtDll.dll") ;
      if(hNtDll)
      {
         RtlGetCompressionWorkSpaceSize = (ptrRtlGetCompressionWorkSpaceSize) GetProcAddress(hNtDll,"RtlGetCompressionWorkSpaceSize") ;
         RtlCompressBuffer = (ptrRtlCompressBuffer) GetProcAddress(hNtDll,"RtlCompressBuffer") ;

         return (TRUE) ;
      }
   }


   return (FALSE) ;
}

int ExitCompress(void)
{
   if(hNtDll)
   {
      FreeLibrary(hNtDll) ;
      hNtDll = NULL ;
      return (TRUE) ;
   }

   return (FALSE) ;
}

LPBYTE LoadThisFile(LPSTR __lpszInputFileName,DWORD *__lpdwBufferSize)
{
   HANDLE   _hInput ;
   DWORD   _dwFileSizeLow, _dwFileSizeHigh ;
   DWORD   _dwTmp ;
   LPBYTE   _lpBuffer, _lpTmp ;
   DWORD   _dwBufferSize ;
   LPSTR   _lpszTmp ;
   char   _szTmp[1024] ;

   _lpBuffer = _lpTmp = NULL ;
   _lpszTmp = _szTmp ;

   *__lpdwBufferSize = 0 ;

   __stosq((unsigned long long *)_lpszTmp,0,sizeof(_szTmp) / ;

   _hInput = CreateFile(__lpszInputFileName,GENERIC_READ,0,NULL,OPEN_EXISTING,FILE_ATTRIBUTE_NORMAL,NULL) ;
   if(_hInput == INVALID_HANDLE_VALUE)
   {
      wsprintf(_lpszTmp,"Cannot open this file: %s",__lpszInputFileName) ;
      Log(_lpszTmp) ;
      return ((LPBYTE) Error(_lpszTmp)) ;
   }

   _dwFileSizeLow = GetFileSize(_hInput,&_dwFileSizeHigh) ;

   if(_dwFileSizeHigh)
   {
      CloseHandle(_hInput) ;

      wsprintf(_lpszTmp,"This file is too big: %s",__lpszInputFileName) ;
      Log(_lpszTmp) ;
      return ((LPBYTE) Error(_lpszTmp)) ;
   }

   if(!_dwFileSizeLow && !_dwFileSizeHigh)
   {
      CloseHandle(_hInput) ;

      wsprintf(_lpszTmp,"This file is empty: %s",__lpszInputFileName) ;
      Log(_lpszTmp) ;
      return ((LPBYTE) Error(_lpszTmp)) ;
   }

   _dwBufferSize = _dwFileSizeLow + 4096 ;

   _lpBuffer = MemoryAlloc(_dwBufferSize) ;
   if(!_lpBuffer)
   {
      CloseHandle(_hInput) ;

      wsprintf(_lpszTmp,"Not enougth memory available for this file: %s",__lpszInputFileName) ;
      Log(_lpszTmp) ;
      return ((LPBYTE) Error(_lpszTmp)) ;
   }

   FlushFileBuffers(_hInput) ;

   if(!ReadFile(_hInput,_lpBuffer,_dwFileSizeLow,&_dwTmp,NULL))
   {
      MemoryFree(_lpBuffer) ;
      CloseHandle(_hInput) ;

      wsprintf(_lpszTmp,"Cannot read this file: %s",__lpszInputFileName) ;
      Log(_lpszTmp) ;
      return ((LPBYTE) Error(_lpszTmp)) ;
   }

   CloseHandle(_hInput) ;

   if(_dwFileSizeLow - _dwTmp)
   {
      MemoryFree(_lpBuffer) ;

      wsprintf(_lpszTmp,"Error while reading this file: %s",__lpszInputFileName) ;
      Log(_lpszTmp) ;
      return ((LPBYTE) Error(_lpszTmp)) ;
   }

   *__lpdwBufferSize = _dwFileSizeLow + 4096 ;

   return (_lpBuffer) ;
}

int CompressMax(LPSTR __lpszInputFileName,LPSTR __lpszOutputFileName)
{
   LPBYTE            _lpUnCompressedBuffer, _lpCompressedBuffer ;
   LPVOID            _lpWorkSpaceSize ;
   DWORD            _dwUnCompressedBufferSize, _dwFinalCompressedSize ;
   DWORD            _dwCompressBufferWorkSpaceSize, _dwCompressFragmentWorkSpaceSize ;
   DWORD            _dwCompressedBufferSize ;
   HANDLE            _hOutput ;
   int               _bRes ;
   COMPRESS         _Compress, *_lpCompress ;
   char            _szTmp[1024] ;
   LPSTR            _lpszTmp = _szTmp ;

   _bRes = FALSE ;
   _lpCompress = &_Compress ;

   __stosq((unsigned long long *)_lpCompress,0,sizeof(COMPRESS) / ;

   _lpUnCompressedBuffer = LoadThisFile(__lpszInputFileName,&_dwUnCompressedBufferSize) ;
   if(_lpUnCompressedBuffer)
   {
      _dwCompressedBufferSize = _dwUnCompressedBufferSize + 4096 ;
      _lpCompressedBuffer = MemoryAlloc(_dwCompressedBufferSize) ;
      if(_lpCompressedBuffer)
      {
         _dwCompressBufferWorkSpaceSize = _dwCompressedBufferSize ;
         _dwCompressFragmentWorkSpaceSize = 0 ;

         _bRes = RtlGetCompressionWorkSpaceSize(   COMPRESSION_FORMAT_LZNT1 | COMPRESSION_ENGINE_MAXIMUM,
                                       &_dwCompressBufferWorkSpaceSize,
                                       &_dwCompressFragmentWorkSpaceSize) ;

         _lpWorkSpaceSize = (LPVOID) MemoryAlloc(_dwCompressBufferWorkSpaceSize) ;
         if(_lpWorkSpaceSize)
         {
            _lpCompress->dwAlgorithme = COMPRESS_NT ;
            _dwFinalCompressedSize = 0 ;

//   https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/content/ntifs/nf-ntifs-rtlcompressbuffer

            _bRes = RtlCompressBuffer(   COMPRESSION_FORMAT_LZNT1 | COMPRESSION_ENGINE_MAXIMUM,
                                 _lpUnCompressedBuffer,
                                 _dwUnCompressedBufferSize,
                                 _lpCompressedBuffer,
                                 _dwCompressedBufferSize,
                                 4096,
                                 &_dwFinalCompressedSize,
                                 _lpWorkSpaceSize) ;

            MemoryFree(_lpWorkSpaceSize) ;

            if(_bRes != STATUS_SUCCESS)
            {
               _lpCompress->dwAlgorithme = COMPRESS_HUFFMAN ;
               _dwFinalCompressedSize = Huffman_Compress(_lpUnCompressedBuffer,_lpCompressedBuffer,_dwUnCompressedBufferSize) ;
            }
         }
         else
         {
            _lpCompress->dwAlgorithme = COMPRESS_HUFFMAN ;
            _dwFinalCompressedSize = Huffman_Compress(_lpUnCompressedBuffer,_lpCompressedBuffer,_dwUnCompressedBufferSize) ;
         }

         if(_dwFinalCompressedSize)
         {
            _hOutput = CreateFile(__lpszOutputFileName,GENERIC_WRITE,0,NULL,CREATE_ALWAYS,FILE_ATTRIBUTE_NORMAL,NULL) ;
            if(_hOutput == INVALID_HANDLE_VALUE)
            {
               FlushFileBuffers(_hOutput) ;
               CloseHandle(_hOutput) ;
               DeleteFile(__lpszOutputFileName) ;

               wsprintf(_lpszTmp,"Cannot open this file: %s",__lpszOutputFileName) ;
               Log(_lpszTmp) ;
               _bRes = Error(_lpszTmp) ;
            }
            else
            {
               _lpCompress->dwSignature = 'MAXI' ;
               _lpCompress->dwMethod = METHOD_COMPRESS ;
               _lpCompress->dwUncompressedFileSize = _dwUnCompressedBufferSize ;
               _lpCompress->dwCompressedFileSize = _dwFinalCompressedSize ;

               lstrcpy(_lpCompress->szUncompressedFileName,__lpszInputFileName) ;

               if(MaxWriteFile(_hOutput,(LPBYTE) _lpCompress,sizeof(COMPRESS)))
               {
                  if(MaxWriteFile(_hOutput,_lpCompressedBuffer,_dwFinalCompressedSize))
                  {
                     FlushFileBuffers(_hOutput) ;
                     _bRes = TRUE ;
                  }
               }
               else
               {
                  wsprintf(_lpszTmp,"Error while writing to file: %s",__lpszOutputFileName) ;
                  Log(_lpszTmp) ;
                  _bRes = Error(_lpszTmp) ;
               }

               CloseHandle(_hOutput) ;

               if(!_bRes)
                  DeleteFile(__lpszOutputFileName) ;
            }
         }
         else
         {
            wsprintf(_lpszTmp,"Error while compressing file: %s",__lpszInputFileName) ;
            Log(_lpszTmp) ;
            _bRes = Error(_lpszTmp) ;
         }

         MemoryFree(_lpCompressedBuffer) ;
      }

      MemoryFree(_lpUnCompressedBuffer) ;
   }

   return (_bRes) ;
}


[Jokaste]

Final decompression function code :

Code: [Select]

#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include <commctrl.h>
#include <Commdlg.h >
#include <Shlwapi.h >
#include <Shellapi.h>
#include "UnCompress.h"
#include "MaxStruct.h"
#include "MaxUtils.h"
#include "huffman.h"
#include "Common.h"

DWORD Error(LPSTR __lpszErrorMessage) ;
DWORD Information(LPSTR __lpszInformationMessage) ;
size_t StringLength (const char * str) ;
LPVOID MemorySetTo0(LPVOID __lpszBuffer,__int64 __qwSize) ;

typedef int (*ptrRtlDecompressBuffer)(USHORT,PUCHAR,ULONG,PUCHAR,ULONG,PULONG) ;
typedef int (*ptrRtlGetCompressionWorkSpaceSize)(USHORT,PULONG,PULONG) ;

ptrRtlGetCompressionWorkSpaceSize   RtlGetCompressionWorkSpaceSize ;
ptrRtlDecompressBuffer            RtlDecompressBuffer ;

//-----------------------------------------------------------------------------------------------
HINSTANCE      hInstance ;
HINSTANCE      hNtDll ;
//-----------------------------------------------------------------------------------------------
int InitUnCompress(void)
{
   if(!hNtDll)
   {
      hNtDll = LoadLibrary("NtDll.dll") ;
      if(hNtDll)
      {
         RtlGetCompressionWorkSpaceSize = (ptrRtlGetCompressionWorkSpaceSize) GetProcAddress(hNtDll,"RtlGetCompressionWorkSpaceSize") ;
         RtlDecompressBuffer = (ptrRtlDecompressBuffer) GetProcAddress(hNtDll,"RtlDecompressBuffer") ;

         return (TRUE) ;
      }
   }

   return (FALSE) ;
}
//-----------------------------------------------------------------------------------------------
int ExitUnCompress(void)
{
   if(hNtDll)
   {
      FreeLibrary(hNtDll) ;
      hNtDll = NULL ;
      return (TRUE) ;
   }

   return (FALSE) ;
}
//-----------------------------------------------------------------------------------------------
int TreatThisFile(LPBYTE __lpInputBuffer,LPSTR __lpszInputFileName)
{
   register   LPBYTE      _lpBytes ;
   register   LPBYTE      _lpOutputBuffer ;
   register   LPCOMPRESS   _lpCompress ;
            HANDLE      _hFile ;
            LPSTR      _lpszTmp ;
            DWORD      _dwTmp, _dwFinalUnCompressedSize ;
            int         _i, _bRes ;
            char      _szTmp[1024] ;
            char      _szExtension[1024] ;
            char      _szFileName[1024] ;

   _lpszTmp = _szTmp ;
   _dwTmp = (DWORD) *(LPDWORD) __lpInputBuffer ;

   if(_dwTmp == 'MAXI')
   {
      _lpCompress = (LPCOMPRESS) __lpInputBuffer ;
      _lpBytes = __lpInputBuffer + sizeof(COMPRESS) ;

      _lpOutputBuffer = VirtualAlloc(NULL,_lpCompress->dwUncompressedFileSize + 4096,MEM_COMMIT|MEM_RESERVE,PAGE_EXECUTE_READWRITE) ;
      if(_lpOutputBuffer)
      {
         if(_lpCompress->dwAlgorithme == COMPRESS_NT)
         {
            _dwFinalUnCompressedSize = 0 ;

            _bRes = RtlDecompressBuffer(COMPRESSION_FORMAT_LZNT1,
                           _lpOutputBuffer,
                           _lpCompress->dwUncompressedFileSize,
                           _lpBytes,
                           _lpCompress->dwCompressedFileSize,
                           &_dwFinalUnCompressedSize) ;
         }
         else
         {
            Huffman_Uncompress(_lpBytes,_lpOutputBuffer,_lpCompress->dwCompressedFileSize,_lpCompress->dwUncompressedFileSize) ;
         }

         PathStripPath(_lpCompress->szUncompressedFileName) ;

         if(PathFileExists(_lpCompress->szUncompressedFileName))
         {
            lstrcpy(_szExtension,PathFindExtension(_lpCompress->szUncompressedFileName)) ;
            lstrcpy(_szFileName,_lpCompress->szUncompressedFileName) ;

            *PathFindExtension(_szFileName) = '\0' ;

            _i = 0 ;

            do
            {
               wsprintf(_lpszTmp,"%s (%3.3d)%s",_szFileName,_i,_szExtension) ;
               _i++ ;
            } while(PathFileExists(_lpszTmp)) ;
         }
         else
         {
            lstrcpy(_lpszTmp,_lpCompress->szUncompressedFileName) ;
         }
         
         _hFile = CreateFile(_lpszTmp,GENERIC_READ|GENERIC_WRITE,0,NULL,CREATE_ALWAYS,FILE_ATTRIBUTE_NORMAL,NULL) ;
         if(_hFile != INVALID_HANDLE_VALUE)
         {
            if(!MaxWriteFile(_hFile,_lpOutputBuffer,_lpCompress->dwUncompressedFileSize - 4096))
            {
               FlushFileBuffers(_hFile) ;
               CloseHandle(_hFile) ;
               DeleteFile(_lpCompress->szUncompressedFileName) ;
               VirtualFree(_lpOutputBuffer,0,MEM_RELEASE) ;
               wsprintf(_lpszTmp,"{0001} : Cannot decompress\nthis file:\n%s",__lpszInputFileName) ;
               return (Error(_lpszTmp)) ;
            }

            FlushFileBuffers(_hFile) ;
            CloseHandle(_hFile) ;
            VirtualFree(_lpOutputBuffer,0,MEM_RELEASE) ;
            return (TRUE) ;
         }
         else
         {
            VirtualFree(_lpOutputBuffer,0,MEM_RELEASE) ;
            wsprintf(_lpszTmp,"{0003} : Cannot create\nthis file:\n%s",__lpszInputFileName) ;
            return (Error(_lpszTmp)) ;
         }
      }
      else
      {
         wsprintf(_lpszTmp,"{0004} : Cannot alloc memory\nfor this file:\n%s",__lpszInputFileName) ;
         return (Error(_lpszTmp)) ;
      }
   }
   else
   {
      wsprintf(_lpszTmp,"{0000} : Bad file:\n%s",__lpszInputFileName) ;
      return (Error(_lpszTmp)) ;
   }

    return (FALSE) ;
}


[TimoVJL]

A GetModuleHandleA("ntdll.dll") is enough as process have already loaded it?

[Jokaste]

You are right.

[Bitbeisser]

In my other algo I use HUFFMANN compression, this is poor, nearly 50%. I will try what you said.
What I am afraid using Huffman is its destructive algo. We know how it does when compressing jpeg.

Complete different issue, which you won't have if you indeed only compress limited range ASCII files...

Ralf