I am looking for a free compression library with no patent and usuable in C under Windows.
For the instance I use the Bcl (Basic Compression Library), I use the hoffman compression but I don't compress more than 50%.
I download files from the NASA, some files have a zize of 23Mb, if I continue my hard disk will be too small. My program which create the SQLite databases compress the source files, but it is not enought. [size=78%]https://sourceforge.net/projects/bcl/files/?source=navbar (https://sourceforge.net/projects/bcl/files/?source=navbar)[/size]
Who has a library for doing this? I don't want GZip nor Bzip.
Thanks to the ghosts
You can do this with,
RtlCompressBuffer
RtlGetCompressionWorkSpaceSize
RtlDecompressBuffer
available via ntdll.dll
https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/content/ntifs/nf-ntifs-rtlcompressbuffer
Link Maximum Compression (https://www.maximumcompression.com) gives some info what to expect.
What kind of data have to compress?
Itis only text. Ascii from 32 to 127
I will try Siekmanski (https://forum.pellesc.de/index.php?action=profile;u=13311)'s solution. Even if the algo is slow, but I hope it will compress better than 50%.
Thank You guys ;)
When I call RtlCompressBuffer it always returns 0xC000000D
Quote0xC000000D
STATUS_INVALID_PARAMETERAn invalid parameter was passed to a service or function. |
int CompressMax(LPSTR __lpszInputFileName,LPSTR __lpszOutputFileName)
{
LPBYTE _lpUnCompressedBuffer, _lpCompressedBuffer ;
LPVOID _lpWorkSpaceSize ;
DWORD _dwUnCompressedBufferSize, _dwFinalCompressedSize ;
DWORD _dwCompressBufferWorkSpaceSize, _dwCompressFragmentWorkSpaceSize ;
DWORD _dwCompressedBufferSize ;
HANDLE _hOutput ;
int _bRes ;
COMPRESS _Compress, *_lpCompress ;
char _szTmp[1024] ;
LPSTR _lpszTmp = _szTmp ;
_bRes = FALSE ;
_lpCompress = &_Compress ;
__stosq((unsigned long long *)_lpCompress,0,sizeof(COMPRESS) / ;
_lpUnCompressedBuffer = LoadThisFile(__lpszInputFileName,&_dwUnCompressedBufferSize) ;
if(_lpUnCompressedBuffer)
{
_dwCompressedBufferSize = _dwUnCompressedBufferSize + 4096 ;
_lpCompressedBuffer = MemoryAlloc(_dwCompressedBufferSize) ;
if(_lpCompressedBuffer)
{
_dwCompressBufferWorkSpaceSize = _dwCompressedBufferSize ;
_dwCompressFragmentWorkSpaceSize = 0 ;
_bRes = RtlGetCompressionWorkSpaceSize( COMPRESSION_ENGINE_MAXIMUM,
&_dwCompressBufferWorkSpaceSize,
&_dwCompressFragmentWorkSpaceSize) ;
_lpWorkSpaceSize = (LPVOID) MemoryAlloc(_dwCompressBufferWorkSpaceSize) ;
if(_lpWorkSpaceSize)
{
_lpCompress->dwAlgorithme = COMPRESS_NT ;
_dwFinalCompressedSize = 0 ;
_bRes = RtlCompressBuffer( COMPRESSION_ENGINE_MAXIMUM,
_lpUnCompressedBuffer,
_dwUnCompressedBufferSize,
_lpCompressedBuffer,
_dwCompressedBufferSize,
4096,
&_dwFinalCompressedSize,
_lpWorkSpaceSize) ;
MemoryFree(_lpWorkSpaceSize) ;
if(_bRes != STATUS_SUCCESS)
I would like some help.
I have found my error. I have added COMPRESSION_FORMAT_LZNT1 to the first parameter.
See the debug image joint to this post. The compression ratio is 46.22%. I expected better!
https://www.maximumcompression.com/data/text.php
Give a link for your example to test.
LZ family could beat it?
PAQ family is just far too slow to be usable, don't except those to finish in same day.
Windows 8 -> use XPRESS+huffman as COMPRESSION_ENGINE_MAXIMUM ?
In my other algo I use HUFFMANN compression, this is poor, nearly 50%. I will try what you said.
What I am afraid using Huffman is its destructive algo. We know how it does when compressing jpeg.
Except COMPRESSION_FORMAT_LZNT1 all the others retunr 0C000025F : STATUS_UNSUPPORTED_COMPRESSION
Final compression function code :
#include <Windows.h>
#include "huffman.h"
#include "MaxStruct.h"
#include "MaxUtils.h"
#include "Log.h"
int Error(LPSTR __lpszMessage) ;
size_t StringLength(const char * str) ;
LPVOID MemorySetTo0(LPVOID __lpszBuffer,__int64 __qwSize) ;
LPVOID MemoryAlloc(DWORD __dwSize) ;
void MemoryFree(LPVOID __lpBuffer) ;
typedef int (*ptrRtlCompressBuffer)(USHORT,PUCHAR,ULONG,PUCHAR,ULONG,ULONG,PULONG,PVOID) ;
typedef int (*ptrRtlGetCompressionWorkSpaceSize)(USHORT,PULONG,PULONG) ;
ptrRtlGetCompressionWorkSpaceSize RtlGetCompressionWorkSpaceSize ;
ptrRtlCompressBuffer RtlCompressBuffer ;
HINSTANCE hNtDll ;
int InitCompress(void)
{
if(!hNtDll)
{
hNtDll = LoadLibrary("NtDll.dll") ;
if(hNtDll)
{
RtlGetCompressionWorkSpaceSize = (ptrRtlGetCompressionWorkSpaceSize) GetProcAddress(hNtDll,"RtlGetCompressionWorkSpaceSize") ;
RtlCompressBuffer = (ptrRtlCompressBuffer) GetProcAddress(hNtDll,"RtlCompressBuffer") ;
return (TRUE) ;
}
}
return (FALSE) ;
}
int ExitCompress(void)
{
if(hNtDll)
{
FreeLibrary(hNtDll) ;
hNtDll = NULL ;
return (TRUE) ;
}
return (FALSE) ;
}
LPBYTE LoadThisFile(LPSTR __lpszInputFileName,DWORD *__lpdwBufferSize)
{
HANDLE _hInput ;
DWORD _dwFileSizeLow, _dwFileSizeHigh ;
DWORD _dwTmp ;
LPBYTE _lpBuffer, _lpTmp ;
DWORD _dwBufferSize ;
LPSTR _lpszTmp ;
char _szTmp[1024] ;
_lpBuffer = _lpTmp = NULL ;
_lpszTmp = _szTmp ;
*__lpdwBufferSize = 0 ;
__stosq((unsigned long long *)_lpszTmp,0,sizeof(_szTmp) / ;
_hInput = CreateFile(__lpszInputFileName,GENERIC_READ,0,NULL,OPEN_EXISTING,FILE_ATTRIBUTE_NORMAL,NULL) ;
if(_hInput == INVALID_HANDLE_VALUE)
{
wsprintf(_lpszTmp,"Cannot open this file: %s",__lpszInputFileName) ;
Log(_lpszTmp) ;
return ((LPBYTE) Error(_lpszTmp)) ;
}
_dwFileSizeLow = GetFileSize(_hInput,&_dwFileSizeHigh) ;
if(_dwFileSizeHigh)
{
CloseHandle(_hInput) ;
wsprintf(_lpszTmp,"This file is too big: %s",__lpszInputFileName) ;
Log(_lpszTmp) ;
return ((LPBYTE) Error(_lpszTmp)) ;
}
if(!_dwFileSizeLow && !_dwFileSizeHigh)
{
CloseHandle(_hInput) ;
wsprintf(_lpszTmp,"This file is empty: %s",__lpszInputFileName) ;
Log(_lpszTmp) ;
return ((LPBYTE) Error(_lpszTmp)) ;
}
_dwBufferSize = _dwFileSizeLow + 4096 ;
_lpBuffer = MemoryAlloc(_dwBufferSize) ;
if(!_lpBuffer)
{
CloseHandle(_hInput) ;
wsprintf(_lpszTmp,"Not enougth memory available for this file: %s",__lpszInputFileName) ;
Log(_lpszTmp) ;
return ((LPBYTE) Error(_lpszTmp)) ;
}
FlushFileBuffers(_hInput) ;
if(!ReadFile(_hInput,_lpBuffer,_dwFileSizeLow,&_dwTmp,NULL))
{
MemoryFree(_lpBuffer) ;
CloseHandle(_hInput) ;
wsprintf(_lpszTmp,"Cannot read this file: %s",__lpszInputFileName) ;
Log(_lpszTmp) ;
return ((LPBYTE) Error(_lpszTmp)) ;
}
CloseHandle(_hInput) ;
if(_dwFileSizeLow - _dwTmp)
{
MemoryFree(_lpBuffer) ;
wsprintf(_lpszTmp,"Error while reading this file: %s",__lpszInputFileName) ;
Log(_lpszTmp) ;
return ((LPBYTE) Error(_lpszTmp)) ;
}
*__lpdwBufferSize = _dwFileSizeLow + 4096 ;
return (_lpBuffer) ;
}
int CompressMax(LPSTR __lpszInputFileName,LPSTR __lpszOutputFileName)
{
LPBYTE _lpUnCompressedBuffer, _lpCompressedBuffer ;
LPVOID _lpWorkSpaceSize ;
DWORD _dwUnCompressedBufferSize, _dwFinalCompressedSize ;
DWORD _dwCompressBufferWorkSpaceSize, _dwCompressFragmentWorkSpaceSize ;
DWORD _dwCompressedBufferSize ;
HANDLE _hOutput ;
int _bRes ;
COMPRESS _Compress, *_lpCompress ;
char _szTmp[1024] ;
LPSTR _lpszTmp = _szTmp ;
_bRes = FALSE ;
_lpCompress = &_Compress ;
__stosq((unsigned long long *)_lpCompress,0,sizeof(COMPRESS) / ;
_lpUnCompressedBuffer = LoadThisFile(__lpszInputFileName,&_dwUnCompressedBufferSize) ;
if(_lpUnCompressedBuffer)
{
_dwCompressedBufferSize = _dwUnCompressedBufferSize + 4096 ;
_lpCompressedBuffer = MemoryAlloc(_dwCompressedBufferSize) ;
if(_lpCompressedBuffer)
{
_dwCompressBufferWorkSpaceSize = _dwCompressedBufferSize ;
_dwCompressFragmentWorkSpaceSize = 0 ;
_bRes = RtlGetCompressionWorkSpaceSize( COMPRESSION_FORMAT_LZNT1 | COMPRESSION_ENGINE_MAXIMUM,
&_dwCompressBufferWorkSpaceSize,
&_dwCompressFragmentWorkSpaceSize) ;
_lpWorkSpaceSize = (LPVOID) MemoryAlloc(_dwCompressBufferWorkSpaceSize) ;
if(_lpWorkSpaceSize)
{
_lpCompress->dwAlgorithme = COMPRESS_NT ;
_dwFinalCompressedSize = 0 ;
// https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/content/ntifs/nf-ntifs-rtlcompressbuffer
_bRes = RtlCompressBuffer( COMPRESSION_FORMAT_LZNT1 | COMPRESSION_ENGINE_MAXIMUM,
_lpUnCompressedBuffer,
_dwUnCompressedBufferSize,
_lpCompressedBuffer,
_dwCompressedBufferSize,
4096,
&_dwFinalCompressedSize,
_lpWorkSpaceSize) ;
MemoryFree(_lpWorkSpaceSize) ;
if(_bRes != STATUS_SUCCESS)
{
_lpCompress->dwAlgorithme = COMPRESS_HUFFMAN ;
_dwFinalCompressedSize = Huffman_Compress(_lpUnCompressedBuffer,_lpCompressedBuffer,_dwUnCompressedBufferSize) ;
}
}
else
{
_lpCompress->dwAlgorithme = COMPRESS_HUFFMAN ;
_dwFinalCompressedSize = Huffman_Compress(_lpUnCompressedBuffer,_lpCompressedBuffer,_dwUnCompressedBufferSize) ;
}
if(_dwFinalCompressedSize)
{
_hOutput = CreateFile(__lpszOutputFileName,GENERIC_WRITE,0,NULL,CREATE_ALWAYS,FILE_ATTRIBUTE_NORMAL,NULL) ;
if(_hOutput == INVALID_HANDLE_VALUE)
{
FlushFileBuffers(_hOutput) ;
CloseHandle(_hOutput) ;
DeleteFile(__lpszOutputFileName) ;
wsprintf(_lpszTmp,"Cannot open this file: %s",__lpszOutputFileName) ;
Log(_lpszTmp) ;
_bRes = Error(_lpszTmp) ;
}
else
{
_lpCompress->dwSignature = 'MAXI' ;
_lpCompress->dwMethod = METHOD_COMPRESS ;
_lpCompress->dwUncompressedFileSize = _dwUnCompressedBufferSize ;
_lpCompress->dwCompressedFileSize = _dwFinalCompressedSize ;
lstrcpy(_lpCompress->szUncompressedFileName,__lpszInputFileName) ;
if(MaxWriteFile(_hOutput,(LPBYTE) _lpCompress,sizeof(COMPRESS)))
{
if(MaxWriteFile(_hOutput,_lpCompressedBuffer,_dwFinalCompressedSize))
{
FlushFileBuffers(_hOutput) ;
_bRes = TRUE ;
}
}
else
{
wsprintf(_lpszTmp,"Error while writing to file: %s",__lpszOutputFileName) ;
Log(_lpszTmp) ;
_bRes = Error(_lpszTmp) ;
}
CloseHandle(_hOutput) ;
if(!_bRes)
DeleteFile(__lpszOutputFileName) ;
}
}
else
{
wsprintf(_lpszTmp,"Error while compressing file: %s",__lpszInputFileName) ;
Log(_lpszTmp) ;
_bRes = Error(_lpszTmp) ;
}
MemoryFree(_lpCompressedBuffer) ;
}
MemoryFree(_lpUnCompressedBuffer) ;
}
return (_bRes) ;
}
Final decompression function code :
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include <commctrl.h>
#include <Commdlg.h >
#include <Shlwapi.h >
#include <Shellapi.h>
#include "UnCompress.h"
#include "MaxStruct.h"
#include "MaxUtils.h"
#include "huffman.h"
#include "Common.h"
DWORD Error(LPSTR __lpszErrorMessage) ;
DWORD Information(LPSTR __lpszInformationMessage) ;
size_t StringLength (const char * str) ;
LPVOID MemorySetTo0(LPVOID __lpszBuffer,__int64 __qwSize) ;
typedef int (*ptrRtlDecompressBuffer)(USHORT,PUCHAR,ULONG,PUCHAR,ULONG,PULONG) ;
typedef int (*ptrRtlGetCompressionWorkSpaceSize)(USHORT,PULONG,PULONG) ;
ptrRtlGetCompressionWorkSpaceSize RtlGetCompressionWorkSpaceSize ;
ptrRtlDecompressBuffer RtlDecompressBuffer ;
//-----------------------------------------------------------------------------------------------
HINSTANCE hInstance ;
HINSTANCE hNtDll ;
//-----------------------------------------------------------------------------------------------
int InitUnCompress(void)
{
if(!hNtDll)
{
hNtDll = LoadLibrary("NtDll.dll") ;
if(hNtDll)
{
RtlGetCompressionWorkSpaceSize = (ptrRtlGetCompressionWorkSpaceSize) GetProcAddress(hNtDll,"RtlGetCompressionWorkSpaceSize") ;
RtlDecompressBuffer = (ptrRtlDecompressBuffer) GetProcAddress(hNtDll,"RtlDecompressBuffer") ;
return (TRUE) ;
}
}
return (FALSE) ;
}
//-----------------------------------------------------------------------------------------------
int ExitUnCompress(void)
{
if(hNtDll)
{
FreeLibrary(hNtDll) ;
hNtDll = NULL ;
return (TRUE) ;
}
return (FALSE) ;
}
//-----------------------------------------------------------------------------------------------
int TreatThisFile(LPBYTE __lpInputBuffer,LPSTR __lpszInputFileName)
{
register LPBYTE _lpBytes ;
register LPBYTE _lpOutputBuffer ;
register LPCOMPRESS _lpCompress ;
HANDLE _hFile ;
LPSTR _lpszTmp ;
DWORD _dwTmp, _dwFinalUnCompressedSize ;
int _i, _bRes ;
char _szTmp[1024] ;
char _szExtension[1024] ;
char _szFileName[1024] ;
_lpszTmp = _szTmp ;
_dwTmp = (DWORD) *(LPDWORD) __lpInputBuffer ;
if(_dwTmp == 'MAXI')
{
_lpCompress = (LPCOMPRESS) __lpInputBuffer ;
_lpBytes = __lpInputBuffer + sizeof(COMPRESS) ;
_lpOutputBuffer = VirtualAlloc(NULL,_lpCompress->dwUncompressedFileSize + 4096,MEM_COMMIT|MEM_RESERVE,PAGE_EXECUTE_READWRITE) ;
if(_lpOutputBuffer)
{
if(_lpCompress->dwAlgorithme == COMPRESS_NT)
{
_dwFinalUnCompressedSize = 0 ;
_bRes = RtlDecompressBuffer(COMPRESSION_FORMAT_LZNT1,
_lpOutputBuffer,
_lpCompress->dwUncompressedFileSize,
_lpBytes,
_lpCompress->dwCompressedFileSize,
&_dwFinalUnCompressedSize) ;
}
else
{
Huffman_Uncompress(_lpBytes,_lpOutputBuffer,_lpCompress->dwCompressedFileSize,_lpCompress->dwUncompressedFileSize) ;
}
PathStripPath(_lpCompress->szUncompressedFileName) ;
if(PathFileExists(_lpCompress->szUncompressedFileName))
{
lstrcpy(_szExtension,PathFindExtension(_lpCompress->szUncompressedFileName)) ;
lstrcpy(_szFileName,_lpCompress->szUncompressedFileName) ;
*PathFindExtension(_szFileName) = '\0' ;
_i = 0 ;
do
{
wsprintf(_lpszTmp,"%s (%3.3d)%s",_szFileName,_i,_szExtension) ;
_i++ ;
} while(PathFileExists(_lpszTmp)) ;
}
else
{
lstrcpy(_lpszTmp,_lpCompress->szUncompressedFileName) ;
}
_hFile = CreateFile(_lpszTmp,GENERIC_READ|GENERIC_WRITE,0,NULL,CREATE_ALWAYS,FILE_ATTRIBUTE_NORMAL,NULL) ;
if(_hFile != INVALID_HANDLE_VALUE)
{
if(!MaxWriteFile(_hFile,_lpOutputBuffer,_lpCompress->dwUncompressedFileSize - 4096))
{
FlushFileBuffers(_hFile) ;
CloseHandle(_hFile) ;
DeleteFile(_lpCompress->szUncompressedFileName) ;
VirtualFree(_lpOutputBuffer,0,MEM_RELEASE) ;
wsprintf(_lpszTmp,"{0001} : Cannot decompress\nthis file:\n%s",__lpszInputFileName) ;
return (Error(_lpszTmp)) ;
}
FlushFileBuffers(_hFile) ;
CloseHandle(_hFile) ;
VirtualFree(_lpOutputBuffer,0,MEM_RELEASE) ;
return (TRUE) ;
}
else
{
VirtualFree(_lpOutputBuffer,0,MEM_RELEASE) ;
wsprintf(_lpszTmp,"{0003} : Cannot create\nthis file:\n%s",__lpszInputFileName) ;
return (Error(_lpszTmp)) ;
}
}
else
{
wsprintf(_lpszTmp,"{0004} : Cannot alloc memory\nfor this file:\n%s",__lpszInputFileName) ;
return (Error(_lpszTmp)) ;
}
}
else
{
wsprintf(_lpszTmp,"{0000} : Bad file:\n%s",__lpszInputFileName) ;
return (Error(_lpszTmp)) ;
}
return (FALSE) ;
}
A GetModuleHandleA("ntdll.dll") is enough as process have already loaded it?
You are right.
Quote from: Jokaste on June 30, 2018, 07:53:30 PM
In my other algo I use HUFFMANN compression, this is poor, nearly 50%. I will try what you said.
What I am afraid using Huffman is its destructive algo. We know how it does when compressing jpeg.
Complete different issue, which you won't have if you indeed only compress limited range ASCII files...
Ralf