Hello the World
I would like to know if there is a limit in number of files to research using these functions.
I have 35 000 files into 145 folders.
After making many attempts the program stops after file #4977
I have checked the file after the last it is ok and when I select this folder the program list all files.
I can say it is not a program bug and all the files are correct.
MayDay.
Please give me ideas.
Hi HellOfMice,
Are you using a recursive funtion to search files? Maybe, you are running out of stack space. You could try to adjust the stack :
/STACK option (POLINK)
Syntax:
/STACK:reserve [, allocate]
Description:
The /STACK option specifies the size of the stack.
The reserve argument specifies the total amount of virtual memory. The default value is 1 MB. The linker will round up the value to the nearest DWORD boundary.
The optional allocate argument is interpreted differently by the operating-systems. In Windows NT it specifies how much memory is allocated each time. The default value is 4 KB.
No it is not that kind of problem. I found other people with this problem when scanning for 40000 files.
In that case use FindFirstFileEx
One case:
Too Many Files Open using FindNextFile and FindFirstFile (https://www.tek-tips.com/threads/too-many-files-open-using-findnextfile-and-findfirstfile.876612/)
my test result:
file 261327 \code\WinSDK\v7.1A\Lib\x64\xpsprint.lib
File count: 261327
Thank You Timo I have a look and I take it. :D :D :D :D
Here is my function
void SearchForFile(HWND __hWnd,LPSTR __lpszCurrentFolderName,LPBYTE __lpColors)
{
register int _iIndexColor ;
register int _iNumberOfColors ;
alignas(__int64) LPDWORD _lpImageBits ;
alignas(__int64) WIN32_FIND_DATA _FileFind ;
alignas(__int64) HANDLE _hFind ;
alignas(__int64) HSTMT _hSQLiteFind ;
alignas(__int64) LPSTR _lpszTemp, _lpszErrorMsg ;
alignas(__int64) LPIMAGEINFOS _lpImageInfos ;
alignas(int) clock_t _InitialTimeStamp, _iCounter ;
alignas(int) int _i, _iImageSize ;
alignas(int) char _szParentFolder[MAX_PATH + 4] ;
alignas(int) char _szImageFile[MAX_PATH + 4] ;
alignas(int) char _szNewImageFile[MAX_PATH + 4] ;
alignas(int) char _szTmp1[256] ;
_InitialTimeStamp = clock() ;
_iCounter = iAnalyzerRecordNumber ;
sqlite3_snprintf(sizeof(_szParentFolder),_szParentFolder,"%s\\*.*",__lpszCurrentFolderName) ;
sqlite3_snprintf(sizeof(_szNewImageFile),_szNewImageFile,"%s\\%%s",__lpszCurrentFolderName) ;
// _hFind=FindFirstFileEx(_szParentFolder,FindExInfoBasic,&_FileFind,FindExSearchNameMatch,NULL,FIND_FIRST_EX_LARGE_FETCH) ;
_hFind = FindFirstFile(_szParentFolder,&_FileFind) ;
if(_hFind == INVALID_HANDLE_VALUE)
return ;
do
{
if(lstrcmp(_FileFind.cFileName,".") != 0)
{
if(lstrcmp(_FileFind.cFileName,"..") != 0)
{
if((_FileFind.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0)
{
if((_FileFind.nFileSizeHigh + _FileFind.nFileSizeLow) > 0)
{
if(_FileFind.nFileSizeHigh == 0)
{
sqlite3_snprintf(sizeof(_szImageFile),_szImageFile,_szNewImageFile,_FileFind.cFileName) ;
_lpImageInfos = LoadImageFromFile(_szImageFile) ;
if(_lpImageInfos)
{
_lpImageBits = (LPDWORD) _lpImageInfos->lpImageBits ;
_iImageSize = _lpImageInfos->BitmapInfo.bmiHeader.bV5SizeImage / 4 ;
// _____________________________________________________________________________________________
// ____________________ Compute the number of colors ___________________________________________
// _____________________________________________________________________________________________
MemorySetTo0(__lpColors,sizeof(BYTE) * MAX_COLORS) ;
_iNumberOfColors = 0 ;
for(_i = 0 ; _i < _iImageSize ; _i++)
{
_iIndexColor = (int) (DWORD) *_lpImageBits ;
if(*(__lpColors + _iIndexColor) == 0)
{
*(__lpColors + _iIndexColor) = 1 ;
_iNumberOfColors++ ;
}
_lpImageBits++ ;
}
_lpImageInfos->SsiInfos.NumberOfColors = _iNumberOfColors ;
// _____________________________________________________________________________________________
// ____________________ Compute the Mean _______________________________________________________
// _____________________________________________________________________________________________
ComputeTheMean(_lpImageInfos,1) ;
// _____________________________________________________________________________________________
// ____________________ Compute deviations from the mean _______________________________________
// _____________________________________________________________________________________________
ComputeTheDeviation(_lpImageInfos,1) ;
// _____________________________________________________________________________________________
// ____________________ Compute Standard Deviation _____________________________________________
// _____________________________________________________________________________________________
ComputeTheStandardDeviation(_lpImageInfos,1) ;
// _____________________________________________________________________________________________
// ____________________ Compute Ratio Image Width / Image Height ______________________________
// _____________________________________________________________________________________________
_lpImageInfos->ImageSource.dRatio = ((double) _lpImageInfos->BitmapInfo.bmiHeader.bV5Width) / ((double) _lpImageInfos->BitmapInfo.bmiHeader.bV5Height) ;
// _____________________________________________________________________________________________
// ____________________ Compute Image Orientation and Form _____________________________________
// _____________________________________________________________________________________________
if(_lpImageInfos->BitmapInfo.bmiHeader.bV5Width > _lpImageInfos->BitmapInfo.bmiHeader.bV5Height)
_lpImageInfos->ImageSource.dwSensImage = IMAGE_ORIENTATION_LANDSCAPE ;
else
{
if(_lpImageInfos->BitmapInfo.bmiHeader.bV5Width == _lpImageInfos->BitmapInfo.bmiHeader.bV5Height)
_lpImageInfos->ImageSource.dwSensImage = IMAGE_ORIENTATION_SQUARE ;
else
_lpImageInfos->ImageSource.dwSensImage = IMAGE_ORIENTATION_PORTRAIT ;
}
_lpImageInfos->SsiInfos.FileTime = _FileFind.ftCreationTime ;
_lpImageInfos->SsiInfos.FileSize = _FileFind.nFileSizeLow ;
Keccak((LPBYTE) _lpImageInfos->lpImageBits,_lpImageInfos->BitmapInfo.bmiHeader.bV5SizeImage,SQLiteBuffer) ;
Hex2Str(SQLiteBuffer,_lpImageInfos->ImageSource.szHashImage,256 / ;
_lpszTemp = ReadWholeFile(_szImageFile) ;
if(_lpszTemp)
{
Keccak((LPBYTE) _lpszTemp,_FileFind.nFileSizeLow,SQLiteBuffer) ;
Hex2Str(SQLiteBuffer,_lpImageInfos->ImageSource.szHashWoleFile,256 / ;
VirtualFree(_lpszTemp,0,MEM_RELEASE) ;
}
// _____________________________________________________________________________________________
// ____________________ Store results __________________________________________________________
// _____________________________________________________________________________________________
_lpImageInfos->SsiInfos.FileTime = _FileFind.ftCreationTime ;
_lpImageInfos->SsiInfos.FileSize = _FileFind.nFileSizeLow ;
ConvertTimeToString(&_FileFind.ftCreationTime,_szTmp1,sizeof(_szTmp1)) ;
/* sqlite3_snprintf(sizeof(SQLiteBuffer),SQLiteBuffer,
"\n*******************************\n"
" File (%5.5d) = %s\n"
" File Time = %s\n"
" File Size = %d bytes\n"
" In memory size = %u bytes\n"
" Image Width x Height = %d x %d\n"
" StdDev = %8.3f,%8.3f,%8.3f\n"
" Deviation = %8.3f,%8.3f,%8.3f\n"
" Mean = %8.3f,%8.3f,%8.3f\n"
" Number of UNIQUE Colors = %d\n"
" Hash Image = %s\n"
" Hash File = %s\n"
"*******************************\n",
iAnalyzerRecordNumber,_szImageFile,_szTmp1,_lpImageInfos->SsiInfos.FileSize,_lpImageInfos->BitmapInfo.bmiHeader.bV5SizeImage,
_lpImageInfos->BitmapInfo.bmiHeader.bV5Width,_lpImageInfos->BitmapInfo.bmiHeader.bV5Height,
_lpImageInfos->ImageSource.StdDevRed,_lpImageInfos->ImageSource.StdDevGreen,_lpImageInfos->ImageSource.StdDevBlue,
_lpImageInfos->ImageSource.DevRed,_lpImageInfos->ImageSource.DevGreen,_lpImageInfos->ImageSource.DevBlue,
_lpImageInfos->ImageSource.MeanRed,_lpImageInfos->ImageSource.MeanGreen,_lpImageInfos->ImageSource.MeanBlue,
_lpImageInfos->SsiInfos.NumberOfColors,_lpImageInfos->ImageSource.szHashImage,_lpImageInfos->ImageSource.szHashWoleFile) ;
Log(SQLiteBuffer) ;*/
MakeGrayImage(__hWnd,_szImageFile,_lpImageInfos,Profile.WorkingImageSize,Profile.GrayConversionMethod,Profile.ResizeMethod) ;
// _____________________________________________________________________________________________
// ____________________ Store the datas ________________________________________________________
// _____________________________________________________________________________________________
sqlite3_snprintf(sizeof(SQLiteBuffer),SQLiteBuffer,"File number %d",iAnalyzerRecordNumber) ;
Log(SQLiteBuffer) ;
sqlite3_snprintf(sizeof(SQLiteBuffer),SQLiteBuffer,szInsertAnalyzer,
iAnalyzerRecordNumber, // RecordNumber
_szImageFile, // FullFileName
_FileFind.ftCreationTime.dwLowDateTime, // ColorsFileCreationDateLow
_FileFind.ftCreationTime.dwHighDateTime, // ColorsFileCreationDateHigh
_FileFind.nFileSizeLow, // ColorsFileSize
_lpImageInfos->ImageSource.szHashWoleFile, // ColorsHashFileContent
_lpImageInfos->ImageSource.szHashImage, // ColorsHashImage
_lpImageInfos->SsiInfos.ImageWidth, // ColorsImageWidth
_lpImageInfos->SsiInfos.ImageHeight, // ColorsImageHeight
_lpImageInfos->ImageSource.dwSensImage, // ColorsImageOrientation
_lpImageInfos->SsiInfos.NumberOfColors, // ColorsNumberUniqueColors
_lpImageInfos->ImageSource.dRatio, // ColorsImageRatio
_lpImageInfos->ImageSource.StdDevRed, // ColorsStdDevRed
_lpImageInfos->ImageSource.StdDevGreen, // ColorsStdDevGreen
_lpImageInfos->ImageSource.StdDevBlue, // ColorsStdDevBlue
_lpImageInfos->ImageSource.DevRed, // ColorsDeviationRed
_lpImageInfos->ImageSource.DevGreen, // ColorsDeviationGreen
_lpImageInfos->ImageSource.DevBlue, // ColorsDeviationBlue
_lpImageInfos->ImageSource.MeanRed, // ColorsMeanRed
_lpImageInfos->ImageSource.MeanGreen, // ColorsMeanGreen
_lpImageInfos->ImageSource.MeanBlue, // ColorsMeanBlue
_lpImageInfos->SsiWorkInfos.ImageWidth, // GrayImageWidth
_lpImageInfos->SsiWorkInfos.ImageHeight, // GrayImageHeight
_lpImageInfos->ImageWork.dwSensImage, // GrayImageOrientation
_lpImageInfos->SsiWorkInfos.NumberOfColors, // GrayNumberUniqueColors
_lpImageInfos->ImageWork.dRatio, // GrayImageRatio
_lpImageInfos->ImageWork.StdDevRed, // GrayStdDevRed
_lpImageInfos->ImageWork.StdDevGreen, // GrayStdDevGreen
_lpImageInfos->ImageWork.StdDevBlue, // GrayStdDevBlue
_lpImageInfos->ImageWork.DevRed, // GrayDeviationRed
_lpImageInfos->ImageWork.DevGreen, // GrayDeviationGreen
_lpImageInfos->ImageWork.DevBlue, // GrayDeviationBlue
_lpImageInfos->ImageWork.MeanRed, // GrayMeanRed
_lpImageInfos->ImageWork.MeanGreen, // GrayMeanGreen
_lpImageInfos->ImageWork.MeanBlue) ; // GrayMeanBlue
// Log(SQLiteBuffer) ;
if(sqlite3_prepare_v2(hSQLite,SQLiteBuffer,-1,&_hSQLiteFind,NULL) != SQLITE_OK)
{
Image_Free(_lpImageInfos) ;
SQLite_Error(__hWnd) ;
Error(__hWnd,"Error while creating INSERT query for table Analyzer") ;
}
if(sqlite3_exec(hSQLite,SQLiteBuffer,NULL,NULL,&_lpszErrorMsg) != SQLITE_OK)
{
Image_Free(_lpImageInfos) ;
Error(__hWnd,_lpszErrorMsg) ;
sqlite3_free(_lpszErrorMsg) ;
Image_Free(_lpImageInfos) ;
break ;
}
iAnalyzerRecordNumber++ ;
Image_Free(_lpImageInfos) ;
}
}
}
}
}
}
} while(FindNextFile(_hFind,&_FileFind) != 0) ;
if(GetLastError() != ERROR_NO_MORE_FILES)
Error(__hWnd,"Cannot continue the research, sorry") ;
_InitialTimeStamp = clock() - _InitialTimeStamp ;
sqlite3_snprintf(sizeof(SQLiteBuffer),SQLiteBuffer,"\n**********************\n%d files treated in %lu clock hits\n**********************\n",iAnalyzerRecordNumber - _iCounter,_InitialTimeStamp) ;
Log(SQLiteBuffer) ;
FindClose(_hFind) ;
return ;
}
Timo I tested you program and it find all the 35 023 files! :D
If I only search the files for all the directories I get the same result. Now I have to search where is the problem.
Thank you for your help
Now my problem is solved.
I searched all the files the stored into a SQLite3 table
When finished
I cross this table to load each file stored in the table and load them one by one to fill another table with more informations
[vendredi 1 novembre 2024 09:23:44] : File #35013 = C:\Users\_______\Documents\# Collections\# Women\# White Background\# White Background-00163.jpg
[vendredi 1 novembre 2024 09:23:44] : File #35014 = C:\Users\_______\Documents\# Collections\# Women\# White Background\# White Background-00164.jpg
[vendredi 1 novembre 2024 09:23:44] : File #35015 = C:\Users\_______\Documents\# Collections\# Women\# White Background\# White Background-00165.jpg
[vendredi 1 novembre 2024 09:23:44] : File #35016 = C:\Users\_______\Documents\# Collections\# Women\# White Background\# White Background-00166.jpg
[vendredi 1 novembre 2024 09:23:44] : File #35017 = C:\Users\_______\Documents\# Collections\# Women\# White Background\# White Background-00167.jpg
[vendredi 1 novembre 2024 09:23:44] : Searching for files ended
[vendredi 1 novembre 2024 09:23:44] : Closing Log file
[vendredi 1 novembre 2024 09:23:44] :
*** Log Stoped ***
Thank you to the people who helped me