binascii.crc32 results not matching
Larry Bates
larry.bates at websafe.com
Sat Dec 10 11:47:58 EST 2005
Peter Hansen wrote:
> Larry Bates wrote:
>
>> I'm trying to get the results of binascii.crc32
>> to match the results of another utility that produces
>> 32 bit unsigned CRCs.
>
>
> What other utility? As Tim says, there are many CRC32s... the
> background notes on this one happen to stumble out at the top of the
> list in response to googling for "zip file crc32 checksum polynomial",
> though I'm sure there are easier ways. The binascii docs say its CRC32
> is compatible with the Zip file checksum, but they don't describe it
> further.
>
> Generally CRCs are described in terms of their "polynomial", though just
> quoting that isn't sufficient to describe their behaviour, but if you
> happen to know the polynomial for your utility, someone else can
> probably point you to a more appropriate routine, or perhaps explain
> what you were doing wrong if the binascii one is actually the right one..
>
> -Peter
>
It was a .DLL written by an employee that has long since
left the company. We want to move the code to Linux for
nightly checking of files. I don't know what to do but
post some long code. See below:
/************************************************************************
INCLUDES
************************************************************************/
#include <windows.h>
#include <string.h>
#include <malloc.h>
#include <stdio.h>
#include <stdlib.h>
#include "filecrc.h"
/************************************************************************
ModuleName: filecrc.c
Author: Syscon Computers - Modified by Barry Weck
Project: PowerBuilder External File CRC function
Date created: May. 18, 1999
Last Modified: Jun. 09, 1999
Module Owner: Syscon Computers
Module description:
This module implements a algorithm for calculating the CRC (Cyclic
Redundency Check) of a binary file. This function is meant to be
compiled as a 16 bit Microsoft Windows (Tm) DLL with either the 16 bit C
compilers of Borland or Microsoft. Compilation under other other
compilers has not been tested. The requirement of "16-bit DLL" is
nessesitated by the version of PowerBuilder used by syscon.
This module was written by a third party and was then modified by
subcontractor Barry Weck. The code is copyrighted and owned by Syscon
Computers, Inc. of Tuscaloosa Alabama (C) 1999.
************************************************************************/
/************************************************************************
DATA
************************************************************************/
static unsigned long ccitt_32[256] =
{
0x00000000UL, 0x77073096UL, 0xee0e612cUL, 0x990951baUL, 0x076dc419UL,
0x706af48fUL, 0xe963a535UL, 0x9e6495a3UL,
0x0edb8832UL, 0x79dcb8a4UL, 0xe0d5e91eUL, 0x97d2d988UL, 0x09b64c2bUL,
0x7eb17cbdUL, 0xe7b82d07UL, 0x90bf1d91UL,
0x1db71064UL, 0x6ab020f2UL, 0xf3b97148UL, 0x84be41deUL, 0x1adad47dUL,
0x6ddde4ebUL, 0xf4d4b551UL, 0x83d385c7UL,
0x136c9856UL, 0x646ba8c0UL, 0xfd62f97aUL, 0x8a65c9ecUL, 0x14015c4fUL,
0x63066cd9UL, 0xfa0f3d63UL, 0x8d080df5UL,
0x3b6e20c8UL, 0x4c69105eUL, 0xd56041e4UL, 0xa2677172UL, 0x3c03e4d1UL,
0x4b04d447UL, 0xd20d85fdUL, 0xa50ab56bUL,
0x35b5a8faUL, 0x42b2986cUL, 0xdbbbc9d6UL, 0xacbcf940UL, 0x32d86ce3UL,
0x45df5c75UL, 0xdcd60dcfUL, 0xabd13d59UL,
0x26d930acUL, 0x51de003aUL, 0xc8d75180UL, 0xbfd06116UL, 0x21b4f4b5UL,
0x56b3c423UL, 0xcfba9599UL, 0xb8bda50fUL,
0x2802b89eUL, 0x5f058808UL, 0xc60cd9b2UL, 0xb10be924UL, 0x2f6f7c87UL,
0x58684c11UL, 0xc1611dabUL, 0xb6662d3dUL,
0x76dc4190UL, 0x01db7106UL, 0x98d220bcUL, 0xefd5102aUL, 0x71b18589UL,
0x06b6b51fUL, 0x9fbfe4a5UL, 0xe8b8d433UL,
0x7807c9a2UL, 0x0f00f934UL, 0x9609a88eUL, 0xe10e9818UL, 0x7f6a0dbbUL,
0x086d3d2dUL, 0x91646c97UL, 0xe6635c01UL,
0x6b6b51f4UL, 0x1c6c6162UL, 0x856530d8UL, 0xf262004eUL, 0x6c0695edUL,
0x1b01a57bUL, 0x8208f4c1UL, 0xf50fc457UL,
0x65b0d9c6UL, 0x12b7e950UL, 0x8bbeb8eaUL, 0xfcb9887cUL, 0x62dd1ddfUL,
0x15da2d49UL, 0x8cd37cf3UL, 0xfbd44c65UL,
0x4db26158UL, 0x3ab551ceUL, 0xa3bc0074UL, 0xd4bb30e2UL, 0x4adfa541UL,
0x3dd895d7UL, 0xa4d1c46dUL, 0xd3d6f4fbUL,
0x4369e96aUL, 0x346ed9fcUL, 0xad678846UL, 0xda60b8d0UL, 0x44042d73UL,
0x33031de5UL, 0xaa0a4c5fUL, 0xdd0d7cc9UL,
0x5005713cUL, 0x270241aaUL, 0xbe0b1010UL, 0xc90c2086UL, 0x5768b525UL,
0x206f85b3UL, 0xb966d409UL, 0xce61e49fUL,
0x5edef90eUL, 0x29d9c998UL, 0xb0d09822UL, 0xc7d7a8b4UL, 0x59b33d17UL,
0x2eb40d81UL, 0xb7bd5c3bUL, 0xc0ba6cadUL,
0xedb88320UL, 0x9abfb3b6UL, 0x03b6e20cUL, 0x74b1d29aUL, 0xead54739UL,
0x9dd277afUL, 0x04db2615UL, 0x73dc1683UL,
0xe3630b12UL, 0x94643b84UL, 0x0d6d6a3eUL, 0x7a6a5aa8UL, 0xe40ecf0bUL,
0x9309ff9dUL, 0x0a00ae27UL, 0x7d079eb1UL,
0xf00f9344UL, 0x8708a3d2UL, 0x1e01f268UL, 0x6906c2feUL, 0xf762575dUL,
0x806567cbUL, 0x196c3671UL, 0x6e6b06e7UL,
0xfed41b76UL, 0x89d32be0UL, 0x10da7a5aUL, 0x67dd4accUL, 0xf9b9df6fUL,
0x8ebeeff9UL, 0x17b7be43UL, 0x60b08ed5UL,
0xd6d6a3e8UL, 0xa1d1937eUL, 0x38d8c2c4UL, 0x4fdff252UL, 0xd1bb67f1UL,
0xa6bc5767UL, 0x3fb506ddUL, 0x48b2364bUL,
0xd80d2bdaUL, 0xaf0a1b4cUL, 0x36034af6UL, 0x41047a60UL, 0xdf60efc3UL,
0xa867df55UL, 0x316e8eefUL, 0x4669be79UL,
0xcb61b38cUL, 0xbc66831aUL, 0x256fd2a0UL, 0x5268e236UL, 0xcc0c7795UL,
0xbb0b4703UL, 0x220216b9UL, 0x5505262fUL,
0xc5ba3bbeUL, 0xb2bd0b28UL, 0x2bb45a92UL, 0x5cb36a04UL, 0xc2d7ffa7UL,
0xb5d0cf31UL, 0x2cd99e8bUL, 0x5bdeae1dUL,
0x9b64c2b0UL, 0xec63f226UL, 0x756aa39cUL, 0x026d930aUL, 0x9c0906a9UL,
0xeb0e363fUL, 0x72076785UL, 0x05005713UL,
0x95bf4a82UL, 0xe2b87a14UL, 0x7bb12baeUL, 0x0cb61b38UL, 0x92d28e9bUL,
0xe5d5be0dUL, 0x7cdcefb7UL, 0x0bdbdf21UL,
0x86d3d2d4UL, 0xf1d4e242UL, 0x68ddb3f8UL, 0x1fda836eUL, 0x81be16cdUL,
0xf6b9265bUL, 0x6fb077e1UL, 0x18b74777UL,
0x88085ae6UL, 0xff0f6a70UL, 0x66063bcaUL, 0x11010b5cUL, 0x8f659effUL,
0xf862ae69UL, 0x616bffd3UL, 0x166ccf45UL,
0xa00ae278UL, 0xd70dd2eeUL, 0x4e048354UL, 0x3903b3c2UL, 0xa7672661UL,
0xd06016f7UL, 0x4969474dUL, 0x3e6e77dbUL,
0xaed16a4aUL, 0xd9d65adcUL, 0x40df0b66UL, 0x37d83bf0UL, 0xa9bcae53UL,
0xdebb9ec5UL, 0x47b2cf7fUL, 0x30b5ffe9UL,
0xbdbdf21cUL, 0xcabac28aUL, 0x53b39330UL, 0x24b4a3a6UL, 0xbad03605UL,
0xcdd70693UL, 0x54de5729UL, 0x23d967bfUL,
0xb3667a2eUL, 0xc4614ab8UL, 0x5d681b02UL, 0x2a6f2b94UL, 0xb40bbe37UL,
0xc30c8ea1UL, 0x5a05df1bUL, 0x2d02ef8dL
};
int WINAPI LibMain(HINSTANCE hInst, WORD wDataSeg, WORD cbHeapSize, LPSTR
lpszCmdLine)
{
if (cbHeapSize != 0)
// UnLockData(0);
return 1;
}
int WINAPI WEP(int nParameter)
{
/*if (nParameter == WEP_SYSTEM_EXIT)
return 1;
else if (nParameter == WEP_FREE_DLL)
return 1;
else*/
return 1;
}
/************************************************************************
Function : CalculateCrc
Description : Calculates CRC for a sequence of BYTEs found in buffer
This routine is iterativly called as long as there are more
chunks of bytes
Parameters : BYTE *buffer - Next chunk of bytes for which a CRC is sought
long count - Number of bytes in this chunk
DWORD crc - Current value of CRC
short *sError - returned error (in case there is an array overrun)
Returns : DWORD CalculateCrc - Calculated CRC
************************************************************************/
DWORD CalculateCrc(BYTE *buffer, long count, DWORD crc, short *sError)
{
DWORD temp1;
DWORD temp2;
int Index;
/* Calculate the 32 bit CRC using the table lookup method. */
while ( count-- != 0 )
{
temp1 = ( crc >> 8 ) & 0x00FFFFFFUL;
Index = (( (int) crc ^ *buffer++ ) & 0xff);
if ((Index < 0) || (Index > 255))
{
*sError = FC_ERR_PROOF_OF_ARRAY_OVERRUN_IN_CRC_CODE;
return(0); /* Time to Panic: array overrun time!*/
}
temp2 = ccitt_32[Index];
crc = temp1 ^ temp2;
}
return( crc );
}
/************************************************************************
Function : long filesize(FILE *stream)
Description : Returns size of file given an already opened file stream
NOTE: Filesize Limit : 2^31
Parameters : FILE *stream - already opened file stream
Returns : Returns size of file
************************************************************************/
long filesize(FILE *stream)
{
long curpos, length;
curpos = ftell(stream);
fseek(stream, 0L, SEEK_END);
length = ftell(stream);
fseek(stream, curpos, SEEK_SET);
return length;
}
/************************************************************************
Function : GetCheckSum
Description : returns the checksum of the file
NOTE: Checksum Range Limit : 0..2^32
NOTE: Filesize Limit : 2^31
Parameters : char* pFileName - name of data file we are calculating CRC for
long lOffset - offset within file
long lLength - how much to check (0 means check entire file
from offset)
short sBufferSize - size of buffer used to read data
short sError - Error Code - see #defines in filecrc.h
unsigned long retval - returned crc of file if no error occured
if error, then this is set to zero
NOTE: Set this to initialize starting
point of CRC
Returns : Ignore the return value - Always equal to 1
************************************************************************/
int WINAPI GetCheckSum(char *pFileName, long lOffset, long lLength,
short sBufferSize, short *sError, unsigned long *retval)
{
BYTE *pBuffer=NULL; /* Current chunk of bytes
from file*/
FILE* pDataFile; /* File Pointer*/
BOOL bDone = FALSE; /* controls main loop*/
long lBytesRead, FileSize, lBytesLeftToRead; /* */
DWORD dwCrc = *retval; /* Current Calculated CRC;
Set to a special value at
the outset
by the caller
*/
*sError = FC_ERR_NOERROR; /* Assume no problems */
*retval=0;
if ((pDataFile = fopen(pFileName,"rb"))==NULL)
*sError = FC_ERR_FILENOTFOUND; /* Can't find the file */
else
{
FileSize = filesize(pDataFile); /* Get Filesize */
if (sBufferSize <= 0)
sBufferSize = FC_DEFAULT_BUFFER_SIZE; /* Sanity check for buffer
size */
/* If Length is 0, then just read the rest of the file FROM THE OFFSET */
if (lLength <= 0)
lLength = FileSize - lOffset;
lBytesLeftToRead = lLength; /* ...and that be how many bytes left to
read, matey; */
if (sBufferSize > lLength) /* OK, since we already checked for neg vals */
sBufferSize = (short)lLength; /* can only happen if lLength <=
"maxShort" */
pBuffer = (BYTE *)malloc(sBufferSize*sizeof(BYTE)); /* Now we can set the
buffer size */
if (lOffset >= FileSize)
*sError = FC_ERR_INVALIDOFFSET; /* oops!, offset past
EOF */
else if ((lOffset+lLength) > FileSize)
*sError = FC_ERR_INVALIDOFFSETPLUSLENGTH; /* oops!, trying to
read past EOF */
else if (pBuffer==NULL)
*sError = FC_ERR_OUTOFMEMORY;
if (*sError == FC_ERR_NOERROR) /* oops!, just plain
out of heap */
{
fseek(pDataFile,lOffset,SEEK_SET); /* start at offset */
while (!bDone)
{
lBytesRead = fread(pBuffer,sizeof(BYTE),sBufferSize,pDataFile);
lBytesLeftToRead -= lBytesRead;
if (lBytesLeftToRead < sBufferSize)
sBufferSize = (short)lBytesLeftToRead; /* just a little bit more
to read next time */
/* get out if (ArrayOverrun in CRC Calculation) or (we are done with
the data stream) */
bDone = (((lBytesRead > 0) &&
((dwCrc = CalculateCrc(pBuffer,lBytesRead,dwCrc,sError)) ==
0) &&
(*sError==FC_ERR_PROOF_OF_ARRAY_OVERRUN_IN_CRC_CODE)) ||
((feof(pDataFile)!=0) || (lBytesLeftToRead <= 0)));
}
if (*sError == FC_ERR_NOERROR)
*retval=dwCrc; /* if there was no error, return the CRC otherwise
make it zero */
}
free(pBuffer); /* clean up */
fclose(pDataFile);
}
return(1);
}
I converted the CalculateCrc function to pure Python and it gives me the
proper results (on small files), but is so slow on large files (we are
processing 650+Mb files) as to be unuseable.
import time
import struct
import os
import binascii
#
# Table for lookups
#
ccitt_32=[
0x00000000L, 0x77073096L, 0xee0e612cL, 0x990951baL, 0x076dc419L,
0x706af48fL, 0xe963a535L, 0x9e6495a3L, 0x0edb8832L, 0x79dcb8a4L,
0xe0d5e91eL, 0x97d2d988L, 0x09b64c2bL, 0x7eb17cbdL, 0xe7b82d07L,
0x90bf1d91L, 0x1db71064L, 0x6ab020f2L, 0xf3b97148L, 0x84be41deL,
0x1adad47dL, 0x6ddde4ebL, 0xf4d4b551L, 0x83d385c7L, 0x136c9856L,
0x646ba8c0L, 0xfd62f97aL, 0x8a65c9ecL, 0x14015c4fL, 0x63066cd9L,
0xfa0f3d63L, 0x8d080df5L, 0x3b6e20c8L, 0x4c69105eL, 0xd56041e4L,
0xa2677172L, 0x3c03e4d1L, 0x4b04d447L, 0xd20d85fdL, 0xa50ab56bL,
0x35b5a8faL, 0x42b2986cL, 0xdbbbc9d6L, 0xacbcf940L, 0x32d86ce3L,
0x45df5c75L, 0xdcd60dcfL, 0xabd13d59L, 0x26d930acL, 0x51de003aL,
0xc8d75180L, 0xbfd06116L, 0x21b4f4b5L, 0x56b3c423L, 0xcfba9599L,
0xb8bda50fL, 0x2802b89eL, 0x5f058808L, 0xc60cd9b2L, 0xb10be924L,
0x2f6f7c87L, 0x58684c11L, 0xc1611dabL, 0xb6662d3dL, 0x76dc4190L,
0x01db7106L, 0x98d220bcL, 0xefd5102aL, 0x71b18589L, 0x06b6b51fL,
0x9fbfe4a5L, 0xe8b8d433L, 0x7807c9a2L, 0x0f00f934L, 0x9609a88eL,
0xe10e9818L, 0x7f6a0dbbL, 0x086d3d2dL, 0x91646c97L, 0xe6635c01L,
0x6b6b51f4L, 0x1c6c6162L, 0x856530d8L, 0xf262004eL, 0x6c0695edL,
0x1b01a57bL, 0x8208f4c1L, 0xf50fc457L, 0x65b0d9c6L, 0x12b7e950L,
0x8bbeb8eaL, 0xfcb9887cL, 0x62dd1ddfL, 0x15da2d49L, 0x8cd37cf3L,
0xfbd44c65L, 0x4db26158L, 0x3ab551ceL, 0xa3bc0074L, 0xd4bb30e2L,
0x4adfa541L, 0x3dd895d7L, 0xa4d1c46dL, 0xd3d6f4fbL, 0x4369e96aL,
0x346ed9fcL, 0xad678846L, 0xda60b8d0L, 0x44042d73L, 0x33031de5L,
0xaa0a4c5fL, 0xdd0d7cc9L, 0x5005713cL, 0x270241aaL, 0xbe0b1010L,
0xc90c2086L, 0x5768b525L, 0x206f85b3L, 0xb966d409L, 0xce61e49fL,
0x5edef90eL, 0x29d9c998L, 0xb0d09822L, 0xc7d7a8b4L, 0x59b33d17L,
0x2eb40d81L, 0xb7bd5c3bL, 0xc0ba6cadL, 0xedb88320L, 0x9abfb3b6L,
0x03b6e20cL, 0x74b1d29aL, 0xead54739L, 0x9dd277afL, 0x04db2615L,
0x73dc1683L, 0xe3630b12L, 0x94643b84L, 0x0d6d6a3eL, 0x7a6a5aa8L,
0xe40ecf0bL, 0x9309ff9dL, 0x0a00ae27L, 0x7d079eb1L, 0xf00f9344L,
0x8708a3d2L, 0x1e01f268L, 0x6906c2feL, 0xf762575dL, 0x806567cbL,
0x196c3671L, 0x6e6b06e7L, 0xfed41b76L, 0x89d32be0L, 0x10da7a5aL,
0x67dd4accL, 0xf9b9df6fL, 0x8ebeeff9L, 0x17b7be43L, 0x60b08ed5L,
0xd6d6a3e8L, 0xa1d1937eL, 0x38d8c2c4L, 0x4fdff252L, 0xd1bb67f1L,
0xa6bc5767L, 0x3fb506ddL, 0x48b2364bL, 0xd80d2bdaL, 0xaf0a1b4cL,
0x36034af6L, 0x41047a60L, 0xdf60efc3L, 0xa867df55L, 0x316e8eefL,
0x4669be79L, 0xcb61b38cL, 0xbc66831aL, 0x256fd2a0L, 0x5268e236L,
0xcc0c7795L, 0xbb0b4703L, 0x220216b9L, 0x5505262fL, 0xc5ba3bbeL,
0xb2bd0b28L, 0x2bb45a92L, 0x5cb36a04L, 0xc2d7ffa7L, 0xb5d0cf31L,
0x2cd99e8bL, 0x5bdeae1dL, 0x9b64c2b0L, 0xec63f226L, 0x756aa39cL,
0x026d930aL, 0x9c0906a9L, 0xeb0e363fL, 0x72076785L, 0x05005713L,
0x95bf4a82L, 0xe2b87a14L, 0x7bb12baeL, 0x0cb61b38L, 0x92d28e9bL,
0xe5d5be0dL, 0x7cdcefb7L, 0x0bdbdf21L, 0x86d3d2d4L, 0xf1d4e242L,
0x68ddb3f8L, 0x1fda836eL, 0x81be16cdL, 0xf6b9265bL, 0x6fb077e1L,
0x18b74777L, 0x88085ae6L, 0xff0f6a70L, 0x66063bcaL, 0x11010b5cL,
0x8f659effL, 0xf862ae69L, 0x616bffd3L, 0x166ccf45L, 0xa00ae278L,
0xd70dd2eeL, 0x4e048354L, 0x3903b3c2L, 0xa7672661L, 0xd06016f7L,
0x4969474dL, 0x3e6e77dbL, 0xaed16a4aL, 0xd9d65adcL, 0x40df0b66L,
0x37d83bf0L, 0xa9bcae53L, 0xdebb9ec5L, 0x47b2cf7fL, 0x30b5ffe9L,
0xbdbdf21cL, 0xcabac28aL, 0x53b39330L, 0x24b4a3a6L, 0xbad03605L,
0xcdd70693L, 0x54de5729L, 0x23d967bfL, 0xb3667a2eL, 0xc4614ab8L,
0x5d681b02L, 0x2a6f2b94L, 0xb40bbe37L, 0xc30c8ea1L, 0x5a05df1bL,
0x2d02ef8dL
]
def CalculateCrc(buffer, crc):
global ccitt_32
#
# Calculate the 32 bit CRC using the table lookup method.
#
unpack=struct.unpack
for c in buffer:
#print 'crc=', crc
#print 'crc >> 8=', (crc >> 8)
temp1=crc >> 8
#print 'temp1=', temp1
Index = (crc ^ unpack('B',c)[0]) & 0xff
#print "Index=", Index
if Index < 0 or Index > 255:
sError = 'FC_ERR_PROOF_OF_ARRAY_OVERRUN_IN_CRC_CODE'
return 0, sError; # Time to Panic: array overrun time!
temp2 = ccitt_32[Index]
crc = temp1 ^ temp2
#print 'crc=', crc
return crc, ''
The algorithm looks very much like the source code for
binascii.crc32 (but I'm not a C programmer).
Thanks for everyone's help.
Regards,
Larry Bates
More information about the Python-list
mailing list