[Python-checkins] CVS: python/dist/src/Objects unicodeobject.c,2.124,2.125

[M.-A. Lemburg]

Update of /cvsroot/python/python/dist/src/Objects
In directory usw-pr-cvs1:/tmp/cvs-serv6841/Objects

Modified Files:
	unicodeobject.c 
Log Message:
Fix for the UTF-8 memory allocation bug and the UTF-8 encoding
bug related to lone high surrogates.



Index: unicodeobject.c
===================================================================
RCS file: /cvsroot/python/python/dist/src/Objects/unicodeobject.c,v
retrieving revision 2.124
retrieving revision 2.125
diff -C2 -d -r2.124 -r2.125
*** unicodeobject.c	2001/12/06 20:03:56	2.124
--- unicodeobject.c	2002/02/06 18:09:02	2.125
***************
*** 1172,1182 ****
      PyObject *v;
      char *p;
!     char *q;
!     Py_UCS4 ch2;
!     unsigned int cbAllocated = 3 * size;
      unsigned int cbWritten = 0;
      int i = 0;
  
!     v = PyString_FromStringAndSize(NULL, cbAllocated);
      if (v == NULL)
          return NULL;
--- 1172,1180 ----
      PyObject *v;
      char *p;
!     unsigned int cbAllocated = 2 * size;
      unsigned int cbWritten = 0;
      int i = 0;
  
!     v = PyString_FromStringAndSize(NULL, cbAllocated + 4);
      if (v == NULL)
          return NULL;
***************
*** 1184,1194 ****
          return v;
  
!     p = q = PyString_AS_STRING(v);
      while (i < size) {
          Py_UCS4 ch = s[i++];
          if (ch < 0x80) {
              *p++ = (char) ch;
              cbWritten++;
          }
          else if (ch < 0x0800) {
              *p++ = 0xc0 | (ch >> 6);
--- 1182,1194 ----
          return v;
  
!     p = PyString_AS_STRING(v);
      while (i < size) {
          Py_UCS4 ch = s[i++];
+ 
          if (ch < 0x80) {
              *p++ = (char) ch;
              cbWritten++;
          }
+ 
          else if (ch < 0x0800) {
              *p++ = 0xc0 | (ch >> 6);
***************
*** 1196,1230 ****
              cbWritten += 2;
          }
-         else if (ch < 0x10000) {
-             /* Check for high surrogate */
-             if (0xD800 <= ch && ch <= 0xDBFF) {
-                 if (i != size) {
-                     ch2 = s[i];
-                     if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {
                          
!                         if (cbWritten >= (cbAllocated - 4)) {
! 			    /* Provide enough room for some more
! 			       surrogates */
! 			    cbAllocated += 4*10;
!                             if (_PyString_Resize(&v, cbAllocated))
  				goto onError;
                          }
  
!                         /* combine the two values */
                          ch = ((ch - 0xD800)<<10 | (ch2-0xDC00))+0x10000;
-                     
                          *p++ = (char)((ch >> 18) | 0xf0);
                          *p++ = (char)(0x80 | ((ch >> 12) & 0x3f));
                          i++;
                          cbWritten += 4;
                      }
                  }
-             }
-             else {
                  *p++ = (char)(0xe0 | (ch >> 12));
-                 cbWritten += 3;
-             }
              *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
              *p++ = (char)(0x80 | (ch & 0x3f));
          } else {
              *p++ = 0xf0 | (ch>>18);
--- 1196,1233 ----
              cbWritten += 2;
          }
                          
!         else {
! 	    
! 	    /* Assure that we have enough room for high order Unicode
! 	       ordinals */
! 	    if (cbWritten >= cbAllocated) {
! 		cbAllocated += 4 * 10;
! 		if (_PyString_Resize(&v, cbAllocated + 4))
  				goto onError;
+ 		p = PyString_AS_STRING(v) + cbWritten;
                          }
  
! 	    if (ch < 0x10000) {
! 		/* Check for high surrogate */
! 		if (0xD800 <= ch && ch <= 0xDBFF && i != size) {
! 		    Py_UCS4 ch2 = s[i];
! 		    /* Check for low surrogate */
! 		    if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {
                          ch = ((ch - 0xD800)<<10 | (ch2-0xDC00))+0x10000;
                          *p++ = (char)((ch >> 18) | 0xf0);
                          *p++ = (char)(0x80 | ((ch >> 12) & 0x3f));
+ 			*p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
+ 			*p++ = (char)(0x80 | (ch & 0x3f));
                          i++;
                          cbWritten += 4;
+ 			continue;
                      }
+ 		    /* Fall through: handles isolated high surrogates */
                  }
                  *p++ = (char)(0xe0 | (ch >> 12));
              *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
              *p++ = (char)(0x80 | (ch & 0x3f));
+ 		cbWritten += 3;
+     
          } else {
              *p++ = 0xf0 | (ch>>18);
***************
*** 1235,1240 ****
  	}
      }
      *p = '\0';
!     if (_PyString_Resize(&v, p - q))
  	goto onError;
      return v;
--- 1238,1244 ----
  	}
      }
+     }
      *p = '\0';
!     if (_PyString_Resize(&v, cbWritten))
  	goto onError;
      return v;