[Python-checkins] python/dist/src/Parser pgenmain.c,2.24,2.25 tokenizer.c,2.53,2.54

[jackjansen@sourceforge.net]

Update of /cvsroot/python/python/dist/src/Parser
In directory usw-pr-cvs1:/tmp/cvs-serv1748/Parser

Modified Files:
	pgenmain.c tokenizer.c 
Log Message:
Mass checkin of universal newline support.
Highlights: import and friends will understand any of \r, \n and \r\n
as end of line. Python file input will do the same if you use mode 'U'.
Everything can be disabled by configuring with --without-universal-newlines.

See PEP278 for details. 


Index: pgenmain.c
===================================================================
RCS file: /cvsroot/python/python/dist/src/Parser/pgenmain.c,v
retrieving revision 2.24
retrieving revision 2.25
diff -C2 -d -r2.24 -r2.25
*** pgenmain.c	11 Sep 2001 16:43:16 -0000	2.24
--- pgenmain.c	14 Apr 2002 20:12:41 -0000	2.25
***************
*** 14,17 ****
--- 14,18 ----
  */
  
+ #include "Python.h"
  #include "pgenheaders.h"
  #include "grammar.h"
***************
*** 182,185 ****
--- 183,196 ----
  	return PyMem_REALLOC(p, n+1);
  }
+ 
+ #ifdef WITH_UNIVERSAL_NEWLINES
+ /* No-nonsense fgets */
+ char *
+ Py_UniversalNewlineFgets(char *buf, int n, FILE *stream, PyObject *fobj)
+ {
+ 	return fgets(buf, n, stream);
+ }
+ #endif
+ 
  
  #include <stdarg.h>

Index: tokenizer.c
===================================================================
RCS file: /cvsroot/python/python/dist/src/Parser/tokenizer.c,v
retrieving revision 2.53
retrieving revision 2.54
diff -C2 -d -r2.53 -r2.54
*** tokenizer.c	30 Aug 2001 20:51:59 -0000	2.53
--- tokenizer.c	14 Apr 2002 20:12:41 -0000	2.54
***************
*** 2,5 ****
--- 2,6 ----
  /* Tokenizer implementation */
  
+ #include "Python.h"
  #include "pgenheaders.h"
  
***************
*** 246,251 ****
  					tok->end = tok->buf + BUFSIZ;
  				}
! 				if (fgets(tok->buf, (int)(tok->end - tok->buf),
! 					  tok->fp) == NULL) {
  					tok->done = E_EOF;
  					done = 1;
--- 247,252 ----
  					tok->end = tok->buf + BUFSIZ;
  				}
! 				if (Py_UniversalNewlineFgets(tok->buf, (int)(tok->end - tok->buf),
! 					  tok->fp, NULL) == NULL) {
  					tok->done = E_EOF;
  					done = 1;
***************
*** 285,291 ****
  				tok->start = curstart < 0 ? NULL :
  					     tok->buf + curstart;
! 				if (fgets(tok->inp,
  					       (int)(tok->end - tok->inp),
! 					       tok->fp) == NULL) {
  					/* Last line does not end in \n,
  					   fake one */
--- 286,292 ----
  				tok->start = curstart < 0 ? NULL :
  					     tok->buf + curstart;
! 				if (Py_UniversalNewlineFgets(tok->inp,
  					       (int)(tok->end - tok->inp),
! 					       tok->fp, NULL) == NULL) {
  					/* Last line does not end in \n,
  					   fake one */