[Python-checkins] python/dist/src/Parser tokenizer.c,2.71,2.72

[jvr@users.sourceforge.net]

Update of /cvsroot/python/python/dist/src/Parser
In directory sc8-pr-cvs1:/tmp/cvs-serv21078/Parser

Modified Files:
	tokenizer.c 
Log Message:
patch 680474 that fixes bug 679880: compile/eval/exec refused utf-8 bom
mark. Added unit test.


Index: tokenizer.c
===================================================================
RCS file: /cvsroot/python/python/dist/src/Parser/tokenizer.c,v
retrieving revision 2.71
retrieving revision 2.72
diff -C2 -d -r2.71 -r2.72
*** tokenizer.c	14 Jan 2003 23:15:22 -0000	2.71
--- tokenizer.c	9 Feb 2003 20:38:48 -0000	2.72
***************
*** 507,511 ****
  
  static int buf_getc(struct tok_state *tok) {
! 	return *tok->str++;
  }
  
--- 507,511 ----
  
  static int buf_getc(struct tok_state *tok) {
! 	return Py_CHARMASK(*tok->str++);
  }
  
***************
*** 514,518 ****
  static void buf_ungetc(int c, struct tok_state *tok) {
  	tok->str--;
! 	assert(*tok->str == c);	/* tok->cur may point to read-only segment */
  }
  
--- 514,518 ----
  static void buf_ungetc(int c, struct tok_state *tok) {
  	tok->str--;
! 	assert(Py_CHARMASK(*tok->str) == c);	/* tok->cur may point to read-only segment */
  }