src/lib/tokenizer.c 1.4

Index: src/lib/tokenizer.c
===================================================================
RCS file: /projects/compbio/cvsroot/kent/src/lib/tokenizer.c,v
retrieving revision 1.3
retrieving revision 1.4
diff -b -B -U 4 -r1.3 -r1.4
--- src/lib/tokenizer.c	14 Jul 2004 05:47:14 -0000	1.3
+++ src/lib/tokenizer.c	20 Nov 2009 08:08:53 -0000	1.4
@@ -42,9 +42,10 @@
 
 void tokenizerReuse(struct tokenizer *tkz)
 /* Reuse token. */
 {
-tkz->reuse = TRUE;
+if (!tkz->eof)
+    tkz->reuse = TRUE;
 }
 
 int tokenizerLineCount(struct tokenizer *tkz)
 /* Return line of current token. */
@@ -69,12 +70,14 @@
     {
     tkz->reuse = FALSE;
     return tkz->string;
     }
+tkz->leadingSpaces = 0;
 for (;;)	/* Skip over white space and comments. */
     {
     int lineSize;
     s = start = skipLeadingSpaces(tkz->linePt);
+    tkz->leadingSpaces += s - tkz->linePt;
     if ((c = start[0]) != 0)
 	{
 	if (tkz->uncommentC && c == '/')
 	     {
@@ -109,8 +112,9 @@
 	{
 	tkz->eof = TRUE;
 	return NULL;
 	}
+    tkz->leadingSpaces += 1;
     tkz->linePt = tkz->curLine;
     }
 if (isalnum(c) || (c == '_'))
     {
@@ -188,13 +192,15 @@
 if (tkz->eof)
     errAbort("Unexpected end of file");
 }
 
-void tokenizerMustHaveNext(struct tokenizer *tkz)
+char *tokenizerMustHaveNext(struct tokenizer *tkz)
 /* Get next token, which must be there. */
 {
-if (tokenizerNext(tkz) == NULL)
+char *s = tokenizerNext(tkz);
+if (s == NULL)
     errAbort("Unexpected end of file");
+return s;
 }
 
 void tokenizerMustMatch(struct tokenizer *tkz, char *string)
 /* Require next token to match string.  Return next token