- /* discard leading & trailing whitespace, but keep inter-word delimeters */
-^[ \t]+ ;
-[ \t]+$ ;
-[ \t]+ return TOKEN_DELIM;
+ /* discard whitespace, unless we're looking for inter-word delimiters */
+{DELIM} { if (yyget_extra(yyscanner)->inter_word) return TOKEN_DELIM; }