/* $Id: lex.c,v 1.13 2009/02/18 13:41:01 kiesling Exp $ */

/*
  This file is part of Ctalk.
  Copyright © 2005-2009  Robert Kiesling, ctalk@ctalklang.org.
  Permission is granted to copy this software provided that this copyright
  notice is included in all source code modules.

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with this program; if not, write to the Free Software Foundation, 
  Inc., 51 Franklin St., Fifth Floor, Boston, MA 02110-1301 USA.
*/

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <errno.h>
#include "lex.h"
#include "lextab.h"
#include "object.h"
#include "message.h"
#include "ctalk.h"
#include "typeof.h"
#include "rtinfo.h"

/* TO DO -
   Use the international abbreviations in iso646.h.
*/


extern int error_line;      /* Declared in errorloc.c. */
extern int error_column;
extern RT_INFO rtinfo;      /* Declared in rtinfo.c.   */

extern int line_info_line;  /* Declared in lineinfo.c. */

extern I_PASS interpreter_pass;  /* Declared in main.c. */

/*
 *  Used for parsing literals.
 */
#define DEFL_ESC_IDX -2
#define ADD_CHR_RESIZE if (end_str == -1) { m -> name[k++] = buf[j]; } else { continue ; } \
                       if (k >= resize_x) {resize_x *= 2; resize_message (m, resize_x); }
/*
 *    Performs lexical scanning.  Returns the lvalue in 
 *    m -> name.  The token types are defined in lex.h.
 */

#define IS_SIGN(c,d) ((c == '-' || c == '+') && \
		      (ASCII_DIGIT (d) || d == '.'))

#define _MPUTCHAR(__m,__c) (__m)->name[0] = (__c); (__m)->name[1] = '\0';

static int gnuc_attribute;  /* Set by __attribute__ label, cleared after )). */

static int preprocess_line = FALSE;  /* Used to determine statement endings -
                                        Set when we encounter a PREPROCESS
					token and cleared at the NEWLINE 
                                        token.  If set, statements end
                                        at the NEWLINE.  Otherwise, statements
                                        end at the SEMICOLON.
				     */
static int comment;                  /* True if within a comment.*/ 

int esc_chr;


int lexical (const char *buf, long long *idx, MESSAGE *m) {

  int c, i, j;
  int numeric_type;
  char tmpbuf[MAXMSG];
  char *q;

  c = buf[(*idx)++];

  /* WHITESPACE */
  /* Newline characters have their own token type, below. */
  if (ascii_chr_tok[c] == WHITESPACE) {
    m -> name[0] = c;
    i = 1;
    while (ascii_chr_tok[(int)buf[*idx]] == WHITESPACE) 
      m -> name[i++] = buf[(*idx)++];
    m -> name[i] = 0;
    m -> tokentype = WHITESPACE;
    return WHITESPACE;
   }

  /* FLOAT, INTEGER, LONG, and LONGLONG                                  */
  /* TO DO - 
     Bring into conformance with inttypes.h, limits.h, and float.h. */
  /* The sign and suffixes, if any, are included in the token.           */
  /* An expression w/o whitespace, eg 3-3, evaluates as a unary minus.   */
  if (ASCII_DIGIT(c) || 
      (c == '.' && ASCII_DIGIT(c)) ||
      (c == '-' && ASCII_DIGIT(c)) ||
#if defined(__GNUC__) && defined(__sparc__) && defined(__svr4__)
      IS_SIGN (c, (int)buf[*idx])) {
#else
      IS_SIGN (c, buf[*idx])) {
#endif
    RADIX radix = decimal;

    (*idx)--;
    j = *idx;
    numeric_type = INTEGER_T;

    /* Account for a signed hexadecimal or octal constant. */
#if defined(__GNUC__) && defined(__sparc__) && defined(__svr4__)
    if (!IS_SIGN (buf[j],(int)buf[j+1])) {
#else
    if (!IS_SIGN (buf[j],buf[j+1])) {
#endif
      if ((buf[j] == (char)'0') && (buf[j + 1] != (char)'.')) {
	if (buf[j+1] == (char) 'x' || buf[j+1] == 'X') {
	  radix = hexadecimal;
	  j += 2;
	} else {
	  if (buf[j+1] && (buf[j+1] >= '0' && buf[j+1] <= '7')) {
	    radix = octal;
	    j += 1;
	  }
	}
      }
    } else {
      if ((buf[j+1] == (char)'0') && (buf[j + 2] != (char)'.')) {
	if (buf[j+2] == (char) 'x' || buf[j+2] == 'X') {
	  radix = hexadecimal;
	  j += 3;
	  _warning ("Signed hexadecimal constant.");
	} else {
	  if (buf[j+2] && (buf[j+2] >= '0' && buf[j+2] <= '7')) {
	    radix = octal;
	    j += 2;
	    _warning ("Signed octal constant.");
	  }
	}
      }
    }

    while (buf[j]) {

#if defined(__GNUC__) && defined(__sparc__) && defined(__svr4__)
      if (buf[j] == (char)'.' && ASCII_DIGIT ((int)buf[j+1]))
#else
      if (buf[j] == (char)'.' && ASCII_DIGIT (buf[j+1]))
#endif
 	numeric_type = DOUBLE_T;

      if ((numeric_type == INTEGER_T) && 
	  ((buf[j] == (char)'l' || buf[j] == (char)'L') &&
	   (buf[j+1] != (char)'l' || buf[j+1] != (char)'L')))
 	numeric_type = LONG_T;

      if (((numeric_type == INTEGER_T) || (numeric_type == LONG_T)) && 
	  ((buf[j] == (char)'l' || buf[j] == (char)'L') && 
	   (buf[j+1] == (char)'l' || buf[j+1] == (char)'L')))
 	numeric_type = LONGLONG_T;

      if ((numeric_type == DOUBLE_T) && 
	  ((buf[j] == (char)'f') || (buf[j] == (char)'F')))
	numeric_type = FLOAT_T;

      if ((numeric_type == DOUBLE_T) && 
	  ((buf[j] == (char)'l') || (buf[j] == (char)'L')))
	numeric_type = LONGDOUBLE_T;

      /* Decimal integers can also have 'u', and 'U' suffixes. */
      /* Account for binary constants here also. */
      if ((radix == decimal) && 
	  (numeric_type == INTEGER_T) && 
#if defined(__GNUC__) && defined(__sparc__) && defined(__svr4__)
	  ! (ASCII_DIGIT ((int)buf[j]) || IS_SIGN(buf[j],(int)buf[j+1]) ||
#else
	  ! (ASCII_DIGIT ((int)buf[j]) || IS_SIGN(buf[j],buf[j+1]) ||
#endif
	     buf[j] == (char)'u' || buf[j] == (char)'U' ||
	     buf[j] == (char)'b' || buf[j] == (char)'B'))
	break;

       if ((radix == decimal) &&
 	  (numeric_type == LONG_T || numeric_type == LONGLONG_T) &&
#if defined(__GNUC__) && defined(__sparc__) && defined(__svr4__)
	   ! (ASCII_DIGIT ((int)buf[j]) || IS_SIGN(buf[j],(int)buf[j+1]) ||
#else
	   ! (ASCII_DIGIT ((int)buf[j]) || IS_SIGN(buf[j],buf[j+1]) ||
#endif
	      buf[j] == (char)'l' || buf[j] == (char)'L' ||
	      buf[j] == (char)'u' || buf[j] == (char)'U' ||
	      buf[j] == (char)'b' || buf[j] == (char)'B'))
	 break;

      /* Octals need range checking. */
#if defined(__GNUC__) && defined(__sparc__) && defined(__svr4__)
	   if ((radix == octal) && ASCII_DIGIT((int)buf[j]) && 
#else
       if ((radix == octal) && ASCII_DIGIT(buf[j]) && 
#endif
	  ((buf[j] < (char)'0') || (buf[j] > (char)'7')))
	_error ("Bad octal constant.");
      if ((radix == octal) && ((buf[j] < (char)'0') || (buf[j] > (char)'7')))
	break;

      if ((radix == hexadecimal) && 
	  (!isxdigit ((int) buf[j]) && 
	   (buf[j] != (char)'l') && (buf[j] != (char)'L')))
	break;

      if (((numeric_type == DOUBLE_T) ||
	   (numeric_type == FLOAT_T) ||
	   (numeric_type == LONGDOUBLE_T))
	  && (!ASCII_DIGIT ((int)buf[j]) &&
	   (buf[j] != '.') &&
	   (buf[j] != 'e') &&
	   (buf[j] != 'E') &&
#if defined(__GNUC__) && defined(__sparc__) && defined(__svr4__)
	   !IS_SIGN (buf[j],(int)buf[j+1])))
#else
	   !IS_SIGN (buf[j],buf[j+1])))
#endif
	break;
	   
      j++;
    }
    substrcpy (m -> name, (char *)buf, *idx, j - *idx);
    *idx = j;
    switch (numeric_type)
      {
      case INTEGER_T:
	m -> tokentype = INTEGER;
	return INTEGER;
	break;
      case DOUBLE_T:
      case FLOAT_T:
      case LONGDOUBLE_T:
 	m -> tokentype = DOUBLE;
 	return DOUBLE;
	break;
      case LONG_T:
	m -> tokentype = LONG;
	return LONG;
	break;
      case LONGLONG_T:
	m -> tokentype = LONGLONG;
	return LONGLONG;
	break;
      }
  }

  /* LABEL */
  if (isalnum ((int)c) || c == '_' || c == '$') {
    (*idx)--;

    q = tmpbuf;
    while (isalnum ((int)buf[*idx]) || (buf[*idx] == '_') 
	   || (buf[*idx] == '$'))
      *q++ = buf[(*idx)++];
    *q = 0;

    strcpy (m -> name, tmpbuf);
    m -> tokentype = LABEL;
    return LABEL;
  }
    
  if (ascii_chr_tok[c]) {
    _MPUTCHAR(m,c)
      m -> tokentype = ascii_chr_tok[c];
    return m -> tokentype;
  }

  /* OPENCCOMENT */
  if (c == '/' && buf[*idx] == '*') {
    strcpy (m -> name, "/*");
    (*idx)++;
     m -> tokentype = OPENCCOMMENT;
     return OPENCCOMMENT;
  }

  /* CLOSECCOMENT */
  if (c == '*' && buf[*idx] == '/') {
    strcpy (m -> name, "*/");
    (*idx)++;
    m -> tokentype = CLOSECCOMMENT;
    return CLOSECCOMMENT;
  }

  /* CPPCOMMENT */
  if (c == '/' && buf[*idx] == '/') {
    q = index (&buf[*idx], '\n');
    strncpy (tmpbuf, &buf[(*idx) - 1], q - &buf[(*idx) - 1]);
    tmpbuf[q - &buf[(*idx) - 1]] = 0;
    strcpy (m -> name, tmpbuf);
    *idx += (q - &buf[*idx]);
    m -> tokentype = CPPCOMMENT;
    return CPPCOMMENT;
  }

  /* ASTERISK, MULT_ASSIGN */
  if (c == '*') {
    switch (buf[*idx]) 
      {
      case '=':
	(*idx)++;
	strcpy (m -> name, "*=");
	m -> tokentype = MULT_ASSIGN;
	return MULT_ASSIGN;
	break;
      default:
	_MPUTCHAR(m,c)
	m -> tokentype = ASTERISK;
	return ASTERISK;
	break;
      }
  }

  /* DIVIDE, DIV_ASSIGN */
  if (c == '/') {
    switch (buf[*idx]) 
      {
      case '=':
	strcpy (m -> name, "/=");
	++(*idx);
	m -> tokentype = DIV_ASSIGN;
	return DIV_ASSIGN;
	break;
      default:
	_MPUTCHAR(m,c)
	m -> tokentype = DIVIDE;
	return DIVIDE;
	break;
      }
  }

  /* LT, ASL, ASL_ASSIGN */
  if (c == '<') {
    switch (buf[*idx])
      {
      case '<':
	(*idx)++;
	switch (buf[*idx])
	  {
	  case '=':
	    (*idx)++;
	    strcpy (m -> name, "<<=");
	    m -> tokentype = ASL_ASSIGN;
	    return ASL_ASSIGN;
	    break;
	  default:
	    strcpy (m -> name, "<<");
	    m -> tokentype = ASL;
	    return ASL;
	    break;
	  }
	break;
      case '=':
	(*idx)++;
	strcpy (m -> name, "<=");
	m -> tokentype = LE;
	return LE;
	break;
      default:
	_MPUTCHAR(m,c)
	m -> tokentype = LT;
	return LT;
	break;
      }
  }

  /* GT, ASR, ASR_ASSIGN, GE */
  if (c == '>') {
    switch (buf[*idx])
      {
      case '>':
	(*idx)++;
	switch (buf[*idx])
	  {
	  case '=':
	    (*idx)++;
	    strcpy (m -> name, ">>=");
	    m -> tokentype = ASR_ASSIGN;
	    return ASR_ASSIGN;
	    break;
	  default:
	    strcpy (m -> name, ">>");
	    m -> tokentype = ASR;
	    return ASR;
	    break;
	  }
	break;
      case '=':
	(*idx)++;
	strcpy (m -> name, ">=");
	m -> tokentype = GE;
	return GE;
	break;
      default:
	_MPUTCHAR(m,c)
	m -> tokentype = GT;
	return GT;
	break;
      }
  }

  /* BIT_XOR, BIT_XOR_ASSIGN */
  if (c == '^') {
    switch (buf[*idx])
      {
      case '=':
	(*idx)++;
	strcpy (m -> name, "^=");
	m -> tokentype = BIT_XOR_ASSIGN;
	return BIT_XOR_ASSIGN;
	break;
      default:
	_MPUTCHAR(m,c)
	m -> tokentype = BIT_XOR;
	return BIT_XOR;
	break;
      }
  }

  /* PERIOD, ELLIPSIS */
  if (c == '.') {
    if (buf[*idx] == '.' && buf[*idx+1] == '.') {
      (*idx) += 2;
      strcpy (m -> name, "...");
      m -> tokentype = ELLIPSIS;
      return ELLIPSIS;
    } else {
      _MPUTCHAR(m,c)
      m -> tokentype = PERIOD;
      return PERIOD;
    }
  }

  /* BIT_OR, BOOLEAN_OR, BIT_OR_ASSIGN */
  if (c == '|') {
    switch (buf[*idx])
      {
      case '|':
	(*idx)++;
	strcpy (m -> name, "||");
	m -> tokentype = BOOLEAN_OR;
	return BOOLEAN_OR;
	break;
      case '=':
	(*idx)++;
	strcpy (m -> name, "|=");
	m -> tokentype = BIT_OR_ASSIGN;
	return BIT_OR_ASSIGN;
	break;
      default:
	_MPUTCHAR(m,c)
	m -> tokentype = BAR;
	return BAR;
	break;
      }
  }

  /* AMPERSAND, BIT_AND_ASSIGN, BOOLEAN_AND */
  if (c == '&') {
    int c_idx;
    switch (buf[*idx])
      {
      case '&':
	(*idx)++;
	strcpy (m -> name, "&&");
	m -> tokentype = BOOLEAN_AND;
	return BOOLEAN_AND;
	break;
      case '=':
	(*idx)++;
	strcpy (m -> name, "&=");
	m -> tokentype = BIT_AND_ASSIGN;
	return BIT_AND_ASSIGN;
	break;
      default:
	_MPUTCHAR(m,c)
	/*
	 *  If the preceding non-whitespace character is an operator,
	 *  then the ampersand is an address-of operator, otherwise
	 *  it is a bitwise and operator.
	 * 
	 *  If the ampersand is the first character of the buffer,
	 *  then it is an address-of operator.
	 */
	if (((*idx) - 1) == 0) {
	  m -> tokentype = AMPERSAND;
	  return AMPERSAND;
	} else {
	  for (c_idx = (*idx) - 2; c_idx >= 0; c_idx--) {
	    if (isspace ((int)buf[c_idx])) 
	      continue;
	    if (IS_C_OP_CHAR (buf[c_idx])) {
	      if ((buf[c_idx] == ')') ||
		  (buf[c_idx] == ']')) {
		m -> tokentype = BIT_AND;
		return BIT_AND;
	      } else {
		m -> tokentype = AMPERSAND;
		return AMPERSAND;
	      }
	    } else {
	      m -> tokentype = BIT_AND;
	      return BIT_AND;
	    }
	  }
	}
	break;
      }
  }

  /* EQ, BOOLEAN_EQ */
  if (c == '=') {
    switch (buf[*idx])
      {
      case '=':
	(*idx)++;
	strcpy (m -> name, "==");
	m -> tokentype = BOOLEAN_EQ;
	return BOOLEAN_EQ;
	break;
      default:
	_MPUTCHAR(m,c)
	m -> tokentype = EQ;
	return EQ;
	break;
      }
  }

  /* EXCLAM, INEQUALITY */
  if (c == '!') {
    switch (buf[*idx]) 
      {
      case '=':
	(*idx)++;
	strcpy (m -> name, "!=");
	m -> tokentype = INEQUALITY;
	return INEQUALITY;
	break;
      default:
	_MPUTCHAR(m,c)
	m -> tokentype = EXCLAM;
	return EXCLAM;
	break;
      }
  }

  /* PLUS, PLUS_ASSIGN, INCREMENT */
  /* Note that this occurs after the numeric constant tokens. */
  if (c == '+') {
    switch (buf[*idx])
      {
      case '=':
	(*idx)++;
	strcpy (m -> name, "+=");
	m -> tokentype = PLUS_ASSIGN;
	return PLUS_ASSIGN;
	break;
      case '+':
	(*idx)++;
	strcpy (m -> name, "++");
	m -> tokentype = INCREMENT;
	return INCREMENT;
	break;
      default:
	_MPUTCHAR(m,c)
	m -> tokentype = PLUS;
	return PLUS;
	break;
      }
  }

  /* MINUS, MINUS_ASSIGN, DECREMENT, DEREF */
  /* Note that this occurs after the checking for numeric constants. */
  if (c == '-') {
    switch (buf[*idx])
      {
      case '=':
	(*idx)++;
	strcpy (m -> name, "-=");
	m -> tokentype = MINUS_ASSIGN;
	return MINUS_ASSIGN;
	break;
      case '-':
	(*idx)++;
	strcpy (m -> name, "--");
	m -> tokentype = DECREMENT;
	return DECREMENT;
	break;
      case '>':
	(*idx)++;
	strcpy (m -> name, "->");
	m -> tokentype = DEREF;
	return DEREF;
	break;
      default:
	_MPUTCHAR(m,c)
	m -> tokentype = MINUS;
	return MINUS;
	break;
      }
  }

  /* PREPROCESS, LITERALIZE, MACRO_CONCAT */
  if (c == '#') {
    /*
     *  We need to ensure that the token is the first *non-whitespace*
     *  token after a newline, or is the first non-whitespace 
     *  token in the text.
     *
     */
    int lookback = ((*idx - 2) > 0) ? *idx - 2 : 0;

    while (((buf[lookback] == ' ') || 
	   (buf[lookback] == '\t')) &&
	   (lookback > 0)) {
      lookback--;
    }

    if ((buf[lookback] == '\n' || lookback == 0)) {

      /* 
       *  If this is a line marker, parse it and set the 
       *  error location.
       */

      if (ASCII_DIGIT ((int)buf[*idx + 1]) && !line_info_line) {
	char *n, linebuf[MAXMSG];
	if ((n = index (&buf[*idx + 1], '\n')) != NULL) {
	  substrcpy (linebuf, (char *)&buf[*idx-1], 0, n - &buf[*idx - 1]);
	} else {
	  strcpy (linebuf, &buf[*idx - 1]);
	}
	line_info_tok (linebuf);
      }
      _MPUTCHAR(m,c)
      m -> tokentype = PREPROCESS;
      preprocess_line = TRUE;
      return PREPROCESS;
    } else {
      if (buf[*idx] == '#') {
	(*idx)++;
	strcpy (m -> name, "##");
	m -> tokentype = MACRO_CONCAT;
	return MACRO_CONCAT;
      } else {
	_MPUTCHAR(m,c)
	m -> tokentype = LITERALIZE;
	return LITERALIZE;
      }
    }
  }

  /* SIZEOF */
  if (c == 's' && !strncmp (&buf[*idx], "izeof", 5)) {
    (*idx) += 5;
    strcpy (m -> name, "sizeof");
    m -> tokentype = SIZEOF;
    return SIZEOF;
  }

  /* LABEL */
  if (isalnum ((int)c) || c == '_' || c == '$') {
    (*idx)--;

    q = tmpbuf;
    while (isalnum ((int)buf[*idx]) || (buf[*idx] == '_') 
	   || (buf[*idx] == '$'))
      *q++ = buf[(*idx)++];
    *q = 0;

    strcpy (m -> name, tmpbuf);
    m -> tokentype = LABEL;
    return LABEL;
  }
    
  /* LITERAL */
  /* Check for quote characters ('"') and escaped quotes within
     the literal. */
  if (c == '"') {
    int j, k, in_str;
    int n_quotes;
    int resize_x;
    int start_str, end_str;

    /*
     *  Esc_chr needs a default value that can't be mistaken
     *  for a <string index> - 1, so use -2.
     */

    for (j = *idx - 1, start_str = *idx - 1, in_str = FALSE, k = 0, resize_x = MAXLABEL,
	   esc_chr = DEFL_ESC_IDX, n_quotes = 0, end_str = -1; 
	 buf[j];
	 j++) {
      switch (buf[j])
	{
	case '\"':
	  if (esc_chr == (j - 1)) {
	    esc_chr = DEFL_ESC_IDX;
	    ADD_CHR_RESIZE
	  } else {
	    ++n_quotes;
	    if (in_str == TRUE) {
	      in_str = FALSE;
	      ADD_CHR_RESIZE
	      end_str = j;
	    } else {
	      in_str = TRUE;
	      ADD_CHR_RESIZE
	    }
	  }
	  break;
	case '\\':
	  esc_chr = (esc_chr == DEFL_ESC_IDX) ? j : DEFL_ESC_IDX;
	  ADD_CHR_RESIZE
	  break;
	default:
	  if (esc_chr != DEFL_ESC_IDX) esc_chr = DEFL_ESC_IDX;
	  ADD_CHR_RESIZE
	  break;
	}
    }

    /*
     *  If the number of quotes to the end of the input is odd,
     *  then we have an unterminated string somehow.
     */

    if (n_quotes % 2) {
      switch (interpreter_pass)
	{
	case run_time_pass:
	  _warning ("%s:%d:warning: Unterminated string constant.\n",
		    ((__getClassLibRead () == True) ? __classLibFileName () : 
		     __argvFileName ()),
		    error_line );
	  break;
	case preprocessing_pass:
	case var_pass:
	case parsing_pass:
	case library_pass:
	case method_pass:
	case expr_check:
	case c_fn_pass:
	  _warning ("%s:%d:warning: Unterminated string constant.\n",
		    __source_filename (), error_line);
	default:
	  break;
	}
    }

    m -> name[k] = 0;
    *idx += strlen (m -> name) - 1;
    m -> tokentype = LITERAL;
    return LITERAL;
  }

  /* LITERAL_CHAR in single quotes.*/
  /*
   *   There should not be much need to re-interpret character
   *   constants, except in __ctalk_to_c_char ().  It is much easier
   *   to keep the characters delimited by single quotes, so we can
   *   distinguish them from labels and escape sequences every time we
   *   retokenize them.
   *
   *   Resize message here if necessary in case there's a buffer
   *   overrun.
   */
  if (c == '\'') {
    int i = 0;
    int resize_x = MAXLABEL;
    m -> name[i++] = c;
    if (!comment) {
      while (1) {
	if (buf[*idx] == ';')
	  _error ("Unterminated character constant line %d, column %d.\n",
		  error_line, error_column);
	m -> name[i++] = buf[(*idx)++];
	if (i >= resize_x) {
	  resize_x *= 2;
	  resize_message (m, resize_x);
	}
	/*
	 *  If quoting an escaped single quote, break
	 *  on the final single quote only.
	 */
	if ((buf[(*idx) - 1] == '\'') && (buf[*idx] != '\''))
	  break;
      }
    }
    m -> tokentype = LITERAL_CHAR;
    return LITERAL_CHAR;
  }

  /* CR - LF  to NEWLINE.
     A newline token can contain a run of newlines, *but it 
     should contain no other characters* between the newlines, 
     otherwise, line numbers will not be calculated correctly 
     in tokenize (), below. */ 
  if (c == '\r' || c == '\n') {
      int buflength = 0;

      if ((*idx >= 2) && buf[*idx - 2] == '\\') {
	strcpy (m -> name, " ");
	/* Increment the error line and reset error column also. */
	++error_line; error_column = 0;
	m -> tokentype = WHITESPACE;
	return WHITESPACE;
      }

      strcpy (m -> name, "\n");

    while (((buf[*idx] == '\r') || (buf[*idx] == '\n')) &&
	   (buflength < (MAXLABEL / 2))) {
      if (buf[*idx] == c) {
	strcat (m -> name, "\n");
	/* Don't increment the error line here - 
	   set_error_location does that for all tokens that 
	   contain newlines. */
      }
      (*idx)++; buflength++;
    }
    error_column = 0;
    if (preprocess_line)
      preprocess_line = FALSE;
    m -> tokentype = NEWLINE;
    return NEWLINE;
  }

  /* MODULUS */
  /* TODO - %= */
  if (c == '%') {
    _MPUTCHAR(m,c)
    m -> tokentype = MODULUS;
    return MODULUS;
  }

  if (c == 0) {
    m -> tokentype = EOS;
    _MPUTCHAR(m,'\n')
    return 0;
  }

  if (c == '\\') {
    /* 
     *  Line splice - replace escape and newline with whitespace.  
     */
    if (!strncmp (&buf[*idx], "\r\n", 2) ||
	!strncmp (&buf[*idx], "\n\r", 2)) {
      strcpy (m -> name, " ");
      m -> tokentype = WHITESPACE;
      (*idx) += 2;
      return WHITESPACE;
    } else {
      if ((buf[*idx] == '\n') || (buf[*idx] == '\r')) {
	strcpy (m -> name, " ");
	m -> tokentype = WHITESPACE;
	++(*idx);
	return WHITESPACE;
      }
    }
  }
  _MPUTCHAR(m,c)
  m -> tokentype = CHAR;
  return c;
}

/* Tokenize a character string and place the messages on the 
   message stack used by the (push() () function.
   Return the last stack pointer.
*/

int tokenize (int (*push)(MESSAGE *), char *buf) {

  long long i = 0L, max;
  int tok, lasttok, lastlangtok;
  MESSAGE *m;
  int stack_ptr = ERROR;             /* Avoid warnings. */
  int comment_open_line = FALSE;

  max = (long long) strlen (buf);

  i = 0; comment = gnuc_attribute = FALSE;
  while (i < max) {

    if ((m = new_message()) == NULL)
      _error ("Tokenize: Invalid new_message.");

    tok = lexical (buf, &i, m);  /* get token */

    lasttok = tok;
    if (tok != WHITESPACE)
      lastlangtok = tok;

    if (tok == OPENCCOMMENT){
      if (comment) {
	_error ("Unterminated comment at line %d", comment_open_line);
      } else {
	comment = TRUE;
	strcpy (m -> name, " ");
	m -> tokentype = WHITESPACE;
	stack_ptr = (push) (m);
        set_error_location (m, &error_line, &error_column);
      }
      continue;
    }

    if (tok == CLOSECCOMMENT) {
      delete_message (m);
      comment = FALSE;
      continue;
    }

    if (comment && m -> tokentype != NEWLINE) {
      delete_message (m);
      continue;
    }

    stack_ptr = (push) (m);
    set_error_location (m, &error_line, &error_column);

  }

  error_reset ();

  return stack_ptr;
}

/*
 *   Return true if the preceding message matches c.
 */
int prefix (MESSAGE_STACK messages, int msg_ptr, int c) {

  MESSAGE *m;

  m = messages[msg_ptr+1];

  if (!m || !IS_MESSAGE (m))
    return FALSE;

  if (m -> name[0] == (char) c)
    return TRUE;

  return FALSE;
}

int set_error_location (MESSAGE *m, int *line, int *col) {

  char *p, *q;

  m -> error_column = *col;
  m -> error_line = *line;
  if (m -> tokentype == NEWLINE) {
    *col = 1;
    *line += strlen (m -> name);
  } else {
    *col += strlen (m -> name);
  }
  if (m -> tokentype == LITERAL) {
    for (p = m -> name, q = index (m -> name, '\n'); ; 
	 q = index (p+1, '\n')) {
      if (!q)
	break;
      *col = 0;
      ++(*line);
      p = q;
    }      
    *col += strlen (p);
  }
  return SUCCESS;
}

/*
 * Collect messages into a buffer for retokenization. 
 */

char *collect_tokens (MESSAGE_STACK messages, int start, int end) {

  int i,
    buflength;
  char *buf;

  for (i = start, buflength = 0; i >= end; i--) {
    if (messages[i] && IS_MESSAGE(messages[i]))
      buflength += strlen (messages[i] -> name);
    else
      break;
  }

  if ((buf = calloc (buflength + 1, sizeof (char))) == NULL)
    _error ("collect_tokens: %s.", strerror (errno));

  for (i = start; i >= end; i--) { 
    if (messages[i] && IS_MESSAGE(messages[i]))
      strcat (buf, messages[i] -> name);
    else
      break;
  }

  return buf;
}

int find_expression_limit (MESSAGE_STACK messages, int expr_start_idx, 
			    int stack_start_idx, int stack_end_idx) {
  int idx, n_parens, n_blocks, n_braces, have_parens;
  int has_fn_form;
  int lookahead_idx;
  int fn_idx;
  MESSAGE *m_tok;

  n_parens = n_blocks = n_braces = have_parens = 0;
  has_fn_form = -1;
  for (idx = expr_start_idx; idx > stack_end_idx; idx--) {
    m_tok = messages[idx];
    if ((lookahead_idx = __ctalkNextLangMsg (messages, idx, stack_end_idx))
 	!= ERROR) {
      switch (M_TOK(messages[lookahead_idx]))
	{
	case ARGSEPARATOR:
 	  if (!n_parens && !have_parens) {
 	    if (obj_expr_is_arg (messages, expr_start_idx,
				 stack_start_idx,
				 stack_end_idx,
 				 &fn_idx) != ERROR) {
 	      return idx;
 	    }
 	  }
 	  break;
	case CLOSEPAREN:
	  if (!n_parens) {
	    return idx;
	  }
	  break;
 	}
    }
    switch (M_TOK(m_tok))
      {
      case LABEL:
	/*
	 *  Check for C function syntax at the first label.
	 */
	if (has_fn_form == -1) {
	  if ((lookahead_idx = __ctalkNextLangMsg (messages, idx, stack_end_idx))
	      != -1) {
	    if (M_TOK(messages[lookahead_idx]) == OPENPAREN) {
	      has_fn_form = 1;
	    } else {
	      has_fn_form = 0;
	    }
	  }
	}
	break;
      case OPENPAREN:
	++n_parens;
	have_parens = TRUE;
	break;
      case CLOSEPAREN:
 	if (--n_parens < 0) {
 	  ++idx;
	  return idx;
	}
	break;
      case ARRAYOPEN:
	++n_blocks;
	break;
      case ARRAYCLOSE:
	if (--n_blocks < 0) {
	  /* Don't include in expression. */
	  ++idx;
	  return idx;
	}
	break;
      case OPENBLOCK:
	++n_braces;
	break;
      case CLOSEBLOCK:
	if (--n_braces < 0)
	  return idx;
	break;
      case SEMICOLON:
	/* Don't include in expression. */
	++idx;
	return idx;
	break;
      case ARGSEPARATOR:
 	if (!n_parens && have_parens && has_fn_form)
 	  return idx;
	break;
      default:
	break;
      }
  }

  return idx;
}

int find_expression_limit_old (MESSAGE_STACK messages, int expr_start_idx, 
			       int stack_end_idx) {
  int idx, n_parens, n_blocks, n_braces, have_parens;
  int has_fn_form;
  int lookahead_idx;
  MESSAGE *m_tok;

  n_parens = n_blocks = n_braces = have_parens = 0;
  has_fn_form = -1;
  for (idx = expr_start_idx; idx > stack_end_idx; idx--) {
    m_tok = messages[idx];
    switch (M_TOK(m_tok))
      {
      case LABEL:
	/*
	 *  Check for C function syntax at the first label.
	 */
	if (has_fn_form == -1) {
	  if ((lookahead_idx = __ctalkNextLangMsg (messages, idx, stack_end_idx))
	      != -1) {
	    if (M_TOK(messages[lookahead_idx]) == OPENPAREN) {
	      has_fn_form = 1;
	    } else {
	      has_fn_form = 0;
	    }
	  }
	}
	break;
      case OPENPAREN:
	++n_parens;
	have_parens = TRUE;
	break;
      case CLOSEPAREN:
 	if (--n_parens < 0) {
 	  ++idx;
	  return idx;
	}
	break;
      case ARRAYOPEN:
	++n_blocks;
	break;
      case ARRAYCLOSE:
	if (--n_blocks < 0) {
	  /* Don't include in expression. */
	  ++idx;
	  return idx;
	}
	break;
      case OPENBLOCK:
	++n_braces;
	break;
      case CLOSEBLOCK:
	if (--n_braces < 0)
	  return idx;
	break;
      case SEMICOLON:
	/* Don't include in expression. */
	++idx;
	return idx;
	break;
      case ARGSEPARATOR:
 	if (!n_parens && have_parens && has_fn_form)
 	  return idx;
	break;
      default:
	break;
      }
  }

  return idx;
}

/*
 *  Collect an expression into a buffer.
 */
char *collect_expression (MESSAGE_STACK messages, int expr_start_idx, 
			  int *expr_end_idx, 
			  int stack_start_idx, 
			  int stack_end_idx) {
  char *exprbuf;
  *expr_end_idx = find_expression_limit (messages, expr_start_idx, 
					 stack_start_idx, stack_end_idx);
  exprbuf = collect_tokens (messages, expr_start_idx, *expr_end_idx);
  return exprbuf;
}

int expr_outer_parens (MESSAGE_STACK messages, int expr_start_idx,
		       int expr_end_idx, int stack_start_idx, 
		       int stack_end_idx, int *start_paren_idx,
		       int *end_paren_idx) {

  int lookback_idx,
    paren_level,
    i,
    expr_limit_tok;
  MESSAGE *m;

  paren_level = 0;
  *start_paren_idx = *end_paren_idx = -1;
  for (lookback_idx = expr_start_idx; lookback_idx <= stack_start_idx; 
	 lookback_idx++) {
    
    m = messages[lookback_idx];

    if (M_ISSPACE(m)) continue;

    expr_limit_tok = M_TOK(m);

    if ((expr_limit_tok == SEMICOLON) ||
	(expr_limit_tok == OPENBLOCK) ||
	(expr_limit_tok == ARRAYOPEN) ||
	(expr_limit_tok == CONDITIONAL) ||
	(expr_limit_tok == COLON)) {
      goto expr_start_found;
    }
  }

 expr_start_found:

  for (i = lookback_idx; i >= expr_start_idx; i--) {
    if (M_TOK(messages[i]) == OPENPAREN) {
      ++paren_level;
      *start_paren_idx = i;
      if ((*end_paren_idx = __ctalkMatchParen (messages, i, stack_end_idx))
	  == ERROR) {
	_warning ("Mismatched parentheses.\n");
      }
    }

    if (M_TOK(messages[i]) == CLOSEPAREN)
    --paren_level;
  }

  return paren_level;
}
