cpp_token_stream.h

Go to the documentation of this file.

#ifndef cpp_token_h_included
#define cpp_token_h_included


//
// Copyright 2002, Lowell Boggs Jr.
//
// This file or directory, containing source code for a computer program,
// is Copyrighted by Lowell Boggs, Jr.  987 Regency Drive, Lewisville
// TX (USA), 75067.  You may use, copy, modify, and distribute this
// source file without charge or obligation so long as you agree to
// the following:
//
//  1.  You must indemnify Lowell Boggs against any and all financial
//      obligations caused by its use, misuse, function, or malfunction.
//      Further, you acknowledge that there is no warranty of any kind,
//      whatsoever.
//
//  2.  You agree not to attempt to patent any portion of this original
//      work -- though you may attempt to patent your own extensions to
//      it if you so choose.
//
//  3.  You keep this copyright notice with the file and all copies
//      of the file and do not change it anyway except language translation.
//
// You are responsible for enforcing your own compliance with these
// conditions and may not use this source file if you cannot agree to the
// above terms and conditions.




//


#include <fstream>
#include <string>
#include <portable_strstream.h>

namespace cxxtls
{


struct CPP_Token
  //
  //
{
  std::string text_;  



  int         type_;  





                
  int         line_;  

  std::string file_;  

  operator std::string const &() const
    //
  {
    return text_;
  }

  enum token_types_
  {
     // one character operators have a value equal to their character value

                
     eof = 0x000, 
     eql = 0x100, 
     neq = 0x101, 
     leq = 0x102, 
     geq = 0x103, 
     sle = 0x104, 
     sre = 0x105, 
     dve = 0x106, 
     ple = 0x107, 
     mne = 0x108, 
     tme = 0x109, 
     mde = 0x10a, 
     adr = 0x10b, 
     orr = 0x10c, 
     ore = 0x10d, 
     ade = 0x10e, 
     inc = 0x10f, 
     dec = 0x110, 
     shr = 0x111, 
     shl = 0x112, 
     lan = 0x113, 
     der = 0x114, 
     teq = 0x115, 
     ceq = 0x116, 
     eli = 0x117, 
     ptr = 0x118, 
     nmb = 0x201, 
     aln = 0x202, 
     inv = 0x300, 
     special_tokens
  };

  static std::string type_name(int type);


};

inline
std::ostream&
operator<<( std::ostream& o, CPP_Token const &r )
{

  return o
         << "\"" << r.file_
         << "\", line " << r.line_
         << " token, '" << r.text_ << "', is of type "
         << CPP_Token::type_name(r.type_);

}

class CPP_Buffer_Token_Source; // see bottom of file

class CPP_Token_Stream_Prep
  //
  //
{
public:

   CPP_Token_Stream_Prep()
   {
   }

   virtual ~CPP_Token_Stream_Prep() {} //< destructor

   virtual void operator() (std::string const &text,
                            std::string const &file,
                            int                line) const = 0;


};


template<class Iterator>
class CPP_Token_Stream
  //
  //
  //
  //
{
  Iterator                     cur_;                   
  Iterator                     end_;                   
  int                          line_;                  
  std::string                  file_;                  
  bool                         initialized_;           
  CPP_Token_Stream_Prep const *preprocessor_handler_;  
  char                         char_type_[256];        
  friend class                 CPP_Buffer_Token_Source;

public:


  CPP_Token_Stream(Iterator const& next,
                   Iterator const& end,
                   std::string filename="",
                   CPP_Token_Stream_Prep const *handler=0
                  )
  :
    cur_(next),
    end_(end),
    line_(1),
    file_(filename),
    initialized_(0),
    preprocessor_handler_(handler)
  {
  }


  void operator() (CPP_Token &token)

  {

    token.text_.resize(0);
    token.type_ = CPP_Token::inv;

    bool first_char = !initialized_;

    if(!initialized_)
    {
      initialize_char_type();
    }

    if(cur_ == end_)
    {
      token.type_ = CPP_Token::eof;
      token.line_ = line_;
      token.file_ = file_;
      return;
    }


    bool already_parsed = false;

    char c='@';

    // skip leading whitespace, comments, and #line directives

    if(first_char)
    {
      scan_till_first_nonblank();
        
      if(cur_ != end_ && *cur_ == '#')
        parse_preprocessor_directive();
    }

    while(cur_ != end_ &&
          ( char_type_[*cur_] == ' '  ||
            *cur_ == '\n' ||
            *cur_ == '/'  ||
            *cur_ == '\\'
          )
         )
    {
      c = *cur_;

      if(c == '\\')
      {
        if(cur_ == end_)    // handle line continuations
          c = ' ';
        else
        {
          ++cur_;
          c = *cur_;
        }
      }

      if(c == '/')
      {
        
        ++cur_;
        
        if(*cur_ == '/')
        {
          // handle end of line comments
        
          while(cur_ != end_ && *cur_ != '\n')
            ++cur_;
        
          if(cur_ != end_)
          {
            ++line_;
            ++cur_;
        
            scan_till_first_nonblank();
        
            if(cur_ != end_ && *cur_ == '#')
              parse_preprocessor_directive();
        
          }
        
          // now we should be at the beginning of the next line
        
        }
        else
        if( *cur_ == '*')
        {
          // handle old style C comments
        
          ++cur_;
        
          while(cur_ != end_)
          {
            // scan for the pair, "*/"
        
            c= *cur_;
        
            if(c == '\n')
            {
              ++line_;
            }
        
            ++cur_;
        
            if(cur_ == end_)
              break;
        
            if(c == '*' && *cur_ == '/')
            {
              ++cur_;
              break;
            }
          }
        
        
        }
        else
        {
          already_parsed = true;
        
          break;
        }
        
      }
      else
      if( c == '\n')
      {
        ++line_;
        


        if(cur_ != end_)
        {
          ++cur_;

          scan_till_first_nonblank();
        
          if(*cur_ == '#')
            parse_preprocessor_directive();
        }
      }
      else
      {
        // handle whitespace
        
        while( cur_ != end_ && char_type_[*cur_] == ' ' )
          ++cur_;
        
      }


    }


    // at this point we know we are not at the end_ of the stream, we can read and
    // classify the first character in the token


    if(cur_ == end_)
    {
      token.type_ = CPP_Token::eof;
      token.line_ = line_;
      token.file_ = file_;
      return;
    }

    if(!already_parsed)
    {
      c = *cur_;
      ++cur_;
    }

    char type = char_type_[c];

    token.text_ +=(c);

    token.line_ = line_;
    token.file_ = file_;

    // now, read any remaining characters in the token

    if(c == '\'')
    { // single character constants

       // the return value will contain the beginning and ending ' marks

       token.type_ = '\'';

       while(cur_ != end_)
       {
         char secondary_character = *cur_;

         if(secondary_character == '\\')
         {
           token.text_ +=('\\');
           ++cur_;
        
           if(cur_ != end_)
           {
             token.text_+=(*cur_);
             ++cur_;
           }
           continue;
         }

         token.text_+=(secondary_character);
         ++cur_;
        
         if(secondary_character == '\n')
         {
           ++line_;
           // no line number override should be occuring because
           // this is technically a bug in the use of the language that
           // I am trying to handle gracefully
         }

         if(secondary_character == '\'')
         {
           break;
         }
       }

    } // single character constants
    else
    if(c == '"')
    { // string constant

      // the return value will contain the leading and trailing double quotes

      token.type_ = '"';

       while(cur_ != end_)
       {
         char secondary_character = *cur_;

         if(secondary_character == '\\')
         {
           token.text_+=('\\');
           ++cur_;
        
           if(cur_ != end_)
           {
             token.text_+=(*cur_);
             ++cur_;
           }
           continue;
         }

         token.text_+=(secondary_character);
         ++cur_;
        
         if(secondary_character == '\n')
           ++line_;
           // no line number override should be occuring because
           // this is technically a bug in the use of the language that
           // I am trying to handle gracefully
        

         if(secondary_character == '"')
         {
           break;
         }
       }

    } // string constant
    else
    if( c == '.')
    {
      token.type_ = c;

      while(*cur_ == '.' && cur_ != end_)
      {
        token.type_ = CPP_Token::eli;
        token.text_ += c;
        c = *cur_;
        ++cur_;
      }

      return;
    }
    else
    if( c == '(' || c == ')' || c == '[' || c == ']' || c == '{' ||
        c == '}'
      )
    {
      token.type_ = c;
      return;
    }
    else
    {
      bool is_number=false;

      if(c >= '0' && c <= '9')
      {
        is_number = true;
        token.type_ = CPP_Token::nmb;
      }
      else
      if( (c >= 'a' && c <= 'z') ||
          (c >= 'A' && c <= 'Z') ||
          (c == '_')
        )
      {
        token.type_ = CPP_Token::aln;
      }
      else
      if( c == '-' && *cur_ == '>')
      {
        token.type_ = CPP_Token::ptr;
        token.text_ += '>';
        ++cur_;
        return;
      }

      while(cur_ != end_)
      { // main loop to aggregate characters of similar types

        char secondary_character = *cur_;

        while( secondary_character == '\\')
        {
          // handle line continuations
        
          ++cur_;
        
          if(cur_ != end_)
            secondary_character = *cur_;
          else
            secondary_character = ' ';
        
          if(secondary_character == '\n')
          {
            ++cur_;
            ++line_;
        
            if(cur_ != end_)
              secondary_character = *cur_;
            else
              secondary_character = ' ';
          }
        
        }

        if(secondary_character == '.' && !is_number && type == 'A')
          break;  // do not include '.' in names


        char secondary_type = char_type_[secondary_character];
        
        if(secondary_type != type)
        {
          if( !  (secondary_character == '=' && type == '>')  )
          {
            break;
          }
        
        }

        token.text_+=(secondary_character);

        ++cur_;

      } // main loop to aggregate characters of similar types

      token.type_ = token_type(token.text_);

    }

  }

  int token_type(std::string const &s)

  







  {

    char c = s[0];

    int size = s.size();

    switch(c)
    {
    case '0': case '1': case '2': case '3': case '4': case '5':
    case '6': case '7': case '8': case '9':
      return CPP_Token::nmb;

    case '=':
      if(size == 2 && s[1] == '=')
        return CPP_Token::eql;
        
      break;

    case ':':
      if(size == 2 && s[1] == ':')
        return CPP_Token::der;
        
      break;

    case '&':
      if(size == 1)
        return '&';
      else
      if(size == 2)
      {
        c = s[1];
        
        if(c == '=')
          return CPP_Token::ade;
        
        if(c == '&')
          return CPP_Token::lan;
        
      }
      return CPP_Token::inv;

    case '|':
      if(size == 1)
        return '|';
      else
      if(size == 2)
      {
        c = s[1];
        
        if(c == '=')
          return CPP_Token::ore;
        
        if(c == '|')
          return CPP_Token::orr;
        
      }
      return CPP_Token::inv;

    case '!':
      if(size == 1)
        return '!';
      else
      if(size == 2)
      {
        c = s[1];
        
        if(c == '=')
          return CPP_Token::neq;
        
      }
      return CPP_Token::inv;

    case '^':
      if(size == 1)
        return '^';
      else
      if(size == 2)
      {
        c = s[1];
        
        if(c == '=')
          return CPP_Token::ceq;
        
      }
      return CPP_Token::inv;

    case '%':
      if(size == 1)
        return '%';
      else
      if(size == 2)
      {
        c = s[1];
        
        if(c == '=')
          return CPP_Token::mde;
        
      }
      return CPP_Token::inv;

    case '/':
      if(size == 1)
        return '/';
      else
      if(size == 2)
      {
        c = s[1];
        
        if(c == '=')
          return CPP_Token::dve;
        
      }
      return CPP_Token::inv;

    case '~':
      if(size == 1)
        return '~';
      else
      if(size == 2)
      {
        c = s[1];
        
        if(c == '=')
          return CPP_Token::teq;
        
      }
      return CPP_Token::inv;

    case '*':
      if(size == 1)
        return '*';
      else
      if(size == 2)
      {
        c = s[1];
        
        if(c == '=')
          return CPP_Token::tme;
        
      }
      return CPP_Token::inv;

    case '+':
      if(size == 1)
        return '+';
      else
      if(size == 2)
      {
        c = s[1];
        
        if(c == '+')
          return CPP_Token::inc;
        
        if(c == '=')
          return CPP_Token::ple;
        
      }
      return CPP_Token::inv;

    case '-':
      if(size == 1)
        return '-';
      else
      if(size == 2)
      {
        c = s[1];
        
        if(c == '-')
          return CPP_Token::dec;
        
        if(c == '=')
          return CPP_Token::mne;
        
      }
      return CPP_Token::inv;

    case '>':
      if(size == 1)
        return '>';
      else
      if(size == 2)
      {
        c = s[1];
        
        if(c == '>')
          return CPP_Token::shr;
        
        if(c == '=')
          return CPP_Token::geq;
        
      }
      else
      if(size == 3)
      {
        return CPP_Token::sre;
      }
      return CPP_Token::inv;

    case '<':
      if(size == 1)
        return '<';
      else
      if(size == 2)
      {
        c = s[1];
        
        if(c == '<')
          return CPP_Token::shl;
        
        if(c == '=')
          return CPP_Token::leq;
        
      }
      else
      if(size == 3)
      {
        return CPP_Token::sle;
      }
      return CPP_Token::inv;
        
    default:

      if( (c >= 'a' && c <= 'z') ||
          (c >= 'A' && c <= 'Z') ||
          c == '_'               ||
          c == '$'
        )
        return CPP_Token::aln;
        
      break;
    }

    if(size == 1)
      return s[0];

    return CPP_Token::inv;

  }

private:

  void initialize_char_type()
    //
    //
  {
    initialized_ = true;

    unsigned i;

    // first, give every character its own unique type

    for(i=0; i < sizeof(char_type_); ++i)  char_type_[i] = i;

    // next define classes of characters

    for(i=0; i < ' '; ++i)   char_type_[i] = 1;   // type 1 is an invalid character
    for(i=128; i < 256; ++i) char_type_[i] = 1;

    char_type_[' '] = ' ';  // a blank is a blank
    char_type_['\t']= ' ';
    char_type_['\r']= ' ';
    char_type_[0x0c]= ' ';  // \f not supported everywhere

    char_type_['\\']= '\\'; // this is a special case


    for(i='A'; i <= 'Z'; ++i) char_type_[i] = 'A';  // type 'A' is alphanumerics
    for(i='a'; i <= 'z'; ++i) char_type_[i] = 'A';  // underscore and '.' are also
    for(i='0'; i <= '9'; ++i) char_type_[i] = 'A';  // treated as alphanumeric -- you must
    char_type_['_'] = 'A';                          // split the returned tokens yourself
    char_type_['.'] = 'A';                          // if you don't want them treated this way.
    char_type_['$'] = 'A';

    char_type_['+'] = '=';  // type '=' refers to operators that can combine with '='
    char_type_['-'] = '=';
    char_type_['*'] = '=';
    char_type_['/'] = '=';
    char_type_['%'] = '=';
    char_type_['&'] = '=';
    char_type_['|'] = '=';
    char_type_['~'] = '=';
    char_type_['^'] = '=';
    char_type_['='] = '=';
    char_type_['<'] = '>';
    char_type_['>'] = '>';
    char_type_['!'] = '=';

  }


  void parse_preprocessor_directive()
    //
    //
    //
  {
    // the stream should contain '# [line] number ["file"] [stackdepth]'

    while(cur_ != end_ && *cur_ == '#')
    {

      ++cur_;

      // skip white spaces before the 'line' keyword or the number

      while(cur_ != end_ && *cur_ == ' ')
        ++cur_;

      if(cur_ == end_)
        return;

      if(*cur_ == 'l')
      {
        // assume this is a line keyword

        while(cur_ != end_ && *cur_ != ' ')
          ++cur_;

        // skip white spaces after the 'line' keyword

        while(cur_ != end_ && *cur_ == ' ')
          ++cur_;
        
        if(cur_ == end_)
          return;
      }

      char c = *cur_;

      if(c < 0 || c > '9')
      {
        // this is a pragma or other ignorable pre-processor directive
        // handle line continuations using either \ or comments that start
        // in the macro body and don't complete on the same line
        
        std::string directive;       // could also be named 'accumulator'
        int         start_line=line_;

        char slash        = ' ';
        bool in_a_comment = false;

        while(cur_ != end_ && *cur_ != '\n')
        {
          slash = *cur_;
          ++cur_;
        
          if(!in_a_comment)
            directive += slash;
        
          if(cur_ != end_)
          {
            if(*cur_ == '*' && slash == '/')
              in_a_comment = true;
            else
            if(*cur_ == '/' && slash == '*')
              in_a_comment = false;
          }
        
        }
        
        if(cur_ != end_)
        {
          ++cur_;  // skip the '\n'
          ++line_; // keep the line_ variable correct

          if(!in_a_comment)
            directive += '\n';

          scan_till_first_nonblank();
        
          while(slash == '\\' || in_a_comment)
          {
            // the next line is a continuation line
        
            while(cur_ != end_ && *cur_ != '\n')
            {
              slash = *cur_;
              ++cur_;
        
              if(!in_a_comment)
                directive += slash;
        
              if(cur_ != end_)
              {
                if(*cur_ == '*' && slash == '/')
                  in_a_comment = true;
                else
                if(*cur_ == '/' && slash == '*')
                  in_a_comment = false;
              }
              else
              {
                in_a_comment=false;
              }
        
            }
        
            if(cur_ != end_)
            {
              ++cur_;  // skip the '\n'
              ++line_; // keep the line_ variable correct
        
              if(!in_a_comment)
                directive += '\n';

              if(scan_till_first_nonblank())
                slash=' ';

            }
            else
            {
             slash=' ';
             in_a_comment=false;
            }
          }
        
        }

        if(preprocessor_handler_)
          (*preprocessor_handler_)(directive, file_, start_line);

        continue;

      }

      // ok, so we should have 'number ["file"]'

      long line=0;

      while(cur_ != end_)
      {
        c = *cur_;

        if(c < '0' || c > '9')
          break;
        
        ++cur_;

        line *= 10;
        line += c - '0';
      }

      line_ = line;

      // so we have eaten the line number, now see if a file name
      // is specified

      // skip white spaces after the line number

      while(cur_ != end_ && *cur_ == ' ')
        ++cur_;

      if(cur_ == end_)
        return;

      if(*cur_ == '\n')
      {
        ++cur_;
        while(cur_ != end_ && char_type_[*cur_] == ' ')
         ++cur_;
        continue;
      }

      // ok we have a filename specified

      ++cur_;  // skip the leading "

      file_.resize(0);

      while(cur_ != end_ && (c = *cur_) != '"' && c != '\n' )
      {
        file_ += c;
        ++cur_;
      }

      while(cur_ != end_ && *cur_ != '\n')
        ++cur_;

      if(cur_ != end_)
      {
        ++cur_;  // skip the \n
        while(cur_ != end_ && char_type_[*cur_] == ' ')
         ++cur_;
      }

    }

  }

  bool scan_till_first_nonblank()
  {
    bool rv = false;

    for(;;)
    {
      // skip blanks and blank lines

      while(cur_ != end_ && char_type_[*cur_] == ' ')
        ++cur_;
        
      if(cur_ == end_ || *cur_ != '\n')
        return rv;
        
      // *cur_ == '\n'

      ++cur_;
      ++line_;
      rv = true;

        
    }

  }

};

struct CPP_Token_Source
{
  virtual void operator()(CPP_Token&) = 0;

  virtual ~CPP_Token_Source() {}
};



class CPP_Stream_Token_Source
: public CPP_Token_Source
//
{
public:

  CPP_Token_Stream<std::istreambuf_iterator<char> > *stream_;

  CPP_Stream_Token_Source(std::istream &istr,
                          std::string filename="",
                          CPP_Token_Stream_Prep const* h=0)
  {

     std::istreambuf_iterator<char> cur(istr);
     std::istreambuf_iterator<char> end;

     stream_ = new CPP_Token_Stream<std::istreambuf_iterator<char> >(cur,end,filename,h);
        
  }

  ~CPP_Stream_Token_Source()
  {
    delete stream_;
  }


  void operator() (CPP_Token &token)
  {
    (*stream_)(token);
  }

};

class CPP_File_Token_Source
: public CPP_Stream_Token_Source
//
//
{
  std::ifstream *input_file_; 

public:

  CPP_File_Token_Source(char const *f = "", CPP_Token_Stream_Prep const* h=0)
  : CPP_Stream_Token_Source( *( input_file_ = new std::ifstream(f) ), f, h )
  {
  }

  CPP_File_Token_Source(std::string const &f, CPP_Token_Stream_Prep const* h=0)
  : CPP_Stream_Token_Source( *( input_file_ = new std::ifstream(f.c_str()) ), f, h )
  {

  }

  ~CPP_File_Token_Source()
  {

    delete input_file_;
  }

  operator bool() const { return bool(*input_file_); } 

};


class CPP_String_Token_Source
: public CPP_Token_Source
{

  std::string copy_; 

  CPP_Token_Stream<std::string::const_iterator> stream_; 

public:

  CPP_String_Token_Source(std::string s)
  : copy_(s),
    stream_(copy_.begin(), copy_.end())
  {
  }

  void operator() (CPP_Token &token)
  {
    stream_(token);
  }

};


class CPP_Buffer_Token_Source
: public CPP_Token_Source
{

  CPP_Token_Stream<char const *> stream_; 

public:

  CPP_Buffer_Token_Source(char const *begin,
                          char const *end,
                          CPP_Token_Stream_Prep const *h=0)
  : stream_(begin, end,"",h)
  {
  }

  CPP_Buffer_Token_Source(char const *begin,
                          char const *end,
                          std::string f,
                          CPP_Token_Stream_Prep const *h=0)
  : stream_(begin, end,f,h)
  {
  }


  void operator() (CPP_Token &token)
  {
    stream_(token);
  }

  char const * cur() const { return stream_.cur_; } 
  char const * end() const { return stream_.end_; } 



};

} // namespace cxxtls
#endif