cpptagdb.cxx

Go to the documentation of this file.

//
// Copyright 2002-2011, Lowell Boggs Jr.
//
// This file or directory, containing source code for a computer program,
// is Copyrighted by Lowell Boggs, Jr.  987 Regency Drive, Lewisville
// TX (USA), 75067.  You may use, copy, modify, and distribute this
// source file without charge or obligation so long as you agree to
// the following:
//
//  1.  You must indemnify Lowell Boggs against any and all financial
//      obligations caused by its use, misuse, function, or malfunction.
//      Further, you acknowledge that there is no warranty of any kind,
//      whatsoever.
//
//  2.  You agree not to attempt to patent any portion of this original
//      work -- though you may attempt to patent your own extensions to
//      it if you so choose.
//
//  3.  You keep this copyright notice with the file and all copies
//      of the file and do not change it anyway except language translation.
//
// You are responsible for enforcing your own compliance with these
// conditions and may not use this source file if you cannot agree to the
// above terms and conditions.




#include <cxxtls/cpp_token_stream.h>
#include <cstdlib>
#include <fstream>
#include <iostream>
#include <cxxtls/file.h>
#include <algorithm>
#include <portable_strstream.h>
#include <portable_io.h>
#include <list>
#include <cxxtls/options.h>
#include <portable_io.h>
#include <string.h>
#include <ctype.h>

using namespace std;
using namespace cxxtls;

static void parse_file(FileName const &);

static ostreambuf_iterator<char>* output;

static bool read_filenames_from_stdin = false; 
static bool log_all_tokens_to_stderr  = false; 
static bool log_filenames_to_stderr   = false; 
static bool grep_style_output         = false; 
static bool log_resumes               = false; 

int main(int argc, char **argv, char **environ)
{

# ifdef _MSC_VER
  
    // force the output to be a binary stream and we'll control eht end of line sequencing our
    // selves
  
    cout.flush();
  
    setmode(1, O_BINARY);

# endif

  output = new ostreambuf_iterator<char>(cout);

  //
  // parse the options
  //

  ProgramOptions ops("-v,-grep,-files,-tokens,-stdin,-resume,-help,-h,--help,-dcl;",
                     argc,
                     argv,
                     environ
                    );

  if(ops.option("-v"))
  {
     // print the version number and quit.

     cout << "cpptagdb version 2.7" << endl;

     exit(1);

     // version 2.7 a.  Fix bugs in the handling of template specializations and the inline keyword
     //                 as applied to declarations.
     // version 2.6 a.  added support for class template specializations.
     //
     // version 2.5 a.  Added support a common boost source code paradigm:  macros to generate class
     //                 names:
     //
     //                   struct MACRO(parm) { /* class body */ };
     //
     // version 2.3 a.  Eliminated the need for the -dcl option.
     //
     // version 2.2 a.  Fixed major bug:  Now it is possible to correctly detect the
     //                 following class name:
     //
     //                     class CursorWindow::Dialog::Impl { ... }
     //
     // version 2.1  a.  Added the -dcl option that lets the user add
     //                  synonyms for __declspec().  
     //
     //              b.  Added script cpptagdb_find_cdecl_synonyms,
     //                  which figures out what the parameters for -dcl
     //                  ought to be.
     //
     //              c.  Added builtin synonyms for boost macros and
     //                  MS visual studio 2010 macros.

     // Prior to version 2.1, I was not keeping track of the versions.



  }


  grep_style_output         = ops.option("-grep");
  log_filenames_to_stderr   = ops.option("-files");
  read_filenames_from_stdin = ops.option("-stdin");
  log_all_tokens_to_stderr  = ops.option("-tokens");
  log_resumes               = ops.option("-resume");


  if(ops.option("-help")  ||
     ops.option("--help") ||
     ops.option("-h")
    )
  {
    FileName program(ops.argv[0]);

    cerr << "Usage:" << endl
         << "  "     << program.basename()
         << "  [options] [.c and .h file names]" << endl
         << "Options:" << endl
         << "  -v       print program version and quit" << endl
         << "  -help    print this help message" << endl
         << "  -h       ditto" << endl
         << "  --help   ditto" << endl
         << "  -stdin   read the .c and .h file names from stdin" << endl
         << "           note:  one filename per line" << endl
         << "  -tokens  log all tokens to stderr" << endl
         << "  -files   log process files to stderr" << endl
         << "  -grep    use grep style output of the symbols" << endl
         << "  -resume  log current scope name when ending classes" << endl
         ;

    exit(0);
  }

  ops.argv.erase(ops.argv.begin());  // remove the program name from the
                                     // non-optional program arguments


  // get the list of file names to work on

  list<FileName> files;

  if(read_filenames_from_stdin)
  {
    string tmp;

    while(!cin.eof())
    {
      tmp.resize(0);

      getline(cin, tmp);

      if(tmp.size() == 0)
        break;
        
        
      files.push_back(tmp);

    }
  }
  else
  {
    size_t i;

    for(i=0; i < ops.argv.size(); ++i)
    {
      files.push_back(ops.argv[i]);
    }

  }

  // parse the named files

  list<FileName>::iterator l = files.begin();
  list<FileName>::iterator g = files.end();


  for(; l != g; ++l )
  {
    if(!l->exists())
    {
      cerr << *l << " error, file does not exist" << endl;
    }
    else
    if(!l->is_dir())
      parse_file(*l);
  }

  exit(0);
}

static void parse_declarations(CPP_Buffer_Token_Source &s);
static void log_symbol(string type, string name, string file, int line, bool include_scope=true);


class DefineHandler
: public CPP_Token_Stream_Prep
{
public:

  void operator() (string const &define, string const &file, int line) const
  {
    string::const_iterator f(define.begin());
    string::const_iterator g(define.end());

    // only handle #defines

    if( f == g || *f != 'd')
      return;

    // skip the word 'define'

    while(f != g && !isspace(*f) ) ++f;

    // skip blanks after define

    while(f != g && ( isspace(*f) || *f == '\n') ) ++f;

    if(f == g)
      return;  // missing define variable name

    // skip past the text in the define variable name

    string::const_iterator l(f);

    while(l != g &&
          (
            (*l >= 'a' && *l <= 'z') ||
            (*l >= 'A' && *l <= 'Z') ||
            (*l >= '0' && *l <= '9') ||
            (*l == '_')
          )
         ) ++l;

    // f,l now defines the name of the define variable

    static string define_name("define");

    log_symbol(define_name, string(f,l), file, line, false);

  }
};



static void parse_file(FileName const& name)
//
// Parse a file and print out the definitions contained
// therein.
//
{
  if(!grep_style_output)
    cout << '+' << name << endl;

  FileContents buffer;

  name.slurp(&buffer);

  if(!buffer.ok())
  {
    cerr << "Error opening " << name << ", " << buffer.error() << endl << flush;
    exit(1);
  }

  DefineHandler dh;

  if(log_filenames_to_stderr)
    cerr << name << endl;

  CPP_Buffer_Token_Source s(buffer.begin(), buffer.end(), name, &dh);

  parse_declarations(s);

}

static CPP_Token                token;
static CPP_Buffer_Token_Source *stream;

inline static void next_token()
//
// get the next token from the stream and leave it 'token'
//
{
  (*stream)(token);

  if(token.type_ == CPP_Token::aln &&
     token.text_[0] == 'o' &&
     token.text_ == "operator"
    )
  {
    (*stream)(token);

    if(token.type_ == '(')
    {
      token.type_ = CPP_Token::aln;

      token.text_ = "operator " + token.text_;

      CPP_Token tmp;

      (*stream)(tmp);

      token.text_ += tmp.text_;
    }
    else
    {
      token.type_ = CPP_Token::aln;

      token.text_ = "operator " + token.text_;
    }



  }


  if(log_all_tokens_to_stderr)
    cerr << token << endl;

}

inline bool eof()
//
// determine if we are at the end of the token stream
//
{
   return token.type_ == CPP_Token::eof;
}

static bool parse_declaration(bool log_func_forward_decls=false);

string outer_scope;

static void parse_declarations(CPP_Buffer_Token_Source &s)
//
// parse and print out all definitions in a file
// (don't use this for parsing nested stuff)
{
  stream = &s;

  for(next_token(); !eof(); )
  {
    outer_scope = "::";

    if(parse_declaration())  // automatically calls next_token()
      break;
  }

  if(!eof())
  {
    cerr << token << ", error:  expected end of file" << endl;
    cerr << __FILE__ << ":" << __LINE__ << endl;
  }

}

#define ASSUMED_TOKEN(type)   if(token.type_ != type)   {     cerr << token << ", error:  expected " << CPP_Token::type_name(type) << endl;     cerr << __FILE__ << ":" << __LINE__ << endl;     return true;   }



// all parsing functions leave the token that terminated them
// in the input stream.

static bool parse_class(bool handling_typedefs=false);
static bool parse_namespace();
static bool parse_typedef();

static bool parse_enum(bool handling_typedefs=false);
static bool parse_extern();
static bool parse_using();
static bool parse_funcvar(bool log_forward_funcs_defs=false,
                          bool handling_typedefs=false);
static void eat_template_parms();
static void eat_function_body();

static bool parse_declaration(bool log_func_forward_decls)
{
  if(token.type_ == ';' ) // allow randomly placed semicolons
  {
    next_token();
    return false;
  }

  if(token.type_ == '{')
  {
    // allow randomly placed {} blocks

    eat_function_body();
    next_token();
    return false;

  }

  if(token.type_ == CPP_Token::eof)
    return true;

  if(token.type_ == '=')  // handle oddly placed variable initializations
  {
     while(token.type_ != ';')
     {
       if(token.type_ == '{')
         eat_function_body();

       next_token();
     }

     return false;
  }


//  if(token.type_ != CPP_Token::der)
//    ASSUMED_TOKEN(CPP_Token::aln);

  if(token.text_ == "template")
  {
    // eat template<parms>

    next_token();  // eat 'template'

    eat_template_parms();

    if(token.type_ == '>')
      next_token();

    // now all references to the template keyword and its parms are
    // gone fromthe stream

  }

  if(token.type_ == CPP_Token::aln && token.text_[0] == 'i' &&
     token.text_ == "inline")
     next_token();

  if(token.text_ == "class"  || token.text_ == "struct" || token.text_ == "union")
    return parse_class();
  else
  if(token.text_ == "typedef" )
    return parse_typedef();
  else
  if(token.text_ == "enum" )
    return parse_enum();
  else
  if(token.text_ == "namespace" )
    return parse_namespace();
  else
  if(token.text_ == "using" )
    return parse_using();
  else
  if(token.text_ == "extern")
    return parse_extern();
  else
  if(token.text_ == "friend")
  {
    next_token(); // eat the friend keyword

    if((token.type_ == CPP_Token::aln) &&
       (token.text_ == "class" ||
        token.text_ == "struct"
       )
      )
    {
      // prevent a misnaming of the scope caused by the
      // parsing of friend declarations.

      while(token.type_ != ';')
        next_token();
    }

    return parse_declaration(log_func_forward_decls);

  }
  else
  if(token.type_ == CPP_Token::aln &&
     token.text_[0] == 'p'         &&
       (token.text_ == "public"    ||
        token.text_ == "private"   ||
        token.text_ == "protected"
       )
    )
  {
    //
    // eat the public:, private:, and protected: syntax so as not
    // to confuse the 'public: typedef int x' as a variable definition.
    //
    next_token();

    if(token.type_ == ':')
      next_token();

    return false;
  }
  else
    return parse_funcvar(log_func_forward_decls);

}


static void log_symbol(string type, string name, string file, int line, bool include_scope)
{
  // the file parameter is unused unless we are doing grep style output

  if(name.size() != 0)
  {
    if(grep_style_output)
    {
      char buffer[40];

      sprintf(buffer, "%d", line);

      copy(file.begin(), file.end(), *output);
      *(*output)++ = ':';

      copy(buffer, buffer + strlen(buffer), *output);
      *(*output)++ = ':';

      *(*output)++ = ' ';

      copy(type.begin(), type.end(), *output);
      *(*output)++ = ' ';

      if(include_scope)
      {
        if(name.begin() != name.end() && *name.begin() != ':')
            copy(outer_scope.begin(), outer_scope.end(), *output);  // only symbols not in the global scope,
                                                                    // prefix them with their scope
      }
        
      copy(name.begin(), name.end(), *output);


      *output = copy_end_of_line(*output);

    }
    else
    {

      *(*output)++ = '-';

      copy(type.begin(), type.end(), *output);
      *(*output)++ = ' ';

      if(include_scope)
      {
        if(name.begin() != name.end() && *name.begin() != ':')
             copy(outer_scope.begin(), outer_scope.end(), *output); // only symbols not in the global scope,
                                                                    // only symbols not in the global scope,   
      }
        
      copy(name.begin(), name.end(), *output);
      *(*output)++ = ' ';

      char buffer[40];

      sprintf(buffer, "%d", line);

      char *scan = buffer;

      while(*scan)
        *(*output)++ = *scan++;

      *output = copy_end_of_line(*output);

    }
  }
}

struct ScopeBinder
//
//
{
  string saved_scope; 

  ScopeBinder(string new_scope)
  {
    saved_scope = outer_scope;
    outer_scope = new_scope;
  }

  ~ScopeBinder()
  {
    outer_scope = saved_scope;

    if(log_resumes)
    {
      string tmp = outer_scope;

      int l = tmp.size();

      if(l > 2 && tmp[l-1] == ':' && tmp[l-2] == ':')
      {
        tmp.erase( tmp.begin() + (l-2), tmp.end() );
      }

      log_symbol("resume", tmp, token.file_, token.line_, false);
    }
  }

};


static void parse_variables_defined(bool log_func_forward_decls=false,
                                    bool handling_typedefs=false
                                   )
// parse and log the names of variables defined as part of a struct
// declaration:
//
//   struct x { ... }  var1, *var2, function(..), array[...], ... ;
{

  if(token.type_ != ';')
    parse_funcvar(log_func_forward_decls, handling_typedefs);

  return;

}
static void eat_function_parms(string *parmtext=0);



static bool parse_class(bool handling_typedefs)
//
//  parse class, union, and struct declarations and keep track
//  of the name of the 'scope' of each nested class
//
{

  string class_type = token.text_;

  next_token();  // eat the 'struct' or 'class' keyword

  string name = token.text_;  // save the class name
  string file = token.file_;
  int    line = token.line_;
  bool   maybeFunc = false;

  if(token.type_ == CPP_Token::der)
  {
      // handle this:
      //
      //  class ::SomeClass { ... }

      next_token();

      ASSUMED_TOKEN(CPP_Token::aln);

      name += token.text_;
      line  = token.line_;


  }


  if(token.type_ == '{')
  {
    name="unnamedstruct"; // handle struct { ... }
  }
  else
  {
      if(token.type_ == '[')
      {
        // someone in plain old c has defined int class[470];
        
        int depth=0;
        
        while(!eof())   // eat the contents of the arrays
        {
          if(token.type_ == '[')
            ++depth;
          else
          if(token.type_ == ']')
          {
            --depth;
            if(depth == 0)
              break;
          }
          next_token();
        }

        next_token();  // eat the closing ]
        
      }
      else if(token.type_ == '(')
      {
        // oops, someone in plain old C has done this:  int class() ...
        
        eat_function_parms();
        next_token();
      }


      if(token.type_ == ';' || token.type_ == ',' )
      {
          // we are not parsing a real class definition
          // but rather a plain old c style variable
          // declaration like 'int class;'
          next_token();
          return false;
      }

    ASSUMED_TOKEN(CPP_Token::aln);

    while(token.type_ == CPP_Token::aln)
    {

        next_token(); // should get here at least once

        if(token.type_ == CPP_Token::aln)
        {
           maybeFunc=true;

           // Assume we parsing something like this:
           //
           //   class MACRO ClassName {... }
           //               ^
           //
           // But we also have to support this:
           //
           //   class SomeClass  function() { return SomeClass(); }
           //                    ^
           //
           // And this
           // 
           //   class MACRO RealClassName { ... }
           //               ^

           name = token.text_;  // save the class name
           file = token.file_;
           line = token.line_;
        }
        else
        if( maybeFunc && token.type_ == ';' )
        {
           log_symbol("variable", name, file, line);
           return false;
        }

    }

    while(token.type_ == CPP_Token::der)
    {
       // handle NAME::NAME::NAME... as the class name, as in
       //
       //   class CursorWindow::Dialog::Impl { ... }

       name += token.text_;
       line  = token.line_;

       next_token();

       if(token.type_ != CPP_Token::aln)
         break;

       name += token.text_;
       line  = token.line_;
       next_token();

    }

  }


  if(token.type_ == ';')
    return false;

  if(token.type_ == CPP_Token::aln ||
     token.type_ == '&'            ||
     token.type_ == '*'
    )
  {
    // this is a variable declaration such as when some does this
    // in plain old c   typedef int class;   class &v1, *v2, x;

    parse_variables_defined(false, handling_typedefs);

    return false;

  }

  if(token.type_ == '(')
  {
      // we have something like this:
      //
      //   class something(...) 
      
      eat_function_parms(); 

      if(token.type_ == ')')
      {
        next_token();

        if(maybeFunc)
        {
            if(token.type_ == '{' ||
               token.type_ == ';' ||
               (token.type_ == CPP_Token::aln && token.text_ == "const")
              )
            {
                // we have something like this:
                //
                //  class  ClassName  functName() const { ... }
                //                                ^     ^
                // we are not defining a class, we are defining a function

                while(   token.type_  != CPP_Token::eof
                      && token.type_  != ';'
                      && token.type_  != '{'
                      && token.type_  != '}'
                     )
                {
                   next_token(); // skip till { or end of declaration

                }

                if(token.type_ == CPP_Token::eof)
                   return true;

                if(token.type_ == '{')
                {
                    log_symbol("function", name, file, line);
                   
                }

                return false;

            }
             
        }

      }
      
      // assume we are seeing something like this:
      //
      //   class compiler_directive(parms)   name { ...

      if(token.type_ == CPP_Token::aln || token.type_ == CPP_Token::der)
      {      
          name = token.text_;  // save the class name
          file = token.file_;
          line = token.line_;
      }
      
      if(token.type_ == CPP_Token::der)
      {
          // we are seeing this:
          //
          //   class directive(parms) ::name ...

          next_token();

          if(token.type_ == CPP_Token::aln)
          {
             name += token.text_;
             line  = token.line_;
          }

          next_token();

      }
      else
      {

        if(token.type_ == '<')
        {
           eat_template_parms();
           if(token.type_ == '>')
              next_token();
        }

        if(token.type_ == '{' || token.type_ == ':' )
          {
            // user has done something like this
            //   class function(parms) { .. }
            //                         ^
            // Normally, this is a syntax error, but if function
            // is really a macro that computes the class name, then, mabye
            // we want to not complain about it...

            if(token.type_ == ':')
              {
                while(   token.type_ != ';'
                      && token.type_ != '{'
                      && token.type_ != CPP_Token::eof
                     )
                  {
                    next_token(); // consume tokens until { or ;
                  }
              }

          }
        else
        if(token.type_ == ';')
        {
             next_token();
             return false; // forward decl of badly formed class def
        }
        else
        if(token.type_ == CPP_Token::der)
        {
           // we are seeing someting like this:
           //   struct Macro(parms)<parms>  ::result<templateParms> : baseMacro(parms) {};
           //                               ^
           //
           // this is a multi-level specialization

           name=""; // we are going to reconstruct the name to refer to result

           while(token.type_ == CPP_Token::der)
           {
              next_token();
              
              ASSUMED_TOKEN(CPP_Token::aln);

              name+= token.text_;
              line = token.line_;
              file = token.file_;

              next_token();
           }

        }
        else
        {
            ASSUMED_TOKEN(CPP_Token::aln);
            next_token();
        }

      }

      while(token.type_ == CPP_Token::der)
      {
         name += token.text_;
         line  = token.line_;
         next_token();

         if(token.type_ != CPP_Token::aln)
            break;

         name += token.text_;
         line  = token.line_;

         next_token();

      }

      // At this point, we are here:
      //
      //   class directive(name) ::name::name::name ...
      //                                            ^

  }
  else
  if(token.type_ == '[' || token.type_ == '=')
  {
     // have syntax like this:
     //
     //   struct T array[40];
     //                 ^

     log_symbol("variable", name, file, line);

     while(    token.type_ != CPP_Token::eof 
           &&  token.type_ != ';'
          )
     {
         next_token();
     }

     return false;

  }
  else
  if(token.type_ == '<')
  {
      // we have something like
      //
      //   template<> struct Class<instanceParms> { ... };

      eat_template_parms();
      if(token.type_ == '>')
         next_token();
     
  }



  // now we know that we really are in a normal c++ class
  // union, or struct declaration.  The classname now gets
  // the added to the scope.


  log_symbol(class_type, name, file, line);

  string newScope;

  static string colon_colon("::");


  if(name[0] != ':')
  {
      newScope = outer_scope + name + colon_colon;
  }
  else
  {
     newScope = name + colon_colon;
  }

  ScopeBinder saved_scope(newScope);

  while(!eof() && token.type_ != '{' && token.type_ != ';')
    next_token();

  if(eof())
    return true;

  if(token.type_ != ';')
    {
      next_token(); // eat }

  while(token.type_ != '}')
  {
    if(token.type_ == '~')   // destructors mess up the parse_declaration logic
    {
      next_token();
      
      if(token.type_ == CPP_Token::aln)
      {
          static string tilde("~");

          token.text_ = tilde + token.text_;
      }

    }

    if(parse_declaration(true))
      return true;
  }
    }

  next_token();

  outer_scope = saved_scope.saved_scope;

  parse_variables_defined(false,handling_typedefs);  // as part of this class declaration

  return false;

}

static void eat_type_name()
//
// parse and discard a typename -- particularly it handles
// the unsigned type's bizare variants.
//
{
  while(token.type_ == CPP_Token::aln &&
        ( (token.text_ == "typename") || (token.text_ == "const") )
       )
  {
    next_token();
  }

  if(token.text_ == "unsigned")
  {
    next_token();

    if(token.type_ == CPP_Token::aln &&
       (token.text_ == "int"    ||
        token.text_ == "short"  ||
        token.text_ == "long"   ||
        token.text_ == "char"
       )
      )
        next_token();
  }
  else
  {
    do
    {
      next_token();
    }
    while(token.type_ == CPP_Token::der);

  }

  eat_template_parms();  // if this is a template type name eat the parms

  while(   (!eof() && token.type_ == '*') 
        || (token.type_ == '&' )
        || (token.type_ == CPP_Token::aln && token.text_ == "const")
       )
  {
    next_token(); // eat normal type modifiers
  }

}


static bool parse_typedef()
{
  next_token();  // eat the word 'typedef'

  if(token.text_ == "class" || token.text_ == "struct" || token.text_ == "union")
    return parse_class(true);

  if(token.text_ == "enum")
    return parse_enum(true);

  eat_type_name();

  parse_variables_defined(false, true);
                                        

  return false;
}

static bool parse_enum(bool handling_typedefs)
//
//  parse and log enumeration declarations
//
//    enum { ... } variables
//    enum name {  ... } variables
//    enum name variables
//    enum [name] { name [=value,]... } [variables]
//
{

  next_token();  // eat 'enum'

  CPP_Token save = token;

  if(token.type_ != '{')
     next_token();

  if(token.type_ == '{')
  {
    static string enum_type("enum");
    static string enum_value("enumeration");

    if(save.text_[0] != '{')
        log_symbol(enum_type, save.text_, save.file_, save.line_);

    // parse { name [=value, ...] }

    next_token();  // eat '{'

    while(token.type_ != '}')
    {
       if(token.type_ == CPP_Token::aln)
       {
         log_symbol(enum_value, token.text_, token.file_, token.line_);
        
         next_token();
        
         if(token.type_ == '=')
         {
           // eat initializer
        
           while(!eof() && token.type_ != ',' && token.type_ != '}')
             next_token();
        
         }
        
       }
       else
         next_token();
    }

    next_token();  // eat the closing '}'

  }

  parse_variables_defined(false, handling_typedefs);

  return false;
}
static bool parse_namespace()
  //
  // handle namespace [optional name] ;
  //
{
  next_token(); // skip the namespace keyword

  ScopeBinder old_scope(outer_scope);

  if(token.type_ == CPP_Token::aln)
  {
    log_symbol("namespace", token.text_, token.file_, token.line_, true);

    outer_scope += token.text_ + "::" ;
    next_token();
  }
  else
  {
    outer_scope = "<filescope>::";
  }

  if(token.type_ == '{')
  {
     next_token();

     while(!eof() && token.type_ != '}')
     {
       if(parse_declaration())
         return true;
     }

     next_token();

     return false;

  }

  return parse_declaration();

}
static bool parse_using()
//
// handle using namespace ;
// or     using namespace::identifier ;
//
{
  next_token();  // eat 'using'

  bool is_namespace(false);

  if(token.text_ == "namespace")
  {
    next_token();
    is_namespace = true;
  }

  CPP_Token tmp(token);

  tmp.text_.resize(0);

  while(!eof() && token.type_ != ';')
  {
    if(token.type_ == CPP_Token::aln ||
       token.type_ == CPP_Token::der
      )
    {
      tmp.text_ += token.text_;
    }

    next_token();
  }

  next_token(); // eat the semicolon

  static string used_symbol("used");

  if(!is_namespace && tmp.text_.size())
    log_symbol(used_symbol, tmp.text_, tmp.file_, tmp.line_);

  return false;
}

static void eat_function_body()
//
// ignore all tokens between matching {}'s.
// (leaves '}' in the stream)
//
{
  int depth=0;

  while(!eof())
  {
     if(token.type_ == '{')
       ++depth;
     else
     if(token.type_ == '}')
     {
       --depth;
       if(depth == 0)
         return;
     }

     next_token();
  }

}

static void eat_function_parms(string *parmtext)
//
// ignore all tokens between matching ()'s.
// (leaves ')' in the stream).  The first alphanumeric
// token between the paren's is stuck in parmtext if
// non-zero
//
{
  int depth=0;

  while(!eof())
  {
     if(token.type_ == '(')
       ++depth;
     else
     if(token.type_ == ')')
     {
       --depth;
       if(depth == 0)
         return;
     }

     if(parmtext && depth == 1 && token.type_ == CPP_Token::aln)
     {
       *parmtext = token.text_;
       parmtext = 0;
     }

     next_token();
  }

}

static void eat_template_parms()
  // parse and discard template parms
  //
  // that is:   < stuff, more_stuff< a, b, c>, ... >
  // Also handles:
  //
  //   class templatename< ... >
  //
{

  int depth=0;

  if(token.type_ == CPP_Token::aln &&
     token.text_ == "class"
    )
  {
    next_token();  // eat 'class' keyword
    next_token();  // eat template specialization name
  }

  if(token.type_ != '<')
    return;

  while(!eof())
  {
     if(token.type_ == '<')
       ++depth;
     else
     if(token.type_ == '>')
     {
       --depth;
       if(depth == 0)
         return;
     }

     next_token();
  }


}

static bool parse_funcvar(bool log_forward_func_declarations,
                          bool handling_typedefs
                         )
//
// parse all function and variable declarations -- and handle
// any leftover trash that occurs whenever you are parsing
// something that is not strictly syntactically correct.
//
// So instead of defining a function like this:
//
//    int function(parms) { body }
//
// This code will accept
//
//    trash ... trash int trash trash ... function(parms) trash .. trash { ... }
//
// It will also parse handle things like this:
//
//    trash struct { members } varname ;
//
// The 'trash' that is excepted includes pretty much any random tokens
// and when '[', '{', the text up to the corresponding closing token
// is simply ignored.
//
{
  // scan tokens until you first encounter one of
  //  (, [, {, struct/class/union/template or ;
  // that tells you that you are defining.

  CPP_Token tmp; // save token just before terminator

  if(token.text_ == "static")
    next_token();

  do
  {



    if(token.type_ == '{')
    {
      // some bizarre code fragment we misunderstood

      eat_function_body();

      next_token();  // eat the trailing '}'


      if(token.type_ == ',')
      {
          static string unknown_type("unknown");
        
          next_token();
          log_symbol(unknown_type, tmp.text_, tmp.file_, tmp.line_);
          return parse_funcvar(log_forward_func_declarations, handling_typedefs);
      }
      return false;

    }

    if(token.type_ == CPP_Token::aln)
    {
      if(token.text_ == "struct"   ||
         token.text_ == "class"    ||
         token.text_ == "union"    ||
         token.text_ == "template" ||
         token.text_ == "extern"
        )
        return parse_declaration();

      tmp = token;

      next_token();

      while(token.type_ == CPP_Token::der)
      {
        tmp.text_ += token.text_;
        next_token();

        if(token.type_ == CPP_Token::aln || token.type_ == '~')
        {
           tmp.text_ += token.text_;
           tmp.line_  = token.line_;
           tmp.file_  = token.file_;

           if(token.type_ == '~')
             {
               next_token();
               if(token.type_ == CPP_Token::aln)
                 {
                   tmp.text_ += token.text_;
                   tmp.line_  = token.line_;
                   tmp.file_  = token.file_;

                 }
             }

           next_token();
        }

      }



      if(token.type_ == '<')
      {
        eat_template_parms();
      }

    }
    else
    if(token.type_ == '=')
    {
      // scoop up and ignore a variable initializer

      while(token.type_ != ',' &&
            token.type_ != ';' &&
            !eof()
           )
      {
        if(token.type_ == '{')
          eat_function_body();
        else
        if(token.type_ == '(')
          eat_function_parms();
        
        next_token();
      }


    }
    else
      if(token.type_ == ';')
        return false; // not end of file (I think)
    else
      next_token();

    if(token.type_ == '~')
    {
      next_token();

      if(token.type_ == CPP_Token::aln)
      {
        static string tilde("~");
        
        token.text_ = tilde + token.text_;
      }

    }

    while( token.type_ == CPP_Token::der) // ::
    {
      next_token();

      if(token.type_ == '~')
      {
        next_token();

        if(token.type_ == CPP_Token::aln)
        {
          static string tilde("~");
        
          token.text_ = tilde + token.text_;

        }

      }

      if(token.type_ != CPP_Token::aln)
        break;
        
      tmp.text_ += "::";
      tmp.text_ += token.text_;
      tmp.line_  = token.line_;
      next_token();
    }

  }
  while(!eof()             &&
        token.type_ != '(' &&
        token.type_ != ';' &&
        token.type_ != ',' &&
        token.type_ != '['
       );

  bool is_function= (token.type_ == '(');

  string parmtext;
  string file = token.file_;
  int    line = token.line_;


  if(token.type_ == '(')
  {
    eat_function_parms(&parmtext);
    next_token();
  }
  else
  if(token.type_ == '[')
  {
     // eat until matching ']' -- and eat it too

     int depth = 0;

     while(!eof())
     {
       if(token.type_ == '[')
         ++depth;
       else
       if(token.type_ == ']')
       {
         --depth;
         if(depth == 0)
           break;
       }

       next_token();

     }

     next_token();

     // we should see either ';' or ',' as the token there

  }


  if(token.type_ == '(')
  {
    // we are dealing with something like a pointer to a function
    // where the following syntax was found:
    //
    //  int (*f)(parms) {}

    tmp.text_ = parmtext;
    tmp.file_ = file;
    tmp.line_ = line;

    eat_function_parms(&parmtext);
    next_token();

  }
  else
  if(token.type_ == ':')
  {
    // ignore the base class initalizers in a constructor's name

    while(!eof() && token.type_ != '{')
      next_token();

  }

  switch(token.type_)
  {
    case CPP_Token::eof:
         return true;

    case ';':

         {
           static string variable_type("variable");
           static string typename_type("typename");
           static string function_type("function");
        
           if(!is_function || log_forward_func_declarations)
            log_symbol( handling_typedefs 
                         ? typename_type 
                         : (is_function 
                            ? function_type
                            : variable_type
                           ),
                       tmp.text_,
                       tmp.file_,
                       tmp.line_
                      );
        
           next_token();
         }
         return false;
        
    case ',':

         {
           static string variable_type("variable");
           static string typename_type("typename");

           // do not log functions at this point -- the best you can have
           // in this code section is a forward declaration and we do not
           // log them -- only bodies.


           if(!is_function)
            log_symbol(handling_typedefs ? typename_type 
                                          : variable_type
                       , 
                       tmp.text_, 
                       tmp.file_, 
                       tmp.line_);
        
           next_token();
           return parse_funcvar(log_forward_func_declarations, handling_typedefs);
         }
    default:

         {
           static string function_type("function");
           static string variable_type("variable");
           static string typename_type("typename");

           if(is_function || (token.type_ != '=') )
               log_symbol(function_type, tmp.text_, tmp.file_, tmp.line_);
           else
               log_symbol(handling_typedefs ? typename_type : variable_type, 
                          tmp.text_, 
                          tmp.file_, 
                          tmp.line_);

         }
        
         // handle a goofy g++ variant syntax
        
           if(token.text_ == "return")
           {
             while(!eof() && token.type_ != '{')
               next_token();
           }
        
         // end goofy g++ variant syntax

         while(!eof() && 
               token.type_ != ';' && 
               token.type_ != '{' &&
               token.type_ != ','
              )
         {
           if(token.type_ == '(')
           {
             string trash;
             eat_function_parms(&trash);
             next_token();
           }
           else
             next_token();
         }

         if(token.type_ == '{')
         {
           eat_function_body();
           next_token();
        
           while(token.type_ == ':')
           {
             // presume we are passing a class definition with #if's in it
             // like this:
             //
             //   class X
             //   {
             //     X()
             //     #if 1
             //        : base1() {}
             //     #else
             //        : base2() {}   <-- the parser is now sitting at the ':'
             //     #endif
        
             while(token.type_ != '{')
               next_token();
        
             eat_function_body();
        
             next_token(); // eat the closing }
        
        
           }
        
         }
         else
             if(token.type_ == ',')
             {
                 next_token();
                 return parse_funcvar(log_forward_func_declarations, handling_typedefs);
             }
          else
               next_token();
        
         break;
        
  }


  return false;
}

static bool parse_extern()
//
//  parse extern variables and extern code definitions:
//
//    extern type varname;
//    extern type funcname;
//    extern "c" declaration;
//    extern "c" { declarations ... }
//
{
  next_token();  // eat 'extern'

  if(token.type_ == '"')
  {
     next_token();

     if(token.type_ != '{')
     {
       return parse_declaration();
     }

     next_token();

     while(token.type_ != '}')
     {
       if(parse_declaration())
         return true;
     }

     next_token();

  }
  else
  {
     return parse_declaration();
  }

  return false;
}