Commits

Anonymous committed 52135bf

Update to JDE 2.2.7.1

Comments (0)

Files changed (28)

 # Boston, MA 02111-1307, USA.
 
 VERSION = 1.06
-AUTHOR_VERSION = 1.3.3
+AUTHOR_VERSION = 1.4beta8
 MAINTAINER = Eric M. Ludlam <zappo@gnu.org>
 PACKAGE = semantic
 PKG_TYPE = regular
-REQUIRES = semantic xemacs-base speedbar
+REQUIRES = eieio xemacs-base xemacs-devel edit-utils speedbar
 CATEGORY = standard
 
 ELCS = document-vars.elc document.elc semantic-bnf.elc semantic-c.elc \
-	semantic-el.elc semantic-imenu.elc semantic-make.elc semantic-mode.elc \
+	semantic-ctxt.elc semantic-el.elc semantic-imenu.elc semantic-make.elc \
 	semantic-sb.elc semantic-util.elc semantic.elc sformat.elc working.elc \
-    senator.elc senator-isearch.elc
+    senator.elc semantic-java.elc semantic-scm.elc semanticdb.elc semantic-load.el
 EXTRA_SOURCES = semantic-util.el
 
 INFO_FILES = $(PACKAGE).info*
-# Simple BNF notation for top-level C elements.
+# C & C++ BNF language specification
 #
-# Copyright (C) 1999, 2000 Eric M. Ludlam
+# Copyright (C) 1999, 2000, 2001 Eric M. Ludlam
 #
 # Author: Eric M. Ludlam <zappo@gnu.org>
 # X-RCS: $Id$
 # Boston, MA 02111-1307, USA.
 #
 # $Log$
-# Revision 1.20  2000/10/30 01:34:23  zappo
-# Tidied up opt-array stuff.
+# Revision 1.37  2001/06/03 14:01:13  zappo
+# Multi-inheritance for classes.
+# Structs like classes for C++.
+# No ; on name spaces.
+# Throws for methods.
+# (Bugs & patches from "Jesper Nordenberg" <mayhem@home.se>
+# and Norbert Lindlbauer <Norbert_Lindlbauer@betaresearch.de>)
 #
-# Revision 1.19  2000/09/27 02:08:04  zappo
-# Moved `type' to be before `function' and `variable' in `declaration'.
+# Revision 1.36  2001/05/25 01:12:10  zappo
+# (expression): Added string.
 #
-# Revision 1.18  2000/09/21 03:35:08  zappo
-# Fixed up struct/union variable declaration section to use new lexical
-# tokens for struct/union.
+# Revision 1.35  2001/05/09 03:25:17  zappo
+# Added inline method support.
+# Store protection elements
+# Change enum parts to be integer variables.
 #
-# Revision 1.17  2000/09/19 04:22:31  zappo
-# Added %keywordtable
+# Revision 1.34  2001/05/01 16:52:40  zappo
+# Revamped lots of summary strings.
+# Added `type' as a possible expansion in classsubparts.
 #
-# Revision 1.16  2000/09/14 19:23:27  zappo
-# Updated to use %token to make new tokens for use in RULES.
+# Revision 1.33  2001/04/21 14:37:55  zappo
+# Spelling error for integer.
 #
-# Revision 1.15  2000/09/11 23:00:32  zappo
-# Use new BNF settings section.
+# Revision 1.32  2001/04/13 02:24:45  zappo
+# Added built in types (void, char, etc) and summaries.
 #
-# Revision 1.14  2000/07/05 14:41:31  zappo
-# Support multiple continuous decl modifiers.
-# Added modifiers into variables.
+# Revision 1.31  2001/04/07 02:43:48  zappo
+# Added name spaces, and references.
 #
-# Revision 1.13  2000/07/01 18:15:31  zappo
-# Updated for new elements in the tokens.
+# Revision 1.30  2001/03/10 16:18:15  zappo
+# Added lots of summaries to %tokens.
+# variables and functions now share declmod and typedecl start
+# match, and then get recombobulated later.  Prevents massive
+# reparsing.
 #
-# Revision 1.12  2000/06/13 14:37:48  zappo
-# The table has moved.
+# Revision 1.29  2001/02/24 15:24:24  zappo
+# Added a few optimizations for structure parts.
+# Used some %tokens in a few spots where I wasn't using them before.
 #
-# Revision 1.11  2000/05/06 01:32:57  zappo
-# Use EXPANDFULL for enums, and for argument lists to functions.
+# Revision 1.28  2001/02/20 20:36:19  zappo
+# Removed unused %put calls.
 #
-# Revision 1.10  2000/04/29 12:54:15  zappo
-# Added support for Linux __P items.
-# Added support for "..." fn prototypes.
+# Revision 1.27  2001/02/09 19:49:26  zappo
+# Added paren testing to argument lists.
 #
-# Revision 1.9  2000/04/28 18:57:49  zappo
-# Added system flag to include files.
+# Revision 1.26  2001/02/09 11:47:32  zappo
+# Added type separation characters.
+# Added all constituents of DECLMOD into their own tokens.
+# Created a rule to create a list of declmods.
 #
-# Revision 1.8  2000/04/25 14:46:42  zappo
-# structparts now uses EXPANDFULL to get positional information for it's
-# sub parts.  This simplified STRUCTSUBPARTS nonterminal
+# Revision 1.25  2001/02/02 04:14:53  zappo
+# Added c++ to the list of language modes.
+# Added lots of operator symbol tokens.
+# Added DECLMOD symbol w/ complex regexp.
+# Added CLASS, OPERATOR, PUBLIC, PRIVATE, and PROTECTED token keywords.
+# Support parsing of a class.
+# Add class bit parsing for functions.
+# Add destructor bit parsing for functions.
+# Allow operator symbols for methods.
+# Added parent and destructor fields to extra-spec for functions.
+# The function rule now returns a function or prototype.
 #
-# Revision 1.7  2000/04/23 15:34:28  zappo
-# Added copyright information.
+# Revision 1.24  2001/01/31 15:26:10  zappo
+# Added `codeblock' rule and `%scopestart'.
 #
-# Revision 1.6  2000/04/20 23:55:03  zappo
-# Macros split from 'variable' nt.
-# Added knr-argument parsing.
+# Revision 1.23  2001/01/24 21:08:41  zappo
+# Added support for new token formats that use ASSOC.
 #
-# Revision 1.5  2000/04/16 22:34:07  zappo
-# Added `filename' for include rules.
+# Revision 1.22  2001/01/06 14:35:40  zappo
+# Put `type' `t' onto some tokens.
+# Struct and enums now match the braces, and return nil.
+# Enum parts now have a token type of 'enum.
+# Added ... to fuction arg lists.
+# Match parens for arg lists.
 #
-# Revision 1.4  1999/12/17 20:53:16  zappo
-# Added a splice , for varnamelist.
-#
-# Revision 1.3  1999/06/06 14:20:24  zappo
-# Fixed up some language definitions.
-#
-# Revision 1.2  1999/05/18 14:08:56  zappo
-# Fixed up the returns of some nonterminals.
+# [...]
 #
 # Revision 1.1  1999/05/17 17:28:30  zappo
 # Initial revision
 #
 
 %start         declaration
+%scopestart    codeblock
 %outputfile    semantic-c.el
 %parsetable    semantic-toplevel-c-bovine-table
 %keywordtable  semantic-c-keyword-table
-%languagemode  c-mode
+%languagemode  (c-mode c++-mode)
 %setupfunction semantic-default-c-setup
 
 %(setq semantic-expand-nonterminal 'semantic-expand-c-nonterminal
        semantic-flex-extensions semantic-flex-c-extensions
        semantic-dependency-include-path semantic-default-c-path
        imenu-create-index-function 'semantic-create-imenu-index
+       semantic-type-relation-separator-character '("." "->")
+       semantic-command-separation-character ";"
        document-comment-start "/*"
        document-comment-line-prefix " *"
        document-comment-end " */"
        )%
 
-%token HASH punctuation "#"
 %token INCLUDE "include"
 %token DEFINE "define"
+%token HASH punctuation "#"
 %token PERIOD punctuation "."
 %token COLON punctuation ":"
 %token SEMICOLON punctuation ";"
 %token STAR punctuation "*"
+%token AMPERSAND punctuation "&"
+%token DIVIDE punctuation "/"
+%token PLUS punctuation "+"
+%token MINUS punctuation "-"
+%token BANG punctuation "!"
 %token EQUAL punctuation "="
+%token LESS punctuation "<"
+%token GREATER punctuation ">"
 %token COMA punctuation ","
+%token TILDE punctuation "~"
+
+%token EXTERN "extern"
+%put EXTERN summary "Declaration Modifier: extern <type> <name> ..."
+%token STATIC "static"
+%put STATIC summary "Declaration Modifier: static <type> <name> ..."
+%token CONST "const"
+%put CONST summary "Declaration Modifier: const <type> <name> ..."
+%token VOLATILE "volatile"
+%put VOLATILE summary "Declaration Modifier: volatile <type> <name> ..."
+%token SIGNED "signed"
+%put SIGNED summary "Numeric Type Modifier: signed <numeric type> <name> ..."
+%token UNSIGNED "unsigned"
+%put UNSIGNED summary "Numeric Type Modifier: unsigned <numeric type> <name> ..."
+
+%token INLINE "inline"
+%put INLINE "Function Modifier: inline <return  type> <name>(...) {...};"
+%token VIRTUAL "virtual"
+%put VIRTUAL summary "Method Modifier: virtual <type> <name>(...) ..."
+
 %token STRUCT "struct"
+%put STRUCT summary "Structure Type Declaration: struct [name] { ... };"
 %token UNION "union"
+%put UNION summary "Union Type Declaration: union [name] { ... };"
 %token ENUM "enum"
+%put ENUM summary "Enumeration Type Declaration: enum [name] { ... };"
 %token TYPEDEF "typedef"
+%put TYPEDEF summary "Arbitrary Type Declaration: typedef <typedeclaration> <name>;"
+%token CLASS "class"
+%put CLASS summary "Class Declaration: class <name>[:parents] { ... };"
+%token NAMESPACE "namespace"
+%put NAMESPACE summary "Namespace Declaration: namespace <name> { ... };"
 
+%token THROW "throw"
+%put THROW summary "<type> <methoddef> (<method args>) throw (<exception>) ..."
 
-declaration : include
-	    | macro
+# Leave these alone for now.
+%token OPERATOR "operator"
+%token PUBLIC "public"
+%token PRIVATE "private"
+%token PROTECTED "protected"
+
+
+# These aren't used for parsing, but is a useful place to describe the keywords.
+%token IF "if"
+%token ELSE "else"
+%put {IF ELSE} summary  "if (<condition>) { code } [ else { code } ]"
+
+%token DO "do"
+%token WHILE "while"
+%put DO summary " do { code } while (<condition>);"
+%put WHILE summary "do { code } while (<condition>); or while (<condition>) { code };"
+
+%token FOR "for"
+%put FOR summary "for(<init>; <condition>; <increment>) { code }"
+
+%token SWITCH "switch"
+%token CASE "case"
+%token DEFAULT "default"
+%put {SWITCH CASE DEFAULT} summary
+"switch (<variable>) { case <constvalue>: code; ... default: code; }"
+
+%token RETURN "return"
+%put RETURN summary "return <value>;"
+
+%token BREAK "break"
+%put BREAK summary "Non-local exit within a loop or switch (for, do/while, switch): break;"
+%token CONTINUE "continue"
+%put CONTINUE summary "Non-local continue within a lool (for, do/while): continue;"
+
+%token SIZEOF "sizeof"
+%put SIZEOF summary "Compile time macro: sizeof(<type or variable>) // size in bytes"
+
+# Types
+%token VOID "void"
+%put VOID summary "Built in typeless type: void"
+%token CHAR "char"
+%put CHAR summary "Integral Character Type: (0 to 256)"
+%token SHORT "short"
+%put SHORT summary "Integral Primitive Type: (-32768 to 32767)"
+%token INT "int"
+%put INT summary "Integral Primitive Type: (-2147483648 to 2147483647)"
+%token LONG "long"
+%put LONG summary "Integral primitive type (-9223372036854775808 to 9223372036854775807)"
+%token FLOAT "float"
+%put FLOAT summary "Primitive floating-point type (single-precision 32-bit IEEE 754)"
+%token DOUBLE "double"
+%put DOUBLE summary "Primitive floating-point type (double-precision 64-bit IEEE 754)"
+
+declaration : macro
 	    | type
-	    | function
-	    | variable
-	    | prototype
+	    | var-or-fun
 	    | define
 	    ;
-		
-include : HASH INCLUDE punctuation "<" filename punctuation ">"
-	  ( ,$4 include t nil )
-        | HASH INCLUDE string
-	  ( (read $3) include nil nil )
-        ;
+
+codeblock : define
+	  | var-or-fun
+	  | type # type is less likely to be used here.
+	  ;
+
+macro : HASH macro-or-include
+	( ,$2 )
+      ;
+
+macro-or-include : DEFINE symbol opt-expression
+		   ( $2 variable nil $3
+			(ASSOC const t)
+			nil
+			)
+		 | INCLUDE LESS filename GREATER
+		   ( ,$3 include t nil )
+		 | INCLUDE string
+		   ( (read $2) include nil nil )
+		 ;
+
+# This is used in struct parts.
+define : HASH DEFINE symbol opt-expression
+	 ( $2 variable nil $3
+	      (ASSOC const t)
+	      nil
+	      )
+       ;
 
 filename : symbol PERIOD symbol
 	   ( (concat $1 $2 $3) )
-         | symbol punctuation "/" filename
+         | symbol DIVIDE filename
 	   ( (concat $1 $2 (car $3)) )
 	 ;
-	  
-structparts : semantic-list
-	      (EXPANDFULL $1 structsubparts)
+
+# In C++, structures can have the same things as classes.
+# So delete this somday in the figure.
+#
+#structparts : semantic-list
+#	      (EXPANDFULL $1 structsubparts)
+#	    ;
+#
+#structsubparts : open-paren "{"
+#		 ( nil )
+#	       | close-paren "}"
+#		 ( nil )
+#	       | var-or-fun
+#	       | define
+#	       # sometimes there are defines in structs.
+#	       ;
+
+classparts : semantic-list
+	     (EXPANDFULL $1 classsubparts)
 	    ;
 
-structsubparts : variable
-	       | define
-	       # sometimes there are defines in structs.
+classsubparts : open-paren "{"
+		( nil )
+	      | close-paren "}"
+		( nil )
+	      | opt-class-protection COLON
+		( ,$1 label )
+	      | var-or-fun
+	      | type
+	      | define	      
+		( ,$1 protection )
+	      # In C++, this label in a classsubpart represents
+	      # PUBLIC or PRIVATE bits.  Ignore them for now.
+	      | EMPTY
+	      ;
+
+opt-class-parents : COLON class-parents
+		    ( $2 )
+		  | EMPTY
+		    ( )
+		  ;
+
+class-parents : opt-class-protection symbol COMA class-parents
+		( ,(cons $2 $4 ) )
+	      | opt-class-protection symbol
+		( $2 )
+	      ;
+
+opt-class-protection : PUBLIC
+		     | PRIVATE
+		     | PROTECTED
+		     ;
+
+namespaceparts : semantic-list
+		 (EXPANDFULL $1 namespacesubparts)
 	       ;
 
+namespacesubparts : open-paren "{"
+		    ( nil )
+		  | close-paren "}"
+		    ( nil )
+		  | type
+                  | var-or-fun
+		  | define
+		  | opt-class-protection COLON
+		    ( $1 protection )
+		  # In C++, this label in a classsubpart represents
+		  # PUBLIC or PRIVATE bits.  Ignore them for now.
+		  | EMPTY
+		  ;
 
 enumparts : semantic-list
 	    (EXPANDFULL $1 enumsubparts)
 	  ;
 
 enumsubparts : symbol opt-assign
-	       ( $1 )
+	       ( $1 variable "int" ,$2 (ASSOC const t) nil)
+	     | open-paren "{"
+	       ( nil )
+	     | close-paren "}"
+	       ( nil )
 	     ;
 
 opt-name : symbol
 	   ( nil )
          ;
 
-typesimple : STRUCT opt-name structparts
-	     ( ,$2 type $1 $3 nil nil nil )
+typesimple : struct-or-class opt-name opt-class-parents classparts
+	     ( ,$2 type ,$1 $4 ,$3 nil nil )
 	   | UNION opt-name structparts
 	     ( ,$2 type $1 $3 nil nil nil )
 	   | ENUM opt-name enumparts
 	     ( $3 type $1 nil $2 nil nil )
            ;
 
+struct-or-class: STRUCT
+	       | CLASS
+	       ;
+
 type : typesimple SEMICOLON
        ( ,$1 )
+     | NAMESPACE symbol namespaceparts
+       ( $2 type $1 $3 nil nil nil )
      ;
 
 opt-stars : STAR opt-stars
 	    ( 0 )
           ;
 
-declmods : symbol "\\(_+\\)?\\(extern\\|static\\|const\\|volitile\\|signed\\|unsigned\\)" declmods
-	   ( ,(cons $1 $2 ) )
-	 | symbol "\\(_+\\)?\\(extern\\|static\\|const\\|volitile\\|signed\\|unsigned\\)"
-	   ( $1 )
+declmods : DECLMOD declmods
+	   ( ,(cons ,(car ,$1) $2 ) )
+	 | DECLMOD
+	   ( ,$1 )
 	 | EMPTY
 	   ()
 	 ;
 
-# dont deal with the stars just yet.
-typeform : typeformbase opt-stars
+DECLMOD : EXTERN
+	| STATIC
+	| CONST
+	| VOLATILE
+	| SIGNED
+	| UNSIGNED
+	| VIRTUAL
+	| INLINE
+	;
+
+# Don't deal with the stars or reference just yet.
+typeform : typeformbase opt-stars opt-ref
 	   ( ,$1 )
 	 ;
 
+# C++: A type can be modified into a reference by "&"
+opt-ref : AMPERSAND
+	| EMPTY
+	;
+
 typeformbase : typesimple
 	       ( ,$1 )
 	     | STRUCT symbol
 	       ( $2 type $1 )
 	     | ENUM symbol
 	       ( $2 type $1 )
+	     | builtintype
+	       ( ,$1 )
 	     | symbol
 	       ( $1 )
 	     ;
-  
+
+builtintype : VOID
+	    | CHAR
+	    | SHORT
+	    | INT
+	    | LONG
+	    | FLOAT
+	    | DOUBLE
+	    ;
+
+var-or-fun : declmods typeform var-or-func-decl
+	     ( ,(semantic-c-reconstitute-token ,$3 $1 $2 ) )
+	   # it is possible for a function to not have a type, and
+	   # it is then assumed to be an int.  How annoying.
+	   | declmods var-or-func-decl
+	     ( ,(semantic-c-reconstitute-token ,$2 $1 nil ) )
+	   ;
+
+var-or-func-decl : opt-class opt-destructor functionname arg-list 
+		   opt-throw
+		   fun-or-proto-end
+		   ( ,$3 'function 
+			 ;; Extra stuff goes in here.
+			 ;; Continue with the stuff we found in
+			 ;; this definition
+			$1 $2 $4 $5)
+		 | varnamelist  SEMICOLON
+		   ( $1 'variable )
+		 ;
+
+opt-throw : THROW semantic-list
+	     ( EXPAND $2 throw-exception-list )
+	   | EMPTY
+	   ;
+
+# Is this true?  I don't actually know.
+throw-exception-list : symbol COMA throw-exception-list
+			( ,(cons $1 $3) )
+		      | symbol close-paren ")"
+			( $1 )
+		     | open-paren "(" throw-exception-list
+			( ,$2 )
+		      ;
+
 opt-bits : COLON symbol
 	   ( $2 )
 	 | EMPTY
 	     ( nil )
 	   ;
 
-macro : HASH DEFINE symbol opt-expression
-	( $3 variable nil t $4 nil nil )
-      ;
-
-variable : variabledef SEMICOLON
-	   ( ,$1 )
-	 ;
-
-variabledef : declmods typeform varnamelist
-	      ( $3 variable $2 (if $1 (string-match "const" (car $1))) nil 
-	      (if (and $1 (string-match "const" (car $1))) (cdr $1) $1) nil )
-	    ;
-
 opt-restrict : symbol "\\<\\(__\\)?restrict\\>"
 	     | EMPTY
 	     ;
 
 # I should store more in this def, but leave it simple for now.
 variablearg : declmods typeform varname
-	      ( (car $3) variable $2 (if $1 (string-match "const" (car $1)))
-	      nil (if (and $1 (string-match "const" (car $1))) (cdr $1) $1)
-	      nil )
+	      ( (car $3) variable $2 nil
+		 (ASSOC const (if (member "const" $1) t nil)
+			typemodifiers (delete "const" $1))
+		 nil
+		 )
 	    ;
 
 varnamelist : varname COMA varnamelist
 	      ( $1 )
 	    ;
 
+opt-class : symbol COLON COLON
+	    ( $1 )
+	  | EMPTY
+	    ( nil )
+	  ;
+
+opt-destructor : TILDE
+		 ( t )
+	       | EMPTY
+		 ( nil )
+	       ;
+
 arg-list : symbol "\\<__?P\\>" semantic-list
 	   (EXPAND $2 arg-list-p)
-	 | semantic-list knr-arguments
+	 | semantic-list "^(" knr-arguments
 	   ( ,$2 )
-	 | semantic-list
+	 | semantic-list "^("
 	   (EXPANDFULL $1 arg-sub-list)
 	 ;
 
 
 arg-sub-list : variablearg
 	       ( ,$1 )
-	     | PERIOD PERIOD PERIOD
-	       close-paren ")"
+	     | PERIOD PERIOD PERIOD close-paren ")"
 	       ( "..." )
+	     | open-paren "("
+	       ( nil )
+	     | close-paren ")"
+	       ( nil )
 	     ;
 
-functiondef : declmods typeform symbol arg-list
-	      ( $3 function $2 $4 $1 nil )
-            ;
+operatorsym : LESS LESS
+	      ( "<<" )
+	    | GREATER GREATER
+	      ( ">>" )
+	    | EQUAL EQUAL
+	      ( "==" )
+	    | LESS EQUAL
+	      ( "<=" )
+	    | GREATER EQUAL
+	      ( ">=" )
+	    | BANG EQUAL
+	      ( "!=" )
+	    | LESS
+	    | GREATER
+	    | STAR
+	    | PLUS
+	    | MINUS
+	    | DIVIDE
+	    | EQUAL
+	    ;
 
-prototype : functiondef SEMICOLON
-	    ( ,$1 )
-	  ;
+functionname : OPERATOR operatorsym
+	       ( ,$2 )
+	     | symbol
+	       ( $1 )
+	     ;
 
-function : functiondef semantic-list
-	   ( ,$1 )
-         ;
+fun-or-proto-end: SEMICOLON
+		  ( t )
+		| semantic-list
+		  ( nil )
+		;
 
 opt-expression : expression
 	       | EMPTY ( nil )
 	       ;
 
-# Use expressiosn for parsing only.  Don't actually return anything
-# for now.  Hopefully we can't fix this later.
+# Use expression for parsing only.  Don't actually return anything
+# for now.  Hopefully we can fix this later.
 expression : symbol
-	     ( nil )
+	     ( )
            | punctuation "[!*&~]" symbol
-	     ( nil )
+	     ( )
+	   | string
+	     ( )
            | semantic-list
-	     ( nil )
+	     ( )
 	   # | expression "+-*/%^|&" expression
 	   # ( nil )
 	   ;
 ;;; document-vars.el --- Default settings for the document package.
 
-;;; Copyright (C) 2000 Eric M. Ludlam
+;;; Copyright (C) 2000, 2001 Eric M. Ludlam
 
 ;; Author: Eric M. Ludlam <zappo@gnu.org>
 ;; Keywords: doc
 (defcustom document-copyright-notice-file nil
   "*A file name containing a copyright notice.
 It will be reformatted in the header to have the correct prefix character.
-See the %N token in document-file-comment"
+See the %N token in `document-file-comment'"
   :group 'document
   :type 'file)
 
 ;;; document.el --- Use the bovinator to aid in generating documentation.
 
-;;; Copyright (C) 2000 Eric M. Ludlam
+;;; Copyright (C) 2000, 2001 Eric M. Ludlam
 
 ;; Author: Eric M. Ludlam <zappo@gnu.org>
 ;; Keywords: doc
 ;; X-RCS: $Id$
 
-;; This file is not part of GNU Emacs.
-
 ;; Semantic is free software; you can redistribute it and/or modify
 ;; it under the terms of the GNU General Public License as published by
 ;; the Free Software Foundation; either version 2, or (at your option)
 ;;   Document then provides some rules for creating English Text based
 ;; on the name of a given function, it's return type, or variable
 ;; type.  It also has output rules for texinfo, or comments.
+;;
+;; NOTE: Some of the user level commands in document.el dealing with
+;; texinfo files have been obsoleted commands in semantic-texi, which
+;; can not insert foriegn tokens.
 
 (require 'sformat)
 (require 'document-vars)
 	(error "No file found for your documentation"))
     (set-buffer (marker-buffer document-current-output-file))
     (goto-char document-current-output-file)
+    (insert "\n")
     (document-insert-texinfo cdi cdib)
+    (insert "\n")
     (setq document-current-output-file (point-marker))
     ))
 
 ;;
 (defun document-insert-texinfo (nonterm buffer)
   "Insert texinfo documentation about NONTERM from BUFFER."
-  (insert "\n")
   (let ((tt (semantic-token-token nonterm)))
     (insert "@"
 	    (cond ((eq tt 'variable)
-		   "defvar")
+		   (if (semantic-token-extra-spec nonterm 'user-visible)
+		       "deffn Option"
+		     "defvar"))
 		  ((eq tt 'function)
-		   "defun")
+		   (if (semantic-token-extra-spec nonterm 'user-visible)
+		       "deffn Command"
+		     "defun"))
 		  ((eq tt 'type)
 		   "deftype")
 		  (t (error "Don't know how to document that")))
 	     (document-generate-documentation nonterm buffer)))
     (insert "\n@end "
 	    (cond ((eq tt 'variable)
-		   "defvar")
+		   (if (semantic-token-extra-spec nonterm 'user-visible)
+		       "deffn"
+		     "defvar"))
 		  ((eq tt 'function)
-		   "defun")
+		   (if (semantic-token-extra-spec nonterm 'user-visible)
+		       "deffn"
+		     "defun"))
 		  ((eq tt 'type)
 		   "deftype"))
-	    "\n")))
+	    )))
 
 (defun document-insert-defun-comment (nonterm buffer)
   "Insert mode-comment documentation about NONTERM from BUFFER."
 ;;
 (defun document-generate-documentation (nonterm buffer)
   "Return a plain string documenting NONTERM from BUFFER."
-  (let ((doc ;; Second, does this thing have docs in the source buffer which
-	 ;; an override method might be able to find?
-	 (semantic-find-documentation nonterm)
-	 ))
-    (if (not doc)
-	(document-generate-new-documentation nonterm buffer)
-      ;; Ok, now lets see what sort of formatting there might be,
-      ;; and see about removing some of it.. (Tables of arguments,
-      ;; and that sort of thing.)
-      nil
-      ;; Return the string.
-      doc)))
+  (save-excursion
+    (set-buffer buffer)
+    (let ((doc ;; Second, does this thing have docs in the source buffer which
+	   ;; an override method might be able to find?
+	   (semantic-find-documentation nonterm)
+	   ))
+      (if (not doc)
+	  (document-generate-new-documentation nonterm buffer)
+	;; Ok, now lets see what sort of formatting there might be,
+	;; and see about removing some of it.. (Tables of arguments,
+	;; and that sort of thing.)
+	nil
+	;; Return the string.
+	doc))))
 
 (defun document-generate-new-documentation (nonterm buffer)
   "Look at elements of NONTERM in BUFFER to make documentation.
   ;; run the elisp version through also.
   (let ((case-fold-search nil))
     (while (string-match
-	    "\\(^\\|[^{]\\)\\<\\([A-Z0-9_]+\\)\\>\\($\\|[^}]\\)"
+	    "\\(^\\|[^{]\\)\\<\\([A-Z0-9_-]+\\)\\>\\($\\|[^}]\\)"
 	    string)
       (setq string (concat (substring string 0 (match-beginning 2))
 			   "@var{"
  `variable' => @code{variable}
  `class'    => @code{class} @xref{class}
  `unknown'  => @code{unknonwn}
+ \"text\"     => ``text''
  'quoteme   => @code{quoteme}
  non-nil    => non-@code{nil}
  t          => @code{t}
     (setq string (replace-match "@code{\\2}" t nil string 2)))
   (while (string-match "\\( \\|^\\)\\(\\(\\(C-\\|M-\\|S-\\)+\\([^ \t\n]\\|RET\\|SPC\\|TAB\\)\\)\\|\\(RET\\|SPC\\|TAB\\)\\)\\( \\|$\\)" string)
     (setq string (replace-match "@kbd{\\2}" t nil string 2)))
+  (while (string-match "\"\\(.+\\)\"" string)
+    (setq string (replace-match "``\\1''" t nil string 0)))
   string)
 
 ;;; Buffer finding and managing
+# BNF grammar for Java
+#
+# Copyright (C) 2000, 2001 Paul F. Kinnucan, Jr.
+# Copyright (C) 2001 David Ponce
+#
+# Author: Paul F. Kinnucan, Jr. <paulk@mathworks.com>
+#         David Ponce <david@dponce.com>
+#
+# $Id$
+#
+# java.bnf is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNU Emacs; see the file COPYING.  If not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+# --------
+# Settings
+# --------
+%outputfile    semantic-java.el
+%parsetable    semantic-toplevel-java-bovine-table
+%keywordtable  semantic-java-keyword-table
+%languagemode  java-mode
+%setupfunction semantic-default-java-setup
+
+%(progn
+   (setq
+    ;; Java is case sensitive
+    semantic-case-fold nil
+    ;; special handling of multiple variable declarations/statement
+    semantic-expand-nonterminal 'semantic-expand-java-nonterminal
+    ;; function to use when creating items in imenu
+    semantic-imenu-summary-function 'semantic-prototype-nonterminal
+    ;; function to use for creating the imenu
+    imenu-create-index-function 'semantic-create-imenu-index
+    ;; Character used to separation a parent/child relationship
+    semantic-type-relation-separator-character '(".")
+    semantic-command-separation-character ";"
+    document-comment-start "/**"
+    document-comment-line-prefix " *"
+    document-comment-end " */"
+    ;; speedbar and imenu buckets name
+    semantic-symbol->name-assoc-list '((type     . "Classes")
+                                       (variable . "Variables")
+                                       (function . "Methods")
+                                       (include  . "Imports")
+                                       (package  . "Package"))
+    ;; Semantic navigation inside 'type children
+    senator-step-at-token-ids '(function variable)
+    )
+   ;; Needed by `semantic-find-doc-snarf-comment'.
+   (set (make-local-variable 'block-comment-end) "\\s-*\\*/")
+   )%
+
+%token ABSTRACT     "abstract"
+%put ABSTRACT summary
+"Class|Method declaration modifier: abstract {class|<type>} <name> ..."
+
+%token BOOLEAN      "boolean"
+%put BOOLEAN summary
+"Primitive logical quantity type (true or false)"
+
+%token BREAK        "break"
+%put BREAK summary
+"break [<label>] ;"
+
+%token BYTE         "byte"
+%put BYTE summary
+"Integral primitive type (-128 to 127)"
+
+%token CASE         "case"
+%put CASE summary
+"switch(<expr>) {case <const-expr>: <stmts> ... }"
+
+%token CATCH        "catch"
+%put CATCH summary
+"try {<stmts>} catch(<parm>) {<stmts>} ... "
+
+%token CHAR         "char"
+%put CHAR summary
+"Integral primitive type ('\u0000' to '\uffff') (0 to 65535)"
+
+%token CLASS        "class"
+%put CLASS summary
+"Class declaration: class <name>"
+
+%token CONST        "const"
+%put CONST summary
+"Unused reserved word"
+
+%token CONTINUE     "continue"
+%put CONTINUE summary
+"continue [<label>] ;"
+
+%token DEFAULT      "default"
+%put DEFAULT summary
+"switch(<expr>) { ... default: <stmts>}"
+
+%token DO           "do"
+%put DO summary
+"do <stmt> while (<expr>);"
+
+%token DOUBLE       "double"
+%put DOUBLE summary
+"Primitive floating-point type (double-precision 64-bit IEEE 754)"
+
+%token ELSE         "else"
+%put ELSE summary
+"if (<expr>) <stmt> else <stmt>"
+
+%token EXTENDS      "extends"
+%put EXTENDS summary
+"SuperClass|SuperInterfaces declaration: extends <name> [, ...]"
+
+%token FINAL        "final"
+%put FINAL summary
+"Class|Member declaration modifier: final {class|<type>} <name> ..."
+
+%token FINALLY      "finally"
+%put FINALLY summary
+"try {<stmts>} ... finally {<stmts>}"
+
+%token FLOAT        "float"
+%put FLOAT summary
+"Primitive floating-point type (single-precision 32-bit IEEE 754)"
+
+%token FOR          "for"
+%put FOR summary
+"for ([<init-expr>]; [<expr>]; [<update-expr>]) <stmt>"
+
+%token GOTO         "goto"
+%put GOTO summary
+"Unused reserved word"
+
+%token IF           "if"
+%put IF summary
+"if (<expr>) <stmt> [else <stmt>]"
+
+%token IMPLEMENTS   "implements"
+%put IMPLEMENTS summary
+"Class SuperInterfaces declaration: implements <name> [, ...]"
+
+%token IMPORT       "import"
+%put IMPORT summary
+"Import package declarations: import <package>"
+
+%token INSTANCEOF   "instanceof"
+
+%token INT          "int"
+%put INT summary
+"Integral primitive type (-2147483648 to 2147483647)"
+
+%token INTERFACE    "interface"
+%put INTERFACE summary
+"Interface declaration: interface <name>"
+
+%token LONG         "long"
+%put LONG summary
+"Integral primitive type (-9223372036854775808 to 9223372036854775807)"
+
+%token NATIVE       "native"
+%put NATIVE summary
+"Method declaration modifier: native <type> <name> ..."
+
+%token NEW          "new"
+
+%token PACKAGE      "package"
+%put PACKAGE summary
+"Package declaration: package <name>"
+
+%token PRIVATE      "private"
+%put PRIVATE summary
+"Access level modifier: private {class|interface|<type>} <name> ..."
+
+%token PROTECTED    "protected"
+%put PROTECTED summary
+"Access level modifier: protected {class|interface|<type>} <name> ..."
+
+%token PUBLIC       "public"
+%put PUBLIC summary
+"Access level modifier: public {class|interface|<type>} <name> ..."
+
+%token RETURN       "return"
+%put RETURN summary
+"return [<expr>] ;"
+
+%token SHORT        "short"
+%put SHORT summary
+"Integral primitive type (-32768 to 32767)"
+
+%token STATIC       "static"
+%put STATIC summary
+"Declaration modifier: static {class|interface|<type>} <name> ..."
+
+%token STRICTFP     "strictfp"
+%put STRICTFP summary
+"Declaration modifier: strictfp {class|interface|<type>} <name> ..."
+
+%token SUPER        "super"
+
+%token SWITCH       "switch"
+%put SWITCH summary
+"switch(<expr>) {[case <const-expr>: <stmts> ...] [default: <stmts>]}"
+
+
+%token SYNCHRONIZED "synchronized"
+%put SYNCHRONIZED summary
+"synchronized (<expr>) ... | Method decl. modifier: synchronized <type> <name> ..."
+
+%token THIS         "this"
+
+%token THROW        "throw"
+%put THROW summary
+"throw <expr> ;"
+
+%token THROWS       "throws"
+%put THROWS summary
+"Method|Constructor declaration: throws <classType>, ..."
+
+%token TRANSIENT    "transient"
+%put TRANSIENT summary
+"Field declaration modifier: transient <type> <name> ..."
+
+%token TRY          "try"
+%put TRY summary
+"try {<stmts>} [catch(<parm>) {<stmts>} ...] [finally {<stmts>}]"
+
+%token VOID         "void"
+%put VOID summary
+"Method return type: void <name> ..."
+
+%token VOLATILE     "volatile"
+%put VOLATILE summary
+"Field declaration modifier: volatile <type> <name> ..."
+
+%token WHILE        "while"
+%put WHILE summary
+"while (<expr>) <stmt> | do <stmt> while (<expr>);"
+  
+# --------------------------
+# Official javadoc line tags
+# --------------------------
+
+# Javadoc tags are identified by a 'javadoc' keyword property.  The
+# value of this property must be itself a property list where the
+# following properties are recognized:
+#
+# - `seq' (mandatory) is the tag sequence number used to check if tags
+#   are correctly ordered in a javadoc comment block.
+#
+# - `usage' (mandatory) is the list of token categories for which this
+#   documentation tag is allowed.
+#
+# - `opt' (optional) if non-nil indicates this is an optional tag.
+#   By default tags are mandatory.
+#
+# - `with-name' (optional) if non-nil indicates that this tag is
+#   followed by an identifier like in "@param <var-name> description"
+#   or "@exception <class-name> description".
+#
+# - `with-ref' (optional) if non-nil indicates that the tag is
+#   followed by a reference like in "@see <reference>".
+
+%token AUTHOR      "author"
+%put   AUTHOR      javadoc (seq 1 usage (type))
+%token VERSION     "version"
+%put   VERSION     javadoc (seq 2 usage (type)) 
+%token PARAM       "param"
+%put   PARAM       javadoc (seq 3 usage (function) with-name t) 
+#%token RETURN      "return"
+%put   RETURN      javadoc (seq 4 usage (function)) 
+%token EXCEPTION   "exception"
+%put   EXCEPTION   javadoc (seq 5 usage (function) with-name t) 
+#%token THROWS      "throws"
+%put   THROWS      javadoc (seq 6 usage (function) with-name t) 
+%token SEE         "see"
+%put   SEE         javadoc (seq 7 usage (type function variable) opt t with-ref t) 
+%token SINCE       "since"
+%put   SINCE       javadoc (seq 8 usage (type function variable) opt t) 
+%token SERIAL      "serial"
+%put   SERIAL      javadoc (seq 9 usage (variable) opt t) 
+%token SERIALDATA  "serialData"
+%put   SERIALDATA  javadoc (seq 10 usage (function) opt t) 
+%token SERIALFIELD "serialField"
+%put   SERIALFIELD javadoc (seq 11 usage (variable) opt t) 
+%token DEPRECATED  "deprecated"
+%put   DEPRECATED  javadoc (seq 12 usage (type function variable) opt t) 
+
+# --------
+# Grammar
+# --------
+bovine-toplevel : package_declaration
+                | import_declaration
+                | type_declaration
+                ;
+  
+number : symbol "[0-9]" punctuation "\\." symbol "[0-9Ee]"
+         punctuation "[-+]" symbol "[0-9fFdD]"
+       | symbol "[0-9]" punctuation "\\." symbol "[0-9EefFdD]"
+       | symbol "[0-9fFdD]"
+       ;
+  
+literal : number
+        | qualified_name
+        | string
+        ;
+
+type : reference_type
+       (,$1)
+     | primitive_type
+       (,$1)
+     ;
+  
+primitive_type : BOOLEAN | BYTE | SHORT | INT | LONG | CHAR | FLOAT | DOUBLE
+               ;
+
+reference_type : array_type
+                 (,$1)
+               | qualified_name
+                 (,$1)
+               ;
+  
+array_type : primitive_type dims
+             ((concat (car $1) (car $2)))
+           | qualified_name dims
+             ((concat (car $1) (car $2)))
+           ;
+
+qualified_name : symbol punctuation "\\." qualified_name
+                 ((concat $1 $2 (car $3)))
+               | symbol
+                 ($1)
+               ;
+
+## TOP-LEVEL ENTRY: package definition.
+##  ("NAME" package DETAIL "DOCSTRING")
+package_declaration : PACKAGE qualified_name punctuation ";"
+                      (,$2 package nil nil)
+                    ;
+  
+
+## TOP-LEVEL ENTRY: import definition.
+##  ("FILE" include SYSTEM "DOCSTRING")
+import_declaration : IMPORT qualified_name punctuation ";"
+                     (,$2 include nil nil)
+                   | IMPORT qualified_name punctuation "\\." punctuation "*" punctuation ";"
+                     ((concat (car $2) $3 $4) include nil nil)
+                   ;
+  
+type_declaration : punctuation ";"
+                 | class_declaration
+                 | interface_declaration
+                 ;
+
+modifiers_opt : modifiers
+                (,$1)
+              | EMPTY
+              ;
+  
+modifiers : modifier modifiers
+            (,(cons (car $1) ,$2))
+          | modifier
+            (,$1)
+          ;
+  
+modifier : PUBLIC | PROTECTED | PRIVATE | STATIC | ABSTRACT
+         | FINAL | NATIVE | SYNCHRONIZED | TRANSIENT | VOLATILE | STRICTFP
+         ;
+
+## TOP-LEVEL ENTRY: class definition.
+##  ("NAME" type "TYPE" PART-LIST PARENTS EXTRA-SPECS "DOCSTRING")
+class_declaration : modifiers_opt CLASS qualified_name class_parents class_body
+                    ( ,$3 type "class" $5 $4
+                         (ASSOC typemodifiers $1)
+                          nil
+                          )
+                  ;
+
+# class_parents := nil | (["extends_this" | nil] ["implements_this1" ... "implements_thisN"])
+class_parents: super interfaces
+               (,(append $1 $2))
+             | interfaces super
+               (,(append $2 $1))
+             | super
+               (,$1)
+             | interfaces
+               (,(cons nil $1))
+             | EMPTY
+             ;
+  
+super : EXTENDS qualified_name
+        (,$2)
+      ;
+  
+interfaces : IMPLEMENTS qualified_name_list
+             (,$2)
+           ;
+  
+qualified_name_list : qualified_name punctuation "," qualified_name_list
+                      (,(cons (car $1) ,$3))
+                    | qualified_name
+                      (,$1)
+                    ;
+  
+class_body : semantic-list # ::= {class_body_declarations}
+             (EXPANDFULL $1 class_body_declarations)
+           ;
+  
+class_body_declarations : class_declaration
+                          (,$1)
+                        | interface_declaration
+                          (,$1)
+                        | field_declaration
+                          (,$1)
+                        | method_declaration
+                          (,$1)
+                        #| static_initializer
+                        | constructor_declaration
+                          (,$1)
+                        #| block
+                        ;
+  
+## TOP-LEVEL ENTRY: variable definition.
+##  ("NAME" variable "TYPE" DEFAULT-VALUE EXTRA-SPECS "DOCSTRING")
+field_declaration : modifiers_opt type variable_declarators punctuation ";"
+                    (,$3 variable ,$2 nil
+                         (ASSOC typemodifiers $1)
+                          nil)
+                  ;
+
+## The following rule is used by `semantic-expand-java-nonterminal' to
+## reparse declaration of multiple variables in the same statement.
+## Reparsing is done with `semantic-bovinate-from-nonterminal-full' to
+## get correct START END information for each variable token.
+field_declaration_multi : modifiers_opt type variable_declarator punctuation ","
+                          (,$3)
+                        | modifiers_opt type variable_declarator punctuation ";"
+                          (,$3)
+                        | variable_declarator punctuation ","
+                          (,$1)
+                        | variable_declarator punctuation ";"
+                          (,$1)
+                        ;
+                         
+variable_declarators : variable_declarator variable_declarators_opt
+                       ((cons (car $1) (car $2)))
+                     ;
+
+variable_declarators_opt: punctuation "," variable_declarators
+                          (,$2)
+                        | EMPTY
+                        ;
+
+variable_declarator : variable_declarator_id variable_assign_opt
+                      (,$1)
+                    ;
+
+variable_assign_opt: punctuation "=" variable_initializer
+                   | EMPTY
+                   ;
+
+variable_declarator_id : symbol dims
+                         ((concat $1 (car $2)))
+                       | symbol
+                         ($1)
+                       ;
+
+variable_initializer : array_initializer
+                     | expression
+                     ;
+
+method_declaration : method_header method_body
+                     (,$1)
+                   ;
+  
+## TOP-LEVEL ENTRY: method definition.
+##  ("NAME" function "TYPE" ARG-LIST EXTRA-SPECS "DOCSTRING")
+method_header : modifiers_opt method_type symbol formal_parameter_list_opt throws_opt
+                ($3 function ,$2 $4
+                    (ASSOC typemodifiers $1 throws $5)
+                     nil)
+              ;
+
+method_type: VOID
+             ($1)
+           | type
+             (,$1)
+           ;
+  
+formal_parameter_list_opt : semantic-list # ::= (formal_parameter_list)
+                            (EXPANDFULL $1 formal_parameter_list)
+                          | EMPTY
+                          ;
+  
+formal_parameter_list : formal_parameter punctuation ","
+                        (,$1)
+                      | formal_parameter
+                        (,$1)
+                      ;
+
+formal_parameter-modifier : FINAL
+                          | EMPTY
+                          ;
+
+## TOP-LEVEL ENTRY: variable definition.
+##  ("NAME" variable "TYPE" DEFAULT-VALUE EXTRA-SPECS "DOCSTRING")
+formal_parameter : formal_parameter-modifier type variable_declarator_id
+                   (,$3 variable ,$2 nil
+                    (ASSOC typemodifiers $1)
+                     nil)
+                 ;
+  
+throws_opt : throws
+             (,$1)
+           | EMPTY
+           ;
+  
+throws : THROWS qualified_name_list
+         (,$2)
+       ;
+  
+method_body : punctuation ";"
+            | block
+            ;
+
+#static_initializer : STATIC block
+#                   ;
+
+## TOP-LEVEL ENTRY: constructor definition.
+##  ("NAME" function "TYPE" ARG-LIST EXTRA-SPECS "DOCSTRING")
+constructor_declaration : modifiers_opt symbol formal_parameter_list_opt throws_opt
+                          constructor_body
+                          ($2 function nil $3
+                              (ASSOC typemodifiers $1 throws $4)
+                               nil)
+                        ;
+  
+constructor_body : block
+                 ;
+
+## TOP-LEVEL ENTRY: interface definition.
+##  ("NAME" type "TYPE" PART-LIST PARENTS EXTRA-SPECS "DOCSTRING")
+interface_declaration : modifiers_opt INTERFACE symbol interface_parents interface_body
+                        ($3 type "interface" $5 $4
+                            (ASSOC typemodifiers $1)
+                             nil)
+                      ;
+
+# interface_parents := nil | ("extends_this1" ... "extends_thisN")
+interface_parents : EXTENDS qualified_name_list
+                    (,$2)
+                  | EMPTY
+                  ;
+  
+interface_body : semantic-list # ::= { interface_body_declarations }
+                 (EXPANDFULL $1 interface_body_declarations)
+               ;
+  
+interface_body_declarations : class_declaration
+                              (,$1)
+                            | interface_declaration
+                              (,$1)
+                            | method_header punctuation ";"
+                              (,$1)
+                            | field_declaration
+                              (,$1)
+                            ;
+  
+array_initializer : semantic-list "\\`{" # ::= {expression, expression, ...}
+                  ;
+  
+block : semantic-list "\\`{" # ::= {statements}
+      ;
+  
+primary : array_creation_expression
+        | primary_no_new_array primary_dim_opt
+        ;
+
+primary_dim_opt : semantic-list "\\`\\["
+                | EMPTY
+                ;
+  
+primary_no_new_array : qualified_name semantic-list "\\`(" # method_invocation
+                     | class_instance_creation_expression
+                     | semantic-list "\\`(" # (expression)
+                     | array_type punctuation "\\." CLASS
+                     | literal
+                     ;
+
+class_instance_creation_expression : NEW qualified_name semantic-list "\\`("
+                                     semantic-list "\\`{" # ::= { class_body_declarations }
+                                   | NEW qualified_name semantic-list "\\`("
+                                   ;
+  
+## array_creation_expression : NEW qualified_name dims array_initializer
+##                           | NEW qualified_name dims #dim_exprs dims_opt
+##                           ;
+array_creation_expression : NEW array_type array_initializer
+                          | NEW array_type #dim_exprs dims_opt
+                          ;
+  
+dims_opt : dims
+           (,$1)
+         | EMPTY
+           (nil)
+         ;
+
+dims: semantic-list "\\`\\[" dims_opt
+      ((concat "[]" (car ,$2)))
+    ;
+  
+field_access : primary punctuation "\\." symbol
+             | qualified_name
+             ;
+  
+postfix_expression : primary postfix_operator_opt
+                   ;
+
+postfix_operator_opt: punctuation "[-+]" punctuation "[-+]"
+                    | EMPTY
+                    ;
+
+unary_expression : punctuation "[-+^!]" unary_expression
+                 | punctuation "[-+]" punctuation "[-+]" unary_expression
+                 | semantic-list "\\`(" unary_expression # cast
+                 | postfix_expression
+                 ;
+
+operator: punctuation "[-+*/%=<>^~&|!?:.]" # added DOT as field/method access operator
+        | INSTANCEOF 
+        ;
+
+operators: operator operators
+         | operator
+         ;
+
+operators_expression_opt: operators expression
+                        | EMPTY
+                        ;
+
+expression: unary_expression operators_expression_opt
+          ;
+
+# End of java.bnf
 # Simple BNF notation for Makefiles.
 #
-# Copyright (C) 1999, 2000 Eric M. Ludlam
+# Copyright (C) 1999, 2000, 2001 Eric M. Ludlam
 #
 # Author: Eric M. Ludlam <zappo@gnu.org>
 # X-RCS: $Id$
 # Boston, MA 02111-1307, USA.
 #
 # $Log$
+# Revision 1.7  2001/04/13 02:01:59  zappo
+# Added a keyword table, and several new tokens and summaries.
+# Added support for the include macro.
+#
+# Revision 1.6  2001/01/24 21:09:21  zappo
+# Added support for new token formats that use ASSOC.
+#
 # Revision 1.5  2000/11/13 21:06:42  zappo
 # Fixed comment.
 #
 
 %start         Makefile
 %outputfile    semantic-make.el
+%keywordtable  semantic-make-keyword-table
 %parsetable    semantic-toplevel-make-bovine-table
-%languagemode  make-mode
+%languagemode  makefile-mode
 %setupfunction semantic-default-make-setup
 %quotemode     backquote
 
        imenu-create-index-function 'semantic-create-imenu-index
        )%
 
+%token IF "if"
+%token ELSE "else"
+%token ENDIF "endif"
+%put { IF ELSE ENDIF } summary "Conditional: if (expression) ... else ... endif"
+%token INCLUDE "include"
+%put INCLUDE summary "Macro: include filename1 filename2 ..."
+
+%token COLON punctuation ":"
+%token PLUS punctuation "+"
+%token EQUAL punctuation "="
 
 Makefile : variable
 	 | rule
 	 | conditional
+	 | include
 	 ;
 
 variable: symbol equals elements
-	  (,$1 variable nil nil ,$3 nil nil)
+	  (,$1 variable nil ,$3 nil nil)
 	;
 
 rule: symbol colons elements commands
       (,$1 function nil ,$3 nil nil)
     ;
 
-conditional: symbol "if" symbol newline
-	     ( nil )
-	   | symbol "else" newline
-	     ( nil )
-	   | symbol "endif" newline
-	     ( nil )
+conditional: IF symbol newline
+	     ( )
+	   | ELSE newline
+	     ( )
+	   | ENDIF newline
+	     ( )
 	   ;
 
-equals: punctuation ":" punctuation "="	()
-      | punctuation "+" punctuation "="	()
-      | punctuation "="	()
+include: INCLUDE symbol elements
+	 (,$2 include nil)
+       ;
+
+equals: COLON EQUAL ()
+      | PLUS EQUAL ()
+      | EQUAL ()
       ;
 
-colons: punctuation ":" punctuation ":" ()
-      | punctuation ":" ()
+colons: COLON COLON ()
+      | COLON ()
       ;
 
 elements: symbol elements
+# Scheme BNF language specification
+#
+# Copyright (C) 2001 Eric M. Ludlam
+#
+# Author: Eric M. Ludlam <zappo@gnu.org>
+# X-RCS: $Id$
+#
+# This is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNU Emacs; see the file COPYING.  If not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+%start         scheme
+%outputfile    semantic-scm.el
+%parsetable    semantic-toplevel-scheme-bovine-table
+%keywordtable  semantic-scheme-keyword-table
+%languagemode  scheme-mode
+%setupfunction semantic-default-scheme-setup
+
+%(setq semantic-symbol->name-assoc-list '( (variable . "Variables")
+					   ;;(type     . "Types")
+					   (function . "Functions")
+					   (include  . "Loads")
+					   (package  . "DefinModule"))
+       imenu-create-index-function 'semantic-create-imenu-index
+       semantic-dependency-include-path semantic-default-scheme-path
+       imenu-create-index-function 'semantic-create-imenu-index
+       document-comment-start ";;"
+       document-comment-line-prefix ";;"
+       document-comment-end "\n"
+       )%
+
+%token DEFINE "define"
+%put DEFINE summary "Function: (define symbol expression)"
+%token DEFINE-MODULE "define-module"
+%put DEFINE-MODULE summary "Function: (define-module (name arg1 ...)) "
+%token LOAD "load"
+%put LOAD summary "Function: (load \"filename\")"
+
+scheme : semantic-list
+	 (EXPAND $1 scheme-list 4)
+       ;
+
+scheme-list : open-paren "(" scheme-in-list
+	      ( ,$2 )
+	    ;
+
+scheme-in-list: DEFINE symbol expression
+		( $2 variable nil $3 nil )
+	      | DEFINE name-args opt-doc
+		( (car ,$2) function (cdr ,$2) nil ,$3 )
+	      | DEFINE-MODULE name-args
+		( (nth (length $2) $2 ) provide nil)
+	      | LOAD string
+		( (file-name-nondirectory (read $2)) require (read $2) )
+	      | symbol
+		( $1 code )
+	      ;
+
+name-args: semantic-list
+	   (EXPAND $1 name-arg-expand)
+	 ;
+
+name-arg-expand : open-paren name-arg-expand
+		  ( ,$2 )
+		| symbol name-arg-expand
+		  ( ,(cons $1 ,$2) )
+		| EMPTY
+		  (  )
+		;
+
+opt-doc : string
+	| EMPTY
+	;
+
+expression : symbol
+	   | semantic-list
+	   | string
+	   ;
 ;;; semantic-bnf.el --- Semantic details for some languages
 
-;;; Copyright (C) 1999, 2000 Eric M. Ludlam
+;;; Copyright (C) 1999, 2000, 2001 Eric M. Ludlam
 
 ;; Author: Eric M. Ludlam <zappo@gnu.org>
 ;; Version: 0.2
 ;; 
 
 (require 'semantic)
+(eval-when-compile
+  (require 'speedbar)
+  (require 'senator))
 
 ;;; Code:
 (defvar semantic-setup-code-delimiters '("^\\s-*;; Code generated from" .
 		    ;; When loading lisp rules, use READ to convert
 		    ;; into a list we can pretty print later.
 		    ,(semantic-lambda
-		      (let ((r (buffer-substring
+		      (let ((r (buffer-substring-no-properties
 				(car (car vals))
 				(cdr (car vals)))))
 			(list (symbol-name (car (read r))) 'setting r))))
-;     (symbol "token" symbol symbol
-;	     ,(semantic-lambda
-;	       (list (nth 1 vals) 'token (nth 2 vals))))
-     (symbol "start" symbol
+				   ;     (symbol "token" symbol symbol
+					;	     ,(semantic-lambda
+	      ;	       (list (nth 1 vals) 'token (nth 2 vals))))
+     (START symbol
+	    ,(semantic-lambda
+	      (list (nth 1 vals) 'start)))
+     (SCOPESTART symbol
+		 ,(semantic-lambda
+		   (list (nth 1 vals) 'scopestart)))
+     (TOKEN symbol string
+	    ,(semantic-lambda
+	      (list (nth 1 vals) 'keyword "symbol" (nth 2 vals))))
+     (TOKEN symbol symbol string
+	    ,(semantic-lambda
+	      (list (nth 1 vals) 'token
+		    (nth 2 vals)  (nth 3 vals))))
+     (PUT symbol symbol put-value
+	  ,(semantic-lambda
+	    (list (nth 1 vals) 'put
+		  (list (nth 1 vals))
+		  (list (apply 'list (nth 2 vals) 'property (nth 3 vals))))))
+     (PUT symbol semantic-list
+	  ,(semantic-lambda
+	    (list (nth 1 vals) 'put
+		  (list (nth 1 vals))
+		  (semantic-bovinate-from-nonterminal-full
+		   (car (nth 2 vals)) (cdr (nth 2 vals))
+		   `put-value-list))))
+     (PUT semantic-list symbol put-value
+	  ,(semantic-lambda
+	    (let ((names (semantic-bovinate-from-nonterminal-full
+			  (car (nth 1 vals)) (cdr (nth 1 vals))
+			  `put-name-list)))
+	      (list (car (car names)) 'put names
+		    (list (apply 'list (nth 2 vals) 'property (nth 3 vals)))))))
+     (PUT semantic-list semantic-list
+	  ,(semantic-lambda
+	    (let ((names (semantic-bovinate-from-nonterminal-full
+			  (car (nth 1 vals)) (cdr (nth 1 vals))
+			  `put-name-list)))
+	      (list (car (car names)) 'put names
+		    (semantic-bovinate-from-nonterminal-full
+		     (car (nth 2 vals)) (cdr (nth 2 vals))
+		     `put-value-list)))))
+     (OUTPUTFILE symbol punctuation "." symbol "\\bel\\b"
+		 ,(semantic-lambda
+		   (list (concat (nth 1 vals) ".el") 'outputfile)))
+     (PARSETABLE symbol
+		 ,(semantic-lambda
+		   (list (nth 1 vals) 'parsetable)))
+     (KEYWORDTABLE symbol
+		   ,(semantic-lambda
+		     (list (nth 1 vals) 'keywordtable)))
+     (LANGUAGEMODE symbol
+		   ,(semantic-lambda
+		     (list (nth 1 vals) 'languagemode)))
+     (LANGUAGEMODE semantic-list
+		   ,(semantic-lambda
+		     (let ((r (buffer-substring-no-properties
+			       (car (nth 1 vals))
+			       (cdr (nth 1 vals)))))
+		       (list r 'languagemode))))
+     (SETUPFUNCTION symbol
+		    ,(semantic-lambda
+		      (list (nth 1 vals) 'setupfunction)))
+     (QUOTEMODE symbol
+		,(semantic-lambda
+		  (list (nth 1 vals) 'quotemode)))
+     )
+    (put-name-list
+     (open-paren ,(semantic-lambda (list nil)))
+     (close-paren ,(semantic-lambda (list nil)))
+     (symbol ,(semantic-lambda (list (nth 0 vals) 'name))))
+    (put-value-list
+     (open-paren ,(semantic-lambda (list nil)))
+     (close-paren ,(semantic-lambda (list nil)))
+     (symbol put-value
 	     ,(semantic-lambda
-	       (list (nth 1 vals) 'start)))
-     (symbol "token" symbol string
-	     ,(semantic-lambda
-	       (list (nth 1 vals) 'keyword "symbol" (nth 2 vals))))
-     (symbol "token" symbol symbol string
-	     ,(semantic-lambda
-	       (list (nth 1 vals) 'token
-		     (nth 2 vals)  (nth 3 vals))))
-     (symbol "outputfile" symbol punctuation "." symbol "\\bel\\b"
-	     ,(semantic-lambda
-	       (list (concat (nth 1 vals) ".el") 'outputfile)))
-     (symbol "parsetable" symbol
-	     ,(semantic-lambda
-	       (list (nth 1 vals) 'parsetable)))
-     (symbol "keywordtable" symbol
-	     ,(semantic-lambda
-	       (list (nth 1 vals) 'keywordtable)))
-     (symbol "languagemode" symbol
-	     ,(semantic-lambda
-	       (list (nth 1 vals) 'languagemode)))
-     (symbol "setupfunction" symbol
-	     ,(semantic-lambda
-	       (list (nth 1 vals) 'setupfunction)))
-     (symbol "quotemode" symbol
-	     ,(semantic-lambda
-	       (list (nth 1 vals) 'quotemode)))
+	       (apply 'list (nth 0 vals) 'property (nth 1 vals))))
      )
+    (put-value
+     (symbol ,(semantic-lambda (list (nth 0 vals))))
+     (string ,(semantic-lambda (list (nth 0 vals))))
+     (semantic-list
+      ,(semantic-lambda (list (semantic-flex-text (cons 1 (nth 0 vals)))))))
     (rule-list
      (match-list lambda-fn rule-or-list
 		 ,(semantic-lambda
     )
 "Bovine table used to convert a BNF language file into a bovine table.")
 
+(defvar semantic-bnf-keyword-table
+  (semantic-flex-make-keyword-table
+   `( ("start" . START)
+      ("scopestart" . SCOPESTART)
+      ("token" . TOKEN)
+      ("put" . PUT)
+      ("outputfile" . OUTPUTFILE)
+      ("parsetable" . PARSETABLE)
+      ("keywordtable" . KEYWORDTABLE)
+      ("languagemode" . LANGUAGEMODE)
+      ("setupfunction" . SETUPFUNCTION)
+      ("quotemode" . QUOTEMODE)
+      )
+   `(("put" summary "%put <keyword> <lisp expression>")
+     ("token" summary "%token <keyword> [syntax] \"matchtext\"")
+     ("start" summary "%start <starting rule name>")
+     ("scopestart" summary "%scopestart <starting scope (code) rule name>")
+     ("languagemode" summary "%languagemode [ lispsymbol | ( lispsym lispsym ...) ]")
+     ))
+  "Keyword table used for Semantic BNF files.")
+
 
 ;;; Conversion routines
 ;;
   "Insert a token expand function based on LST."
   (let ((argv (1- (string-to-int (substring (symbol-name (car (cdr lst)))
 					    1)))))
-    (insert "\n")
+    (insert "\n ")
     (insert "(semantic-bovinate-from-nonterminal "
 	    "(car (nth " (int-to-string argv) " vals)) "
 	    "(cdr (nth " (int-to-string argv) " vals)) "
 	    "'" (symbol-name (car (cdr (cdr lst))))
-	    ")\n")))
+	    ")\n ")))
 
 (defun semantic-bnf-EXPANDFULL (lst)
   "Insert a token full expand function based on LST."
   (let ((argv (1- (string-to-int (substring (symbol-name (car (cdr lst)))
 					    1)))))
-    (insert "\n")
+    (insert "\n ")
     (insert "(semantic-bovinate-from-nonterminal-full "
 	    "(car (nth " (int-to-string argv) " vals)) "
 	    "(cdr (nth " (int-to-string argv) " vals)) "
 	    "'" (symbol-name (car (cdr (cdr lst))))
-	    ")\n")))
+	    ")\n ")))
+
+(defun semantic-bnf-ASSOC (lst quotemode)
+  "Handle an ASSOC list based on LST.
+QUOTEMODE is the current mode of quotation."
+  (let ((lst (cdr lst))
+	l)
+    (while lst
+      ;; quote the key
+      (setq l   (cons (list 'quote (car lst)) l)
+	    lst (cdr lst))
+      ;; push the value
+      (if lst
+	  (setq l   (cons (car lst) l)
+		lst (cdr lst))))
+    ;; substitute ASSOC by call to semantic-bovinate-make-assoc-list
+    ;; and do BNF lambda substitution on the whole expression
+    (semantic-bnf-lambda-substitute
+     (cons 'semantic-bovinate-make-assoc-list (nreverse l)) quotemode t)))
 
 (defun semantic-bnf-lambda-substitute (lst quotemode &optional inplace)
   "Insert LST substituting based on rules for the BNF converter.
 	      (semantic-bnf-lambda-substitute (car lst) quotemode nil)
 	      (insert ")")
 	      (setq lst nil inplace nil))
-	  (insert "(list")
-	  (setq inplace t))
-	)
-    (if inplace (insert " (")))
+	  (if (and (= (length lst) 1) (symbolp (car lst)))
+	      (progn
+		(insert " '" (symbol-name (car lst)))
+		(setq lst nil inplace nil))
+	    (insert "(list")
+	    (setq inplace t))
+	  )))
   (cond ((eq (car lst) 'EXPAND)
 	 (semantic-bnf-EXPAND lst))
 	((eq (car lst) 'EXPANDFULL)
 	 (semantic-bnf-EXPANDFULL lst))
+	((eq (car lst) 'ASSOC)
+	 (semantic-bnf-ASSOC lst quotemode))
 	(t
+	 (if inplace (insert " ("))
 	 (let ((inlist nil))
 	   (while lst
 	     (cond ((eq (car lst) nil)
 		      (if (and (not inlist) (not inplace))
 			  (progn (insert " (list")
 				 (setq inlist t)))
-		      (if (and inplace (not fn) (not (eq (car (car lst)) 'EXPAND)))
-			  (insert " (append"))
-		      (semantic-bnf-lambda-substitute (car lst) quotemode (and fn (not (eq fn 'quote))))
-		      (if (and inplace (not fn) (not (eq (car (car lst)) 'EXPAND)))
-			  (insert  ")"))
+;		      (if (and inplace (not fn) (not (eq (car (car lst)) 'EXPAND)))
+;			  (insert " (append"))
+		      (semantic-bnf-lambda-substitute (car lst) quotemode t);(and fn (not (eq fn 'quote))))
+;		      (if (and inplace (not fn) (not (eq (car (car lst)) 'EXPAND)))
+;			  (insert  ")"))
 		      ))
 		   ((symbolp (car lst))
 		    (let ((n (symbol-name (car lst))) ;the name
 					   (setq inlist nil))))
 			      (insert " (nth " (int-to-string val) " vals)")
 			      (if (and (not x) (not inplace)) (setq inlist t)))
-			  (if (and (not inlist) (not inplace))
+			  (if (and (not inlist) (not inplace) )
 			      (progn (insert " (list")
 				     (setq inlist t)))
-			  (insert " " (if inplace "" "'") n)))))
+			  (insert " "
+				  (if (or inplace (eq (car lst) t)) "" "'")
+				  n; " "
+				  )))))
 		   (t
 		    (if (and (not inlist) (not inplace))
 			(progn (insert " (list")
 			       (setq inlist t)))
 		    (insert (format " %S" (car lst)))))
 	     (setq lst (cdr lst)))
-	   (if inlist (insert ")")))))
-  (if inplace (insert ")")))
+	   (if inlist (insert ")")))
+	   (if inplace (insert ")"))))
+  )
 
 (defun semantic-bnf-lambda-convert (semliststr vals quotemode)
   "Convert SEMLISTSTR into Lisp code based on VALS.
       ;; We converted the lambda string into a list.  Now write it
       ;; out as the bovine lambda expression, and do macro-like
       ;; conversion upon it.
-      (insert "\n")
+      (insert "\n ")
       (cond ((eq (car slsr) 'EXPAND)
-	     (insert ",(lambda (vals start end)\n")
+	     (insert ",(lambda (vals start end)\n ")
 	     (semantic-bnf-EXPAND slsr)
 	     )
 	    ((and (listp (car slsr))
 	     ;; Use a simpler expander
 	     )
 	    (t
-	     (insert ",(semantic-lambda\n")
+	     (insert " ,(semantic-lambda\n ")
 	     (semantic-bnf-lambda-substitute slsr quotemode)
 	     ))
       (insert ")"))))
 
-(defun semantic-bnf-to-bovine (file tokstream &optional start)
-  "Insert the BNF file FILE into the current buffer as a bovine table.
-Inserts the token stream TOKSTREAM, and uses START is the starting token."
+(defun semantic-bnf-to-bovine (tokstream &optional start scopestart)
+  "Insert the BNF TOKSTREAM into the current buffer as a bovine table.
+Optional argument START is the token to start with.
+Optional argument SCOPESTART is the token to start subscopes with."
   (interactive "FBNF file: ")
   (let ((tl (float (length tokstream)))
 	(tokens (semantic-find-nonterminal-by-token 'token tokstream))
 	       (matches (car (cdr (cdr (cdr rule))))))
 	  (when (eq (car (cdr rule)) 'rule)
 	    (insert "(")
-	    (if (and start (string= start (car rule)))
-		(insert "bovine-toplevel")
-	      (insert (car rule)))
-	    (insert "\n")
+	    (cond ((and start (string= start (car rule)))
+		   (insert "bovine-toplevel"))
+		  ((and scopestart (string= scopestart (car rule)))
+		   (insert "bovine-inner-scope"))
+		  (t
+		   (insert (car rule))))
+	    (insert "\n ")
 	    (while matches
 	      (let* ((mla (car matches))
 		     (lamb (car mla))
 			(insert " " (car ml))))
 		    (setq ml (cdr ml))))
 		(semantic-bnf-lambda-convert lamb (car (cdr mla)) quotemode)
-		(insert ")\n"))
+		(insert ")\n "))
 	      (setq matches (cdr matches)))
-	    (insert ") ; end " (car rule) "\n")))
+	    (insert ") ; end " (car rule) "\n ")))
 	(setq tokstream (cdr tokstream))
 	(working-status (* 100.0 (- 1.0 (/ (float (length tokstream)) tl)))))
       (working-status t))
-    (insert ")\n")
+    (insert ")\n ")
     ))
 
 ;;; Output File hacks
 parse table variable."
   (let ((mode (semantic-find-nonterminal-by-token 'languagemode tokstream)))
     (if mode
-	(intern (semantic-token-name (car mode)))
-      (semantic-bnf-find-languagemode-old))))
+	(let ((m (read (semantic-token-name (car mode)))))
+	  (if (listp m)
+	      m
+	    (list m)))
+      (list (semantic-bnf-find-languagemode-old)))))
 
 (defun semantic-bnf-find-setup-code (tokstream sourcefile)
   "Find the setup code based on TOKSTREAM.
 	    (let ((m (string-match ";"
 				   (car semantic-setup-code-delimiters))))
 	      (insert (substring (car semantic-setup-code-delimiters) m))
-	      (insert " " sourcefile "\n")
+	      (insert " " sourcefile "\n  ")
 	      (save-excursion;; save in the middle
-		(insert "\n" (substring (cdr semantic-setup-code-delimiters)
+		(insert "\n " (substring (cdr semantic-setup-code-delimiters)
 					m))
-		(insert " " sourcefile "\n"))
+		(insert " " sourcefile "\n "))
 	      t)
 	    ))))))
+
+(defvar semantic-bnf-indent-table t
+  "Non nil means to indent the large table during creation.")
+
+(defun semantic-bnf-generate-and-load-no-indent ()
+  "Call `semantic-bnf-genrate-and-load' without indenting the table."
+  (interactive)
+  (let ((semantic-bnf-indent-table nil))
+    (semantic-bnf-generate-and-load)))
   
 (defun semantic-bnf-generate-and-load ()
   "Take the current BNF, auto-generate it into a table, and load it."
 	 (keydest (semantic-bnf-find-keyword-destination tok))
 	 (mode (semantic-bnf-find-languagemode tok))
 	 (start (semantic-find-nonterminal-by-token 'start tok))
+	 (scopestart (semantic-find-nonterminal-by-token 'scopestart tok))
+	 (setup-fn (semantic-find-nonterminal-by-token 'setupfunction tok))
 	 )
     (if (not dest)
 	(error "You must specify a destination table in your BNF file"))
 	    (delete-region (point) (save-excursion (forward-sexp 1) (point))))
 	(delete-blank-lines)
 	(let ((key (semantic-find-nonterminal-by-token 'keyword tok))
+	      keys
+	      (put (semantic-find-nonterminal-by-token 'put tok))
 	      (start (point)))
 	  (if (not key)
-	      (insert "nil\n")
+	      (insert "nil\n ")
 	    (insert "(semantic-flex-make-keyword-table \n `(")
+	    ;; Get all the keys
 	    (while key
-	      (insert " (" (nth 3 (car key)) " . " (car (car key)) ")\n")
+	      (insert " (" (nth 3 (car key)) " . " (car (car key)) ")\n ")
 	      (setq key (cdr key)))
-	    (insert "))\n"))
+	    (insert ")\n  '(\n ")
+	    ;; Now get all properties
+	    (while put
+	      (setq keys (nth 2 (car put)))
+	      (while keys
+		(setq key (semantic-find-nonterminal-by-token 'keyword tok))
+		(let ((a (assoc (if (listp (car keys))
+				    (car (car keys))
+				  (car keys))
+				key)))
+		  (if (not a) (error "Token %s not found" (car keys)))
+		  (let ((pairs (nth 3 (car put))))
+		    (while pairs
+		      (insert "  ("
+			      (nth 3 a) " "
+			      (car (car pairs)) " "
+			      (car (cdr (cdr (car pairs)))) ")\n ")
+		      (setq pairs (cdr pairs)))))
+		(setq keys (cdr keys)))
+	      (setq put (cdr put)))
+	    (insert "))\n "))
 	  (save-excursion
 	  (indent-region start (point) nil)))
 	(eval-defun nil))
 	   (when var
 	     ;; The bovine table
 	     (insert "(setq semantic-toplevel-bovine-table "
-		     (semantic-token-name (car var)) ")\n"))
+		     (semantic-token-name (car var)) "\n ")
+	     (insert "semantic-toplevel-bovine-table-source \""
+		     fname "\")\n")
+	     )
 	   ;; Keytable setup
 	   (when key
 	     (insert "(setq semantic-flex-keywords-obarray "
-		     (semantic-token-name (car key)) ")\n"))
+		     (semantic-token-name (car key)) ")\n "))
+	   ;; Is there more than one major mode?
+	   (if (and (listp mode) (> (length mode) 1))
+	       (insert "(setq semantic-equivalent-major-modes '"
+		       (format "%S" mode) ")\n"))
 	   ;; Add in user specified settings
 	   (let ((settings (semantic-find-nonterminal-by-token 'setting tok)))
 	     (while settings
 	       (insert (nth 2 (car settings)))
-	       (insert "\n")
+	       (insert "\n ")
 	       (setq settings (cdr settings))))
 	   (point))
 	 nil)
       (if (looking-at "\\s-*\\(`?(\\|nil\\)")
 	  (delete-region (point) (save-excursion (forward-sexp 1) (point))))
       (delete-blank-lines)
-      (semantic-bnf-to-bovine (buffer-file-name bb) tok
-			      (if start (semantic-token-name (car start))))
-      (message "Indenting table....")
-      (save-excursion
-	(indent-region (progn (re-search-backward "(defvar")
-			      (goto-char (match-beginning 0))
-			      (point))
-		       (progn (forward-sexp 1) (point))
-		       nil))
+      (semantic-bnf-to-bovine
+       tok (if start (semantic-token-name (car start)))
+       (if scopestart (semantic-token-name (car scopestart))))
+      (if semantic-bnf-indent-table
+	  (save-excursion
+	    (message "Indenting table....")
+	    (indent-region (progn (re-search-backward "(defvar")
+				  (goto-char (match-beginning 0))
+				  (point))
+			   (progn (forward-sexp 1) (point))
+			   nil)))
       (eval-defun nil))
     (message "Done.")
-    (if mode
-	(save-excursion
-	  (let ((bufs (buffer-list)))
-	    (while bufs
-	      (set-buffer (car bufs))
-	      (if (eq major-mode mode)
-		  (funcall mode))
-	      (setq bufs (cdr bufs))))))))
+    (when mode
+      (save-excursion
+	(let ((bufs (buffer-list)))
+	  (while bufs
+	    (set-buffer (car bufs))
+	    (if (member major-mode mode)
+		(progn
+		  (if setup-fn
+		      (funcall (intern (semantic-token-name (car setup-fn))))
+		    (funcall mode)))
+	      )
+	    (setq bufs (cdr bufs)))))
+      )))
+
+(defun semantic-bnf-generate-one-rule ()
+  "Generate code for one rule in a temporary buffer."
+  (interactive)
+  (semantic-bovinate-toplevel t)
+  (let ((r (semantic-current-nonterminal)))
+    (if (or (not r) (not (eq (semantic-token-token r) 'rule)))
+	(error "No rule to expand nearby"))
+    (pop-to-buffer "*Rule Expansion*" t)
+    (save-excursion
+      (set-buffer "*Rule Expansion*")
+      (erase-buffer)
+      (insert "Expanding rule [" (semantic-token-name r) "]\n\n")
+      (semantic-bnf-to-bovine (list r)))))
+
+;;; Debugging support
+;;
+;; Source level debugging if a BNF table requires a few simple functions.
+(defun semantic-bnf-skip-string-or-comment ()
+  "Return non-nil if point was moved after a string or comment."
+  (let ((state (parse-partial-sexp (save-excursion
+                                     (beginning-of-line) (point))
+                                   (point))))
+    (cond ((nth 3 state) ;; string
+           (re-search-backward "\\s\"")
+           (forward-sexp)
+           t)
+          ((nth 4 state) ;; comment
+           (forward-line)
+           t)
+          (t
+           nil))))
+
+(defun semantic-bnf-find-state-position (rule matchlistindex matchindex)
+  "Find the current debugger position in the current buffer.
+RULE is a symbol representing the rule name we are currently in.
+MATCHLISTINDEX is the index to the current match list being tested.
+MATCHINDEX is the index into the matchlist being tested."
+  (let* ((start (car (semantic-find-nonterminal-by-token 'start (current-buffer))))
+	 (sn (symbol-name rule))
+	 (findme (if (and start (eq rule 'bovine-toplevel))
+		     (semantic-token-name start)
+		   (symbol-name rule)))
+	 (r (semantic-find-nonterminal-by-name
+	     findme (semantic-find-nonterminal-by-token
+		     'rule (current-buffer)))))
+    (if (not r)
+	(error "Semantic debugger error: Cannot find rule %s" findme))
+    ;; Find the rule
+    (goto-char (semantic-token-start r))
+    ;; find the matchlist
+    (re-search-forward ":\\s-*")
+    (while (/= matchlistindex 0)
+      (re-search-forward "\\s-*|\\s-*")
+      ;; If point is in a comment or a string skip it
+      (or (semantic-bnf-skip-string-or-comment)
+          (setq matchlistindex (1- matchlistindex)))
+      )
+    ;; find the specific token we are matching
+    (while (/= matchindex 0)
+      (when (semantic-bnf-looking-at-%token-not-keyword)
+	(setq matchindex (1- matchindex)))
+      (forward-sexp 1)
+      (setq matchindex (1- matchindex))
+      )
+    (skip-chars-forward " \t\n")
+    ;; Leave the cursor here, and let them highlight if for us
+    (current-buffer)
+    ))
+
+(defun semantic-bnf-looking-at-%token-not-keyword ()
+  "Return non-nil if the token following the cursor is a %token.
+Some tokens are keywords.  Make sure we know the difference."
+  (when (looking-at "\\s-*\\(\\(\\w\\|\\s_\\)+\\)")
+    (semantic-find-nonterminal-by-name
+     (match-string 1)
+     (semantic-find-nonterminal-by-token 'token (current-buffer)))))
+
+(defun semantic-bnf-find-source-on-load-path (sourcefile)
+  "Find the BNF file SOURCEFILE on the Emacs `load-path'.
+Once found, put it in a buffer, and return it."
+  (let ((sf (locate-library sourcefile)))
+    (if sf (find-file-noselect sf)))
+  )
 
 ;;; Semantic BNF mode
 ;;
   (modify-syntax-entry ?\; "." semantic-bnf-syntax-table)
   (modify-syntax-entry ?\" "\"" semantic-bnf-syntax-table)
   (modify-syntax-entry ?- "_" semantic-bnf-syntax-table)
-  (modify-syntax-entry ?( "(" semantic-bnf-syntax-table)
-  (modify-syntax-entry ?) ")(" semantic-bnf-syntax-table)
   (modify-syntax-entry ?# "<" semantic-bnf-syntax-table)