svn commit - mysqldoc@docsrva: r8819 - in trunk/tools: . MySQL MySQL/XML

[mcbrown]

Author: mcbrown
Date: 2007-11-21 18:48:29 +0100 (Wed, 21 Nov 2007)
New Revision: 8819

Log:
Some changes related to the arbitrary parser: 

 - Moved the parsing of dynxml source into a string format and moved the process into a
module so that we can call it from elsewhere
 - Updated the parser to be a front end to the module
 - Some minor tweaks to the arbitrary parser
 - Updated the arbitrary parser so that it uses the DynXML parser to parse content during
the arbitrary parsing process
 - Some other fixes to the parser to handle some recent ID map and other changes
 - Update to ID map to expose the relative prefix at ID level (required by the Arby
parser)




Added:
   trunk/tools/MySQL/DynXMLParse.pm
Modified:
   trunk/tools/MySQL/IDMap.pm
   trunk/tools/MySQL/XML/Arbitrary.pm
   trunk/tools/arbitrary-parser.pl
   trunk/tools/dynxml-parser.pl


Added: trunk/tools/MySQL/DynXMLParse.pm
===================================================================
--- trunk/tools/MySQL/DynXMLParse.pm	                        (rev 0)
+++ trunk/tools/MySQL/DynXMLParse.pm	2007-11-21 17:48:29 UTC (rev 8819)
Changed blocks: 1, Lines Added: 145, Lines Deleted: 0; 4612 bytes

@@ -0,0 +1,145 @@
+package MySQL::DynXMLParse;
+
+# DynXMLParse.pm
+#
+# generic parser for all dynamic document elements
+
+# Martin MC Brown
+# mc@stripped
+# 2007-22-21
+
+use strict;
+use warnings;
+
+# Load the MySQL specific modules
+
+use lib qw(./tools/ ../tools ../../tools ../../../trunk/tools);
+use MySQL::DynXML;
+
+# Dynamic doc types to parse
+
+use MySQL::DynXML::Optvar;
+use MySQL::DynXML::Opfuncs;
+use MySQL::DynXML::ReservedWords;
+use MySQL::DynXML::Changelog;
+
+# Load the local modules
+
+use IO::File;
+use vars qw(@ISA @EXPORT);
+require Exporter;
+@ISA = qw(Exporter);
+
+@EXPORT = qw(inplace_parse);
+
+sub inplace_parse
+{
+    my ($inplacefile,$options) = @_;
+
+# Initialize the list of known DynXML types, but don't instantiate the related class
+
+    my $DynXMLParsers = {
+        optvar => undef,
+        opfuncs => undef,
+        reservedwords => undef,
+        changelog => undef,
+        openbugs => undef,
+    };
+
+    my $dynxml = new MySQL::DynXML();
+
+    my $sources = {};
+    
+    my @matches = ($inplacefile =~ m{(<para
condition="dynamic:.*?"\s+role=".*?"/>)}msg);
+    my @titlematches = ($inplacefile =~ m{(<title
condition="dynamic:.*?"\s+role=".*?"/>)}msg);
+    push @matches,@titlematches;
+
+    my $tdcache = {};
+
+    foreach my $item (@matches)
+    {
+        my $parser_options = deep_copy($options);
+        my ($inserttype,$tabletype) = ($item =~ m/condition="dynamic:(.*?):(.*?)"/);
+
+        my $string = '';
+
+        if ($inserttype eq 'optvar')
+        {
+            if (!exists($DynXMLParsers->{optvar}) || 
+                !defined($DynXMLParsers->{optvar}))
+            {
+                $DynXMLParsers->{optvar} = new
MySQL::DynXML::Optvar($dynxml,$parser_options);
+            }
+            $string = $DynXMLParsers->{optvar}->generate($parser_options,$item);
+        }
+        elsif ($inserttype eq 'opfuncs')
+        {
+            if (!exists($DynXMLParsers->{opfuncs}) || 
+                !defined($DynXMLParsers->{opfuncs}))
+            {
+                $DynXMLParsers->{opfuncs} = new
MySQL::DynXML::Opfuncs($dynxml,$parser_options);
+            }
+            $string = $DynXMLParsers->{opfuncs}->generate($parser_options,$item);
+        }
+        elsif ($inserttype eq 'reservedwords')
+        {
+            if (!exists($DynXMLParsers->{reserveredwords}) || 
+                !defined($DynXMLParsers->{reservedwords}))
+            {
+                $DynXMLParsers->{reservedwords} = new
MySQL::DynXML::ReservedWords($dynxml,$parser_options);
+            }
+            $string =
$DynXMLParsers->{reservedwords}->generate($parser_options,$item);
+        }
+        elsif ($inserttype eq 'changelog')
+        {
+            if (!exists($DynXMLParsers->{changelog}) || 
+                !defined($DynXMLParsers->{changelog}))
+            {
+                $DynXMLParsers->{changelog} = new
MySQL::DynXML::Changelog($dynxml,$parser_options);
+            }
+            $string = $DynXMLParsers->{changelog}->generate($parser_options,$item);
+        }
+        elsif ($inserttype eq 'openbugs')
+        {
+            if (!exists($DynXMLParsers->{openbugs}) || 
+                !defined($DynXMLParsers->{openbugs}))
+            {
+                $parser_options->{usecustombasedir} = 'open-bugs';
+                $DynXMLParsers->{changelog} = new
MySQL::DynXML::Changelog($dynxml,$parser_options);
+            }
+            $string = $DynXMLParsers->{changelog}->generate($parser_options,$item);
+        }
+        else
+        {
+            print STDERR "$0: WARNING: Dynamic datatype $inserttype not recognized\n";
+        }
+        my $qitem = quotemeta($item);
+        
+        $inplacefile =~ s/$qitem/$string/g;
+    }
+
+    my @lines;
+    push @lines,sprintf('<remark role="dependency"
condition="%s"/>',$options->{infile});
+
+    for my $dynxmltype (keys %{$DynXMLParsers})
+    {
+        push @lines,$DynXMLParsers->{$dynxmltype}->generate_dependencies()
+            if (defined($DynXMLParsers->{$dynxmltype}));
+    }
+
+    my $deptext = join("\n",@lines);
+    $inplacefile =~ s{<remark role="dynamic-dependency-list"/>}{$deptext};
+
+    return($inplacefile);
+}
+
+sub deep_copy {
+    my $this = shift;
+    if (not ref $this) {
+        $this;
+    } elsif (ref $this eq 'ARRAY') {
+        [map deep_copy($_), @$this];
+    } elsif (ref $this eq 'HASH') {
+        +{map { $_ => deep_copy($this->{$_}) } keys %$this};
+    } else { die 'what type is $_?' }
+}


Modified: trunk/tools/MySQL/IDMap.pm
===================================================================
--- trunk/tools/MySQL/IDMap.pm	2007-11-21 17:23:53 UTC (rev 8818)
+++ trunk/tools/MySQL/IDMap.pm	2007-11-21 17:48:29 UTC (rev 8819)
Changed blocks: 1, Lines Added: 1, Lines Deleted: 0; 522 bytes

@@ -560,6 +560,7 @@
                   adds => [$prefix],
                   fullpath => $self->{fileinfo}->{fullpath},
                   relpath => $self->{fileinfo}->{relpath},
+                  relprefix => $self->{fileinfo}->{relprefix},
     };
     
     if (exists($self->{byid}->{$id}))


Modified: trunk/tools/MySQL/XML/Arbitrary.pm
===================================================================
--- trunk/tools/MySQL/XML/Arbitrary.pm	2007-11-21 17:23:53 UTC (rev 8818)
+++ trunk/tools/MySQL/XML/Arbitrary.pm	2007-11-21 17:48:29 UTC (rev 8819)
Changed blocks: 5, Lines Added: 21, Lines Deleted: 9; 2601 bytes

@@ -1,7 +1,9 @@
 package MySQL::XML::Arbitrary;
+
 # Load the standard MySQL Libraries
 
 use MySQL;
+use MySQL::DynXMLParse;
 
 # Ensure we have enabled error checking
 

@@ -30,6 +32,7 @@
                   'idmap' => $options->{idmap},
                   'id' => 'arbitrary',
                   'debug' => $options->{debug},
+                  'srcdir' => $options->{srcdir},
               }, $class;
 }
 

@@ -369,15 +372,20 @@
     foreach (split(/[\r\n]+/,$filetxt))
     {
         $incomment = 1 if ($_ =~ m/<!--/);
-        $incomment = 0 if ($incomment && ($_ =~ /-->/));
-        next if ($incomment);
+        if ($incomment && ($_ =~ m/-->/))
+        {
+            $incomment = 0;
+            next;
+        }
+        next if ($incomment > 0);
+
         if ($_ =~ m{SYSTEM\s*?"(.*?)"})
         {
             my $loc = $1;
             if (!-e $loc)
             {
 # Prepend the source file prefix to the file:
-                my $newloc =
"$self->{idmap}->{byid}->{$spec->{id}}->{prefix}/$loc";
+                my $newloc =
"$self->{idmap}->{byid}->{$spec->{id}}->{relprefix}/$loc";
                 $_ =~ s{SYSTEM\s*?".*?"}{SYSTEM "$newloc"};
             }
         }

@@ -388,7 +396,7 @@
             if (!-e $loc)
             {
 # Prepend the source file prefix to the file:
-                my $newloc =
"$self->{idmap}->{byid}->{$spec->{id}}->{prefix}/$loc";
+                my $newloc =
"$self->{idmap}->{byid}->{$spec->{id}}->{relprefix}/$loc";
                 $_ =~ s{$srctype=".*?"}{$srctype="$newloc"};
             }
         }

@@ -547,17 +555,21 @@
     {
         die("PANIC: Don't seem to have any lines to write out\n");
     }
+
+    my $inplacefile = join("\n",@{$spec->{lines}});
         
-    open(DEST,">$destfile") or die "Couldn't open the section file ($destfile: $!)\n";
-    print DEST (join("\n",@{$spec->{lines}}));
-    close(DEST);
+    my $fh = new IO::File($destfile,'w') or die "Couldn't open the section file
($destfile: $!)\n";
+    binmode($fh,':utf8');
+    print $fh (inplace_parse($inplacefile,
+                             {srcdir => $self->{srcdir},
+                              infile => $destfile,
+                             }));
+    $fh->close();
 
     $self->print_error("********\n********\n*******\n\n");
 
     return sprintf('<xi:include href="%s"
xmlns:xi="http://www.w3.org/2001/XInclude"/>',
                    $destfile);
-
-
 }
 
 1;


Modified: trunk/tools/arbitrary-parser.pl
===================================================================
--- trunk/tools/arbitrary-parser.pl	2007-11-21 17:23:53 UTC (rev 8818)
+++ trunk/tools/arbitrary-parser.pl	2007-11-21 17:48:29 UTC (rev 8819)
Changed blocks: 3, Lines Added: 14, Lines Deleted: 5; 1860 bytes

@@ -29,16 +29,17 @@
     die "ERROR: ID maps must be version 1 or later\nPlease change to trunk and 'make
clean', then return here and 'make idmap.refs'";
 }
 
-my ($help,$debug) = (0,0);
+my ($help,$debug,$srcdir) = (0,0,undef);
 
 GetOptions("help" => \$help,
            "debug" => \$debug,
+           "srcdir=s" => \$srcdir,
            );
 
-if ($help)
+if ($help or !defined($srcdir))
 {
     print <<EOF;
-genarbelements.pl [--debug] arbspecfile xmlembedfile
+genarbelements.pl [--debug] --srcdir=DYNXML-BASE arbspecfile xmlembedfile
 EOF
     exit 0;
 }

@@ -47,7 +48,9 @@
 my $template = shift or die "You must supply the name of the template to process";
 
 my $my_handler = MySQL::XML::Arbitrary->new({idmap => $idmap,
-                                             debug => $debug});
+                                             debug => $debug, 
+                                             srcdir => $srcdir,
+                                            });
 
 XML::Parser::PerlSAX->new->parse(Source => { SystemId => $file}, 
                                  Handler => $my_handler);

@@ -57,8 +60,14 @@
 close(TEMPLATE);
 foreach my $id (keys %{$my_handler->{fragments}})
 {
+    print STDERR "Replacing fragment $id in template\n" if ($debug);
+    
     my $inclist = join("\n",@{$my_handler->{fragments}->{$id}});
-    $templatesrc =~ s/<!--SRCFILE $id-->/$inclist/g;
+    my $changes = ($templatesrc =~ s/<!--SRCFILE $id-->/$inclist/g);
+    if ($changes == 0)
+    {
+        print STDERR "Couldn't replace the fragment $id\n";
+    }
 }
 open(XMLFILE,">$template.final") or die "Couldn't create XML file\n";
 print XMLFILE $templatesrc;


Modified: trunk/tools/dynxml-parser.pl
===================================================================
--- trunk/tools/dynxml-parser.pl	2007-11-21 17:23:53 UTC (rev 8818)
+++ trunk/tools/dynxml-parser.pl	2007-11-21 17:48:29 UTC (rev 8819)
Changed blocks: 4, Lines Added: 11, Lines Deleted: 127; 5497 bytes

@@ -2,7 +2,7 @@
 
 # dynxml-parser.pl
 # 
-# generic parser for all dynamic document elements
+# Interface to the dynxml parser
 
 # Martin MC Brown
 # mc@stripped

@@ -15,21 +15,13 @@
 
 use lib qw(./tools/ ../tools ../../tools ../../../trunk/tools);
 use MySQL::DynXML;
+use MySQL::DynXMLParse;
 
-# Dynamic doc types to parse
-
-use MySQL::DynXML::Optvar;
-use MySQL::DynXML::Opfuncs;
-use MySQL::DynXML::ReservedWords;
-use MySQL::DynXML::Changelog;
-
 # Load the local modules
 
 use Getopt::Long;
 use IO::File;
 
-my $dynxml = new MySQL::DynXML();
-
 my ($opt_infile,$opt_outfile,$opt_srcdir) = (undef,undef,undef);
 
 GetOptions("infile=s" => \$opt_infile,

@@ -48,17 +40,17 @@
                srcdir => $opt_srcdir,
            };
 
-# Initialize the list of known DynXML types, but don't instantiate the related class
+my $fh = new IO::File($options->{infile}) or die "Error: Can't open source file
($options->{infile}): $!\n";
+binmode($fh,':utf8');
+my $inplacefile = join('',<$fh>);
+$fh->close();
 
-my $DynXMLParsers = {
-    optvar => undef,
-    opfuncs => undef,
-    reservedwords => undef,
-    changelog => undef,
-    openbugs => undef,
-};
+my $finaltext = inplace_parse($inplacefile,$options);
 
-inplace_parse($options);
+$fh = new IO::File($options->{outfile},'w') or die "Error: Can't open destination file
($options->{outfile}: $!\n";    
+binmode($fh,':utf8');
+print $fh $finaltext;
+$fh->close();
 
 sub show_help
 {

@@ -85,111 +77,3 @@
         exit 0;
     }
 }
-
-sub inplace_parse
-{
-    my ($options) = @_;
-
-    my $fh = new IO::File($options->{infile}) or die "Error: Can't open source file
($options->{infile}): $!\n";
-    binmode($fh,':utf8');
-    my $inplacefile = join('',<$fh>);
-    $fh->close();
-
-    my $sources = {};
-    
-    my @matches = ($inplacefile =~ m{(<para
condition="dynamic:.*?"\s+role=".*?"/>)}msg);
-    my @titlematches = ($inplacefile =~ m{(<title
condition="dynamic:.*?"\s+role=".*?"/>)}msg);
-    push @matches,@titlematches;
-
-    my $tdcache = {};
-
-    foreach my $item (@matches)
-    {
-        my $parser_options = deep_copy($options);
-        my ($inserttype,$tabletype) = ($item =~ m/condition="dynamic:(.*?):(.*?)"/);
-
-        my $string = '';
-
-        if ($inserttype eq 'optvar')
-        {
-            if (!exists($DynXMLParsers->{optvar}) || 
-                !defined($DynXMLParsers->{optvar}))
-            {
-                $DynXMLParsers->{optvar} = new
MySQL::DynXML::Optvar($dynxml,$parser_options);
-            }
-            $string = $DynXMLParsers->{optvar}->generate($parser_options,$item);
-        }
-        elsif ($inserttype eq 'opfuncs')
-        {
-            if (!exists($DynXMLParsers->{opfuncs}) || 
-                !defined($DynXMLParsers->{opfuncs}))
-            {
-                $DynXMLParsers->{opfuncs} = new
MySQL::DynXML::Opfuncs($dynxml,$parser_options);
-            }
-            $string = $DynXMLParsers->{opfuncs}->generate($parser_options,$item);
-        }
-        elsif ($inserttype eq 'reservedwords')
-        {
-            if (!exists($DynXMLParsers->{reserveredwords}) || 
-                !defined($DynXMLParsers->{reservedwords}))
-            {
-                $DynXMLParsers->{reservedwords} = new
MySQL::DynXML::ReservedWords($dynxml,$parser_options);
-            }
-            $string =
$DynXMLParsers->{reservedwords}->generate($parser_options,$item);
-        }
-        elsif ($inserttype eq 'changelog')
-        {
-            if (!exists($DynXMLParsers->{changelog}) || 
-                !defined($DynXMLParsers->{changelog}))
-            {
-                $DynXMLParsers->{changelog} = new
MySQL::DynXML::Changelog($dynxml,$parser_options);
-            }
-            $string = $DynXMLParsers->{changelog}->generate($parser_options,$item);
-        }
-        elsif ($inserttype eq 'openbugs')
-        {
-            if (!exists($DynXMLParsers->{openbugs}) || 
-                !defined($DynXMLParsers->{openbugs}))
-            {
-                $parser_options->{usecustombasedir} = 'open-bugs';
-                $DynXMLParsers->{changelog} = new
MySQL::DynXML::Changelog($dynxml,$parser_options);
-            }
-            $string = $DynXMLParsers->{changelog}->generate($parser_options,$item);
-        }
-        else
-        {
-            print STDERR "$0: WARNING: Dynamic datatype $inserttype not recognized\n";
-        }
-        my $qitem = quotemeta($item);
-        
-        $inplacefile =~ s/$qitem/$string/g;
-    }
-
-    my @lines;
-    push @lines,sprintf('<remark role="dependency"
condition="%s"/>',$options->{infile});
-
-    for my $dynxmltype (keys %{$DynXMLParsers})
-    {
-        push @lines,$DynXMLParsers->{$dynxmltype}->generate_dependencies()
-            if (defined($DynXMLParsers->{$dynxmltype}));
-    }
-
-    my $deptext = join("\n",@lines);
-    $inplacefile =~ s{<remark role="dynamic-dependency-list"/>}{$deptext};
-
-    $fh = new IO::File($options->{outfile},'w') or die "Error: Can't open destination
file ($options->{outfile}: $!\n";    
-    binmode($fh,':utf8');
-    print $fh $inplacefile;
-    $fh->close();
-}
-
-sub deep_copy {
-    my $this = shift;
-    if (not ref $this) {
-        $this;
-    } elsif (ref $this eq 'ARRAY') {
-        [map deep_copy($_), @$this];
-    } elsif (ref $this eq 'HASH') {
-        +{map { $_ => deep_copy($this->{$_}) } keys %$this};
-    } else { die 'what type is $_?' }
-}