From: Date: November 21 2007 6:48pm Subject: svn commit - mysqldoc@docsrva: r8819 - in trunk/tools: . MySQL MySQL/XML List-Archive: http://lists.mysql.com/commits/38225 Message-Id: <200711211748.lALHmThr011920@docsrva.mysql.com> Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Author: mcbrown Date: 2007-11-21 18:48:29 +0100 (Wed, 21 Nov 2007) New Revision: 8819 Log: Some changes related to the arbitrary parser: - Moved the parsing of dynxml source into a string format and moved the process into a module so that we can call it from elsewhere - Updated the parser to be a front end to the module - Some minor tweaks to the arbitrary parser - Updated the arbitrary parser so that it uses the DynXML parser to parse content during the arbitrary parsing process - Some other fixes to the parser to handle some recent ID map and other changes - Update to ID map to expose the relative prefix at ID level (required by the Arby parser) Added: trunk/tools/MySQL/DynXMLParse.pm Modified: trunk/tools/MySQL/IDMap.pm trunk/tools/MySQL/XML/Arbitrary.pm trunk/tools/arbitrary-parser.pl trunk/tools/dynxml-parser.pl Added: trunk/tools/MySQL/DynXMLParse.pm =================================================================== --- trunk/tools/MySQL/DynXMLParse.pm (rev 0) +++ trunk/tools/MySQL/DynXMLParse.pm 2007-11-21 17:48:29 UTC (rev 8819) Changed blocks: 1, Lines Added: 145, Lines Deleted: 0; 4612 bytes @@ -0,0 +1,145 @@ +package MySQL::DynXMLParse; + +# DynXMLParse.pm +# +# generic parser for all dynamic document elements + +# Martin MC Brown +# mc@stripped +# 2007-22-21 + +use strict; +use warnings; + +# Load the MySQL specific modules + +use lib qw(./tools/ ../tools ../../tools ../../../trunk/tools); +use MySQL::DynXML; + +# Dynamic doc types to parse + +use MySQL::DynXML::Optvar; +use MySQL::DynXML::Opfuncs; +use MySQL::DynXML::ReservedWords; +use MySQL::DynXML::Changelog; + +# Load the local modules + +use IO::File; +use vars qw(@ISA @EXPORT); +require Exporter; +@ISA = qw(Exporter); + +@EXPORT = qw(inplace_parse); + +sub inplace_parse +{ + my ($inplacefile,$options) = @_; + +# Initialize the list of known DynXML types, but don't instantiate the related class + + my $DynXMLParsers = { + optvar => undef, + opfuncs => undef, + reservedwords => undef, + changelog => undef, + openbugs => undef, + }; + + my $dynxml = new MySQL::DynXML(); + + my $sources = {}; + + my @matches = ($inplacefile =~ m{()}msg); + my @titlematches = ($inplacefile =~ m{()}msg); + push @matches,@titlematches; + + my $tdcache = {}; + + foreach my $item (@matches) + { + my $parser_options = deep_copy($options); + my ($inserttype,$tabletype) = ($item =~ m/condition="dynamic:(.*?):(.*?)"/); + + my $string = ''; + + if ($inserttype eq 'optvar') + { + if (!exists($DynXMLParsers->{optvar}) || + !defined($DynXMLParsers->{optvar})) + { + $DynXMLParsers->{optvar} = new MySQL::DynXML::Optvar($dynxml,$parser_options); + } + $string = $DynXMLParsers->{optvar}->generate($parser_options,$item); + } + elsif ($inserttype eq 'opfuncs') + { + if (!exists($DynXMLParsers->{opfuncs}) || + !defined($DynXMLParsers->{opfuncs})) + { + $DynXMLParsers->{opfuncs} = new MySQL::DynXML::Opfuncs($dynxml,$parser_options); + } + $string = $DynXMLParsers->{opfuncs}->generate($parser_options,$item); + } + elsif ($inserttype eq 'reservedwords') + { + if (!exists($DynXMLParsers->{reserveredwords}) || + !defined($DynXMLParsers->{reservedwords})) + { + $DynXMLParsers->{reservedwords} = new MySQL::DynXML::ReservedWords($dynxml,$parser_options); + } + $string = $DynXMLParsers->{reservedwords}->generate($parser_options,$item); + } + elsif ($inserttype eq 'changelog') + { + if (!exists($DynXMLParsers->{changelog}) || + !defined($DynXMLParsers->{changelog})) + { + $DynXMLParsers->{changelog} = new MySQL::DynXML::Changelog($dynxml,$parser_options); + } + $string = $DynXMLParsers->{changelog}->generate($parser_options,$item); + } + elsif ($inserttype eq 'openbugs') + { + if (!exists($DynXMLParsers->{openbugs}) || + !defined($DynXMLParsers->{openbugs})) + { + $parser_options->{usecustombasedir} = 'open-bugs'; + $DynXMLParsers->{changelog} = new MySQL::DynXML::Changelog($dynxml,$parser_options); + } + $string = $DynXMLParsers->{changelog}->generate($parser_options,$item); + } + else + { + print STDERR "$0: WARNING: Dynamic datatype $inserttype not recognized\n"; + } + my $qitem = quotemeta($item); + + $inplacefile =~ s/$qitem/$string/g; + } + + my @lines; + push @lines,sprintf('<remark role="dependency" condition="%s"/>',$options->{infile}); + + for my $dynxmltype (keys %{$DynXMLParsers}) + { + push @lines,$DynXMLParsers->{$dynxmltype}->generate_dependencies() + if (defined($DynXMLParsers->{$dynxmltype})); + } + + my $deptext = join("\n",@lines); + $inplacefile =~ s{<remark role="dynamic-dependency-list"/>}{$deptext}; + + return($inplacefile); +} + +sub deep_copy { + my $this = shift; + if (not ref $this) { + $this; + } elsif (ref $this eq 'ARRAY') { + [map deep_copy($_), @$this]; + } elsif (ref $this eq 'HASH') { + +{map { $_ => deep_copy($this->{$_}) } keys %$this}; + } else { die 'what type is $_?' } +} Modified: trunk/tools/MySQL/IDMap.pm =================================================================== --- trunk/tools/MySQL/IDMap.pm 2007-11-21 17:23:53 UTC (rev 8818) +++ trunk/tools/MySQL/IDMap.pm 2007-11-21 17:48:29 UTC (rev 8819) Changed blocks: 1, Lines Added: 1, Lines Deleted: 0; 522 bytes @@ -560,6 +560,7 @@ adds => [$prefix], fullpath => $self->{fileinfo}->{fullpath}, relpath => $self->{fileinfo}->{relpath}, + relprefix => $self->{fileinfo}->{relprefix}, }; if (exists($self->{byid}->{$id})) Modified: trunk/tools/MySQL/XML/Arbitrary.pm =================================================================== --- trunk/tools/MySQL/XML/Arbitrary.pm 2007-11-21 17:23:53 UTC (rev 8818) +++ trunk/tools/MySQL/XML/Arbitrary.pm 2007-11-21 17:48:29 UTC (rev 8819) Changed blocks: 5, Lines Added: 21, Lines Deleted: 9; 2601 bytes @@ -1,7 +1,9 @@ package MySQL::XML::Arbitrary; + # Load the standard MySQL Libraries use MySQL; +use MySQL::DynXMLParse; # Ensure we have enabled error checking @@ -30,6 +32,7 @@ 'idmap' => $options->{idmap}, 'id' => 'arbitrary', 'debug' => $options->{debug}, + 'srcdir' => $options->{srcdir}, }, $class; } @@ -369,15 +372,20 @@ foreach (split(/[\r\n]+/,$filetxt)) { $incomment = 1 if ($_ =~ m/<!--/); - $incomment = 0 if ($incomment && ($_ =~ /-->/)); - next if ($incomment); + if ($incomment && ($_ =~ m/-->/)) + { + $incomment = 0; + next; + } + next if ($incomment > 0); + if ($_ =~ m{SYSTEM\s*?"(.*?)"}) { my $loc = $1; if (!-e $loc) { # Prepend the source file prefix to the file: - my $newloc = "$self->{idmap}->{byid}->{$spec->{id}}->{prefix}/$loc"; + my $newloc = "$self->{idmap}->{byid}->{$spec->{id}}->{relprefix}/$loc"; $_ =~ s{SYSTEM\s*?".*?"}{SYSTEM "$newloc"}; } } @@ -388,7 +396,7 @@ if (!-e $loc) { # Prepend the source file prefix to the file: - my $newloc = "$self->{idmap}->{byid}->{$spec->{id}}->{prefix}/$loc"; + my $newloc = "$self->{idmap}->{byid}->{$spec->{id}}->{relprefix}/$loc"; $_ =~ s{$srctype=".*?"}{$srctype="$newloc"}; } } @@ -547,17 +555,21 @@ { die("PANIC: Don't seem to have any lines to write out\n"); } + + my $inplacefile = join("\n",@{$spec->{lines}}); - open(DEST,">$destfile") or die "Couldn't open the section file ($destfile: $!)\n"; - print DEST (join("\n",@{$spec->{lines}})); - close(DEST); + my $fh = new IO::File($destfile,'w') or die "Couldn't open the section file ($destfile: $!)\n"; + binmode($fh,':utf8'); + print $fh (inplace_parse($inplacefile, + {srcdir => $self->{srcdir}, + infile => $destfile, + })); + $fh->close(); $self->print_error("********\n********\n*******\n\n"); return sprintf('<xi:include href="%s" xmlns:xi="http://www.w3.org/2001/XInclude"/>', $destfile); - - } 1; Modified: trunk/tools/arbitrary-parser.pl =================================================================== --- trunk/tools/arbitrary-parser.pl 2007-11-21 17:23:53 UTC (rev 8818) +++ trunk/tools/arbitrary-parser.pl 2007-11-21 17:48:29 UTC (rev 8819) Changed blocks: 3, Lines Added: 14, Lines Deleted: 5; 1860 bytes @@ -29,16 +29,17 @@ die "ERROR: ID maps must be version 1 or later\nPlease change to trunk and 'make clean', then return here and 'make idmap.refs'"; } -my ($help,$debug) = (0,0); +my ($help,$debug,$srcdir) = (0,0,undef); GetOptions("help" => \$help, "debug" => \$debug, + "srcdir=s" => \$srcdir, ); -if ($help) +if ($help or !defined($srcdir)) { print <<EOF; -genarbelements.pl [--debug] arbspecfile xmlembedfile +genarbelements.pl [--debug] --srcdir=DYNXML-BASE arbspecfile xmlembedfile EOF exit 0; } @@ -47,7 +48,9 @@ my $template = shift or die "You must supply the name of the template to process"; my $my_handler = MySQL::XML::Arbitrary->new({idmap => $idmap, - debug => $debug}); + debug => $debug, + srcdir => $srcdir, + }); XML::Parser::PerlSAX->new->parse(Source => { SystemId => $file}, Handler => $my_handler); @@ -57,8 +60,14 @@ close(TEMPLATE); foreach my $id (keys %{$my_handler->{fragments}}) { + print STDERR "Replacing fragment $id in template\n" if ($debug); + my $inclist = join("\n",@{$my_handler->{fragments}->{$id}}); - $templatesrc =~ s/<!--SRCFILE $id-->/$inclist/g; + my $changes = ($templatesrc =~ s/<!--SRCFILE $id-->/$inclist/g); + if ($changes == 0) + { + print STDERR "Couldn't replace the fragment $id\n"; + } } open(XMLFILE,">$template.final") or die "Couldn't create XML file\n"; print XMLFILE $templatesrc; Modified: trunk/tools/dynxml-parser.pl =================================================================== --- trunk/tools/dynxml-parser.pl 2007-11-21 17:23:53 UTC (rev 8818) +++ trunk/tools/dynxml-parser.pl 2007-11-21 17:48:29 UTC (rev 8819) Changed blocks: 4, Lines Added: 11, Lines Deleted: 127; 5497 bytes @@ -2,7 +2,7 @@ # dynxml-parser.pl # -# generic parser for all dynamic document elements +# Interface to the dynxml parser # Martin MC Brown # mc@stripped @@ -15,21 +15,13 @@ use lib qw(./tools/ ../tools ../../tools ../../../trunk/tools); use MySQL::DynXML; +use MySQL::DynXMLParse; -# Dynamic doc types to parse - -use MySQL::DynXML::Optvar; -use MySQL::DynXML::Opfuncs; -use MySQL::DynXML::ReservedWords; -use MySQL::DynXML::Changelog; - # Load the local modules use Getopt::Long; use IO::File; -my $dynxml = new MySQL::DynXML(); - my ($opt_infile,$opt_outfile,$opt_srcdir) = (undef,undef,undef); GetOptions("infile=s" => \$opt_infile, @@ -48,17 +40,17 @@ srcdir => $opt_srcdir, }; -# Initialize the list of known DynXML types, but don't instantiate the related class +my $fh = new IO::File($options->{infile}) or die "Error: Can't open source file ($options->{infile}): $!\n"; +binmode($fh,':utf8'); +my $inplacefile = join('',<$fh>); +$fh->close(); -my $DynXMLParsers = { - optvar => undef, - opfuncs => undef, - reservedwords => undef, - changelog => undef, - openbugs => undef, -}; +my $finaltext = inplace_parse($inplacefile,$options); -inplace_parse($options); +$fh = new IO::File($options->{outfile},'w') or die "Error: Can't open destination file ($options->{outfile}: $!\n"; +binmode($fh,':utf8'); +print $fh $finaltext; +$fh->close(); sub show_help { @@ -85,111 +77,3 @@ exit 0; } } - -sub inplace_parse -{ - my ($options) = @_; - - my $fh = new IO::File($options->{infile}) or die "Error: Can't open source file ($options->{infile}): $!\n"; - binmode($fh,':utf8'); - my $inplacefile = join('',<$fh>); - $fh->close(); - - my $sources = {}; - - my @matches = ($inplacefile =~ m{(<para condition="dynamic:.*?"\s+role=".*?"/>)}msg); - my @titlematches = ($inplacefile =~ m{(<title condition="dynamic:.*?"\s+role=".*?"/>)}msg); - push @matches,@titlematches; - - my $tdcache = {}; - - foreach my $item (@matches) - { - my $parser_options = deep_copy($options); - my ($inserttype,$tabletype) = ($item =~ m/condition="dynamic:(.*?):(.*?)"/); - - my $string = ''; - - if ($inserttype eq 'optvar') - { - if (!exists($DynXMLParsers->{optvar}) || - !defined($DynXMLParsers->{optvar})) - { - $DynXMLParsers->{optvar} = new MySQL::DynXML::Optvar($dynxml,$parser_options); - } - $string = $DynXMLParsers->{optvar}->generate($parser_options,$item); - } - elsif ($inserttype eq 'opfuncs') - { - if (!exists($DynXMLParsers->{opfuncs}) || - !defined($DynXMLParsers->{opfuncs})) - { - $DynXMLParsers->{opfuncs} = new MySQL::DynXML::Opfuncs($dynxml,$parser_options); - } - $string = $DynXMLParsers->{opfuncs}->generate($parser_options,$item); - } - elsif ($inserttype eq 'reservedwords') - { - if (!exists($DynXMLParsers->{reserveredwords}) || - !defined($DynXMLParsers->{reservedwords})) - { - $DynXMLParsers->{reservedwords} = new MySQL::DynXML::ReservedWords($dynxml,$parser_options); - } - $string = $DynXMLParsers->{reservedwords}->generate($parser_options,$item); - } - elsif ($inserttype eq 'changelog') - { - if (!exists($DynXMLParsers->{changelog}) || - !defined($DynXMLParsers->{changelog})) - { - $DynXMLParsers->{changelog} = new MySQL::DynXML::Changelog($dynxml,$parser_options); - } - $string = $DynXMLParsers->{changelog}->generate($parser_options,$item); - } - elsif ($inserttype eq 'openbugs') - { - if (!exists($DynXMLParsers->{openbugs}) || - !defined($DynXMLParsers->{openbugs})) - { - $parser_options->{usecustombasedir} = 'open-bugs'; - $DynXMLParsers->{changelog} = new MySQL::DynXML::Changelog($dynxml,$parser_options); - } - $string = $DynXMLParsers->{changelog}->generate($parser_options,$item); - } - else - { - print STDERR "$0: WARNING: Dynamic datatype $inserttype not recognized\n"; - } - my $qitem = quotemeta($item); - - $inplacefile =~ s/$qitem/$string/g; - } - - my @lines; - push @lines,sprintf('<remark role="dependency" condition="%s"/>',$options->{infile}); - - for my $dynxmltype (keys %{$DynXMLParsers}) - { - push @lines,$DynXMLParsers->{$dynxmltype}->generate_dependencies() - if (defined($DynXMLParsers->{$dynxmltype})); - } - - my $deptext = join("\n",@lines); - $inplacefile =~ s{<remark role="dynamic-dependency-list"/>}{$deptext}; - - $fh = new IO::File($options->{outfile},'w') or die "Error: Can't open destination file ($options->{outfile}: $!\n"; - binmode($fh,':utf8'); - print $fh $inplacefile; - $fh->close(); -} - -sub deep_copy { - my $this = shift; - if (not ref $this) { - $this; - } elsif (ref $this eq 'ARRAY') { - [map deep_copy($_), @$this]; - } elsif (ref $this eq 'HASH') { - +{map { $_ => deep_copy($this->{$_}) } keys %$this}; - } else { die 'what type is $_?' } -}