| List: | Commits | « Previous MessageNext Message » | |
| From: | mcbrown | Date: | October 17 2007 5:59pm |
| Subject: | svn commit - mysqldoc@docsrva: r8138 - in trunk/tools: . MySQL/DynXML | ||
| View as plain text | |||
Author: mcbrown Date: 2007-10-17 17:59:46 +0200 (Wed, 17 Oct 2007) New Revision: 8138 Log: Committing some final functionality into the Dynamic Changelog = Adding some in-line checking of version information during parsing - Adding specific higlight tags and text - Changing the way sort order is determined (using a fixed list for hightlight tags) - Added support for custom version entry messages - use ver="5.1." - in the <message> tag - Added CVE specific tag and processing - Made sure that an entry that doesn't start with <para> still gets it's highlight - Made sure that an entry that doesn't end with <para> still gets it's bug/CVE info - Adding support for the new elements (custom message, introducedby,bugseealso,regressionof,identifiedin) to parser Improved the conversion script Modified: trunk/tools/MySQL/DynXML/Changelog.pm trunk/tools/MySQL/DynXML/ChangelogParser.pm trunk/tools/MySQL/DynXML/ChangelogVersionParser.pm trunk/tools/convert-newsfile-tochangelog.pl Modified: trunk/tools/MySQL/DynXML/Changelog.pm =================================================================== --- trunk/tools/MySQL/DynXML/Changelog.pm 2007-10-17 15:42:48 UTC (rev 8137) +++ trunk/tools/MySQL/DynXML/Changelog.pm 2007-10-17 15:59:46 UTC (rev 8138) Changed blocks: 3, Lines Added: 73, Lines Deleted: 10; 4352 bytes @@ -25,6 +25,23 @@ my $class = ref($self) || $self; my $dynxmlcore = shift; my $options = shift; + + my $highlighttext = { + 'api' => 'API', + 'clusterapi' => 'Cluster API', + 'clusterreplication' => 'Cluster Replication', + 'diskdata' => 'Disk Data', + 'importantchange' => 'Important Change', + 'importantnote' => 'Important Note', + 'incompatiblechange' => 'Incompatible Change', + 'packaging' => 'Packaging', + 'partitioning' => 'Partitioning', + 'pluginapi' => 'Plugin API', + 'replication' => 'Replication', + 'securityenhancement' => 'Security Enhancement', + 'securityfix' => 'Security Fix', + 'cluster' => 'MySQL Cluster', + }; return bless { dynxmlcore => $dynxmlcore, @@ -33,6 +50,7 @@ versioncache => {}, sources => {}, dependencyreq => 0, + highlighttext => $highlighttext, }, $class; } @@ -213,22 +231,67 @@ } $counter++; - my $message = $logentries->{$md5}->{message}; + my $message = ''; + + + if (exists($logentries->{$md5}->{messagebyversion}->{$ver}) && + defined($logentries->{$md5}->{messagebyversion}->{$ver})) + { + $message = $logentries->{$md5}->{messagebyversion}->{$ver}; + } + elsif (exists($logentries->{$md5}->{message}) && + defined($logentries->{$md5}->{message})) + { + $message = $logentries->{$md5}->{message}; + } + else + { + print STDERR ("FATAL: Couldn't find a logentry message for version $ver (bug(s): ", + join(', ',@{$logentries->{$md5}->{bugs}}),")\n"); + exit(1); + } - if (scalar @{$logentries->{$md5}->{bugs}}) + if ((exists($logentries->{$md5}->{bugs}) && scalar @{$logentries->{$md5}->{bugs}}) || + (exists($logentries->{$md5}->{cve}) && scalar @{$logentries->{$md5}->{cve}})) { - my $bugslist = sprintf('(%s)',join(', ',map { $_ = "Bug \#$_" } @{$logentries->{$md5}->{bugs}})); - $message =~ s|(.*)(</para>)\s*$|$1 $bugslist$2|; + my @bugcve; + + push(@bugcve,map { $_ = "Bug \#$_" } @{$logentries->{$md5}->{bugs}}) + if ((exists($logentries->{$md5}->{bugs}) && scalar @{$logentries->{$md5}->{bugs}})); + + push(@bugcve,@{$logentries->{$md5}->{cve}}) + if (exists($logentries->{$md5}->{cve}) && scalar @{$logentries->{$md5}->{cve}}); + + my $bugslist = sprintf('(%s)',join(', ',@bugcve)); + + if ( $message =~ m!</para>\s*$!) + { + $message =~ s|(.*)(</para>)\s*$|$1 $bugslist$2|; + } + else + { + $message .= sprintf('<para>%s</para>',$bugslist); + } } - if (exists($logentries->{$md5}->{tags}->{incompatiblechange})) + foreach my $highlight (keys %{$logentries->{$md5}->{tagsbytype}->{highlight}}) { - $message =~ s#^(\s+<para>)#$1<emphasis role="bold">Incompatible change:</emphasis>#; + my $ht = join(', ',keys %{$logentries->{$md5}->{tagsbytype}->{highlight}}); + my $text = $self->{highlighttext}->{$highlight}; + + if ($message =~ m|^\s*<para>|) + { + $message =~ s!^(\s*<para>)!$1<emphasis role="bold">$text</emphasis>:!; + } + else + { + $message = sprintf('<para><emphasis role="bold">%s</emphasis></para>%s', + $text, + $message); + } + } - if (exists($logentries->{$md5}->{tags}->{securityfix})) - { - $message =~ s#^(\s+<para>)#$1<emphasis role="bold">Security Fix:</emphasis>#; - } + print $outfile ('<listitem>',$message,'</listitem>',"\n"); } print $outfile "</itemizedlist>\n" if ($counter); Modified: trunk/tools/MySQL/DynXML/ChangelogParser.pm =================================================================== --- trunk/tools/MySQL/DynXML/ChangelogParser.pm 2007-10-17 15:42:48 UTC (rev 8137) +++ trunk/tools/MySQL/DynXML/ChangelogParser.pm 2007-10-17 15:59:46 UTC (rev 8138) Changed blocks: 6, Lines Added: 83, Lines Deleted: 13; 5028 bytes @@ -19,15 +19,36 @@ my $self = shift; my $class = ref($self) || $self; + my $highlight_base = 100000; + + my $highlight_precedence = { + 'api' => 200, + 'clusterapi' => 225, + 'clusterreplication' => 250, + 'diskdata' => 275, + 'importantchange' => 425, + 'importantnote' => 400, + 'incompatiblechange' => 450, + 'packaging' => 360, + 'partitioning' => 350, + 'pluginapi' => 325, + 'replication' => 300, + 'securityenhancement' => 475, + 'securityfix' => 500, + 'cluster' => 375, + }; + return bless { - 'logentriesbyver' => {}, - 'currententry' => {}, - 'currenttext' => [], - 'remapxmldata' => 0, - 'savecdata' => 0, - 'counter' => 0, - 'custom' => {}, - }, $class; + 'highlightbase' => $highlight_base, + 'highlightprecedence' => $highlight_precedence, + 'logentriesbyver' => {}, + 'currententry' => {}, + 'currenttext' => [], + 'remapxmldata' => 0, + 'savecdata' => 0, + 'counter' => 0, + 'custom' => {}, + }, $class; } sub start_element @@ -37,7 +58,8 @@ if ($element->{Name} eq 'logentry') { $self->{currententry} = {type => $element->{Attributes}->{entrytype}, - message => '', + message => undef, + messagebyversion => {}, versions => {}, tags => {}, bugs => [], @@ -57,6 +79,13 @@ { $self->{savecdata} = 1; $self->{remapxmldata} = 1; + $self->{msgversion} = undef; + + if (exists($element->{Attributes}->{ver})) + { + $self->{msgversion} = $element->{Attributes}->{ver}; + } + return; } elsif ($element->{Name} eq 'version') @@ -79,6 +108,26 @@ { push @{$self->{currententry}->{bugs}},$element->{Attributes}->{bugid}; } + elsif ($element->{Name} eq 'introducedby') + { + push @{$self->{currententry}->{introducedby}},$element->{Attributes}->{bugid}; + } + elsif ($element->{Name} eq 'seealsobug') + { + push @{$self->{currententry}->{bugseealso}},$element->{Attributes}->{bugid}; + } + elsif ($element->{Name} eq 'regressionof') + { + push @{$self->{currententry}->{regressionof}},$element->{Attributes}->{bugid}; + } + elsif ($element->{Name} eq 'identifiedin') + { + push @{$self->{currententry}->{identifiedin}},$element->{Attributes}->{ver}; + } + elsif ($element->{Name} eq 'cve') + { + push @{$self->{currententry}->{cve}},$element->{Attributes}->{ref}; + } if ($self->{remapxmldata}) { @@ -103,9 +152,22 @@ $rank += 200000000 if ($self->{currententry}->{type} eq 'feature'); $rank += 100000000 if ($self->{currententry}->{type} eq 'bug'); $rank += 90000000 if ($self->{currententry}->{type} eq 'other'); - $rank += 50000000 if (exists($self->{currententry}->{tags}->{securityfix})); - $rank += 25000000 if (exists($self->{currententry}->{tags}->{incompatiblechange})); - $rank += 12500000 if (exists($self->{currententry}->{tags}->{performance})); + + foreach my $ht (keys %{$self->{highlightprecedence}}) + { + if (exists($self->{currententry}->{tagsbytype}->{highlight}->{$ht})) + { + if (exists($self->{highlightprecedence}->{$ht})) + { + $rank += ($self->{highlightprecedence}->{$ht}*$self->{highlightbase}); + } + else + { + print STDERR "WARNING: Couldn't find a highlight order for $ht\n"; + } + } + } + if (exists($self->{currententry}->{bugs}->[0])) { $rank += $self->{currententry}->{bugs}->[0]; @@ -132,11 +194,19 @@ } if ($element->{Name} eq 'message') { - $self->{currententry}->{message} = join('',@{$self->{currenttext}}); + if (exists($self->{msgversion}) && defined($self->{msgversion})) + { + $self->{currententry}->{messagebyversion}->{$self->{msgversion}} = join('',@{$self->{currenttext}}); + } + else + { + $self->{currententry}->{message} = join('',@{$self->{currenttext}}); + } $self->{savecdata} = 0; $self->{remapxmldata} = 0; $self->{currenttext} = [], + $self->{msgversion} = undef; return; } if ($element->{Name} eq 'values') Modified: trunk/tools/MySQL/DynXML/ChangelogVersionParser.pm =================================================================== --- trunk/tools/MySQL/DynXML/ChangelogVersionParser.pm 2007-10-17 15:42:48 UTC (rev 8137) +++ trunk/tools/MySQL/DynXML/ChangelogVersionParser.pm 2007-10-17 15:59:46 UTC (rev 8138) Changed blocks: 1, Lines Added: 15, Lines Deleted: 0; 1101 bytes @@ -37,6 +37,21 @@ } elsif ($element->{Name} eq 'versionentry') { + my $reqcount = 0; + + if (!exists($element->{Attributes}->{ver})) + { + print STDERR "FATAL: There must be a version attached to a version entry\n"; + exit(1); + } + if (!exists($element->{Attributes}->{reldate})) + { + print STDERR ("FATAL: There must be a reldate attached to a version entry ", + $element->{Attributes}->{ver}, + "\n"); + exit(1); + } + $self->{currententry} = {version => $element->{Attributes}->{ver}, fullversion => $element->{Attributes}->{ver}, reldate => $element->{Attributes}->{reldate}, Modified: trunk/tools/convert-newsfile-tochangelog.pl =================================================================== --- trunk/tools/convert-newsfile-tochangelog.pl 2007-10-17 15:42:48 UTC (rev 8137) +++ trunk/tools/convert-newsfile-tochangelog.pl 2007-10-17 15:59:46 UTC (rev 8138) Changed blocks: 8, Lines Added: 257, Lines Deleted: 23; 13274 bytes @@ -45,6 +45,62 @@ exit(1); } +my $highlights = { + 'incompatible change' => 'incompatiblechange', + 'security fix' => 'securityfix', + 'mysql cluster replication' => 'clusterreplication', + 'cluster replication' => 'clusterreplication', + 'ndb cluster' => 'cluster', + 'apis' => 'api', + 'cluster apis' => 'clusterapi', + 'ndb api' => 'clusterapi', + 'disk data' => 'diskdata', + 'replication' => 'replication', + 'important change' => 'importantchange', + 'important note' => 'importantnote', + 'packaging changes' => 'packaging', + 'partitioning' => 'partitioning', + 'plugin api' => 'pluginapi', + 'security fix' => 'securityfix', + 'security enhancement' => 'securityenhancement', +}; + +my $mantagmatches = { + 'partitioning' => 'partitioning', + 'partitioned' => 'partitioning', + 'replication' => 'replication', + 'replicating' => 'replication', + 'binary log' => 'binary log', + 'stored routine' => 'stored routine', + 'trigger' => 'triggers', + 'stored procedure' => 'stored procedure', + 'character set' => 'character sets', + 'AIX' => 'AIX', + 'Solaris' => 'Solaris', + 'Windows' => 'Windows', + 'collation' => 'collation', + 'Mac OS X' => 'Mac OS X', + 'HP-UX' => 'HP-UX', + 'transaction' => 'transactions', + 'yaSSL' => 'SSL', + 'SSL' => 'SSL', + 'Unicode' => 'character sets', + 'Cygwin' => 'Cygwin', + 'subquery' => 'subqueries', + 'subqueries' => 'subqueries', + 'event scheduler' => 'events', + 'full-text' => 'full text', + 'cluster' => 'cluster', + 'replicate' => 'replication', + 'slave server' => 'replication', + 'master server' => 'replication', + 'statement-based' => 'statement-based logging', + 'statement based' => 'statement-based logging', + 'row-based' => 'row-based logging', + 'row based' => 'row-based logging', +}; + + my $dynxml = MySQL::DynXML->new(); my $logentries = {}; my $bybugid = {}; @@ -174,19 +230,21 @@ my $entrystring = $bug->toString; my $bugseq = ''; if ($entrystring =~ m/\((Bug\s+\#\d+.*?)\)/ms) +# if ($entrystring =~ m/\(.*?(Bug\s+\#\d+.*?).*?\)/ms) { - $bugseq = $1; + $bugseq = $1; } + print STDERR "Found $bugseq\n"; + my (@bugs) = (()); if ($bugseq) { (@bugs) = ($bugseq =~ m/Bug\s+\#(\d+)/msg); } - my $incompatible = 0; - my $securityfix = 0; - + my $highlighttags = []; + # Extract the child elements and build the block that will be inserted # into the <message> block in the changelog XML file @@ -200,28 +258,164 @@ if ($bugseq) { - $message =~ s/\($bugseq\)//g; + $message =~ s/\(\s*$bugseq\s*\)//g; } # Identify and remove specific tags as these will be tracked differently - if ($message =~ m{(<emphasis[^>]*?>Incompatible change</emphasis>:?)}i) + foreach my $highlight (keys %{$highlights}) { - $message =~ s/$1//g; - $incompatible = 1; + if ($message =~ m{(<emphasis[^>]*?>($highlight)</emphasis>:?)}i) + { + my $srctext = $2; + $message =~ s/$1//g; + if (exists($highlights->{lc($srctext)})) + { + push @{$highlighttags},$highlights->{lc($srctext)}; + } + else + { + print STDERR "WARN: Couldn't find a highlight conversion for $srctext\n"; + } + } + if ($message =~ m{(<literal>($highlight)</literal>:?)}i) + { + my $srctext = $2; + $message =~ s/$1//g; + if (exists($highlights->{lc($srctext)})) + { + push @{$highlighttags},$highlights->{lc($srctext)}; + } + else + { + print STDERR "WARN: Couldn't find a highlight conversion for $srctext\n"; + } + } } - if ($message =~ m{(<emphasis[^>]*?>Security Fix</emphasis>:?)}i) + +# Extract potential manual tags from the log message + +# Look for options + my $manualtags = {}; + + my $msgcopy = $message; + $msgcopy =~ s/\s+/ /msg; + + my @options = (); + + @options = ($msgcopy =~ m!<option>(.*?)</option>!g); + foreach my $option (@options) { - $message =~ s/$1//g; - $securityfix = 1; + $option =~ s/^-+//g; + $option =~ s/=.*//g; + $manualtags->{$option} = 1; } +# For commands, we strip any potential value + + @options = ($msgcopy =~ m!<command>(.*?)</command>!g); + foreach my $option (@options) + { + $option =~ s/ .*//; + $manualtags->{$option} = 1; + } + +# For literal, we try and determine if it's all upper case, else it's ignored + + @options = ($msgcopy =~ m!<literal>(.*?)</literal>!g); + foreach my $option (@options) + { + if ($option =~ m/^[A-Z _()]{3,}$/) + { + $option =~ s/\(\)$//g; + $manualtags->{$option} = 1; + } + if ($option =~ m/^[a-z_()]{3,}$/i) + { + $option =~ s/\(\)$//g; + $manualtags->{$option} = 1; + } + if ($option =~ m/^(innodb|myisam|example|federated|bdb|merge|arvhive|memory|heap|csv|blackhole)$/i) + { + $manualtags->{$option} = 1; + } + } + +# Functions + @options = ($msgcopy =~ m!<function>(.*?)</function>!g); + foreach my $option (@options) + { + if ($option =~ m/^[A-Za-z _()]+$/) + { + $option =~ s/\(\)$//g; + $manualtags->{$option} = 1; + } + } + +# Embedded functions + @options = ($msgcopy =~ m!([A-Z_]+\()!g); + foreach my $option (@options) + { + if ($option =~ m/^[A-Z_]{3,}$/) + { + $option =~ s/\($//g; + $manualtags->{$option} = 1; + } + } + +# Filenames + @options = ($msgcopy =~ m!<filename>(.*?)</filename>!g); + foreach my $option (@options) + { + if ($option =~ m/^[A-Z._()]{3,}$/i) + { + $manualtags->{$option} = 1; + } + } + +# Classes + @options = ($msgcopy =~ m!<class>(.*?)</class>!g); + foreach my $option (@options) + { + if ($option =~ m/^[A-Z _()]{3,}$/i) + { + $option =~ s/\(\)$//g; + $manualtags->{$option} = 1; + } + } + +# Extract CVE entries: + + my @cve = ($msgcopy =~ m/(CVE[0-9-]+)/g); + + foreach my $cve (@cve) + { + $message =~ s/$cve//g; + } + +# Process the manual tag search matches + + foreach my $mantag (keys %{$mantagmatches}) + { + if ($msgcopy =~ m/$mantag/i) + { + $manualtags->{$mantagmatches->{$mantag}} = 1; + } + } + + +# Finish up the formatting of the entry + my $classtags = {}; if (defined($outertag)) { $classtags->{$outertag} = 1; } +# Final cleanup on the message + + $message =~ s!\([\s,]*\)\s+</para>!</para>!msg; + # Calculate and MD5 string for the entry, to use for comparison # and matching across multiple versions @@ -246,9 +440,11 @@ }, message => $message, bugs => {}, + cve => \@cve, class => $currclass, - incompatiblechange => $incompatible, - securityfix => $securityfix}; + highlighttags => $highlighttags, + manualtags => $manualtags, + }; # Add each bug individually @@ -258,9 +454,7 @@ $bybugid->{$bug} = $md5; } } - } - } } } @@ -268,7 +462,6 @@ { print STDERR "Ignoring section $ref\n" if ($opt_debug); } - } } @@ -281,24 +474,63 @@ <changelog> EOF + my $counter = 0; +my $tagcounter = 0; + foreach my $id (keys %{$logentries}) { + $counter++; + my $entry = $logentries->{$id}; printf $dest ('<logentry entrytype="%s">',$entry->{class}); - if ($entry->{incompatiblechange} || $entry->{securityfix}) + + if (scalar keys %{$entry->{manualtags}}) { - print $dest ('<tags>'); - print $dest '<highlight type="incompatiblechange"/>' if ($entry->{incompatiblechange}); - print $dest '<highlight type-"securityfix"/>' if ($entry->{securityfix}); - print $dest ('</tags>'); + $tagcounter++; } - if (scalar keys %{$entry->{bugs}}) +# else +# { +# print STDERR "Couldn't extract anything from $entry->{message}\n"; +# } + + if (scalar @{$entry->{highlighttags}} || + scalar keys %{$entry->{manualtags}}) { +# Write out highlight tags + + my $tagunique = {}; + map { $tagunique->{$_} = 1 } @{$entry->{highlighttags}}; + print $dest ('<tags>',"\n"); + foreach my $tag (keys %{$tagunique}) + { + printf $dest ('<highlight type="%s"/>%s',$tag,"\n"); + } + + +# Write out determined manual tags + + foreach my $tag (keys %{$entry->{manualtags}}) + { + printf $dest ('<manual type="%s"/>%s',$tag,"\n"); + } + +# Finish + print $dest ('</tags>',"\n"); + } + if (scalar keys %{$entry->{bugs}} || + scalar @{$entry->{cve}}) + { print $dest "<bugs>\n"; + foreach my $bug (keys %{$entry->{bugs}}) { - printf $dest ('<fixes bugid="%s"/>',$bug); + printf $dest ('<fixes bugid="%s"/>%s',$bug,"\n"); } + foreach my $cve (@{$entry->{cve}}) + { + printf $dest ('<cve ref="%s"/>%s',$cve,"\n"); + } + print $dest "</bugs>\n"; } @@ -373,3 +605,5 @@ print $dest ("</versionlog>\n"); close($dest); + +print "Wrote $counter ($tagcounter had tags)\n";
| Thread | ||
|---|---|---|
| • svn commit - mysqldoc@docsrva: r8138 - in trunk/tools: . MySQL/DynXML | mcbrown | 17 Oct |
© 1995-2008 MySQL AB, 2008-2009 Sun Microsystems, Inc.
Page generated in 0.703 sec. using MySQL 5.1.14-beta-log