Quellcode-Bibliothek updateOperatorDictionary.pl Sprache: Shell

#!/usr/bin/perl
# -*- Mode: Perl; tab-width: 2; indent-tabs-mode: nil; -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

use XML::LibXSLT;
use XML::LibXML;
use LWP::Simple;

# output files
$FILE_UNICODE = "unicode.xml";
$FILE_DICTIONARY = "dictionary.xml";
$FILE_DIFFERENCES = "differences.txt";
$FILE_NEW_DICTIONARY = "new_dictionary.txt";
$FILE_SYNTAX_ERRORS = "syntax_errors.txt";

# our dictionary (property file)
$MOZ_DICTIONARY = "mathfont.properties";

# dictionary provided by the W3C in "XML Entity Definitions for Characters"
$WG_DICTIONARY_URL = "https://raw.githubusercontent.com/w3c/xml-entities/gh-pages/unicode.xml";

# XSL stylesheet to extract relevant data from the dictionary
$DICTIONARY_XSL = "operatorDictionary.xsl";

# dictionary provided by the W3C transformed with operatorDictionary.xsl
$WG_DICTIONARY = $FILE_DICTIONARY;

if (!($#ARGV >= 0 &&
      ((($ARGV[0] eq "download") && $#ARGV <= 1) ||
       (($ARGV[0] eq "compare") && $#ARGV <= 1) ||
       (($ARGV[0] eq "check") && $#ARGV <= 0) ||
       (($ARGV[0] eq "clean") && $#ARGV <= 0)))) {
    &usage;
}

if ($ARGV[0] eq "download") {
    if ($#ARGV == 1) {
        $WG_DICTIONARY_URL = $ARGV[1];
    }
    print "Downloading $WG_DICTIONARY_URL...\n";
    getstore($WG_DICTIONARY_URL, $FILE_UNICODE);

    print "Converting $FILE_UNICODE into $FILE_DICTIONARY...\n";
    my $xslt = XML::LibXSLT->new();
    my $source = XML::LibXML->load_xml(location => $FILE_UNICODE);
    my $style_doc = XML::LibXML->load_xml(location => $DICTIONARY_XSL,
                                          no_cdata=>1);
    my $stylesheet = $xslt->parse_stylesheet($style_doc);
    my $results = $stylesheet->transform($source);
    open($file, ">$FILE_DICTIONARY") || die ("Couldn't open $FILE_DICTIONARY!");
    print $file $stylesheet->output_as_bytes($results);
    close($file);
    exit 0;
}

if ($ARGV[0] eq "clean") {
    unlink($FILE_UNICODE,
           $FILE_DICTIONARY,
           $FILE_DIFFERENCES,
           $FILE_NEW_DICTIONARY,
           $FILE_SYNTAX_ERRORS);
    exit 0;
}

if ($ARGV[0] eq "compare" && $#ARGV == 1) {
    $WG_DICTIONARY = $ARGV[1];
}

################################################################################
# structure of the dictionary used by this script:
# - key: same as in mathfont.properties
# - table:
#    index | value
#      0   | description
#      1   | lspace
#      2   | rspace
#      4   | largeop
#      5   | movablelimits
#      6   | stretchy
#      7   | separator
#      8   | accent
#      9   | fence
#     10   | symmetric
#     13   | direction

# 1) build %moz_hash from $MOZ_DICTIONARY

print "loading $MOZ_DICTIONARY...\n";
open($file, $MOZ_DICTIONARY) || die ("Couldn't open $MOZ_DICTIONARY!");

print "building dictionary...\n";
while (<$file>) {
    next unless (m/^operator\.(.*)$/);
    (m/^([\w|\.|\\]*)\s=\s(.*)\s#\s(.*)$/);

    # 1.1) build the key
    $key = $1;

    # 1.2) build the array
    $_ = $2;
    @value = ();
    $value[0] = $3;
    if (m/^(.*)lspace:(\d)(.*)$/) { $value[1] = $2; } else { $value[1] = "5"; }
    if (m/^(.*)rspace:(\d)(.*)$/) { $value[2] = $2; } else { $value[2] = "5"; }
    $value[4] = (m/^(.*)largeop(.*)$/);
    $value[5] = (m/^(.*)movablelimits(.*)$/);
    $value[6] = (m/^(.*)stretchy(.*)$/);
    $value[7] = (m/^(.*)separator(.*)$/);
    $value[8] = (m/^(.*)accent(.*)$/);
    $value[9] = (m/^(.*)fence(.*)$/);
    $value[10] = (m/^(.*)symmetric(.*)$/);
    if (m/^(.*)direction:([a-z]*)(.*)$/) { $value[13] = $2; }
    else { $value[13] = ""; }

    # 1.3) save the key and value
    $moz_hash{$key} = [ @value ];
}

close($file);

################################################################################
# 2) If mode "check", verify validity of our operator dictionary and quit.
#    If mode "compare", go to step 3)

if ($ARGV[0] eq "check") {
    print "checking operator dictionary...\n";
    open($file_syntax_errors, ">$FILE_SYNTAX_ERRORS") ||
        die ("Couldn't open $FILE_SYNTAX_ERRORS!");

    $nb_errors = 0;
    $nb_warnings = 0;
    @moz_keys = (keys %moz_hash);
    # check the validity of our private data
    while ($key = pop(@moz_keys)) {

        if ($key =~ /\\u.+\\u.+\\u.+/) {
            $valid = 0;
            $nb_errors++;
            print $file_syntax_errors "error: \"$key\" has more than 2 characters\n";
        }

        if ($key =~ /\\u20D2\./ || $key =~ /\\u0338\./) {
            $valid = 0;
            $nb_errors++;
            print $file_syntax_errors "error: \"$key\" ends with character U+20D2 or U+0338\n";
        }

        @moz = @{ $moz_hash{$key} };
        $entry = &generateEntry($key, @moz);
        $valid = 1;

        if (!(@moz[13] eq "" ||
              @moz[13] eq "horizontal" ||
              @moz[13] eq "vertical")) {
            $valid = 0;
            $nb_errors++;
            print $file_syntax_errors "error: invalid direction \"$moz[13]\"\n";
        }

        if (@moz[4] && !(@moz[13] eq "vertical")) {
            $valid = 0;
            $nb_errors++;
            print $file_syntax_errors "error: operator is largeop but does not have vertical direction\n";
        }

        if (!$valid) {
            print $file_syntax_errors $entry;
            print $file_syntax_errors "\n";
        }
    }

    # check that all forms have the same direction.
    @moz_keys = (keys %moz_hash);
    while ($key = pop(@moz_keys)) {

        if (@{ $moz_hash{$key} }) {
            # the operator has not been removed from the hash table yet.

            $_ = $key;
            (m/^([\w|\.|\\]*)\.(prefix|infix|postfix)$/);
            $key_prefix = "$1.prefix";
            $key_infix = "$1.infix";
            $key_postfix = "$1.postfix";
            @moz_prefix = @{ $moz_hash{$key_prefix} };
            @moz_infix = @{ $moz_hash{$key_infix} };
            @moz_postfix = @{ $moz_hash{$key_postfix} };

            $same_direction = 1;

            if (@moz_prefix) {
                if (@moz_infix &&
                    !($moz_infix[13] eq $moz_prefix[13])) {
                    $same_direction = 0;
                }
                if (@moz_postfix &&
                    !($moz_postfix[13] eq $moz_prefix[13])) {
                    $same_direction = 0;
                }
            }
            if (@moz_infix) {
                if (@moz_postfix &&
                    !($moz_postfix[13] eq $moz_infix[13])) {
                    $same_direction = 0;
                }
            }

            if (!$same_direction) {
                $nb_errors++;
                print  $file_syntax_errors
                    "error: operator has a stretchy form, but all forms";
                print  $file_syntax_errors
                    " have not the same direction\n";
                if (@moz_prefix) {
                    $_ = &generateEntry($key_prefix, @moz_prefix);
                    print $file_syntax_errors $_;
                }
                if (@moz_infix) {
                    $_ = &generateEntry($key_infix, @moz_infix);
                    print $file_syntax_errors $_;
                }
                if (@moz_postfix) {
                    $_ = &generateEntry($key_postfix, @moz_postfix);
                    print $file_syntax_errors $_;
                }
                print $file_syntax_errors "\n";
            }

            if (@moz_prefix) {
                delete $moz_hash{$key.prefix};
            }
            if (@moz_infix) {
                delete $moz_hash{$key_infix};
            }
            if (@moz_postfix) {
                delete $moz_hash{$key_postfix};
            }
        }
    }

    close($file_syntax_errors);
    print "\n";
    if ($nb_errors > 0 || $nb_warnings > 0) {
        print "$nb_errors error(s) found\n";
        print "$nb_warnings warning(s) found\n";
        print "See output file $FILE_SYNTAX_ERRORS.\n\n";
    } else {
        print "No error found.\n\n";
    }

    exit 0;
}

################################################################################
# 3) build %wg_hash and @wg_keys from the page $WG_DICTIONARY

print "loading $WG_DICTIONARY...\n";
my $parser = XML::LibXML->new();
my $doc = $parser->parse_file($WG_DICTIONARY);

print "building dictionary...\n";
@wg_keys = ();

foreach my $entry ($doc->findnodes('/root/entry')) {
    # 3.1) build the key
    $key = "operator.";

    $_ = $entry->getAttribute("unicode");

    # Skip non-BMP Arabic characters that are handled specially.
    if ($_ == "U1EEF0" || $_ == "U1EEF1") {
        next;
    }

    $_ = "$_-";
    while (m/^U?0(\w*)-(.*)$/) {
        # Concatenate .\uNNNN
        $key = "$key\\u$1";
        $_ = $2;
    }

    $_ = $entry->getAttribute("form"); # "Form"
    $key = "$key.$_";

    # 3.2) build the array
    @value = ();
    $value[0] = lc($entry->getAttribute("description"));
    $value[1] = $entry->getAttribute("lspace");
    if ($value[1] eq "") { $value[1] = "5"; }
    $value[2] = $entry->getAttribute("rspace");
    if ($value[2] eq "") { $value[2] = "5"; }

    $_ = $entry->getAttribute("properties");
    $value[4] = (m/^(.*)largeop(.*)$/);
    $value[5] = (m/^(.*)movablelimits(.*)$/);
    $value[6] = (m/^(.*)stretchy(.*)$/);
    $value[7] = (m/^(.*)separator(.*)$/);
    $value[9] = (m/^(.*)fence(.*)$/);
    $value[10] = (m/^(.*)symmetric(.*)$/);

    # not stored in the WG dictionary
    $value[8] = ""; # accent
    $value[13] = ""; # direction

    # 3.3) save the key and value
    push(@wg_keys, $key);
    $wg_hash{$key} = [ @value ];
}
@wg_keys = reverse(@wg_keys);

################################################################################
# 4) Compare the two dictionaries and output the result

print "comparing dictionaries...\n";
open($file_differences, ">$FILE_DIFFERENCES") ||
    die ("Couldn't open $FILE_DIFFERENCES!");
open($file_new_dictionary, ">$FILE_NEW_DICTIONARY") ||
    die ("Couldn't open $FILE_NEW_DICTIONARY!");

$conflicting = 0; $conflicting_stretching = 0;
$new = 0; $new_stretching = 0;
$obsolete = 0; $obsolete_stretching = 0;
$unchanged = 0;

# 4.1) look to the entries of the WG dictionary
while ($key = pop(@wg_keys)) {

    @wg = @{ $wg_hash{$key} };
    delete $wg_hash{$key};
    $wg_value = &generateCommon(@wg);

    if (exists($moz_hash{$key})) {
        # entry is in both dictionary
        @moz = @{ $moz_hash{$key} };
        delete $moz_hash{$key};
        $moz_value = &generateCommon(@moz);
        if ($moz_value ne $wg_value) {
            # conflicting entry
            print $file_differences "[conflict]";
            $conflicting++;
            if ($moz[6] != $wg[6]) {
                print $file_differences "[stretching]";
                $conflicting_stretching++;
            }
            print $file_differences " - $key ($wg[0])\n";
            print $file_differences "-$moz_value\n+$wg_value\n\n";
            $_ = &completeCommon($wg_value, $key, @moz, @wg);
            print $file_new_dictionary $_;
        } else {
            # unchanged entry
            $unchanged++;
            $_ = &completeCommon($wg_value, $key, @moz, @wg);
            print $file_new_dictionary $_;
        }
    } else {
        # we don't have this entry in our dictionary yet
        print $file_differences "[new entry]";
        $new++;
        if ($wg[6]) {
            print $file_differences "[stretching]";
            $new_stretching++;
        }
        print $file_differences " - $key ($wg[0])\n";
        print $file_differences "-\n+$wg_value\n\n";
        $_ = &completeCommon($wg_value, $key, (), @wg);
        print $file_new_dictionary $_;
    }
}

print $file_new_dictionary
    "\n# Entries below are not part of the official MathML dictionary\n\n";
# 4.2) look in our dictionary the remaining entries
@moz_keys = (keys %moz_hash);
@moz_keys = reverse(sort(@moz_keys));

while ($key = pop(@moz_keys)) {
    @moz = @{ $moz_hash{$key} };
    $moz_value = &generateCommon(@moz);
    print $file_differences "[obsolete entry]";
    $obsolete++;
    if ($moz[6]) {
        print $file_differences "[stretching]";
        $obsolete_stretching++;
    }
    print $file_differences " - $key ($moz[0])\n";
    print $file_differences "-$moz_value\n+\n\n";
    $_ = &completeCommon($moz_value, $key, (), @moz);
    print $file_new_dictionary $_;
}

close($file_differences);
close($file_new_dictionary);

print "\n";
print "- $obsolete obsolete entries ";
print "($obsolete_stretching of them are related to stretching)\n";
print "- $unchanged unchanged entries\n";
print "- $conflicting conflicting entries ";
print "($conflicting_stretching of them are related to stretching)\n";
print "- $new new entries ";
print "($new_stretching of them are related to stretching)\n";
print "\nSee output files $FILE_DIFFERENCES and $FILE_NEW_DICTIONARY.\n\n";
print "After having modified the dictionary, please run";
print "./updateOperatorDictionary check\n\n";
exit 0;

################################################################################
sub usage {
    # display the accepted command syntax and quit
    print "usage:\n";
    print " ./updateOperatorDictionary.pl download [unicode.xml]\n";
    print " ./updateOperatorDictionary.pl compare [dictionary.xml]\n";
    print " ./updateOperatorDictionary.pl check\n";
    print " ./updateOperatorDictionary.pl clean\n";
    exit 0;
}

sub generateCommon {
    # helper function to generate the string of data shared by both dictionaries
    my(@v) = @_;
    $entry = "lspace:$v[1] rspace:$v[2]";
    if ($v[4]) { $entry = "$entry largeop"; }
    if ($v[5]) { $entry = "$entry movablelimits"; }
    if ($v[6]) { $entry = "$entry stretchy"; }
    if ($v[7]) { $entry = "$entry separator"; }
    if ($v[9]) { $entry = "$entry fence"; }
    if ($v[10]) { $entry = "$entry symmetric"; }
    return $entry;
}

sub completeCommon {
    # helper to add key and private data to generateCommon
    my($entry, $key, @v_moz, @v_wg) = @_;

    $entry = "$key = $entry";

    if ($v_moz[8]) { $entry = "$entry accent"; }
    if ($v_moz[13]) { $entry = "$entry direction:$v_moz[13]"; }

    if ($v_moz[0]) {
        # keep our previous comment
        $entry = "$entry # $v_moz[0]";
    } else {
        # otherwise use the description given by the WG
        $entry = "$entry # $v_wg[0]";
    }

    $entry = "$entry\n";
    return $entry;
}

sub generateEntry {
    # helper function to generate an entry of our operator dictionary
    my($key, @moz) = @_;
    $entry = &generateCommon(@moz);
    $entry = &completeCommon($entry, $key, @moz, @moz);
    return $entry;
}

quality79%

¤ Die Informationen auf dieser Webseite wurden nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit, noch Qualität der bereit gestellten Informationen zugesichert.0.35Bemerkung: (vorverarbeitet) ¤

*Bot Zugriff

Wurzel

Suchen

Beweissystem der NASA

Beweissystem Isabelle

NIST Cobol Testsuite

Cephes Mathematical Library

Wiener Entwicklungsmethode

Haftungshinweis

Die Informationen auf dieser Webseite wurden nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit, noch Qualität der bereit gestellten Informationen zugesichert.

Bemerkung:

Die farbliche Syntaxdarstellung ist noch experimentell.