Quelle UnicodeTools.gi Sprache: unbekannt

Spracherkennung für: .gi vermutete Sprache: Unknown {[0] [0] [0]} [Methode: Schwerpunktbildung, einfache Gewichte, sechs Dimensionen]

#############################################################################
##
#W UnicodeTools.gi GAPDoc Frank Lübeck
##
##
#Y Copyright (C) 2007, Frank Lübeck, Lehrstuhl D für Mathematik,
#Y RWTH Aachen
##
## The files UnicodeTools.g{d,i} contain utilities for converting text
## between different encodings. They introduce unicode strings and
## characters as GAP objects.
##

# reading some extendible tables of translations of unicode characters
ReadPackage("GAPDoc", "lib/UnicodeTabs.g");

# UNICODE_RECODE is a is a record. For some string enc, describing a
# character encoding, UNICODE_RECODE.(enc) is a function which translates
# a GAP string in encoding enc to a list of integers describing the unicode
# codepoints of the characters in the string.
#
# UNICODE_RECODE.TABLES contains some embeddings of 8 bit code pages into
# unicode as lists of length 256, codepoint of character i in [0..255] in
# position i+1.

# normalize encoding names
UNICODE_RECODE.NormalizedEncodings := rec(
 latin1 := "ISO-8859-1",
 latin2 := "ISO-8859-2",
 latin3 := "ISO-8859-3",
 latin4 := "ISO-8859-4",
 latin5 := "ISO-8859-9",
 latin6 := "ISO-8859-10",
 latin7 := "ISO-8859-13",
 latin8 := "ISO-8859-14",
 latin9 := "ISO-8859-15",
 latin0 := "ISO-8859-15",
 utf8 := "UTF-8",
 UTF8 := "UTF-8",
 ASCII := "ANSI_X3.4-1968",
 US\-ASCII := "ANSI_X3.4-1968",
 xml := "XML",
 url := "URL",
 URL := "URL",
 percent := "URL",
);
UNICODE_RECODE.f := function()
 local f, nam, i;
 f := function(nam)
 UNICODE_RECODE.NormalizedEncodings.(nam) := nam;
 UNICODE_RECODE.Decoder.(nam) := function(str)
 return UNICODE_RECODE.TABLES.(nam){List(str, INT_CHAR)+1};
 end;
 end;
 for i in Concatenation([1..11],[13..15]) do
 f(Concatenation("ISO-8859-",String(i)));
 od;
 f("ANSI_X3.4-1968");

 UNICODE_RECODE.NormalizedEncodings.("UTF-8") := "UTF-8";
 UNICODE_RECODE.NormalizedEncodings.("XML") := "XML";
 for nam in RecNames(UNICODE_RECODE.NormalizedEncodings) do
 UNICODE_RECODE.NormalizedEncodings.(LowercaseString(nam)) :=
 UNICODE_RECODE.NormalizedEncodings.(nam);
 od;
end;
# can be used for case insensitive normalization
UNICODE_RECODE.NormalizedEncoding := function(str)
 local lstr;
 lstr := LowercaseString(str);
 if not IsBound(UNICODE_RECODE.NormalizedEncodings.(lstr)) then
 return fail;
 else
 return UNICODE_RECODE.NormalizedEncodings.(lstr);
 fi;
end;

UNICODE_RECODE.f();
Unbind(UNICODE_RECODE.f);
# slightly more efficient for latin1:
UNICODE_RECODE.Decoder.("ISO-8859-1") := function(str)
 return List(str, INT_CHAR);
end;
# helper function; arg: str[, start], translate single UTF-8 character
# to its unicode number
UNICODE_RECODE.UnicodeUTF8Char := function(arg)
 local str, i, a, i1, i2, i3;
 str := arg[1];
 if Length(arg)>1 then
 i := arg[2];
 else
 i := 1;
 fi;
 a := INT_CHAR(str[i]);
 if a < 128 then
 return a;
 elif a < 224 then
 if i = Length(str) then return fail; fi;
 i1 := INT_CHAR(str[i+1]);
 if i1 < 128 or i1 > 191 then
 return fail;
 fi;
 return (a mod 192)*64 + (i1 mod 64);
 elif a < 240 then
 if Length(str) < i+2 then return fail; fi;
 i1 := INT_CHAR(str[i+1]);
 i2 := INT_CHAR(str[i+2]);
 if i1 < 128 or i2 < 128 or i1 > 191 or i2 > 191 then
 return fail;
 fi;
 return (a mod 224)*4096 + (i1 mod 64)*64
 + (i2 mod 64);
 else
 if Length(str) < i+3 then return fail; fi;
 i1 := INT_CHAR(str[i+1]);
 i2 := INT_CHAR(str[i+2]);
 i3 := INT_CHAR(str[i+3]);
 if i1<128 or i2<128 or i3<128 or i1>191 or i2>191 or i3>191 then
 return fail;
 fi;
 return (a mod 240)*262144 + (i1 mod 64)*4096
 + (i2 mod 64)*64
 + (i3 mod 64);
 fi;
end;
## UNICODE_RECODE.Decoder.("UTF-8") := function(str)
## local res, c, i;
## res := [];
## for i in [1..Length(str)] do
## c := INT_CHAR(str[i]);
## if c < 128 or c > 191 then
## Add(res, UNICODE_RECODE.UnicodeUTF8Char(str, i));
## fi;
## od;
## if fail in res then return fail; fi;
## return res;
## end;
UNICODE_RECODE.Decoder.("UTF-8") := function(str)
 local res, i, n;
 res := [];
 i := 1;
 while i <= Length(str) do
 n := UNICODE_RECODE.UnicodeUTF8Char(str, i);
 if n = fail then
 return fail;
 elif n < 128 then
 i := i+1;
 elif n < 2048 then
 i := i+2;
 elif n < 65536 then
 i := i+3;
 else
 i := i+4;
 fi;
 Add(res, n);
 od;
 if fail in res then return fail; fi;
 return res;
end;

UNICODE_RECODE.Decoder.("XML") := function(str)
 local res, i, j, n;
 res := [];
 i := 1;
 while i <= Length(str) do
 if str[i] = '&' and i < Length(str) and str[i+1] = '#' then
 j := Position(str, ';', i);
 n := str{[i+2..j-1]};
 if n[1] = 'x' then
 n := IntHexString(n{[2..Length(n)]});
 else
 n := Int(n);
 fi;
 Add(res, n);
 i := j+1;
 else
 Add(res, INT_CHAR(str[i]));
 i := i+1;
 fi;
 od;
 return res;
end;

UNICODE_RECODE.Decoder.("URL") := function(str)
 local res, i;
 res := "";
 i := 1;
 while i <= Length(str) do
 if str[i] = '%' then
 Add(res, CHAR_INT(IntHexString(str{[i+1,i+2]})));
 i := i+3;
 else
 Add(res, str[i]);
 i := i+1;
 fi;
 od;
 return IntListUnicodeString(Unicode(res, "UTF-8"));
end;

################################################
UNICODE_RECODE.TABLES :=
rec(
 ANSI_X3\.4\-1968 := [0..127],
 ISO\-8859\-1 := [ 0 .. 255 ],
 ISO\-8859\-2 := [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52,
 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70,
 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88,
 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104,
 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118,
 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132,
 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146,
 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 260,
 728, 321, 163, 317, 346, 166, 167, 352, 350, 356, 377, 172, 381, 379,
 175, 261, 731, 322, 179, 318, 347, 711, 183, 353, 351, 357, 378, 733,
 382, 380, 340, 192, 193, 258, 195, 313, 262, 198, 268, 200, 280, 202,
 282, 204, 205, 270, 272, 323, 327, 210, 211, 336, 213, 214, 344, 366,
 217, 368, 219, 220, 354, 222, 341, 224, 225, 259, 227, 314, 263, 230,
 269, 232, 281, 234, 283, 236, 237, 271, 273, 324, 328, 242, 243, 337,
 245, 246, 345, 367, 249, 369, 251, 252, 355, 729, 255 ],
 ISO\-8859\-3 := [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52,
 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70,
 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88,
 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104,
 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118,
 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132,
 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146,
 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 294,
 728, 162, 163, 164, 292, 166, 167, 304, 350, 286, 308, 172, 173, 379,
 175, 295, 177, 178, 179, 180, 293, 182, 183, 305, 351, 287, 309, 188,
 189, 380, 191, 192, 193, 194, 195, 266, 264, 198, 199, 200, 201, 202,
 203, 204, 205, 206, 207, 208, 209, 210, 211, 288, 213, 214, 284, 216,
 217, 218, 219, 364, 348, 222, 223, 224, 225, 226, 227, 267, 265, 230,
 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 289,
 245, 246, 285, 248, 249, 250, 251, 365, 349, 729, 255 ],
 ISO\-8859\-4 := [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52,
 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70,
 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88,
 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104,
 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118,
 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132,
 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146,
 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 260,
 312, 342, 163, 296, 315, 166, 167, 352, 274, 290, 358, 172, 381, 174,
 175, 261, 731, 343, 179, 297, 316, 711, 183, 353, 275, 291, 359, 330,
 382, 331, 256, 192, 193, 194, 195, 196, 197, 302, 268, 200, 280, 202,
 278, 204, 205, 298, 272, 325, 332, 310, 211, 212, 213, 214, 215, 370,
 217, 218, 219, 360, 362, 222, 257, 224, 225, 226, 227, 228, 229, 303,
 269, 232, 281, 234, 279, 236, 237, 299, 273, 326, 333, 311, 243, 244,
 245, 246, 247, 371, 249, 250, 251, 361, 363, 729, 255 ],
 ISO\-8859\-5 := [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52,
 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70,
 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88,
 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104,
 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118,
 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132,
 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146,
 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 1025,
 1026, 1027, 1028, 1029, 1030, 1031, 1032, 1033, 1034, 1035, 1036, 172,
 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047, 1048, 1049,
 1050, 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, 1059, 1060, 1061,
 1062, 1063, 1064, 1065, 1066, 1067, 1068, 1069, 1070, 1071, 1072, 1073,
 1074, 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085,
 1086, 1087, 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097,
 1098, 1099, 1100, 1101, 1102, 1103, 8470, 1105, 1106, 1107, 1108, 1109,
 1110, 1111, 1112, 1113, 1114, 1115, 1116, 167, 1118, 1119, 255 ],
 ISO\-8859\-6 := [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52,
 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70,
 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88,
 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104,
 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118,
 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132,
 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146,
 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160,
 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 1548, 172, 173, 174,
 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 1563, 187, 188,
 189, 1567, 191, 1569, 1570, 1571, 1572, 1573, 1574, 1575, 1576, 1577,
 1578, 1579, 1580, 1581, 1582, 1583, 1584, 1585, 1586, 1587, 1588, 1589,
 1590, 1591, 1592, 1593, 1594, 218, 219, 220, 221, 222, 1600, 1601,
 1602, 1603, 1604, 1605, 1606, 1607, 1608, 1609, 1610, 1611, 1612, 1613,
 1614, 1615, 1616, 1617, 1618, 242, 243, 244, 245, 246, 247, 248, 249,
 250, 251, 252, 253, 254, 255 ],
 ISO\-8859\-7 := [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52,
 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70,
 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88,
 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104,
 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118,
 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132,
 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146,
 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 701,
 700, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 8213,
 175, 176, 177, 178, 900, 901, 902, 182, 904, 905, 906, 186, 908, 188,
 910, 911, 912, 913, 914, 915, 916, 917, 918, 919, 920, 921, 922, 923,
 924, 925, 926, 927, 928, 929, 209, 931, 932, 933, 934, 935, 936, 937,
 938, 939, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, 950, 951,
 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 962, 963, 964, 965,
 966, 967, 968, 969, 970, 971, 972, 973, 974, 254, 255 ],
 ISO\-8859\-8 := [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52,
 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70,
 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88,
 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104,
 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118,
 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132,
 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146,
 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160,
 161, 162, 163, 164, 165, 166, 167, 168, 215, 170, 171, 172, 173, 8254,
 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 247, 186, 187, 188,
 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202,
 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216,
 217, 218, 219, 220, 221, 8215, 1488, 1489, 1490, 1491, 1492, 1493,
 1494, 1495, 1496, 1497, 1498, 1499, 1500, 1501, 1502, 1503, 1504, 1505,
 1506, 1507, 1508, 1509, 1510, 1511, 1512, 1513, 1514, 250, 251, 252,
 253, 254, 255 ],
 ISO\-8859\-9 := [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52,
 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70,
 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88,
 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104,
 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118,
 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132,
 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146,
 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160,
 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174,
 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188,
 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202,
 203, 204, 205, 206, 286, 208, 209, 210, 211, 212, 213, 214, 215, 216,
 217, 218, 219, 304, 350, 222, 223, 224, 225, 226, 227, 228, 229, 230,
 231, 232, 233, 234, 235, 236, 237, 238, 287, 240, 241, 242, 243, 244,
 245, 246, 247, 248, 249, 250, 251, 305, 351, 254, 255 ],
 ISO\-8859\-10 := [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33,
 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51,
 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69,
 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87,
 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103,
 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117,
 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131,
 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145,
 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159,
 260, 274, 290, 298, 296, 310, 166, 315, 272, 352, 358, 381, 172, 362,
 330, 175, 261, 275, 291, 299, 297, 311, 182, 316, 273, 353, 359, 382,
 8213, 363, 331, 256, 192, 193, 194, 195, 196, 197, 302, 268, 200, 280,
 202, 278, 204, 205, 206, 207, 325, 332, 210, 211, 212, 213, 360, 215,
 370, 217, 218, 219, 220, 221, 222, 257, 224, 225, 226, 227, 228, 229,
 303, 269, 232, 281, 234, 279, 236, 237, 238, 239, 326, 333, 242, 243,
 244, 245, 361, 247, 371, 249, 250, 251, 252, 253, 312, 255 ],
 ISO\-8859\-11 := [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33,
 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51,
 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69,
 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87,
 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103,
 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117,
 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131,
 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145,
 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159,
 3585, 3586, 3587, 3588, 3589, 3590, 3591, 3592, 3593, 3594, 3595, 3596,
 3597, 3598, 3599, 3600, 3601, 3602, 3603, 3604, 3605, 3606, 3607, 3608,
 3609, 3610, 3611, 3612, 3613, 3614, 3615, 3616, 3617, 3618, 3619, 3620,
 3621, 3622, 3623, 3624, 3625, 3626, 3627, 3628, 3629, 3630, 3631, 3632,
 3633, 3634, 3635, 3636, 3637, 3638, 3639, 3640, 3641, 3642, 218, 219,
 220, 221, 3647, 3648, 3649, 3650, 3651, 3652, 3653, 3654, 3655, 3656,
 3657, 3658, 3659, 3660, 3661, 3662, 3663, 3664, 3665, 3666, 3667, 3668,
 3669, 3670, 3671, 3672, 3673, 3674, 3675, 251, 252, 253, 254, 255 ],
 ISO\-8859\-13 := [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33,
 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51,
 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69,
 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87,
 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103,
 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117,
 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131,
 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145,
 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159,
 8221, 161, 162, 163, 8222, 165, 166, 216, 168, 342, 170, 171, 172, 173,
 198, 175, 176, 177, 178, 8220, 180, 181, 182, 248, 184, 343, 186, 187,
 188, 189, 230, 260, 302, 256, 262, 195, 196, 280, 274, 268, 200, 377,
 278, 290, 310, 298, 315, 352, 323, 325, 210, 332, 212, 213, 214, 370,
 321, 346, 362, 219, 379, 381, 222, 261, 303, 257, 263, 227, 228, 281,
 275, 269, 232, 378, 279, 291, 311, 299, 316, 353, 324, 326, 242, 333,
 244, 245, 246, 371, 322, 347, 363, 251, 380, 382, 8217, 255 ],
 ISO\-8859\-14 := [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33,
 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51,
 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69,
 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87,
 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103,
 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117,
 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131,
 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145,
 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159,
 7682, 7683, 162, 266, 267, 7690, 166, 7808, 168, 7810, 7691, 7922, 172,
 173, 376, 7710, 7711, 288, 289, 7744, 7745, 181, 7766, 7809, 7767,
 7811, 7776, 7923, 7812, 7813, 7777, 191, 192, 193, 194, 195, 196, 197,
 198, 199, 200, 201, 202, 203, 204, 205, 206, 372, 208, 209, 210, 211,
 212, 213, 7786, 215, 216, 217, 218, 219, 220, 374, 222, 223, 224, 225,
 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 373,
 240, 241, 242, 243, 244, 245, 7787, 247, 248, 249, 250, 251, 252, 375,
 254, 255 ],
 ISO\-8859\-15 := [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33,
 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51,
 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69,
 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87,
 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103,
 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117,
 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131,
 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145,
 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159,
 160, 161, 162, 8364, 164, 352, 166, 353, 168, 169, 170, 171, 172, 173,
 174, 175, 176, 177, 178, 381, 180, 181, 182, 382, 184, 185, 186, 338,
 339, 376, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201,
 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215,
 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229,
 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243,
 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255 ] );
# reverse tables are only generated when needed
UNICODE_RECODE.TABLES.reverse := rec();
## # this created the code tables for ISO-8859:
## for i in Concatenation([1..11],[13..15]) do
## fn := Concatenation("iso8859-",String(i),".txt");
## str := StringFile(fn);
## str := SplitString(str,"","\n");
## str := List(str, a-> SplitString(a,""," \t"));
## str := List(str, a-> List([1,2], k-> IntHexString(
## Filtered(a[k], x-> not x in "=+U"))));
## str := Filtered(str, a-> a[1]<>a[2]);
## fn := Concatenation("ISO-8859-",String(i));
## res := [0..255];
## for a in str do
## res[a[1]] := a[2];
## od;
## UNICODE_RECODE.TABLES.(fn) := res;
## od;
################################################

# wrap and cache integers as unicode characters
InstallMethod(UChar, [IsInt], function(n)
 local res;
 if not IsInt(n) or n < 0 or n > 2097151 then
 return fail;
 fi;
 if IsBound(UNICODECHARCACHE[n]) then
 return UNICODECHARCACHE[n];
 fi;
 res := rec(codepoint := n);
 Objectify(UnicodeCharacterType, res);
 UNICODECHARCACHE[n] := res;
 return res;
end);
# interpret GAP characters as latin 1 encoded
InstallMethod(UChar, [IsChar], function(c)
 return UChar(INT_CHAR(c));
end);

# viewing and printing unicode characters
InstallMethod(ViewObj, [IsUnicodeCharacter], function(c)
 Print("'", UNICODE_RECODE.UTF8UnicodeChar(c!.codepoint), "'");
end);
InstallMethod(PrintObj, [IsUnicodeCharacter], function(c)
 Print("UChar(",c!.codepoint,")");
end);
# \=
InstallMethod(\=, [IsUnicodeCharacter, IsUnicodeCharacter],
function(c, d)
 return c!.codepoint = d!.codepoint;
end);
InstallOtherMethod(Int, [IsUnicodeCharacter], function(uc)
 return uc!.codepoint;
end);

## <#GAPDoc Label="Unicode">
## <ManSection>
## <Heading>Unicode Strings and Characters</Heading>
## <Oper Name="Unicode" Arg="list[, encoding]"/>
## <Oper Name="UChar" Arg="num"/>
## <Filt Name="IsUnicodeString" />
## <Filt Name="IsUnicodeCharacter" />
## <Func Name="IntListUnicodeString" Arg="ustr" />
##
## <Description>
## Unicode characters are described by their <Emph>codepoint</Emph>, an
## integer in the range from <M>0</M> to <M>2^{21}-1</M>.
## For details about unicode, see <URL>https://www.unicode.org</URL>.
##
## The function <Ref Oper="UChar"/> wraps an integer <A>num</A> into
## a &GAP; object lying in the filter <Ref Filt="IsUnicodeCharacter"/>.
## Use <C>Int</C> to get the codepoint back. The argument <A>num</A> can
## also be a &GAP; character which is then translated to an integer via
## <Ref BookName="Reference" Func="IntChar"/>. 
##
## <Ref Oper="Unicode" /> produces a &GAP; object in the filter
## <Ref Filt="IsUnicodeString"/>. This is a wrapped list of integers
## for the unicode characters in the string. The function <Ref
## Func="IntListUnicodeString"/> gives access to this list of integers.
## Basic list functionality is available for <Ref Filt="IsUnicodeString"/>
## elements. The entries are in <Ref Filt="IsUnicodeCharacter"/>.
## The argument <A>list</A> for <Ref Oper="Unicode"/> is either a list of
## integers or a &GAP; string. In the latter case an <A>encoding</A> can be
## specified as string, its default is <C>"UTF-8"</C>. 
##
## <Index>URL encoding</Index><Index>RFC 3986</Index>
## Currently supported encodings can be found in
## <C>UNICODE_RECODE.NormalizedEncodings</C> (ASCII,
## ISO-8859-X, UTF-8 and aliases). The encoding <C>"XML"</C> means an ASCII
## encoding in which non-ASCII characters are specified by XML character
## entities. The encoding <C>"URL"</C> is for URL-encoded (also called
## percent-encoded strings, as specified in RFC 3986
## (<URL Text="see here">https://www.ietf.org/rfc/rfc3986.txt</URL>).
## The listed encodings <C>"LaTeX"</C> and aliases
## cannot be used with <Ref Oper="Unicode" />.
## See the operation <Ref Oper="Encode"/> for mapping a unicode string
## to a &GAP; string.
## <Example>
## gap> ustr := Unicode("a and \366", "latin1");
## Unicode("a and ö")
## gap> ustr = Unicode("a and ö", "XML");
## true
## gap> IntListUnicodeString(ustr);
## [ 97, 32, 97, 110, 100, 32, 246 ]
## gap> ustr[7];
## 'ö'
## </Example>
## </Description>
## </ManSection>
##
## <#/GAPDoc>

# NC method, assume that l is (plain?) list of integers in correct range
InstallMethod(Unicode, [IsList], function(l)
 local res;
 res := [l];
 Objectify(UnicodeStringType, res);
 return res;
end);
# extract the list of integers
InstallGlobalFunction("IntListUnicodeString", function(ustr)
 return ustr![1];
end);

InstallMethod(Unicode, [IsString, IsString], function(str, enc)
 local res;
 if Length(str) > 0 and not IsStringRep(str) then
## Info(InfoWarning, 1, "#W Changing argument to IsStringRep");
## Info(InfoWarning, 2, ":\n ", str);
## Info(InfoWarning, 1, "\n");
## ConvertToStringRep(str);
 str := ShallowCopy(str);
 ConvertToStringRep(str);
 fi;
 if not IsBound(UNICODE_RECODE.NormalizedEncodings.(enc)) then
 Error("Sorry, only the following encodings are supported for 'Unicode':\n",
 RecNames(UNICODE_RECODE.Decoder), "\n");
 fi;
 enc := UNICODE_RECODE.NormalizedEncodings.(enc);
 res := UNICODE_RECODE.Decoder.(enc)(str);
 if res = fail then
 return fail;
 fi;
 return Unicode(res);
end);
# just a string as argument is assumed to be in UTF-8 encoding
InstallMethod(Unicode, [IsStringRep], function(str)
 return Unicode(str, "UTF-8");
end);

# view and print
InstallMethod(ViewObj, [IsUnicodeString], function(ustr)
 local l;
 l := IntListUnicodeString(ustr);
 if Length(l) > 40 then
 l := l{[1..37]};
 Append(l, [46,46,46]);
 fi;
 Print("Unicode(");
 ViewObj(Concatenation(List(l, UNICODE_RECODE.UTF8UnicodeChar)));
 Print(")");
end);
InstallMethod(PrintObj, [IsUnicodeString], function(ustr)
 Print("Unicode(");
 PrintObj(Concatenation(List(IntListUnicodeString(ustr),
 UNICODE_RECODE.UTF8UnicodeChar)));
 Print(")");
end);

# the *basic* list operations
InstallMethod(Length, [IsUnicodeString], function(ustr)
 return Length(IntListUnicodeString(ustr));
end);

InstallMethod(\[\], [IsUnicodeString, IsPosInt], function(ustr, i)
 return UChar(IntListUnicodeString(ustr)[i]);
end);

InstallOtherMethod(\[\]\:\=, [IsUnicodeString and IsMutable,
 IsPosInt, IsUnicodeCharacter],
function(ustr, i, x)
 local l;
 if i > Length(ustr)+1 then
 Error("no unicode string assignment at position ",i,"\n");
 fi;
 l := IntListUnicodeString(ustr);
 l[i] := x!.codepoint;
end);

InstallMethod(Unbind\[\], [IsUnicodeString and IsMutable, IsPosInt],
function(ustr, i)
 local l;
 if i < Length(ustr) then
 Error("can only unbind last character in unicode string\n");
 fi;
 if i = Length(ustr) then
 l := IntListUnicodeString(ustr);
 Unbind(l[Length(l)]);
 fi;
end);

# let ShallowCopy produce a unicode string
InstallMethod(ShallowCopy, [IsUnicodeString], function(ustr)
 return Unicode(ShallowCopy(IntListUnicodeString(ustr)));
end);
# let sublists be unicode strings
InstallMethod(\{\}, [IsUnicodeString, IsList], function(ustr, poss)
 return Unicode(IntListUnicodeString(ustr){poss});
end);

# a better Append for efficiency
InstallMethod(Append, [IsUnicodeString and IsMutable, IsUnicodeString],
function(ustr, ustr2)
 Append(IntListUnicodeString(ustr), IntListUnicodeString(ustr2));
end);

# better \= for efficiency
InstallMethod(\=, [IsUnicodeString, IsUnicodeString], function(ustr1, ustr2)
 return IntListUnicodeString(ustr1) = IntListUnicodeString(ustr2);
end);

# better Position, PositionSublist
InstallMethod(Position, [IsUnicodeString, IsUnicodeCharacter],
function(ustr, c)
 return Position(IntListUnicodeString(ustr), c!.codepoint);
end);
InstallMethod(Position, [IsUnicodeString, IsUnicodeCharacter, IsInt],
function(ustr, c, pos)
 return Position(IntListUnicodeString(ustr), c!.codepoint, pos);
end);
InstallOtherMethod(PositionSublist, [IsUnicodeString, IsUnicodeString],
function(ustr, ustr2)
 return PositionSublist(IntListUnicodeString(ustr),
 IntListUnicodeString(ustr2));
end);
InstallMethod(PositionSublist, [IsUnicodeString, IsUnicodeString, IsInt],
function(ustr, ustr2, pos)
 return PositionSublist(IntListUnicodeString(ustr),
 IntListUnicodeString(ustr2), pos);
end);

## <#GAPDoc Label="Encode">
## <ManSection>
## <Oper Name="Encode" Arg="ustr[, encoding]" />
## <Returns>a &GAP; string</Returns>
## <Func Name="SimplifiedUnicodeString" Arg='ustr[, encoding][, "single"]' />
## <Returns>a unicode string</Returns>
## <Func Name="LowercaseUnicodeString" Arg="ustr" />
## <Returns>a unicode string</Returns>
## <Func Name="UppercaseUnicodeString" Arg="ustr" />
## <Returns>a unicode string</Returns>
## <Var Name="LaTeXUnicodeTable" />
## <Var Name="SimplifiedUnicodeTable" />
## <Var Name="LowercaseUnicodeTable" />
##
## <Description>
## The operation <Ref Oper="Encode"/> translates a unicode string <A>ustr</A>
## into a &GAP; string in some specified <A>encoding</A>. The default
## encoding is <C>"UTF-8"</C>. 
##
## Supported encodings can be found in
## <C>UNICODE_RECODE.NormalizedEncodings</C>. Except for some cases
## mentioned below characters which are not available in the target
## encoding are substituted by '?' characters.
##
## If the <A>encoding</A> is <C>"URL"</C> (see <Ref Oper="Unicode"/>) then
## an optional argument <A>encreserved</A> can be given, it must be a list
## of reserved characters which should be percent encoded; the default is
## to encode only the <C>%</C> character.
##
## The encoding <C>"LaTeX"</C> substitutes
## non-ASCII characters and &LaTeX; special characters by &LaTeX; code
## as given in an ordered list
## <C>LaTeXUnicodeTable</C> of pairs [codepoint, string]. If you have a
## unicode character for which no substitution is contained in that list,
## you will get a warning and the translation is <C>Unicode(nr)</C>.
## In this case find a substitution and add a
## corresponding [codepoint, string]
## pair to <C>LaTeXUnicodeTable</C> using <Ref BookName="reference"
## Oper="AddSet"/>. Also, please, tell the &GAPDoc; authors about your
## addition, such that we can extend the list <C>LaTeXUnicodeTable</C>.
## (Most of the initial entries were generated from lists in the
## &TeX; projects enc&TeX; and <C>ucs</C>.)
## There are some variants of this encoding:
## <C>"LaTeXleavemarkup"</C> does
## the same translations for non-ASCII characters but leaves the &LaTeX;
## special characters (e.g., any &LaTeX; commands) as they are.
## <C>"LaTeXUTF8"</C> does not give a warning about unicode characters
## without explicit translation, instead it translates the character
## to its <C>UTF-8</C> encoding. Make sure to setup your &LaTeX; document such
## that all these characters are understood.
## <C>"LaTeXUTF8leavemarkup"</C> is a combination of the last two variants.
##
## Note that the <C>"LaTeX"</C> encoding can only be used with <Ref
## Oper="Encode"/> but not for the opposite translation with <Ref
## Oper="Unicode" /> (which would need far too complicated heuristics).
##
## The function <Ref Func="SimplifiedUnicodeString"/> can be used to
## substitute many non-ASCII characters by related ASCII characters
## or strings (e.g., by a corresponding character without accents).
## The argument <A>ustr</A> and the result are unicode strings, if
## <A>encoding</A> is <C>"ASCII"</C> then all non-ASCII characters are
## translated, otherwise only the non-latin1 characters. If the string
## <C>"single"</C> in an argument then only substitutions are considered
## which don't make the result string longer. The translations are stored
## in a sorted list <C>SimplifiedUnicodeTable</C>. Its entries are of the
## form <C>[codepoint, trans1, trans2, ...]</C>. Here <C>trans1</C> and so
## on is either an integer for the codepoint of a substitution character or
## it is a list of codepoint integers. If you are missing characters in
## this list and know a sensible ASCII approximation, then add an entry
## (with <Ref BookName="reference" Oper="AddSet"/>) and tell the &GAPDoc;
## authors about it. (The initial content of <C>SimplifiedUnicodeTable</C>
## was mainly generated from the <Q><C>transtab</C></Q> tables by Markus
## Kuhn.)
##
## The function <Ref Func="LowercaseUnicodeString"/> gets and returns a
## unicode string and translates each uppercase character to its
## corresponding lowercase version. This function uses a list
## <C>LowercaseUnicodeTable</C> of pairs of codepoint integers.
## This list was generated using the file <F>UnicodeData.txt</F> from the
## unicode definition (field 14 in each row).
##
## The function <Ref Func="UppercaseUnicodeString"/> does the similar
## translation to uppercase characters.
##
## <Example>
## gap> ustr := Unicode("a and ö", "XML");
## Unicode("a and ö")
## gap> SimplifiedUnicodeString(ustr, "ASCII");
## Unicode("a and oe")
## gap> SimplifiedUnicodeString(ustr, "ASCII", "single");
## Unicode("a and o")
## gap> ustr2 := UppercaseUnicodeString(ustr);;
## gap> Print(Encode(ustr2, GAPInfo.TermEncoding), "\n");
## A AND Ö
## </Example>
## </Description>
## </ManSection>
##
## <#/GAPDoc>
##

# helper function for encoding a unicode character to UTF-8
UNICODE_RECODE.UTF8UnicodeChar := function(n)
 local res, a, b, c, d;
 res := "";
 if n < 0 then
 return fail;
 elif n < 128 then
 Add(res, CHAR_INT(n));
 elif n < 2048 then
 a := n mod 64;
 b := (n - a) / 64;
 Add(res, CHAR_INT(b + 192));
 Add(res, CHAR_INT(a + 128));
 elif n < 65536 then
 a := n mod 64;
 n := (n - a)/64;
 b := n mod 64;
 c := (n - b)/64;
 Add(res, CHAR_INT(c + 224));
 Add(res, CHAR_INT(b + 128));
 Add(res, CHAR_INT(a + 128));
 elif n < 2097152 then
 a := n mod 64;
 n := (n - a)/64;
 b := n mod 64;
 n := (n - b)/64;
 c := n mod 64;
 d := (n - c)/64;
 Add(res, CHAR_INT(d + 240));
 Add(res, CHAR_INT(c + 128));
 Add(res, CHAR_INT(b + 128));
 Add(res, CHAR_INT(a + 128));
 else
 return fail;
 fi;
 return res;
end;
# encode unicode string to GAP string in UTF-8 encoding
UNICODE_RECODE.Encoder.("UTF-8") := function(ustr)
 local res, f, n;
 res := "";
 f := UNICODE_RECODE.UTF8UnicodeChar;
 for n in IntListUnicodeString(ustr) do
 Append(res, f(n));
 od;
 return res;
end;
# non-ASCII characters to XML character entities
UNICODE_RECODE.Encoder.("XML") := function(ustr)
 local res, n;
 res := "";
 for n in IntListUnicodeString(ustr) do
 if n < 128 then
 Add(res, CHAR_INT(n));
 else
 Append(res, Concatenation("&#x", LowercaseString(HexStringInt(n)),";"));
 fi;
 od;
 return res;
end;

UNICODE_RECODE.RFC3986Unreserved := Set(List(
 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.~",
 INT_CHAR));
UNICODE_RECODE.RFC3986Reserved := Set(List("!*'();:@&=+$,/?%#[]", INT_CHAR));
UNICODE_RECODE.Encoder.("URL") := function(arg)
 local ustr, encreserved, res, s, i, j;
 ustr := arg[1];
 # also allow UTF-8 GAP string
 if not IsUnicodeString(ustr) then
 ustr := Unicode(ustr);
 fi;
 if Length(arg) > 1 then
 encreserved := arg[2];
 else
 encreserved := "%";
 fi;
 if IsString(encreserved) then
 encreserved := Set(List(encreserved, INT_CHAR));
 fi;
 res := "";
 for i in IntListUnicodeString(ustr) do
 if i in UNICODE_RECODE.RFC3986Unreserved then
 Add(res, CHAR_INT(i));
 elif i in UNICODE_RECODE.RFC3986Reserved and not i in encreserved then
 Add(res, CHAR_INT(i));
 elif i < 128 then
 Add(res, '%');
 if i < 16 then
 Add(res, '0');
 fi;
 Append(res, HexStringInt(i));
 else
 s := Encode(Unicode([i]), "UTF-8");
 for j in List(s, INT_CHAR) do
 Add(res, '%');
 Append(res, HexStringInt(j));
 od;
 fi;
 od;
 return res;
end;

# non-ASCII characters to LaTeX code, if known from LaTeXUnicodeTable
# args: unicodestring[, leavemarkup[, leaveutf8]]
UNICODE_RECODE.Encoder.("LaTeX") := function(arg)
 local ustr, leavemarkup, tt, res, pos, s, n, leaveutf8, al;
 ustr := arg[1];
 if Length(arg) > 1 then
 leavemarkup := arg[2];
 else
 leavemarkup := false;
 fi;
 if Length(arg) > 2 then
 leaveutf8 := arg[3];
 else
 leaveutf8 := false;
 fi;
 tt := LaTeXUnicodeTable;
 res := "";
 for n in IntListUnicodeString(ustr) do
 al := Set([ 35, 36, 37, 38, 45, 60, 62, 92, 94, 95, 123, 125, 126]);
 pos := Position(al, n);
 if pos <> fail and not leavemarkup then
 Append(res, tt[pos][2]);
 elif n < 128 then
 Add(res, CHAR_INT(n));
 else
 pos := PositionSorted(tt, [n]);
 if IsBound(tt[pos]) and tt[pos][1] = n then
 Append(res, tt[pos][2]);
 elif leaveutf8 = true then
 Append(res, UNICODE_RECODE.UTF8UnicodeChar(n));
 else
 s := Encode(Unicode([n]), GAPInfo.TermEncoding);
 Info(InfoWarning, 1,
 "#W Missing LaTeX translation of unicode character ",
 String(n), ":", s, ",\nadd to LaTeXUnicodeTable\n");
 Append(res, Concatenation("Unicode(", String(n), ")"));
 fi;
 fi;
 od;
 return res;
end;
UNICODE_RECODE.NormalizedEncodings.LaTeX := "LaTeX";
UNICODE_RECODE.NormalizedEncodings.latex := "LaTeX";
UNICODE_RECODE.NormalizedEncodings.BibTeX := "LaTeX";

UNICODE_RECODE.Encoder.("LaTeXleavemarkup") := function(ustr)
 return UNICODE_RECODE.Encoder.("LaTeX")(ustr, true);
end;
UNICODE_RECODE.NormalizedEncodings.LaTeXleavemarkup := "LaTeXleavemarkup";
UNICODE_RECODE.NormalizedEncodings.latexleavemarkup := "LaTeXleavemarkup";
UNICODE_RECODE.NormalizedEncodings.BibTeXleavemarkup := "LaTeXleavemarkup";
UNICODE_RECODE.Encoder.("LaTeXUTF8leavemarkup") := function(ustr)
 return UNICODE_RECODE.Encoder.("LaTeX")(ustr, true, true);
end;
UNICODE_RECODE.NormalizedEncodings.LaTeXUTF8leavemarkup := "LaTeXUTF8leavemarkup";
UNICODE_RECODE.NormalizedEncodings.BibTeXUTF8leavemarkup := "LaTeXUTF8leavemarkup";
UNICODE_RECODE.Encoder.("LaTeXUTF8") := function(ustr)
 return UNICODE_RECODE.Encoder.("LaTeX")(ustr, false, true);
end;
UNICODE_RECODE.NormalizedEncodings.LaTeXUTF8 := "LaTeXUTF8";
UNICODE_RECODE.NormalizedEncodings.BibTeXUTF8:= "LaTeXUTF8";

InstallGlobalFunction(SimplifiedUnicodeString, function(arg)
 local ustr, single, max, tt, res, pos, a, f, n;
 ustr := arg[1];
 # at most single character substitutions?
 single := false;
 # maximal untouched character (255 for latin1 and 127 for ASCII)
 max := 255;
 if "single" in arg then
 single := true;
 fi;
 if "ascii" in arg or "ASCII" in arg or "ANSI_X3.4-1968" in arg then
 max := 127;
 fi;
 tt := SimplifiedUnicodeTable;
 res := [];
 for n in IntListUnicodeString(ustr) do
 if n <= max then
 Add(res, n);
 else
 pos := PositionSorted(tt, [n]);
 if IsBound(tt[pos]) and tt[pos][1] = n then
 a := tt[pos];
 f := Filtered([2..Length(a)], i-> (IsInt(a[i]) and a[i] <= max)
 or (IsList(a[i]) and ForAll(a[i], j-> j <= max)));
 if single then
 f := Filtered(f, i-> IsInt(a[i]) or Length(a[i]) <= 1);
 fi;
 if Length(f) > 0 then
 a := a[f[1]];
 if IsInt(a) then
 Add(res, a);
 else
 Append(res, a);
 fi;
 else
 # ? is '?'
 Add(res, 63);
 fi;
 else
 Add(res, 63);
 fi;
 fi;
 od;
 return Unicode(res);
end);

InstallGlobalFunction(LowercaseUnicodeString, function(ustr)
 local res, tt, pos, i;
 res := ShallowCopy(IntListUnicodeString(ustr));
 tt := LowercaseUnicodeTable;
 for i in [1..Length(res)] do
 pos := PositionSorted(tt, [res[i]]);
 if IsBound(tt[pos]) and tt[pos][1] = res[i] then
 res[i] := tt[pos][2];
 fi;
 od;
 return Unicode(res);
end);
InstallGlobalFunction(UppercaseUnicodeString, function(ustr)
 local res, UppercaseUnicodeTable, tt, pos, i;
 res := ShallowCopy(IntListUnicodeString(ustr));
 if not IsBound(UppercaseUnicodeTable) then
 UppercaseUnicodeTable := Set(List(LowercaseUnicodeTable, a-> [a[2],a[1]]));
 fi;
 tt := UppercaseUnicodeTable;
 for i in [1..Length(res)] do
 pos := PositionSorted(tt, [res[i]]);
 if IsBound(tt[pos]) and tt[pos][1] = res[i] then
 res[i] := tt[pos][2];
 fi;
 od;
 return Unicode(res);
end);

# ISO-8859 cases, substitute '?' for unknown characters
UNICODE_RECODE.f := function()
 local nam, i;
 for i in Concatenation([1..11],[13..15]) do
 nam := MakeImmutable(Concatenation("ISO-8859-", String(i)));
 UNICODE_RECODE.Encoder.(nam) := function(ustr)
 local t, s, res, pos, c;
 if not IsBound(UNICODE_RECODE.TABLES.reverse.(nam)) then
 t := [0..255];
 s := ShallowCopy(UNICODE_RECODE.TABLES.(nam));
 SortParallel(s, t);
 UNICODE_RECODE.TABLES.reverse.(nam) := [s, t];
 fi;
 t := UNICODE_RECODE.TABLES.reverse.(nam);
 res := [];
 for c in IntListUnicodeString(ustr) do
 if c < 160 then
 Add(res, c);
 else
 pos := PositionSorted(t[1], c);
 if pos = fail then
 Add(res, 63); # '?'
 else
 Add(res, t[2][pos]);
 fi;
 fi;
 od;
 return STRING_SINTLIST(res);
 end;
 od;
end;
UNICODE_RECODE.f();
Unbind(UNICODE_RECODE.f);
UNICODE_RECODE.Encoder.("ANSI_X3.4-1968") := function(ustr)
 local res;
 res := List(IntListUnicodeString(ustr), function(i)
 if i < 128 then return i; else return 63; fi; end);
 return STRING_SINTLIST(res);
end;

InstallMethod(Encode, [IsUnicodeString, IsString], function(ustr, enc)
 if not IsBound(UNICODE_RECODE.NormalizedEncodings.(enc)) then
 Error("Sorry, only the following encodings are supported for Encode:\n",
 RecNames(UNICODE_RECODE.Encoder), "\n");
 fi;
 enc := UNICODE_RECODE.NormalizedEncodings.(enc);
 return UNICODE_RECODE.Encoder.(enc)(ustr);
end);
# generic dispatcher for encoding depending on extra data (e.g. used with "URL"
InstallOtherMethod(Encode, [IsUnicodeString, IsString, IsObject],
function(ustr, enc, data)
 if not IsBound(UNICODE_RECODE.NormalizedEncodings.(enc)) then
 Error("Sorry, only the following encodings are supported for Encode:\n",
 RecNames(UNICODE_RECODE.Encoder), "\n");
 fi;
 enc := UNICODE_RECODE.NormalizedEncodings.(enc);
 return UNICODE_RECODE.Encoder.(enc)(ustr, data);
end);

# here the default is UTF-8 encoding
InstallMethod(Encode, [IsUnicodeString], function(ustr)
 return UNICODE_RECODE.Encoder.("UTF-8")(ustr);
end);

## <#GAPDoc Label="WidthUTF8String">
## <ManSection >
## <Heading>Lengths of UTF-8 strings</Heading>
## <Func Arg="str" Name="WidthUTF8String" />
## <Func Arg="str" Name="NrCharsUTF8String" />
## <Returns>an integer</Returns>
## <Description>
## Let <A>str</A> be a &GAP; string with text in UTF-8 encoding. There are
## three <Q>lengths</Q> of such a string which must be distinguished. The
## operation <Ref BookName="reference" Oper="Length"/> returns the number of
## bytes and so the memory occupied by <A>str</A>. The function <Ref
## Func="NrCharsUTF8String"/> returns the number of unicode characters in
## <A>str</A>, that is the length of <C>Unicode(<A>str</A>)</C>. 
##
## In many applications the function <Ref Func="WidthUTF8String"/> is more
## interesting, it returns the number of columns needed by the string if
## printed to a terminal. This takes into account that some unicode
## characters are combining characters and that there are wide characters
## which need two columns (e.g., for Chinese or Japanese). (To be precise:
## This implementation assumes that there are no control characters in
## <A>str</A> and uses the character width returned by the <C>wcwidth</C>
## function in the GNU C-library called with UTF-8 locale.)
## <Example>
## gap> # A, German umlaut u, B, zero width space, C, newline
## gap> str := Encode( Unicode( "AüBC\n", "XML" ) );;
## gap> Print(str);
## AüBC
## gap> # umlaut u needs two bytes and the zero width space three
## gap> Length(str);
## 9
## gap> NrCharsUTF8String(str);
## 6
## gap> # zero width space and newline don't contribute to width
## gap> WidthUTF8String(str);
## 4
## </Example>
## </Description>
## </ManSection>
## <#/GAPDoc>
InstallGlobalFunction(NrCharsUTF8String, function(str)
 local n, nc, c;
 n := 0;
 for c in str do
 nc := INT_CHAR(c);
 if nc < 128 or nc > 191 then
 n := n+1;
 fi;
 od;
 return n;
end);

InstallGlobalFunction(WidthUTF8String, function(str)
 local ustr, res, pos, i;
 if not IsUnicodeString(str) then
 ustr := Unicode(str, "UTF-8");
 if ustr = fail then
 Error("string str is not in UTF-8 encoding.\n");
 else
 str := ustr;
 fi;
 fi;
 str := IntListUnicodeString(str);
 res := 0;
 for i in str do
 if i > 31 and i < 127 then
 res := res+1;
 else
 pos := PositionSorted(WidthUnicodeTable, [i]);
 if not IsBound(WidthUnicodeTable[pos]) or WidthUnicodeTable[pos][1] <> i
 then
 pos := pos-1;
 fi;
 res := res + WidthUnicodeTable[pos][2];
 fi;
 od;
 return res;
end);

## <#GAPDoc Label="InitialSubstringUTF8String">
## <ManSection >
## <Func Arg="str, maxwidth[, suf]" Name="InitialSubstringUTF8String" />
## <Returns>UTF-8 encoded string</Returns>
## <Description>
## The arguments <A>str</A> and <A>suf</A> each must be a &GAP; string with
## text in UTF-8 encoding or a unicode string. The argument <A>suf</A> is
## optional and its default value is the empty string. If the visible width
## of <A>str</A> is at most <A>maxwidth</A> then <A>str</A> is returned as
## UTF-8 encoded &GAP; string. Otherwise, <A>suf</A> is appended to the
## maximal initial substring of <A>str</A> such that the total visible
## width of the result is at most <A>maxwidth</A>.
##
## <Example>
## gap> # A, German umlaut u, B, zero width space, C, newline
## gap> str := Encode( Unicode( "AüBC\n", "XML" ) );;
## gap> ini := InitialSubstringUTF8String(str, 3);;
## gap> WidthUTF8String(ini);
## 3
## gap> IntListUnicodeString(Unicode(ini));
## [ 65, 252, 66, 8203 ]
## gap> l := Unicode([ 23380, 22827, 23376 ] );; # three chars of width 2
## gap> s := InitialSubstringUTF8String(l, 4, "*");;
## gap> WidthUTF8String(s);
## 3
## </Example>
## </Description>
## </ManSection>
## <#/GAPDoc>
InstallGlobalFunction(InitialSubstringUTF8String, function(str, len, lsuf...)
 local u, suf, wsuf, cut, ints, sum, pos, j;
 if not IsUnicodeString(str) then
 u := Unicode(str, "UTF-8");
 if u = fail then
 Error("string str is not in UTF-8 encoding");
 fi;
 ints := IntListUnicodeString(u);
 else
 ints := IntListUnicodeString(str);
 fi;
 if Length(lsuf) > 0 then
 if IsUnicodeString(lsuf[1]) then
 suf := Encode(lsuf[1], "UTF-8");
 else
 suf := lsuf[1];
 fi;
 else
 suf := "";
 fi;
 wsuf := WidthUTF8String(suf);
 sum := 0;
 cut := -1;
 for j in [1..Length(ints)] do
 if ints[j] > 31 and ints[j] < 127 then
 sum := sum + 1;
 else
 pos := PositionSorted(WidthUnicodeTable, [ints[j]]);
 if not IsBound( WidthUnicodeTable[ pos ] ) or
 WidthUnicodeTable[pos][1] <> ints[j] then
 pos := pos-1;
 fi;
 sum := sum + WidthUnicodeTable[pos][2];
 fi;
 if sum+wsuf > len and cut < 0 then
 cut := j-1;
 fi;
 if sum > len then
 return Concatenation(Encode(Unicode(ints{[1..cut]})), suf);
 fi;
 od;
 if IsString(str) then
 return str;
 else
 return Encode(str, "UTF-8");
 fi;
end);

# Not (yet?) documented utility to translate a latin1 or UTF-8 encoded
# GAP string to a string with lowercase ASCII characters. Can be used for
# sorting and searching, allowing some freedom for the input of non-ASCII
# characters.
InstallGlobalFunction(LowerASCIIString, function(str)
 local u;
 # heuristic to distinguish UTF-8 and latin1
 u := Unicode(str);
 if u = fail then
 u := Unicode(str, "latin1");
 fi;
 u := SimplifiedUnicodeString(u, "ASCII");
 u := LowercaseUnicodeString(u);
 return Encode(u);
end);

# overwrite library method if sensible depending on term encoding and
# encoding of string
InstallMethod(ViewObj, "IsString", true, [IsString and IsFinite],0,
function(s)
 local u, c;
 u := Unicode(s, GAPInfo.TermEncoding);
 if u <> fail then
 Print("\"");
 u := IntListUnicodeString(u);
 for c in u do
 if c = 34 then
 Print("\\\"");
 elif c = 92 then
 Print("\\\\");
 elif c < 32 then
 Print(SPECIAL_CHARS_VIEW_STRING[2][c+1]);
 else
 Print(Encode(Unicode([c]), GAPInfo.TermEncoding));
 fi;
 od;
 Print("\"");
 else
 PrintObj(s);
 fi;
end);

Quelle UnicodeTools.gi Sprache: unbekannt

[zur Elbe Produktseite wechseln0.670QuellennavigatorsAnalyse erneut starten2026-06-14]