diff --git a/entities/global.ent b/entities/global.ent
deleted file mode 100644
index b5e94d1f1..000000000
--- a/entities/global.ent
+++ /dev/null
@@ -1,655 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-superglobals'>
-__autoload'>
-
-PECL'>
-
-
-
-
-
-true'>
-false'>
-null'>
-
-array'>
-int'>
-string'>
-bool'>
-float'>
-object'>
-resource'>
-mixed'>
-never'>
-
-
-throw'>
-try'>
-catch'>
-finally'>
-
-
-function'>
-return'>
-
-
-namespace'>
-use'>
-
-
-const'>
-declare'>
-foreach'>
-global'>
-instanceof'>
-new'>
-yield'>
-yield from'>
-
-intcallbackmixedamixedb'>
-
-php.ini'>
-.user.ini'>
-httpd.conf'>
-.htaccess'>
-
-
-API'>
-ASCII'>
-CA'>
-CGI'>
-CLI'>
-CLI SAPI'>
-cURL'>
-DER'>
-DNS'>
-DOM'>
-FPM'>
-FTP'>
-FTPS'>
-HMAC'>
-HTML'>
-HTTP'>
-HTTPS'>
-HSTS'>
-IP'>
-IPv4'>
-IPv6'>
-JSON'>
-LDAP'>
-PDO'>
-PEM'>
-Phar'>
-PHP'>
-POP3'>
-RTSP'>
-SAPI'>
-SFTP'>
-SMTP'>
-SPL'>
-SSH'>
-SSL'>
-TCP'>
-TLS'>
-UDP'>
-URI'>
-URL'>
-XML'>
diff --git a/scripts/entities.php b/scripts/entities.php
index a4d373137..8005862ea 100644
--- a/scripts/entities.php
+++ b/scripts/entities.php
@@ -12,27 +12,42 @@
+----------------------------------------------------------------------+
| Authors: André L F S Bacci |
+----------------------------------------------------------------------+
-| Description: Collect individual entities into an entities.ent file. |
+| Description: Collect individual entities into an .entities.ent file. |
+----------------------------------------------------------------------+
# Mental model, or things that I would liked to know 20 years prior
XML Entity processing has more in common with DOMDocumentFragment than
-DOMElement. In other words, simple text and multi roots XML entities
+DOMElement. In other words, simple text and multi rooted XML files
are valid contents, whereas they are not valid XML documents.
Also, namespaces do not automatically "cross" between a parent
document and their includes, even if they are included in the same
-file, as local textual entities. They are, for all intended purposes,
-separated documents, with separated namespaces and have *expected*
-different *default* namespaces.
+file, as local textual entities. s are, for all intended
+purposes, separated documents, with separated namespaces and have
+*expected* different default namespaces.
So each one of, possibly multiple, "root" XML elements inside an
fragment need to be annotated with default namespace, even if the
"root" element occurs surrounded by text. For example:
- "texttext", need one namespace, or it is invalid, and;
-- "", need TWO namespaces, or it is also invalid.
+
+# Output
+
+This script collects bundled and individual entity files (detailed
+below), at some expected relative paths, and generates an
+.entities.ent file, in a sibling position to manual.xml.in.
+
+The output .entities.ent file has no duplications, so collection
+order is important to keep the necessary operational semantics. Here,
+newer loaded entities takes priority (overwrites) over previous one.
+Note that this is the reverse of convention, where
+duplicated entity names are ignored. The priority order used here
+is important to allow detecting cases where "constant" entities
+are being overwriten, or if translatable entities are missing
+translations.
# Individual tracked entities, or `.xml` files at `entities/`
@@ -44,20 +59,22 @@
and second, this allows normal revision tracking on then, without
requiring weird changes on `revcheck.php`.
-# Small entities, group tracked (future)
+# Bundled entities files, group tracked
-For very small textual entities, down to simple text words, that may
-never change, having tracking for each instance is an overkill.
+For very small textual entities, down to simple text words or single
+tag elements, that may never change, individual entity tracking is
+an overkill. This script also loads bundled entities files, at
+some expected locations, with specific semantics.
-It's planned to have new `manual.ent` and `website.ent` files
-on each doc language, that internally are valid XML documents and
-also replicates namespace declarations used on manual.xml.in, so
-it will possible migrate the current infrastructure
-to something that is more consumable for XML toolage (and will
-avoid most of it not all XML namespacing hell).
+These bundle files are really normal XML files, correctly annotated
+with XML namespaces used on manual, so any individual exported entity
+have corret XML namespace annotations. These bundle entity files
+are revcheck tracked normaly, but are not included in manual.xml.in,
+as they only participate in general entity loading, described above.
-These small files are to be splited into entities/ as individial
-.tmp text files, for normal inclusion on manual.
+- global.ent - expected untranslated
+- manual.ent - expected translated
+- lang/entities/* - expected translated
*/
@@ -65,86 +82,248 @@
ini_set( 'display_startup_errors' , 1 );
error_reporting( E_ALL );
+const PARTIAL_IMPL = true; // For while spliting and bundle convertion are incomplete
+
if ( count( $argv ) < 2 || in_array( '--help' , $argv ) || in_array( '-h' , $argv ) )
{
- fwrite( STDERR , "\nUsage: {$argv[0]} entitiesDir [entitiesDir]\n\n" );
+ fwrite( STDERR , "\nUsage: {$argv[0]} [--debug] entitiesDir [entitiesDir]\n\n" );
return;
}
-$filename = __DIR__ . "/../.entities.ent"; // sibling of .manual.xml
-touch( $filename ); // empty file at minimum, and because
-$filename = realpath( $filename ); // realpath() fails if file does not exist.
-
-$entities = []; // all entities, already replaced
-$expected = []; // entities that are expected to be replaced/translated
-$foundcnt = []; // tracks how many times entity name was found
+$filename = Entities::rotateOutputFile();
$langs = [];
-$detail = false;
+$normal = true; // configure.php mode
+$debug = false; // detailed output
for( $idx = 1 ; $idx < count( $argv ) ; $idx++ )
- if ( $argv[$idx] == "--detail" )
- $detail = true;
+ if ( $argv[$idx] == "--debug" )
+ {
+ $normal = false;
+ $debug = true;
+ }
else
$langs[] = $argv[$idx];
-if ( ! $detail )
- print "Creating file $filename...";
+if ( $normal )
+ print "Creating .entities.ent...";
+else
+ print "Creating .entities.ent in debug mode.\n";
-for ( $run = 0 ; $run < count( $langs ) ; $run++ )
- parseDir( $langs[$run] , ( count( $langs ) && $run == 0 ) );
+loadEnt( __DIR__ . "/../global.ent" , global: true );
+foreach( $langs as $lang )
+{
+ loadEnt( __DIR__ . "/../../$lang/global.ent" , global: true );
+ loadEnt( __DIR__ . "/../../$lang/manual.ent" , translate: true );
+ loadEnt( __DIR__ . "/../../$lang/remove.ent" , remove: true );
+ loadDir( $langs , $lang );
+}
-dump( $filename , $entities );
-[$all, $unt, $over] = verifyReplaced( $detail );
+Entities::writeOutputFile();
+Entities::checkReplaces( $debug );
-if ( ! $detail )
+echo " done: " , Entities::$countTotalGenerated , " entities";
+if ( Entities::$countUnstranslated > 0 )
+ echo ", " , Entities::$countUnstranslated , " untranslated";
+if ( Entities::$countConstantReplaced > 0 )
+ echo ", " , Entities::$countConstantReplaced , " global replaced";
+if ( Entities::$countRemoveReplaced > 0 )
+ echo ", " , Entities::$countRemoveReplaced , " to be removed";
+echo ".\n";
+
+exit;
+
+class EntityData
{
- echo " done";
- if ( $unt + $over > 0 )
- echo ": $all entities, $unt untranslated, $over overwrites.";
- echo "\n";
+ public function __construct(
+ public string $path ,
+ public string $name ,
+ public string $text ) {}
}
-exit;
-function parseDir( string $dir , bool $expectedReplaced )
+class Entities
+{
+ public static int $countConstantReplaced = 0;
+ public static int $countUnstranslated = 0;
+ public static int $countRemoveReplaced = 0;
+ public static int $countTotalGenerated = 0;
+
+ private static string $filename = __DIR__ . "/../.entities.ent"; // sibling of .manual.xml
+
+ private static array $entities = []; // All entities, overwriten
+ private static array $global = []; // Entities from global.ent files
+ private static array $replace = []; // Entities expected replaced / translated
+ private static array $remove = []; // Entities expected removed
+ private static array $count = []; // Name / Count
+ private static array $slow = []; // External entities, slowless, overwrite
+
+ static function put( string $path , string $name , string $text , bool $global = false , bool $replace = false , bool $remove = false )
+ {
+ $entity = new EntityData( $path , $name , $text );
+ Entities::$entities[ $name ] = $entity;
+
+ if ( $global )
+ Entities::$global[ $name ] = $name;
+
+ if ( $replace )
+ Entities::$replace[ $name ] = $name;
+
+ if ( $remove )
+ Entities::$remove[ $name ] = $name;
+
+ if ( ! isset( Entities::$count[$name] ) )
+ Entities::$count[$name] = 1;
+ else
+ Entities::$count[$name]++;
+ }
+
+ static function slow( string $path )
+ {
+ if ( isset( $slow[$path] ) )
+ fwrite( STDERR , "External entity file physically overwrote: $path\n" );
+ $slow[ $path ] = $path;
+ }
+
+ static function rotateOutputFile()
+ {
+ if ( file_exists( Entities::$filename ) )
+ unlink( Entities::$filename );
+ touch( Entities::$filename );
+
+ Entities::$filename = realpath( Entities::$filename ); // only full paths on XML
+ }
+
+ static function writeOutputFile()
+ {
+ saveEntitiesFile( Entities::$filename , Entities::$entities );
+ }
+
+ static function checkReplaces( bool $debug )
+ {
+ Entities::$countTotalGenerated = count( Entities::$entities );
+ Entities::$countConstantReplaced = 0;
+ Entities::$countUnstranslated = 0;
+ Entities::$countRemoveReplaced = 0;
+
+ foreach( Entities::$entities as $name => $text )
+ {
+ $replaced = Entities::$count[$name] - 1;
+ $expectedConstant = in_array( $name , Entities::$global );
+ $expectedReplaced = in_array( $name , Entities::$replace );
+ $expectedRemoved = in_array( $name , Entities::$remove );
+
+ if ( $expectedConstant && $replaced != 0 )
+ {
+ Entities::$countConstantReplaced++;
+ if ( $debug )
+ print "Expected global, replaced $replaced times:\t$name\n";
+ }
+
+ if ( $expectedReplaced && $replaced != 1 )
+ {
+ Entities::$countUnstranslated++;
+ if ( $debug )
+ print "Expected translated, replaced $replaced times:\t$name\n";
+ }
+
+ elseif ( $expectedRemoved && $replaced != 0 )
+ {
+ Entities::$countRemoveReplaced++;
+ if ( $debug )
+ print "Expected removed, replaced $replaced times:\t$name\n";
+ }
+ }
+ }
+}
+
+function loadEnt( string $path , bool $global = false , bool $translate = false , bool $remove = false )
{
- if ( ! is_dir( $dir ) )
- exit( "Not a directory: $dir\n" );
+ $absolute = realpath( $path );
+ if ( $absolute === false )
+ if ( PARTIAL_IMPL )
+ return;
+ else
+ exit( "Not directory: $path\n" );
+ $path = $absolute;
+
+ $text = file_get_contents( $path );
+ $text = str_replace( "&" , "&" , $text );
+
+ $dom = new DOMDocument( '1.0' , 'utf8' );
+ if ( ! $dom->loadXML( $text ) )
+ die( "XML load failed for $path\n" );
+
+ $xpath = new DOMXPath( $dom );
+ $list = $xpath->query( "/*/*" );
+
+ foreach( $list as $ent )
+ {
+ // weird, namespace correting, DOMNodeList -> DOMDocumentFragment
+ $other = new DOMDocument( '1.0' , 'utf8' );
+
+ foreach( $ent->childNodes as $node )
+ $other->appendChild( $other->importNode( $node , true ) );
+
+ $name = $ent->getAttribute( "name" );
+ $text = $other->saveXML();
+
+ $text = str_replace( "&" , "&" , $text );
+ $lines = explode( "\n" , $text );
+ array_shift( $lines ); // remove XML declaration
+ array_pop( $lines ); // remove spurious EOL
+ $text = implode( "\n" , $lines );
+
+ Entities::put( $path , $name , $text , $global , $translate , $remove );
+ }
+}
+
+function loadDir( array $langs , string $lang )
+{
+ global $debug;
+
+ $dir = __DIR__ . "/../../$lang/entities";
+ $dir = realpath( $dir );
+ if ( $dir === false || ! is_dir( $dir ) )
+ if ( PARTIAL_IMPL )
+ {
+ if ( $debug )
+ print "Not a directory: $dir\n";
+ return;
+ }
+ else
+ exit( "Not directory: $dir\n" );
- $count = 0;
$files = scandir( $dir );
+ $expectedReplaced = array_search( $lang , $langs ) > 0;
foreach( $files as $file )
{
- if ( str_starts_with( $file , '.' ) )
- continue;
-
$path = realpath( "$dir/$file" );
if ( is_dir( $path ) )
continue;
+ if ( str_starts_with( $file , '.' ) )
+ continue;
$text = file_get_contents( $path );
- validateStore( $path , $text , $expectedReplaced );
- $count++;
- }
+ $text = rtrim( $text , "\n" );
- global $detail;
- if ( $detail )
- echo "$count files on $dir\n";
+ loadXml( $path , $text , $expectedReplaced );
+ }
}
-function validateStore( string $path , string $text , bool $expectedReplaced )
+function loadXml( string $path , string $text , bool $expectedReplaced )
{
- $trim = trim( $text );
- if ( strlen( $trim ) == 0 )
+ if ( trim( $text ) == "" )
{
- // Yes, there are empty entities, and they are valid entities, but not valid XML.
- // see: en/language-snippets.ent mongodb.note.queryable-encryption-preview
- push( $path , $text , $expectedReplaced , true );
+ fwrite( STDERR , "\n Empty entity '$path'. Should it be in remove.ent?\n" );
+ Entities::put( $pat , $text , replace: $expectedReplaced );
return;
}
+ $info = pathinfo( $path );
+ $name = $info["filename"];
+
$frag = "$text";
$dom = new DOMDocument( '1.0' , 'utf8' );
@@ -169,47 +348,15 @@ function validateStore( string $path , string $text , bool $expectedReplaced )
return;
}
- push( $path , $text , $expectedReplaced );
-}
-
-class EntityData
-{
- public function __construct(
- public string $path ,
- public string $name ,
- public string $text ) {}
+ Entities::put( $path , $name , $text , replace: $expectedReplaced );
}
-function push( string $path , string $text , bool $expectedReplaced )
+function saveEntitiesFile( string $filename , array $entities )
{
-
- global $entities;
- global $expected;
- global $foundcnt;
-
- $info = pathinfo( $path );
- $name = $info["filename"];
-
- if ( $expectedReplaced )
- $expected[] = $name;
-
- if ( ! isset( $foundcnt[$name] ) )
- $foundcnt[$name] = 1;
- else
- $foundcnt[$name]++;
-
- $entity = new EntityData( $path , $name , $text );
- $entities[$name] = $entity;
-}
-
-function dump( string $filename , array $entities )
-{
- // In PHP 8.4 may be possible to construct an extended
- // DOMEntity class with writable properties. For now,
- // creating entities files directly by hand.
+ $tmpDir = __DIR__ . "/entities";
$file = fopen( $filename , "w" );
- fputs( $file , "\n\n\n" );
+ fputs( $file , "\n\n\n" );
foreach( $entities as $name => $entity )
{
@@ -228,42 +375,18 @@ function dump( string $filename , array $entities )
// as an external file to avoid (re)quotation hell.
if ( $quote == "" )
+ {
+ if ( $entity->path == "" )
+ {
+ $entity->path = $tmpDir . "/{$entity->path}.tmp";
+ file_put_contents( $entity->path , $text );
+ }
fputs( $file , "path}'>\n\n" );
+ Entities::slow( $entity->path );
+ }
else
fputs( $file , "\n\n" );
}
fclose( $file );
}
-
-function verifyReplaced( bool $outputDetail )
-{
- global $entities;
- global $expected;
- global $foundcnt;
-
- $countUntranslated = 0;
- $countConstantChanged = 0;
-
- foreach( $entities as $name => $text )
- {
- $replaced = $foundcnt[$name] - 1 ;
- $expectedReplaced = in_array( $name , $expected );
-
- if ( $expectedReplaced && $replaced != 1 )
- {
- $countUntranslated++;
- if ( $outputDetail )
- print "Expected translated, replaced $replaced times:\t$name\n";
- }
-
- elseif ( ! $expectedReplaced && $replaced != 0 )
- {
- $countConstantChanged++;
- if ( $outputDetail )
- print "Unexpected replaced, replaced $replaced times:\t$name\n";
- }
- }
-
- return [count( $entities ), $countUntranslated, $countConstantChanged];
-}
diff --git a/scripts/split-ent.php b/scripts/split-ent.php
deleted file mode 100644
index 582f159ff..000000000
--- a/scripts/split-ent.php
+++ /dev/null
@@ -1,93 +0,0 @@
- |
-+----------------------------------------------------------------------+
-| Description: Split an .ent file into individual files. |
-+----------------------------------------------------------------------+
-
-See `entities.php` source for detailed rationale.
-
-Use this for spliting `language-snippets-ent` or other "big" entities
-files into individual .xml files.
-
-Leave hash/user empty to generate files without revtag (doc-en). For
-translators, open issues instructing running this script with filled
-the generated hash and local user (or '_').
-
-After spliting, add the new directory entities/ with they contents,
-and remove `language-snippets-ent`, in one go.
-
-*/
-
-ini_set( 'display_errors' , 1 );
-ini_set( 'display_startup_errors' , 1 );
-error_reporting( E_ALL );
-
-if ( count( $argv ) < 4 )
- die(" Syntax: php $argv[0] infile outdir [hash user]\n" );
-
-$infile = $argv[1];
-$outdir = $argv[2];
-$hash = $argv[3] ?? "";
-$user = $argv[4] ?? "";
-
-$content = file_get_contents( $infile );
-$entities = [];
-
-// Parse
-
-$pos1 = 0;
-while ( true )
-{
- $pos1 = strpos( $content , " $text )
-{
- $file = "$outdir/$name.xml";
- if ( file_exists( $file ) )
- exit( "Name colision: $file\n" );
-}
-
-// Write
-
-foreach( $entities as $name => $text )
-{
- $file = "$outdir/$name.xml";
- file_put_contents( $file , $text );
-}
-
-$total = count( $entities );
-print "Generated $total files.\n";