attempt to streamline TeX-compliant expansions, back off from Alignment

lib/LaTeXML/Core/Gullet.pm

@@ -244,7 +244,7 @@ sub unread {
 #  will step to the next input stream (Mouth) if one is available,
 # If $commentsok is true, will also pass comments.
 sub readXToken {
-  my ($self, $toplevel, $commentsok) = @_;
+  my ($self, $toplevel, $commentsok, $undefined_allowed) = @_;
   $toplevel = 1 unless defined $toplevel;
   return shift(@{ $$self{pending_comments} }) if $commentsok && @{ $$self{pending_comments} };
   my ($token, $cc, $defn);
@@ -272,7 +272,7 @@ sub readXToken {
                 : ($r eq 'LaTeXML::Core::Tokens' ? @$_
                   : Fatal('misdefined', $r, undef, "Expected a Token, got " . Stringify($_))))) }
             @{$r}); } }
-    elsif ($cc == CC_CS && !(LaTeXML::Core::State::lookupMeaning($STATE, $token))) {
+    elsif ($cc == CC_CS && !$undefined_allowed && !(LaTeXML::Core::State::lookupMeaning($STATE, $token))) {
       Error('undefined', $token, $self, "The token " . Stringify($token) . " is not defined during expansion. Consuming it and proceeding, expect trouble...");
       return; }
     else {

lib/LaTeXML/Core/Parameter.pm

@@ -152,7 +152,7 @@ sub revert {
 
 __END__
 
-=pod 
+=pod
 
 =head1 NAME
 

lib/LaTeXML/Package/TeX.pool.ltxml

@@ -2804,8 +2804,11 @@ DefPrimitiveI('\@@open@inner@column', undef, sub {
     my @savedtokens = ();
     $$colspec{empty} = 0;           # Assume the column isn't empty
                                     # Scan for leading \omit, skipping over (& saving) \hline.
+         # TODO: This loop of expansions actually allows for undefined command sequences to be present
+      #      as its timing is a little artificial. E.g. it would read locally scoped macros before they are
+      #      activated by the before=> templates of columns.
 
-    while (my $tok = $gullet->readXToken(0)) {
+    while (my $tok = $gullet->readXToken(0, 0, 1)) {
       if ($tok->equals(T_SPACE)) { }    # Skip leading space
       elsif (grep { $tok->equals($_) } @line_tokens) {    # Save line commands
         push(@lines, $stomach->invokeToken($tok)); }

lib/LaTeXML/Package/calc.sty.ltxml

@@ -25,6 +25,13 @@ use LaTeXML::Package;
 ## to expand its arguments, we could define each macro to expand into its name, and have chekcs on
 ## "ToString($op) eq 'ratio' ", etc.
 
+DefMacro('\minof',    '\noexpand\minof');
+DefMacro('\maxof',    '\noexpand\maxof');
+DefMacro('\widthof',  '\noexpand\widthof');
+DefMacro('\heightof', '\noexpand\heightof');
+DefMacro('\ratio',    '\noexpand\ratio');
+DefMacro('\real',     '\noexpand\real');
+
 # \setcounter{<ctr>}{<integer expression>}
 DefPrimitive('\setcounter{}{}', sub {
     my ($stomach, $ctr, $arg) = @_;
@@ -147,7 +154,7 @@ sub readTerm {
 sub readValue {
   my ($gullet, $type) = @_;
   $gullet->skipSpaces;
-  my $peek = $gullet->readToken();
+  my $peek = $gullet->readXToken();
   # Evaluate <Text Dimen Factors>
   if (Equals($peek, T_CS('\widthof'))) {
     my $box = Digest($gullet->readArg);
@@ -203,7 +210,7 @@ sub readValue {
 sub readMinMax {
   my ($gullet, $type) = @_;
   $gullet->skipSpaces;
-  my $peek = $gullet->readToken();
+  my $peek = $gullet->readXToken();
   if (Equals($peek, T_CS('\minof'))) {
     my $x = readExpression($gullet, $type, $gullet->readArg);
     my $y = readExpression($gullet, $type, $gullet->readArg);
@@ -219,7 +226,7 @@ sub readMinMax {
 sub readTextDimenFactor {
   my ($gullet) = @_;
   $gullet->skipSpaces;
-  my $peek = $gullet->readToken();
+  my $peek = $gullet->readXToken();
   if (Equals($peek, T_CS('\widthof'))) {
     my $box = Digest($gullet->readArg);
     return Dimension(0); }
@@ -247,7 +254,7 @@ sub readParenthesized {
 sub readReal {
   my ($gullet) = @_;
   $gullet->skipSpaces;
-  my $peek = $gullet->readToken();
+  my $peek = $gullet->readXToken();
   if (Equals($peek, T_CS('\real'))) {
     my $arg = $gullet->readArg;
     return $gullet->readingFromMouth(LaTeXML::Core::Mouth->new(), sub {

lib/LaTeXML/Package/siunitx.sty.ltxml

@@ -102,8 +102,8 @@ DefPrimitiveI('\lx@six@initialize', undef, sub {
       InputDefinitions('siunitx-abbreviations', type => 'cfg', noltxml => 1); }
     if (six_getBool('binary-units')) {
       InputDefinitions('siunitx-binary', type => 'cfg', noltxml => 1); }
-    # if (six_getBool('free-standing-units')) { # ??
-    six_enableUnitMacros(six_getBool('overwrite-functions'));    # }
+    if (six_getBool('free-standing-units')) {
+      six_enableUnitMacros(six_getBool('overwrite-functions')); }
     return; });
 
 AtBeginDocument('\lx@six@initialize');
@@ -503,7 +503,9 @@ sub six_parse_begin {
   my $stomach = $STATE->getStomach;
   $stomach->bgroup;
   six_setup($kv) if $kv;
-  map { Let($_, T_CS('\relax')); } six_get('input-protect-tokens')->unlist;
+  for my $token (six_get('input-protect-tokens')->unlist) {
+    my $name = $token->getCSName;
+    Let($token, T_OTHER($name)); }
   return; }
 
 sub six_parse_end {
@@ -1161,7 +1163,7 @@ DefMacro('\lx@six@highlight{}', '\lx@six@unitobject@arg{highlight}{#1}');
 DefColumnType('s Optional', sub {
     my ($gullet, $kv) = @_;
     $LaTeXML::BUILD_TEMPLATE->addColumn(
-      before => Tokens(T_CS('\lx@si@column@begin'), ($kv ? (T_OTHER('['), $kv, T_OTHER(']')) : ())),
+      before => Tokens(T_CS('\lx@sicolumn@begin'), ($kv ? (T_OTHER('['), $kv, T_OTHER(']')) : ()), T_LETTER("sismall")),
       #      after => Tokens(T_CS('\lx@si@column@end')),
       #      align => 'char:' . ToString(Digest(T_CS('\nprt@decimal')))
     );
@@ -1170,25 +1172,30 @@ DefColumnType('s Optional', sub {
 DefColumnType('S Optional', sub {
     my ($gullet, $kv) = @_;
     $LaTeXML::BUILD_TEMPLATE->addColumn(
-      before => Tokens(T_CS('\lx@SI@column@begin'), ($kv ? (T_OTHER('['), $kv, T_OTHER(']')) : ())),
+      before => Tokens(T_CS('\lx@sicolumn@begin'), ($kv ? (T_OTHER('['), $kv, T_OTHER(']')) : ()), T_LETTER("SILARGE")),
       #      after => Tokens(T_CS('\lx@SI@column@end')),
       #      align => 'char:' . ToString(Digest(T_CS('\nprt@decimal'))));
     );
     return; });
 
-DefMacro('\lx@si@column@begin Optional XUntil:\@@close@inner@column',
-  '\def\@@eat@space{}\lx@table@si[#1]{#2}\@@close@inner@column');
-DefMacro('\lx@SI@column@begin Optional XUntil:\@@close@inner@column',
-  '\def\@@eat@space{}\lx@table@num[#1]{#2}\@@close@inner@column');
+DefMacro('\lx@sicolumn@begin OptionalKeyVals:SIX Token', sub {
+    my ($gullet, $kv, $type) = @_;
+    six_parse_begin($gullet, $kv);
+    if (ToString($type) eq 'sismall') {
+      six_enableUnitMacros(1);
+      return (T_CS('\lx@column@next'), T_CS('\lx@table@si')); }
+    else {
+      return (T_CS('\lx@column@next'), T_CS('\lx@table@num')); }
+});
+DefMacro('\lx@column@next Token XUntil:\@@close@inner@column', '\def\@@eat@space{}#1{#2}\@@close@inner@column');
 
 # TODO: These two need to deal better with unrecognized arguments.
 # Generally they should NOT be in math mode... (but sometimes, still?)
 
 # Similar to \num, no error...
 # color treated a bit differently?
-DefMacro('\lx@table@num OptionalKeyVals:SIX {}', sub {
-    my ($gullet, $kv, $number) = @_;
-    six_parse_begin($gullet, $kv);
+DefMacro('\lx@table@num {}', sub {
+    my ($gullet, $number) = @_;
     my @tokens  = Expand($number)->unlist;
     my $doparse = six_getBool('parse-numbers');
     # Deal with recognizing "surrounding material"
@@ -1226,10 +1233,8 @@ DefMacro('\lx@table@num OptionalKeyVals:SIX {}', sub {
       @post); });
 
 # similar to \si
-DefMacro('\lx@table@si OptionalKeyVals:SIX {}', sub {
-    my ($gullet, $kv, $units) = @_;
-    six_parse_begin($gullet, $kv);
-    six_enableUnitMacros(1);
+DefMacro('\lx@table@si {}', sub {
+    my ($gullet, $units) = @_;
     my @tokens = Expand($units)->unlist;
     my @pre    = ();
     my @post   = ();