Improve Make lexer (#1285)

This commit improves the Make lexer in the following ways:

- use`\t` in place of` \s`;
- support syntax variant `${}` as well as `$()`;
- support all built-in functions, not just `$(shell...)`;
- support include directive;
- support conditional directives `ifdef`, `ifndef`, `ifeq`, `ifneq`, 
  `else`, `endif`;
- support substitution references when expanding variables;
- permit macro names to start with digits; and
- register additional filename extensions.

lib/rouge/lexers/make.rb

@@ -8,9 +8,18 @@ class Make < RegexLexer
       desc "Makefile syntax"
       tag 'make'
       aliases 'makefile', 'mf', 'gnumake', 'bsdmake'
-      filenames '*.make', 'Makefile', 'makefile', 'Makefile.*', 'GNUmakefile'
+      filenames '*.make', '*.mak', '*.mk', 'Makefile', 'makefile', 'Makefile.*', 'GNUmakefile', '*,fe1'
       mimetypes 'text/x-makefile'
 
+      def self.functions
+        @functions ||= %w(
+          abspath addprefix addsuffix and basename call dir error eval file
+          filter filter-out findstring firstword flavor foreach if join lastword
+          notdir or origin patsubst realpath shell sort strip subst suffix value
+          warning wildcard word wordlist words
+        )
+      end
+
       # TODO: Add support for special keywords
       # bsd_special = %w(
       #   include undef error warning if else elif endif for endfor
@@ -28,15 +37,25 @@ def initialize(opts={})
 
         rule %r/#.*?\n/, Comment
 
-        rule %r/(export)(\s+)(?=[a-zA-Z0-9_\${}\t -]+\n)/ do
+        rule %r/([-s]?include)((?:[\t ]+[^\t\n #]+)+)/ do
+          groups Keyword, Literal::String::Other
+        end
+
+        rule %r/(ifn?def|ifn?eq)([\t ]+)([^#\n]+)/ do
+          groups Keyword, Text, Name::Variable
+        end
+
+        rule %r/(?:else|endif)[\t ]*(?=[#\n])/, Keyword
+
+        rule %r/(export)([\t ]+)(?=[\w\${}()\t -]+\n)/ do
           groups Keyword, Text
           push :export
         end
 
-        rule %r/export\s+/, Keyword
+        rule %r/export[\t ]+/, Keyword
 
         # assignment
-        rule %r/([a-zA-Z0-9_${}.-]+)(\s*)([!?:+]?=)/m do |m|
+        rule %r/([\w${}().-]+)([\t ]*)([!?:+]?=)/m do |m|
           token Name::Variable, m[1]
           token Text, m[2]
           token Operator, m[3]
@@ -52,9 +71,9 @@ def initialize(opts={})
       end
 
       state :export do
-        rule %r/[\w\${}-]/, Name::Variable
+        rule %r/[\w\${}()-]/, Name::Variable
         rule %r/\n/, Text, :pop!
-        rule %r/\s+/, Text
+        rule %r/[\t ]+/, Text
       end
 
       state :block_header do
@@ -70,6 +89,14 @@ def initialize(opts={})
       end
 
       state :block_body do
+        rule %r/(ifn?def|ifn?eq)([\t ]+)([^#\n]+)(#.*)?(\n)/ do
+          groups Keyword, Text, Name::Variable, Comment, Text
+        end
+
+        rule %r/(else|endif)([\t ]*)(#.*)?(\n)/ do
+          groups Keyword, Text, Comment, Text
+        end
+
         rule %r/(\t[\t ]*)([@-]?)/ do
           groups Text, Punctuation
           push :shell_line
@@ -80,22 +107,22 @@ def initialize(opts={})
 
       state :shell do
         # macro interpolation
-        rule %r/\$\(\s*[a-z_]\w*\s*\)/i, Name::Variable
-        # $(shell ...)
-        rule %r/(\$\()(\s*)(shell)(\s+)/m do
+        rule %r/\$[({][\t ]*\w[\w:=%.]*[\t ]*[)}]/i, Name::Variable
+        # function invocation
+        rule %r/(\$[({])([\t ]*)(#{Make.functions.join('|')})([\t ]+)/m do
           groups Name::Function, Text, Name::Builtin, Text
           push :shell_expr
         end
 
         rule(/\\./m) { delegate @shell }
-        stop = /\$\(|\(|\)|\\|$/
+        stop = /\$\(|\$\{|\(|\)|\}|\\|$/
         rule(/.+?(?=#{stop})/m) { delegate @shell }
         rule(stop) { delegate @shell }
       end
 
       state :shell_expr do
-        rule(/\(/) { delegate @shell; push }
-        rule %r/\)/, Name::Variable, :pop!
+        rule(/[({]/) { delegate @shell; push }
+        rule %r/[)}]/, Name::Function, :pop!
         mixin :shell
       end
 

spec/lexers/make_spec.rb

@@ -9,6 +9,9 @@
 
     it 'guesses by filename' do
       assert_guess :filename => 'foo.make'
+      assert_guess :filename => 'bar.mak'
+      assert_guess :filename => 'baz.mk'
+      assert_guess :filename => 'Make,fe1'
       assert_guess :filename => 'Makefile'
       assert_guess :filename => 'makefile'
       assert_guess :filename => 'Makefile.in'

spec/visual/samples/make

@@ -55,3 +55,60 @@ commands_switch:	all platform
 
 frameworkaltinstallunixtools:
 	cd Mac && $(MAKE) altinstallunixtools DESTDIR="$(DESTDIR)"
+
+include SubMake.make # Insert lines from another file (which must exist)
+-include Maybe.make  # Next 2 similar, but not an error if they doon't exist
+sinclude DynamicDependencies.make
+
+# Function expansions (including $(shell)) can occur anywhere
+$(shell echo VARIABLE) := $(shell echo value)
+
+$(shell echo $(TARGET1)) $(TARGET2) target3: \
+	$(shell echo $(DEPENDENCY1)) $(DEPENDENCY2) dependency3
+	touch $(shell echo target{1..3})
+	@echo $(shell echo 1)
+	@echo $(shell echo 2 )
+	@echo ${shell echo 3}
+	@echo ${shell echo 4 }
+
+# Many other built-in functions exist
+OBJECT_GOALS = $(filter %.o,${MAKECMDGOALS})
+
+# User-defined functions can be used via $(call)
+reverse = $(2) $(1)
+foo = $(call reverse,a,b) # foo contains 'b a'
+
+# Substitution references can be used when expanding variables
+SOURCE = $(OBJECT:.o=.c)
+
+# Alternative syntax for substitution references
+foo = input1.txt
+bar = $(foo:input%.txt=output%.bin) # set to 'output1.bin'
+
+# Conditionals
+ifdef THING
+THING2 = $(THING)
+else
+THING2 = default
+endif
+
+ifeq ($(TARGET),special) # syntax variant: parentheses and separating comma
+TARGET = something_else
+else
+	ifneq '$(TARGET)' "don't" # syntax variant: either argument can be single or double quoted
+TARGET = be_silly
+	endif # and can be indented, even with tabs (so long as not in a recipe)
+endif
+
+count:
+	echo one
+	echo two
+ifndef QUIET # conditionals can happen within recipes
+	echo miss a few
+endif
+	echo one hundred
+
+# Dollars must be escaped if a literal one is required
+
+print_path:
+	echo $$PATH