Merge branch 'master' of https://github.com/pygments/pygments

AUTHORS

@@ -237,5 +237,7 @@ Other contributors, listed alphabetically, are:
 * Hubert Gruniaux -- C and C++ lexer improvements
 * Thomas Symalla -- AMDGPU Lexer
 * 15b3 -- Image Formatter improvements
+* Fabian Neumann -- CDDL lexer
+* Thomas Duboucher -- CDDL lexer
 
 Many thanks for all contributions!

CHANGES

@@ -39,8 +39,26 @@ Version 2.8.0
 - Changed setuptools to use a declarative config through ``setup.cfg``.
   Building Pygments now requires setuptools 39.2+.
 - Added markdown to MarkdownLexer aliases (#1687)
+- Changed line number handling
+
+  * In ``<table>`` based output, the ``td.linenos`` element will have either a
+    ``normal`` or ``special`` class attached. Previously, only ``special`` line
+    numbers got a class. This prevents styles from getting applied twice -
+    once via ``<pre>``, once via ``<span class="special">``. This also means
+    that ``td.linenos pre`` is no longer styled, instead, use
+    ``td.linenos .normal`` and ``td.linenos .special``.
+  * In the "inline" style, the DOM element order was changed. The line number
+    is added first, then the line is wrapped is wrapped by the highlighter.
+    This fixes lines not being fully highlighted.
+  * The visual output for inline and non-inline line numbers & highlighting,
+    as well as class-based and inline styling is now consistent.
+  * Line number styles are set to ``background-color: transparent`` and
+    ``color: inherit`` by default. This works much better with dark styles
+    which don't have colors set for line numbers.
+
 - Removed "raw" alias from RawTokenLexer, so that it cannot be
   selected by alias.
+- Fixed RawTokenLexer to work in Python 3 and handle exceptions.
 - Added prompt colors to the Solarized theme (#1529)
 
 Version 2.7.4

doc/languages.rst

@@ -220,6 +220,7 @@ Other markup
 * BBCode
 * CapDL
 * `Cap'n Proto <https://capnproto.com>`_
+* `CDDL <https://datatracker.ietf.org/doc/rfc8610/>`_
 * CMake
 * `Csound <https://csound.com>`_ scores
 * CSS

pygments/formatters/html.py

@@ -552,9 +552,9 @@ def get_background_style_defs(self, arg=None):
     def get_linenos_style_defs(self):
         lines = [
             'pre { %s }' % self._pre_style,
-            'td.linenos pre { %s }' % self._linenos_style,
+            'td.linenos .normal { %s }' % self._linenos_style,
             'span.linenos { %s }' % self._linenos_style,
-            'td.linenos pre.special { %s }' % self._linenos_special_style,
+            'td.linenos .special { %s }' % self._linenos_special_style,
             'span.linenos.special { %s }' % self._linenos_special_style,
         ]
 
@@ -682,7 +682,7 @@ def _wrap_tablelinenos(self, inner):
                 if special_line:
                     style = ' class="special"'
                 else:
-                    style = ''
+                    style = ' class="normal"'
 
             if style:
                 line = '<span%s>%s</span>' % (style, line)
@@ -930,11 +930,16 @@ def format_unencoded(self, tokensource, outfile):
         linewise, e.g. line number generators.
         """
         source = self._format_lines(tokensource)
+
+        # As a special case, we wrap line numbers before line highlighting
+        # so the line numbers get wrapped in the highlighting tag.
+        if not self.nowrap and self.linenos == 2:
+            source = self._wrap_inlinelinenos(source)
+
         if self.hl_lines:
             source = self._highlight_lines(source)
+
         if not self.nowrap:
-            if self.linenos == 2:
-                source = self._wrap_inlinelinenos(source)
             if self.lineanchors:
                 source = self._wrap_lineanchors(source)
             if self.linespans:

pygments/formatters/img.py

@@ -452,6 +452,16 @@ def _get_text_color(self, style):
             fill = '#000'
         return fill
 
+    def _get_text_bg_color(self, style):
+        """
+        Get the correct background color for the token from the style.
+        """
+        if style['bgcolor'] is not None:
+            bg_color = '#' + style['bgcolor']
+        else:
+            bg_color = None
+        return bg_color
+
     def _get_style_font(self, style):
         """
         Get the correct font for the style.
@@ -474,14 +484,15 @@ def _draw_linenumber(self, posno, lineno):
             str(lineno).rjust(self.line_number_chars),
             font=self.fonts.get_font(self.line_number_bold,
                                      self.line_number_italic),
-            fill=self.line_number_fg,
+            text_fg=self.line_number_fg,
+            text_bg=None,
         )
 
-    def _draw_text(self, pos, text, font, **kw):
+    def _draw_text(self, pos, text, font, text_fg, text_bg):
         """
         Remember a single drawable tuple to paint later.
         """
-        self.drawables.append((pos, text, font, kw))
+        self.drawables.append((pos, text, font, text_fg, text_bg))
 
     def _create_drawables(self, tokensource):
         """
@@ -506,7 +517,8 @@ def _create_drawables(self, tokensource):
                         self._get_text_pos(linelength, lineno),
                         temp,
                         font = self._get_style_font(style),
-                        fill = self._get_text_color(style)
+                        text_fg = self._get_text_color(style),
+                        text_bg = self._get_text_bg_color(style),
                     )
                     temp_width, temp_hight = self.fonts.get_text_size(temp)
                     linelength += temp_width
@@ -576,8 +588,11 @@ def format(self, tokensource, outfile):
                 y = self._get_line_y(linenumber - 1)
                 draw.rectangle([(x, y), (x + rectw, y + recth)],
                                fill=self.hl_color)
-        for pos, value, font, kw in self.drawables:
-            draw.text(pos, value, font=font, **kw)
+        for pos, value, font, text_fg, text_bg in self.drawables:
+            if text_bg:
+                text_size = draw.textsize(text=value, font=font)
+                draw.rectangle([pos[0], pos[1], pos[0] + text_size[0], pos[1] + text_size[1]], fill=text_bg)
+            draw.text(pos, value, font=font, fill=text_fg)
         im.save(outfile, self.image_format.upper())
 
 

pygments/formatters/latex.py

@@ -299,13 +299,13 @@ def rgbcolor(col):
                 cmndef += (r'\def\$$@tc##1{\textcolor[rgb]{%s}{##1}}' %
                            rgbcolor(ndef['color']))
             if ndef['border']:
-                cmndef += (r'\def\$$@bc##1{\setlength{\fboxsep}{0pt}'
-                           r'\fcolorbox[rgb]{%s}{%s}{\strut ##1}}' %
+                cmndef += (r'\def\$$@bc##1{{\setlength{\fboxsep}{-\fboxrule}'
+                           r'\fcolorbox[rgb]{%s}{%s}{\strut ##1}}}' %
                            (rgbcolor(ndef['border']),
                             rgbcolor(ndef['bgcolor'])))
             elif ndef['bgcolor']:
-                cmndef += (r'\def\$$@bc##1{\setlength{\fboxsep}{0pt}'
-                           r'\colorbox[rgb]{%s}{\strut ##1}}' %
+                cmndef += (r'\def\$$@bc##1{{\setlength{\fboxsep}{0pt}'
+                           r'\colorbox[rgb]{%s}{\strut ##1}}}' %
                            rgbcolor(ndef['bgcolor']))
             if cmndef == '':
                 continue
@@ -321,8 +321,7 @@ def get_style_defs(self, arg=''):
         cp = self.commandprefix
         styles = []
         for name, definition in self.cmd2def.items():
-            styles.append(r'\expandafter\def\csname %s@tok@%s\endcsname{%s}' %
-                          (cp, name, definition))
+            styles.append(r'\@namedef{%s@tok@%s}{%s}' % (cp, name, definition))
         return STYLE_TEMPLATE % {'cp': self.commandprefix,
                                  'styles': '\n'.join(styles)}
 
@@ -342,7 +341,8 @@ def format_unencoded(self, tokensource, outfile):
                           (start and ',firstnumber=%d' % start or '') +
                           (step and ',stepnumber=%d' % step or ''))
         if self.mathescape or self.texcomments or self.escapeinside:
-            outfile.write(',codes={\\catcode`\\$=3\\catcode`\\^=7\\catcode`\\_=8}')
+            outfile.write(',codes={\\catcode`\\$=3\\catcode`\\^=7'
+                          '\\catcode`\\_=8\\relax}')
         if self.verboptions:
             outfile.write(',' + self.verboptions)
         outfile.write(']\n')

pygments/formatters/other.py

@@ -87,34 +87,32 @@ def format(self, tokensource, outfile):
             import gzip
             outfile = gzip.GzipFile('', 'wb', 9, outfile)
 
-            def write(text):
-                outfile.write(text.encode())
-            flush = outfile.flush
+            write = outfile.write
+            flush = outfile.close
         elif self.compress == 'bz2':
             import bz2
             compressor = bz2.BZ2Compressor(9)
 
             def write(text):
-                outfile.write(compressor.compress(text.encode()))
+                outfile.write(compressor.compress(text))
 
             def flush():
                 outfile.write(compressor.flush())
                 outfile.flush()
         else:
-            def write(text):
-                outfile.write(text.encode())
+            write = outfile.write
             flush = outfile.flush
 
         if self.error_color:
             for ttype, value in tokensource:
-                line = "%s\t%r\n" % (ttype, value)
+                line = b"%r\t%r\n" % (ttype, value)
                 if ttype is Token.Error:
                     write(colorize(self.error_color, line))
                 else:
                     write(line)
         else:
             for ttype, value in tokensource:
-                write("%s\t%r\n" % (ttype, value))
+                write(b"%r\t%r\n" % (ttype, value))
         flush()
 
 

pygments/lexers/_mapping.py

@@ -78,6 +78,7 @@
     'CapDLLexer': ('pygments.lexers.esoteric', 'CapDL', ('capdl',), ('*.cdl',), ()),
     'CapnProtoLexer': ('pygments.lexers.capnproto', "Cap'n Proto", ('capnp',), ('*.capnp',), ()),
     'CbmBasicV2Lexer': ('pygments.lexers.basic', 'CBM BASIC V2', ('cbmbas',), ('*.bas',), ()),
+    'CddlLexer': ('pygments.lexers.cddl', 'CDDL', ('cddl',), ('*.cddl',), ('text/x-cddl',)),
     'CeylonLexer': ('pygments.lexers.jvm', 'Ceylon', ('ceylon',), ('*.ceylon',), ('text/x-ceylon',)),
     'Cfengine3Lexer': ('pygments.lexers.configs', 'CFEngine3', ('cfengine3', 'cf3'), ('*.cf',), ()),
     'ChaiscriptLexer': ('pygments.lexers.scripting', 'ChaiScript', ('chai', 'chaiscript'), ('*.chai',), ('text/x-chaiscript', 'application/x-chaiscript')),

pygments/lexers/basic.py

@@ -351,10 +351,10 @@ class CbmBasicV2Lexer(RegexLexer):
         ]
     }
 
-    def analyse_text(self, text):
+    def analyse_text(text):
         # if it starts with a line number, it shouldn't be a "modern" Basic
         # like VB.net
-        if re.match(r'\d+', text):
+        if re.match(r'^\d+', text):
             return 0.2