Print debugging information on assertion error

.gitattributes

@@ -0,0 +1,3 @@
+# test p8 cart files should not have their line endings changed on checkout
+# use 'binary' settings for p8 cart files to force this
+*.p8 binary

.github/workflows/python-app.yml

@@ -0,0 +1,64 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Tests
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+permissions:
+  contents: read
+
+jobs:
+  test-ubuntu:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        pip install --upgrade pip
+        pip install -r pydevtools.txt
+        pip install flake8 pytest
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with pytest
+      run: |
+        pytest
+
+  test-windows:
+
+    runs-on: windows-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        pip install --upgrade pip
+        pip install -r pydevtools.txt
+        pip install flake8 pytest
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with pytest
+      run: |
+        pytest

.gitignore

@@ -13,5 +13,6 @@ htmlcov/
 docs/_build/
 .idea/
 .pytest_cache/
+build/
 
 .DS_Store

README.md

@@ -1,5 +1,7 @@
 # picotool: Tools and Python libraries for manipulating PICO-8 game files
 
+[![tests](https://github.com/scottnm/picotool/actions/workflows/python-app.yml/badge.svg)](https://github.com/scottnm/picotool/actions/workflows/python-app.yml)
+
 [PICO-8](http://www.lexaloffle.com/pico-8.php) is a _fantasy game console_ by [Lexaloffle Games](http://www.lexaloffle.com/). The PICO-8 runtime environment runs _cartridges_ (or _carts_): game files containing code, graphics, sound, and music data. The console includes a built-in editor for writing games. Game cartridge files can be played in a browser, and can be posted to the Lexaloffle bulletin board or exported to any website.
 
 `picotool` is a suite of tools and libraries for building and manipulating PICO-8 game cartridge files. The suite is implemented in, and requires, [Python 3](https://www.python.org/). The tools can examine and transform cartridges in various ways, and you can implement your own tools to access and modify cartridge data with the Python libraries.

pico8/build/build.py

@@ -89,7 +89,7 @@ def _walk_FunctionCall(self, node):
             if (not isinstance(arg_exps[0], parser.ExpValue) or
                     not isinstance(arg_exps[0].value, lexer.TokString)):
                 self._error_at_node('require() first argument must be a '
-                                    'string literal', node)
+                                    f'string literal, but first argument is {arg_exps[0]}', node)
             require_path = arg_exps[0].value.value
 
             use_game_loop = False
@@ -266,9 +266,19 @@ def do_build(args):
                     result.lua = lua.Lua.from_lines(
                         infh, version=game.DEFAULT_VERSION)
                     package_lua = {}
-                    _evaluate_require(
-                        result.lua, file_path=fn, package_lua=package_lua,
-                        lua_path=getattr(args, 'lua_path', None))
+
+                    # ADDED: try-except
+                    try:
+                        _evaluate_require(
+                            result.lua, file_path=fn, package_lua=package_lua,
+                            lua_path=getattr(args, 'lua_path', None))
+                    except LuaBuildError as e:
+                        print(f"LuaBuildError caught on _evaluate_require with top file: {fn}")
+                        print("The error may be located in any file recursively required, comment out until you find the culprit.")
+                        print(f"msg: {e.msg}")
+                        print(f"token: {e.token}")
+                        print("Passing error upward")
+                        raise
 
                     if getattr(args, 'optimize_tokens', False):
                         # TODO: perform const subst, dead code elim, taking

pico8/game/formatter/p8.py

@@ -3,6 +3,7 @@
 __all__ = [
     'P8Formatter',
     'InvalidP8HeaderError',
+    'InvalidP8VersionError',
     'InvalidP8SectionError',
     'P8IncludeNotFound',
     'P8IncludeOutsideOfAllowedDirectory',
@@ -24,8 +25,8 @@
 from ...music.music import Music
 
 HEADER_TITLE_STR = b'pico-8 cartridge // http://www.pico-8.com\n'
-HEADER_VERSION_RE = re.compile(br'version (\d+)\n')
-SECTION_DELIM_RE = re.compile(br'__(\w+)__\n')
+HEADER_VERSION_RE = re.compile(br'version (\d+)\r?\n')
+SECTION_DELIM_RE = re.compile(br'__(\w+)__\r?\n')
 INCLUDE_LINE_RE = re.compile(
     br'\s*#include\s+(\S+)(\.p8\.png|\.p8|\.lua)(\:\d+)?')
 PICO8_CART_PATHS = [
@@ -39,8 +40,22 @@
 class InvalidP8HeaderError(util.InvalidP8DataError):
     """Exception for invalid .p8 file header."""
 
+    def __init__(self, bad_header, expected_header):
+        self.bad_header = bad_header
+        self.expected_header = expected_header
+
     def __str__(self):
-        return 'Invalid .p8: missing or corrupt header'
+        return 'Invalid .p8: missing or corrupt header. Found {self.bad_header!r}, expected {self.expected_header!r}'
+
+
+class InvalidP8VersionError(util.InvalidP8DataError):
+    """Exception for invalid .p8 version header."""
+
+    def __init__(self, bad_version_line):
+        self.bad_version_line = bad_version_line 
+
+    def __str__(self):
+        return ('Invalid .p8: invalid version header. found "%s"' % self.bad_version_line)
 
 
 class InvalidP8SectionError(util.InvalidP8DataError):
@@ -68,12 +83,13 @@ class InvalidP8Include(util.InvalidP8DataError):
 
 def _get_raw_data_from_p8_file(instr, filename=None):
     header_title_str = instr.readline()
-    if header_title_str != HEADER_TITLE_STR:
-        raise InvalidP8HeaderError()
+    # use rstrip to normalize line endings
+    if header_title_str.rstrip() != HEADER_TITLE_STR.rstrip():
+        raise InvalidP8HeaderError(header_title_str, HEADER_TITLE_STR)
     header_version_str = instr.readline()
     version_m = HEADER_VERSION_RE.match(header_version_str)
     if version_m is None:
-        raise InvalidP8HeaderError()
+        raise InvalidP8VersionError(header_version_str)
     version = int(version_m.group(1))
 
     # (section is a text str.)
@@ -308,7 +324,7 @@ def to_file(
         for line in game.lua.to_lines(
                 writer_cls=lua_writer_cls,
                 writer_args=lua_writer_args):
-            outstr.write(bytes(lua.p8scii_to_unicode(line), 'utf-8'))
+            outstr.write(line)
             ended_in_newline = line.endswith(b'\n')
         if not ended_in_newline:
             outstr.write(b'\n')

pico8/gfx/gfx.py

@@ -84,7 +84,8 @@ def from_lines(cls, lines, version):
         """
         datastrs = []
         for line in lines:
-            if len(line) != 129:
+            # Each line of the GFX section is 128 characters followed by either an LF or CRLF line ending
+            if len(line.rstrip()) != 128:
                 continue
 
             larray = list(line.rstrip())

pico8/lua/lexer.py

@@ -161,9 +161,7 @@ def code(self):
             escaped_chrs = []
             for c in self._data:
                 c = bytes([c])
-                if c in _STRING_REVERSE_ESCAPES:
-                    escaped_chrs.append(b'\\' + _STRING_REVERSE_ESCAPES[c])
-                elif c == self._quote:
+                if c == self._quote:
                     escaped_chrs.append(b'\\' + c)
                 else:
                     escaped_chrs.append(c)
@@ -376,18 +374,6 @@ def _process_token(self, s):
                     i += 1
                     break
 
-                if c == b'\\':
-                    # Escape character.
-                    num_m = re.match(br'\d{1,3}', s[i+1:])
-                    if num_m:
-                        c = bytes([int(num_m.group(0))])
-                        i += len(num_m.group(0))
-                    else:
-                        next_c = s[i+1:i+2]
-                        if next_c in _STRING_ESCAPES:
-                            c = _STRING_ESCAPES[next_c]
-                            i += 1
-
                 self._in_string.append(c)
                 i += 1
 

pico8/lua/lua.py

@@ -344,24 +344,49 @@ def get_char_count(self):
 
     def get_token_count(self):
         c = 0
-        for t in self._lexer._tokens:
+        prev_op=False #whether previous non whitespace token was an operator (for unary -)
+        unary_minus_ops=parser.BINOP_PATS + parser.UNOP_PATS+tuple(lexer.TokSymbol(x) for x in
+          (b'&=', b'|=', b'^^=', b'<<=', b'>>=', b'>>>=', b'<<>=', b'>><=', b'\\=',
+          b'..=', b'+=', b'-=', b'*=', b'/=', b'%=', b'^=',
+          b'(', b',', b'{', b'[' ,b'=') ) # these ops are not 100% accurate to how pico8 does it (pico8's list is slightly different)
+          #but all the edge cases left are pretty niche
+        for i,t in enumerate(self._lexer._tokens):
             # TODO: As of 0.1.8, "1 .. 5" is three tokens, "1..5" is one token
+            if (isinstance(t, lexer.TokSpace) or
+                  isinstance(t, lexer.TokNewline) or
+                  isinstance(t, lexer.TokComment)):
+              continue
+
             if (t.matches(lexer.TokSymbol(b':')) or
                     t.matches(lexer.TokSymbol(b'.')) or
                     t.matches(lexer.TokSymbol(b')')) or
                     t.matches(lexer.TokSymbol(b']')) or
                     t.matches(lexer.TokSymbol(b'}')) or
+                    t.matches(lexer.TokSymbol(b',')) or
+                    t.matches(lexer.TokSymbol(b';')) or
                     t.matches(lexer.TokKeyword(b'local')) or
                     t.matches(lexer.TokKeyword(b'end'))):
                 # PICO-8 generously does not count these as tokens.
                 pass
+            elif ((t.matches(lexer.TokSymbol(b'-')) or t.matches(lexer.TokSymbol(b'~'))) and
+                i+1<len(self._lexer._tokens) and
+                self._lexer._tokens[i+1].matches(lexer.TokNumber) and
+                prev_op):
+                #unary - and ~
+                pass
+
+
+
             elif t.matches(lexer.TokNumber) and t._data.find(b'e') != -1:
                 # PICO-8 counts 'e' part of number as a separate token.
                 c += 2
-            elif (not isinstance(t, lexer.TokSpace) and
-                  not isinstance(t, lexer.TokNewline) and
-                  not isinstance(t, lexer.TokComment)):
+            else:
                 c += 1
+            prev_op=False
+            for op in unary_minus_ops:
+                if t.matches(op):
+                    prev_op=True
+
         return c
 
     def get_line_count(self):

pico8/tool.py

@@ -10,6 +10,8 @@
 import sys
 import traceback
 
+from pico8.game.formatter.p8 import _get_raw_data_from_p8_file
+
 from . import util
 from .build import build
 from .game import file
@@ -185,7 +187,7 @@ def listrawlua(args):
                 raw_lua = data.code.split(b'\n')
         elif fname.endswith('.p8'):
             with open(fname, 'rb') as fh:
-                data = game.Game.get_raw_data_from_p8_file(fh, fname)
+                data = _get_raw_data_from_p8_file(fh, fname)
                 raw_lua = data.section_lines['lua']
         else:
             util.error('{}: must be .p8 or .p8.png\n'.format(fname))

pydevtools.txt

@@ -6,6 +6,8 @@ flake8
 mypy
 sphinx
 pytest
+pytest-xdist
 coverage
 ipython
 rstcheck
+pypng

setup.py

@@ -22,6 +22,7 @@
         'mypy',
         'pypng',
         'pytest',
+        'pytest-xdist',
     ],
     entry_points={
         'console_scripts': [

tests/pico8/build/build_test.py

@@ -5,8 +5,6 @@
 import shutil
 import tempfile
 import unittest
-from unittest.mock import Mock
-from unittest.mock import patch
 
 from pico8.build import build
 from pico8.game import game

tests/pico8/game/formatter/formatter_test.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+import unittest
+
+from pico8.game.formatter.p8 import P8Formatter
+from pico8.game import game
+from pico8.lua import lua
+from io import BytesIO
+
+class TestP8Formatter(unittest.TestCase):
+	def testEmojisArePreservedThroughFormatting(self):
+		lua_code = 'print(\"Hello Emoji 🅾️\")'
+		lua_code_utf8_bytes = lua_code.encode('utf-8')
+		emoji_bytes = '🅾️'.encode('utf-8')
+
+		output_cart = game.Game.make_empty_game('dummy.p8')
+		output_cart.lua = lua.Lua.from_lines([lua_code_utf8_bytes], version=8)
+
+		cart_stream = BytesIO()
+
+		formatter = P8Formatter()
+		formatter.to_file(
+			output_cart,
+			cart_stream,
+			lua_writer_cls = lua.LuaEchoWriter)
+
+		cart_bytes = cart_stream.getvalue()
+
+		assert(emoji_bytes in cart_bytes)