Feature/abi merge master again

.flake8

@@ -11,6 +11,7 @@ ignore =
 per-file-ignores =
     pyteal/compiler/optimizer/__init__.py:          F401
     examples/application/asset.py:                  F403, F405
+    examples/application/opup.py:                   F403, F405
     examples/application/security_token.py:         F403, F405
     examples/application/vote.py:                   F403, F405
     examples/signature/atomic_swap.py:              F403, F405
@@ -23,9 +24,7 @@ per-file-ignores =
     examples/signature/recurring_swap_deploy.py:    F403, F405
     pyteal/__init__.py:                             F401, F403
     pyteal/ir/ops.py:                               E221
-    tests/module_test.py:                           F401, F403
-    tests/*.py:                                     I252
+    tests/unit/module_test.py:                      F401, F403
 
 # from flake8-tidy-imports
 ban-relative-imports = true
-

.github/workflows/build.yml

@@ -13,7 +13,7 @@ jobs:
     container: python:${{ matrix.python }}
     strategy:
       matrix:
-        python: ['3.10']
+        python: ["3.10"]
     steps:
       - run: python3 --version
       - name: Check out code
@@ -25,19 +25,68 @@ jobs:
       - name: Build and Test
         run: make build-and-test
 
+  run-integration-tests:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python: [ "3.10" ]
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "${{ matrix.python }}"
+      - name: Test Python version
+        run: |
+          installed="$(python --version)"
+          expected="${{ matrix.python }}"
+          echo $installed
+          [[ $installed =~ "Python ${expected}" ]] && echo "Configured Python" || (echo "Failed to configure Python" && exit 1)
+      - name: Local ACT Only - Install required os level applications
+        if: ${{ env.ACT }}
+        run: |
+          sudo apt update -y
+          sudo apt install -y curl
+          sudo apt -y install ca-certificates curl gnupg lsb-release
+          sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+          sudo echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
+            $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt update
+          sudo apt -y install docker-ce docker-ce-cli containerd.io
+          sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
+          sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
+          docker-compose --version
+      - name: Create sandbox
+        run: make sandbox-dev-up
+      - name: Install python dependencies
+        run: make setup-development
+      - name: Build, Unit Tests and Integration Tests
+        run: make all-tests
+      - name: Stop running images
+        run: make sandbox-dev-stop
+
   build-docset:
     runs-on: ubuntu-20.04
-    container: python:3.10  # Needs `make`, can't be slim
+    strategy:
+      matrix:
+        python: [ "3.10" ]
     steps:
       - name: Check out code
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "${{ matrix.python }}"
       - name: Install python dependencies
         run: make setup-docs
       - name: Make docs
         run: make bundle-docs
       - name: Archive docset
+        if: ${{ !env.ACT }}
         uses: actions/upload-artifact@v2
         with:
           name: pyteal.docset

.gitignore

@@ -51,9 +51,8 @@ coverage.xml
 .hypothesis/
 .pytest_cache/
 
-# Tests generating TEAL output to compared against an expected example.
-tests/**/*.teal
-!tests/**/*_expected.teal
+# Generated by unit tests - usually for diffs against expected:
+**/generated
 
 # Translations
 *.mo
@@ -133,5 +132,9 @@ dmypy.json
 .idea
 .vscode
 
+# comma seperated vals report files
+*.csv
+!tests/**/*_example.csv
+
 # mac OS
 .DS_Store

CONTRIBUTING.md

@@ -43,7 +43,7 @@ A sibling module is defined as a different child module of the parent module. Fo
 
 #### In Runtime Code
 
-A file in this codebase that requires another module/file in this codebase, should import the absolute path of the module/file, not the relative one, using the `from X import Y` method.
+When a runtime file in this codebase needs to import another module/file in this codebase, you should import the absolute path of the module/file, not the relative one, using the `from X import Y` method.
 
 With regard to modules, there are two ways to import an object from this codebase:
 

Makefile

@@ -1,5 +1,7 @@
+# ---- Setup ---- #
+
 setup-development:
-	pip install -e.[development]
+	pip install -e .[development]
 
 setup-docs: setup-development
 	pip install -r docs/requirements.txt
@@ -8,6 +10,11 @@ setup-docs: setup-development
 setup-wheel:
 	pip install wheel
 
+generate-init:
+	python -m scripts.generate_init
+
+# ---- Docs and Distribution ---- #
+
 bdist-wheel:
 	python setup.py sdist bdist_wheel
 
@@ -20,8 +27,7 @@ bundle-docs: bundle-docs-clean
 	doc2dash --name pyteal --index-page index.html --online-redirect-url https://pyteal.readthedocs.io/en/ _build/html && \
 	tar -czvf pyteal.docset.tar.gz pyteal.docset
 
-generate-init:
-	python -m scripts.generate_init
+# ---- Code Quality ---- #
 
 check-generate-init:
 	python -m scripts.generate_init --check
@@ -33,17 +39,50 @@ black:
 flake8:
 	flake8 $(ALLPY)
 
+# TODO: add `tests` to $MYPY when graviton respects mypy (version 🐗) 
 MYPY = pyteal scripts
 mypy:
 	mypy $(MYPY)
 
 lint: black flake8 mypy
 
+# ---- Unit Tests (no algod) ---- #
+
+# TODO: add blackbox_test.py to multithreaded tests when following issue has been fixed https://github.com/algorand/pyteal/issues/199
+NUM_PROCS = auto
 test-unit:
-	pytest
+	pytest -n $(NUM_PROCS) --durations=10 -sv pyteal tests/unit --ignore tests/unit/blackbox_test.py --ignore tests/unit/user_guide_test.py
+	pytest -n 1 -sv tests/unit/blackbox_test.py tests/unit/user_guide_test.py
 
 build-and-test: check-generate-init lint test-unit
 
-# Extras:
+# ---- Integration Test (algod required) ---- #
+
+sandbox-dev-up:
+	docker-compose up -d algod
+
+sandbox-dev-stop:
+	docker-compose stop algod
+
+integration-run:
+	pytest -n $(NUM_PROCS) --durations=10 -sv tests/integration
+
+test-integration: integration-run
+
+all-tests: build-and-test test-integration
+
+# ---- Local Github Actions Simulation via `act` ---- #
+# assumes act is installed, e.g. via `brew install act`
+
+ACT_JOB = run-integration-tests
+local-gh-job:
+	act -j $(ACT_JOB)
+
+local-gh-simulate:
+	act
+
+
+# ---- Extras ---- #
+
 coverage:
 	pytest --cov-report html --cov=pyteal

README.md

@@ -60,18 +60,30 @@ Pip install PyTeal in editable state with dependencies:
   * `pip install -e.[development]`
   * Note, that if you're using `zsh` you'll need to escape the brackets: `pip install -e.\[development\]`
 
+Format code:
+
+* `black .`
+
+Lint using flake8:
+
+* `flake8 docs examples pyteal scripts tests *.py`
+
 Type checking using mypy:
 
-* `mypy pyteal`
+* `mypy pyteal scripts`
 
-Run tests:
+Run unit tests:
 
-* `pytest`
+* `pytest pyteal tests/unit`
 
-Format code:
+Run integration tests (assumes a developer-mode `algod` is available on port 4001):
 
-* `black .`
+* `pytest tests/integration`
 
-Lint using flake8:
+Stand up developer-mode algod on ports 4001, 4002 and `tealdbg` on port 9392 (assumes [Docker](https://www.docker.com/) is available on your system):
 
-* `flake8 docs examples pyteal scripts tests *.py`
+* `docker-compose up -d`
+
+Tear down and clean up resources for the developer-mode algod stood up above:
+
+* `docker-compose down`

docker-compose.yml

@@ -0,0 +1,9 @@
+version: '3'
+
+services:
+  algod:
+    image: makerxau/algorand-sandbox-dev:latest
+    ports:
+      - "4001:4001"
+      - "4002:4002"
+      - "9392:9392"

docs/accessing_transaction_field.rst

@@ -47,8 +47,8 @@ Operator
 :any:`Txn.global_num_byte_slices() <TxnObject.global_num_byte_slices>`           :code:`TealType.uint64`   3                Maximum global byte strings in app schema
 :any:`Txn.local_num_uints() <TxnObject.local_num_uints>`                         :code:`TealType.uint64`   3                Maximum local integers in app schema
 :any:`Txn.local_num_byte_slices() <TxnObject.local_num_byte_slices>`             :code:`TealType.uint64`   3                Maximum local byte strings in app schema
-:any:`Txn.accounts <TxnObject.accounts>`                                         :code:`TealType.bytes[]`  2                Array of application accounts
-:any:`Txn.assets <TxnObject.assets>`                                             :code:`TealType.uint64[]` 3                Array of application assets
+:any:`Txn.accounts <TxnObject.accounts>`                                         :code:`TealType.bytes[]`  2                Array of accounts available to the application
+:any:`Txn.assets <TxnObject.assets>`                                             :code:`TealType.uint64[]` 3                Array of assets available to the application
 :any:`Txn.applications <TxnObject.applications>`                                 :code:`TealType.uint64[]` 3                Array of applications
 :any:`Txn.clear_state_program() <TxnObject.clear_state_program>`                 :code:`TealType.bytes`    2
 :any:`Txn.extra_program_pages() <TxnObject.extra_program_pages>`                 :code:`TealType.uint64`   4                Number of extra program pages for app
@@ -223,7 +223,7 @@ Information about the current state of the blockchain can be obtained using the
 Operator                                    Type                    Min TEAL Version Notes
 =========================================== ======================= ================ =============================================================
 :any:`Global.min_txn_fee()`                 :code:`TealType.uint64` 2                in microAlgos
-:any:`Global.min_balance()`                 :code:`TealType.uint64` 2                in mircoAlgos
+:any:`Global.min_balance()`                 :code:`TealType.uint64` 2                in microAlgos
 :any:`Global.max_txn_life()`                :code:`TealType.uint64` 2                number of rounds
 :any:`Global.zero_address()`                :code:`TealType.bytes`  2                32 byte address of all zero bytes
 :any:`Global.group_size()`                  :code:`TealType.uint64` 2                number of txns in this atomic transaction group, at least 1

docs/assets.rst

@@ -126,6 +126,7 @@ Expression                        Type                    Description
 :any:`AssetParam.name()`          :code:`TealType.bytes`  The name of the asset.
 :any:`AssetParam.url()`           :code:`TealType.bytes`  A URL associated with the asset.
 :any:`AssetParam.metadataHash()`  :code:`TealType.bytes`  A 32-byte hash associated with the asset.
+:any:`AssetParam.creator()`       :code:`TealType.bytes`  The address of the asset's creator account.
 :any:`AssetParam.manager()`       :code:`TealType.bytes`  The address of the asset's manager account.
 :any:`AssetParam.reserve()`       :code:`TealType.bytes`  The address of the asset's reserve account.
 :any:`AssetParam.freeze()`        :code:`TealType.bytes`  The address of the asset's freeze account.

docs/index.rst

@@ -44,6 +44,7 @@ PyTeal **hasn't been security audited**. Use it at your own risk.
    assets
    versions
    compiler_optimization
+   opup
 
 .. toctree::
    :maxdepth: 3

docs/opup.rst

@@ -0,0 +1,95 @@
+.. _opup:
+
+OpUp:  Budget Increase Utility
+=================================
+
+Some opcode budget is consumed during execution of every Algorand Smart Contract because every TEAL
+instruction has a corresponding cost. In order for the evaluation to succeed, the budget consumed must not
+exceed the budget provided. This constraint may introduce a problem for expensive contracts that quickly
+consume the initial budget of 700. The OpUp budget increase utility provides a workaround using NoOp inner
+transactions that increase the transaction group's pooled compute budget. The funding for issuing the inner
+transactions is provided by the contract doing the issuing, not the user calling the contract, so the
+contract must have enough funds for this purpose. Note that there is a context specific limit to the number
+of inner transactions issued in a transaction group so budget cannot be increased arbitrarily. The available
+budget when using the OpUp utility will need to be high enough to execute the TEAL code that issues the inner
+transactions. A budget of ~20 is enough for most use cases. 
+
+Usage
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The :any:`pyteal.OpUp` utility is available in two modes: :any:`Explicit` and :any:`OnCall`:
+
+================= ===================================================================================
+OpUp Mode         Description
+================= ===================================================================================
+:code:`Explicit`  Calls a user provided external app when more budget is requested.
+:code:`OnCall`    Creates and immediately deletes the app to be called when more budget is requested.
+================= ===================================================================================
+
+
+:any:`Explicit` has the benefit of constructing more lightweight inner transactions, but requires the
+target app ID to be provided in the foreign apps array field of the transaction and the :any:`pyteal.OpUp`
+constructor in order for it to be accessible. :any:`OnCall` is easier to use, but has slightly more overhead
+because the target app must be created and deleted during the evaluation of an app call.
+
+Ensure Budget
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:any:`pyteal.OpUp.ensure_budget` attempts to ensure that the available budget is at least the budget requested by
+the caller. If there aren't enough funds for issuing the inner transactions or the inner transaction limit
+is exceeded, the evaluation will fail. The typical usage pattern is to insert the :any:`pyteal.OpUp.ensure_budget`
+call just before a particularly heavyweight subroutine or expression. Keep in mind that the required budget
+expression will be evaluated before the inner transactions are issued so it may be prudent to avoid expensive
+expressions, which may exhaust the budget before it can be increased.
+
+In the example below, the :py:meth:`pyteal.Ed25519Verify` expression is used, which costs 1,900.
+
+.. code-block:: python
+
+    # The application id to be called when more budget is requested. This should be
+    # replaced with an id provided by the developer.
+    target_app_id = Int(1)
+
+    # OnCall mode works the exact same way, just omit the target_app_id
+    opup = OpUp(OpUpMode.Explicit, target_app_id)
+    program = Seq(
+        If(Txn.application_id() != Int(0)).Then(
+            Seq(
+                opup.ensure_budget(Int(2000)),
+                Assert(Ed25519Verify(args[0], args[1], args[2])),
+            )
+        ),
+        Approve(),
+    )
+
+Maximize Budget
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:any:`pyteal.OpUp.maximize_budget` attempts to issue as many inner transactions as possible with the given fee.
+This essentially maximizes the available budget while putting a ceiling on the amount of fee spent. Just
+as with :any:`pyteal.OpUp.ensure_budget`, the evaluation will fail if there aren't enough funds for issuing the
+inner transactions or the inner transaction limit is exceeded. This method may be preferred to
+:any:`pyteal.OpUp.ensure_budget` when the fee spent on increasing budget needs to be capped or if the developer
+would rather just maximize the available budget instead of doing in depth cost analysis on the program.
+
+In the example below, the fee is capped at 3,000 microAlgos for increasing the budget. This works out to 3 inner
+transactions being issued, each increasing the available budget by ~700.
+
+.. code-block:: python
+
+    target_app_id = Int(1) # the application id to be called when more budget is requested
+
+    # OnCall mode works the exact same way, just omit the target_app_id
+    opup = OpUp(OpUpMode.Explicit, target_app_id)
+    program = Seq(
+        If(Txn.application_id() != Int(0)).Then(
+            Seq(
+                opup.maximize_budget(Int(3000)),
+                Assert(Ed25519Verify(args[0], args[1], args[2])),
+            )
+        ),
+        Approve(),
+    )
+
+If budget increase requests appear multiple times in the program, it may be a good idea to wrap the
+invocation in a PyTeal Subroutine to improve code reuse and reduce the size of the compiled program.