Import Functionality

demo/example.py

@@ -17,7 +17,7 @@ def get_assignment_map_from_checkpoint(tvars, init_checkpoint):
         # inline test
         itest().given(name, "a:0").check_eq(m.group(1), "a")
         # a failing inline test
-        # itest().given(name, "a:0").check_eq(m.group(1), "aaa")
+       # itest().given(name, "a:0").check_eq(m.group(1), "aaa")
         if m is not None:
             name = m.group(1)
         name_to_variable[name] = var

meeting-notes/10-31 Meeting Notes.txt

@@ -0,0 +1,27 @@
+- Are tests run directly after parsing, or are they amended to a file first?
+-- Ad hoc file
+
+- Need more information on the control flow of the module?
+-- Will ask
+-- Entry point is VisitExpr
+--- Traverses ast tree, if current node is expr, it triggers the function
+--- If the expression is an inline test class, it starts with itest() and will trigger past inline test
+
+- Where does module determine that a line in the module is an inline test?
+-- Will collect all calls and will keep collect all relevant st nodes in file; at some point, will try to pass nodes into inline tests
+
+- What is the root node of the ast in collect_inline_test_calls()?
+-- Inline test expression
+-- Inline tests are generated separately with each individual itest() call
+
+# Ideas
+- Find should return list of tests AND imports
+- Add import_list to ExtractInlineTest class; would search for imports in the module at the same time as inline tests
+- Furthermore, collect import calls while collecting ineline_test_calls in "parse_inline_test"
+- From research, found that you could get list of imports loaded by module by getting intersection between system modules and module imports, though that would be redundant since we can just access the source code directly
+- Can define strings for "import", "from", and "as"; would like to know if ast's group code by line
+
+
+Import statements may be in classes or functions
+
+Add references to imported libraries to ensure they are imported properly

meeting-notes/11-14 Meeting Notes.txt

@@ -0,0 +1,16 @@
+Split progress so far, and push each as a separate request:
+- Constructor
+- Imports
+- Diff Given
+
+-Start another fork on the original pytest-inline repo
+-Put constructor changes onto new fork
+-Push request
+-Repeat for other two components
+
+Diff Given:
+- Multiple scenes to worry about
+- May need to up the best version, then develop off that
+
+Clean up as separate requests, push them, then add more concrete tests for more inline tests
+- More tests from constructor to inline

meeting-notes/11-7 Meeting Notes.txt

@@ -0,0 +1,29 @@
+Cases where libraries are not installed:
+- Throw bad error message
+- Should be the developer's duty to set up their environment properly with the libraries they want to use
+- Add to pytest configuration file
+
+Devices would eventually be moved out of constructor
+For differential tests, we would introduce diff_input function
+
+diff_given():
+- Parameters:
+- Specific
+-- Passes in differential test inputs
+-- Want to eliminate the devices constructor; devices would only support specific inputs
+-- Supports any differential input variable to be given
+--- Example: testing on devices, written as diff_given(devices, ["cuda", "cpu"])
+--- Reference how given is written right now
+-- Would be from the inline test call itself
+-- Devices is only accepted first argument in diff_given
+--- Should be equivalent to what it was
+--- Can try to extend for any first argument
+-- Can be used together with given()
+--- Some variables shouldn't be given multiple values; not changing the values of multiple inputs
+--- Use given() to specific input value for some variables
+--- Use diff_given() for inputs that need to be varied
+-- Refer to project proposal for format
+-- Order can be varied
+
+Fork original repo, then copy changes over
+

pyproject.toml

@@ -88,4 +88,4 @@ select = [
 ]
 
 [tool.ruff.lint.isort]
-known-first-party = ["inline"]
+known-first-party = ["inline"]

src/inline/inline.py

@@ -10,6 +10,7 @@ def __init__(
         tag: List = [],
         disabled: bool = False,
         timeout: float = -1.0,
+        devices: List = None,  
     ):
         """
         Initialize Inline object with test name / parametrized flag
@@ -20,6 +21,8 @@ def __init__(
         :param tag: tags to group tests
         :param disabled: whether the test is disabled
         :param timeout: seconds to timeout the test, must be a float
+        :param devices: list of devices to run differential testing on (e.g., ["cpu", "cuda", "mps"])
+                   if None, differential testing is disabled
         """
 
     def given(self, variable, value):
@@ -32,6 +35,16 @@ def given(self, variable, value):
         """
         return self
 
+    def diff_given(self, variable, value):
+        """
+        Set value to a variable for differential testing.
+
+        :param variable: a variable name
+        :param value: a value that will be assigned to the variable
+        :returns: Inline object
+        """
+        return self
+
     def check_eq(self, actual_value, expected_value):
         """
         Assert whether two values equal
@@ -42,6 +55,19 @@ def check_eq(self, actual_value, expected_value):
         :raises: AssertionError
         """
         return self
+
+    def diff_test(self, outputs):
+        """
+        Assert whether outputs are consistent across different devices.
+        This method compares the outputs from different devices specified in the constructor.
+
+        :param outputs: a dictionary mapping device names to their outputs, or a single output value
+                       if a single value is provided, the test will run the computation on all devices
+                       and compare against this reference value
+        :returns: Inline object
+        :raises: AssertionError if outputs differ across devices
+        """
+        return self
 
     def check_neq(self, actual_value, expected_value):
         """