Add module 1 tasks 1 and task 2

Author: hugo-valle

.gitignore

@@ -68,6 +68,9 @@ instance/
 
 # Virtualenv
 venv/
+venv36/
+venv37/
+venv38/
 .venv/
 
 # Datasets

conftest.py

@@ -1,133 +1,182 @@
-import re
 import ast
+import json
 
 import parso
 import pytest
 
+from collections import OrderedDict
+from types import GeneratorType as generator
+from itertools import chain
 from pathlib import Path
-from redbaron import RedBaron
-from redbaron.utils import indent
 
+from objectpath import Tree
+from mongoquery import Query
 
-class SourceCode:
-    def __init__(self, exists, code):
-        self.exists = exists
-        self.code = code
+from tests.nodes import convert_node, flatten
+from tests.template import Template
 
 
 class Parser:
-    def __init__(self, filename):
-        self.code = ""
-        self.message = ""
+    def __init__(self, file_name, nodes):
+
+        sensor = Path.cwd() / "sensor"
+        # ext = sensor / "extensions"
+
+        self.data = {
+            "success": True,
+            "full_path": "",
+            "message": "",
+            "start_pos": 0,
+            "nodes": nodes,
+        }
+
+        if file_name is not None:
+            path = lambda root, fn: root / "{}.py".format(fn)
+            # if file_name == "menu" or file_name == "stats":
+                # full_path = path(ext, file_name)
+            if file_name == "sensor":
+                full_path = path(Path.cwd(), file_name)
+            else:
+                full_path = path(sensor, file_name)
+
+            grammar = parso.load_grammar()
+            module = grammar.parse(path=full_path)
+            self.data["success"] = len(grammar.iter_errors(module)) == 0
 
-        error_message = ""
-        error_start_pos = ""
+        if self.data["success"]:
+            self.data["message"] = "Syntax: valid"
+            if file_name is not None:
+                self.data["nodes"] = convert_node(ast.parse(full_path.read_text()))
+                self.data["code"] = full_path.read_text()
 
-        if filename == "sensor":
-            file_path = Path.cwd() / "sensor.py"
         else:
-            file_path = Path.cwd() / "sensor" / "{}.py".format(filename)
+            self.data["message"] = grammar.iter_errors(module)[0].message
+            self.data["start_pos"] = grammar.iter_errors(module)[0].start_pos[0]
 
-        grammar = parso.load_grammar()
-        module = grammar.parse(path=file_path.resolve())
-        self.success = len(grammar.iter_errors(module)) == 0
+    @property
+    def nodes(self):
+        return self.data["nodes"]
 
-        if self.success:
-            with open(file_path.resolve(), "r") as source_code:
-                self.code = RedBaron(source_code.read())
-        else:
-            error_message = grammar.iter_errors(module)[0].message
-            error_start_pos = grammar.iter_errors(module)[0].start_pos[0]
-            self.message = "{} on or around line {} in `{}`.".format(
-                error_message, error_start_pos, file_path.name
-            )
+    n = nodes
+
+    @property
+    def success(self):
+        return self.data["success"]
+
+    @property
+    def code(self):
+        return self.data["code"]
+
+    def query(self, pattern):
+        nodes = Template(pattern).process(self.code)
+        if isinstance(nodes, list) and len(nodes) == 1:
+            nodes = nodes[0]
+
+        return Parser(None, nodes)
 
-    def get_by_name(self, type, name, code=None):
-        if code is None:
-            item = self.code.find_all(type, lambda node: node.name == name)
+    def query_raw(self, pattern):
+        nodes = Template(pattern).process(self.code, True)
+        if isinstance(nodes, list) and len(nodes) == 1:
+            nodes = [flatten(convert_node(node)) for node in nodes[0].body]
+        return Parser(None, nodes)
+
+    def last_line(self):
+        return flatten(self.nodes["body"][-1])
+
+    @property
+    def message(self):
+        return "{} on or around line {} in `{}`.".format(
+            self.data["message"], self.data["start_pos"], self.data["full_path"]
+        )
+
+    def match(self, template):
+        return Parser(None, list(filter(Query(template).match, self.nodes)))
+
+    def execute(self, expr):
+        result = Tree(self.nodes).execute(expr)
+        if isinstance(result, (generator, chain, map)):
+            process = list(result)
+            return (
+                Parser(None, process[0]) if len(process) == 1 else Parser(None, process)
+            )
         else:
-            item = code.find_all(type, lambda node: node.name == name)
+            return Parser(None, result)
+
+    ex = execute
+
+    def exists(self):
+        return bool(self.nodes)
 
-        return SourceCode(True, item[0]) if len(item) > 0 else SourceCode(False, [])
+    def calls(self):
+        nodes = self.execute("$.body[@.type is 'Expr' and @.value.type is 'Call']").n
+        node_list = [nodes] if isinstance(nodes, dict) else nodes
 
-    def get_call(self, value, code):
-        call = code.find("call", lambda node: node.previous.value == value)
-        return SourceCode(True, call) if call is not None and len(call) > 0 else SourceCode(False, [])
+        return Parser(None, [flatten(node) for node in node_list])
 
-    def get_args(self, code):
-        return list(
-            code.find_all("call_argument").map(
-                lambda node: str(node.target) + ":" + str(node.value).replace("'", '"')
+    def assign_(self):
+        return Parser(None, [flatten(self.execute("$.body[@.type is 'Assign']").n)])
+
+    def assigns(self):
+        return Parser(
+            None,
+            [flatten(node) for node in self.execute("$.body[@.type is 'Assign']").n],
+        )
+
+    def globals(self, name):
+        return name in self.execute("$.body[@.type is 'Global'].names").n
+
+    def defines(self, name):
+        return self.execute(
+            "$.body[@.type is 'FunctionDef' and @.name is '{}'].(name, args, body, decorator_list)".format(
+                name
             )
         )
 
-    def get_by_value(self, type, value, code=None):
-        if code is None:
-            item = self.code.find_all(type, lambda node: str(node.target) == value)
-        else:
-            item = code.find_all(type, lambda node: str(node.target) == value)
-        return SourceCode(True, item[0]) if len(item) > 0 else SourceCode(False, [])
-
-    def get_imports(self):
-        imports = []
-        self.code.find_all(
-            "import",
-            lambda node: node.find_all(
-                "dotted_as_name", lambda node: imports.append(str(node))
-            ),
+    def class_(self, name):
+        return self.execute(
+            "$.body[@.type is 'ClassDef' and @.name is '{}'].(name, args, body)".format(
+                name
+            )
         )
-        return imports
-
-    def get_from_import(self, value):
-        imports = self.code.find_all(
-            "from_import",
-            lambda node: "".join(list(node.value.node_list.map(lambda node: str(node))))
-            == value,
-        ).find_all("name_as_name")
-        return list(imports.map(lambda node: node.value))
-
-    def flatten(self, dictionary):
-        def _flatten(node):
-            trimmed = re.sub(r"\"|'", "", node.key.value)
-            flattened = []
-            if node.value.type is "list":
-                for item in node.value.node_list:
-                    if item.type is not "comma":
-                        flattened.append("{}:{}".format(trimmed, str(item)))
-            else:
-                flattened.append("{}:{}".format(trimmed, node.value.value))
-
-            return flattened
-
-        items = list(dictionary.find_all("dictitem").map(lambda node: _flatten(node)))
-        return [item for sublist in items for item in sublist]
-
-    def get_conditional(self, values, type, nested=False):
-        def flat(node):
-            if node.type == "comparison":
-                return "{}:{}:{}".format(
-                    str(node.first).replace("'", '"'),
-                    str(node.value).replace(" ", ":"),
-                    str(node.second).replace("'", '"'),
-                )
-            elif node.type == "unitary_operator":
-                return "{}:{}".format(
-                    str(node.value), str(node.target).replace("'", '"')
-                )
-
-        nodes = self.code.value if nested else self.code
-        for value in values:
-            final_node = nodes.find_all(type).find(
-                ["comparison", "unitary_operator"], lambda node: flat(node) == value
+
+    def decorators(self):
+        return Parser(None, [flatten(self.execute("$.decorator_list").n)])
+
+    def returns(self, name):
+        return name == self.execute("$.body[@.type is 'Return'].value.id").n
+
+    def returns_call(self):
+        return Parser(None, [flatten(self.execute("$.body[@.type is 'Return']").n)])
+
+    def method(self, name):
+        return self.execute(
+            "$..body[@.type is 'FunctionDef' and @.name is '{}']".format(name)
+        )
+
+    def has_arg(self, name, pos=0):
+        nodes = self.execute("$.args.args.arg").n
+        return nodes[pos] if isinstance(nodes, list) else nodes
+
+    def imports(self, name):
+        return name in self.execute("$.body[@.type is 'Import'].names..name").n
+
+    def for_(self):
+        for_body = self.execute("$.body[@.type is 'For'].body").n
+        iterators = self.execute("$.body[@.type is 'For'].(target, iter)").n
+        return Parser(None, [flatten(for_body), flatten(iterators)])
+
+    def from_imports(self, mod, alias):
+        nodes = self.execute(
+            "$.body[@.type is 'ImportFrom' and @.module is '{}'].names..name".format(
+                mod
             )
-            if final_node is not None:
-                return final_node
-        return None
+        ).n
+        return alias in (nodes if isinstance(nodes, list) else [nodes])
 
 
 @pytest.fixture
 def parse():
-    def _parse(filename):
-        return Parser(filename)
+    def _parse(file_name):
+        return Parser(file_name, {})
 
     return _parse

datasets/SENSOR_ROOM1.csv

@@ -1,4 +1,4 @@
-id,date,time,temperature,humidity,energy_usage,particulate,room
+id,date,time,temperature,humidity,energy_usage,particulate,area
 1,5/7/2020,0:01,60,0.8,0xffe,6.2E+00,1
 2,5/7/2020,0:01,60,0.8,0xff3,6.3E+00,1
 3,5/7/2020,0:04,60,0.8,0xfef,6.5E+00,1

datasets/SENSOR_ROOM2.csv

@@ -1,4 +1,4 @@
-id,date,time,temperature,humidity,energy_usage,particulate,room
+id,date,time,temperature,humidity,energy_usage,particulate,area
 1,5/7/2020,0:01,50,0.8,0xffc,8.1E+00,2
 2,5/7/2020,0:02,50,0.53,0xffb,8.1E+00,2
 3,5/7/2020,0:02,50,0.82,0xff3,8.5E+00,2

requirements.txt

@@ -1,10 +1,9 @@
 docutils==0.16
-Markdown==3.1.1
-parso==0.6.0
-pytest==5.3.5
+Markdown==3.2.1
+mongoquery==1.3.6
+objectpath==0.6.1
+parso==0.6.2
+pytest==5.4.1
 pytest-json-report==1.2.1
-pytest-metadata==1.8.0
-pytest-sugar==0.9.2
-PyYAML==5.3
-redbaron==0.9.2
-typer==0.0.8
+PyYAML==5.3.1
+typer==0.1.1

sensor.py

@@ -0,0 +1,7 @@
+# Module 1: Load data from files
+import os
+import glob
+import csv
+
+def load_sensor_data():
+    sensor_data = []