From a6327c630c74f8437341b8e6feffacba88f8f2a7 Mon Sep 17 00:00:00 2001 From: Jay Date: Wed, 24 May 2023 21:41:16 -0400 Subject: [PATCH] Flip exec and shell chars (#7) --- Makefile | 2 +- README.md | 50 +++++++++++++++---------------- examples/demo.py | 37 ++++++++++++----------- examples/hello.py | 2 +- run.py => examples/run.py | 2 +- pybash/transformer.py | 24 ++++++++++----- pyproject.toml | 2 +- test_pybash.py | 63 +++++++++++++++++++++++---------------- 8 files changed, 101 insertions(+), 81 deletions(-) rename run.py => examples/run.py (76%) diff --git a/Makefile b/Makefile index d7806a8..0c0431d 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ check: lint test -SOURCE_FILES=pybash test_pybash.py run.py +SOURCE_FILES=pybash test_pybash.py install: pip install -e . diff --git a/README.md b/README.md index 5f8519c..75883b9 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,9 @@ ![PyPI](https://img.shields.io/pypi/v/pybash) ![GitHub](https://img.shields.io/github/license/jaykv/pybash) -Streamline bash-command execution from python with a new syntax. It combines the simplicity of writing bash scripts with the flexibility of python. Under the hood, any line or variable assignment starting with `>` or surrounded by parentheses is transformed to python `subprocess` calls and then injected into `sys.meta_path` as an import hook. All possible thanks to the wonderful [ideas](https://github.com/aroberge/ideas) project! +Streamline bash-command execution from python with a new syntax. It combines the simplicity of writing bash scripts with the flexibility of python. Under the hood, any line or variable assignment starting with `$` or surrounded by parentheses is transformed to python `subprocess` calls and then injected into `sys.meta_path` as an import hook. All possible thanks to the wonderful [ideas](https://github.com/aroberge/ideas) project! -For security and performance reasons, PyBash will NOT execute as shell, unless explicitly specified with a `$` instead of a single `>` before the command. While running commands as shell can be convenient, it can also spawn security risks if you're not too careful. If you're curious about the transformations, look at the [unit tests](test_pybash.py) for some quick examples. +For security and performance reasons, PyBash will NOT execute as shell, unless explicitly specified with a `>` instead of a single `$` before the command. While running commands as shell can be convenient, it can also spawn security risks if you're not too careful. If you're curious about the transformations, look at the [unit tests](test_pybash.py) for some quick examples. Note: this is a mainly experimental library. @@ -20,7 +20,7 @@ Note: this is a mainly experimental library. ```python from pybash.transformer import transform -transform(">echo hello world") # returns the python code for the bash command as string +transform("$echo hello world") # returns the python code for the bash command as string ``` ## As script runner @@ -31,20 +31,20 @@ transform(">echo hello world") # returns the python code for the bash command as ```py # text = "HELLO WORLD" ->echo f{text} +$echo f{text} ``` ### Run script: ```bash -$ python -m pybash hello.py +python -m pybash hello.py ``` # Supported transforms ### 1. Simple execution with output ```python ->python --version ->echo \\nthis is an echo +$python --version +$echo \\nthis is an echo ``` outputs: ``` @@ -55,7 +55,7 @@ this is an echo ### 2. Set output to variable and parse ```python -out = >cat test.txt +out = $cat test.txt test_data = out.decode('utf-8').strip() print(test_data.replace("HELLO", "HOWDY")) ``` @@ -66,7 +66,7 @@ HOWDY WORLD ### 3. Wrapped, in-line execution and parsing ```python -print((>cat test.txt).decode('utf-8').strip()) +print(($cat test.txt).decode('utf-8').strip()) ``` outputs: ``` @@ -75,12 +75,12 @@ HELLO WORLD ### 4. Redirection ```python ->echo "hello" >> test4.txt +$echo "hello" >> test4.txt ``` ### 5. Pipe chaining ```python ->cat test.txt | sed 's/HELLO/HOWDY/g' | sed 's/HOW/WHY/g' | sed 's/WHY/WHEN/g' +$cat test.txt | sed 's/HELLO/HOWDY/g' | sed 's/HOW/WHY/g' | sed 's/WHY/WHEN/g' ``` outputs: ``` @@ -89,25 +89,25 @@ WHENDY WORLD ### 6. Redirection chaining ```python ->cat test.txt | sed 's/HELLO/HOWDY\\n/g' > test1.txt >> test2.txt > test3.txt +$cat test.txt | sed 's/HELLO/HOWDY\\n/g' > test1.txt >> test2.txt > test3.txt ``` ### 7. Chaining pipes and redirection- works in tandem! ```python ->cat test.txt | sed 's/HELLO/HOWDY\\n/g' > test5.txt +$cat test.txt | sed 's/HELLO/HOWDY\\n/g' > test5.txt ``` ### 8. Input redirection ```python ->sort < test.txt >> sorted_test.txt +$sort < test.txt >> sorted_test.txt ``` ```python ->sort < test.txt | sed 's/SORT/TEST\\n/g' +$sort < test.txt | sed 's/SORT/TEST\\n/g' ``` ### 9. Glob patterns with shell ```python -$ls .github/* +>ls .github/* ``` ### 10. Direct interpolation @@ -118,21 +118,21 @@ Denoted by {{code here}}. Interpolated as direct code replace. The value/output command = "status" def get_option(command): return "-s" if command == "status" else "-v" ->git {{command}} {{get_option(command)}} +$git {{command}} {{get_option(command)}} display_type = "labels" ->kubectl get pods --show-{{display_type}}=true +$kubectl get pods --show-{{display_type}}=true ## BAD option = "-s -v" ->git status {{option}} +$git status {{option}} options = ['-s', '-v'] ->git status {{" ".join(options)}} +$git status {{" ".join(options)}} # use dynamic interpolation options = {'version': '-v'} ->git status {{options['version']}} +$git status {{options['version']}} ``` ### 11. f-string interpolation @@ -143,22 +143,22 @@ Denoted by f{ any python variable, function call, or expression here }. Interpol # git -h options = {'version': '-v', 'help': '-h'} ->git f{options['h']} +$git f{options['h']} # kubectl get pods --show-labels -n coffee namespace = "coffee" ->kubectl get pods f{"--" + "-".join(['show', 'labels'])} -n f{namespace} +$kubectl get pods f{"--" + "-".join(['show', 'labels'])} -n f{namespace} ## BAD option = "-s -v" ->git status f{option} +$git status f{option} ``` #### Also works inside methods! ```python # PYBASH DEMO # def cp_test(): - >cp test.txt test_copy.txt + $cp test.txt test_copy.txt cp_test() ``` diff --git a/examples/demo.py b/examples/demo.py index 8806203..f0be85f 100644 --- a/examples/demo.py +++ b/examples/demo.py @@ -1,59 +1,60 @@ # PYBASH DEMO # +# run with either: python run.py or python -m pybash demo.py # dynamic interpolation options = {'version': '-v', 'help': '-h'} ->git f{options['help']} +$git f{options['help']} namespace = "coffee" ->kubectl get pods f{"--" + "-".join(['show', 'labels'])} --namespace f{namespace} +$kubectl get pods f{"--" + "-".join(['show', 'labels'])} --namespace f{namespace} # static interpolation git_command = "status" option = "-v" ->git {{git_command}} {{option}} +$git {{git_command}} {{option}} #TODO: -#a = >echo "hello" >> test.txt -#a = >sort < test.txt | grep "HELLO" +#a = $echo "hello" >> test.txt +#a = $sort < test.txt | grep "HELLO" #print(f"test: {a}") # 1. use inside methods ->echo "SORT TEST\nHELLO WORLD" > test.txt +$echo "SORT TEST\nHELLO WORLD" > test.txt def cp_test(): print("1. >cp test.txt test_copy.txt") - >cp test.txt test_copy.txt + $cp test.txt test_copy.txt cp_test() # 2. simple bash command execution with output print("2. >python --version\n >echo \\nthis is an echo") ->python --version ->echo \\nthis is an echo +$python --version +$echo \\nthis is an echo # 3. set output to python variable directly -out = >cat test.txt +out = $cat test.txt test_data = out.decode('utf-8').strip() print(test_data.replace("HELLO", "HOWDY")) # 4. wrapped, in-line execution -print((>cat test.txt).decode('utf-8').strip()) +print(($cat test.txt).decode('utf-8').strip()) # 5. redirection ->echo "hello" >> test4.txt +$echo "hello" >> test4.txt # 6. pipe chaining ->cat test.txt | sed 's/HELLO/HOWDY/g' | sed 's/HOW/WHY/g' | sed 's/WHY/WHEN/g' +$cat test.txt | sed 's/HELLO/HOWDY/g' | sed 's/HOW/WHY/g' | sed 's/WHY/WHEN/g' # 7. chaining pipes and redirection- works with all! ->cat test.txt | sed 's/HELLO/HOWDY\\n/g' > test5.txt +$cat test.txt | sed 's/HELLO/HOWDY\\n/g' > test5.txt # 8. chained redirection ->cat test.txt | sed 's/HELLO/HOWDY\\n/g' > test1.txt >> test2.txt > test3.txt +$cat test.txt | sed 's/HELLO/HOWDY\\n/g' > test1.txt >> test2.txt > test3.txt # 9. input redirection ->sort < test.txt >> sorted_test.txt ->sort < test.txt | sed 's/SORT/TEST\\n/g' +$sort < test.txt >> sorted_test.txt +$sort < test.txt | sed 's/SORT/TEST\\n/g' # 10. Glob patterns -$ls .github/* +>ls ./* diff --git a/examples/hello.py b/examples/hello.py index 7e0731c..da6631a 100644 --- a/examples/hello.py +++ b/examples/hello.py @@ -1,3 +1,3 @@ text = "HELLO WORLD" ->echo f{text} \ No newline at end of file +$ echo f{text} \ No newline at end of file diff --git a/run.py b/examples/run.py similarity index 76% rename from run.py rename to examples/run.py index 71c6991..5542d87 100755 --- a/run.py +++ b/examples/run.py @@ -9,4 +9,4 @@ if __name__ == "__main__": import demo - print(f"running {demo}") + print(f"~ completed running {demo}") diff --git a/pybash/transformer.py b/pybash/transformer.py index 06c2160..3f03810 100644 --- a/pybash/transformer.py +++ b/pybash/transformer.py @@ -10,7 +10,7 @@ class InvalidInterpolation(Exception): class Processor: - command_char = ">" + command_char = "$" def __init__(self, token: token_utils.Token) -> None: self.token = token @@ -78,7 +78,7 @@ def direct_interpolate(string: str) -> str: class Shelled(Processor): # $ls .github/* - command_char = "$" + command_char = ">" def transform(self) -> token_utils.Token: command_str = " ".join(self.command) @@ -107,12 +107,13 @@ def parse(self) -> None: def transform(self) -> None: pipeline_command = Pipeline(self.command).parse_command(variablized=True) + if pipeline_command != self.command: self.token.string = pipeline_command self.token.string += ';' if pipeline_command[-1] != ';' else '' self.token.string += ' '.join(self.parsed_line[: self.start_index]) + ' cmd1\n' else: - self.token.string = ' '.join(self.parsed_line[: self.start_index]) + self.token.string = ' '.join(self.parsed_line[: self.start_index]) + ' ' self.token.string += Commander.build_subprocess_list_cmd("check_output", self.command) + '\n' @@ -128,7 +129,7 @@ def transform(self) -> token_utils.Token: # shlex strips out single quotes and double quotes-- use raw_line for the code around the wrapped command self.token.string = ( ' '.join(self.raw_line[: self.start_index]) - + self.raw_line[self.start_index][: self.raw_line[self.start_index].index('>')] + + self.raw_line[self.start_index][: self.raw_line[self.start_index].index('$')] ) self.token.string += ( Commander.build_subprocess_list_cmd("check_output", self.command) @@ -329,7 +330,7 @@ def get_start_index(parsed_line: list) -> int: int: starting index """ for i, val in enumerate(parsed_line): - if '>' in val: + if '$' in val: return i return 0 @@ -339,7 +340,7 @@ def get_bash_command( parsed_line: list, start_index: Union[int, None] = None, wrapped: Union[bool, None] = None, - command_char: str = ">", + command_char: str = "$", ) -> list: """Parses line to bash command @@ -361,6 +362,8 @@ def get_bash_command( # > may be at the beginning or somewhere in the middle of this arg # examples: >ls, print(>cat => strip up to and including > command[0] = command[0][command[0].index(command_char) + 1 :].strip() + if command[0] == '': + del command[0] # remove everything after and including first )- not part of the command if wrapped: @@ -412,8 +415,8 @@ def build_subprocess_list_cmd(method: str, args: list, **kwargs) -> str: return command -TOKENIZERS = {"$": Shelled, ">": Execed} -GREEDY_TOKENIZERS = {"= >": Variablized, "(>": Wrapped} +TOKENIZERS = {">": Shelled, "$": Execed} +GREEDY_TOKENIZERS = {"= $": Variablized, "($": Wrapped} def transform(source, **_kwargs): @@ -443,3 +446,8 @@ def transform(source, **_kwargs): new_tokens.extend(line) return token_utils.untokenize(new_tokens) + + +if __name__ == "__main__": + pyscript = transform("test = $ echo hi") + print(pyscript) diff --git a/pyproject.toml b/pyproject.toml index 404e88b..a0f13c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "PyBash" -version = "0.3.1" +version = "0.3.2" description = ">execute bash commands from python easily" authors = ["Jay "] readme = "README.md" diff --git a/test_pybash.py b/test_pybash.py index c369fef..a6b03ac 100644 --- a/test_pybash.py +++ b/test_pybash.py @@ -5,28 +5,37 @@ def test_single_exec(): - assert run_bash('>ls -la') == 'subprocess.run(["ls","-la"])\n' - assert run_bash('>python --version') == 'subprocess.run(["python","--version"])\n' + assert run_bash('$ls -la') == 'subprocess.run(["ls","-la"])\n' + assert run_bash('$python --version') == 'subprocess.run(["python","--version"])\n' + assert run_bash('$ python --version') == 'subprocess.run(["python","--version"])\n' def test_var_exec(): - assert run_bash('a = >echo "test 123"') == 'a =subprocess.check_output(["echo","test 123"])\n' + assert run_bash('a = $echo "test 123"') == 'a = subprocess.check_output(["echo","test 123"])\n' + assert run_bash('a = $ echo "test 123"') == 'a = subprocess.check_output(["echo","test 123"])\n' + + +def test_wrapped_exec(): + assert ( + run_bash('print(($ cat test.txt).decode("utf-8").strip())') + == 'print((subprocess.check_output(["cat","test.txt"])).decode("utf-8").strip())\n' + ) def test_inline_exec(): - assert run_bash('print(str(>echo test 123))') == 'print(str(subprocess.check_output(["echo","test","123"])))\n' + assert run_bash('print(str($echo test 123))') == 'print(str(subprocess.check_output(["echo","test","123"])))\n' def test_inline_dot_exec(): assert ( - run_bash("print((>cat test.txt).decode('utf-8').strip())") + run_bash("print(($cat test.txt).decode('utf-8').strip())") == 'print((subprocess.check_output(["cat","test.txt"])).decode(\'utf-8\').strip())\n' ) def test_method_exec(): src = '''def cp_test(): - >cp test.txt test_copy.txt + $cp test.txt test_copy.txt ''' assert ( run_bash(src) @@ -38,91 +47,93 @@ def test_method_exec(): def test_pipe_basic(): assert ( - run_bash(">cat test.txt | sed 's/HELLO/HOWDY/g'") + run_bash("$cat test.txt | sed 's/HELLO/HOWDY/g'") == 'cmd1 = subprocess.Popen(["cat","test.txt"], stdout=subprocess.PIPE); cmd2 = subprocess.run(["sed","s/HELLO/HOWDY/g"], stdin=cmd1.stdout)\n' ) assert ( - run_bash(">cat test.txt > test2.txt") + run_bash("$cat test.txt > test2.txt") == 'fout = open("test2.txt", "wb"); cmd1 = subprocess.run(["cat","test.txt"], stdout=fout)\n' ) def test_pipe_redirect(): assert ( - run_bash(">cat test.txt | sed 's/HELLO/HOWDY/g' > test2.txt") + run_bash("$cat test.txt | sed 's/HELLO/HOWDY/g' > test2.txt") == 'cmd1 = subprocess.Popen(["cat","test.txt"], stdout=subprocess.PIPE);fout = open("test2.txt", "wb"); cmd1 = subprocess.run(["sed","s/HELLO/HOWDY/g"], stdout=fout, stdin=cmd1.stdout)\n' ) def test_pipe_pipe_pipe(): assert ( - run_bash(">cat test.txt | sed 's/HELLO/HOWDY/g' | sed 's/HOW/WHY/g' | sed 's/WHY/WHEN/g'") + run_bash("$cat test.txt | sed 's/HELLO/HOWDY/g' | sed 's/HOW/WHY/g' | sed 's/WHY/WHEN/g'") == 'cmd1 = subprocess.Popen(["cat","test.txt"], stdout=subprocess.PIPE);cmd1 = subprocess.Popen(["sed","s/HELLO/HOWDY/g"], stdout=subprocess.PIPE, stdin=cmd1.stdout);cmd1 = subprocess.Popen(["sed","s/HOW/WHY/g"], stdout=subprocess.PIPE, stdin=cmd1.stdout); cmd2 = subprocess.run(["sed","s/WHY/WHEN/g"], stdin=cmd1.stdout)\n' ) def test_pipe_chained_redirect(): assert ( - run_bash(">cat test.txt | sed 's/HELLO/HOWDY\\n/g' > test1.txt >> test2.txt > test3.txt") + run_bash("$cat test.txt | sed 's/HELLO/HOWDY\\n/g' > test1.txt >> test2.txt > test3.txt") == 'cmd1 = subprocess.Popen(["cat","test.txt"], stdout=subprocess.PIPE);fout = open("test1.txt", "wb"); cmd1 = subprocess.run(["sed","s/HELLO/HOWDY\\n/g"], stdout=fout, stdin=cmd1.stdout);fout7 = open("test2.txt", "ab"); cmd1 = subprocess.run(["cat","test1.txt"], stdout=fout7);fout9 = open("test3.txt", "wb"); cmd1 = subprocess.run(["cat","test2.txt"], stdout=fout9)\n' ) def test_input_redirect(): - assert run_bash(">sort < test.txt") == 'fout = open("test.txt", "r"); cmd1 = subprocess.run(["sort"], stdin=fout)\n' + assert run_bash("$sort < test.txt") == 'fout = open("test.txt", "r"); cmd1 = subprocess.run(["sort"], stdin=fout)\n' assert ( - run_bash(">sort < test.txt | sed 's/SORT/WHAT/g'") + run_bash("$sort < test.txt | sed 's/SORT/WHAT/g'") == 'fout = open("test.txt", "r"); cmd1 = subprocess.Popen(["sort"], stdin=fout, stdout=subprocess.PIPE);cmd2 = subprocess.run(["sed","s/SORT/WHAT/g"], stdin=cmd1.stdout)\n' ) assert ( - run_bash(">sort < test.txt | sed 's/SORT/WHAT/g' | sed 's/WHAT/WHY/g'") + run_bash("$sort < test.txt | sed 's/SORT/WHAT/g' | sed 's/WHAT/WHY/g'") == 'fout = open("test.txt", "r"); cmd1 = subprocess.Popen(["sort"], stdin=fout, stdout=subprocess.PIPE);cmd1 = subprocess.Popen(["sed","s/SORT/WHAT/g"], stdout=subprocess.PIPE, stdin=cmd1.stdout); cmd2 = subprocess.run(["sed","s/WHAT/WHY/g"], stdin=cmd1.stdout)\n' ) assert ( - run_bash(">sort < test.txt | sed 's/SORT/WHAT/g' | sed 's/WHAT/WHY/g' > iredirect_end.txt") + run_bash("$sort < test.txt | sed 's/SORT/WHAT/g' | sed 's/WHAT/WHY/g' > iredirect_end.txt") == 'fout = open("test.txt", "r"); cmd1 = subprocess.Popen(["sort"], stdin=fout, stdout=subprocess.PIPE);cmd1 = subprocess.Popen(["sed","s/SORT/WHAT/g"], stdout=subprocess.PIPE, stdin=cmd1.stdout);fout = open("iredirect_end.txt", "wb"); cmd1 = subprocess.run(["sed","s/WHAT/WHY/g"], stdout=fout, stdin=cmd1.stdout)\n' ) assert ( - run_bash(">sort < test.txt > test_wb_redirect.txt") + run_bash("$sort < test.txt > test_wb_redirect.txt") == 'fout = open("test.txt", "r"); cmd1 = subprocess.Popen(["sort"], stdin=fout, stdout=subprocess.PIPE);fout = open("test_wb_redirect.txt", "wb"); fout.write(cmd1.stdout.read());\n' ) assert ( - run_bash(">sort < test.txt >> test_ab_redirect.txt") + run_bash("$sort < test.txt >> test_ab_redirect.txt") == 'fout = open("test.txt", "r"); cmd1 = subprocess.Popen(["sort"], stdin=fout, stdout=subprocess.PIPE);fout = open("test_ab_redirect.txt", "ab"); fout.write(cmd1.stdout.read());\n' ) def test_shell_commands(): - assert run_bash("$ls .github/*") == 'subprocess.run("ls .github/*", shell=True)\n' + assert run_bash(">ls .github/*") == 'subprocess.run("ls .github/*", shell=True)\n' def test_direct_interpolate(): - assert run_bash(">git {{command}} {{option}}") == 'subprocess.run(["git","" + command + "","" + option + ""])\n' + assert run_bash("$git {{command}} {{option}}") == 'subprocess.run(["git","" + command + "","" + option + ""])\n' assert ( - run_bash(">git {{command}} {{process(option)}}") + run_bash("$git {{command}} {{process(option)}}") == 'subprocess.run(["git","" + command + "","" + process(option) + ""])\n' ) assert ( - run_bash(">k get pods --show-{{display_type}}=true") + run_bash("$k get pods --show-{{display_type}}=true") == 'subprocess.run(["k","get","pods","--show-" + display_type + "=true"])\n' ) def test_fstring_interpolate(): assert ( - run_bash(">kubectl get pods f{\"--\" + \"-\".join(['show', 'labels'])} -n f{ namespace }") + run_bash("$kubectl get pods f{\"--\" + \"-\".join(['show', 'labels'])} -n f{ namespace }") == 'subprocess.run(["kubectl","get","pods","" + f"""{"--" + "-".join([\'show\', \'labels\'])}""" + "","-n","" + f"""{ namespace }""" + ""])\n' ) - assert run_bash(">git f{options['h']}") == 'subprocess.run(["git","" + f"""{options[\'h\']}""" + ""])\n' + assert run_bash("$git f{options['h']}") == 'subprocess.run(["git","" + f"""{options[\'h\']}""" + ""])\n' def test_invalid_interpolate(): with pytest.raises(InvalidInterpolation): - assert run_bash(">git {{command}} {{ option }}") - assert run_bash(">git {{command}} {{'\"'.join(option)}}") - assert run_bash(">git {{command}} {{a['key']}}") + assert run_bash("$git {{command}} {{ option }}") + assert run_bash("$git {{command}} {{'\"'.join(option)}}") + assert run_bash("$git {{command}} {{a['key']}}") def test_no_parse(): assert run_bash('if 5 > 4:') == 'if 5 > 4:' assert run_bash('if (pred1 and pred2) > 0:') == 'if (pred1 and pred2) > 0:' + assert run_bash('print("$(echo hi)")') == 'print("$(echo hi)")' + assert run_bash('a =$(echo "test 123")') == 'a =$(echo "test 123")'