New translation modules architecture (#678)

* feat: initial commit

* fix: validity tests

* fix: some of the stdlib errors

* fix: some tests

* fix: stdlib tests

* fix: additional error

* fix: cargo clippy

* fix: wrong quoting

* fix: wrong init value

* fix: block rendering

* feat: improve block formatting

* fix: shellcheck issues

* feat: add more tests

* fix: shellcheck issues

* fix: clippy error

* feat: add ternary tests

* fix: escaping of slices

* impl From<String> for CommentFragment

* simplify path for use

* refactor match for readability

Co-authored-by: Huw Walters <huw.walters@gmail.com>

* refactor &*self.index -> self.index.as_ref()

* add trailing comma to match block

* rigidly match all requirements to be 1 1 0

* remove a redundant blank line

* simplify echoing the array

* reword unreachable code

* refactor constructor to give every function its own line

* refactor: render -> to_string

* fix: trailing commas

* refactor: rename fragment structures

* refactor: symbols and block fn name

* refactor: add context to text escaped regular character test

* refactor: remove padding in doc comments

* fix: comment and compound

* refactor: remove unnecessary files

* refactor: compount -> list

* refactor: set_quoted -> with_quotes

* refactor: iter loop

* refactor: fragments(raw:) -> raw_fragment()

* refactor: formatting

* refactor: improve readability

* refactor: variable

* refactor: finalize addressing concerns

* fix: clippy errors

---------

Co-authored-by: b1ek <me@blek.codes>
Co-authored-by: Huw Walters <huw.walters@gmail.com>

Author: 3 people

src/compiler.rs

@@ -1,6 +1,7 @@
 extern crate chrono;
 use crate::docs::module::DocumentationModule;
 use crate::modules::block::Block;
+use crate::modules::prelude::{BlockFragment, FragmentRenderable};
 use crate::translate::check_all_blocks;
 use crate::translate::module::TranslateModule;
 use crate::utils::{ParserMetadata, TranslateMetadata};
@@ -165,9 +166,9 @@ impl AmberCompiler {
         let ast_forest = self.get_sorted_ast_forest(block, &meta);
         let mut meta_translate = TranslateMetadata::new(meta, &self.options);
         let time = Instant::now();
-        let mut result = vec![];
+        let mut result = BlockFragment::new(Vec::new(), false);
         for (_path, block) in ast_forest {
-            result.push(block.translate(&mut meta_translate));
+            result.append(block.translate(&mut meta_translate));
         }
         if Self::env_flag_set(AMBER_DEBUG_TIME) {
             let pathname = self.path.clone().unwrap_or(String::from("unknown"));
@@ -178,7 +179,7 @@ impl AmberCompiler {
             );
         }
 
-        let mut result = result.join("\n") + "\n";
+        let mut result = result.to_string(&mut meta_translate);
 
         let filters = self.options.no_proc.iter()
             .map(|x| WildMatchPattern::new(x))

src/modules/block.rs

@@ -1,16 +1,16 @@
 use std::collections::VecDeque;
 use std::ops::Index;
 
+use crate::modules::prelude::*;
 use heraclitus_compiler::prelude::*;
 use itertools::Itertools;
-use crate::docs::module::DocumentationModule;
-use crate::utils::{metadata::ParserMetadata, TranslateMetadata};
-use crate::translate::module::TranslateModule;
 use super::statement::stmt::Statement;
 
 #[derive(Debug, Clone)]
 pub struct Block {
-    pub statements: Vec<Statement>
+    pub statements: Vec<Statement>,
+    pub should_indent: bool,
+    pub needs_noop: bool,
 }
 
 impl Block {
@@ -23,14 +23,26 @@ impl Block {
     pub fn push_statement(&mut self, statement: Statement) {
         self.statements.push(statement);
     }
+
+    pub fn with_needs_noop(mut self) -> Self {
+        self.needs_noop = true;
+        self
+    }
+
+    pub fn with_no_indent(mut self) -> Self {
+        self.should_indent = false;
+        self
+    }
 }
 
 impl SyntaxModule<ParserMetadata> for Block {
     syntax_name!("Block");
 
     fn new() -> Self {
         Block {
-            statements: vec![]
+            statements: vec![],
+            should_indent: true,
+            needs_noop: false,
         }
     }
 
@@ -61,20 +73,21 @@ impl SyntaxModule<ParserMetadata> for Block {
 }
 
 impl TranslateModule for Block {
-    fn translate(&self, meta: &mut TranslateMetadata) -> String {
+    fn translate(&self, meta: &mut TranslateMetadata) -> FragmentKind {
         // Save the current statement queue and create a new one
         let mut new_queue = VecDeque::new();
         std::mem::swap(&mut meta.stmt_queue, &mut new_queue);
-        meta.increase_indent();
-        let result = if self.is_empty() {
-            ":".to_string()
-        } else {
-            self.statements.iter()
-                .map(|statement| statement.translate(meta))
-                .filter(|translation| !translation.trim().is_empty())
-                .collect::<Vec<_>>().join("\n")
+        let result = {
+            let mut statements = vec![];
+            for statement in &self.statements {
+                let statement = statement.translate(meta);
+                statements.extend(meta.stmt_queue.drain(..));
+                statements.push(statement);
+            }
+            BlockFragment::new(statements, self.should_indent)
+                .with_needs_noop(self.needs_noop)
+                .to_frag()
         };
-        meta.decrease_indent();
         // Restore the old statement queue
         std::mem::swap(&mut meta.stmt_queue, &mut new_queue);
         result

src/modules/builtin/cd.rs

@@ -1,22 +1,19 @@
-use heraclitus_compiler::prelude::*;
+use crate::fragments;
 use crate::modules::expression::expr::Expr;
-use crate::docs::module::DocumentationModule;
+use heraclitus_compiler::prelude::*;
+use crate::modules::prelude::*;
 use crate::modules::types::{Type, Typed};
-use crate::translate::module::TranslateModule;
-use crate::utils::{ParserMetadata, TranslateMetadata};
 
 #[derive(Debug, Clone)]
 pub struct Cd {
-    value: Expr
+    value: Expr,
 }
 
 impl SyntaxModule<ParserMetadata> for Cd {
     syntax_name!("ChangeDirectory");
 
     fn new() -> Self {
-        Cd {
-            value: Expr::new()
-        }
+        Cd { value: Expr::new() }
     }
 
     fn parse(&mut self, meta: &mut ParserMetadata) -> SyntaxResult {
@@ -35,9 +32,8 @@ impl SyntaxModule<ParserMetadata> for Cd {
 }
 
 impl TranslateModule for Cd {
-    fn translate(&self, meta: &mut TranslateMetadata) -> String {
-        let value = self.value.translate(meta);
-        format!("cd {} || exit", value)
+    fn translate(&self, meta: &mut TranslateMetadata) -> FragmentKind {
+        fragments!("cd ", self.value.translate(meta), " || exit")
     }
 }
 

src/modules/builtin/echo.rs

@@ -1,20 +1,19 @@
-use heraclitus_compiler::prelude::*;
-use crate::docs::module::DocumentationModule;
+use crate::fragments;
 use crate::modules::expression::expr::Expr;
-use crate::translate::module::TranslateModule;
-use crate::utils::{ParserMetadata, TranslateMetadata};
+use crate::modules::prelude::*;
+use heraclitus_compiler::prelude::*;
 
 #[derive(Debug, Clone)]
 pub struct Echo {
-    value: Box<Expr>
+    value: Box<Expr>,
 }
 
 impl SyntaxModule<ParserMetadata> for Echo {
     syntax_name!("Log");
 
     fn new() -> Self {
         Echo {
-            value: Box::new(Expr::new())
+            value: Box::new(Expr::new()),
         }
     }
 
@@ -26,9 +25,8 @@ impl SyntaxModule<ParserMetadata> for Echo {
 }
 
 impl TranslateModule for Echo {
-    fn translate(&self, meta: &mut TranslateMetadata) -> String {
-        let value = self.value.translate(meta);
-        format!("echo {}", value)
+    fn translate(&self, meta: &mut TranslateMetadata) -> FragmentKind {
+        fragments!("echo ", self.value.translate(meta))
     }
 }
 

src/modules/builtin/exit.rs

@@ -1,22 +1,19 @@
-use heraclitus_compiler::prelude::*;
+use crate::fragments;
 use crate::modules::expression::expr::Expr;
-use crate::docs::module::DocumentationModule;
+use crate::modules::prelude::*;
 use crate::modules::types::{Type, Typed};
-use crate::translate::module::TranslateModule;
-use crate::utils::{ParserMetadata, TranslateMetadata};
+use heraclitus_compiler::prelude::*;
 
 #[derive(Debug, Clone)]
 pub struct Exit {
-    code: Option<Expr>
+    code: Option<Expr>,
 }
 
 impl SyntaxModule<ParserMetadata> for Exit {
     syntax_name!("Exit");
 
     fn new() -> Self {
-        Exit {
-            code: None
-        }
+        Exit { code: None }
     }
 
     fn parse(&mut self, meta: &mut ParserMetadata) -> SyntaxResult {
@@ -43,11 +40,11 @@ impl SyntaxModule<ParserMetadata> for Exit {
 }
 
 impl TranslateModule for Exit {
-    fn translate(&self, meta: &mut TranslateMetadata) -> String {
-        let code = self.code.as_ref()
+    fn translate(&self, meta: &mut TranslateMetadata) -> FragmentKind {
+        let exit_code = self.code.as_ref()
             .map(|expr| expr.translate(meta))
-            .unwrap_or_else(|| "0".to_string());
-        format!("exit {}", code)
+            .unwrap_or(fragments!("0"));
+        fragments!("exit ", exit_code)
     }
 }
 

src/modules/builtin/len.rs

@@ -1,6 +1,6 @@
-use crate::docs::module::DocumentationModule;
 use crate::modules::expression::expr::Expr;
 use crate::modules::expression::unop::UnOp;
+use crate::modules::prelude::*;
 use crate::modules::types::{Type, Typed};
 use crate::translate::module::TranslateModule;
 use crate::utils::{ParserMetadata, TranslateMetadata};
@@ -50,21 +50,12 @@ impl SyntaxModule<ParserMetadata> for Len {
 }
 
 impl TranslateModule for Len {
-    fn translate(&self, meta: &mut TranslateMetadata) -> String {
+    fn translate(&self, meta: &mut TranslateMetadata) -> FragmentKind {
         let value = self.value.translate(meta);
-        if self.value.get_type() == Type::Text {
-            meta.stmt_queue.push_back(format!("__AMBER_LEN={value}"));
-            return String::from("\"${#__AMBER_LEN}\"")
-        }
-        // Case for Array passed as a reference
-        if value.starts_with("\"${!") {
-                meta.stmt_queue.push_back(format!("__AMBER_LEN=({value})"));
-                String::from("\"${#__AMBER_LEN[@]}\"")
-        } else {
-            format!("\"${{#{}", value.trim_start_matches("\"${"))
-                .trim_end()
-                .to_string()
-        }
+        let id = meta.gen_value_id();
+        meta.push_intermediate_variable_lazy("__length", Some(id), Type::Num, value)
+            .with_length_getter(true)
+            .to_frag()
     }
 }
 

src/modules/builtin/lines.rs

@@ -1,9 +1,11 @@
-use heraclitus_compiler::prelude::*;
-use crate::docs::module::DocumentationModule;
+use crate::fragments;
+use crate::raw_fragment;
 use crate::modules::expression::expr::Expr;
 use crate::modules::types::{Type, Typed};
 use crate::translate::module::TranslateModule;
 use crate::utils::metadata::{ParserMetadata, TranslateMetadata};
+use crate::modules::prelude::*;
+use heraclitus_compiler::prelude::*;
 
 #[derive(Debug, Clone)]
 pub struct LinesInvocation {
@@ -12,7 +14,7 @@ pub struct LinesInvocation {
 
 impl Typed for LinesInvocation {
     fn get_type(&self) -> Type {
-        Type::Array(Box::new(Type::Text))
+        Type::array_of(Type::Text)
     }
 }
 
@@ -21,7 +23,7 @@ impl SyntaxModule<ParserMetadata> for LinesInvocation {
 
     fn new() -> Self {
         LinesInvocation {
-            path: Box::new(None)
+            path: Box::new(None),
         }
     }
 
@@ -33,7 +35,10 @@ impl SyntaxModule<ParserMetadata> for LinesInvocation {
         syntax(meta, &mut path)?;
         token(meta, ")")?;
         if path.get_type() != Type::Text {
-            let msg = format!("Expected value of type 'Text' but got '{}'", path.get_type());
+            let msg = format!(
+                "Expected value of type 'Text' but got '{}'",
+                path.get_type()
+            );
             return error!(meta, tok, msg);
         }
         self.path = Box::new(Some(path));
@@ -42,34 +47,30 @@ impl SyntaxModule<ParserMetadata> for LinesInvocation {
 }
 
 impl TranslateModule for LinesInvocation {
-    fn translate(&self, meta: &mut TranslateMetadata) -> String {
-        let name = format!("__AMBER_ARRAY_{}", meta.gen_value_id());
+    fn translate(&self, meta: &mut TranslateMetadata) -> FragmentKind {
         let temp = format!("__AMBER_LINE_{}", meta.gen_value_id());
-        let path = (*self.path).as_ref()
-            .map(|p| p.translate_eval(meta, false))
-            .unwrap_or_default();
-        let quote = meta.gen_quote();
-        let dollar = meta.gen_dollar();
+        let path = (*self.path)
+            .as_ref()
+            .map(|p| p.translate(meta))
+            .expect("Cannot read lines without provided path");
         let indent = TranslateMetadata::single_indent();
-        let block = [
-            format!("{name}=()"),
-            format!("while IFS= read -r {temp}; do"),
-            format!("{indent}{name}+=(\"${temp}\")"),
-            format!("done <{path}"),
-        ].join("\n");
-        meta.stmt_queue.push_back(block);
-        format!("{quote}{dollar}{{{name}[@]}}{quote}")
+        let id = meta.gen_value_id();
+        let value = meta.push_intermediate_variable_lazy("__array", Some(id), Type::array_of(Type::Text), FragmentKind::Empty);
+        meta.stmt_queue.extend([
+            raw_fragment!("while IFS= read -r {temp}; do"),
+            raw_fragment!("{indent}{}+=(\"${}\")", value.get_name(), temp),
+            fragments!("done <", path),
+        ]);
+        value.to_frag()
     }
 }
 
 impl LinesInvocation {
-    pub fn surround_iter(&self, meta: &mut TranslateMetadata, name: &str) -> (String, String) {
-        let path = (*self.path).as_ref()
+    pub fn translate_path(&self, meta: &mut TranslateMetadata) -> FragmentKind {
+        (*self.path)
+            .as_ref()
             .map(|p| p.translate(meta))
-            .unwrap_or_default();
-        let prefix = format!("while IFS= read -r {name}; do");
-        let suffix = format!("done <{path}");
-        (prefix, suffix)
+            .expect("Cannot read lines without provided path in iterator loop")
     }
 }