[OKL] Adds @directive preprocessor attribute

Makefile

@@ -157,7 +157,7 @@ bin-tests: unit-tests
 
 $(testPath)/bin/%:$(testPath)/src/%.cpp $(outputs)
 	@mkdir -p $(abspath $(dir $@))
-	$(compiler) $(testFlags) $(pthreadFlag) -o $@ $(flags) $< $(paths) $(linkerFlags) -L$(OCCA_DIR)/lib -locca
+	$(compiler) $(testFlags) $(pthreadFlag) -o $@ -Wl,-rpath,$(libPath) $(flags) $< $(paths) $(linkerFlags) -L$(OCCA_DIR)/lib -locca
 #=================================================
 
 

include/occa/lang/preprocessor.hpp

@@ -130,7 +130,13 @@ namespace occa {
 
       void processToken(token_t *token);
       void processIdentifier(identifierToken &token);
-      void processOperator(operatorToken &token);
+
+      void processOperator(operatorToken &opToken);
+      void processHashOperator(operatorToken &opToken);
+      void processAttributeOperator(operatorToken &opToken);
+      void freeAttributeOperatorTokens(token_t &opToken,
+                                       token_t &directiveToken,
+                                       std::list<token_t*> &prevOutputCache);
 
       bool lineIsTrue(identifierToken &directive,
                       bool &isTrue);

include/occa/lang/tokenizer.hpp

@@ -96,7 +96,7 @@ namespace occa {
       int peekForOperator();
 
       void getIdentifier(std::string &value);
-      void getString(std::string &value,
+      bool getString(std::string &value,
                      const int encoding = 0);
       void getRawString(std::string &value);
 

src/lang/macro.cpp

@@ -624,7 +624,7 @@ namespace occa {
 
         if (!token) {
           errorOn(&source,
-                  "Not able to find closing )");
+                  "Not able to find a closing )");
           break;
         }
 

src/lang/preprocessor.cpp

@@ -531,14 +531,21 @@ namespace occa {
       }
     }
 
-    void preprocessor_t::processOperator(operatorToken &token) {
-      if ((token.opType() != operatorType::hash) ||
-          !passedNewline) {
-        pushOutput(&token);
-        return;
+    void preprocessor_t::processOperator(operatorToken &opToken) {
+      const opType_t &op = opToken.opType();
+      // Make sure the line starts with #
+      if ((op == operatorType::hash) && passedNewline) {
+        return processHashOperator(opToken);
       }
-      delete &token;
+      // Test for @directive("...")
+      if (op == operatorType::attribute) {
+        return processAttributeOperator(opToken);
+      }
+      pushOutput(&opToken);
+    }
 
+    void preprocessor_t::processHashOperator(operatorToken &opToken) {
+      delete &opToken;
       if (inputIsEmpty()) {
         return;
       }
@@ -590,6 +597,145 @@ namespace occa {
       delete directive;
     }
 
+    void preprocessor_t::processAttributeOperator(operatorToken &opToken) {
+      // Check for @[directive]
+      token_t *directiveToken = getSourceToken();
+      const bool isDirective = (
+        (token_t::safeType(directiveToken) & tokenType::identifier) &&
+        (directiveToken->to<identifierToken>().value == "directive")
+      );
+      if (!isDirective) {
+        // Push the [@] token directly
+        pushOutput(&opToken);
+        // The next token still needs to be processed
+        if (directiveToken) {
+          pushInput(directiveToken);
+        }
+        return;
+      }
+
+      // Freeze our outputs and expand the rest of our symbols
+      std::list<token_t*> prevOutputCache = outputCache;
+      outputCache.clear();
+
+      // Make sure we have a tokenizer
+      tokenizer_t *tokenizer = (tokenizer_t*) getInput("tokenizer_t");
+      if (!tokenizer) {
+        warningOn(directiveToken,
+                  "Unable to apply @directive due to the lack of a tokenizer");
+        return freeAttributeOperatorTokens(opToken,
+                                           *directiveToken,
+                                           prevOutputCache);
+      }
+
+      // Try and get the 3 @directive[(]["..."][)] tokens
+      const int currentTokenizerErrors = tokenizer->errors;
+      const int currentErrors = errors;
+      while (!inputIsEmpty() && outputCache.size() < 3) {
+        fetchNext();
+        if ((errors > currentErrors) ||
+            (tokenizer->errors > currentTokenizerErrors)) {
+          return freeAttributeOperatorTokens(opToken,
+                                             *directiveToken,
+                                             prevOutputCache);
+        }
+      }
+      if (outputCache.size() < 3) {
+        errorOn(directiveToken,
+                "@directive expects a string argument");
+        return freeAttributeOperatorTokens(opToken,
+                                           *directiveToken,
+                                           prevOutputCache);
+      }
+
+      // Check for the proper 3 @directive[(]["..."][)] tokens
+      token_t *parenStartToken = outputCache.front();
+      outputCache.pop_front();
+      token_t *contentToken = outputCache.front();
+      outputCache.pop_front();
+      token_t *parenEndToken = outputCache.front();
+      outputCache.pop_front();
+
+      if (
+        !(token_t::safeOperatorType(parenStartToken) & operatorType::parenthesesStart)
+        || !(token_t::safeType(contentToken) & tokenType::string)
+        || !(token_t::safeOperatorType(parenEndToken) & operatorType::parenthesesEnd)
+      ) {
+        errorOn(directiveToken,
+                "@directive expects a string argument");
+        delete parenStartToken;
+        delete contentToken;
+        delete parenEndToken;
+        skipToNewline();
+        return freeAttributeOperatorTokens(opToken,
+                                           *directiveToken,
+                                           prevOutputCache);
+      }
+
+      stringToken &sourceToken = contentToken->to<stringToken>();
+      const std::string source = strip(sourceToken.value);
+      const int sourceLength = (int) source.size();
+
+      // Make sure @directive content starts with a #
+      bool missingStartHash = (!sourceLength || source[0] != '#');
+      // Make sure there are no newlines inserted
+      bool hasNewlines = (
+        !missingStartHash
+        && source.find("\\n") != std::string::npos
+      );
+
+      // Print error message if needed
+      if (missingStartHash || hasNewlines) {
+        if (missingStartHash) {
+          errorOn(contentToken,
+                  "@directive expects a string argument which starts with #");
+        } else {
+          errorOn(contentToken,
+                  "@directive expects a string argument cannot contain newlines");
+        }
+        delete parenStartToken;
+        delete contentToken;
+        delete parenEndToken;
+        return freeAttributeOperatorTokens(opToken,
+                                           *directiveToken,
+                                           prevOutputCache);
+      }
+
+      tokenVector newTokens;
+      tokenizer->tokenize(newTokens,
+                          sourceToken.origin,
+                          source);
+
+      incrementNewline();
+      pushInput(new newlineToken(sourceToken.origin));
+      const int newTokenCount = (int) newTokens.size();
+      for (int i = (newTokenCount - 1); i >= 0; --i) {
+        pushInput(newTokens[i]);
+      }
+
+      delete &opToken;
+      delete directiveToken;
+
+      // Bring back the original output cache
+      outputCache = prevOutputCache;
+    }
+
+    void preprocessor_t::freeAttributeOperatorTokens(token_t &opToken,
+                                                     token_t &directiveToken,
+                                                     std::list<token_t*> &prevOutputCache) {
+      delete &opToken;
+      delete &directiveToken;
+
+      // Delete output tokens
+      while (!outputCache.empty()) {
+        delete outputCache.front();
+        outputCache.pop_front();
+      }
+      while (!prevOutputCache.empty()) {
+        delete prevOutputCache.front();
+        prevOutputCache.pop_front();
+      }
+    }
 
     bool preprocessor_t::lineIsTrue(identifierToken &directive,
                                     bool &isTrue) {

src/lang/tokenizer.cpp

@@ -168,6 +168,7 @@ namespace occa {
 
     void tokenizer_t::printError(const std::string &message) {
       origin.printError(message);
+      ++errors;
     }
 
     void tokenizer_t::setLine(const int line) {
@@ -473,28 +474,33 @@ namespace occa {
       pop();
     }
 
-    void tokenizer_t::getString(std::string &value,
+    bool tokenizer_t::getString(std::string &value,
                                 const int encoding) {
       if (encoding & encodingType::R) {
         getRawString(value);
-        return;
+        return true;
       }
       if (*fp.start != '"') {
-        return;
+        return false;
       }
-      push();
+      // Skip ["]
       ++fp.start;
+
       push();
       skipTo("\"\n");
+
+      // Handle error outside of here
       if (*fp.start == '\n') {
+        printError("Not able to find a closing \"");
         pop();
-        popAndRewind();
-        return;
+        return false;
       }
+
       value = unescape(str(), '"');
       pop();
-      pop();
       ++fp.start;
+
+      return true;
     }
 
     void tokenizer_t::getRawString(std::string &value) {
@@ -679,12 +685,8 @@ namespace occa {
         return NULL;
       }
 
-      const char *start = fp.start;
       std::string value, udf;
-      getString(value, encoding);
-      if (fp.start == start) {
-        printError("Not able to find closing \"");
-        pop();
+      if (!getString(value, encoding)) {
         return NULL;
       }
 
@@ -713,7 +715,7 @@ namespace occa {
       push();
       skipTo("'\n");
       if (*fp.start == '\n') {
-        printError("Not able to find closing '");
+        printError("Not able to find a closing '");
         popAndRewind();
         pop();
         return NULL;
@@ -758,7 +760,7 @@ namespace occa {
         push();
         skipTo(">\n");
         if (*fp.start == '\n') {
-          printError("Not able to find closing >");
+          printError("Not able to find a closing >");
           pop();
           pop();
           return NULL;