add firecrawl

Author: bboynton97

.gitignore

@@ -123,6 +123,7 @@ celerybeat.pid
 
 # Environments
 .env
+**/.env
 .venv
 env/
 venv/
@@ -167,3 +168,5 @@ ex/
 **/ex/
 cookiecutter.json
 
+examples/tests/
+examples/tests/**/*

agentstack/cli/cli.py

@@ -225,16 +225,6 @@ def ask_project_details(slug_name: Optional[str] = None) -> dict:
         inquirer.Text("version", message="What's the initial version", default="0.1.0"),
         inquirer.Text("description", message="Enter a description for your project"),
         inquirer.Text("author", message="Who's the author (your name)?"),
-        inquirer.List(
-            "license",
-            message="License?",
-            choices=[
-                "MIT",
-                "Apache-2.0",
-                "GPL",
-                "other",
-            ],
-        ),
     ]
 
     return inquirer.prompt(questions)
@@ -246,7 +236,7 @@ def insert_template(project_details: dict, framework_name: str, design: dict):
                                        description=project_details["description"],
                                        author_name=project_details["author"],
                                        version=project_details["version"],
-                                       license=project_details["license"],
+                                       license="MIT",
                                        year=datetime.now().year)
 
     project_structure = ProjectStructure()

agentstack/templates/crewai/tools/browserbase.py

@@ -1,3 +1,3 @@
 from crewai_tools import BrowserbaseLoadTool
 
-Browserbase = BrowserbaseLoadTool(text_content=True)
+browserbase = BrowserbaseLoadTool(text_content=True)

agentstack/templates/crewai/tools/firecrawl.py

@@ -0,0 +1,50 @@
+from crewai_tools import tool
+from firecrawl import FirecrawlApp
+import os
+
+app = FirecrawlApp(api_key=os.getenv('FIRECRAWL_API_KEY'))
+
+
+@tool
+def web_scrape(url: str):
+    """
+    Scrape a url and return markdown. Use this to read a singular page and web_crawl only if you
+    need to read all other links as well.
+    """
+    scrape_result = app.scrape_url(url, params={'formats': ['markdown']})
+    return scrape_result.markdown
+
+
+@tool
+def web_crawl(url: str):
+    """
+    Scrape a url and crawl through other links from that page, scraping their contents.
+    This tool returns a crawl_id that you will need to use after waiting for a period of time
+    to retrieve the final contents. You should attempt to accomplish another task while waiting
+    for the crawl to complete.
+
+    Crawl will ignore sublinks of a page if they aren’t children of the url you provide.
+    So, the website.com/other-parent/blog-1 wouldn’t be returned if you crawled website.com/blogs/.
+    """
+
+    crawl_status = app.crawl_url(
+        url,
+        params={
+            'limit': 100,
+            'scrapeOptions': {'formats': ['markdown']}
+        },
+        poll_interval=30
+    )
+
+    return crawl_status
+
+
+@tool
+def retrieve_web_crawl(crawl_id: str):
+    """
+    Retrieve the results of a previously started web crawl. Crawls take time to process
+    so be sure to only use this tool some time after initiating a crawl. The result
+    will tell you if the crawl is finished. If it is not, wait some more time then try again.
+    """
+    return app.check_crawl_status(crawl_id)
+

agentstack/tools/browserbase.json

@@ -2,5 +2,5 @@
   "name": "browserbase",
   "package": "poetry add browserbase crewai-tools",
   "env": "BROWSERBASE_API_KEY=...\nBROWSERBASE_PROJECT_ID=...",
-  "tools": ["Browserbase"]
+  "tools": ["browserbase"]
 }

agentstack/tools/firecrawl.json

@@ -0,0 +1,6 @@
+{
+  "name": "firecrawl",
+  "package": "poetry add firecrawl-py",
+  "env": "FIRECRAWL_API_KEY=...",
+  "tools": ["web_scrape", "web_crawl", "retrieve_web_crawl"]
+}