summarizing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!apt-get update && apt-get install ffmpeg tmux vim -y\n",
    "!pip3 install moviepy openai python-dotenv pydub pytubefix openai-whisper llama-index llama-index-llms-openai llama-index-llms-ollama llama-index-embeddings-ollama \n",
    "!pip3 install flash-attn --no-build-isolation\n",
    "!curl -fsSL https://ollama.com/install.sh | sh\n",
    "\n",
    "# !chmod +x /usr/bin/ollama\n",
    "# !useradd -r -s /bin/false -m -d /usr/share/ollama ollama\n",
    "# !pip3 install openpyxl sentencepiece protobuf evaluate rouge_score absl-py tensorboardX bitsandbytes peft accelerate python-dotenv dspy-ai graspologic\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, pickle, gc, subprocess\n",
    "from dotenv import load_dotenv\n",
    "from video_transcription import split_audio_into_chunks, video_to_audio, transcribe_audio_chunks, get_transcription_model, download_video\n",
    "from llama_index.core import Document\n",
    "# import torch\n",
    "\n",
    "load_dotenv(\"/workspace/repos/agentic-ai/.env\")\n",
    "OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\")\n",
    "os.environ[\"OPENAI_API_KEY\"] = OPENAI_API_KEY\n",
    "\n",
    "video_path=\"/workspace/data/video1512218125.mp4\"\n",
    "# video_path=\"https://www.youtube.com/watch?v=Z07Ewop7rQA\"\n",
    "audio_output_path=\"/workspace/data/video_audio.mp3\"\n",
    "transcribe_output_dir=\"/workspace/data\"\n",
    "file_save_path=\"/workspace/data/transcription.pkl\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(file_save_path):\n",
    "    print(\"Loading transcription model...\")\n",
    "    transcribe_model = get_transcription_model(open_source_model=True)\n",
    "\n",
    "    print(\"Processing video...\")\n",
    "    if 'youtube' in video_path:\n",
    "        print(\"   Downloading video from youtube...\")\n",
    "        download_video(video_url=video_path, audio_output_path=audio_output_path)\n",
    "    else:\n",
    "        video_to_audio(video_path=video_path, audio_output_path=audio_output_path)\n",
    "\n",
    "    print(\"Splitting audio into chunks...\")\n",
    "    split_audio_into_chunks(audio_output_path=audio_output_path, transcribe_output_dir=transcribe_output_dir, max_chunk_size_mb=24)\n",
    "\n",
    "    print(\"Transcribing audio chunks...\")\n",
    "    documents = transcribe_audio_chunks(model=transcribe_model, chunk_dir=\"/workspace/data\", file_save_path=file_save_path)\n",
    "\n",
    "    del transcribe_model\n",
    "    # torch.cuda.empty_cache()\n",
    "    gc.collect()\n",
    "else:\n",
    "    with open(file_save_path, 'rb') as f:\n",
    "        documents = pickle.load(f)\n",
    "    \n",
    "if isinstance(documents[0], dict):\n",
    "    documents = [Document(text=chunk[\"text\"]) for chunk in documents]\n",
    "\n",
    "full_doc = \" \".join([doc.text for doc in documents])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core import Settings\n",
    "from llama_index.llms.ollama import Ollama\n",
    "from llama_index.llms.openai import OpenAI as LOpenAI\n",
    "\n",
    "# model_name, ctx_len = \"gpt-4o\", 128000\n",
    "# model_name, ctx_len = \"llama3.1:latest\", 128000\n",
    "model_name, ctx_len = \"hermes3:8b\", 128000\n",
    "\n",
    "if model_name == \"gpt-4o\":\n",
    "    llm = LOpenAI(model=model_name, max_tokens=4000)\n",
    "else:\n",
    "    try: \n",
    "        print(\"Pulling Ollama model...\")\n",
    "        sub_out = subprocess.run(['ollama', 'pull', model_name], capture_output=True, text=True)\n",
    "    except Exception as e: \n",
    "        print(f\"Error pulling model: Is the Ollama server running?\\n{e}\")\n",
    "    \n",
    "    addtion_kwargs = {\"max_new_tokens\": 4000}\n",
    "    # system_prompt = \"You are an expert at answering questions about rules and regulations regarding Title 17—Commodity and Securities Exchanges: CHAPTER II—SECURITIES AND EXCHANGE COMMISSION. Please provide a summary of the following text, and cite any sections, rules, acts or laws (e.g. § 230.503, § 240.13a-15, Act (15 U.S.C. 781), Investment Company Act of 1940) from context that support the answer. Be detailed in your response.\"\n",
    "    llm = Ollama(model=model_name, url=\"http://127.0.0.1:11434\", context_window=ctx_len, model_type=\"chat\", is_function_calling_model=False, \n",
    "                request_timeout=1000.0, additional_kwargs=addtion_kwargs) # system_prompt=system_prompt\n",
    "    print(llm.metadata)\n",
    "\n",
    "Settings.llm = llm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tree summarizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core.response_synthesizers import TreeSummarize\n",
    "summarizer = TreeSummarize(llm=llm, verbose=True)\n",
    "# prompt_summary = \"You are a professional executive of AlphaTrAI. Your job is to summarize this text in great detail from a video transcription. The summary will be distributed to investors and stakeholders, so give a lot of details and examples from the transcription.\"\n",
    "prompt_summary = f\"\"\"You are a professional executive at AlphaTrAI. Your job is to summarize the text from a video transcription. The summary will be a memo distributed to investors and stakeholders. Be sure it the memo has the following items:\n",
    "1. Extract all the names of new hires and their position, and/or new advisors mentioned in the transcription.\n",
    "2. Create a section to mention the personnel new to AlphaTrAI.\n",
    "3. Include other highlights and progress made by AlphaTrAI.\n",
    "4. Ensure the memo and ensure it is factual, optimistic, and any values mention come directly from the text. \n",
    "\n",
    "The transcription is as follows:\\n{full_doc}\"\"\"\n",
    "\n",
    "response = await summarizer.aget_response(prompt_summary, [doc.text for doc in documents])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(response)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LLM direct summarization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_summary = f\"\"\"You are a professional executive at AlphaTrAI. Your job is to summarize the text from a video transcription. The summary will be a memo distributed to investors and stakeholders. Be sure it the memo has the following items:\n",
    "1. Extract all the names of new hires and their position, and/or new advisors mentioned in the transcription.\n",
    "2. Include other highlights and progress made by AlphaTrAI.\n",
    "3. Ensure the memo is professional, fluid, factual, and optimistic. \n",
    "\n",
    "The transcription is as follows:\\n{full_doc}\"\"\"\n",
    "\n",
    "response = llm.complete(prompt_summary, max_tokens=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(response.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Agentic Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip3 install llama-index-embeddings-huggingface llama-index-vector-stores-neo4jvector llama-index-graph-stores-neo4j\n",
    "!apt install dialog apt-utils -y (done above)\n",
    "!wget -O - https://debian.neo4j.com/neotechnology.gpg.key | gpg --dearmor -o /etc/apt/keyrings/neotechnology.gpg\n",
    "!echo 'deb [signed-by=/etc/apt/keyrings/neotechnology.gpg] https://debian.neo4j.com stable latest' | tee -a /etc/apt/sources.list.d/neo4j.list\n",
    "!apt list -a neo4j\n",
    "!add-apt-repository universe -y\n",
    "!apt install neo4j=1:5.22.0 -y\n",
    "!echo \"neo4j-enterprise neo4j/question select I ACCEPT\" | debconf-set-selections\n",
    "!echo \"neo4j-enterprise neo4j/license note\" | debconf-set-selections\n",
    "!apt install openjdk-17-jre -y\n",
    "!cd /var/lib/neo4j/plugins/ && wget https://github.com/neo4j/apoc/releases/download/5.22.0/apoc-5.22.0-core.jar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_neo4j_password('bewaretheneo')\n",
    "add_lines_to_conf()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dotenv import load_dotenv\n",
    "load_dotenv()\n",
    "\n",
    "from llama_index.core.agent import ReActAgent\n",
    "# from llama_index.llms.openai import OpenAI\n",
    "from llama_index.core.tools import FunctionTool\n",
    "from rag_utils import create_neo4j_graph_store, create_neo4j_graphrag, neo4j_query, set_neo4j_password, add_lines_to_conf\n",
    "from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n",
    "\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "llm.is_function_calling_model = True\n",
    "\n",
    "embed_model_name = \"Alibaba-NLP/gte-Qwen2-1.5B-instruct\"\n",
    "print(\"loading embed model...\")\n",
    "embed_model = HuggingFaceEmbedding(model_name=embed_model_name)\n",
    "\n",
    "Settings.embed_model = embed_model\n",
    "Settings.chunk_size = 300\n",
    "Settings.chunk_overlap = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Literal\n",
    "from llama_index.core.indices.property_graph import SchemaLLMPathExtractor\n",
    "\n",
    "entities = Literal[\"PEOPLE\", \n",
    "                   \"PLACE\"\n",
    "]\n",
    "\n",
    "relations = Literal[\n",
    "    \"ROLE\",\n",
    "    \"COMPANY\"\n",
    "]\n",
    "\n",
    "validation_schema = {\n",
    "    \"People\": [\"ROLE\"],\n",
    "    \"Place\": [\"COMPANY\"],\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Settings.chunk_size = 300\n",
    "Settings.chunk_overlap = 50\n",
    "\n",
    "kg_extractor = SchemaLLMPathExtractor(\n",
    "    llm=llm,\n",
    "    possible_entities=entities,\n",
    "    possible_relations=relations,\n",
    "    kg_validation_schema=validation_schema,\n",
    "    strict=True,  # if false, will allow triples outside of the schema\n",
    "    num_workers=4,\n",
    "    max_triplets_per_chunk=10,\n",
    ")\n",
    "\n",
    "graph_store = create_neo4j_graph_store(neo_url=\"bolt://localhost:7687\", \n",
    "                                       password=os.getenv(\"NEO4J_PWD\"), \n",
    "                                       config={\"connection_timeout\": 240, \"connection_acquisition_timeout\": 240, \"max_connection_pool_size\": 1000})\n",
    "neo4j_query(graph_store, query=\"\"\"MATCH (n) DETACH DELETE n\"\"\")\n",
    "\n",
    "\n",
    "graph_index = create_neo4j_graphrag(documents, llm, embed_model, kg_extractor, graph_store)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core.tools import QueryEngineTool, ToolMetadata\n",
    "\n",
    "query_engine_tools = QueryEngineTool(\n",
    "            query_engine=graph_index,\n",
    "            metadata=ToolMetadata(\n",
    "                name=\"graph_tool\",\n",
    "                description=(\n",
    "                    \"Useful for finding people names and roles, and the company they work for.\"\n",
    "                ),\n",
    "            ),\n",
    "        ),\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip3 install llama-agents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_agents import (\n",
    "    AgentService,\n",
    "    ToolService,\n",
    "    MetaServiceTool,\n",
    "    ControlPlaneServer,\n",
    "    SimpleMessageQueue,\n",
    "    AgentOrchestrator,\n",
    ")\n",
    "\n",
    "from llama_index.core.agent import FunctionCallingAgentWorker\n",
    "from llama_index.core.agent import ReActAgentWorker, ReActAgent\n",
    "\n",
    "\n",
    "\n",
    "# create our multi-agent framework components\n",
    "message_queue = SimpleMessageQueue()\n",
    "control_plane = ControlPlaneServer(\n",
    "    message_queue=message_queue,\n",
    "    orchestrator=AgentOrchestrator(llm=llm),\n",
    ")\n",
    "\n",
    "# define Tool Service\n",
    "tool_service = ToolService(\n",
    "    message_queue=message_queue,\n",
    "    tools=[query_engine_tools],#, adding_tool],\n",
    "    running=True,\n",
    "    step_interval=0.5,\n",
    ")\n",
    "\n",
    "# define meta-tools here\n",
    "meta_tools = [\n",
    "    await MetaServiceTool.from_tool_service(\n",
    "        t.metadata.name,\n",
    "        message_queue=message_queue,\n",
    "        tool_service=tool_service,\n",
    "    )\n",
    "    for t in [query_engine_tools]#, adding_tool]\n",
    "]\n",
    "\n",
    "\n",
    "# define Agent and agent service\n",
    "# worker1 = FunctionCallingAgentWorker.from_tools(\n",
    "worker1 = ReActAgentWorker.from_tools(\n",
    "    meta_tools,\n",
    "    llm=llm,\n",
    ")\n",
    "agent1 = worker1.as_agent()\n",
    "agent_server_1 = AgentService(\n",
    "    agent=agent1,\n",
    "    message_queue=message_queue,\n",
    "    description=\"Summarize a transcription as a memo for investors and stakeholders.\",\n",
    "    service_name=\"summarize_transcription\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "asset_list = [\"Vanguard Group\",\t\"BlackRock\",\t\"State Street Global\",\t\"Fidelity Investments\",\t\"BNY Mellon\",\t\"Goldman Sachs Group\",\t\"J.P. Morgan Asset & Wealth\",\t\"Legal & General Investment\",\t\"Wellington Mgmt.\",\t\"Amundi\",\t\"Prudential Financial\",\t\"Geode Capital Mgmt.\",\t\"PIMCO\",\t\"Northern Trust Asset Mgmt.\",\t\"Nuveen\",\t\"Capital Group\",\t\"T. Rowe Price Associates\",\t\"AXA Investment\",\t\"Franklin Templeton\",\t\"Federated Hermes\",\t\"Invescos\",\t\"Dimensional Fund Advisors\",\t\"MetLife Investment Mgmt.\",\t\"Morgan Stanley Inv. Mgmt.\",\t\"New York Life Investments\",\t\"Schroders\",\t\"Principal Global Investors\",\t\"KKR\",\t\"DWS\",\t\"Macquarie Asset Mgmt.\",\t\"Brookfield Asset Mgmt.\",\t\"Allspring Global Investments\",\t\"BNP Paribas Asset Mgmt.\",\t\"Asset Management One\",\t\"Mercer\",\t\"Barings\",\t\"Aegon Asset Mgmt.\",\t\"AllianceBernstein\",\t\"Neuberger Berman\",\t\"Ares Mgmt.\",\t\"Columbia Threadneedle\",\t\"NISA Investment\",\t\"Voya Investment Mgmt.\",\t\"MassMutual\",\t\"Aviva Investors\",\t\"SEI Investments\",\t\"Manulife Investment\",\t\"SLC Management\",\t\"Russell Investments\",\t\"Loomis, Sayles\",\t\"Baillie Gifford Overseas\",\t\"Dodge & Cox\",\t\"TCW Group\",\t\"MFS Investment\",\t\"RBC Global Asset Mgmt.\",\t\"Mesirow\",\t\"Guggenheim Investments\",\t\"Wilmington Trust\",\t\"WTW Investment Services\",\t\"Conning\",\t\"Empower Investments\",\t\"PFM Asset Mgmt.\",\t\"CBRE Investment Mgmt.\",\t\"TD Global Invest. Solutions\",\t\"IFM Investors\",\t\"Arrowstreet Capital\",\t\"Nomura Asset Mgmt.\",\t\"Oaktree Capital\",\t\"Payden & Rygel\",\t\"Lazard Asset Mgmt.\",\t\"Victory Capital\",\t\"Artisan Partners\",\t\"PRIMECAP\",\t\"Man Group\",\t\"Robeco\",\t\"Baird Advisors\",\t\"Hamilton Lane\",\t\"Ninety One\",\t\"Partners Group\",\t\"Starwood Capital\",\t\"AQR Capital Mgmt.\",\t\"Acadian Asset Mgmt.\",\t\"Prologis\",\t\"LSV Asset Mgmt.\",\t\"StepStone Group\",\t\"American Century\",\t\"Charles Schwab Investment\",\t\"RhumbLine Advisers\",\t\"Pathway Capital\",\t\"Boston Partners\",\t\"Record Currency Mgmt.\",\t\"Income Research & Mgmt.\",\t\"AEW Capital\",\t\"Pictet Asset Mgmt.\",\t\"First Sentier Investors\",\t\"Hines\",\t\"New England Asset Mgmt.\",\t\"Alan Biller and Associates\",\t\"PPM America\",\t\"LaSalle Investment\",\t\"GCM Grosvenor\",\t\"PineBridge Investments\",\t\"PNC Financial\",\t\"CC&L Financial Group\",\t\"Fort Washington\",\t\"Dai-ichi Life Holdings\",\t\"Oak Hill Advisors\",\t\"William Blair\",\t\"Putnam Investments\",\t\"Ashmore Group\",\t\"Heitman\",\t\"Harrison Street\",\t\"Grantham, Mayo v. Otterloo\",\t\"PAG\",\t\"Harris Associates\",\t\"Adams Street Partners\",\t\"Sterling Capital\",\t\"GoldenTree Asset Mgmt.\",\t\"Mondrian Investment\",\t\"Angelo, Gordon\",\t\"Nikko Asset Mgmt.\",\t\"Harding Loevner\",\t\"Brown Advisory\",\t\"Portfolio Advisors\",\t\"Fisher Investments\",\t\"Cohen & Steers\",\t\"Marathon-London\",\t\"Harbor Capital Advisors\",\t\"Aristotle Capital Mgmt.\",\t\"SECOR Asset Mgmt.\",\t\"Stockbridge Capital Group\",\t\"PanAgora Asset Mgmt.\",\t\"Pzena Investment\",\t\"Causeway Capital\",\t\"Colchester Global Investors\",\t\"MissionSquare Investments\",\t\"Hayfin Capital Mgmt.\",\t\"ORIX USA\",\t\"CIBC Asset Mgmt.\",\t\"Los Angeles Capital\",\t\"Shenkman Group\",\t\"Jarislowsky Fraser\",\t\"EARNEST Partners\",\t\"Knights of Columbus Asset\",\t\"Strategic Investment Group\",\t\"Commonfund\",\t\"Rockpoint Group\",\t\"Hotchkis & Wiley\",\t\"AAM\",\t\"CIM Group*\",\t\"Beutel, Goodman\",\t\"Nomura Corporate Research\",\t\"Scout Investments\",\t\"Calamos Advisors\",\t\"ACORE Capital\",\t\"PCCP\",\t\"Guardian Capital\",\t\"DuPont Capital\",\t\"Canyon Partners\",\t\"Kayne Anderson Rudnick\",\t\"Polen Capital\",\t\"TA Realty\",\t\"Sustainable Growth Advisers\",\t\"MFG Asset Mgmt.\",\t\"Unigestion\",\t\"Intech\",\t\"Eagle Capital\",\t\"Garcia Hamilton\",\t\"Sprucegrove Investment\",\t\"Longfellow Investment\",\t\"Axiom Investors\",\t\"King Street Capital\",\t\"Wasatch Global Investors\",\t\"Boyd Watterson\",\t\"Champlain Investment\",\t\"Crestline Investors\",\t\"Callan\",\t\"Pacific Asset Mgmt.\",\t\"Fuller & Thaler\",\t\"Cantillon Capital Mgmt.\",\t\"Jacobs Levy Equity\",\t\"Brandes Investment\",\t\"Fayez Sarofim\",\t\"Sit Investment\",\t\"Cliffwater\",\t\"Intercontinental Real Estate\",\t\"Walton Street Capital\",\t\"Beacon Capital\",\t\"Rockwood Capital\",\t\"Breckinridge Capital\",\t\"Beach Point Capital\",\t\"Amalgamated Bank\",\t\"American Realty Advisors\",\t\"Abbott Capital\",\t\"Eagle Asset Mgmt.\",\t\"Westfield Capital\",\t\"Driehaus Capital\",\t\"CenterSquare Investment\",\t\"Segall Bryant & Hamill\",\t\"Polaris Capital\",\t\"Grayscale Investments\",\t\"GW&K Investment\",\t\"CornerStone Partners\",\t\"Westbrook Partners\",\t\"Bahl & Gaynor\",\t\"Sage Advisory Services\",\t\"Yousif Capital\",\t\"Commerce Trust\",\t\"Zacks Investment\",\t\"Stone Harbor Investment\",\t\"Brown Capital\",\t\"Cooke & Bieler\",\t\"L&B Realty\",\t\"Jensen Investment\",\t\"Burgundy Asset Mgmt.\",\t\"Pugh Capital\",\t\"Mill Creek Residential\",\t\"Global Endowment Mgmt.\",\t\"Ullico Investment\",\t\"London Co.\",\t\"GAMCO Investors\",\t\"Matthews Asia\",\t\"Capital Fund Mgmt.\",\t\"Waterfall Asset Mgmt.\",\t\"Westwood Global\",\t\"Frontier Capital\",\t\"Christian Brothers\",\t\"Manning & Napier\",\t\"Ariel Investments\",\t\"Washington Capital\",\t\"TimesSquare Capital\",\t\"Ramirez Asset Mgmt.\",\t\"Altrinsic Global Advisors\",\t\"National Real Estate\",\t\"Advent Capital\",\t\"CS McKee\",\t\"WEDGE Capital\",\t\"Newfleet Asset Mgmt.\",\t\"National Investment\",\t\"TT International\",\t\"Prima Capital Advisors\",\t\"Alger\",\t\"Silver Creek Capital\",\t\"River Road Asset Mgmt.\",\t\"Agincourt Capital\",\t\"Diamond Hill Capital\",\t\"AGF Investments\",\t\"Sentinel Real Estate\",\t\"Ceredex Value Advisors\",\t\"CoreCommodity\",\t\"LCM Partners\",\t\"Madison Realty\",\t\"Silvercrest Asset Mgmt.\",\t\"White Oak Global Advisors\",\t\"Luther King Capital\",\t\"Equus Capital\",\t\"Hardman Johnston Global\",\t\"AFL-CIO Housing Trust\",\t\"Corbin Capital\",\t\"City of London\",\t\"Spider Mgmt.\",\t\"M3 Capital\",\t\"Davis Advisors\",\t\"Torchlight Investors\",\t\"Stephens Inv. Mgmt. Group\",\t\"Great Lakes Advisors\",\t\"Congress Asset Mgmt.\",\t\"Parnassus Investments\",\t\"Dana Investment\",\t\"Martingale Asset Mgmt.\",\t\"Madison Investments\",\t\"Richmond Capital\",\t\"Camden Asset Mgmt.\",\t\"400 Capital Mgmt.\",\t\"Glenmede Investment\",\t\"Lyrical Asset Mgmt.\",\t\"Gramercy\",\t\"D.F. Dent\",\t\"Resource Mgmt.\",\t\"DePrince, Race & Zollo\",\t\"Fiduciary Mgmt./Milwaukee\",\t\"Duff & Phelps\",\t\"AFL-CIO Building Trust\",\t\"Johnson Asset Mgmt.\",\t\"LM Capital Group\",\t\"Conestoga Capital\",\t\"Sierra Investment\",\t\"Baird Equity Asset Mgmt.\",\t\"Forest Investment\",\t\"Carmel Partners\",\t\"Atalanta Sosnoff Capital\",\t\"Jackson Square Partners\",\t\"Peregrine Capital\",\t\"Todd Asset Mgmt.\",\t\"Hoisington Investment\",\t\"GlobeFlex Capital\",\t\"Kornitzer Capital\",\t\"Patron Capital\",\t\"Emerald Advisers\",\t\"Mar Vista Investment\",\t\"Stacey Braun Associates\",\t\"Leading Edge Investment\",\t\"Kennedy Capital\",\t\"Security Capital Research\",\t\"Riverbridge Partners\",\t\"Cardinal Capital\",\t\"Granahan Investment\",\t\"Dolan McEniry\",\t\"Angel Oak Capital\",\t\"Global Forest Partners\",\t\"Channing Capital\",\t\"ClariVest Asset Mgmt.\",\t\"Twin Bridge Capital\",\t\"Evanston Capital\",\t\"Aristotle Capital Boston\",\t\"Systematic Financial\",\t\"Palisade Capital\",\t\"Algert Global\",\t\"Hillsdale Investment\",\t\"Prime Group\",\t\"3650 REIT\",\t\"GTIS Partners\",\t\"Bivium Capital\",\t\"Molpus Woodlands Group\",\t\"Winthrop Capital\",\t\"Zevenbergen Capital\",\t\"Dalton Investments\",\t\"Hood River Capital\",\t\"Trillium Asset Mgmt.\",\t\"Bowen, Hanes\",\t\"EAM Investors\",\t\"Verger Capital\",\t\"GIA Partners\",\t\"Weatherbie Capital\",\t\"Oberweis Asset Mgmt.\",\t\"Zazove Associates\",\t\"Wexford Capital\",\t\"Timberland Inv. Resources\",\t\"Sawgrass Asset Mgmt.\",\t\"SSI Investment\",\t\"NewSouth Capital\",\t\"Foundry Partners\",\t\"Sound Shore Mgmt.\",\t\"KBS\",\t\"HS Management\",\t\"Silvant Capital\",\t\"Millstreet Capital\",\t\"Redwood Investments\",\t\"Genter Capital\",\t\"Smith Group Asset Mgmt.\",\t\"Sarofim Realty\",\t\"Edgar Lomax\",\t\"Covenant Capital Group\",\t\"Heartland Advisors\",\t\"Adelante Capital\",\t\"Rice Hall James\",\t\"Cramer Rosenthal McGlynn\",\t\"Quest Investment\",\t\"Penn Capital\",\t\"Ranger Investments\",\t\"Associated Capital Group\",\t\"Cornerstone Investment\",\t\"Smith Affiliated Capital\",\t\"Logan Capital\",\t\"Wilbanks, Smith & Thomas\",\t\"Orleans Capital\",\t\"Thornburg Investment\",\t\"Karpus Investment\",\t\"StoneRidge PMG Advisors\",\t\"Tributary Capital\",\t\"Mairs & Power\",\t\"Bridgeway Capital\",\t\"Granite Investment\",\t\"Ativo Capital Mgmt.\",\t\"Nicholas Investment\",\t\"Sasco Capital\",\t\"CS Capital\",\t\"TWIN Capital\",\t\"CI Inverness\",\t\"Miller/Howard Investments\",\t\"Belle Haven Investments\",\t\"Montag & Caldwell\",\t\"Anchor Capital\",\t\"Wedgewood Partners\",\t\"Wright Investors' Service\",\t\"Phocas Financial\",\t\"TSCG Investors\",\t\"Pier Capital\",\t\"GLOBALT\",\t\"Van Hulzen Asset Mgmt.\",\t\"SKBA Capital Mgmt.\",\t\"Domain Timber Advisors\",\t\"Speece Thorson Capital\",\t\"Redstone Advisors\",\t\"Aristotle Credit Partners\",\t\"TerraCotta Group\",\t\"Farr, Miller & Washington\",\t\"SouthernSun Asset Mgmt.\",\t\"Gifford Fong Associates\",\t\"Denali Advisors\",\t\"KDP Asset Mgmt.\",\t\"AMI Asset Mgmt.\",\t\"Semper Capital\",\t\"Renaissance Investment\",\t\"ZWJ Investment Counsel\",\t\"Campbell Newman Asset\",\t\"Gateway Investment\",\t\"SMH Capital Advisors\",\t\"Argent Capital\",\t\"Chicago Capital\",\t\"Osborne Partners\",\t\"Oak Associates\",\t\"Windham Capital\",\t\"Bridge City\",\t\"Strategy Asset Managers\",\t\"Kingdon Capital\",\t\"Glovista Investments\",\t\"Winslow Asset Mgmt.\",\t\"Hahn Capital\",\t\"Affinity Investment Advisors\",\t\"Teton Advisors\",\t\"Abner, Herrman & Brock\",\t\"NovaPoint Capital\",\t\"Paradigm Capital\",\t\"Flippin, Bruce & Porter\",\t\"Kestrel Investment\",\t\"Tom Johnson Investment\",\t\"Argus Investors' Counsel\",\t\"Branson, Fowlkes/Russell\",\t\"Robinson Value Mgmt.\",\t\"Chase Investment Counsel\",\t\"Nicholas Co.\",\t\"Cadinha\",\t\"Pacific West Land\",]\n",
    "import os, re, time\n",
    "import numpy as np\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv('/workspace/repos/agentic-ai/.env')\n",
    "from llama_index.llms.openai import OpenAI as LOpenAI\n",
    "from llama_index.retrievers.you import YouRetriever\n",
    "from llama_index.core.query_engine import RetrieverQueryEngine\n",
    "from llama_index.core import Settings\n",
    "\n",
    "model_name, ctx_len = \"gpt-4o-2024-08-06\", 128000\n",
    "you_api_key = os.environ[\"YDC_API_KEY\"]\n",
    "\n",
    "# openai_key = os.getenv(\"OPENAI_API_KEY\")\n",
    "# os.environ[\"OPENAI_API_KEY\"] = openai_key\n",
    "\n",
    "# llm = LOpenAI(model=model_name, max_tokens=8000)\n",
    "# Settings.llm = llm\n",
    "\n",
    "from llama_index.llms.ollama import Ollama\n",
    "\n",
    "model_name, ctx_len = \"llama3.1:8b-instruct-q8_0\", 128000\n",
    "addtion_kwargs = {\"max_new_tokens\": 200} #, 'temperature': 0.8}\n",
    "prompt = \"Your job is to extract information from the given text. Be concise in your answers. Do not provide any information that is not in the text.\"\n",
    "llm = Ollama(model=model_name, url=\"http://127.0.0.1:11434\", context_window=ctx_len, model_type=\"chat\", is_function_calling_model=False, \n",
    "                request_timeout=4000.0, **addtion_kwargs) #, system_prompt=system_prompt) additional_kwargs=addtion_kwargs,\n",
    "\n",
    "retriever = YouRetriever(api_key=you_api_key, endpoint=\"search\", num_web_results=3)\n",
    "query_engine = RetrieverQueryEngine.from_args(retriever, llm=llm)\n",
    "\n",
    "ceos=[]\n",
    "coos=[]\n",
    "cios=[]\n",
    "presidents=[]\n",
    "execs = [ceos, coos, cios, presidents]\n",
    "exec_strs = [\"Chief Executive Officer (CEO)\", \"Chief Operating Officer (COO)\", \"Chief Investment Officer (CIO)\", \"President\", \"10-Year Performance\", \"Key Funds\", \"Assets Under Management (AUM)\", \"Primary Address\"]\n",
    "results = []\n",
    "\n",
    "start=time.time()\n",
    "for i,a in enumerate(asset_list):\n",
    "    company_execs=[a]\n",
    "    for exec_type, exec_str in zip(execs, exec_strs[:4]):\n",
    "        system_prompt = f\"\"\"\n",
    "        Who is the current {exec_str} at {a}?\n",
    "        Only return the name.\n",
    "        \"\"\"\n",
    "        response = query_engine.query(system_prompt)\n",
    "        # print(str(response))\n",
    "        # response = llm.complete(system_prompt)\n",
    "        # response_check = llm.complete(f\"Extract and only output the name in the following text.\\n\\n{response.text}\")\n",
    "        company_execs.append(response.response)\n",
    "        \n",
    "    return_prompt=f\"\"\"What is the 10-year annualized return for {a}? Return the annualized percentage in this format: \"10-year return: 13.5%\".\"\"\"\n",
    "    company_execs.append(query_engine.query(return_prompt).response)\n",
    "    funds_prompt = f\"\"\"What are the key funds for {a}? Only return the names of the funds.\"\"\"\n",
    "    company_execs.append(query_engine.query(funds_prompt).response)\n",
    "    aum_prompt = f\"\"\"What is the current Assets Under Management (AUM) for {a}? Only return the total amount in USD.\"\"\"\n",
    "    company_execs.append(query_engine.query(aum_prompt).response)\n",
    "    address_prompt = f\"\"\"What is the primary address for {a}? Only return the address.\"\"\"\n",
    "    company_execs.append(query_engine.query(address_prompt).response)\n",
    "\n",
    "    if i%25==0:\n",
    "        end=time.time()\n",
    "        print(f\"{np.round((end-start)/60, 2)} Processed {i+1}/{len(asset_list)} companies...\")\n",
    "    \n",
    "    results.append(company_execs)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "\n",
    "# Sample list of lists\n",
    "\n",
    "\n",
    "# Specify the file name\n",
    "filename = '/workspace/data/asset_execs2.csv'\n",
    "header = [\"Company Name\"]+exec_strs\n",
    "# Open the file in write mode\n",
    "with open(filename, 'w', newline='') as file:\n",
    "    writer = csv.writer(file)\n",
    "    \n",
    "    # Write each row to the CSV file\n",
    "    for row in [header]+results:\n",
    "        writer.writerow(row)\n",
    "\n",
    "print(f\"Data has been written to {filename}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "asset_list = [\"Vanguard Group\",\t\"BlackRock\",\t\"State Street Global\",\t\"Fidelity Investments\",\t\"BNY Mellon\",\t\"Goldman Sachs Group\",\t\"J.P. Morgan Asset & Wealth\",\t\"Legal & General Investment\",\t\"Wellington Mgmt.\",\t\"Amundi\",\t\"Prudential Financial\",\t\"Geode Capital Mgmt.\",\t\"PIMCO\",\t\"Northern Trust Asset Mgmt.\",\t\"Nuveen\",\t\"Capital Group\",\t\"T. Rowe Price Associates\",\t\"AXA Investment\",\t\"Franklin Templeton\",\t\"Federated Hermes\",\t\"Invescos\",\t\"Dimensional Fund Advisors\",\t\"MetLife Investment Mgmt.\",\t\"Morgan Stanley Inv. Mgmt.\",\t\"New York Life Investments\",\t\"Schroders\",\t\"Principal Global Investors\",\t\"KKR\",\t\"DWS\",\t\"Macquarie Asset Mgmt.\",\t\"Brookfield Asset Mgmt.\",\t\"Allspring Global Investments\",\t\"BNP Paribas Asset Mgmt.\",\t\"Asset Management One\",\t\"Mercer\",\t\"Barings\",\t\"Aegon Asset Mgmt.\",\t\"AllianceBernstein\",\t\"Neuberger Berman\",\t\"Ares Mgmt.\",\t\"Columbia Threadneedle\",\t\"NISA Investment\",\t\"Voya Investment Mgmt.\",\t\"MassMutual\",\t\"Aviva Investors\",\t\"SEI Investments\",\t\"Manulife Investment\",\t\"SLC Management\",\t\"Russell Investments\",\t\"Loomis, Sayles\",\t\"Baillie Gifford Overseas\",\t\"Dodge & Cox\",\t\"TCW Group\",\t\"MFS Investment\",\t\"RBC Global Asset Mgmt.\",\t\"Mesirow\",\t\"Guggenheim Investments\",\t\"Wilmington Trust\",\t\"WTW Investment Services\",\t\"Conning\",\t\"Empower Investments\",\t\"PFM Asset Mgmt.\",\t\"CBRE Investment Mgmt.\",\t\"TD Global Invest. Solutions\",\t\"IFM Investors\",\t\"Arrowstreet Capital\",\t\"Nomura Asset Mgmt.\",\t\"Oaktree Capital\",\t\"Payden & Rygel\",\t\"Lazard Asset Mgmt.\",\t\"Victory Capital\",\t\"Artisan Partners\",\t\"PRIMECAP\",\t\"Man Group\",\t\"Robeco\",\t\"Baird Advisors\",\t\"Hamilton Lane\",\t\"Ninety One\",\t\"Partners Group\",\t\"Starwood Capital\",\t\"AQR Capital Mgmt.\",\t\"Acadian Asset Mgmt.\",\t\"Prologis\",\t\"LSV Asset Mgmt.\",\t\"StepStone Group\",\t\"American Century\",\t\"Charles Schwab Investment\",\t\"RhumbLine Advisers\",\t\"Pathway Capital\",\t\"Boston Partners\",\t\"Record Currency Mgmt.\",\t\"Income Research & Mgmt.\",\t\"AEW Capital\",\t\"Pictet Asset Mgmt.\",\t\"First Sentier Investors\",\t\"Hines\",\t\"New England Asset Mgmt.\",\t\"Alan Biller and Associates\",\t\"PPM America\",\t\"LaSalle Investment\",\t\"GCM Grosvenor\",\t\"PineBridge Investments\",\t\"PNC Financial\",\t\"CC&L Financial Group\",\t\"Fort Washington\",\t\"Dai-ichi Life Holdings\",\t\"Oak Hill Advisors\",\t\"William Blair\",\t\"Putnam Investments\",\t\"Ashmore Group\",\t\"Heitman\",\t\"Harrison Street\",\t\"Grantham, Mayo v. Otterloo\",\t\"PAG\",\t\"Harris Associates\",\t\"Adams Street Partners\",\t\"Sterling Capital\",\t\"GoldenTree Asset Mgmt.\",\t\"Mondrian Investment\",\t\"Angelo, Gordon\",\t\"Nikko Asset Mgmt.\",\t\"Harding Loevner\",\t\"Brown Advisory\",\t\"Portfolio Advisors\",\t\"Fisher Investments\",\t\"Cohen & Steers\",\t\"Marathon-London\",\t\"Harbor Capital Advisors\",\t\"Aristotle Capital Mgmt.\",\t\"SECOR Asset Mgmt.\",\t\"Stockbridge Capital Group\",\t\"PanAgora Asset Mgmt.\",\t\"Pzena Investment\",\t\"Causeway Capital\",\t\"Colchester Global Investors\",\t\"MissionSquare Investments\",\t\"Hayfin Capital Mgmt.\",\t\"ORIX USA\",\t\"CIBC Asset Mgmt.\",\t\"Los Angeles Capital\",\t\"Shenkman Group\",\t\"Jarislowsky Fraser\",\t\"EARNEST Partners\",\t\"Knights of Columbus Asset\",\t\"Strategic Investment Group\",\t\"Commonfund\",\t\"Rockpoint Group\",\t\"Hotchkis & Wiley\",\t\"AAM\",\t\"CIM Group*\",\t\"Beutel, Goodman\",\t\"Nomura Corporate Research\",\t\"Scout Investments\",\t\"Calamos Advisors\",\t\"ACORE Capital\",\t\"PCCP\",\t\"Guardian Capital\",\t\"DuPont Capital\",\t\"Canyon Partners\",\t\"Kayne Anderson Rudnick\",\t\"Polen Capital\",\t\"TA Realty\",\t\"Sustainable Growth Advisers\",\t\"MFG Asset Mgmt.\",\t\"Unigestion\",\t\"Intech\",\t\"Eagle Capital\",\t\"Garcia Hamilton\",\t\"Sprucegrove Investment\",\t\"Longfellow Investment\",\t\"Axiom Investors\",\t\"King Street Capital\",\t\"Wasatch Global Investors\",\t\"Boyd Watterson\",\t\"Champlain Investment\",\t\"Crestline Investors\",\t\"Callan\",\t\"Pacific Asset Mgmt.\",\t\"Fuller & Thaler\",\t\"Cantillon Capital Mgmt.\",\t\"Jacobs Levy Equity\",\t\"Brandes Investment\",\t\"Fayez Sarofim\",\t\"Sit Investment\",\t\"Cliffwater\",\t\"Intercontinental Real Estate\",\t\"Walton Street Capital\",\t\"Beacon Capital\",\t\"Rockwood Capital\",\t\"Breckinridge Capital\",\t\"Beach Point Capital\",\t\"Amalgamated Bank\",\t\"American Realty Advisors\",\t\"Abbott Capital\",\t\"Eagle Asset Mgmt.\",\t\"Westfield Capital\",\t\"Driehaus Capital\",\t\"CenterSquare Investment\",\t\"Segall Bryant & Hamill\",\t\"Polaris Capital\",\t\"Grayscale Investments\",\t\"GW&K Investment\",\t\"CornerStone Partners\",\t\"Westbrook Partners\",\t\"Bahl & Gaynor\",\t\"Sage Advisory Services\",\t\"Yousif Capital\",\t\"Commerce Trust\",\t\"Zacks Investment\",\t\"Stone Harbor Investment\",\t\"Brown Capital\",\t\"Cooke & Bieler\",\t\"L&B Realty\",\t\"Jensen Investment\",\t\"Burgundy Asset Mgmt.\",\t\"Pugh Capital\",\t\"Mill Creek Residential\",\t\"Global Endowment Mgmt.\",\t\"Ullico Investment\",\t\"London Co.\",\t\"GAMCO Investors\",\t\"Matthews Asia\",\t\"Capital Fund Mgmt.\",\t\"Waterfall Asset Mgmt.\",\t\"Westwood Global\",\t\"Frontier Capital\",\t\"Christian Brothers\",\t\"Manning & Napier\",\t\"Ariel Investments\",\t\"Washington Capital\",\t\"TimesSquare Capital\",\t\"Ramirez Asset Mgmt.\",\t\"Altrinsic Global Advisors\",\t\"National Real Estate\",\t\"Advent Capital\",\t\"CS McKee\",\t\"WEDGE Capital\",\t\"Newfleet Asset Mgmt.\",\t\"National Investment\",\t\"TT International\",\t\"Prima Capital Advisors\",\t\"Alger\",\t\"Silver Creek Capital\",\t\"River Road Asset Mgmt.\",\t\"Agincourt Capital\",\t\"Diamond Hill Capital\",\t\"AGF Investments\",\t\"Sentinel Real Estate\",\t\"Ceredex Value Advisors\",\t\"CoreCommodity\",\t\"LCM Partners\",\t\"Madison Realty\",\t\"Silvercrest Asset Mgmt.\",\t\"White Oak Global Advisors\",\t\"Luther King Capital\",\t\"Equus Capital\",\t\"Hardman Johnston Global\",\t\"AFL-CIO Housing Trust\",\t\"Corbin Capital\",\t\"City of London\",\t\"Spider Mgmt.\",\t\"M3 Capital\",\t\"Davis Advisors\",\t\"Torchlight Investors\",\t\"Stephens Inv. Mgmt. Group\",\t\"Great Lakes Advisors\",\t\"Congress Asset Mgmt.\",\t\"Parnassus Investments\",\t\"Dana Investment\",\t\"Martingale Asset Mgmt.\",\t\"Madison Investments\",\t\"Richmond Capital\",\t\"Camden Asset Mgmt.\",\t\"400 Capital Mgmt.\",\t\"Glenmede Investment\",\t\"Lyrical Asset Mgmt.\",\t\"Gramercy\",\t\"D.F. Dent\",\t\"Resource Mgmt.\",\t\"DePrince, Race & Zollo\",\t\"Fiduciary Mgmt./Milwaukee\",\t\"Duff & Phelps\",\t\"AFL-CIO Building Trust\",\t\"Johnson Asset Mgmt.\",\t\"LM Capital Group\",\t\"Conestoga Capital\",\t\"Sierra Investment\",\t\"Baird Equity Asset Mgmt.\",\t\"Forest Investment\",\t\"Carmel Partners\",\t\"Atalanta Sosnoff Capital\",\t\"Jackson Square Partners\",\t\"Peregrine Capital\",\t\"Todd Asset Mgmt.\",\t\"Hoisington Investment\",\t\"GlobeFlex Capital\",\t\"Kornitzer Capital\",\t\"Patron Capital\",\t\"Emerald Advisers\",\t\"Mar Vista Investment\",\t\"Stacey Braun Associates\",\t\"Leading Edge Investment\",\t\"Kennedy Capital\",\t\"Security Capital Research\",\t\"Riverbridge Partners\",\t\"Cardinal Capital\",\t\"Granahan Investment\",\t\"Dolan McEniry\",\t\"Angel Oak Capital\",\t\"Global Forest Partners\",\t\"Channing Capital\",\t\"ClariVest Asset Mgmt.\",\t\"Twin Bridge Capital\",\t\"Evanston Capital\",\t\"Aristotle Capital Boston\",\t\"Systematic Financial\",\t\"Palisade Capital\",\t\"Algert Global\",\t\"Hillsdale Investment\",\t\"Prime Group\",\t\"3650 REIT\",\t\"GTIS Partners\",\t\"Bivium Capital\",\t\"Molpus Woodlands Group\",\t\"Winthrop Capital\",\t\"Zevenbergen Capital\",\t\"Dalton Investments\",\t\"Hood River Capital\",\t\"Trillium Asset Mgmt.\",\t\"Bowen, Hanes\",\t\"EAM Investors\",\t\"Verger Capital\",\t\"GIA Partners\",\t\"Weatherbie Capital\",\t\"Oberweis Asset Mgmt.\",\t\"Zazove Associates\",\t\"Wexford Capital\",\t\"Timberland Inv. Resources\",\t\"Sawgrass Asset Mgmt.\",\t\"SSI Investment\",\t\"NewSouth Capital\",\t\"Foundry Partners\",\t\"Sound Shore Mgmt.\",\t\"KBS\",\t\"HS Management\",\t\"Silvant Capital\",\t\"Millstreet Capital\",\t\"Redwood Investments\",\t\"Genter Capital\",\t\"Smith Group Asset Mgmt.\",\t\"Sarofim Realty\",\t\"Edgar Lomax\",\t\"Covenant Capital Group\",\t\"Heartland Advisors\",\t\"Adelante Capital\",\t\"Rice Hall James\",\t\"Cramer Rosenthal McGlynn\",\t\"Quest Investment\",\t\"Penn Capital\",\t\"Ranger Investments\",\t\"Associated Capital Group\",\t\"Cornerstone Investment\",\t\"Smith Affiliated Capital\",\t\"Logan Capital\",\t\"Wilbanks, Smith & Thomas\",\t\"Orleans Capital\",\t\"Thornburg Investment\",\t\"Karpus Investment\",\t\"StoneRidge PMG Advisors\",\t\"Tributary Capital\",\t\"Mairs & Power\",\t\"Bridgeway Capital\",\t\"Granite Investment\",\t\"Ativo Capital Mgmt.\",\t\"Nicholas Investment\",\t\"Sasco Capital\",\t\"CS Capital\",\t\"TWIN Capital\",\t\"CI Inverness\",\t\"Miller/Howard Investments\",\t\"Belle Haven Investments\",\t\"Montag & Caldwell\",\t\"Anchor Capital\",\t\"Wedgewood Partners\",\t\"Wright Investors' Service\",\t\"Phocas Financial\",\t\"TSCG Investors\",\t\"Pier Capital\",\t\"GLOBALT\",\t\"Van Hulzen Asset Mgmt.\",\t\"SKBA Capital Mgmt.\",\t\"Domain Timber Advisors\",\t\"Speece Thorson Capital\",\t\"Redstone Advisors\",\t\"Aristotle Credit Partners\",\t\"TerraCotta Group\",\t\"Farr, Miller & Washington\",\t\"SouthernSun Asset Mgmt.\",\t\"Gifford Fong Associates\",\t\"Denali Advisors\",\t\"KDP Asset Mgmt.\",\t\"AMI Asset Mgmt.\",\t\"Semper Capital\",\t\"Renaissance Investment\",\t\"ZWJ Investment Counsel\",\t\"Campbell Newman Asset\",\t\"Gateway Investment\",\t\"SMH Capital Advisors\",\t\"Argent Capital\",\t\"Chicago Capital\",\t\"Osborne Partners\",\t\"Oak Associates\",\t\"Windham Capital\",\t\"Bridge City\",\t\"Strategy Asset Managers\",\t\"Kingdon Capital\",\t\"Glovista Investments\",\t\"Winslow Asset Mgmt.\",\t\"Hahn Capital\",\t\"Affinity Investment Advisors\",\t\"Teton Advisors\",\t\"Abner, Herrman & Brock\",\t\"NovaPoint Capital\",\t\"Paradigm Capital\",\t\"Flippin, Bruce & Porter\",\t\"Kestrel Investment\",\t\"Tom Johnson Investment\",\t\"Argus Investors' Counsel\",\t\"Branson, Fowlkes/Russell\",\t\"Robinson Value Mgmt.\",\t\"Chase Investment Counsel\",\t\"Nicholas Co.\",\t\"Cadinha\",\t\"Pacific West Land\",]\n",
    "import os, re\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv('/workspace/repos/agentic-ai/.env')\n",
    "from llama_index.llms.openai import OpenAI as LOpenAI\n",
    "from llama_index.retrievers.you import YouRetriever\n",
    "from llama_index.core.query_engine import RetrieverQueryEngine\n",
    "from llama_index.core import Settings\n",
    "from llama_index.llms.ollama import Ollama\n",
    "\n",
    "model_name, ctx_len = \"gpt-4o-2024-08-06\", 128000\n",
    "you_api_key = os.environ[\"YDC_API_KEY\"]\n",
    "\n",
    "openai_key = os.getenv(\"OPENAI_API_KEY\")\n",
    "os.environ[\"OPENAI_API_KEY\"] = openai_key\n",
    "\n",
    "model_name, ctx_len = \"llama3.1:8b-instruct-q8_0\", 128000\n",
    "addtion_kwargs = {\"max_new_tokens\": 8000, 'temperature': 0.9}\n",
    "llm = Ollama(model=model_name, url=\"http://127.0.0.1:11434\", context_window=ctx_len, model_type=\"chat\", is_function_calling_model=False, \n",
    "                request_timeout=4000.0, **addtion_kwargs) #, system_prompt=system_prompt) additional_kwargs=addtion_kwargs,\n",
    "# llm = LOpenAI(model=model_name, max_tokens=8000)\n",
    "# Settings.llm = llm\n",
    "\n",
    "retriever = YouRetriever(api_key=you_api_key, endpoint=\"search\", num_web_results=3)\n",
    "# query_engine = RetrieverQueryEngine.from_args(retriever, llm=llm)\n",
    "\n",
    "ceos=[]\n",
    "coos=[]\n",
    "cios=[]\n",
    "presidents=[]\n",
    "execs = [ceos, coos, cios, presidents]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "collect=[]\n",
    "# exec_strs = [\"Fund Manager\", \"Chief Executive Officer\", \"Chief Operating Officer\", \"Chief Investment Officer\", \"President\", \"Performance\", \"Key Funds\", \"Assets Under Management (AUM)\", \"Primary Address\"]\n",
    "a=\"Capital Group\"\n",
    "for exec_type, exec_str in zip(execs, exec_strs[:4]):\n",
    "    system_prompt = f\"\"\"\n",
    "    Find the names of the following executives at {a}:\n",
    "    1. {exec_strs[0]}\n",
    "    2. {exec_strs[1]}\n",
    "    3. {exec_strs[2]}\n",
    "    4. {exec_strs[3]}\n",
    "    Only return the names in order.\n",
    "    \"\"\"\n",
    "    response = retriever.retrieve(system_prompt)\n",
    "    # response = query_engine.query(system_prompt)\n",
    "    collect.append(response.response)\n",
    "    # print(str(response))\n",
    "    # response = llm.complete(system_prompt)\n",
    "    # response_check = llm.complete(f\"Extract and only output the name in the following text.\\n\\n{response.text}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip3 install duckduckgo-search langchain-community playwright && playwright install && playwright install-deps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from duckduckgo_search import DDGS\n",
    "from langchain_community.document_loaders import AsyncChromiumLoader\n",
    "from langchain_community.document_transformers import BeautifulSoupTransformer\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()\n",
    "\n",
    "def get_page(urls):\n",
    "    loader = AsyncChromiumLoader(urls)\n",
    "    html = loader.load()\n",
    "\n",
    "    bs_transformer = BeautifulSoupTransformer()\n",
    "    docs_transformed = bs_transformer.transform_documents(html, tags_to_extract=[\"p\"], remove_unwanted_tags=[\"a\"])\n",
    "\n",
    "    return docs_transformed\n",
    "\n",
    "\n",
    "def truncate(text):\n",
    "    words = text.split()\n",
    "    truncated = \" \".join(words[:400])\n",
    "\n",
    "    return truncated\n",
    "\n",
    "def ddg_search(query):\n",
    "    results = DDGS().text(query, max_results=5)\n",
    "    urls = []\n",
    "    for result in results:\n",
    "        url = result['href']\n",
    "        urls.append(url)\n",
    "\n",
    "    docs = get_page(urls)\n",
    "\n",
    "    content = []\n",
    "    for doc in docs:\n",
    "        page_text = re.sub(\"\\n\\n+\", \"\\n\", doc.page_content)\n",
    "        text = truncate(page_text)\n",
    "        content.append(text)\n",
    "\n",
    "    return content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "a=\"Capital Group\"\n",
    "system_prompt = f\"\"\"\n",
    "Find the names of the following executives at {a}:\n",
    "1. {exec_strs[0]}\n",
    "2. {exec_strs[1]}\n",
    "3. {exec_strs[2]}\n",
    "4. {exec_strs[3]}\n",
    "Only return the names in order.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# results = ddg_search(system_prompt)\n",
    "results = ddg_search(\"Who is the CEO at Capital Group?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(results[4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up ollama\n",
    "!apt-get update && apt-get install tmux vim -y\n",
    "!pip3 install --ignore-installed llama-index llama-parse llama_deploy llama-index-llms-huggingface llama-index-embeddings-huggingface llama-index-llms-ollama llama-index-embeddings-ollama llama-index-vector-stores-neo4jvector llama-index-graph-stores-neo4j llama-index-finetuning llama-index-utils-workflow llama-index-readers-file llama-index-retrievers-you\n",
    "!pip3 install sentencepiece protobuf evaluate rouge_score absl-py tensorboardX bitsandbytes peft accelerate python-dotenv graspologic fpdf2\n",
    "!pip3 install flash-attn --no-build-isolation\n",
    "!curl -fsSL https://ollama.com/install.sh | sh\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip3 install free-proxy fake-useragent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import urllib, requests, random, time\n",
    "from concurrent.futures import ThreadPoolExecutor, as_completed\n",
    "from fp.fp import FreeProxy\n",
    "from fake_useragent import UserAgent\n",
    "\n",
    "\n",
    "# def extract_proxies(html):\n",
    "#     soup = BeautifulSoup(html, 'html.parser')\n",
    "#     text = soup.get_text()\n",
    "    \n",
    "#     # Regular expression to match IP addresses and ports\n",
    "#     proxy_pattern = re.compile(r'\\b(?:[0-9]{1,3}\\.){3}[0-9]{1,3}:[0-9]{1,5}\\b')\n",
    "#     proxies = proxy_pattern.findall(text)\n",
    "    \n",
    "#     return proxies\n",
    "\n",
    "# def fetch_proxies():\n",
    "#     url = \"https://free-proxy-list.net/\"\n",
    "#     response = requests.get(url)\n",
    "#     soup = BeautifulSoup(response.text, 'html.parser')\n",
    "#     ips_ports = extract_proxies(soup.text)\n",
    "#     proxies = []\n",
    "\n",
    "#     for ip_port in ips_ports:\n",
    "#         proxy = f\"http://{ip_port}\"\n",
    "#         proxies.append(proxy)\n",
    "    \n",
    "#     return proxies\n",
    "\n",
    "\n",
    "class GoogleSearch:\n",
    "    def __init__(self, query: str) -> None:\n",
    "        self.query = query\n",
    "        escaped_query = urllib.parse.quote_plus(query)\n",
    "        self.URL = f\"https://www.google.com/search?q={escaped_query}\"\n",
    "\n",
    "        self.headers = {\n",
    "            \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3538.102 Safari/537.36\"\n",
    "        }\n",
    "        self.ua = UserAgent()\n",
    "        self.proxy = None\n",
    "        self.links = self.get_initial_links()\n",
    "        self.all_page_data = self.all_pages()\n",
    "\n",
    "    def get_random_user_agent(self):\n",
    "        self.headers[\"User-Agent\"] = self.ua.random\n",
    "        return None\n",
    "    \n",
    "    def get_random_proxy(self):\n",
    "        # return FreeProxy(https=True, google=True, timeout=1.0).get(repeat=False) country_id=['US', 'GB', 'CA', 'AU'], \n",
    "        return FreeProxy(google=True, https=True, timeout=20.0).get()\n",
    "\n",
    "    def clean_urls(self, anchors: list[str]) -> list[str]:\n",
    "\n",
    "        links: list[str] = []\n",
    "        for a in anchors:\n",
    "            links.append(\n",
    "                list(filter(lambda l: l.startswith(\"url=http\"), a[\"href\"].split(\"&\")))\n",
    "            )\n",
    "\n",
    "        links = [\n",
    "            link.split(\"url=\")[-1]\n",
    "            for sublist in links\n",
    "            for link in sublist\n",
    "            if len(link) > 0\n",
    "        ]\n",
    "\n",
    "        return links\n",
    "\n",
    "    def read_url_page(self, url: str) -> str:\n",
    "        self.get_random_user_agent()\n",
    "        # response = requests.get(url, headers=self.headers)\n",
    "        response = requests.get(url, headers=self.headers, proxies={\"https\": self.proxy, \"http\": self.proxy})\n",
    "        print(\"read page\", response.status_code)\n",
    "        response.raise_for_status()\n",
    "        soup = BeautifulSoup(response.text, \"html.parser\")\n",
    "        return soup.get_text(strip=True)\n",
    "\n",
    "    def get_initial_links(self) -> list[str]:\n",
    "        \"\"\"\n",
    "        scrape google for the query with keyword based search\n",
    "        \"\"\"\n",
    "        # print(\"Searching Google...\")\n",
    "        self.proxy = self.get_random_proxy()\n",
    "        self.get_random_user_agent()\n",
    "        response = requests.get(self.URL, headers=self.headers, proxies={\"https\": self.proxy, \"http\": self.proxy}) #\"https\": proxy, \n",
    "        print(\"inital links\", response.status_code)\n",
    "        # response = requests.get(self.URL, headers=self.headers)\n",
    "        # Print the response status and content for debugging\n",
    "        code429 = 0\n",
    "        while True:\n",
    "            if response.status_code in [429, 400]:\n",
    "                print(f\"Got a {response.status_code}. Retrying.\")\n",
    "                retry_after = int(response.headers.get(\"Retry-After\", 4))  # Default to 60 seconds if not provided\n",
    "                print(f\"Rate limited. Retrying after {retry_after} seconds.\")\n",
    "                time.sleep(retry_after)\n",
    "                proxy = self.get_random_proxy()\n",
    "                print(proxy)\n",
    "                self.get_random_user_agent()\n",
    "                response = requests.get(self.URL, headers=self.headers, proxies={\"https\": proxy, \"http\": proxy}) #\"https\": proxy, \n",
    "                code429 += 1\n",
    "                if code429 > 5:\n",
    "                    print(\"Too many 429s. Exiting.\", response.status_code)\n",
    "                    break\n",
    "            else:\n",
    "                print(\"Finally got through\", response.status_code)\n",
    "                break\n",
    "\n",
    "        soup = BeautifulSoup(response.text, \"html.parser\")\n",
    "        anchors = soup.find_all(\"a\", href=True)\n",
    "        return self.clean_urls(anchors)\n",
    "\n",
    "    def all_pages(self) -> list[tuple[str, str]]:\n",
    "        print(self.links)\n",
    "        data: list[tuple[str, str]] = []\n",
    "        with ThreadPoolExecutor(max_workers=4) as executor:\n",
    "\n",
    "            future_to_url = {\n",
    "                executor.submit(self.read_url_page, url): url for url in self.links[:3]\n",
    "            }\n",
    "            for future in as_completed(future_to_url):\n",
    "                url = future_to_url[future]\n",
    "                try:\n",
    "                    output = future.result()\n",
    "                    data.append((url, output))\n",
    "\n",
    "                except requests.exceptions.HTTPError as e:\n",
    "                    print(e)\n",
    "\n",
    "        return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "asset_list = [\"Vanguard Group\",\t\"BlackRock\",\t\"State Street Global\",\t\"Fidelity Investments\",\t\"BNY Mellon\",\t\"Goldman Sachs Group\",\t\"J.P. Morgan Asset & Wealth\",\t\"Legal & General Investment\",\t\"Wellington Mgmt.\",\t\"Amundi\",\t\"Prudential Financial\",\t\"Geode Capital Mgmt.\",\t\"PIMCO\",\t\"Northern Trust Asset Mgmt.\",\t\"Nuveen\",\t\"Capital Group\",\t\"T. Rowe Price Associates\",\t\"AXA Investment\",\t\"Franklin Templeton\",\t\"Federated Hermes\",\t\"Invescos\",\t\"Dimensional Fund Advisors\",\t\"MetLife Investment Mgmt.\",\t\"Morgan Stanley Inv. Mgmt.\",\t\"New York Life Investments\",\t\"Schroders\",\t\"Principal Global Investors\",\t\"KKR\",\t\"DWS\",\t\"Macquarie Asset Mgmt.\",\t\"Brookfield Asset Mgmt.\",\t\"Allspring Global Investments\",\t\"BNP Paribas Asset Mgmt.\",\t\"Asset Management One\",\t\"Mercer\",\t\"Barings\",\t\"Aegon Asset Mgmt.\",\t\"AllianceBernstein\",\t\"Neuberger Berman\",\t\"Ares Mgmt.\",\t\"Columbia Threadneedle\",\t\"NISA Investment\",\t\"Voya Investment Mgmt.\",\t\"MassMutual\",\t\"Aviva Investors\",\t\"SEI Investments\",\t\"Manulife Investment\",\t\"SLC Management\",\t\"Russell Investments\",\t\"Loomis, Sayles\",\t\"Baillie Gifford Overseas\",\t\"Dodge & Cox\",\t\"TCW Group\",\t\"MFS Investment\",\t\"RBC Global Asset Mgmt.\",\t\"Mesirow\",\t\"Guggenheim Investments\",\t\"Wilmington Trust\",\t\"WTW Investment Services\",\t\"Conning\",\t\"Empower Investments\",\t\"PFM Asset Mgmt.\",\t\"CBRE Investment Mgmt.\",\t\"TD Global Invest. Solutions\",\t\"IFM Investors\",\t\"Arrowstreet Capital\",\t\"Nomura Asset Mgmt.\",\t\"Oaktree Capital\",\t\"Payden & Rygel\",\t\"Lazard Asset Mgmt.\",\t\"Victory Capital\",\t\"Artisan Partners\",\t\"PRIMECAP\",\t\"Man Group\",\t\"Robeco\",\t\"Baird Advisors\",\t\"Hamilton Lane\",\t\"Ninety One\",\t\"Partners Group\",\t\"Starwood Capital\",\t\"AQR Capital Mgmt.\",\t\"Acadian Asset Mgmt.\",\t\"Prologis\",\t\"LSV Asset Mgmt.\",\t\"StepStone Group\",\t\"American Century\",\t\"Charles Schwab Investment\",\t\"RhumbLine Advisers\",\t\"Pathway Capital\",\t\"Boston Partners\",\t\"Record Currency Mgmt.\",\t\"Income Research & Mgmt.\",\t\"AEW Capital\",\t\"Pictet Asset Mgmt.\",\t\"First Sentier Investors\",\t\"Hines\",\t\"New England Asset Mgmt.\",\t\"Alan Biller and Associates\",\t\"PPM America\",\t\"LaSalle Investment\",\t\"GCM Grosvenor\",\t\"PineBridge Investments\",\t\"PNC Financial\",\t\"CC&L Financial Group\",\t\"Fort Washington\",\t\"Dai-ichi Life Holdings\",\t\"Oak Hill Advisors\",\t\"William Blair\",\t\"Putnam Investments\",\t\"Ashmore Group\",\t\"Heitman\",\t\"Harrison Street\",\t\"Grantham, Mayo v. Otterloo\",\t\"PAG\",\t\"Harris Associates\",\t\"Adams Street Partners\",\t\"Sterling Capital\",\t\"GoldenTree Asset Mgmt.\",\t\"Mondrian Investment\",\t\"Angelo, Gordon\",\t\"Nikko Asset Mgmt.\",\t\"Harding Loevner\",\t\"Brown Advisory\",\t\"Portfolio Advisors\",\t\"Fisher Investments\",\t\"Cohen & Steers\",\t\"Marathon-London\",\t\"Harbor Capital Advisors\",\t\"Aristotle Capital Mgmt.\",\t\"SECOR Asset Mgmt.\",\t\"Stockbridge Capital Group\",\t\"PanAgora Asset Mgmt.\",\t\"Pzena Investment\",\t\"Causeway Capital\",\t\"Colchester Global Investors\",\t\"MissionSquare Investments\",\t\"Hayfin Capital Mgmt.\",\t\"ORIX USA\",\t\"CIBC Asset Mgmt.\",\t\"Los Angeles Capital\",\t\"Shenkman Group\",\t\"Jarislowsky Fraser\",\t\"EARNEST Partners\",\t\"Knights of Columbus Asset\",\t\"Strategic Investment Group\",\t\"Commonfund\",\t\"Rockpoint Group\",\t\"Hotchkis & Wiley\",\t\"AAM\",\t\"CIM Group*\",\t\"Beutel, Goodman\",\t\"Nomura Corporate Research\",\t\"Scout Investments\",\t\"Calamos Advisors\",\t\"ACORE Capital\",\t\"PCCP\",\t\"Guardian Capital\",\t\"DuPont Capital\",\t\"Canyon Partners\",\t\"Kayne Anderson Rudnick\",\t\"Polen Capital\",\t\"TA Realty\",\t\"Sustainable Growth Advisers\",\t\"MFG Asset Mgmt.\",\t\"Unigestion\",\t\"Intech\",\t\"Eagle Capital\",\t\"Garcia Hamilton\",\t\"Sprucegrove Investment\",\t\"Longfellow Investment\",\t\"Axiom Investors\",\t\"King Street Capital\",\t\"Wasatch Global Investors\",\t\"Boyd Watterson\",\t\"Champlain Investment\",\t\"Crestline Investors\",\t\"Callan\",\t\"Pacific Asset Mgmt.\",\t\"Fuller & Thaler\",\t\"Cantillon Capital Mgmt.\",\t\"Jacobs Levy Equity\",\t\"Brandes Investment\",\t\"Fayez Sarofim\",\t\"Sit Investment\",\t\"Cliffwater\",\t\"Intercontinental Real Estate\",\t\"Walton Street Capital\",\t\"Beacon Capital\",\t\"Rockwood Capital\",\t\"Breckinridge Capital\",\t\"Beach Point Capital\",\t\"Amalgamated Bank\",\t\"American Realty Advisors\",\t\"Abbott Capital\",\t\"Eagle Asset Mgmt.\",\t\"Westfield Capital\",\t\"Driehaus Capital\",\t\"CenterSquare Investment\",\t\"Segall Bryant & Hamill\",\t\"Polaris Capital\",\t\"Grayscale Investments\",\t\"GW&K Investment\",\t\"CornerStone Partners\",\t\"Westbrook Partners\",\t\"Bahl & Gaynor\",\t\"Sage Advisory Services\",\t\"Yousif Capital\",\t\"Commerce Trust\",\t\"Zacks Investment\",\t\"Stone Harbor Investment\",\t\"Brown Capital\",\t\"Cooke & Bieler\",\t\"L&B Realty\",\t\"Jensen Investment\",\t\"Burgundy Asset Mgmt.\",\t\"Pugh Capital\",\t\"Mill Creek Residential\",\t\"Global Endowment Mgmt.\",\t\"Ullico Investment\",\t\"London Co.\",\t\"GAMCO Investors\",\t\"Matthews Asia\",\t\"Capital Fund Mgmt.\",\t\"Waterfall Asset Mgmt.\",\t\"Westwood Global\",\t\"Frontier Capital\",\t\"Christian Brothers\",\t\"Manning & Napier\",\t\"Ariel Investments\",\t\"Washington Capital\",\t\"TimesSquare Capital\",\t\"Ramirez Asset Mgmt.\",\t\"Altrinsic Global Advisors\",\t\"National Real Estate\",\t\"Advent Capital\",\t\"CS McKee\",\t\"WEDGE Capital\",\t\"Newfleet Asset Mgmt.\",\t\"National Investment\",\t\"TT International\",\t\"Prima Capital Advisors\",\t\"Alger\",\t\"Silver Creek Capital\",\t\"River Road Asset Mgmt.\",\t\"Agincourt Capital\",\t\"Diamond Hill Capital\",\t\"AGF Investments\",\t\"Sentinel Real Estate\",\t\"Ceredex Value Advisors\",\t\"CoreCommodity\",\t\"LCM Partners\",\t\"Madison Realty\",\t\"Silvercrest Asset Mgmt.\",\t\"White Oak Global Advisors\",\t\"Luther King Capital\",\t\"Equus Capital\",\t\"Hardman Johnston Global\",\t\"AFL-CIO Housing Trust\",\t\"Corbin Capital\",\t\"City of London\",\t\"Spider Mgmt.\",\t\"M3 Capital\",\t\"Davis Advisors\",\t\"Torchlight Investors\",\t\"Stephens Inv. Mgmt. Group\",\t\"Great Lakes Advisors\",\t\"Congress Asset Mgmt.\",\t\"Parnassus Investments\",\t\"Dana Investment\",\t\"Martingale Asset Mgmt.\",\t\"Madison Investments\",\t\"Richmond Capital\",\t\"Camden Asset Mgmt.\",\t\"400 Capital Mgmt.\",\t\"Glenmede Investment\",\t\"Lyrical Asset Mgmt.\",\t\"Gramercy\",\t\"D.F. Dent\",\t\"Resource Mgmt.\",\t\"DePrince, Race & Zollo\",\t\"Fiduciary Mgmt./Milwaukee\",\t\"Duff & Phelps\",\t\"AFL-CIO Building Trust\",\t\"Johnson Asset Mgmt.\",\t\"LM Capital Group\",\t\"Conestoga Capital\",\t\"Sierra Investment\",\t\"Baird Equity Asset Mgmt.\",\t\"Forest Investment\",\t\"Carmel Partners\",\t\"Atalanta Sosnoff Capital\",\t\"Jackson Square Partners\",\t\"Peregrine Capital\",\t\"Todd Asset Mgmt.\",\t\"Hoisington Investment\",\t\"GlobeFlex Capital\",\t\"Kornitzer Capital\",\t\"Patron Capital\",\t\"Emerald Advisers\",\t\"Mar Vista Investment\",\t\"Stacey Braun Associates\",\t\"Leading Edge Investment\",\t\"Kennedy Capital\",\t\"Security Capital Research\",\t\"Riverbridge Partners\",\t\"Cardinal Capital\",\t\"Granahan Investment\",\t\"Dolan McEniry\",\t\"Angel Oak Capital\",\t\"Global Forest Partners\",\t\"Channing Capital\",\t\"ClariVest Asset Mgmt.\",\t\"Twin Bridge Capital\",\t\"Evanston Capital\",\t\"Aristotle Capital Boston\",\t\"Systematic Financial\",\t\"Palisade Capital\",\t\"Algert Global\",\t\"Hillsdale Investment\",\t\"Prime Group\",\t\"3650 REIT\",\t\"GTIS Partners\",\t\"Bivium Capital\",\t\"Molpus Woodlands Group\",\t\"Winthrop Capital\",\t\"Zevenbergen Capital\",\t\"Dalton Investments\",\t\"Hood River Capital\",\t\"Trillium Asset Mgmt.\",\t\"Bowen, Hanes\",\t\"EAM Investors\",\t\"Verger Capital\",\t\"GIA Partners\",\t\"Weatherbie Capital\",\t\"Oberweis Asset Mgmt.\",\t\"Zazove Associates\",\t\"Wexford Capital\",\t\"Timberland Inv. Resources\",\t\"Sawgrass Asset Mgmt.\",\t\"SSI Investment\",\t\"NewSouth Capital\",\t\"Foundry Partners\",\t\"Sound Shore Mgmt.\",\t\"KBS\",\t\"HS Management\",\t\"Silvant Capital\",\t\"Millstreet Capital\",\t\"Redwood Investments\",\t\"Genter Capital\",\t\"Smith Group Asset Mgmt.\",\t\"Sarofim Realty\",\t\"Edgar Lomax\",\t\"Covenant Capital Group\",\t\"Heartland Advisors\",\t\"Adelante Capital\",\t\"Rice Hall James\",\t\"Cramer Rosenthal McGlynn\",\t\"Quest Investment\",\t\"Penn Capital\",\t\"Ranger Investments\",\t\"Associated Capital Group\",\t\"Cornerstone Investment\",\t\"Smith Affiliated Capital\",\t\"Logan Capital\",\t\"Wilbanks, Smith & Thomas\",\t\"Orleans Capital\",\t\"Thornburg Investment\",\t\"Karpus Investment\",\t\"StoneRidge PMG Advisors\",\t\"Tributary Capital\",\t\"Mairs & Power\",\t\"Bridgeway Capital\",\t\"Granite Investment\",\t\"Ativo Capital Mgmt.\",\t\"Nicholas Investment\",\t\"Sasco Capital\",\t\"CS Capital\",\t\"TWIN Capital\",\t\"CI Inverness\",\t\"Miller/Howard Investments\",\t\"Belle Haven Investments\",\t\"Montag & Caldwell\",\t\"Anchor Capital\",\t\"Wedgewood Partners\",\t\"Wright Investors' Service\",\t\"Phocas Financial\",\t\"TSCG Investors\",\t\"Pier Capital\",\t\"GLOBALT\",\t\"Van Hulzen Asset Mgmt.\",\t\"SKBA Capital Mgmt.\",\t\"Domain Timber Advisors\",\t\"Speece Thorson Capital\",\t\"Redstone Advisors\",\t\"Aristotle Credit Partners\",\t\"TerraCotta Group\",\t\"Farr, Miller & Washington\",\t\"SouthernSun Asset Mgmt.\",\t\"Gifford Fong Associates\",\t\"Denali Advisors\",\t\"KDP Asset Mgmt.\",\t\"AMI Asset Mgmt.\",\t\"Semper Capital\",\t\"Renaissance Investment\",\t\"ZWJ Investment Counsel\",\t\"Campbell Newman Asset\",\t\"Gateway Investment\",\t\"SMH Capital Advisors\",\t\"Argent Capital\",\t\"Chicago Capital\",\t\"Osborne Partners\",\t\"Oak Associates\",\t\"Windham Capital\",\t\"Bridge City\",\t\"Strategy Asset Managers\",\t\"Kingdon Capital\",\t\"Glovista Investments\",\t\"Winslow Asset Mgmt.\",\t\"Hahn Capital\",\t\"Affinity Investment Advisors\",\t\"Teton Advisors\",\t\"Abner, Herrman & Brock\",\t\"NovaPoint Capital\",\t\"Paradigm Capital\",\t\"Flippin, Bruce & Porter\",\t\"Kestrel Investment\",\t\"Tom Johnson Investment\",\t\"Argus Investors' Counsel\",\t\"Branson, Fowlkes/Russell\",\t\"Robinson Value Mgmt.\",\t\"Chase Investment Counsel\",\t\"Nicholas Co.\",\t\"Cadinha\",\t\"Pacific West Land\",]\n",
    "import os, re\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv('/workspace/repos/agentic-ai/.env')\n",
    "from llama_index.llms.openai import OpenAI as LOpenAI\n",
    "# from llama_index.retrievers.you import YouRetriever\n",
    "# from llama_index.core.query_engine import RetrieverQueryEngine\n",
    "from llama_index.core import Settings\n",
    "from llama_index.llms.ollama import Ollama\n",
    "\n",
    "# model_name, ctx_len = \"gpt-4o-2024-08-06\", 128000\n",
    "# you_api_key = os.environ[\"YDC_API_KEY\"]\n",
    "\n",
    "# openai_key = os.getenv(\"OPENAI_API_KEY\")\n",
    "# os.environ[\"OPENAI_API_KEY\"] = openai_key\n",
    "\n",
    "model_name, ctx_len = \"llama3.1:8b-instruct-q8_0\", 128000\n",
    "addtion_kwargs = {\"max_new_tokens\": 8000, 'temperature': 0.8}\n",
    "llm = Ollama(model=model_name, url=\"http://127.0.0.1:11434\", context_window=ctx_len, model_type=\"chat\", is_function_calling_model=False, \n",
    "                request_timeout=4000.0, **addtion_kwargs) #, system_prompt=system_prompt) additional_kwargs=addtion_kwargs,\n",
    "# llm = LOpenAI(model=model_name, max_tokens=8000)\n",
    "# Settings.llm = llm\n",
    "\n",
    "# retriever = YouRetriever(api_key=you_api_key, endpoint=\"search\", num_web_results=3)\n",
    "# query_engine = RetrieverQueryEngine.from_args(retriever, llm=llm)\n",
    "\n",
    "ceos=[]\n",
    "coos=[]\n",
    "cios=[]\n",
    "presidents=[]\n",
    "execs = [ceos, coos, cios, presidents]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def get_first_page_context(search_prompt):\n",
    "    results = GoogleSearch(search_prompt)\n",
    "    page_data = results.all_page_data\n",
    "    context = \"\"\n",
    "    for page in page_data:\n",
    "        context += page[1]\n",
    "        break\n",
    "    return context, page_data\n",
    "\n",
    "results = []\n",
    "exec_strs = [\"Chief Executive Officer (CEO)\", \"Chief Operating Officer (COO)\", \"Chief Investment Officer (CIO)\", \"President\", \"10-Year Performance\", \"Key Funds\", \"Assets Under Management (AUM)\", \"Primary Address\"]\n",
    "\n",
    "\n",
    "for i,a in enumerate(asset_list):\n",
    "    a = \"Vanguard\"\n",
    "    company_execs=[a]\n",
    "    for exec_type, exec_str in zip(execs, exec_strs[:4]):\n",
    "        search_prompt = f\"\"\"Who is the {exec_str} of {a}?\"\"\"\n",
    "        context = get_first_page_context(search_prompt)\n",
    "        answer_prompt = f\"\"\"Given the following text, extract the name of the {exec_str}.\n",
    "                            Only output the name.\\n\\n{context}\"\"\"\n",
    "        response = llm.complete(answer_prompt)\n",
    "        company_execs.append(response.text)\n",
    "\n",
    "    return_prompt=f\"\"\"What is the 10-year annualized return at {a}?\"\"\"\n",
    "    context = get_first_page_context(return_prompt)\n",
    "    answer_prompt = f\"\"\"Given the following text, extract the 10-year annualized return.\n",
    "                        Only output the return percent.\\n\\n{context}\"\"\"\n",
    "    company_execs.append(llm.complete(answer_prompt).text)\n",
    "    funds_prompt = f\"\"\"What are the most popular funds at {a}?\"\"\"\n",
    "    context = get_first_page_context(funds_prompt)\n",
    "    answer_prompt = f\"\"\"Given the following text, extract the fund names.\n",
    "                        Only output a list of the fund names.\\n\\n{context}\"\"\"\n",
    "    company_execs.append(llm.complete(answer_prompt).text)\n",
    "    aum_prompt = f\"\"\"What is the total Assets Under Management (AUM) in USD at {a}?\"\"\"\n",
    "    context = get_first_page_context(aum_prompt)\n",
    "    answer_prompt = f\"\"\"Given the following text, extract the AUM.\n",
    "                        Only output the AUM in USD.\\n\\n{context}\"\"\"\n",
    "    company_execs.append(llm.complete(answer_prompt).text)\n",
    "    address_prompt = f\"\"\"What is the primary address at {a}?\"\"\"\n",
    "    context = get_first_page_context(address_prompt)\n",
    "    answer_prompt = f\"\"\"Given the following text, extract the primary address.\n",
    "                        Only output the address.\\n\\n{context}\"\"\"\n",
    "    company_execs.append(llm.complete(answer_prompt).text)\n",
    "    break\n",
    "    # if i%25==0:\n",
    "    #     print(f\"Processed {i+1}/{len(asset_list)} companies...\")\n",
    "    # results.append(company_execs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exec_strs = [\"Chief Executive Officer (CEO)\", \"Chief Operating Officer (COO)\", \"Chief Investment Officer (CIO)\", \"President\", \"10-Year Performance\", \"Key Funds\", \"Assets Under Management (AUM)\", \"Primary Address\"]\n",
    "\n",
    "def get_first_page_context(search_prompt):\n",
    "    results = GoogleSearch(search_prompt)\n",
    "    page_data = results.all_page_data\n",
    "    context = \"\"\n",
    "    for page in page_data:\n",
    "        context += page[1]\n",
    "        break\n",
    "    return context, page_data\n",
    "\n",
    "company_execs=[]\n",
    "a = \"Vanguard\"\n",
    "funds_prompt = f\"\"\"What are the most popular funds at {a}?\"\"\"\n",
    "context, page_data = get_first_page_context(funds_prompt)\n",
    "answer_prompt = f\"\"\"Given the following text, extract the fund names.\n",
    "                    Only output a list of the fund names.\\n\\n{context}\"\"\"\n",
    "company_execs.append(llm.complete(answer_prompt).text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "url = \"https://api.proxyscrape.com/v4/free-proxy-list/get?request=display_proxies&proxy_format=protocolipport&format=text\"  \n",
    "header = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3538.102 Safari/537.36'}\n",
    "response = requests.get(url, headers=header)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0'}\n",
    "# http://50.207.199.81:80"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "header = [\"Company Name\"]+exec_strs\n",
    "for h,c in zip(header, company_execs):\n",
    "    print(f\"{h}: {c}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "page_data = results.all_page_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(page_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(page_data[0][1][:2000]+page_data[1][1][:2000]+page_data[2][1][:2000]+page_data[3][1][:2000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "context = \"\"\n",
    "for page in page_data:\n",
    "    context += page[1][:2000]\n",
    "# context = page_data[0][1][:1000]+page_data[1][1][:1000]+page_data[2][1][:1000]\n",
    "response = llm.complete(f\"\"\"Given the following text, extract the name of the CEO.\n",
    "                         Only output the name.\\n\\n{context}\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "header=[\"Company Name\"]+exec_strs\n",
    "bads=0\n",
    "for r in results:\n",
    "    for elem in r:\n",
    "        if len(elem.split())>4:\n",
    "            bads+=1\n",
    "            print(\" \".join(elem.split()[-3:]))\n",
    "            break\n",
    "print(bads)\n",
    "results=[r[0] for r in results]\n",
    "for i in range(len(results)):\n",
    "    for j in range(1,5):\n",
    "        # results[i][j] = results[i][j].strip().replace(\"'\", \"\")\n",
    "        # if results[i][j].endswith(\".\"):\n",
    "        #     results[i][j] = results[i][j][:-1]\n",
    "        # if results[i][j].endswith(\",\"):\n",
    "        #     results[i][j] = results[i][j][:-1]\n",
    "        # if \" \".join(results[i][j].split()[-3:]).startswith(\"is\"):\n",
    "        #     results[i][j] = \" \".join(results[i][j].split()[-3:])[2:].strip()\n",
    "        # if \" \".join(results[i][j].split()[-3:]).startswith((\"is\", \"have\", \"and\", \"a different\", \"my\")):\n",
    "        #     results[i][j] = \" \".join(results[i][j].split()[-3:])[2:].strip()\n",
    "        if len(results[i][j].split())>4:\n",
    "            results[i][j] = \" \".join(results[i][j].split()[-3:])[2:].replace('\"', '').strip()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "\n",
    "# Sample list of lists\n",
    "\n",
    "\n",
    "# Specify the file name\n",
    "filename = '/workspace/data/asset_execs2.csv'\n",
    "header = [\"Company Name\"]+exec_strs\n",
    "# Open the file in write mode\n",
    "with open(filename, 'w', newline='') as file:\n",
    "    writer = csv.writer(file)\n",
    "    \n",
    "    # Write each row to the CSV file\n",
    "    for row in [header]+results:\n",
    "        writer.writerow(row)\n",
    "\n",
    "print(f\"Data has been written to {filename}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "asset_list = [\"Vanguard Group\",\t\"BlackRock\",\t\"State Street Global\",\t\"Fidelity Investments\",\t\"BNY Mellon\",\t\"Goldman Sachs Group\",\t\"J.P. Morgan Asset & Wealth\",\t\"Legal & General Investment\",\t\"Wellington Mgmt.\",\t\"Amundi\",\t\"Prudential Financial\",\t\"Geode Capital Mgmt.\",\t\"PIMCO\",\t\"Northern Trust Asset Mgmt.\",\t\"Nuveen\",\t\"Capital Group\",\t\"T. Rowe Price Associates\",\t\"AXA Investment\",\t\"Franklin Templeton\",\t\"Federated Hermes\",\t\"Invescos\",\t\"Dimensional Fund Advisors\",\t\"MetLife Investment Mgmt.\",\t\"Morgan Stanley Inv. Mgmt.\",\t\"New York Life Investments\",\t\"Schroders\",\t\"Principal Global Investors\",\t\"KKR\",\t\"DWS\",\t\"Macquarie Asset Mgmt.\",\t\"Brookfield Asset Mgmt.\",\t\"Allspring Global Investments\",\t\"BNP Paribas Asset Mgmt.\",\t\"Asset Management One\",\t\"Mercer\",\t\"Barings\",\t\"Aegon Asset Mgmt.\",\t\"AllianceBernstein\",\t\"Neuberger Berman\",\t\"Ares Mgmt.\",\t\"Columbia Threadneedle\",\t\"NISA Investment\",\t\"Voya Investment Mgmt.\",\t\"MassMutual\",\t\"Aviva Investors\",\t\"SEI Investments\",\t\"Manulife Investment\",\t\"SLC Management\",\t\"Russell Investments\",\t\"Loomis, Sayles\",\t\"Baillie Gifford Overseas\",\t\"Dodge & Cox\",\t\"TCW Group\",\t\"MFS Investment\",\t\"RBC Global Asset Mgmt.\",\t\"Mesirow\",\t\"Guggenheim Investments\",\t\"Wilmington Trust\",\t\"WTW Investment Services\",\t\"Conning\",\t\"Empower Investments\",\t\"PFM Asset Mgmt.\",\t\"CBRE Investment Mgmt.\",\t\"TD Global Invest. Solutions\",\t\"IFM Investors\",\t\"Arrowstreet Capital\",\t\"Nomura Asset Mgmt.\",\t\"Oaktree Capital\",\t\"Payden & Rygel\",\t\"Lazard Asset Mgmt.\",\t\"Victory Capital\",\t\"Artisan Partners\",\t\"PRIMECAP\",\t\"Man Group\",\t\"Robeco\",\t\"Baird Advisors\",\t\"Hamilton Lane\",\t\"Ninety One\",\t\"Partners Group\",\t\"Starwood Capital\",\t\"AQR Capital Mgmt.\",\t\"Acadian Asset Mgmt.\",\t\"Prologis\",\t\"LSV Asset Mgmt.\",\t\"StepStone Group\",\t\"American Century\",\t\"Charles Schwab Investment\",\t\"RhumbLine Advisers\",\t\"Pathway Capital\",\t\"Boston Partners\",\t\"Record Currency Mgmt.\",\t\"Income Research & Mgmt.\",\t\"AEW Capital\",\t\"Pictet Asset Mgmt.\",\t\"First Sentier Investors\",\t\"Hines\",\t\"New England Asset Mgmt.\",\t\"Alan Biller and Associates\",\t\"PPM America\",\t\"LaSalle Investment\",\t\"GCM Grosvenor\",\t\"PineBridge Investments\",\t\"PNC Financial\",\t\"CC&L Financial Group\",\t\"Fort Washington\",\t\"Dai-ichi Life Holdings\",\t\"Oak Hill Advisors\",\t\"William Blair\",\t\"Putnam Investments\",\t\"Ashmore Group\",\t\"Heitman\",\t\"Harrison Street\",\t\"Grantham, Mayo v. Otterloo\",\t\"PAG\",\t\"Harris Associates\",\t\"Adams Street Partners\",\t\"Sterling Capital\",\t\"GoldenTree Asset Mgmt.\",\t\"Mondrian Investment\",\t\"Angelo, Gordon\",\t\"Nikko Asset Mgmt.\",\t\"Harding Loevner\",\t\"Brown Advisory\",\t\"Portfolio Advisors\",\t\"Fisher Investments\",\t\"Cohen & Steers\",\t\"Marathon-London\",\t\"Harbor Capital Advisors\",\t\"Aristotle Capital Mgmt.\",\t\"SECOR Asset Mgmt.\",\t\"Stockbridge Capital Group\",\t\"PanAgora Asset Mgmt.\",\t\"Pzena Investment\",\t\"Causeway Capital\",\t\"Colchester Global Investors\",\t\"MissionSquare Investments\",\t\"Hayfin Capital Mgmt.\",\t\"ORIX USA\",\t\"CIBC Asset Mgmt.\",\t\"Los Angeles Capital\",\t\"Shenkman Group\",\t\"Jarislowsky Fraser\",\t\"EARNEST Partners\",\t\"Knights of Columbus Asset\",\t\"Strategic Investment Group\",\t\"Commonfund\",\t\"Rockpoint Group\",\t\"Hotchkis & Wiley\",\t\"AAM\",\t\"CIM Group*\",\t\"Beutel, Goodman\",\t\"Nomura Corporate Research\",\t\"Scout Investments\",\t\"Calamos Advisors\",\t\"ACORE Capital\",\t\"PCCP\",\t\"Guardian Capital\",\t\"DuPont Capital\",\t\"Canyon Partners\",\t\"Kayne Anderson Rudnick\",\t\"Polen Capital\",\t\"TA Realty\",\t\"Sustainable Growth Advisers\",\t\"MFG Asset Mgmt.\",\t\"Unigestion\",\t\"Intech\",\t\"Eagle Capital\",\t\"Garcia Hamilton\",\t\"Sprucegrove Investment\",\t\"Longfellow Investment\",\t\"Axiom Investors\",\t\"King Street Capital\",\t\"Wasatch Global Investors\",\t\"Boyd Watterson\",\t\"Champlain Investment\",\t\"Crestline Investors\",\t\"Callan\",\t\"Pacific Asset Mgmt.\",\t\"Fuller & Thaler\",\t\"Cantillon Capital Mgmt.\",\t\"Jacobs Levy Equity\",\t\"Brandes Investment\",\t\"Fayez Sarofim\",\t\"Sit Investment\",\t\"Cliffwater\",\t\"Intercontinental Real Estate\",\t\"Walton Street Capital\",\t\"Beacon Capital\",\t\"Rockwood Capital\",\t\"Breckinridge Capital\",\t\"Beach Point Capital\",\t\"Amalgamated Bank\",\t\"American Realty Advisors\",\t\"Abbott Capital\",\t\"Eagle Asset Mgmt.\",\t\"Westfield Capital\",\t\"Driehaus Capital\",\t\"CenterSquare Investment\",\t\"Segall Bryant & Hamill\",\t\"Polaris Capital\",\t\"Grayscale Investments\",\t\"GW&K Investment\",\t\"CornerStone Partners\",\t\"Westbrook Partners\",\t\"Bahl & Gaynor\",\t\"Sage Advisory Services\",\t\"Yousif Capital\",\t\"Commerce Trust\",\t\"Zacks Investment\",\t\"Stone Harbor Investment\",\t\"Brown Capital\",\t\"Cooke & Bieler\",\t\"L&B Realty\",\t\"Jensen Investment\",\t\"Burgundy Asset Mgmt.\",\t\"Pugh Capital\",\t\"Mill Creek Residential\",\t\"Global Endowment Mgmt.\",\t\"Ullico Investment\",\t\"London Co.\",\t\"GAMCO Investors\",\t\"Matthews Asia\",\t\"Capital Fund Mgmt.\",\t\"Waterfall Asset Mgmt.\",\t\"Westwood Global\",\t\"Frontier Capital\",\t\"Christian Brothers\",\t\"Manning & Napier\",\t\"Ariel Investments\",\t\"Washington Capital\",\t\"TimesSquare Capital\",\t\"Ramirez Asset Mgmt.\",\t\"Altrinsic Global Advisors\",\t\"National Real Estate\",\t\"Advent Capital\",\t\"CS McKee\",\t\"WEDGE Capital\",\t\"Newfleet Asset Mgmt.\",\t\"National Investment\",\t\"TT International\",\t\"Prima Capital Advisors\",\t\"Alger\",\t\"Silver Creek Capital\",\t\"River Road Asset Mgmt.\",\t\"Agincourt Capital\",\t\"Diamond Hill Capital\",\t\"AGF Investments\",\t\"Sentinel Real Estate\",\t\"Ceredex Value Advisors\",\t\"CoreCommodity\",\t\"LCM Partners\",\t\"Madison Realty\",\t\"Silvercrest Asset Mgmt.\",\t\"White Oak Global Advisors\",\t\"Luther King Capital\",\t\"Equus Capital\",\t\"Hardman Johnston Global\",\t\"AFL-CIO Housing Trust\",\t\"Corbin Capital\",\t\"City of London\",\t\"Spider Mgmt.\",\t\"M3 Capital\",\t\"Davis Advisors\",\t\"Torchlight Investors\",\t\"Stephens Inv. Mgmt. Group\",\t\"Great Lakes Advisors\",\t\"Congress Asset Mgmt.\",\t\"Parnassus Investments\",\t\"Dana Investment\",\t\"Martingale Asset Mgmt.\",\t\"Madison Investments\",\t\"Richmond Capital\",\t\"Camden Asset Mgmt.\",\t\"400 Capital Mgmt.\",\t\"Glenmede Investment\",\t\"Lyrical Asset Mgmt.\",\t\"Gramercy\",\t\"D.F. Dent\",\t\"Resource Mgmt.\",\t\"DePrince, Race & Zollo\",\t\"Fiduciary Mgmt./Milwaukee\",\t\"Duff & Phelps\",\t\"AFL-CIO Building Trust\",\t\"Johnson Asset Mgmt.\",\t\"LM Capital Group\",\t\"Conestoga Capital\",\t\"Sierra Investment\",\t\"Baird Equity Asset Mgmt.\",\t\"Forest Investment\",\t\"Carmel Partners\",\t\"Atalanta Sosnoff Capital\",\t\"Jackson Square Partners\",\t\"Peregrine Capital\",\t\"Todd Asset Mgmt.\",\t\"Hoisington Investment\",\t\"GlobeFlex Capital\",\t\"Kornitzer Capital\",\t\"Patron Capital\",\t\"Emerald Advisers\",\t\"Mar Vista Investment\",\t\"Stacey Braun Associates\",\t\"Leading Edge Investment\",\t\"Kennedy Capital\",\t\"Security Capital Research\",\t\"Riverbridge Partners\",\t\"Cardinal Capital\",\t\"Granahan Investment\",\t\"Dolan McEniry\",\t\"Angel Oak Capital\",\t\"Global Forest Partners\",\t\"Channing Capital\",\t\"ClariVest Asset Mgmt.\",\t\"Twin Bridge Capital\",\t\"Evanston Capital\",\t\"Aristotle Capital Boston\",\t\"Systematic Financial\",\t\"Palisade Capital\",\t\"Algert Global\",\t\"Hillsdale Investment\",\t\"Prime Group\",\t\"3650 REIT\",\t\"GTIS Partners\",\t\"Bivium Capital\",\t\"Molpus Woodlands Group\",\t\"Winthrop Capital\",\t\"Zevenbergen Capital\",\t\"Dalton Investments\",\t\"Hood River Capital\",\t\"Trillium Asset Mgmt.\",\t\"Bowen, Hanes\",\t\"EAM Investors\",\t\"Verger Capital\",\t\"GIA Partners\",\t\"Weatherbie Capital\",\t\"Oberweis Asset Mgmt.\",\t\"Zazove Associates\",\t\"Wexford Capital\",\t\"Timberland Inv. Resources\",\t\"Sawgrass Asset Mgmt.\",\t\"SSI Investment\",\t\"NewSouth Capital\",\t\"Foundry Partners\",\t\"Sound Shore Mgmt.\",\t\"KBS\",\t\"HS Management\",\t\"Silvant Capital\",\t\"Millstreet Capital\",\t\"Redwood Investments\",\t\"Genter Capital\",\t\"Smith Group Asset Mgmt.\",\t\"Sarofim Realty\",\t\"Edgar Lomax\",\t\"Covenant Capital Group\",\t\"Heartland Advisors\",\t\"Adelante Capital\",\t\"Rice Hall James\",\t\"Cramer Rosenthal McGlynn\",\t\"Quest Investment\",\t\"Penn Capital\",\t\"Ranger Investments\",\t\"Associated Capital Group\",\t\"Cornerstone Investment\",\t\"Smith Affiliated Capital\",\t\"Logan Capital\",\t\"Wilbanks, Smith & Thomas\",\t\"Orleans Capital\",\t\"Thornburg Investment\",\t\"Karpus Investment\",\t\"StoneRidge PMG Advisors\",\t\"Tributary Capital\",\t\"Mairs & Power\",\t\"Bridgeway Capital\",\t\"Granite Investment\",\t\"Ativo Capital Mgmt.\",\t\"Nicholas Investment\",\t\"Sasco Capital\",\t\"CS Capital\",\t\"TWIN Capital\",\t\"CI Inverness\",\t\"Miller/Howard Investments\",\t\"Belle Haven Investments\",\t\"Montag & Caldwell\",\t\"Anchor Capital\",\t\"Wedgewood Partners\",\t\"Wright Investors' Service\",\t\"Phocas Financial\",\t\"TSCG Investors\",\t\"Pier Capital\",\t\"GLOBALT\",\t\"Van Hulzen Asset Mgmt.\",\t\"SKBA Capital Mgmt.\",\t\"Domain Timber Advisors\",\t\"Speece Thorson Capital\",\t\"Redstone Advisors\",\t\"Aristotle Credit Partners\",\t\"TerraCotta Group\",\t\"Farr, Miller & Washington\",\t\"SouthernSun Asset Mgmt.\",\t\"Gifford Fong Associates\",\t\"Denali Advisors\",\t\"KDP Asset Mgmt.\",\t\"AMI Asset Mgmt.\",\t\"Semper Capital\",\t\"Renaissance Investment\",\t\"ZWJ Investment Counsel\",\t\"Campbell Newman Asset\",\t\"Gateway Investment\",\t\"SMH Capital Advisors\",\t\"Argent Capital\",\t\"Chicago Capital\",\t\"Osborne Partners\",\t\"Oak Associates\",\t\"Windham Capital\",\t\"Bridge City\",\t\"Strategy Asset Managers\",\t\"Kingdon Capital\",\t\"Glovista Investments\",\t\"Winslow Asset Mgmt.\",\t\"Hahn Capital\",\t\"Affinity Investment Advisors\",\t\"Teton Advisors\",\t\"Abner, Herrman & Brock\",\t\"NovaPoint Capital\",\t\"Paradigm Capital\",\t\"Flippin, Bruce & Porter\",\t\"Kestrel Investment\",\t\"Tom Johnson Investment\",\t\"Argus Investors' Counsel\",\t\"Branson, Fowlkes/Russell\",\t\"Robinson Value Mgmt.\",\t\"Chase Investment Counsel\",\t\"Nicholas Co.\",\t\"Cadinha\",\t\"Pacific West Land\",]\n",
    "import os, re, subprocess\n",
    "from dotenv import load_dotenv\n",
    "from llama_index.llms.openai import OpenAI as LOpenAI\n",
    "from llama_index.core import Settings\n",
    "from llama_index.llms.ollama import Ollama\n",
    "load_dotenv('/workspace/repos/agentic-ai/.env')\n",
    "\n",
    "model_name, ctx_len = \"llama3.1:8b-instruct-q8_0\", 128000\n",
    "try:\n",
    "    # Check if the model is present\n",
    "    result = subprocess.run(['ollama', 'list'], capture_output=True, text=True, check=True)\n",
    "    output = result.stdout\n",
    "    if model_name not in output:\n",
    "        subprocess.run(['ollama', 'pull', model_name], check=True)\n",
    "        print(f\"Model {model_name} pulled successfully.\")\n",
    "    else:\n",
    "        print(f\"Model {model_name} is already present.\")\n",
    "except subprocess.CalledProcessError as e:\n",
    "    print(f\"An error occurred: {e}\")\n",
    "addtion_kwargs = {\"max_new_tokens\": 2000}#, 'temperature': 0.8}\n",
    "# system_prompt = \"Your task is to extract relevant information from the provided text based on the given question. Keep your answers brief and to the point. Do not include calculations, explanations, or any information not explicitly stated in the text.\"\n",
    "llm = Ollama(model=model_name, url=\"http://127.0.0.1:11434\", context_window=ctx_len, model_type=\"chat\", is_function_calling_model=False, \n",
    "             request_timeout=4000.0, **addtion_kwargs) #, system_prompt=system_prompt) additional_kwargs=addtion_kwargs,\n",
    "print(llm.metadata)\n",
    "\n",
    "# openai_key = os.getenv(\"OPENAI_API_KEY\")\n",
    "# os.environ[\"OPENAI_API_KEY\"] = openai_key\n",
    "# llm = LOpenAI(model=model_name, max_tokens=8000)\n",
    "# Settings.llm = llm\n",
    "\n",
    "# retriever = YouRetriever(api_key=you_api_key, endpoint=\"search\", num_web_results=3)\n",
    "# query_engine = RetrieverQueryEngine.from_args(retriever, llm=llm)\n",
    "\n",
    "ceos=[]\n",
    "coos=[]\n",
    "cios=[]\n",
    "presidents=[]\n",
    "execs = [ceos, coos, cios, presidents]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from crawl4ai import AsyncWebCrawler\n",
    "import nest_asyncio\n",
    "import asyncio\n",
    "from fake_useragent import UserAgent\n",
    "# https://pypi.org/project/googlesearch-python/\n",
    "from googlesearch import search\n",
    "import csv\n",
    "import random\n",
    "\n",
    "nest_asyncio.apply()\n",
    "\n",
    "verbose=False\n",
    "\n",
    "async def main(urls):\n",
    "    content = []\n",
    "    ua = UserAgent()\n",
    "    async with AsyncWebCrawler(verbose=verbose) as crawler:\n",
    "        for url in urls:\n",
    "            result = await crawler.arun(url=url, word_count_threshold=2000, user_agent=ua.random, verbose=verbose)\n",
    "            content.append(result)\n",
    "    return content\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    content = asyncio.run(main())\n",
    "\n",
    "\n",
    "def get_url_info(queries, num_results=3, num_words=2000, description_only=True):\n",
    "    urls = []\n",
    "    if isinstance(queries, str):\n",
    "        queries = [queries]\n",
    "    for query in queries:\n",
    "        for url in search(query, \n",
    "                          sleep_interval=random.randint(2, 5), \n",
    "                          num_results=num_results,\n",
    "                          timeout=1000,\n",
    "                          advanced=True):\n",
    "            if description_only:\n",
    "                urls.append(url.description)\n",
    "            else:\n",
    "                urls.append(url.url)\n",
    "\n",
    "    context = \"\"\n",
    "\n",
    "    if description_only:\n",
    "        for con in urls:\n",
    "            if con is not None:\n",
    "                context += \"\\n\" + con\n",
    "            else:\n",
    "                continue\n",
    "    else:\n",
    "        content = asyncio.run(main(urls))\n",
    "        for con in content:\n",
    "            if con.markdown is not None:\n",
    "                truncated_content = \" \".join(con.markdown.split()[:num_words])\n",
    "                context += \"\\n\" + truncated_content\n",
    "            else:\n",
    "                continue\n",
    "\n",
    "    return context\n",
    "\n",
    "def make_csv(results, filename, header=None, verbose=True):\n",
    "    # Check if the file exists\n",
    "    file_exists = os.path.isfile(filename)\n",
    "\n",
    "    # Open the file in append mode if it exists, otherwise write mode\n",
    "    with open(filename, 'a' if file_exists else 'w', newline='') as file:\n",
    "        writer = csv.writer(file)\n",
    "        \n",
    "        # Write the header only if the file does not exist\n",
    "        if not file_exists and header is not None:\n",
    "            writer.writerow(header)\n",
    "        \n",
    "        # Write each row to the CSV file\n",
    "        for row in results:\n",
    "            writer.writerow(row)\n",
    "\n",
    "    if verbose: \n",
    "        print(f\"Data has been {'appended to' if file_exists else 'written to'} {filename}\")\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "string1=\"\"\"Here is an outline on how to write a letter of recommendation for graduate school:\n",
    "\n",
    "**I. Introduction**\n",
    "\n",
    "* Begin with a formal greeting and address the recommender's title (e.g. Dr./Professor, Dean)\n",
    "* Clearly state your relationship with the student (e.g. academic advisor, research mentor, colleague)\n",
    "\n",
    "**II. Overview of the Student's Qualities and Abilities**\n",
    "\n",
    "* Summarize the student's academic achievements, research experience, or relevant skills\n",
    "* Highlight their strengths, such as:\n",
    "\t+ Academic ability and potential for success in graduate school\n",
    "\t+ Research skills and contributions to projects or teams\n",
    "\t+ Leadership, teamwork, or communication abilities\n",
    "\t+ Adaptability, resilience, or other relevant personal qualities\n",
    "\n",
    "**III. Specific Examples and Stories**\n",
    "\n",
    "* Provide 2-3 specific examples that illustrate the student's skills, abilities, or achievements\n",
    "* Use anecdotes or case studies to demonstrate the student's:\n",
    "\t+ Critical thinking and problem-solving abilities\n",
    "\t+ Ability to work independently or as part of a team\n",
    "\t+ Capacity for original thought and innovation\n",
    "\t+ Communication skills, including writing, speaking, or presenting\n",
    "\n",
    "**IV. Why the Student is a Good Fit for Graduate School**\n",
    "\n",
    "* Explain why you believe the student is well-suited for graduate study in their chosen field\n",
    "* Highlight their interests, goals, and motivations for pursuing graduate education\n",
    "* Emphasize how their research experiences, skills, or personal qualities make them a strong candidate for admission\n",
    "\n",
    "**V. Conclusion**\n",
    "\n",
    "* Reiterate your support for the student's application\n",
    "* Close with a positive statement about the student's potential and future success in graduate school\n",
    "\n",
    "**VI. Final Details**\n",
    "\n",
    "* Include any additional information that may be relevant to the application (e.g. letters of recommendation from previous supervisors or colleagues)\n",
    "* Proofread carefully to ensure accuracy, clarity, and professionalism\n",
    "\n",
    "**VII. Tips for Writers**\n",
    "\n",
    "* Be specific and provide concrete examples to support your recommendations\n",
    "* Use a clear, concise writing style and avoid jargon or technical terms that may be unfamiliar to non-experts\n",
    "* Keep the letter focused on the student's abilities and potential, rather than their personal characteristics or demographics\n",
    "\n",
    "**VIII. Final Checklist**\n",
    "\n",
    "* Ensure you have:\n",
    "\t+ A strong relationship with the student (ideally 1-2 years)\n",
    "\t+ First-hand knowledge of the student's academic and research experiences\n",
    "\t+ A clear understanding of the graduate program and its requirements\n",
    "\t+ Time to write a thorough, well-researched letter\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "string2 = \"\"\"Here is a general outline on how to write a letter of recommendation for graduate school:\n",
    "\n",
    "**I. Introduction**\n",
    "\n",
    "* Begin by stating the purpose of the letter and the context in which you're writing it\n",
    "* Mention your relationship with the applicant (e.g., professor, advisor, colleague)\n",
    "* Provide a brief overview of the applicant's background and qualifications\n",
    "\n",
    "**II. Academic Ability and Potential**\n",
    "\n",
    "* Describe the applicant's academic abilities and strengths\n",
    "* Discuss their potential for success in graduate school\n",
    "* Highlight any notable achievements or accomplishments they've made during their undergraduate studies\n",
    "\n",
    "**III. Relevant Skills and Experiences**\n",
    "\n",
    "* Outline the applicant's relevant skills and experiences that make them a strong candidate for graduate study\n",
    "* Discuss how these skills and experiences have prepared them for the challenges of graduate school\n",
    "* Emphasize their ability to apply theoretical knowledge in practical settings\n",
    "\n",
    "**IV. Personal Qualities and Characteristics**\n",
    "\n",
    "* Describe the applicant's personal qualities and characteristics that make them a good fit for graduate school\n",
    "* Discuss their motivation, work ethic, and commitment to their field of study\n",
    "* Highlight any notable personal achievements or awards they've received\n",
    "\n",
    "**V. Graduate School Potential**\n",
    "\n",
    "* Explain why you think the applicant is well-suited for graduate study in their chosen field\n",
    "* Describe the specific graduate program they're applying to and how it aligns with their goals and interests\n",
    "* Mention any relevant research opportunities, internships, or other experiences that would enhance their education\n",
    "\n",
    "**VI. Conclusion**\n",
    "\n",
    "* Summarize your overall impression of the applicant's qualifications and potential\n",
    "* Reiterate why you think they are a strong candidate for graduate school\n",
    "* Close by expressing your confidence in the applicant's ability to succeed at the next level of academic study.\n",
    "\n",
    "**VII. Additional Tips**\n",
    "\n",
    "* Use specific examples and anecdotes to illustrate the applicant's skills and abilities\n",
    "* Avoid generic or formulaic language, and try to convey your unique perspective on the applicant's strengths and potential\n",
    "* Proofread carefully to ensure that the letter is free of errors in grammar, punctuation, and spelling.\n",
    "\n",
    "**VIII. Formatting and Content Guidelines**\n",
    "\n",
    "* Typically, letters of recommendation should be 1-2 pages in length\n",
    "* Use a formal business or academic tone throughout the letter\n",
    "* Include your contact information (name, title, department, institution) at the top of the page\n",
    "\n",
    "Remember to tailor your letter to the specific graduate program and requirements, and to provide as much detail as possible about the applicant's qualifications and potential. Good luck!\n",
    "\"\"\"\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "csv_file = '/workspace/data/asset_execs_ours.csv'\n",
    "data = pd.read_csv(csv_file)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.duplicated().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.drop_duplicates(inplace=True, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv(csv_file, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "leftovers = list(set(asset_list) - set(data['Company Name'].values.tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "asset_list == data['Company Name'].values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# Assuming data is your DataFrame and asset_list is your list\n",
    "data['Company Name'] = pd.Categorical(data['Company Name'], categories=asset_list, ordered=True)\n",
    "data = data.sort_values('Company Name')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['Company Name'].values.tolist()==asset_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "asset_list.index(\"Todd Asset Mgmt.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "exec_strs = [\"Chief Executive Officer (CEO)\", \"Chief Operations Officer (COO)\", \"Chief Investment Officer (CIO)\", \"President\", \"10-Year Performance\", \"Key Funds\", \"Assets Under Management (AUM)\", \"Primary Address\"]\n",
    "filename = '/workspace/data/asset_execs_ours.csv'\n",
    "header = [\"Company Name\"]+exec_strs\n",
    "\n",
    "start_date = \"2024-01-01\"\n",
    "# end_date = \"2024-10-08\"\n",
    "query_qualifier = f\"after:{start_date}\"\n",
    "# query_qualifier = f\"after:{start_date} before:{end_date}\"\n",
    "results = []\n",
    "context_results = []\n",
    "intermittent_save = []\n",
    "start_time = time.time()\n",
    "for i,a in enumerate(leftovers):\n",
    "    item_start_time = time.time()\n",
    "    contexts = []\n",
    "    company_execs=[a]\n",
    "    a_qual = f\"the asset managemet company {a}\"\n",
    "    for exec_type, exec_str in zip(execs, exec_strs[:4]):\n",
    "\n",
    "        # Construct the custom search query with date range\n",
    "        query = f\"\"\"From {a_qual}'s executive team, who is the {exec_str} at {a_qual}?\"\"\"\n",
    "        custom_query = f\"{query} {query_qualifier}\"\n",
    "        context = get_url_info(custom_query, num_results=3)\n",
    "        contexts.append(a + \" \" + exec_str + \"\\n\" + context)\n",
    "        # response = llm.complete(f\"\"\"Given the following text, extract the name of the {exec_str} at {a_qual}.\n",
    "        response = llm.complete(f\"\"\"Given the following text, what is the name of the {exec_str} at {a_qual}?\n",
    "                                    Only output the name.\\n\\nText:\\n{context}\"\"\")\n",
    "                                    \n",
    "        company_execs.append(response.text)\n",
    "    \n",
    "    num_results=5\n",
    "    num_words=2000\n",
    "    return_prompt=f\"\"\"What is the 10-year return for {a_qual}?\"\"\"\n",
    "    return_query = f\"{return_prompt} {query_qualifier}\"\n",
    "    context = get_url_info(return_query, num_results=num_results, num_words=num_words)\n",
    "    contexts.append(a + \" \" + return_query + \"\\n\" + context)\n",
    "    # answer_prompt = f\"\"\"Given the following text, extract the 10-year return for {a_qual}.\n",
    "    answer_prompt = f\"\"\"Given the following text, what is the 10-year return for {a_qual}?\n",
    "                        Only output the return.\\n\\nText:\\n{context}\"\"\"\n",
    "    company_execs.append(llm.complete(answer_prompt).text)\n",
    "\n",
    "    funds_prompt = f\"\"\"What are the key funds for {a_qual}?\"\"\"\n",
    "    funds_query = f\"{funds_prompt} {query_qualifier}\"\n",
    "    context = get_url_info(funds_query, num_results=3, num_words=num_words, description_only=False)\n",
    "    contexts.append(a + \" \" + funds_query + \"\\n\" + context)\n",
    "    # answer_prompt = f\"\"\"Given the following text, extract the names of the funds for {a_qual}.\n",
    "    answer_prompt = f\"\"\"Given the following text, what are the names of the funds for {a_qual}?\n",
    "                        Only output the names.\\n\\nText:\\n{context}\"\"\"\n",
    "    company_execs.append(llm.complete(answer_prompt).text)\n",
    "\n",
    "    aum_prompt = f\"\"\"What is the total Assets Under Management (AUM) in USD for {a_qual}?\"\"\"\n",
    "    aum_query = f\"{aum_prompt} {query_qualifier}\"\n",
    "    context = get_url_info(aum_query, num_results=num_results, num_words=1000)\n",
    "    contexts.append(a + \" \" + aum_query + \"\\n\" + context)\n",
    "    # answer_prompt = f\"\"\"Given the following text, extract the total Assets Under Management (AUM) for {a_qual}.\n",
    "    answer_prompt = f\"\"\"Given the following text, what is the current total Assets Under Management (AUM) at {a_qual}?\n",
    "                        Only output the total AUM.\\n\\nText:\\n{context}\"\"\"\n",
    "    company_execs.append(llm.complete(answer_prompt).text)\n",
    "\n",
    "    address_prompt = f\"\"\"What is the primary address for {a_qual}?\"\"\"\n",
    "    address_query = f\"{address_prompt} {query_qualifier}\"\n",
    "    context = get_url_info(address_query, num_results=num_results, num_words=500)\n",
    "    contexts.append(a + \" \" + address_query + \"\\n\" + context)\n",
    "    # answer_prompt = f\"\"\"Given the following text, extract the primary address for {a_qual}.\n",
    "    answer_prompt = f\"\"\"Given the following text, what is the primary address for {a_qual}?\n",
    "                        Only output the address.\\n\\nText:\\n{context}\"\"\"\n",
    "    company_execs.append(llm.complete(answer_prompt).text)\n",
    "    intermittent_save.append(company_execs)\n",
    "    \n",
    "    # Print the estimated remaining time at regular intervals\n",
    "    if i % 14 == 0 or i == len(asset_list) - 1:\n",
    "        # Periodically save results\n",
    "        header = [\"Company Name\"]+exec_strs if os.path.exists(filename) else None\n",
    "        make_csv(intermittent_save, filename, header=None, verbose=True)\n",
    "        intermittent_save = []\n",
    "        # Record the time after processing the current item\n",
    "        item_end_time = time.time()\n",
    "        # Calculate the average processing time per item\n",
    "        average_processing_time = (item_end_time - start_time) / (i + 1)\n",
    "        # Estimate the remaining time\n",
    "        remaining_items = len(asset_list) - (i + 1)\n",
    "        estimated_remaining_time = (remaining_items * average_processing_time)/60\n",
    "\n",
    "        print(f\"Processed {i+1}/{len(asset_list)} companies...\")\n",
    "        if estimated_remaining_time > 60:\n",
    "            print(f\"Estimated remaining time: {estimated_remaining_time/60:.2f} hours\")\n",
    "        else:\n",
    "            print(f\"Estimated remaining time: {estimated_remaining_time:.2f} minutes\")\n",
    "        \n",
    "\n",
    "    context_results.append(contexts)\n",
    "    results.append(company_execs)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "query = f\"\"\"What is the name of the Chief Operating Officer of BNY Mellon?\"\"\"\n",
    "urls = []\n",
    "for url in search(query, pause=random.randint(2, 5), stop=2):\n",
    "    urls.append(url)\n",
    "print(len(urls))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_csv(results, filename, header=None, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}