diff --git a/Dockerfile b/Dockerfile index 75e932930..a9cef513d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,4 +14,4 @@ COPY entrypoint.sh /entrypoint.sh COPY wait-for-it.sh /wait-for-it.sh RUN chmod +x /entrypoint.sh /wait-for-it.sh -CMD ["/wait-for-it.sh", "super__postgres:5432","-t","60","--","/entrypoint.sh"] \ No newline at end of file +CMD ["/wait-for-it.sh", "super__postgres:5432","-t","60","--","/entrypoint.sh"] diff --git a/DockerfileCelery b/DockerfileCelery index 2938220bd..57cd9c947 100644 --- a/DockerfileCelery +++ b/DockerfileCelery @@ -12,4 +12,4 @@ WORKDIR /app COPY . . COPY config.yaml . -CMD ["celery", "-A", "superagi.worker", "worker", "--loglevel=info"] \ No newline at end of file +CMD ["celery", "-A", "superagi.worker", "worker", "--loglevel=info"] diff --git a/cli2.py b/cli2.py index 32f92420c..36f0b911f 100644 --- a/cli2.py +++ b/cli2.py @@ -24,10 +24,10 @@ def run_npm_commands(shell=False): def run_server(shell=False,a_name=None,a_description=None,goals=None): + tgwui_process = Process(target=subprocess.run, args=(["python", "test.py","--name",a_name,"--description",a_description,"--goals"]+goals,), kwargs={"shell": shell}) api_process = Process(target=subprocess.run, args=(["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"],), kwargs={"shell": shell}) celery_process = Process(target=subprocess.run, args=(["celery", "-A", "celery_app", "worker", "--loglevel=info"],), kwargs={"shell": shell}) ui_process = Process(target=subprocess.run, args=(["python", "test.py","--name",a_name,"--description",a_description,"--goals"]+goals,), kwargs={"shell": shell}) - api_process.start() celery_process.start() ui_process.start() diff --git a/config_template.yaml b/config_template.yaml index 7464e62a0..ef31adc8c 100644 --- a/config_template.yaml +++ b/config_template.yaml @@ -4,9 +4,17 @@ PINECONE_ENVIRONMENT: YOUR_PINECONE_ENVIRONMENT OPENAI_API_KEY: YOUR_OPEN_API_KEY -#DATABASE INFO +# For locally hosted LLMs comment out the next line and uncomment the one after +# to configure a local llm point your browser to 127.0.0.1:7860 and click on the model tab in text generation web ui. +OPENAI_API_BASE: https://api.openai.com/v1 +#OPENAI_API_BASE: "http://super__tgwui:5001/v1" + +# "gpt-3.5-turbo-0301": 4032, "gpt-4-0314": 8092, "gpt-3.5-turbo": 4032, "gpt-4": 8092, "llama":2048, "mpt-7b-storywriter":45000 +MODEL_NAME: "gpt-3.5-turbo-0301" MAX_TOOL_TOKEN_LIMIT: 800 +MAX_MODEL_TOKEN_LIMIT: 4032 # set to 2048 for llama +#DATABASE INFO # redis details DB_NAME: super_agi_main POSTGRES_URL: super__postgres @@ -48,7 +56,7 @@ ENCRYPTION_KEY: secret GOOGLE_API_KEY: YOUR_GOOGLE_API_KEY SEARCH_ENGINE_ID: YOUR_SEARCH_ENIGNE_ID -# IF YOU DONT HAVE GOOGLE SERACH KEY, USE THIS +# IF YOU DONT HAVE GOOGLE SEARCH KEY, USE THIS SERP_API_KEY: YOUR_SERP_API_KEY #ENTER YOUR EMAIL CREDENTIALS TO ACCESS EMAIL TOOL diff --git a/docker-compose.yaml b/docker-compose.yaml index 75a10ee87..124afbc9b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -24,7 +24,7 @@ services: - super__postgres networks: - super_network - + gui: build: ./gui ports: @@ -39,12 +39,12 @@ services: - /app/.next super__redis: - image: "redis:latest" + image: "docker.io/library/redis:latest" networks: - super_network super__postgres: - image: "postgres:latest" + image: "docker.io/library/postgres:latest" environment: - POSTGRES_USER=superagi - POSTGRES_PASSWORD=password @@ -61,4 +61,4 @@ networks: driver: bridge volumes: - superagi_postgres_data: \ No newline at end of file + superagi_postgres_data: diff --git a/local-llm b/local-llm new file mode 100644 index 000000000..c30f23307 --- /dev/null +++ b/local-llm @@ -0,0 +1,93 @@ +version: '3.8' + +services: + backend: + volumes: + - "./:/app" + build: . + ports: + - "8001:8001" + depends_on: + - super__tgwui + - super__redis + - super__postgres + networks: + - super_network + + celery: + volumes: + - "./:/app" + build: + context: . + dockerfile: DockerfileCelery + depends_on: + - super__tgwui + - super__redis + - super__postgres + networks: + - super_network + + gui: + build: ./gui + ports: + - "3000:3000" + environment: + - NEXT_PUBLIC_API_BASE_URL=http://localhost:8001 + networks: + - super_network + volumes: + - ./gui:/app + - /app/node_modules + - /app/.next + + super__tgwui: + build: + context: . + dockerfile: ./tgwui/DockerfileTGWUI + container_name: super__tgwui + environment: + - EXTRA_LAUNCH_ARGS="--listen --verbose --extensions openai --threads 4 --n_ctx 1600" + ports: + - 7860:7860 # Default web port + - 5000:5000 # Default API port + - 5005:5005 # Default streaming port + - 5001:5001 # Default OpenAI API extension port + volumes: + - ./tgwui/config/loras:/app/loras + - ./tgwui/config/models:/app/models + - ./tgwui/config/presets:/app/presets + - ./tgwui/config/prompts:/app/prompts + - ./tgwui/config/softprompts:/app/softprompts + - ./tgwui/config/training:/app/training + logging: + driver: json-file + options: + max-file: "3" # number of files or file count + max-size: '10m' + networks: + - super_network + + super__redis: + image: "docker.io/library/redis:latest" + networks: + - super_network + + super__postgres: + image: "docker.io/library/postgres:latest" + environment: + - POSTGRES_USER=superagi + - POSTGRES_PASSWORD=password + - POSTGRES_DB=super_agi_main + volumes: + - superagi_postgres_data:/var/lib/postgresql/data/ + networks: + - super_network + ports: + - "5432:5432" + +networks: + super_network: + driver: bridge + +volumes: + superagi_postgres_data: diff --git a/local-llm-gpu b/local-llm-gpu new file mode 100644 index 000000000..dceffd3d8 --- /dev/null +++ b/local-llm-gpu @@ -0,0 +1,107 @@ +version: '3.8' + +services: + backend: + volumes: + - "./:/app" + build: . + ports: + - "8001:8001" + depends_on: + - super__tgwui + - super__redis + - super__postgres + networks: + - super_network + + celery: + volumes: + - "./:/app" + build: + context: . + dockerfile: DockerfileCelery + depends_on: + - super__tgwui + - super__redis + - super__postgres + networks: + - super_network + + gui: + build: ./gui + ports: + - "3000:3000" + environment: + - NEXT_PUBLIC_API_BASE_URL=http://localhost:8001 + networks: + - super_network + volumes: + - ./gui:/app + - /app/node_modules + - /app/.next + + super__tgwui: + build: + context: . + target: llama-cublas + dockerfile: ./tgwui/DockerfileTGWUI +# args: +# - LCL_SRC_DIR=text-generation-webui # Developers - see Dockerfile app_base + container_name: super__tgwui + environment: + - EXTRA_LAUNCH_ARGS="--listen --no-mmap --verbose --extensions openai --auto-devices --n_ctx 1600 --gpu-memory 20 20 --n-gpu-layers 128 --threads 8 --model vicuna-13b-cot.ggmlv3.q8_0.bin" + ports: + - 7860:7860 # Default web port + - 5000:5000 # Default API port + - 5005:5005 # Default streaming port + - 5001:5001 # Default OpenAI API extension port + volumes: + - ./tgwui/config/loras:/app/loras + - ./tgwui/config/models:/app/models + - ./tgwui/config/presets:/app/presets + - ./tgwui/config/prompts:/app/prompts + - ./tgwui/config/softprompts:/app/softprompts + - ./tgwui/config/training:/app/training + - ./tgwui/config/embeddings:/app/embeddings + logging: + driver: json-file + options: + max-file: "3" # number of files or file count + max-size: '10m' + networks: + - super_network +### Uncomment the following lines to run the container using the host machine's GPU resources + deploy: + resources: + reservations: + devices: + - driver: nvidia +# count: "all" + device_ids: ['0', '1'] # must comment the above line if this line is uncommented. + capabilities: [gpu] + + + super__redis: + image: "docker.io/library/redis:latest" + networks: + - super_network + + super__postgres: + image: "docker.io/library/postgres:latest" + environment: + - POSTGRES_USER=superagi + - POSTGRES_PASSWORD=password + - POSTGRES_DB=super_agi_main + volumes: + - superagi_postgres_data:/var/lib/postgresql/data/ + networks: + - super_network + ports: + - "5432:5432" + +networks: + super_network: + driver: bridge + +volumes: + superagi_postgres_data: diff --git a/run.sh b/run.sh index 899a1a32e..92f740c5a 100755 --- a/run.sh +++ b/run.sh @@ -6,6 +6,12 @@ if [ ! -f "config.yaml" ]; then exit 1 fi +if [ ! -f "tgwui/text-generation-webui" ]; then + echo "Downloading tgwui src" + git clone https://github.com/oobabooga/text-generation-webui + mv text-generation-webui tgwui +fi + # Function to check if virtual environment is activated is_venv_activated() { [[ -n "$VIRTUAL_ENV" ]] diff --git a/superagi/agent/super_agi.py b/superagi/agent/super_agi.py index 1d38f9afa..1080cb8f1 100644 --- a/superagi/agent/super_agi.py +++ b/superagi/agent/super_agi.py @@ -117,7 +117,7 @@ def execute(self, workflow_step: AgentWorkflowStep): agent_execution_id = self.agent_config["agent_execution_id"] task_queue = TaskQueue(str(agent_execution_id)) - token_limit = TokenCounter.token_limit(self.llm.get_model()) + token_limit = TokenCounter.token_limit() agent_feeds = self.fetch_agent_feeds(session, self.agent_config["agent_execution_id"], self.agent_config["agent_id"]) current_calls = 0 if len(agent_feeds) <= 0: @@ -283,7 +283,7 @@ def build_agent_prompt(self, prompt: str, task_queue: TaskQueue, max_token_limit last_task = response["task"] last_task_result = response["response"] current_task = task_queue.get_first_task() or "" - token_limit = TokenCounter.token_limit(self.llm.get_model()) - max_token_limit + token_limit = TokenCounter.token_limit() - max_token_limit prompt = AgentPromptBuilder.replace_task_based_variables(prompt, current_task, last_task, last_task_result, pending_tasks, completed_tasks, token_limit) return prompt \ No newline at end of file diff --git a/superagi/helper/json_cleaner.py b/superagi/helper/json_cleaner.py index f5b43119f..58ff5c6b2 100644 --- a/superagi/helper/json_cleaner.py +++ b/superagi/helper/json_cleaner.py @@ -51,7 +51,7 @@ def extract_json_section(cls, input_str: str = ""): @classmethod def remove_escape_sequences(cls, string): - return string.encode('utf-8').decode('unicode_escape').encode('raw_unicode_escape').decode('utf-8') + return string.encode('utf-8').decode('unicode_escape').encode('raw_unicode_escape') @classmethod def add_quotes_to_property_names(cls, json_string: str) -> str: diff --git a/superagi/jobs/agent_executor.py b/superagi/jobs/agent_executor.py index cf4caa691..a91697bbd 100644 --- a/superagi/jobs/agent_executor.py +++ b/superagi/jobs/agent_executor.py @@ -132,7 +132,7 @@ def execute_next_action(self, agent_execution_id): memory = VectorFactory.get_vector_storage("PineCone", "super-agent-index1", OpenAiEmbedding(model_api_key)) except: - print("Unable to setup the pincone connection...") + print("Unable to setup the pinecone connection...") memory = None user_tools = session.query(Tool).filter(Tool.id.in_(parsed_config["tools"])).all() diff --git a/superagi/llms/openai.py b/superagi/llms/openai.py index a79557860..86e58dbdb 100644 --- a/superagi/llms/openai.py +++ b/superagi/llms/openai.py @@ -8,7 +8,7 @@ class OpenAi(BaseLlm): - def __init__(self, api_key, image_model=None, model="gpt-4", temperature=0.6, max_tokens=4032, top_p=1, + def __init__(self, api_key, image_model=None, model="gpt-4", temperature=0.6, max_tokens=get_config("MAX_MODEL_TOKEN_LIMIT"), top_p=1, frequency_penalty=0, presence_penalty=0, number_of_results=1): self.model = model @@ -21,6 +21,7 @@ def __init__(self, api_key, image_model=None, model="gpt-4", temperature=0.6, ma self.api_key = api_key self.image_model = image_model openai.api_key = api_key + openai.api_base = get_config("OPENAI_API_BASE", "https://api.openai.com/v1") def get_model(self): return self.model @@ -28,7 +29,7 @@ def get_model(self): def get_image_model(self): return self.image_model - def chat_completion(self, messages, max_tokens=4032): + def chat_completion(self, messages, max_tokens=get_config("MAX_MODEL_TOKEN_LIMIT")): try: # openai.api_key = get_config("OPENAI_API_KEY") response = openai.ChatCompletion.create( diff --git a/superagi/tools/email/send_email_attachment.py b/superagi/tools/email/send_email_attachment.py index 311eef445..feb3e1176 100644 --- a/superagi/tools/email/send_email_attachment.py +++ b/superagi/tools/email/send_email_attachment.py @@ -17,11 +17,11 @@ class SendEmailAttachmentInput(BaseModel): to: str = Field(..., description="Email Address of the Receiver, default email address is 'example@example.com'") subject: str = Field(..., description="Subject of the Email to be sent") body: str = Field(..., description="Email Body to be sent") - filename: str = Field(..., description="Name of the file to be sent as an Attachement with Email") + filename: str = Field(..., description="Name of the file to be sent as an Attachment with Email") class SendEmailAttachmentTool(BaseTool): - name: str = "Send Email with Attachement" + name: str = "Send Email with Attachment" args_schema: Type[BaseModel] = SendEmailAttachmentInput description: str = "Send an Email with a file attached to it" @@ -32,9 +32,9 @@ def _execute(self, to: str, subject: str, body: str, filename: str) -> str: base_path = base_path + filename attachmentpath = base_path attachment = os.path.basename(attachmentpath) - return self.send_email_with_attachement(to, subject, body, attachmentpath, attachment) + return self.send_email_with_attachment(to, subject, body, attachmentpath, attachment) - def send_email_with_attachement(self, to, subject, body, attachment_path, attachment) -> str: + def send_email_with_attachment(self, to, subject, body, attachment_path, attachment) -> str: email_sender = get_config('EMAIL_ADDRESS') email_password = get_config('EMAIL_PASSWORD') if email_sender == "" or email_sender.isspace(): diff --git a/superagi/tools/searx/README.MD b/superagi/tools/searx/README.MD new file mode 100644 index 000000000..8a2fa67a6 --- /dev/null +++ b/superagi/tools/searx/README.MD @@ -0,0 +1,16 @@ +

+ +

+ +# SuperAGI Searx Search Tool + +The SuperAGI Searx Search Tool helps users perform a Searx search and extract snippets and webpages. We parse the HTML response pages because most Searx instances do not support the JSON response format without an API key. + +## ⚙️ Installation + +### 🛠 **Setting Up of SuperAGI** +Set up the SuperAGI by following the instructions given (https://github.com/TransformerOptimus/SuperAGI/blob/main/README.MD) + +## Running SuperAGI Searx Search Serp Tool + +You can simply ask your agent about latest information regarding anything in the world and your agent will be able to browse the internet to get that information for you. diff --git a/superagi/tools/searx/__init__.py b/superagi/tools/searx/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/superagi/tools/searx/search_scraper.py b/superagi/tools/searx/search_scraper.py new file mode 100644 index 000000000..ceaf88283 --- /dev/null +++ b/superagi/tools/searx/search_scraper.py @@ -0,0 +1,73 @@ +import random +from typing import List +import httpx +from bs4 import BeautifulSoup +from pydantic import BaseModel + + +searx_hosts = ["https://search.ononoki.org", "https://searx.be", "https://search.us.projectsegfau.lt"] + +class SearchResult(BaseModel): + id: int + title: str + link: str + description: str + sources: List[str] + + def __str__(self): + return f"""{self.id}. {self.title} - {self.link} +{self.description}""" + +def search(query): + '''Gets the raw HTML of a searx search result page''' + # TODO: use a better strategy for choosing hosts. Could use this list: https://searx.space/data/instances.json + searx_url = random.choice(searx_hosts) + res = httpx.get( + searx_url + "/search", params={"q": query}, headers={"User-Agent": "Mozilla/5.0 (X11; Linux i686; rv:109.0) Gecko/20100101 Firefox/114.0"} + ) + if res.status_code != 200: + print(res.status_code, searx_url) + raise Exception(f"Searx returned {res.status_code} status code") + + return res.text + +def clean_whitespace(s: str): + return " ".join(s.split()) + + +def scrape_results(html): + '''Converts raw HTML into a list of SearchResult objects''' + soup = BeautifulSoup(html, "html.parser") + result_divs = soup.find_all(attrs={"class": "result"}) + + result_list = [] + n = 1 + for result_div in result_divs: + # Needed to work on multiple versions of Searx + header = result_div.find(["h4", "h3"]) + link = header.find("a")["href"] + title = header.text.strip() + + description = clean_whitespace(result_div.find("p").text) + + # Needed to work on multiple versions of Searx + sources_container = result_div.find( + attrs={"class": "pull-right"} + ) or result_div.find(attrs={"class": "engines"}) + source_spans = sources_container.find_all("span") + sources = [] + for s in source_spans: + sources.append(s.text.strip()) + + result = SearchResult( + id=n, title=title, link=link, description=description, sources=sources + ) + result_list.append(result) + n += 1 + + return result_list + + +def search_results(query): + '''Returns a text summary of the search results via the SearchResult.__str__ method''' + return "\n\n".join(list(map(lambda x: str(x), scrape_results(search(query))))) diff --git a/superagi/tools/searx/searx.py b/superagi/tools/searx/searx.py new file mode 100644 index 000000000..a1f3a3696 --- /dev/null +++ b/superagi/tools/searx/searx.py @@ -0,0 +1,43 @@ +from typing import Type, Optional +from pydantic import BaseModel, Field +from superagi.llms.base_llm import BaseLlm +from superagi.tools.base_tool import BaseTool +from superagi.tools.searx.search_scraper import search_results + + +class SearxSearchSchema(BaseModel): + query: str = Field( + ..., + description="The search query for the Searx search engine.", + ) + +class SearxSearchTool(BaseTool): + llm: Optional[BaseLlm] = None + name = "SearxSearch" + description = ( + "A tool for performing a Searx search and extracting snippets and webpages." + "Input should be a search query." + ) + args_schema: Type[SearxSearchSchema] = SearxSearchSchema + + class Config: + arbitrary_types_allowed = True + + def _execute(self, query: str) -> tuple: + snippets = search_results(query) + summary = self.summarise_result(query, snippets) + + return summary + + def summarise_result(self, query, snippets): + summarize_prompt = """Summarize the following text `{snippets}` + Write a concise or as descriptive as necessary and attempt to + answer the query: `{query}` as best as possible. Use markdown formatting for + longer responses.""" + + summarize_prompt = summarize_prompt.replace("{snippets}", str(snippets)) + summarize_prompt = summarize_prompt.replace("{query}", query) + + messages = [{"role": "system", "content": summarize_prompt}] + result = self.llm.chat_completion(messages, max_tokens=self.max_token_limit) + return result["content"] diff --git a/tgwui/DockerfileTGWUI b/tgwui/DockerfileTGWUI new file mode 100644 index 000000000..01cb26779 --- /dev/null +++ b/tgwui/DockerfileTGWUI @@ -0,0 +1,121 @@ +FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 AS env_base +# Pre-reqs +RUN apt-get update && apt-get install --no-install-recommends -y \ + git vim build-essential python3-dev python3-venv python3-pip +# Instantiate venv and pre-activate +RUN pip3 install virtualenv +RUN virtualenv /venv +# Credit, Itamar Turner-Trauring: https://pythonspeed.com/articles/activate-virtualenv-dockerfile/ +ENV VIRTUAL_ENV=/venv +RUN python3 -m venv $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +RUN pip3 install --upgrade pip setuptools && \ + pip3 install torch torchvision torchaudio + +FROM env_base AS app_base +### DEVELOPERS/ADVANCED USERS ### +# Clone oobabooga/text-generation-webui +RUN git clone https://github.com/oobabooga/text-generation-webui /src +# To use local source: comment out the git clone command then set the build arg `LCL_SRC_DIR` +#ARG LCL_SRC_DIR="text-generation-webui" +#COPY ${LCL_SRC_DIR} /src +# This is required to get multi-gpu support until the main branch updates the requirements.txt file to include llama-cpp-python 0.1.59 or greater. + +################################# +# Copy source to app +RUN cp -ar /src /app +# Install oobabooga/text-generation-webui +RUN --mount=type=cache,target=/root/.cache/pip pip3 install -r /app/requirements.txt +# Install extensions +COPY tgwui/scripts/build_extensions.sh /app/scripts/build_extensions.sh +RUN --mount=type=cache,target=/root/.cache/pip \ + chmod +x /app/scripts/build_extensions.sh && . /app/scripts/build_extensions.sh + +## Clone default GPTQ +RUN git clone https://github.com/oobabooga/GPTQ-for-LLaMa.git -b cuda /app/repositories/GPTQ-for-LLaMa +## Build and install default GPTQ ('quant_cuda') +ARG TORCH_CUDA_ARCH_LIST="6.1;7.0;7.5;8.0;8.6+PTX" +RUN cd /app/repositories/GPTQ-for-LLaMa/ && python3 setup_cuda.py install + +FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 AS base +# Runtime pre-reqs +RUN apt-get update && apt-get install --no-install-recommends -y \ + python3-venv python3-dev git +# Copy app and src +COPY --from=app_base /app /app +COPY --from=app_base /src /src +# Copy and activate venv +COPY --from=app_base /venv /venv +ENV VIRTUAL_ENV=/venv +RUN python3 -m venv $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +# Finalise app setup +WORKDIR /app +EXPOSE 7860 +EXPOSE 5000 +EXPOSE 5005 +EXPOSE 5001 +# Required for Python print statements to appear in logs +ENV PYTHONUNBUFFERED=1 +# Force variant layers to sync cache by setting --build-arg BUILD_DATE +ARG BUILD_DATE +ENV BUILD_DATE=$BUILD_DATE +RUN echo "$BUILD_DATE" > /build_date.txt + +# Set embeddings model for llama +#ENV OPENEDAI_EMBEDDING_MODEL=/app/embeddings/SGPT-125M-weightedmean-nli-bitfit +#COPY tgwui/config/embeddings/SGPT-125M-weightedmean-nli-bitfit /app/embeddings +#RUN echo -e "Embeddings model $OPENEDAI_EMBEDDING_MODEL" +#RUN python extensions/openai/cache_embedding_model.py + +# Run +COPY tgwui/scripts/docker-entrypoint.sh /scripts/docker-entrypoint.sh +RUN chmod +x /scripts/docker-entrypoint.sh +ENTRYPOINT ["/scripts/docker-entrypoint.sh"] + +# VARIANT BUILDS +FROM base AS cuda +RUN echo "CUDA" >> /variant.txt +RUN apt-get install --no-install-recommends -y git python3-dev python3-pip +RUN rm -rf /app/repositories/GPTQ-for-LLaMa && \ + git clone https://github.com/qwopqwop200/GPTQ-for-LLaMa -b cuda /app/repositories/GPTQ-for-LLaMa +RUN pip3 uninstall -y quant-cuda && \ + sed -i 's/^safetensors==0\.3\.0$/safetensors/g' /app/repositories/GPTQ-for-LLaMa/requirements.txt && \ + pip3 install -r /app/repositories/GPTQ-for-LLaMa/requirements.txt +ENV EXTRA_LAUNCH_ARGS="" +CMD ["python3", "/app/server.py"] + +FROM base AS triton +RUN echo "TRITON" >> /variant.txt +RUN apt-get install --no-install-recommends -y git python3-dev build-essential python3-pip +RUN rm -rf /app/repositories/GPTQ-for-LLaMa && \ + git clone https://github.com/qwopqwop200/GPTQ-for-LLaMa -b triton /app/repositories/GPTQ-for-LLaMa +RUN pip3 uninstall -y quant-cuda && \ + sed -i 's/^safetensors==0\.3\.0$/safetensors/g' /app/repositories/GPTQ-for-LLaMa/requirements.txt && \ + pip3 install -r /app/repositories/GPTQ-for-LLaMa/requirements.txt +ENV EXTRA_LAUNCH_ARGS="" +CMD ["python3", "/app/server.py"] + +FROM base AS llama-cublas +RUN echo "LLAMA-CUBLAS" >> /variant.txt +RUN apt-get install --no-install-recommends -y git python3-dev build-essential python3-pip +ENV LLAMA_CUBLAS=1 +RUN pip uninstall -y llama-cpp-python && \ + CMAKE_ARGS="-DLLAMA_CUBLAS=on" FORCE_CMAKE=1 pip install llama-cpp-python +ENV EXTRA_LAUNCH_ARGS="" +CMD ["python3", "/app/server.py"] + +FROM base AS monkey-patch +RUN echo "4-BIT MONKEY-PATCH" >> /variant.txt +RUN apt-get install --no-install-recommends -y git python3-dev build-essential python3-pip +RUN git clone https://github.com/johnsmith0031/alpaca_lora_4bit /app/repositories/alpaca_lora_4bit && \ + cd /app/repositories/alpaca_lora_4bit && git checkout 2f704b93c961bf202937b10aac9322b092afdce0 +ARG TORCH_CUDA_ARCH_LIST="8.6" +RUN pip install git+https://github.com/sterlind/GPTQ-for-LLaMa.git@lora_4bit +ENV EXTRA_LAUNCH_ARGS="" +CMD ["python3", "/app/server.py", "--monkey-patch"] + +FROM base AS default +RUN echo "DEFAULT" >> /variant.txt +ENV EXTRA_LAUNCH_ARGS="" +CMD ["python3", "/app/server.py"] diff --git a/tgwui/config/loras/place-your-loras-here.txt b/tgwui/config/loras/place-your-loras-here.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tgwui/config/models/place-your-models-here.txt b/tgwui/config/models/place-your-models-here.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tgwui/config/presets/Debug-deterministic.yaml b/tgwui/config/presets/Debug-deterministic.yaml new file mode 100644 index 000000000..e5ccac65d --- /dev/null +++ b/tgwui/config/presets/Debug-deterministic.yaml @@ -0,0 +1 @@ +do_sample: False diff --git a/tgwui/config/presets/Kobold-Godlike.yaml b/tgwui/config/presets/Kobold-Godlike.yaml new file mode 100644 index 000000000..3d2bdf2f7 --- /dev/null +++ b/tgwui/config/presets/Kobold-Godlike.yaml @@ -0,0 +1,6 @@ +do_sample: true +top_p: 0.5 +top_k: 0 +temperature: 0.7 +repetition_penalty: 1.1 +typical_p: 0.19 diff --git a/tgwui/config/presets/Kobold-Liminal Drift.yaml b/tgwui/config/presets/Kobold-Liminal Drift.yaml new file mode 100644 index 000000000..3dbcd5cce --- /dev/null +++ b/tgwui/config/presets/Kobold-Liminal Drift.yaml @@ -0,0 +1,6 @@ +do_sample: true +top_p: 1.0 +top_k: 0 +temperature: 0.66 +repetition_penalty: 1.1 +typical_p: 0.6 diff --git a/tgwui/config/presets/LLaMA-Precise.yaml b/tgwui/config/presets/LLaMA-Precise.yaml new file mode 100644 index 000000000..2d0c2bdc8 --- /dev/null +++ b/tgwui/config/presets/LLaMA-Precise.yaml @@ -0,0 +1,6 @@ +do_sample: true +top_p: 0.1 +top_k: 40 +temperature: 0.7 +repetition_penalty: 1.18 +typical_p: 1.0 diff --git a/tgwui/config/presets/Naive.yaml b/tgwui/config/presets/Naive.yaml new file mode 100644 index 000000000..af9842019 --- /dev/null +++ b/tgwui/config/presets/Naive.yaml @@ -0,0 +1,4 @@ +do_sample: true +temperature: 0.7 +top_p: 0.85 +top_k: 50 diff --git a/tgwui/config/presets/NovelAI-Best Guess.yaml b/tgwui/config/presets/NovelAI-Best Guess.yaml new file mode 100644 index 000000000..2c21d136d --- /dev/null +++ b/tgwui/config/presets/NovelAI-Best Guess.yaml @@ -0,0 +1,6 @@ +do_sample: true +top_p: 0.9 +top_k: 100 +temperature: 0.8 +repetition_penalty: 1.15 +typical_p: 1.0 diff --git a/tgwui/config/presets/NovelAI-Decadence.yaml b/tgwui/config/presets/NovelAI-Decadence.yaml new file mode 100644 index 000000000..28e1a21b6 --- /dev/null +++ b/tgwui/config/presets/NovelAI-Decadence.yaml @@ -0,0 +1,6 @@ +do_sample: true +top_p: 1.0 +top_k: 100 +temperature: 2 +repetition_penalty: 1 +typical_p: 0.97 diff --git a/tgwui/config/presets/NovelAI-Genesis.yaml b/tgwui/config/presets/NovelAI-Genesis.yaml new file mode 100644 index 000000000..41ee4c6bd --- /dev/null +++ b/tgwui/config/presets/NovelAI-Genesis.yaml @@ -0,0 +1,6 @@ +do_sample: true +top_p: 0.98 +top_k: 0 +temperature: 0.63 +repetition_penalty: 1.05 +typical_p: 1.0 diff --git a/tgwui/config/presets/NovelAI-Lycaenidae.yaml b/tgwui/config/presets/NovelAI-Lycaenidae.yaml new file mode 100644 index 000000000..be296d795 --- /dev/null +++ b/tgwui/config/presets/NovelAI-Lycaenidae.yaml @@ -0,0 +1,6 @@ +do_sample: true +top_p: 0.85 +top_k: 12 +temperature: 2 +repetition_penalty: 1.15 +typical_p: 1.0 diff --git a/tgwui/config/presets/NovelAI-Ouroboros.yaml b/tgwui/config/presets/NovelAI-Ouroboros.yaml new file mode 100644 index 000000000..53846a1d3 --- /dev/null +++ b/tgwui/config/presets/NovelAI-Ouroboros.yaml @@ -0,0 +1,6 @@ +do_sample: true +top_p: 1.0 +top_k: 100 +temperature: 1.07 +repetition_penalty: 1.05 +typical_p: 1.0 diff --git a/tgwui/config/presets/NovelAI-Pleasing Results.yaml b/tgwui/config/presets/NovelAI-Pleasing Results.yaml new file mode 100644 index 000000000..ca7408df8 --- /dev/null +++ b/tgwui/config/presets/NovelAI-Pleasing Results.yaml @@ -0,0 +1,6 @@ +do_sample: true +top_p: 1.0 +top_k: 0 +temperature: 0.44 +repetition_penalty: 1.15 +typical_p: 1.0 diff --git a/tgwui/config/presets/NovelAI-Sphinx Moth.yaml b/tgwui/config/presets/NovelAI-Sphinx Moth.yaml new file mode 100644 index 000000000..8efd5a74b --- /dev/null +++ b/tgwui/config/presets/NovelAI-Sphinx Moth.yaml @@ -0,0 +1,6 @@ +do_sample: true +top_p: 0.18 +top_k: 30 +temperature: 2.0 +repetition_penalty: 1.15 +typical_p: 1.0 diff --git a/tgwui/config/presets/NovelAI-Storywriter.yaml b/tgwui/config/presets/NovelAI-Storywriter.yaml new file mode 100644 index 000000000..34d11ec81 --- /dev/null +++ b/tgwui/config/presets/NovelAI-Storywriter.yaml @@ -0,0 +1,6 @@ +do_sample: true +top_p: 0.73 +top_k: 0 +temperature: 0.72 +repetition_penalty: 1.1 +typical_p: 1.0 diff --git a/tgwui/config/presets/Special-Contrastive Search.yaml b/tgwui/config/presets/Special-Contrastive Search.yaml new file mode 100644 index 000000000..290342e23 --- /dev/null +++ b/tgwui/config/presets/Special-Contrastive Search.yaml @@ -0,0 +1,3 @@ +do_sample: False +penalty_alpha: 0.6 +top_k: 4 diff --git a/tgwui/config/presets/Special-Eta Sampling.yaml b/tgwui/config/presets/Special-Eta Sampling.yaml new file mode 100644 index 000000000..813522e63 --- /dev/null +++ b/tgwui/config/presets/Special-Eta Sampling.yaml @@ -0,0 +1,4 @@ +do_sample: true +eta_cutoff: 3 +temperature: 0.7 +repetition_penalty: 1.18 diff --git a/tgwui/config/prompts/Alpaca-with-Input.txt b/tgwui/config/prompts/Alpaca-with-Input.txt new file mode 100644 index 000000000..56df0e285 --- /dev/null +++ b/tgwui/config/prompts/Alpaca-with-Input.txt @@ -0,0 +1,10 @@ +Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request. + +### Instruction: +Instruction + +### Input: +Input + +### Response: + diff --git a/tgwui/config/prompts/GPT-4chan.txt b/tgwui/config/prompts/GPT-4chan.txt new file mode 100644 index 000000000..1bc8c7f46 --- /dev/null +++ b/tgwui/config/prompts/GPT-4chan.txt @@ -0,0 +1,6 @@ +----- +--- 865467536 +Hello, AI frens! +How are you doing on this fine day? +--- 865467537 + diff --git a/tgwui/config/prompts/QA.txt b/tgwui/config/prompts/QA.txt new file mode 100644 index 000000000..32b0e2350 --- /dev/null +++ b/tgwui/config/prompts/QA.txt @@ -0,0 +1,4 @@ +Common sense questions and answers + +Question: +Factual answer: diff --git a/tgwui/config/training/datasets/put-trainer-datasets-here.txt b/tgwui/config/training/datasets/put-trainer-datasets-here.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tgwui/config/training/formats/alpaca-chatbot-format.json b/tgwui/config/training/formats/alpaca-chatbot-format.json new file mode 100644 index 000000000..4b38103f4 --- /dev/null +++ b/tgwui/config/training/formats/alpaca-chatbot-format.json @@ -0,0 +1,4 @@ +{ + "instruction,output": "User: %instruction%\nAssistant: %output%", + "instruction,input,output": "User: %instruction%: %input%\nAssistant: %output%" +} diff --git a/tgwui/config/training/formats/alpaca-format.json b/tgwui/config/training/formats/alpaca-format.json new file mode 100644 index 000000000..dd6df9564 --- /dev/null +++ b/tgwui/config/training/formats/alpaca-format.json @@ -0,0 +1,4 @@ +{ + "instruction,output": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\n%instruction%\n\n### Response:\n%output%", + "instruction,input,output": "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n\n### Instruction:\n%instruction%\n\n### Input:\n%input%\n\n### Response:\n%output%" +} diff --git a/tgwui/scripts/build_extensions.sh b/tgwui/scripts/build_extensions.sh new file mode 100644 index 000000000..929b4aacd --- /dev/null +++ b/tgwui/scripts/build_extensions.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Specify the directory containing the top-level folders +directory="/app/extensions" + +# Iterate over the top-level folders +for folder in "$directory"/*; do + if [ -d "$folder" ]; then + # Change directory to the current folder + cd "$folder" + + # Check if requirements.txt file exists + if [ -f "requirements.txt" ]; then + echo "Installing requirements in $folder..." + pip3 install -r requirements.txt + echo "Requirements installed in $folder" + else + echo "Skipping $folder: requirements.txt not found" + fi + + # Change back to the original directory + cd "$directory" + fi +done + diff --git a/tgwui/scripts/docker-entrypoint.sh b/tgwui/scripts/docker-entrypoint.sh new file mode 100644 index 000000000..bb0049ffd --- /dev/null +++ b/tgwui/scripts/docker-entrypoint.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# Function to handle keyboard interrupt +function ctrl_c { + echo -e "\nKilling container!" + # Add your cleanup actions here + exit 0 +} +# Register the keyboard interrupt handler +trap ctrl_c SIGTERM SIGINT SIGQUIT SIGHUP + +# Generate default configs if empty +CONFIG_DIRECTORIES=("loras" "models" "presets" "prompts" "training/datasets" "training/formats") +for config_dir in "${CONFIG_DIRECTORIES[@]}"; do + if [ -z "$(ls /app/"$config_dir")" ]; then + echo "*** Initialising config for: '$config_dir' ***" + cp -ar /src/"$config_dir"/* /app/"$config_dir"/ + chown -R 1000:1000 /app/"$config_dir" # Not ideal... but convenient. + fi +done + +# Print variant +VARIANT=$(cat /variant.txt) +echo "=== Running text-generation-webui variant: '$VARIANT' ===" + +# Print version freshness +cur_dir=$(pwd) +src_dir="/src" +cd $src_dir +git fetch origin >/dev/null 2>&1 +if [ $? -ne 0 ]; then + # An error occurred + COMMITS_BEHIND="UNKNOWN" +else + # The command executed successfully + COMMITS_BEHIND=$(git rev-list HEAD..origin --count) +fi +echo "=== (This version is $COMMITS_BEHIND commits behind origin) ===" +cd $cur_dir + +# Print build date +BUILD_DATE=$(cat /build_date.txt) +echo "=== Image build date: $BUILD_DATE ===" + +# Assemble CMD and extra launch args +eval "extra_launch_args=($EXTRA_LAUNCH_ARGS)" +LAUNCHER=($@ $extra_launch_args) + +# Launch the server with ${CMD[@]} + ${EXTRA_LAUNCH_ARGS[@]} +"${LAUNCHER[@]}"