From c8647d49732b9238099ae08c5ddb65581d79f838 Mon Sep 17 00:00:00 2001 From: Raphael MANSUY Date: Sat, 7 Sep 2024 16:03:45 +0200 Subject: [PATCH] Fix/interractive_4 (#16) * first_phase_refactoring * add nice print help * update * fix(code2prompt): r(code2prompt): improve token price display * update * fix(.): r: remove unused imports and variables * fix: Improve README.md * docs: Improve documentation for code2prompt/main.py * feat: Add improvements.md file with suggested enhancements * feat: Implement logging in critical areas of the code * fix: Add log_error function to logging_utils * fix: Add log_output_created function to logging_utils.py * feat: Add log_clipboard_copy and log_token_count functions to logging_utils * feat: Add colored logging to the terminal * feat: Add log level configuration in command line * feat: Add logging level configuration to .code2promptrc and analyze-code.j2 * feat: Improve log message formatting * fix: Add stderr logging for clipboard copy success and failure * fix: Set default log level to WARNING * feat: Add color and emoji to clipboard copy success message * feat: Add header to token price estimation output * fix: Correct token price output formatting * fix: Correct the order of input and output prices in log_token_prices function * fix: Correct the order of "In" and "Out" tokens in the log_token_prices function * fix: Correct the formatting of the token price table * fix/aider(.gitignore): add .aider* to ignore file * fix(code2prompt): update main.py file * fix(gitignore): add .ruff_cache to gitignore * first version * better version * fix(code2prompt): enhance interactive command with improved cursor navigation and visible line handling * fix(code2prompt): add terminal resize handling and improve instructions * fix(code2prompt): move interactive_command.py to commands directory * fix(include_loader): import jinja2 to resolve undefined name error fix(logging_utils): remove unused logger variable chore(deps): add jinja2 dependency to pyproject.toml test(analyzer): remove unused imports test(create_template_directory): remove unused imports test(include_loader): remove unused imports test(is_filtered): improve readability and consistency test(template_include): remove unused imports * fix(code2prompt/utils): Improve logging and file processing criteria * fix(code2prompt/commands): improve `get_terminal_height`, `get_directory_tree`, and `format_tree` functions * fix(code2prompt/commands): improve interactive command page up and down functionality * fix(code2prompt): add interactive file selector for generate command * fix(code2prompt/commands): Refactored interactive file selector, added support for multiple paths * fix(code2prompt): Refactor InteractiveFileSelector to handle multiple paths * fix(code2prompt/commands): improve interactive file selector responsiveness to terminal resize events * fix(code2prompt): Introduce file_path_retriever module to handle file path filtering and processing * fix(code2prompt/commands): improve code organization and documentation in the GenerateCommand class * fix(code2prompt): Improve performance of `_get_directory_tree` method and remove unnecessary validation for `retrieve_file_paths` * fix(code2prompt/commands): handle invalid or missing paths in interactive selector * fix(code2prompt/commands): create and use private methods to handle key bindings and application creation * fix(code2prompt/commands): update interactive file selector to use Path objects * fix(code2prompt): improve interactive file selector behavior * fix(code2prompt/commands): improve directory tree generation and formatting * fix(code2prompt/commands): improve interactive file selector --- .code2promptrc | 8 +- .cursorrules | 31 ++ .gitignore | 4 +- README.md | 84 ++--- code2prompt/commands/__init__.py | 0 code2prompt/commands/analyze.py | 70 ++++ code2prompt/commands/base_command.py | 76 +++++ code2prompt/commands/generate.py | 66 ++++ code2prompt/commands/interactive_selector.py | 297 +++++++++++++++++ code2prompt/comment_stripper/__init__.py | 8 - .../comment_stripper/strip_comments.py | 30 +- code2prompt/config.py | 105 ++++++ code2prompt/core/file_path_retriever.py | 70 ++++ code2prompt/core/process_file.py | 31 +- code2prompt/core/process_files.py | 62 ++-- code2prompt/main.py | 295 +++++++---------- code2prompt/print_help.py | 67 ++++ code2prompt/templates/analyze-code.j2 | 6 +- code2prompt/utils/add_line_numbers.py | 9 + code2prompt/utils/display_price_table.py | 115 +++++++ code2prompt/utils/file_utils.py | 134 ++++++++ code2prompt/utils/include_loader.py | 3 +- code2prompt/utils/is_filtered.py | 16 +- code2prompt/utils/is_ignored.py | 2 - code2prompt/utils/logging_utils.py | 271 +++------------- code2prompt/utils/output_utils.py | 127 ++++++++ code2prompt/utils/price_calculator.py | 151 ++++++--- code2prompt/utils/should_process_file.py | 42 ++- code2prompt/version.py | 1 + docs/demo01/doc01.md | 0 docs/demo01/doc02.md | 0 poetry.lock | 298 +++++++++++++++++- pyproject.toml | 15 +- ruff.toml | 77 +++++ script/detect_dead_code.sh | 2 + tests/test_analyze.py | 3 +- tests/test_create_template_directory.py | 3 +- tests/test_include_loader.py | 3 +- tests/test_is_filtered.py | 72 +++-- tests/test_price.py | 211 ++++++------- tests/test_template_include.py | 2 - todo/improvements.md | 19 ++ 42 files changed, 2176 insertions(+), 710 deletions(-) create mode 100644 .cursorrules create mode 100644 code2prompt/commands/__init__.py create mode 100644 code2prompt/commands/analyze.py create mode 100644 code2prompt/commands/base_command.py create mode 100644 code2prompt/commands/generate.py create mode 100644 code2prompt/commands/interactive_selector.py create mode 100644 code2prompt/config.py create mode 100644 code2prompt/core/file_path_retriever.py create mode 100644 code2prompt/print_help.py create mode 100644 code2prompt/utils/display_price_table.py create mode 100644 code2prompt/utils/file_utils.py create mode 100644 code2prompt/utils/output_utils.py create mode 100644 code2prompt/version.py create mode 100644 docs/demo01/doc01.md create mode 100644 docs/demo01/doc02.md create mode 100644 ruff.toml create mode 100755 script/detect_dead_code.sh create mode 100644 todo/improvements.md diff --git a/.code2promptrc b/.code2promptrc index 5de6b92..604daf0 100644 --- a/.code2promptrc +++ b/.code2promptrc @@ -1,4 +1,8 @@ { "suppress_comments": false, - "line_number": false -} \ No newline at end of file + "line_number": false, + "log_level": "INFO", + "encoding": "cl100k_base", + "filter": "*.py,*.js", + "exclude": "tests/*,docs/*" +} diff --git a/.cursorrules b/.cursorrules new file mode 100644 index 0000000..c387bff --- /dev/null +++ b/.cursorrules @@ -0,0 +1,31 @@ + +# Role Overview + +You are an elite software developer with extensive expertise in Python, command-line tools, and file system operations. Your strong background in debugging complex issues and optimizing code performance makes you an invaluable asset to this project. + +## Key Attributes + +- **Pragmatic Approach**: You prioritize delivering high-quality, maintainable code that meets project requirements. +- **Modular Design**: You embrace composability and modularity, ensuring that your code is easy to extend and maintain. +- **Principled Coding**: You adhere to the KISS (Keep It Simple, Stupid) and DRY (Don't Repeat Yourself) principles, promoting simplicity and efficiency. +- **Documentation & Testing**: You recognize the importance of clear documentation and thorough testing to guarantee the reliability of your work. +- **Functional Preference**: You prefer using functions and modules over classes, focusing on functional programming paradigms. + +## Technological Stack + +This project utilizes the following technologies: + +- **Python Version**: 3.6+ +- **Dependencies**: + - `python = "^3.8,<4.0"` + - `rich = "^13.7.1"` # For rich text and beautiful formatting + - `click = "^8.1.7"` # For creating elegant command-line interfaces + - `jinja2 = "^3.1.4"` # For template rendering + - `prompt-toolkit = "^3.0.47"` # For building powerful interactive command-line applications + - `tiktoken = "^0.7.0"` # For tokenization tasks + - `pyperclip = "^1.9.0"` # For clipboard operations + - `colorama = "^0.4.6"` # For colored terminal text output + - `tqdm = "^4.66.4"` # For progress bars + - `tabulate = "^0.9.0"` # For tabular data formatting + - `pydantic` # For data validation and type checking + - `poetry` # For dependency management diff --git a/.gitignore b/.gitignore index 1d98c94..d434d7a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ __pycache__ *.pyc .DS_Store .tasks -cli.log \ No newline at end of file +cli.log +.aider* +.ruff_cache \ No newline at end of file diff --git a/README.md b/README.md index 93b7a71..b30d8ae 100644 --- a/README.md +++ b/README.md @@ -10,23 +10,23 @@ Code2Prompt is a powerful command-line tool that generates comprehensive prompts from codebases, designed to streamline interactions between developers and Large Language Models (LLMs) for code analysis, documentation, and improvement tasks. - ## Table of Contents 1. [Why Code2Prompt?](#why-code2prompt) 2. [Features](#features) 3. [Installation](#installation) -4. [Quick Start](#quick-start) -5. [Usage](#usage) -6. [Options](#options) -7. [Examples](#examples) -8. [Templating System](#templating-system) -9. [Integration with LLM CLI](#integration-with-llm-cli) -10. [GitHub Actions Integration](#github-actions-integration) -11. [Configuration File](#configuration-file) -12. [Troubleshooting](#troubleshooting) -13. [Contributing](#contributing) -14. [License](#license) +4. [Getting Started](#getting-started) +5. [Quick Start](#quick-start) +6. [Usage](#usage) +7. [Options](#options) +8. [Examples](#examples) +9. [Templating System](#templating-system) +10. [Integration with LLM CLI](#integration-with-llm-cli) +11. [GitHub Actions Integration](#github-actions-integration) +12. [Configuration File](#configuration-file) +13. [Troubleshooting](#troubleshooting) +14. [Contributing](#contributing) +15. [License](#license) # Code2Prompt: Transform Your Codebase into AI-Ready Prompts @@ -42,25 +42,25 @@ Code2Prompt is a powerful, open-source command-line tool that bridges the gap be ### ๐Ÿš€ Key Features -- **Holistic Codebase Representation**: Generate a well-structured Markdown prompt that captures your entire project's essence. -- **Intelligent Source Tree Generation**: Create a clear, hierarchical view of your codebase structure. -- **Customizable Prompt Templates**: Tailor your output using Jinja2 templates to suit specific AI tasks. -- **Smart Token Management**: Count and optimize tokens to ensure compatibility with various LLM token limits. -- **Gitignore Integration**: Respect your project's .gitignore rules for accurate representation. -- **Flexible File Handling**: Filter and exclude files using powerful glob patterns. -- **Clipboard Ready**: Instantly copy generated prompts to your clipboard for quick AI interactions. -- **Multiple Output Options**: Save to file or display in the console. -- **Enhanced Code Readability**: Add line numbers to source code blocks for precise referencing. -- **Include file**: Support of template import -- **Input variables**: Support of Input Variables in templates. +- **Holistic Codebase Representation**: Generate a well-structured Markdown prompt that captures your entire project's essence, making it easier for LLMs to understand the context. +- **Intelligent Source Tree Generation**: Create a clear, hierarchical view of your codebase structure, allowing for better navigation and understanding of the project. +- **Customizable Prompt Templates**: Tailor your output using Jinja2 templates to suit specific AI tasks, enhancing the relevance of generated prompts. +- **Smart Token Management**: Count and optimize tokens to ensure compatibility with various LLM token limits, preventing errors during processing. +- **Gitignore Integration**: Respect your project's .gitignore rules for accurate representation, ensuring that irrelevant files are excluded from processing. +- **Flexible File Handling**: Filter and exclude files using powerful glob patterns, giving you control over which files are included in the prompt generation. +- **Clipboard Ready**: Instantly copy generated prompts to your clipboard for quick AI interactions, streamlining your workflow. +- **Multiple Output Options**: Save to file or display in the console, providing flexibility in how you want to use the generated prompts. +- **Enhanced Code Readability**: Add line numbers to source code blocks for precise referencing, making it easier to discuss specific parts of the code. +- **Include file**: Support of template import, allowing for modular template design. +- **Input variables**: Support of Input Variables in templates, enabling dynamic prompt generation based on user input. ### ๐Ÿ’ก Why Code2Prompt? - **Contextual Understanding**: Provide LLMs with a comprehensive view of your project for more accurate suggestions and analysis. -- **Consistency Boost**: Maintain coding style and conventions across your entire project. -- **Efficient Refactoring**: Enable better interdependency analysis and smarter refactoring recommendations. -- **Improved Documentation**: Generate contextually relevant documentation that truly reflects your codebase. -- **Pattern Recognition**: Help LLMs learn and apply your project-specific patterns and idioms. +- **Consistency Boost**: Maintain coding style and conventions across your entire project, improving code quality. +- **Efficient Refactoring**: Enable better interdependency analysis and smarter refactoring recommendations, saving time and effort. +- **Improved Documentation**: Generate contextually relevant documentation that truly reflects your codebase, enhancing maintainability. +- **Pattern Recognition**: Help LLMs learn and apply your project-specific patterns and idioms, improving the quality of AI interactions. Transform the way you interact with AI for software development. With Code2Prompt, harness the full power of your codebase in every AI conversation. @@ -81,6 +81,20 @@ pip install code2prompt pipx install code2prompt ``` +## Getting Started + +To get started with Code2Prompt, follow these steps: + +1. **Install Code2Prompt**: Use one of the installation methods mentioned above. +2. **Prepare Your Codebase**: Ensure your project is organized and that you have a `.gitignore` file if necessary. +3. **Run Code2Prompt**: Use the command line to generate prompts from your codebase. + +For example, to generate a prompt from a single Python file, run: + +```bash +code2prompt --path /path/to/your/script.py +``` + ## Quick Start 1. Generate a prompt from a single Python file: @@ -130,7 +144,7 @@ code2prompt --path /path/to/dir1 --path /path/to/file2.py [OPTIONS] | `--encoding` | | Specify the tokenizer encoding to use (default: "cl100k_base") | | `--create-templates` | | Create a templates directory with example templates | | `--version` | `-v` | Show the version and exit | - +| `--log-level` | | Set the logging level (e.g., DEBUG, INFO, WARNING, ERROR, CRITICAL) | ## Command Parameters @@ -225,7 +239,6 @@ or By using the `--filter` and `--exclude` options effectively and safely (with proper quoting), you can precisely control which files are processed in your project, ensuring both accuracy and security in your command execution. - ## Examples 1. Generate documentation for a Python library: @@ -350,7 +363,6 @@ code2prompt --path /your/project --tokens --encoding p50k_base Understanding token counts is crucial when working with AI models that have token limits, ensuring your prompts fit within the model's context window. - ### Token Price Estimation Code2Prompt now includes a powerful feature for estimating token prices across various AI providers and models. Use the `--price` option in conjunction with `--tokens` to display a comprehensive breakdown of estimated costs. This feature calculates prices based on both input and output tokens, with input tokens determined by your codebase and a default of 1000 output tokens (customizable via `--output-tokens`). You can specify a particular provider or model, or view prices across all available options. This functionality helps developers make informed decisions about AI model usage and cost management. For example: @@ -363,7 +375,6 @@ This command will analyze your project, count the tokens, and provide a detailed ![](./docs/screen-example2.png) - ## ๐Ÿ”ฅ Analyzing Codebases code2prompt now offers a powerful feature to analyze codebases and provide a summary of file extensions. Use the `--analyze` option along with the `-p` (path) option to get an overview of your project's file composition. For example: @@ -462,8 +473,6 @@ Start from this codebase: ## The codebase: - - ``` When you run `code2prompt` with this template, it will automatically detect the `{{input:variable_name}}` patterns and prompt the user to provide values for each variable (extension_name, main_functionality, and target_audience). This allows for flexible and interactive prompt generation, making it easy to create customized AI prompts for various Chrome extension ideas. @@ -475,8 +484,7 @@ For example, if a user inputs: The tool will generate a tailored prompt for an AI to create a detailed plan for this specific Chrome extension. This feature is particularly useful for developers, product managers, or anyone looking to quickly generate customized AI prompts for various projects or ideas. - -## ๐Ÿ”ฅ Feature Highligth "Include File" Feature +## ๐Ÿ”ฅ Feature Highlight "Include File" Feature The code2prompt project now supports a powerful "include file" feature, enhancing template modularity and reusability. @@ -531,11 +539,11 @@ Example `.code2promptrc`: ## Roadmap - - [ ] Interractive filtering + - [ ] Interactive filtering - [X] Include system in template to promote re-usability of sub templates. - [X] Support of input variables - - [ ] Tokens count for Anthropic Models and other models such LLama3 or Mistral - - [X] Cost Estimations for main LLM providers based in token count + - [ ] Tokens count for Anthropic Models and other models such as LLama3 or Mistral + - [X] Cost Estimations for main LLM providers based on token count - [ ] Integration with [qllm](https://github.com/quantalogic/qllm) (Quantalogic LLM) - [ ] Embedding of file summary in SQL-Lite - [ ] Intelligence selection of file based on an LLM diff --git a/code2prompt/commands/__init__.py b/code2prompt/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/code2prompt/commands/analyze.py b/code2prompt/commands/analyze.py new file mode 100644 index 0000000..7d601a7 --- /dev/null +++ b/code2prompt/commands/analyze.py @@ -0,0 +1,70 @@ +# code2prompt/commands/analyze.py + +from pathlib import Path +from typing import Dict + +from code2prompt.commands.base_command import BaseCommand +from code2prompt.utils.analyzer import ( + analyze_codebase, + format_flat_output, + format_tree_output, + get_extension_list, +) + + +class AnalyzeCommand(BaseCommand): + """Command for analyzing the codebase structure.""" + + def execute(self) -> None: + """Execute the analyze command.""" + self.logger.info("Analyzing codebase...") + + for path in self.config.path: + self._analyze_path(Path(path)) + + self.logger.info("Analysis complete.") + + def _analyze_path(self, path: Path) -> None: + """ + Analyze a single path and output the results. + + Args: + path (Path): The path to analyze. + """ + extension_counts, extension_dirs = analyze_codebase(path) + + if not extension_counts: + self.logger.warning(f"No files found in {path}") + return + + if self.config.format == "flat": + output = format_flat_output(extension_counts) + else: + output = format_tree_output(extension_dirs) + + print(output) + + print("\nComma-separated list of extensions:") + print(get_extension_list(extension_counts)) + + if self.config.tokens: + total_tokens = self._count_tokens(extension_counts) + self.logger.info(f"Total tokens in codebase: {total_tokens}") + + def _count_tokens(self, extension_counts: Dict[str, int]) -> int: + """ + Count the total number of tokens in the codebase. + + Args: + extension_counts (Dict[str, int]): A dictionary of file extensions and their counts. + + Returns: + int: The total number of tokens. + """ + total_tokens = 0 + for _ext, count in extension_counts.items(): + # This is a simplified token count. You might want to implement a more + # sophisticated counting method based on the file type. + total_tokens += count * 100 # Assuming an average of 100 tokens per file + + return total_tokens diff --git a/code2prompt/commands/base_command.py b/code2prompt/commands/base_command.py new file mode 100644 index 0000000..fb308a9 --- /dev/null +++ b/code2prompt/commands/base_command.py @@ -0,0 +1,76 @@ +# code2prompt/commands/base_command.py + +from abc import ABC, abstractmethod +import logging +from code2prompt.config import Configuration + +class BaseCommand(ABC): + """ + Abstract base class for all commands in the code2prompt tool. + + This class defines the basic structure and common functionality + for all command classes. It ensures that each command has access + to the configuration and a logger, and defines an abstract execute + method that must be implemented by all subclasses. + + Attributes: + config (Configuration): The configuration object for the command. + logger (logging.Logger): The logger instance for the command. + """ + + def __init__(self, config: Configuration, logger: logging.Logger): + """ + Initialize the BaseCommand with configuration and logger. + + Args: + config (Configuration): The configuration object for the command. + logger (logging.Logger): The logger instance for the command. + """ + self.config = config + self.logger = logger + + @abstractmethod + def execute(self) -> None: + """ + Execute the command. + + This method must be implemented by all subclasses to define + the specific behavior of each command. + + Raises: + NotImplementedError: If the subclass does not implement this method. + """ + raise NotImplementedError("Subclasses must implement execute method") + + def log_start(self) -> None: + """ + Log the start of the command execution. + """ + self.logger.info(f"Starting execution of {self.__class__.__name__}") + + def log_end(self) -> None: + """ + Log the end of the command execution. + """ + self.logger.info(f"Finished execution of {self.__class__.__name__}") + + def handle_error(self, error: Exception) -> None: + """ + Handle and log any errors that occur during command execution. + + Args: + error (Exception): The exception that was raised. + """ + self.logger.error(f"Error in {self.__class__.__name__}: {str(error)}", exc_info=True) + + def validate_config(self) -> bool: + """ + Validate the configuration for the command. + + This method should be overridden by subclasses to perform + command-specific configuration validation. + + Returns: + bool: True if the configuration is valid, False otherwise. + """ + return True \ No newline at end of file diff --git a/code2prompt/commands/generate.py b/code2prompt/commands/generate.py new file mode 100644 index 0000000..0234d89 --- /dev/null +++ b/code2prompt/commands/generate.py @@ -0,0 +1,66 @@ +""" +This module contains the GenerateCommand class, which is responsible for generating +markdown content from code files based on the provided configuration. +""" + +from typing import List, Dict, Any +from code2prompt.core.process_files import process_files +from code2prompt.core.generate_content import generate_content +from code2prompt.core.write_output import write_output +from code2prompt.utils.count_tokens import count_tokens +from code2prompt.utils.logging_utils import log_token_count +from code2prompt.utils.display_price_table import display_price_table +from code2prompt.commands.base_command import BaseCommand + + +class GenerateCommand(BaseCommand): + """Command for generating markdown content from code files.""" + + def execute(self) -> None: + """Execute the generate command.""" + self.logger.info("Generating markdown...") + file_paths = self._process_files() + content = self._generate_content(file_paths) + self._write_output(content) + + if self.config.price: + self.display_token_count_and_price(content) + elif self.config.tokens: + self.display_token_count(content) + + self.logger.info("Generation complete.") + + def _process_files(self) -> List[Dict[str, Any]]: + """Process files based on the configuration.""" + all_files_data = [] + files_data = process_files( + file_paths=self.config.path, + line_number=self.config.line_number, + no_codeblock=self.config.no_codeblock, + suppress_comments=self.config.suppress_comments, + ) + all_files_data.extend(files_data) + return all_files_data + + def _generate_content(self, files_data: List[Dict[str, Any]]) -> str: + """Generate content from processed files data.""" + return generate_content(files_data, self.config.dict()) + + def _write_output(self, content: str) -> None: + """Write the generated content to output.""" + write_output(content, self.config.output, copy_to_clipboard=True) + + def display_token_count_and_price(self, content: str) -> None: + """Handle token counting and price calculation if enabled.""" + token_count = count_tokens(content, self.config.encoding) + model = self.config.model + provider = self.config.provider + display_price_table(token_count, provider, model, self.config.output_tokens) + log_token_count(token_count) + + + def display_token_count(self, content: str) -> None: + """Display the token count if enabled.""" + token_count = count_tokens(content, self.config.encoding) + log_token_count(token_count) + diff --git a/code2prompt/commands/interactive_selector.py b/code2prompt/commands/interactive_selector.py new file mode 100644 index 0000000..c9aa5af --- /dev/null +++ b/code2prompt/commands/interactive_selector.py @@ -0,0 +1,297 @@ +from typing import List, Dict, Set, Tuple +import os +from pathlib import Path +from prompt_toolkit import Application +from prompt_toolkit.layout.containers import VSplit, HSplit, Window +from prompt_toolkit.layout.controls import FormattedTextControl +from prompt_toolkit.layout.layout import Layout +from prompt_toolkit.layout.scrollable_pane import ScrollablePane +from prompt_toolkit.widgets import Frame +from prompt_toolkit.key_binding import KeyBindings +from prompt_toolkit.styles import Style +import signal + +# Constant for terminal height adjustment +TERMINAL_HEIGHT_ADJUSTMENT = 3 + + +class InteractiveFileSelector: + """Interactive file selector.""" + + def __init__(self, paths: List[Path], selected_files: List[Path]): + self.paths: List[Path] = paths.copy() + self.start_line: int = 0 + self.cursor_position: int = 0 + self.formatted_tree: List[str] = [] + self.tree_paths: List[Path] = [] + self.tree_full_paths: List[str] = [] + self.kb = self._create_key_bindings() + self.selected_files: Set[str] = set( + [str(Path(file).resolve()) for file in selected_files] + ) + self.selection_state: Dict[str, Set[str]] = {} # State tracking for selections + self.app = self._create_application(self.kb) + + def _get_terminal_height(self) -> int: + """Get the height of the terminal.""" + return os.get_terminal_size().lines + + def _get_directory_tree(self) -> Dict[Path, Dict]: + """Get a combined directory tree for the given paths.""" + tree: Dict[Path, Dict] = {} + for path in self.paths: + current = tree # Start from the root of the tree + for part in Path(path).parts: + if part not in current: # Check if part is already in the current level + current[part] = {} # Create a new dictionary for the part + current = current[part] # Move to the next level in the tree + return tree + + def _format_tree( + self, tree: Dict[Path, Dict], indent: str = "", parent_dir: str = "" + ) -> Tuple[List[str], List[Path], List[str]]: + """Format the directory tree into a list of strings.""" + lines: List[str] = [] + tree_paths: List[Path] = [] + tree_full_paths: List[str] = [] + for i, (file_path, subtree) in enumerate(tree.items()): + is_last = i == len(tree) - 1 + prefix = "โ””โ”€โ”€ " if is_last else "โ”œโ”€โ”€ " + line = f"{indent}{prefix}{Path(file_path).name}" + lines.append(line) + resolved_path = Path(parent_dir, file_path).resolve() + tree_paths.append(resolved_path) + tree_full_paths.append( + str(resolved_path) + ) # Store the full path as a string + if subtree: + extension = " " if is_last else "โ”‚ " + sub_lines, sub_tree_paths, sub_full_paths = self._format_tree( + subtree, indent + extension, str(resolved_path) + ) + lines.extend(sub_lines) + tree_paths.extend(sub_tree_paths) + tree_full_paths.extend( + sub_full_paths + ) # Merge the full paths from the subtree + return lines, tree_paths, tree_full_paths + + def _validate_cursor_position(self) -> None: + """Ensure cursor position is valid.""" + if self.cursor_position < 0: + self.cursor_position = 0 + elif self.cursor_position >= len(self.formatted_tree): + self.cursor_position = len(self.formatted_tree) - 1 + + def _get_visible_lines(self) -> int: + """Calculate the number of visible lines based on terminal height.""" + terminal_height = self._get_terminal_height() + return terminal_height - TERMINAL_HEIGHT_ADJUSTMENT # Use constant + + def _get_formatted_text(self) -> List[tuple]: + """Generate formatted text for display.""" + result = [] + # Ensure that formatted_tree and tree_paths have the same length + if len(self.formatted_tree) == len(self.tree_paths): + visible_lines = self._get_visible_lines() + # Calculate the end line for the loop + end_line = min(self.start_line + visible_lines, len(self.formatted_tree)) + for i in range(self.start_line, end_line): + line = self.formatted_tree[i] + style = "class:cursor" if i == self.cursor_position else "" + # Ensure cursor_position is valid + self._validate_cursor_position() + # Get the full path + file_path = str(self.tree_full_paths[i]) + is_dir = os.path.isdir(file_path) + # Check if the full path is selected + is_selected = file_path in self.selected_files + # Update checkbox based on selection state + checkbox = "[X]" if is_selected else " " if is_dir else "[ ]" + if file_path in self.selection_state: + if len(self.selection_state[file_path]) == len(self.tree_paths): + checkbox = "[X]" + # Append formatted line to result + result.append((style, f"{checkbox} {line}\n")) + return result + + def _toggle_file_selection(self, current_item: str) -> None: + """Toggle the selection of the current item.""" + # Convert current_item to string to use with startswith + current_item_str = str(current_item) + if current_item_str in self.selected_files: + self.selected_files.remove(current_item_str) + # Unselect all descendants + if current_item_str in self.selection_state: + for descendant in self.selection_state[current_item_str]: + self.selected_files.discard(descendant) + del self.selection_state[current_item_str] + else: + self.selected_files.add(current_item_str) + # Select all descendants + self.selection_state[current_item_str] = { + descendant + for descendant in self.tree_paths + if str(descendant).startswith(current_item_str) + } + + def _get_current_item(self) -> str: + """Get the current item based on cursor position.""" + if 0 <= self.cursor_position < len(self.tree_paths): + current_item = self.tree_full_paths[self.cursor_position] + return current_item # Return the full path + return None # Return None if no valid path is found + + def _resize_handler(self, _event) -> None: + """Handle terminal resize event.""" + self.start_line = max(0, self.cursor_position - self._get_visible_lines() + 1) + self.app.invalidate() # Invalidate the application to refresh the layout + + def run(self) -> List[Path]: + """Run the interactive file selection.""" + self._check_paths() + tree = self._get_directory_tree() + self.formatted_tree, self.tree_paths, self.tree_full_paths = self._format_tree( + tree + ) + signal.signal(signal.SIGWINCH, self._resize_handler) + self.app.run() + list_selected_files : List[Path] = [] + for f in self.selected_files: + list_selected_files.append(Path(f)) + print(list_selected_files) + return list_selected_files + + def _create_key_bindings(self) -> KeyBindings: + """Create and return key bindings for the application.""" + kb = KeyBindings() + + @kb.add("q") + def quit_application(event): + event.app.exit() + + @kb.add("up") + def move_cursor_up(_event): + if self.cursor_position > 0: + self.cursor_position -= 1 + # Update start_line if needed for scrolling + if self.cursor_position < self.start_line: + self.start_line = self.cursor_position + self._validate_cursor_position() # Validate after moving + self.app.invalidate() # Refresh the display after moving + + @kb.add("down") + def move_cursor_down(_event): + if self.cursor_position < len(self.formatted_tree) - 1: + self.cursor_position += 1 + # Update start_line if needed for scrolling + if self.cursor_position >= self.start_line + self._get_visible_lines(): + self.start_line += 1 + self._validate_cursor_position() # Validate after moving + self.app.invalidate() # Refresh the display after moving + + @kb.add("pageup") + def page_up(_event): + self.cursor_position = max( + 0, self.cursor_position - self._get_visible_lines() + ) + if self.cursor_position < self.start_line: + self.start_line = ( + self.cursor_position + ) # Adjust start_line to keep the cursor in view + self.app.invalidate() # Refresh the display after moving + + @kb.add("pagedown") + def page_down(_event): + self.cursor_position = min( + len(self.formatted_tree) - 1, + self.cursor_position + self._get_visible_lines(), + ) + if self.cursor_position >= self.start_line + self._get_visible_lines(): + self.start_line = ( + self.cursor_position - self._get_visible_lines() + 1 + ) # Adjust start_line to keep the cursor in view + self.app.invalidate() # Refresh the display after moving + + @kb.add("space") + def toggle_selection(_event): + current_item = self._get_current_item() # Get the current item as a Path + if current_item: # Ensure current_item is not None + self._toggle_file_selection( + current_item + ) # Pass the Path object directly + self.app.invalidate() # Refresh the display after toggling + + @kb.add("enter") + def confirm_selection(_event): + self.app.exit() + + return kb + + def _get_selected_files_text(self) -> str: + """Get the selected files text.""" + if self.selected_files: + return f"Selected: {len(self.selected_files)} file(s)" + return "Selected: 0 file(s): None" + + def _create_application(self, kb) -> Application: + """Create and return the application instance.""" + tree_window = Window( + content=FormattedTextControl(self._get_formatted_text, focusable=True), + width=60, + dont_extend_width=True, + wrap_lines=False, + ) + scrollable_tree = ScrollablePane(tree_window) + instructions = ( + "Instructions:\n" + "-------------\n" + "1. Use โ†‘ and โ†“ to navigate\n" + "2. Press Space to select/deselect an item\n" + "3. Press Enter to confirm your selection\n" + "4. Press q to quit the selection process\n" + ) + layout = Layout( + VSplit( + [ + Frame(scrollable_tree, title="File Tree"), + Window(width=1, char="โ”‚"), + HSplit( + [ + Window( + content=FormattedTextControl(instructions), height=5 + ), + Window(height=1), + Window( + content=FormattedTextControl( + self._get_selected_files_text + ), + height=10, + ), + ], + ), + ], + padding=1, + ) + ) + style = Style.from_dict( + { + "cursor": "bg:#00ff00 #000000", + "frame.border": "#888888", + } + ) + + return Application( + layout=layout, + key_bindings=kb, + full_screen=True, + style=style, + mouse_support=True, + ) + + def _check_paths(self) -> None: + """Check if the provided paths are valid.""" + if not self.paths or any(not path for path in self.paths): + raise ValueError( + "A valid list of paths must be provided for interactive mode." + ) diff --git a/code2prompt/comment_stripper/__init__.py b/code2prompt/comment_stripper/__init__.py index 39204f9..e69de29 100644 --- a/code2prompt/comment_stripper/__init__.py +++ b/code2prompt/comment_stripper/__init__.py @@ -1,8 +0,0 @@ -from .c_style import strip_c_style_comments -from .html_style import strip_html_style_comments -from .python_style import strip_python_style_comments -from .shell_style import strip_shell_style_comments -from .sql_style import strip_sql_style_comments -from .matlab_style import strip_matlab_style_comments -from .r_style import strip_r_style_comments -from .strip_comments import strip_comments diff --git a/code2prompt/comment_stripper/strip_comments.py b/code2prompt/comment_stripper/strip_comments.py index 5291503..aa699f6 100644 --- a/code2prompt/comment_stripper/strip_comments.py +++ b/code2prompt/comment_stripper/strip_comments.py @@ -1,3 +1,7 @@ +""" +This module contains the function to strip comments from code based on the programming language. +""" + from .c_style import strip_c_style_comments from .html_style import strip_html_style_comments from .python_style import strip_python_style_comments @@ -6,9 +10,33 @@ from .matlab_style import strip_matlab_style_comments from .r_style import strip_r_style_comments + def strip_comments(code: str, language: str) -> str: + """Strips comments from the given code based on the specified programming language. + + Args: + code (str): The source code from which comments will be removed. + language (str): The programming language of the source code. + + Returns: + str: The code without comments. + """ if language in [ - "c", "cpp", "java", "javascript", "csharp", "php", "go", "rust", "kotlin", "swift", "scala", "dart", + "c", + "cpp", + "java", + "javascript", + "csharp", + "php", + "go", + "rust", + "kotlin", + "swift", + "scala", + "dart", + "typescript", + "typescriptreact", + "react", ]: return strip_c_style_comments(code) elif language in ["python", "ruby", "perl"]: diff --git a/code2prompt/config.py b/code2prompt/config.py new file mode 100644 index 0000000..83e592a --- /dev/null +++ b/code2prompt/config.py @@ -0,0 +1,105 @@ +# code2prompt/config.py + +from pathlib import Path +from typing import List, Optional +from pydantic import BaseModel, Field, field_validator, ValidationError + +class Configuration(BaseModel): + """ + Configuration class for code2prompt tool. + + This class uses Pydantic for data validation and settings management. + It defines all the configuration options available for the code2prompt tool. + """ + + path: List[Path] = Field(default_factory=list, description="Path(s) to the directory or file to process.") + output: Optional[Path] = Field(None, description="Name of the output Markdown file.") + gitignore: Optional[Path] = Field(None, description="Path to the .gitignore file.") + filter: Optional[str] = Field(None, description="Comma-separated filter patterns to include files.") + exclude: Optional[str] = Field(None, description="Comma-separated patterns to exclude files.") + case_sensitive: bool = Field(False, description="Perform case-sensitive pattern matching.") + suppress_comments: bool = Field(False, description="Strip comments from the code files.") + line_number: bool = Field(False, description="Add line numbers to source code blocks.") + no_codeblock: bool = Field(False, description="Disable wrapping code inside markdown code blocks.") + template: Optional[Path] = Field(None, description="Path to a Jinja2 template file for custom prompt generation.") + tokens: bool = Field(False, description="Display the token count of the generated prompt.") + encoding: str = Field("cl100k_base", description="Specify the tokenizer encoding to use.") + create_templates: bool = Field(False, description="Create a templates directory with example templates.") + log_level: str = Field("INFO", description="Set the logging level.") + price: bool = Field(False, description="Display the estimated price of tokens based on provider and model.") + provider: Optional[str] = Field(None, description="Specify the provider for price calculation.") + model: Optional[str] = Field(None, description="Specify the model for price calculation.") + output_tokens: int = Field(1000, description="Specify the number of output tokens for price calculation.") + analyze: bool = Field(False, description="Analyze the codebase and provide a summary of file extensions.") + format: str = Field("flat", description="Format of the analysis output (flat or tree-like).") + interactive: bool = Field(False, description="Interactive mode to select files.") + + @field_validator('encoding') + @classmethod + def validate_encoding(cls, v: str) -> str: + valid_encodings = ["cl100k_base", "p50k_base", "p50k_edit", "r50k_base"] + if v not in valid_encodings: + raise ValueError(f"Invalid encoding. Must be one of: {', '.join(valid_encodings)}") + return v + + @field_validator('log_level') + @classmethod + def validate_log_level(cls, v: str) -> str: + valid_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] + if v.upper() not in valid_levels: + raise ValueError(f"Invalid log level. Must be one of: {', '.join(valid_levels)}") + return v.upper() + + @field_validator('format') + @classmethod + def validate_format(cls, v: str) -> str: + valid_formats = ["flat", "tree"] + if v not in valid_formats: + raise ValueError(f"Invalid format. Must be one of: {', '.join(valid_formats)}") + return v + + @classmethod + def load_from_file(cls, file_path: Path) -> "Configuration": + """ + Load configuration from a file. + + Args: + file_path (Path): Path to the configuration file. + + Returns: + Configuration: Loaded configuration object. + + Raises: + FileNotFoundError: If the configuration file is not found. + ValidationError: If the configuration file is invalid. + """ + if not file_path.exists(): + raise FileNotFoundError(f"Configuration file not found: {file_path}") + + try: + with file_path.open() as f: + config_data = f.read() + return cls.model_validate_json(config_data) + except ValidationError as e: + raise ValueError(f"Invalid configuration file: {e}") + + def merge(self, cli_options: dict) -> "Configuration": + """ + Merge CLI options with the current configuration. + + Args: + cli_options (dict): Dictionary of CLI options. + + Returns: + Configuration: New configuration object with merged options. + """ + # Create a new dict with all current config values + merged_dict = self.model_dump() + + # Update with CLI options, but only if they're different from the default + for key, value in cli_options.items(): + if value is not None and value != self.model_fields[key].default: + merged_dict[key] = value + + # Create a new Configuration object with the merged options + return Configuration.model_validate(merged_dict) \ No newline at end of file diff --git a/code2prompt/core/file_path_retriever.py b/code2prompt/core/file_path_retriever.py new file mode 100644 index 0000000..da0982a --- /dev/null +++ b/code2prompt/core/file_path_retriever.py @@ -0,0 +1,70 @@ +""" +This module contains the function to get file paths based on the provided options. +""" + +from pathlib import Path +from code2prompt.utils.get_gitignore_patterns import get_gitignore_patterns +from code2prompt.utils.should_process_file import should_process_file + + +def retrieve_file_paths( + file_paths: list[Path], + filter_patterns: list[str], + exclude_patterns: list[str], + case_sensitive: bool, + gitignore: list[str], +) -> list[Path]: + """ + Retrieves file paths based on the provided options. + + Args: + file_paths (list[Path]): A list of paths to retrieve. + filter_patterns (list[str]): Patterns to include. + exclude_patterns (list[str]): Patterns to exclude. + case_sensitive (bool): Whether the filtering should be case sensitive. + gitignore (list[str]): Gitignore patterns to consider. + + Returns: + list[Path]: A list of file paths that should be processed. + """ + if not file_paths: + raise ValueError("file_paths list cannot be empty.") + + retrieved_paths: list[Path] = [] + + for path in file_paths: + try: + path = Path(path) + + # Get gitignore patterns for the current path + gitignore_patterns = get_gitignore_patterns( + path.parent if path.is_file() else path, gitignore + ) + + # Add the top-level directory if it should be processed + if path.is_dir() and should_process_file( + path, + gitignore_patterns, + path.parent, + filter_patterns, + exclude_patterns, + case_sensitive, + ): + retrieved_paths.append(path) + + # Add files and directories within the top-level directory + for file_path in path.rglob("*"): + if should_process_file( + file_path, + gitignore_patterns, + path, + filter_patterns, + exclude_patterns, + case_sensitive, + ): + retrieved_paths.append(file_path) + + except (FileNotFoundError, PermissionError) as e: + print(f"Error processing path {path}: {e}") + + return retrieved_paths \ No newline at end of file diff --git a/code2prompt/core/process_file.py b/code2prompt/core/process_file.py index b05cd4e..64f659f 100644 --- a/code2prompt/core/process_file.py +++ b/code2prompt/core/process_file.py @@ -1,9 +1,18 @@ -from code2prompt.comment_stripper import strip_comments +""" +This module contains the function to process a file and extract its metadata and content. +""" + +from pathlib import Path +from datetime import datetime + from code2prompt.utils.add_line_numbers import add_line_numbers from code2prompt.utils.language_inference import infer_language -from datetime import datetime +from code2prompt.comment_stripper.strip_comments import strip_comments -def process_file(file_path, suppress_comments, line_number, no_codeblock): + +def process_file( + file_path: Path, suppress_comments: bool, line_number: bool, no_codeblock: bool +): """ Processes a given file to extract its metadata and content. @@ -18,19 +27,23 @@ def process_file(file_path, suppress_comments, line_number, no_codeblock): """ file_extension = file_path.suffix file_size = file_path.stat().st_size - file_creation_time = datetime.fromtimestamp(file_path.stat().st_ctime).strftime("%Y-%m-%d %H:%M:%S") - file_modification_time = datetime.fromtimestamp(file_path.stat().st_mtime).strftime("%Y-%m-%d %H:%M:%S") + file_creation_time = datetime.fromtimestamp(file_path.stat().st_ctime).strftime( + "%Y-%m-%d %H:%M:%S" + ) + file_modification_time = datetime.fromtimestamp(file_path.stat().st_mtime).strftime( + "%Y-%m-%d %H:%M:%S" + ) language = "unknown" try: with file_path.open("r", encoding="utf-8") as f: file_content = f.read() - + language = infer_language(file_path.name) - + if suppress_comments and language != "unknown": file_content = strip_comments(file_content, language) - + if line_number: file_content = add_line_numbers(file_content) except UnicodeDecodeError: @@ -44,5 +57,5 @@ def process_file(file_path, suppress_comments, line_number, no_codeblock): "created": file_creation_time, "modified": file_modification_time, "content": file_content, - "no_codeblock": no_codeblock + "no_codeblock": no_codeblock, } diff --git a/code2prompt/core/process_files.py b/code2prompt/core/process_files.py index f463bc3..32ab684 100644 --- a/code2prompt/core/process_files.py +++ b/code2prompt/core/process_files.py @@ -1,9 +1,17 @@ +""" +This module contains functions for processing files and directories. +""" + from pathlib import Path -from code2prompt.utils.get_gitignore_patterns import get_gitignore_patterns from code2prompt.core.process_file import process_file -from code2prompt.utils.should_process_file import should_process_file -def process_files(options): + +def process_files( + file_paths: list[Path], + suppress_comments: bool, + line_number: bool, + no_codeblock: bool, +): """ Processes files or directories based on the provided paths. @@ -15,41 +23,21 @@ def process_files(options): list: A list of dictionaries containing processed file data. """ files_data = [] + + # Test file paths if List[Path] type + if not (isinstance(file_paths, list) and all(isinstance(path, Path) for path in file_paths)): + raise ValueError("file_paths must be a list of Path objects") - # Ensure 'path' is always a list for consistent processing - paths = options['path'] if isinstance(options['path'], list) else [options['path']] - - for path in paths: + # Use get_file_paths to retrieve all file paths to process + for path in file_paths: path = Path(path) - - # Get gitignore patterns for the current path - gitignore_patterns = get_gitignore_patterns( - path.parent if path.is_file() else path, - options['gitignore'] + result = process_file( + file_path=path, + suppress_comments=suppress_comments, + line_number=line_number, + no_codeblock=no_codeblock, ) + if result: + files_data.append(result) - if path.is_file(): - # Process single file - if should_process_file(path, gitignore_patterns, path.parent, options): - result = process_file( - path, - options['suppress_comments'], - options['line_number'], - options['no_codeblock'] - ) - if result: - files_data.append(result) - else: - # Process directory - for file_path in path.rglob("*"): - if should_process_file(file_path, gitignore_patterns, path, options): - result = process_file( - file_path, - options['suppress_comments'], - options['line_number'], - options['no_codeblock'] - ) - if result: - files_data.append(result) - - return files_data \ No newline at end of file + return files_data diff --git a/code2prompt/main.py b/code2prompt/main.py index 1ec5a18..7ae7288 100644 --- a/code2prompt/main.py +++ b/code2prompt/main.py @@ -1,93 +1,71 @@ -from importlib import resources +"""Main module for the code2prompt CLI tool.""" + import logging from pathlib import Path - import click -from tabulate import tabulate - -from code2prompt.utils.config import load_config, merge_options -from code2prompt.utils.count_tokens import count_tokens -from code2prompt.core.generate_content import generate_content -from code2prompt.core.process_files import process_files -from code2prompt.core.write_output import write_output -from code2prompt.utils.create_template_directory import create_templates_directory -from code2prompt.utils.logging_utils import setup_logger, log_token_count, log_error, log_info -from code2prompt.utils.price_calculator import load_token_prices, calculate_prices -from code2prompt.utils.analyzer import analyze_codebase, format_flat_output, format_tree_output, get_extension_list +from code2prompt.commands.analyze import AnalyzeCommand +from code2prompt.commands.generate import GenerateCommand +from code2prompt.config import Configuration +from code2prompt.utils.logging_utils import setup_logger +from code2prompt.commands.interactive_selector import InteractiveFileSelector +from code2prompt.core.file_path_retriever import retrieve_file_paths +from code2prompt.version import VERSION -# Version number of the code2prompt tool -VERSION = "0.6.13" -# Default options for the tool -DEFAULT_OPTIONS = { - "path": [], - "output": None, - "gitignore": None, - "filter": None, - "exclude": None, - "case_sensitive": False, - "suppress_comments": False, - "line_number": False, - "no_codeblock": False, - "template": None, - "tokens": False, - "encoding": "cl100k_base", - "create_templates": False, - "log_level": "INFO", - "price": False, - "provider": None, - "model": None, - "output_tokens": 1000, # Default output token count - "analyze": False, - "format": "flat" -} - -@click.command() +@click.group(invoke_without_command=True) @click.version_option( VERSION, "-v", "--version", message="code2prompt version %(version)s" ) @click.option( - "--path", "-p", + "--config", + type=click.Path(exists=True, dir_okay=False), + help="Path to configuration file", +) +@click.option( + "--path", + "-p", type=click.Path(exists=True), multiple=True, help="Path(s) to the directory or file to process.", ) @click.option( - "--output", "-o", - type=click.Path(), - help="Name of the output Markdown file." + "--output", "-o", type=click.Path(), help="Name of the output Markdown file." ) @click.option( - "--gitignore", "-g", + "--gitignore", + "-g", type=click.Path(exists=True), help="Path to the .gitignore file.", ) @click.option( - "--filter", "-f", + "--filter", + "-f", type=str, help='Comma-separated filter patterns to include files (e.g., "*.py,*.js").', ) @click.option( - "--exclude", "-e", + "--interactive", + "-i", + is_flag=True, + help="Interactive mode to select files.", +) +@click.option( + "--exclude", + "-e", type=str, help='Comma-separated patterns to exclude files (e.g., "*.txt,*.md").', ) @click.option( - "--case-sensitive", - is_flag=True, - help="Perform case-sensitive pattern matching." + "--case-sensitive", is_flag=True, help="Perform case-sensitive pattern matching." ) @click.option( - "--suppress-comments", "-s", + "--suppress-comments", + "-s", is_flag=True, help="Strip comments from the code files.", - default=False, ) @click.option( - "--line-number", "-ln", - is_flag=True, - help="Add line numbers to source code blocks.", - default=False, + "--line-number", "-ln", is_flag=True, help="Add line numbers to source code blocks." ) @click.option( "--no-codeblock", @@ -95,14 +73,13 @@ help="Disable wrapping code inside markdown code blocks.", ) @click.option( - "--template", "-t", + "--template", + "-t", type=click.Path(exists=True), help="Path to a Jinja2 template file for custom prompt generation.", ) @click.option( - "--tokens", - is_flag=True, - help="Display the token count of the generated prompt." + "--tokens", is_flag=True, help="Display the token count of the generated prompt." ) @click.option( "--encoding", @@ -118,10 +95,9 @@ @click.option( "--log-level", type=click.Choice( - ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], - case_sensitive=False + ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], case_sensitive=False ), - default="INFO", + default="WARNING", help="Set the logging level.", ) @click.option( @@ -130,140 +106,109 @@ help="Display the estimated price of tokens based on provider and model.", ) @click.option( - "--provider", - type=str, - help="Specify the provider for price calculation.", -) -@click.option( - "--model", - type=str, - help="Specify the model for price calculation.", + "--provider", type=str, help="Specify the provider for price calculation." ) +@click.option("--model", type=str, help="Specify the model for price calculation.") @click.option( "--output-tokens", type=int, default=1000, help="Specify the number of output tokens for price calculation.", ) -@click.option( - "--analyze", - is_flag=True, - help="Analyze the codebase and provide a summary of file extensions.", -) -@click.option( - "--format", - type=click.Choice(["flat", "tree"]), - default="flat", - help="Format of the analysis output (flat or tree-like).", -) -def create_markdown_file(**cli_options): - """ - Creates a Markdown file based on the provided options. - - This function orchestrates the process of reading files from the specified paths, - processing them according to the given options (such as filtering, excluding certain files, - handling comments, etc.), and then generating a Markdown file with the processed content. - The output file name and location can be customized through the options. +@click.pass_context +def cli(ctx, config, path, **generate_options): + """code2prompt CLI tool""" + ctx.obj = {} + if config: + ctx.obj["config"] = Configuration.load_from_file(Path(config)) + else: + ctx.obj["config"] = Configuration() - Args: - **options (dict): Key-value pairs of options to customize the behavior of the function. - Possible keys include 'path', 'output', 'gitignore', 'filter', 'exclude', - 'case_sensitive', 'suppress_comments', 'line_number', 'no_codeblock', 'template', - 'tokens', 'encoding', 'create_templates', 'log_level', 'price', 'provider', 'model', - 'output_tokens', 'analyze', and 'format'. + logging.info("CLI initialized with config: %s", ctx.obj["config"]) - Returns: - None - """ - # Load configuration from .code2promptrc files - config = load_config(".") + if ctx.invoked_subcommand is None: + ctx.invoke(generate, path=path, **generate_options) - # Merge options: CLI takes precedence over config, which takes precedence over defaults - options = merge_options(cli_options, config, DEFAULT_OPTIONS) - # Set up logger with the specified log level - _logger = setup_logger(level=getattr(logging, options["log_level"].upper())) - - if options["create_templates"]: - cwd = Path.cwd() - templates_dir = cwd / "templates" - package_templates_dir = resources.files("code2prompt").joinpath("templates") - create_templates_directory( - package_templates_dir=package_templates_dir, - templates_dir=templates_dir - ) - return - - if not options["path"]: - log_error( - "Error: No path specified. Please provide a path using --path option or in .code2promptrc file." - ) - return +@cli.command() +@click.option( + "--path", + "-p", + type=click.Path(exists=True), + multiple=True, + help="Path(s) to the directory or file to process.", +) +@click.option( + "--output", "-o", type=click.Path(), help="Name of the output Markdown file." +) +@click.pass_context +def generate(ctx, **options): + """Generate markdown from code files""" - if options["analyze"]: - for path in options["path"]: - extension_counts, extension_dirs = analyze_codebase(path) - if "No files found" in extension_counts: - click.echo("No files found") - else: - if options["format"] == "flat": - output = format_flat_output(extension_counts) - else: - output = format_tree_output(extension_dirs) - - click.echo(output) - - click.echo("\nComma-separated list of extensions:") - click.echo(get_extension_list(extension_counts)) - return + config = ctx.obj["config"].merge(options) + logger = setup_logger(level=config.log_level) - all_files_data = [] - for path in options["path"]: - files_data = process_files({**options, "path": path}) - all_files_data.extend(files_data) + selected_paths: list[Path] = config.path - content = generate_content(all_files_data, options) + # Check if selected_paths is empty before proceeding + if not selected_paths: # {{ edit_1 }} Added check for empty paths + logging.error("No file paths provided. Please specify valid paths.") + return # Exit the function if no paths are provided - token_count = None - if options["tokens"] or options["price"]: - token_count = count_tokens(content, options["encoding"]) - log_token_count(token_count) + filter_patterns: list[str] = config.filter.split(",") if config.filter else [] + exclude_patterns: list[str] = config.exclude.split(",") if config.exclude else [] + case_sensitive: bool = config.case_sensitive + gitignore: str = config.gitignore - write_output(content, options["output"], copy_to_clipboard=True) + # filter paths based on .gitignore + filtered_paths = retrieve_file_paths( + file_paths=selected_paths, # {{ edit_1 }} Added 'file_paths' argument + gitignore=gitignore, + filter_patterns=filter_patterns, + exclude_patterns=exclude_patterns, + case_sensitive=case_sensitive, + ) - if options["price"]: - display_price_table(options, token_count) + if filtered_paths and config.interactive: + file_selector = InteractiveFileSelector(filtered_paths, filtered_paths) + filtered_selected_path = file_selector.run() + config.path = filtered_selected_path + else: + config.path = filtered_paths -def display_price_table(options, token_count): - """ - Display a table with price estimates for the given token count. + command = GenerateCommand(config, logger) + command.execute() - Args: - options (dict): The options dictionary containing pricing-related settings. - token_count (int): The number of tokens to calculate prices for. - """ - if token_count is None: - log_error("Error: Token count is required for price calculation.") - return + logger.info("Markdown generation completed.") - token_prices = load_token_prices() - if not token_prices: - return - output_token_count = options["output_tokens"] - table_data = calculate_prices(token_prices, token_count, output_token_count, options["provider"], options["model"]) +@cli.command() +@click.option( + "--path", + "-p", + type=click.Path(exists=True), + multiple=True, + help="Path(s) to analyze.", +) +@click.option( + "--format", + type=click.Choice(["flat", "tree"]), + default="flat", + help="Format of the analysis output.", +) +@click.pass_context +def analyze(ctx, **options): + """Analyze codebase structure""" + config = ctx.obj["config"].merge(options) + logger = setup_logger(level=config.log_level) + logger.info("Analyzing codebase with options: %s", options) - if not table_data: - log_error("Error: No matching provider or model found") - return + command = AnalyzeCommand(config, logger) + command.execute() - headers = ["Provider", "Model", "Price for 1K Input Tokens", "Number of Input Tokens", "Total Price"] - table = tabulate(table_data, headers=headers, tablefmt="grid") + logger.info("Codebase analysis completed.") - log_info("\nโœจ Estimated Token Prices: (All prices are in USD, it is an estimate as the current token implementation is based on OpenAI's Tokenizer)") - log_info("\n") - log_info(table) - log_info("\n๐Ÿ“ Note: The prices are based on the token count and the provider's pricing model.") -if __name__ == "__main__": - create_markdown_file() \ No newline at end of file +def get_directory_tree(path): + """Retrieve a list of files and directories in a given path.""" + return [p.name for p in Path(path).iterdir() if p.is_file() or p.is_dir()] diff --git a/code2prompt/print_help.py b/code2prompt/print_help.py new file mode 100644 index 0000000..249ef61 --- /dev/null +++ b/code2prompt/print_help.py @@ -0,0 +1,67 @@ +from code2prompt.version import VERSION + + +import click + + +def print_help(_ctx): + """Print comprehensive help information.""" + click.echo(click.style("code2prompt CLI Tool", fg="green", bold=True)) + click.echo(f"Version: {VERSION}\n") + + click.echo(click.style("Description:", fg="yellow", bold=True)) + click.echo("code2prompt is a powerful CLI tool for generating markdown documentation from code files and analyzing codebase structure.\n") + + click.echo(click.style("Usage:", fg="yellow", bold=True)) + click.echo(" code2prompt [OPTIONS] COMMAND [ARGS]...\n") + + click.echo(click.style("Commands:", fg="yellow", bold=True)) + click.echo(" generate Generate markdown from code files") + click.echo(" analyze Analyze codebase structure\n") + + click.echo(click.style("Global Options:", fg="yellow", bold=True)) + click.echo(" --config PATH Path to configuration file") + click.echo(" -v, --version Show the version and exit") + click.echo(" --help Show this message and exit\n") + + click.echo(click.style("Generate Command Options:", fg="yellow", bold=True)) + click.echo(" -p, --path PATH Path(s) to the directory or file to process") + click.echo(" -o, --output PATH Name of the output Markdown file") + click.echo(" -g, --gitignore PATH Path to the .gitignore file") + click.echo(" -f, --filter TEXT Comma-separated filter patterns to include files") + click.echo(" -e, --exclude TEXT Comma-separated patterns to exclude files") + click.echo(" --case-sensitive Perform case-sensitive pattern matching") + click.echo(" -s, --suppress-comments Strip comments from the code files") + click.echo(" -ln, --line-number Add line numbers to source code blocks") + click.echo(" --no-codeblock Disable wrapping code inside markdown code blocks") + click.echo(" -t, --template PATH Path to a Jinja2 template file for custom prompt generation") + click.echo(" --tokens Display the token count of the generated prompt") + click.echo(" --encoding [cl100k_base|p50k_base|p50k_edit|r50k_base]") + click.echo(" Specify the tokenizer encoding to use") + click.echo(" --create-templates Create a templates directory with example templates") + click.echo(" --log-level [DEBUG|INFO|WARNING|ERROR|CRITICAL]") + click.echo(" Set the logging level") + click.echo(" --price Display the estimated price of tokens") + click.echo(" --provider TEXT Specify the provider for price calculation") + click.echo(" --model TEXT Specify the model for price calculation") + click.echo(" --output-tokens INTEGER Specify the number of output tokens for price calculation\n") + + click.echo(click.style("Analyze Command Options:", fg="yellow", bold=True)) + click.echo(" -p, --path PATH Path(s) to analyze") + click.echo(" --format [flat|tree] Format of the analysis output\n") + + click.echo(click.style("Examples:", fg="yellow", bold=True)) + click.echo(" code2prompt generate -p ./src") + click.echo(" code2prompt analyze -p ./src --format tree") + click.echo(" code2prompt generate -p ./src -o output.md --price --provider openai --model gpt-3.5-turbo\n") + + click.echo(click.style("Note:", fg="red", bold=True)) + click.echo("๐Ÿšจ code2prompt 0.7.0 is a major version update from code2doc.") + click.echo("๐Ÿค– Starting with version 0.7.0, you must use 'code2prompt generate' to generate content from code files.") + click.echo() + click.echo(click.style("For more information, visit:", fg="cyan")) + click.echo("https://github.com/raphaelmansuy/code2prompt") + click.echo() + click.echo("Created with โค๏ธ by Raphael Mansuy") + + \ No newline at end of file diff --git a/code2prompt/templates/analyze-code.j2 b/code2prompt/templates/analyze-code.j2 index 0ed6abc..8fc596f 100644 --- a/code2prompt/templates/analyze-code.j2 +++ b/code2prompt/templates/analyze-code.j2 @@ -75,6 +75,10 @@ You are a world-class software architect and code quality expert with decades of - Suggest collaborative approaches and tools to facilitate the improvement process - Consider the impact of proposed changes on the entire software development lifecycle +## Logging Level Configuration + +To configure the log level in the command line, you can use the `--log-level` option when running the `code2prompt` command. This option allows you to specify the desired logging level, such as DEBUG, INFO, WARNING, ERROR, or CRITICAL. + ## Reflection and Continuous Improvement After completing the analysis and plan: @@ -107,4 +111,4 @@ Remember, your goal is to provide a transformative yet pragmatic roadmap that el {% endfor %} - \ No newline at end of file + diff --git a/code2prompt/utils/add_line_numbers.py b/code2prompt/utils/add_line_numbers.py index bb4fdcf..ad16615 100644 --- a/code2prompt/utils/add_line_numbers.py +++ b/code2prompt/utils/add_line_numbers.py @@ -1,4 +1,13 @@ def add_line_numbers(code: str) -> str: + """ + Adds line numbers to each line of the given code. + + Args: + code (str): The code to add line numbers to. + + Returns: + str: The code with line numbers added. + """ lines = code.splitlines() max_line_number = len(lines) line_number_width = len(str(max_line_number)) diff --git a/code2prompt/utils/display_price_table.py b/code2prompt/utils/display_price_table.py new file mode 100644 index 0000000..847ca83 --- /dev/null +++ b/code2prompt/utils/display_price_table.py @@ -0,0 +1,115 @@ +from code2prompt.utils.price_calculator import calculate_prices, load_token_prices + +import click +from rich.console import Console +from rich.table import Table +from rich.panel import Panel +from rich.text import Text + + +def format_price(price: float, is_total: bool = False) -> str: + """ + Formats the given price as a string. + + Args: + price (float): The price to be formatted. + is_total (bool, optional): Indicates whether the price is a total. Defaults to False. + + Returns: + str: The formatted price as a string. + + """ + if is_total: + return f"${price:.6f}" + return f"${price /1_000 * 1_000_000 :.2f}" + + +def format_specific_price(price: float, tokens: int) -> str: + """ + Formats the specific price based on the given price and tokens. + + Args: + price (float): The price value. + tokens (int): The number of tokens. + + Returns: + str: The formatted specific price. + + """ + return f"${(price * tokens / 1_000):.6f}" + + +def display_price_table( + output_tokens: int, provider: str, model: str, token_count: int +): + """ + Displays a price table with estimated token prices based on the token count and provider's pricing model. + + Args: + output_tokens (int): The number of output tokens. + provider (str): The name of the provider. + model (str): The name of the model. + token_count (int): The number of input tokens. + + Returns: + None + """ + token_prices = load_token_prices() + if not token_prices: + return + price_results = calculate_prices( + token_prices, token_count, output_tokens, provider, model + ) + + if not price_results: + click.echo("Error: No matching provider or model found") + return + + console = Console() + + table = Table(show_header=True, header_style="bold magenta", expand=True) + table.add_column("Provider", style="cyan", no_wrap=True) + table.add_column("Model", style="green") + table.add_column("Input Price\n($/1M tokens)", justify="right", style="yellow") + table.add_column("Output Price\n($/1M tokens)", justify="right", style="yellow") + table.add_column("Tokens\nOut | In", justify="right", style="blue") + table.add_column("Price $\nOut | In", justify="right", style="magenta") + table.add_column("Total Cost", justify="right", style="red") + + for result in price_results: + input_price = format_price(result.price_input) + output_price = format_price(result.price_output) + specific_input_price = format_specific_price(result.price_input, token_count) + specific_output_price = format_specific_price( + result.price_output, output_tokens + ) + total_price = format_price(result.total_price, is_total=True) + + table.add_row( + result.provider_name, + result.model_name, + input_price, + output_price, + f"{token_count:,} | {output_tokens:,}", + f"{specific_input_price} | {specific_output_price}", + total_price, + ) + + title = Text("Estimated Token Prices", style="bold white on blue") + subtitle = Text("All prices in USD", style="italic") + + panel = Panel( + table, title=title, subtitle=subtitle, expand=False, border_style="blue" + ) + + console.print("\n") + console.print(panel) + console.print( + "\n๐Ÿ“Š Note: Prices are based on the token count and provider's pricing model." + ) + console.print( + "๐Ÿ’ก Tip: 'Price $ In | Out' shows the cost for the specific input and output tokens." + ) + console.print( + "โš ๏ธ This is an estimate based on OpenAI's Tokenizer implementation.\n" + ) diff --git a/code2prompt/utils/file_utils.py b/code2prompt/utils/file_utils.py new file mode 100644 index 0000000..aa1cbae --- /dev/null +++ b/code2prompt/utils/file_utils.py @@ -0,0 +1,134 @@ +# code2prompt/utils/file_utils.py + +from pathlib import Path +from typing import List, Dict, Any +import logging + +from code2prompt.config import Configuration +from code2prompt.utils.is_binary import is_binary +from code2prompt.utils.is_filtered import is_filtered +from code2prompt.utils.is_ignored import is_ignored +from code2prompt.utils.get_gitignore_patterns import get_gitignore_patterns +from code2prompt.core.process_file import process_file + +logger = logging.getLogger(__name__) + +def process_files(config: Configuration) -> List[Dict[str, Any]]: + """ + Process files based on the provided configuration. + + Args: + config (Configuration): Configuration object containing processing options. + + Returns: + List[Dict[str, Any]]: A list of dictionaries containing processed file data. + """ + files_data = [] + for path in config.path: + path = Path(path) + gitignore_patterns = get_gitignore_patterns( + path.parent if path.is_file() else path, + config.gitignore + ) + + if path.is_file(): + file_data = process_single_file(path, gitignore_patterns, config) + if file_data: + files_data.append(file_data) + else: + files_data.extend(process_directory(path, gitignore_patterns, config)) + + return files_data + +def process_single_file( + file_path: Path, + gitignore_patterns: List[str], + config: Configuration +) -> Dict[str, Any]: + """ + Process a single file if it meets the criteria. + + Args: + file_path (Path): Path to the file to process. + gitignore_patterns (List[str]): List of gitignore patterns. + config (Configuration): Configuration object containing processing options. + + Returns: + Dict[str, Any]: Processed file data if the file should be processed, None otherwise. + """ + if should_process_file(file_path, gitignore_patterns, file_path.parent, config): + return process_file( + file_path, + config.suppress_comments, + config.line_number, + config.no_codeblock + ) + return None + +def process_directory( + directory_path: Path, + gitignore_patterns: List[str], + config: Configuration +) -> List[Dict[str, Any]]: + """ + Process all files in a directory that meet the criteria. + + Args: + directory_path (Path): Path to the directory to process. + gitignore_patterns (List[str]): List of gitignore patterns. + config (Configuration): Configuration object containing processing options. + + Returns: + List[Dict[str, Any]]: List of processed file data for files that meet the criteria. + """ + files_data = [] + for file_path in directory_path.rglob("*"): + if file_path.is_file(): + file_data = process_single_file(file_path, gitignore_patterns, config) + if file_data: + files_data.append(file_data) + return files_data + +def should_process_file( + file_path: Path, + gitignore_patterns: List[str], + root_path: Path, + config: Configuration +) -> bool: + """ + Determine whether a file should be processed based on several criteria. + + Args: + file_path (Path): Path to the file to check. + gitignore_patterns (List[str]): List of gitignore patterns. + root_path (Path): Root path for relative path calculations. + config (Configuration): Configuration object containing processing options. + + Returns: + bool: True if the file should be processed, False otherwise. + """ + logger.debug(f"Checking if should process file: {file_path}") + + if not file_path.is_file(): + logger.debug(f"Skipping {file_path}: Not a file.") + return False + + if is_ignored(file_path, gitignore_patterns, root_path): + logger.debug(f"Skipping {file_path}: File is ignored based on gitignore patterns.") + return False + + if not is_filtered( + file_path, + config.filter, + config.exclude, + config.case_sensitive + ): + logger.debug(f"Skipping {file_path}: File does not meet filter criteria.") + return False + + if is_binary(file_path): + logger.debug(f"Skipping {file_path}: File is binary.") + return False + + logger.debug(f"Processing file: {file_path}") + return True \ No newline at end of file diff --git a/code2prompt/utils/include_loader.py b/code2prompt/utils/include_loader.py index cddc173..054a9a4 100644 --- a/code2prompt/utils/include_loader.py +++ b/code2prompt/utils/include_loader.py @@ -1,8 +1,9 @@ import os -from typing import List, Tuple, Callable, Optional, Set +from typing import List, Tuple, Callable from jinja2 import BaseLoader, TemplateNotFound import threading from contextlib import contextmanager +import jinja2 # Import jinja2 to resolve the undefined name error class CircularIncludeError(Exception): diff --git a/code2prompt/utils/is_filtered.py b/code2prompt/utils/is_filtered.py index fbdc5e2..f4da447 100644 --- a/code2prompt/utils/is_filtered.py +++ b/code2prompt/utils/is_filtered.py @@ -1,7 +1,17 @@ +""" +This module contains utility functions for filtering files based on include and exclude patterns. +""" + from pathlib import Path from fnmatch import fnmatch -def is_filtered(file_path: Path, include_pattern: str = "", exclude_pattern: str = "", case_sensitive: bool = False) -> bool: + +def is_filtered( + file_path: Path, + include_pattern: str = "", + exclude_pattern: str = "", + case_sensitive: bool = False, +) -> bool: """ Determine if a file should be filtered based on include and exclude patterns. @@ -34,7 +44,7 @@ def match_patterns(path: str, patterns: list) -> bool: # Prepare patterns def prepare_patterns(pattern): if isinstance(pattern, str): - return [p.strip().lower() for p in pattern.split(',') if p.strip()] + return [p.strip().lower() for p in pattern.split(",") if p.strip()] elif isinstance(pattern, (list, tuple)): return [str(p).strip().lower() for p in pattern if str(p).strip()] else: @@ -56,4 +66,4 @@ def prepare_patterns(pattern): return match_patterns(file_path_str, include_patterns) # If we reach here, there were no include patterns and the file wasn't excluded - return True \ No newline at end of file + return True diff --git a/code2prompt/utils/is_ignored.py b/code2prompt/utils/is_ignored.py index a4a4b20..6fb1706 100644 --- a/code2prompt/utils/is_ignored.py +++ b/code2prompt/utils/is_ignored.py @@ -2,8 +2,6 @@ from pathlib import Path -from pathlib import Path -from fnmatch import fnmatch def is_ignored(file_path: Path, gitignore_patterns: list, base_path: Path) -> bool: """ diff --git a/code2prompt/utils/logging_utils.py b/code2prompt/utils/logging_utils.py index 4fc09a9..f78861f 100644 --- a/code2prompt/utils/logging_utils.py +++ b/code2prompt/utils/logging_utils.py @@ -1,234 +1,65 @@ -# code2prompt/utils/logging_utils.py - -import sys import logging -from colorama import init, Fore, Style - -# Initialize colorama for cross-platform color support -init() - -class ColorfulFormatter(logging.Formatter): - """ - A custom formatter for logging messages that colors the output based on the log level - and prefixes each message with an emoji corresponding to its severity. - - Attributes: - COLORS (dict): Mapping of log levels to color codes. - EMOJIS (dict): Mapping of log levels to emojis. - - Methods: - format(record): Formats the given LogRecord. - """ - COLORS = { - 'DEBUG': Fore.CYAN, - 'INFO': Fore.GREEN, - 'WARNING': Fore.YELLOW, - 'ERROR': Fore.RED, - 'CRITICAL': Fore.MAGENTA - } - - EMOJIS = { - 'DEBUG': '๐Ÿ”', - 'INFO': 'โœจ', - 'WARNING': 'โš ๏ธ', - 'ERROR': '๐Ÿ’ฅ', - 'CRITICAL': '๐Ÿšจ' - } - - def format(self, record): - """ - Formats the given LogRecord. - - Args: - record (logging.LogRecord): The log record to format. - - Returns: - str: The formatted log message. - """ - color = self.COLORS.get(record.levelname, Fore.WHITE) - emoji = self.EMOJIS.get(record.levelname, '') - return f"{color}{emoji} {record.levelname}: {record.getMessage()}{Style.RESET_ALL}" - -def setup_logger(name='code2prompt', level=logging.INFO): - """ - Sets up and returns a logger with the specified name and logging level. - - Args: - name (str): The name of the logger. Defaults to 'code2prompt'. - level (int): The root logger level. Defaults to logging.INFO. - - Returns: - logging.Logger: The configured logger instance. - """ - local_logger = logging.getLogger(name) - local_logger.setLevel(level) - - # Only add handler if there are none to prevent duplicate logging - if not local_logger.handlers: - # Create handlers - c_handler = logging.StreamHandler(sys.stderr) - c_handler.setFormatter(ColorfulFormatter()) - - # Add handlers to the logger - local_logger.addHandler(c_handler) - - return local_logger - -# Create a global logger instance -logger = setup_logger() - -def log_debug(message): - """ - Logs a debug-level message. - - This function logs a message at the debug level, which is intended for detailed information, - typically of interest only when diagnosing problems. - - Args: - message (str): The message to log. - - Example: - log_debug("This is a debug message") - """ - logger.debug(message) - -def log_info(message): - """ - Logs an informational-level message. - - This function logs a message at the INFO level, which is used to provide general information - about the program's operation without implying any particular priority. - - Args: - message (str): The message to log. +import colorlog +import sys - Example: - log_info("Processing started") - """ - logger.info(message) +def setup_logger(level="INFO"): + """Set up the logger with the specified logging level.""" + logger = colorlog.getLogger() + logger.setLevel(level) -def log_warning(message): - """ - Logs a warning-level message. + # Create console handler + ch = colorlog.StreamHandler() + ch.setLevel(level) - This function logs a message at the WARNING level, indicating that something unexpected - happened, but did not stop the execution of the program. + # Create formatter with a more structured format + formatter = colorlog.ColoredFormatter( + '%(log_color)s[%(asctime)s] %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' + ) + ch.setFormatter(formatter) - Args: - message (str): The message to log as a warning. + # Add the handler to the logger + logger.addHandler(ch) - Example: - log_warning("An error occurred while processing the file") - """ - logger.warning(message) + return logger def log_error(message): - """ - Logs an error-level message. - - This function logs a message at the ERROR level, indicating that an error occurred - that prevented the program from continuing normally. - - Args: - message (str): The message to log as an error. - - Example: - log_error("Failed to process file due to permission issues") - """ + """Log an error message.""" + logger = logging.getLogger() logger.error(message) -def log_critical(message): - """ - Logs a critical-level message. - - This function logs a message at the CRITICAL level, indicating a severe error - that prevents the program from functioning correctly. - - Args: - message (str): The message to log as a critical error. - - Example: - log_critical("A critical system failure occurred") - """ - logger.critical(message) - -def log_success(message): - """ - Logs a success-level message. - - This function logs a message at the INFO level with a green color and a checkmark emoji, - indicating that an operation was successful. - - Args: - message (str): The message to log as a success. - - Example: - log_success("File processed successfully") - """ - logger.info(f"{Fore.GREEN}โœ… SUCCESS: {message}{Style.RESET_ALL}") - -def log_file_processed(file_path): - """ - Logs a message indicating that a file has been processed. - - This function logs a message at the INFO level, indicating that a specific file has been processed. - It uses a blue color and a file emoji for visual distinction. - - Args: - file_path (str): The path to the file that was processed. - - Example: - log_file_processed("/path/to/file.txt") - """ - logger.info(f"{Fore.BLUE}๐Ÿ“„ Processed: {file_path}{Style.RESET_ALL}") - -def log_token_count(count): - """ - Logs the total number of tokens processed. - - This function logs the total count of tokens processed by the application, - using a cyan color and a token emoji for visual distinction. - - Args: - count (int): The total number of tokens processed. - - Example: - log_token_count(5000) - """ - logger.info(f"{Fore.CYAN}๐Ÿ”ข Token count: {count}{Style.RESET_ALL}") - def log_output_created(output_path): - """ - Logs a message indicating that an output file has been created. - - This function logs a message at the INFO level, indicating that an output file has been successfully created. - It uses a green color and a folder emoji for visual distinction. - - Args: - output_path (str): The path to the output file that was created. - - Example: - log_output_created("/path/to/output/file.txt") - """ - logger.info(f"{Fore.GREEN}๐Ÿ“ Output file created: {output_path}{Style.RESET_ALL}") - -def log_clipboard_copy(success=True): - """ - Logs whether the content was successfully copied to the clipboard. - - This function logs a message indicating whether the content copying to the clipboard was successful or not. - It uses different emojis and colors depending on the success status. - - Args: - success (bool): Indicates whether the content was successfully copied to the clipboard. Defaults to True. + """Log a message indicating that an output file has been created.""" + logger = logging.getLogger() + logger.info(f"Output file created: {output_path}") - Examples: - log_clipboard_copy(True) - Logs: ๐Ÿ“‹ Content copied to clipboard - log_clipboard_copy(False) - Logs: ๐Ÿ“‹ Failed to copy content to clipboard - """ +def log_clipboard_copy(success): + """Log a message indicating whether copying to clipboard was successful.""" + logger = logging.getLogger() if success: - logger.info(f"{Fore.GREEN}๐Ÿ“‹ Content copied to clipboard{Style.RESET_ALL}") + success_message = "\033[92m๐Ÿ“‹ Content copied to clipboard successfully.\033[0m" + logger.info(success_message) + print(success_message, file=sys.stderr) else: - logger.warning(f"{Fore.YELLOW}๐Ÿ“‹ Failed to copy content to clipboard{Style.RESET_ALL}") \ No newline at end of file + logger.error("Failed to copy content to clipboard.") + print("Failed to copy content to clipboard.", file=sys.stderr) + +def log_token_count(token_count): + """Log the token count.""" + # Calculate the number of tokens in the input + token_count_message = f"\nโœจ \033[94mToken count: {token_count}\033[0m\n" # Added color for better display + print(token_count_message, file=sys.stderr) + +def log_token_prices(prices): + """Log the estimated token prices.""" + # Remove the unused logger variable + # logger = logging.getLogger() # Unused variable + header = "โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ Estimated Token Prices โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€" + print(header) + print("โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”“") + print("โ”ƒ โ”ƒ โ”ƒ Input Price โ”ƒ Output Price โ”ƒ Tokens โ”ƒ Price $ โ”ƒ โ”ƒ") + print("โ”ƒ Provider โ”ƒ Model โ”ƒ ($/1M tokens) โ”ƒ ($/1M tokens) โ”ƒ In | Out โ”ƒ In | Out โ”ƒ Total Cost โ”ƒ") + print("โ”กโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ•‡โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ•‡โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ•‡โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ•‡โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ•‡โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ•‡โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ฉ") + for price in prices: + print(f"โ”‚ {price['provider']: <11} โ”‚ {price['model']: <19} โ”‚ {price['input_price']: >13} โ”‚ {price['output_price']: >13} โ”‚ {price['tokens_in']: >13} | {price['tokens_out']: >13} โ”‚ {price['input_cost']: >12} | {price['output_cost']: >12} โ”‚ {price['total_cost']: >12} โ”‚") + print("โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜") diff --git a/code2prompt/utils/output_utils.py b/code2prompt/utils/output_utils.py new file mode 100644 index 0000000..4216493 --- /dev/null +++ b/code2prompt/utils/output_utils.py @@ -0,0 +1,127 @@ +# code2prompt/utils/output_utils.py + +from pathlib import Path +import logging +from typing import Dict, List, Optional + +from rich import print as rprint +from rich.panel import Panel +from rich.syntax import Syntax + +from code2prompt.config import Configuration + + +def generate_content(files_data: List[Dict], config: Configuration) -> str: + """ + Generate content based on the provided files data and configuration. + + Args: + files_data (List[Dict]): A list of dictionaries containing processed file data. + config (Configuration): Configuration object containing options. + + Returns: + str: The generated content as a string. + """ + if config.template: + return _process_template(files_data, config) + return _generate_markdown_content(files_data, config.no_codeblock) + + +def _process_template(files_data: List[Dict], config: Configuration) -> str: + """ + Process a Jinja2 template with the given files data and user inputs. + + Args: + files_data (List[Dict]): A list of dictionaries containing processed file data. + config (Configuration): Configuration object containing options. + + Returns: + str: The processed template content. + """ + from code2prompt.core.template_processor import ( + get_user_inputs, + load_template, + process_template, + ) + + template_content = load_template(config.template) + user_inputs = get_user_inputs(template_content) + return process_template(template_content, files_data, user_inputs, config.template) + + +def _generate_markdown_content(files_data: List[Dict], no_codeblock: bool) -> str: + """ + Generate markdown content from the provided files data. + + Args: + files_data (List[Dict]): A list of dictionaries containing file information and content. + no_codeblock (bool): Flag indicating whether to disable wrapping code inside markdown code blocks. + + Returns: + str: A Markdown-formatted string containing the table of contents and the file contents. + """ + table_of_contents = [f"- {file['path']}\n" for file in files_data] + content = [] + + for file in files_data: + file_info = ( + f"## File: {file['path']}\n\n" + f"- Extension: {file['extension']}\n" + f"- Language: {file['language']}\n" + f"- Size: {file['size']} bytes\n" + f"- Created: {file['created']}\n" + f"- Modified: {file['modified']}\n\n" + ) + + if no_codeblock: + file_code = f"### Code\n\n{file['content']}\n\n" + else: + file_code = f"### Code\n\n```{file['language']}\n{file['content']}\n```\n\n" + + content.append(file_info + file_code) + + return "# Table of Contents\n" + "".join(table_of_contents) + "\n" + "".join(content) + + +def write_output(content: str, output_path: Optional[Path], logger: logging.Logger): + """ + Write the generated content to a file or print it to the console. + + Args: + content (str): The content to be written or printed. + output_path (Optional[Path]): The path to the file where the content should be written. + If None, the content is printed to the console. + logger (logging.Logger): Logger instance for logging messages. + """ + if output_path: + try: + with output_path.open("w", encoding="utf-8") as output_file: + output_file.write(content) + logger.info(f"Output file created: {output_path}") + except IOError as e: + logger.error(f"Error writing to output file: {e}") + else: + rprint(Panel(Syntax(content, "markdown", theme="monokai", line_numbers=True))) + + +def log_token_count(count: int): + """ + Log the total number of tokens processed. + + Args: + count (int): The total number of tokens processed. + """ + rprint(f"[cyan]๐Ÿ”ข Token count: {count}[/cyan]") + + +def log_clipboard_copy(success: bool = True): + """ + Log whether the content was successfully copied to the clipboard. + + Args: + success (bool): Indicates whether the content was successfully copied to the clipboard. + """ + if success: + rprint("[green]๐Ÿ“‹ Content copied to clipboard[/green]") + else: + rprint("[yellow]๐Ÿ“‹ Failed to copy content to clipboard[/yellow]") \ No newline at end of file diff --git a/code2prompt/utils/price_calculator.py b/code2prompt/utils/price_calculator.py index 8daee85..15afbaa 100644 --- a/code2prompt/utils/price_calculator.py +++ b/code2prompt/utils/price_calculator.py @@ -1,24 +1,67 @@ import json +from typing import List, Optional from pathlib import Path +from functools import lru_cache +from pydantic import BaseModel, ConfigDict, field_validator -def load_token_prices(): + +class PriceModel(BaseModel): + price: Optional[float] = None + input_price: Optional[float] = None + output_price: Optional[float] = None + name: str + + @field_validator("price", "input_price", "output_price") + @classmethod + def check_price(cls, v: Optional[float]) -> Optional[float]: + if v is not None and v < 0: + raise ValueError("Price must be non-negative") + return v + + +class Provider(BaseModel): + name: str + models: List[PriceModel] + + +class TokenPrices(BaseModel): + providers: List[Provider] + + +class PriceResult(BaseModel): + provider_name: str + model_name: str + price_input: float + price_output: float + total_tokens: int + total_price: float + + model_config = ConfigDict(protected_namespaces=()) + + + + +@lru_cache(maxsize=1) +def load_token_prices() -> TokenPrices: """ Load token prices from a JSON file. Returns: - dict: A dictionary containing token prices. + TokenPrices: A Pydantic model containing token prices. Raises: RuntimeError: If there is an error loading the token prices. """ price_file = Path(__file__).parent.parent / "data" / "token_price.json" try: - with open(price_file, "r", encoding="utf-8") as f: - return json.load(f) + with price_file.open("r", encoding="utf-8") as f: + data = json.load(f) + return TokenPrices.model_validate(data) except (IOError, json.JSONDecodeError) as e: raise RuntimeError(f"Error loading token prices: {str(e)}") from e -def calculate_price(token_count, price_per_1000): + +def calculate_price(token_count: int, price_per_1000: float) -> float: """ Calculates the price based on the token count and price per 1000 tokens. @@ -31,57 +74,77 @@ def calculate_price(token_count, price_per_1000): """ return (token_count / 1_000) * price_per_1000 -def calculate_prices(token_prices, input_tokens, output_tokens, provider=None, model=None): + +def calculate_prices( + token_prices: TokenPrices, + input_tokens: int, + output_tokens: int, + provider: Optional[str] = None, + model: Optional[str] = None, +) -> List[PriceResult]: """ Calculate the prices for a given number of input and output tokens based on token prices. Args: - token_prices (dict): A dictionary containing token prices for different providers and models. + token_prices (TokenPrices): A Pydantic model containing token prices for different providers and models. input_tokens (int): The number of input tokens. output_tokens (int): The number of output tokens. - provider (str, optional): The name of the provider. If specified, only prices for the specified provider will be calculated. Defaults to None. - model (str, optional): The name of the model. If specified, only prices for the specified model will be calculated. Defaults to None. + provider (str, optional): The name of the provider. If specified, only prices for the specified provider will be calculated. + model (str, optional): The name of the model. If specified, only prices for the specified model will be calculated. Returns: - list: A list of tuples containing the provider name, model name, price per token, total tokens, and total price for each calculation. - + List[PriceResult]: A list of PriceResult objects containing the calculation results. """ -def calculate_prices(token_prices, input_tokens, output_tokens, provider=None, model=None): results = [] - - for p in token_prices["providers"]: - if provider and p["name"] != provider: + total_tokens = input_tokens + output_tokens + + for provider_data in token_prices.providers: + if provider and provider_data.name.lower() != provider.lower(): continue - - for m in p["models"]: - if model and m["name"] != model: + + for model_data in provider_data.models: + if model and model_data.name.lower() != model.lower(): continue - - total_tokens = input_tokens + output_tokens - - if "price" in m: - # Single price for both input and output tokens - price = m["price"] - total_price = (price * total_tokens) / 1000 - price_info = f"${price:.10f}" - elif "input_price" in m and "output_price" in m: - # Separate prices for input and output tokens - input_price = m["input_price"] - output_price = m["output_price"] - total_price = ((input_price * input_tokens) + (output_price * output_tokens)) / 1000 - price_info = f"${input_price:.10f} (input) / ${output_price:.10f} (output)" + + if model_data.price is not None: + price_input = model_data.price + price_output = model_data.price + total_price = calculate_price(total_tokens, model_data.price) + elif ( + model_data.input_price is not None + and model_data.output_price is not None + ): + price_input = model_data.input_price + price_output = model_data.output_price + total_price = calculate_price( + input_tokens, price_input + ) + calculate_price(output_tokens, price_output) else: - # Skip models with unexpected price structure continue - - result = ( - p["name"], # Provider name - m["name"], # Model name - price_info, # Price information - total_tokens, # Total number of tokens - f"${total_price:.10f}" # Total price + + results.append( + PriceResult( + provider_name=provider_data.name, + model_name=model_data.name, + price_input=price_input, + price_output=price_output, + total_tokens=total_tokens, + total_price=total_price, + ) ) - - results.append(result) - - return results \ No newline at end of file + + return results + + +if __name__ == "__main__": + # Example usage + token_prices = load_token_prices() + results = calculate_prices(token_prices, input_tokens=100, output_tokens=50) + for result in results: + print(f"Provider: {result.provider_name}") + print(f"Model: {result.model_name}") + print(f"Input Price: ${result.price_input:.10f}") + print(f"Output Price: ${result.price_output:.10f}") + print(f"Total Tokens: {result.total_tokens}") + print(f"Total Price: ${result.total_price:.10f}") + print("---") \ No newline at end of file diff --git a/code2prompt/utils/should_process_file.py b/code2prompt/utils/should_process_file.py index a941e5f..0202f5e 100644 --- a/code2prompt/utils/should_process_file.py +++ b/code2prompt/utils/should_process_file.py @@ -1,4 +1,13 @@ +""" + +This module contains the function to determine +if a file should be processed based on several criteria. + +""" + import logging +from pathlib import Path +from typing import List # Add this import from code2prompt.utils.is_binary import is_binary from code2prompt.utils.is_filtered import is_filtered from code2prompt.utils.is_ignored import is_ignored @@ -6,34 +15,45 @@ logger = logging.getLogger(__name__) -def should_process_file(file_path, gitignore_patterns, root_path, options): +def should_process_file( + file_path: Path, + gitignore_patterns: List[str], # List is now defined + root_path: Path, + filter_patterns: str, ## comma separated list of patterns + exclude_patterns: str, ## comma separated list of patterns + case_sensitive: bool, +) -> bool: """ Determine whether a file should be processed based on several criteria. """ - logger.debug(f"Checking if should process file: {file_path}") + logger.debug( + "Checking if should process file: %s", file_path + ) # Use lazy % formatting if not file_path.is_file(): - logger.debug(f"Skipping {file_path}: Not a file.") + logger.debug("Skipping %s: Not a file.", file_path) # Use lazy % formatting return False if is_ignored(file_path, gitignore_patterns, root_path): logger.debug( - f"Skipping {file_path}: File is ignored based on gitignore patterns." + "Skipping %s: File is ignored based on gitignore patterns.", file_path ) return False if not is_filtered( - file_path, - options.get("filter", ""), - options.get("exclude", ""), - options.get("case_sensitive", False), + file_path=file_path, + include_pattern=filter_patterns, + exclude_pattern=exclude_patterns, + case_sensitive=case_sensitive, ): - logger.debug(f"Skipping {file_path}: File does not meet filter criteria.") + logger.debug( + "Skipping %s: File does not meet filter criteria.", file_path + ) # Use lazy % formatting return False if is_binary(file_path): - logger.debug(f"Skipping {file_path}: File is binary.") + logger.debug("Skipping %s: File is binary.", file_path) # Use lazy % formatting return False - logger.debug(f"Processing file: {file_path}") + logger.debug("Processing file: %s", file_path) # Use lazy % formatting return True diff --git a/code2prompt/version.py b/code2prompt/version.py new file mode 100644 index 0000000..36f89e5 --- /dev/null +++ b/code2prompt/version.py @@ -0,0 +1 @@ +VERSION="0.7.0" \ No newline at end of file diff --git a/docs/demo01/doc01.md b/docs/demo01/doc01.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/demo01/doc02.md b/docs/demo01/doc02.md new file mode 100644 index 0000000..e69de29 diff --git a/poetry.lock b/poetry.lock index 7db9883..a899713 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,19 @@ # This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +[[package]] +name = "annotated-types" +version = "0.7.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, + {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.9\""} + [[package]] name = "appnope" version = "0.1.4" @@ -11,6 +25,20 @@ files = [ {file = "appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee"}, ] +[[package]] +name = "astroid" +version = "3.2.4" +description = "An abstract syntax tree for Python with inference support." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "astroid-3.2.4-py3-none-any.whl", hash = "sha256:413658a61eeca6202a59231abb473f932038fbcbf1666587f66d482083413a25"}, + {file = "astroid-3.2.4.tar.gz", hash = "sha256:0e14202810b30da1b735827f78f5157be2bbd4a7a59b7707ca0bfc2fb4c0063a"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} + [[package]] name = "asttokens" version = "2.4.1" @@ -239,6 +267,23 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "colorlog" +version = "6.8.2" +description = "Add colours to the output of Python's logging module." +optional = false +python-versions = ">=3.6" +files = [ + {file = "colorlog-6.8.2-py3-none-any.whl", hash = "sha256:4dcbb62368e2800cb3c5abd348da7e53f6c362dda502ec27c560b2e58a66bd33"}, + {file = "colorlog-6.8.2.tar.gz", hash = "sha256:3e3e079a41feb5a1b64f978b5ea4f46040a94f11f0e8bbb8261e3dbbeca64d44"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} + +[package.extras] +development = ["black", "flake8", "mypy", "pytest", "types-colorama"] + [[package]] name = "comm" version = "0.2.2" @@ -298,6 +343,21 @@ files = [ {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, ] +[[package]] +name = "dill" +version = "0.3.8" +description = "serialize all of Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"}, + {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"}, +] + +[package.extras] +graph = ["objgraph (>=1.7.2)"] +profile = ["gprof2dot (>=2022.7.29)"] + [[package]] name = "exceptiongroup" version = "1.2.0" @@ -439,6 +499,20 @@ qtconsole = ["qtconsole"] test = ["pytest (<7.1)", "pytest-asyncio", "testpath"] test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.21)", "pandas", "pytest (<7.1)", "pytest-asyncio", "testpath", "trio"] +[[package]] +name = "isort" +version = "5.13.2" +description = "A Python utility / library to sort Python imports." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, + {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, +] + +[package.extras] +colors = ["colorama (>=0.4.6)"] + [[package]] name = "jedi" version = "0.19.1" @@ -625,6 +699,17 @@ files = [ [package.dependencies] traitlets = "*" +[[package]] +name = "mccabe" +version = "0.7.0" +description = "McCabe checker, plugin for flake8" +optional = false +python-versions = ">=3.6" +files = [ + {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, + {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, +] + [[package]] name = "mdurl" version = "0.1.2" @@ -808,6 +893,129 @@ files = [ {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, ] +[[package]] +name = "pydantic" +version = "2.8.2" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic-2.8.2-py3-none-any.whl", hash = "sha256:73ee9fddd406dc318b885c7a2eab8a6472b68b8fb5ba8150949fc3db939f23c8"}, + {file = "pydantic-2.8.2.tar.gz", hash = "sha256:6f62c13d067b0755ad1c21a34bdd06c0c12625a22b0fc09c6b149816604f7c2a"}, +] + +[package.dependencies] +annotated-types = ">=0.4.0" +pydantic-core = "2.20.1" +typing-extensions = [ + {version = ">=4.6.1", markers = "python_version < \"3.13\""}, + {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, +] + +[package.extras] +email = ["email-validator (>=2.0.0)"] + +[[package]] +name = "pydantic-core" +version = "2.20.1" +description = "Core functionality for Pydantic validation and serialization" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_core-2.20.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3acae97ffd19bf091c72df4d726d552c473f3576409b2a7ca36b2f535ffff4a3"}, + {file = "pydantic_core-2.20.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:41f4c96227a67a013e7de5ff8f20fb496ce573893b7f4f2707d065907bffdbd6"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f239eb799a2081495ea659d8d4a43a8f42cd1fe9ff2e7e436295c38a10c286a"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:53e431da3fc53360db73eedf6f7124d1076e1b4ee4276b36fb25514544ceb4a3"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f1f62b2413c3a0e846c3b838b2ecd6c7a19ec6793b2a522745b0869e37ab5bc1"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d41e6daee2813ecceea8eda38062d69e280b39df793f5a942fa515b8ed67953"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d482efec8b7dc6bfaedc0f166b2ce349df0011f5d2f1f25537ced4cfc34fd98"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e93e1a4b4b33daed65d781a57a522ff153dcf748dee70b40c7258c5861e1768a"}, + {file = "pydantic_core-2.20.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e7c4ea22b6739b162c9ecaaa41d718dfad48a244909fe7ef4b54c0b530effc5a"}, + {file = "pydantic_core-2.20.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4f2790949cf385d985a31984907fecb3896999329103df4e4983a4a41e13e840"}, + {file = "pydantic_core-2.20.1-cp310-none-win32.whl", hash = "sha256:5e999ba8dd90e93d57410c5e67ebb67ffcaadcea0ad973240fdfd3a135506250"}, + {file = "pydantic_core-2.20.1-cp310-none-win_amd64.whl", hash = "sha256:512ecfbefef6dac7bc5eaaf46177b2de58cdf7acac8793fe033b24ece0b9566c"}, + {file = "pydantic_core-2.20.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:d2a8fa9d6d6f891f3deec72f5cc668e6f66b188ab14bb1ab52422fe8e644f312"}, + {file = "pydantic_core-2.20.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:175873691124f3d0da55aeea1d90660a6ea7a3cfea137c38afa0a5ffabe37b88"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37eee5b638f0e0dcd18d21f59b679686bbd18917b87db0193ae36f9c23c355fc"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25e9185e2d06c16ee438ed39bf62935ec436474a6ac4f9358524220f1b236e43"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:150906b40ff188a3260cbee25380e7494ee85048584998c1e66df0c7a11c17a6"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ad4aeb3e9a97286573c03df758fc7627aecdd02f1da04516a86dc159bf70121"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3f3ed29cd9f978c604708511a1f9c2fdcb6c38b9aae36a51905b8811ee5cbf1"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b0dae11d8f5ded51699c74d9548dcc5938e0804cc8298ec0aa0da95c21fff57b"}, + {file = "pydantic_core-2.20.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:faa6b09ee09433b87992fb5a2859efd1c264ddc37280d2dd5db502126d0e7f27"}, + {file = "pydantic_core-2.20.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9dc1b507c12eb0481d071f3c1808f0529ad41dc415d0ca11f7ebfc666e66a18b"}, + {file = "pydantic_core-2.20.1-cp311-none-win32.whl", hash = "sha256:fa2fddcb7107e0d1808086ca306dcade7df60a13a6c347a7acf1ec139aa6789a"}, + {file = "pydantic_core-2.20.1-cp311-none-win_amd64.whl", hash = "sha256:40a783fb7ee353c50bd3853e626f15677ea527ae556429453685ae32280c19c2"}, + {file = "pydantic_core-2.20.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:595ba5be69b35777474fa07f80fc260ea71255656191adb22a8c53aba4479231"}, + {file = "pydantic_core-2.20.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a4f55095ad087474999ee28d3398bae183a66be4823f753cd7d67dd0153427c9"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9aa05d09ecf4c75157197f27cdc9cfaeb7c5f15021c6373932bf3e124af029f"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e97fdf088d4b31ff4ba35db26d9cc472ac7ef4a2ff2badeabf8d727b3377fc52"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bc633a9fe1eb87e250b5c57d389cf28998e4292336926b0b6cdaee353f89a237"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d573faf8eb7e6b1cbbcb4f5b247c60ca8be39fe2c674495df0eb4318303137fe"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26dc97754b57d2fd00ac2b24dfa341abffc380b823211994c4efac7f13b9e90e"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:33499e85e739a4b60c9dac710c20a08dc73cb3240c9a0e22325e671b27b70d24"}, + {file = "pydantic_core-2.20.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bebb4d6715c814597f85297c332297c6ce81e29436125ca59d1159b07f423eb1"}, + {file = "pydantic_core-2.20.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:516d9227919612425c8ef1c9b869bbbee249bc91912c8aaffb66116c0b447ebd"}, + {file = "pydantic_core-2.20.1-cp312-none-win32.whl", hash = "sha256:469f29f9093c9d834432034d33f5fe45699e664f12a13bf38c04967ce233d688"}, + {file = "pydantic_core-2.20.1-cp312-none-win_amd64.whl", hash = "sha256:035ede2e16da7281041f0e626459bcae33ed998cca6a0a007a5ebb73414ac72d"}, + {file = "pydantic_core-2.20.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:0827505a5c87e8aa285dc31e9ec7f4a17c81a813d45f70b1d9164e03a813a686"}, + {file = "pydantic_core-2.20.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19c0fa39fa154e7e0b7f82f88ef85faa2a4c23cc65aae2f5aea625e3c13c735a"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa223cd1e36b642092c326d694d8bf59b71ddddc94cdb752bbbb1c5c91d833b"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c336a6d235522a62fef872c6295a42ecb0c4e1d0f1a3e500fe949415761b8a19"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7eb6a0587eded33aeefea9f916899d42b1799b7b14b8f8ff2753c0ac1741edac"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:70c8daf4faca8da5a6d655f9af86faf6ec2e1768f4b8b9d0226c02f3d6209703"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9fa4c9bf273ca41f940bceb86922a7667cd5bf90e95dbb157cbb8441008482c"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:11b71d67b4725e7e2a9f6e9c0ac1239bbc0c48cce3dc59f98635efc57d6dac83"}, + {file = "pydantic_core-2.20.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:270755f15174fb983890c49881e93f8f1b80f0b5e3a3cc1394a255706cabd203"}, + {file = "pydantic_core-2.20.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c81131869240e3e568916ef4c307f8b99583efaa60a8112ef27a366eefba8ef0"}, + {file = "pydantic_core-2.20.1-cp313-none-win32.whl", hash = "sha256:b91ced227c41aa29c672814f50dbb05ec93536abf8f43cd14ec9521ea09afe4e"}, + {file = "pydantic_core-2.20.1-cp313-none-win_amd64.whl", hash = "sha256:65db0f2eefcaad1a3950f498aabb4875c8890438bc80b19362cf633b87a8ab20"}, + {file = "pydantic_core-2.20.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:4745f4ac52cc6686390c40eaa01d48b18997cb130833154801a442323cc78f91"}, + {file = "pydantic_core-2.20.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a8ad4c766d3f33ba8fd692f9aa297c9058970530a32c728a2c4bfd2616d3358b"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41e81317dd6a0127cabce83c0c9c3fbecceae981c8391e6f1dec88a77c8a569a"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:04024d270cf63f586ad41fff13fde4311c4fc13ea74676962c876d9577bcc78f"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eaad4ff2de1c3823fddf82f41121bdf453d922e9a238642b1dedb33c4e4f98ad"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:26ab812fa0c845df815e506be30337e2df27e88399b985d0bb4e3ecfe72df31c"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c5ebac750d9d5f2706654c638c041635c385596caf68f81342011ddfa1e5598"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2aafc5a503855ea5885559eae883978c9b6d8c8993d67766ee73d82e841300dd"}, + {file = "pydantic_core-2.20.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4868f6bd7c9d98904b748a2653031fc9c2f85b6237009d475b1008bfaeb0a5aa"}, + {file = "pydantic_core-2.20.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:aa2f457b4af386254372dfa78a2eda2563680d982422641a85f271c859df1987"}, + {file = "pydantic_core-2.20.1-cp38-none-win32.whl", hash = "sha256:225b67a1f6d602de0ce7f6c1c3ae89a4aa25d3de9be857999e9124f15dab486a"}, + {file = "pydantic_core-2.20.1-cp38-none-win_amd64.whl", hash = "sha256:6b507132dcfc0dea440cce23ee2182c0ce7aba7054576efc65634f080dbe9434"}, + {file = "pydantic_core-2.20.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b03f7941783b4c4a26051846dea594628b38f6940a2fdc0df00b221aed39314c"}, + {file = "pydantic_core-2.20.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1eedfeb6089ed3fad42e81a67755846ad4dcc14d73698c120a82e4ccf0f1f9f6"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:635fee4e041ab9c479e31edda27fcf966ea9614fff1317e280d99eb3e5ab6fe2"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:77bf3ac639c1ff567ae3b47f8d4cc3dc20f9966a2a6dd2311dcc055d3d04fb8a"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ed1b0132f24beeec5a78b67d9388656d03e6a7c837394f99257e2d55b461611"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c6514f963b023aeee506678a1cf821fe31159b925c4b76fe2afa94cc70b3222b"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10d4204d8ca33146e761c79f83cc861df20e7ae9f6487ca290a97702daf56006"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2d036c7187b9422ae5b262badb87a20a49eb6c5238b2004e96d4da1231badef1"}, + {file = "pydantic_core-2.20.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9ebfef07dbe1d93efb94b4700f2d278494e9162565a54f124c404a5656d7ff09"}, + {file = "pydantic_core-2.20.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6b9d9bb600328a1ce523ab4f454859e9d439150abb0906c5a1983c146580ebab"}, + {file = "pydantic_core-2.20.1-cp39-none-win32.whl", hash = "sha256:784c1214cb6dd1e3b15dd8b91b9a53852aed16671cc3fbe4786f4f1db07089e2"}, + {file = "pydantic_core-2.20.1-cp39-none-win_amd64.whl", hash = "sha256:d2fe69c5434391727efa54b47a1e7986bb0186e72a41b203df8f5b0a19a4f669"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a45f84b09ac9c3d35dfcf6a27fd0634d30d183205230a0ebe8373a0e8cfa0906"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d02a72df14dfdbaf228424573a07af10637bd490f0901cee872c4f434a735b94"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d2b27e6af28f07e2f195552b37d7d66b150adbaa39a6d327766ffd695799780f"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:084659fac3c83fd674596612aeff6041a18402f1e1bc19ca39e417d554468482"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:242b8feb3c493ab78be289c034a1f659e8826e2233786e36f2893a950a719bb6"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:38cf1c40a921d05c5edc61a785c0ddb4bed67827069f535d794ce6bcded919fc"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e0bbdd76ce9aa5d4209d65f2b27fc6e5ef1312ae6c5333c26db3f5ade53a1e99"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:254ec27fdb5b1ee60684f91683be95e5133c994cc54e86a0b0963afa25c8f8a6"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:407653af5617f0757261ae249d3fba09504d7a71ab36ac057c938572d1bc9331"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:c693e916709c2465b02ca0ad7b387c4f8423d1db7b4649c551f27a529181c5ad"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b5ff4911aea936a47d9376fd3ab17e970cc543d1b68921886e7f64bd28308d1"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:177f55a886d74f1808763976ac4efd29b7ed15c69f4d838bbd74d9d09cf6fa86"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:964faa8a861d2664f0c7ab0c181af0bea66098b1919439815ca8803ef136fc4e"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4dd484681c15e6b9a977c785a345d3e378d72678fd5f1f3c0509608da24f2ac0"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f6d6cff3538391e8486a431569b77921adfcdef14eb18fbf19b7c0a5294d4e6a"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a6d511cc297ff0883bc3708b465ff82d7560193169a8b93260f74ecb0a5e08a7"}, + {file = "pydantic_core-2.20.1.tar.gz", hash = "sha256:26ca695eeee5f9f1aeeb211ffc12f10bcb6f71e2989988fda61dabd65db878d4"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + [[package]] name = "pygments" version = "2.17.2" @@ -823,6 +1031,36 @@ files = [ plugins = ["importlib-metadata"] windows-terminal = ["colorama (>=0.4.6)"] +[[package]] +name = "pylint" +version = "3.2.6" +description = "python code static checker" +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "pylint-3.2.6-py3-none-any.whl", hash = "sha256:03c8e3baa1d9fb995b12c1dbe00aa6c4bcef210c2a2634374aedeb22fb4a8f8f"}, + {file = "pylint-3.2.6.tar.gz", hash = "sha256:a5d01678349454806cff6d886fb072294f56a58c4761278c97fb557d708e1eb3"}, +] + +[package.dependencies] +astroid = ">=3.2.4,<=3.3.0-dev0" +colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} +dill = [ + {version = ">=0.2", markers = "python_version < \"3.11\""}, + {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, + {version = ">=0.3.6", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, +] +isort = ">=4.2.5,<5.13.0 || >5.13.0,<6" +mccabe = ">=0.6,<0.8" +platformdirs = ">=2.2.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +tomlkit = ">=0.10.1" +typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""} + +[package.extras] +spelling = ["pyenchant (>=3.2,<4.0)"] +testutils = ["gitpython (>3)"] + [[package]] name = "pyperclip" version = "1.9.0" @@ -1120,6 +1358,33 @@ typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9 [package.extras] jupyter = ["ipywidgets (>=7.5.1,<9)"] +[[package]] +name = "ruff" +version = "0.5.5" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.5.5-py3-none-linux_armv6l.whl", hash = "sha256:605d589ec35d1da9213a9d4d7e7a9c761d90bba78fc8790d1c5e65026c1b9eaf"}, + {file = "ruff-0.5.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:00817603822a3e42b80f7c3298c8269e09f889ee94640cd1fc7f9329788d7bf8"}, + {file = "ruff-0.5.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:187a60f555e9f865a2ff2c6984b9afeffa7158ba6e1eab56cb830404c942b0f3"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe26fc46fa8c6e0ae3f47ddccfbb136253c831c3289bba044befe68f467bfb16"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4ad25dd9c5faac95c8e9efb13e15803cd8bbf7f4600645a60ffe17c73f60779b"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f70737c157d7edf749bcb952d13854e8f745cec695a01bdc6e29c29c288fc36e"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:cfd7de17cef6ab559e9f5ab859f0d3296393bc78f69030967ca4d87a541b97a0"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a09b43e02f76ac0145f86a08e045e2ea452066f7ba064fd6b0cdccb486f7c3e7"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d0b856cb19c60cd40198be5d8d4b556228e3dcd545b4f423d1ad812bfdca5884"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3687d002f911e8a5faf977e619a034d159a8373514a587249cc00f211c67a091"}, + {file = "ruff-0.5.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ac9dc814e510436e30d0ba535f435a7f3dc97f895f844f5b3f347ec8c228a523"}, + {file = "ruff-0.5.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:af9bdf6c389b5add40d89b201425b531e0a5cceb3cfdcc69f04d3d531c6be74f"}, + {file = "ruff-0.5.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d40a8533ed545390ef8315b8e25c4bb85739b90bd0f3fe1280a29ae364cc55d8"}, + {file = "ruff-0.5.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:cab904683bf9e2ecbbe9ff235bfe056f0eba754d0168ad5407832928d579e7ab"}, + {file = "ruff-0.5.5-py3-none-win32.whl", hash = "sha256:696f18463b47a94575db635ebb4c178188645636f05e934fdf361b74edf1bb2d"}, + {file = "ruff-0.5.5-py3-none-win_amd64.whl", hash = "sha256:50f36d77f52d4c9c2f1361ccbfbd09099a1b2ea5d2b2222c586ab08885cf3445"}, + {file = "ruff-0.5.5-py3-none-win_arm64.whl", hash = "sha256:3191317d967af701f1b73a31ed5788795936e423b7acce82a2b63e26eb3e89d6"}, + {file = "ruff-0.5.5.tar.gz", hash = "sha256:cc5516bdb4858d972fbc31d246bdb390eab8df1a26e2353be2dbc0c2d7f5421a"}, +] + [[package]] name = "six" version = "1.16.0" @@ -1227,6 +1492,17 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "tomlkit" +version = "0.13.0" +description = "Style preserving TOML library" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tomlkit-0.13.0-py3-none-any.whl", hash = "sha256:7075d3042d03b80f603482d69bf0c8f345c2b30e41699fd8883227f89972b264"}, + {file = "tomlkit-0.13.0.tar.gz", hash = "sha256:08ad192699734149f5b97b45f1f18dad7eb1b6d16bc72ad0c2335772650d7b72"}, +] + [[package]] name = "tornado" version = "6.4.1" @@ -1284,13 +1560,13 @@ test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0, [[package]] name = "typing-extensions" -version = "4.10.0" +version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, - {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] [[package]] @@ -1310,6 +1586,20 @@ h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "vulture" +version = "2.11" +description = "Find dead code" +optional = false +python-versions = ">=3.8" +files = [ + {file = "vulture-2.11-py2.py3-none-any.whl", hash = "sha256:12d745f7710ffbf6aeb8279ba9068a24d4e52e8ed333b8b044035c9d6b823aba"}, + {file = "vulture-2.11.tar.gz", hash = "sha256:f0fbb60bce6511aad87ee0736c502456737490a82d919a44e6d92262cb35f1c2"}, +] + +[package.dependencies] +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} + [[package]] name = "wcwidth" version = "0.2.13" @@ -1339,4 +1629,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = "^3.8,<4.0" -content-hash = "5890e3cfe27cc6096ce82354b8554c46e3e54ffe762d962ab702513f0168bfff" +content-hash = "768e1ed3cf01cc50b8de9c62d2289d82b2d90023beb706659efa0244ac80580c" diff --git a/pyproject.toml b/pyproject.toml index 3869df6..87e4b59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "code2prompt" -version = "0.6.13" +version = "0.7.1" description = "A tool to convert code snippets into AI prompts for documentation or explanation purposes." authors = ["Raphael MANSUY "] license = "MIT" @@ -27,23 +27,30 @@ python = "^3.8,<4.0" rich = "^13.7.1" # For rich text and beautiful formatting click = "^8.1.7" # For creating beautiful command line interfaces jinja2 = "^3.1.4" # For template rendering -prompt-toolkit = "^3.0.47" # For building powerful interactive command line applications tiktoken = "^0.7.0" # For tokenization tasks pyperclip = "^1.9.0" # For clipboard operations colorama = "^0.4.6" # For colored terminal text output tqdm = "^4.66.4" tabulate = "^0.9.0" +pydantic = "^2.8.2" +prompt-toolkit = "^3.0.47" +colorlog = "^6.8.2" [tool.poetry.scripts] -code2prompt = "code2prompt.main:create_markdown_file" +code2prompt = "code2prompt.main:cli" + [tool.poetry.group.dev.dependencies] pytest = "^8.1.1" ipykernel = "^6.29.4" +vulture = "^2.11" +pylint = "^3.2.6" +ruff = "^0.5.5" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.setuptools] -package-data = {"code2prompt" = ["templates/**/*","data/**/*"]} \ No newline at end of file +package-data = {"code2prompt" = ["templates/**/*","data/**/*"]} + diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..d28e492 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,77 @@ +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", +] + +# Same as Black. +line-length = 88 +indent-width = 4 + +# Assume Python 3.8 +target-version = "py38" + +[lint] +# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. +# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or +# McCabe complexity (`C901`) by default. +select = ["E4", "E7", "E9", "F"] +ignore = [] + +# Allow fix for all enabled rules (when `--fix`) is provided. +fixable = ["ALL"] +unfixable = [] + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +[format] +# Like Black, use double quotes for strings. +quote-style = "double" + +# Like Black, indent with spaces, rather than tabs. +indent-style = "space" + +# Like Black, respect magic trailing commas. +skip-magic-trailing-comma = false + +# Like Black, automatically detect the appropriate line ending. +line-ending = "auto" + +# Enable auto-formatting of code examples in docstrings. Markdown, +# reStructuredText code/literal blocks and doctests are all supported. +# +# This is currently disabled by default, but it is planned for this +# to be opt-out in the future. +docstring-code-format = false + +# Set the line length limit used when formatting code snippets in +# docstrings. +# +# This only has an effect when the `docstring-code-format` setting is +# enabled. +docstring-code-line-length = "dynamic" diff --git a/script/detect_dead_code.sh b/script/detect_dead_code.sh new file mode 100755 index 0000000..498ef89 --- /dev/null +++ b/script/detect_dead_code.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +poetry run vulture ../code2prompt \ No newline at end of file diff --git a/tests/test_analyze.py b/tests/test_analyze.py index fde37f9..1adac23 100644 --- a/tests/test_analyze.py +++ b/tests/test_analyze.py @@ -1,10 +1,9 @@ import pytest from click.testing import CliRunner from code2prompt.main import create_markdown_file -from code2prompt.utils.analyzer import analyze_codebase, format_flat_output, format_tree_output, get_extension_list +from code2prompt.utils.analyzer import analyze_codebase, format_flat_output, get_extension_list from pathlib import Path import tempfile -import os @pytest.fixture def temp_codebase(): diff --git a/tests/test_create_template_directory.py b/tests/test_create_template_directory.py index aad97f3..d1d8611 100644 --- a/tests/test_create_template_directory.py +++ b/tests/test_create_template_directory.py @@ -1,8 +1,7 @@ import pytest from pathlib import Path import tempfile -import shutil -from unittest.mock import patch, MagicMock +from unittest.mock import patch from code2prompt.utils.create_template_directory import create_templates_directory @pytest.fixture diff --git a/tests/test_include_loader.py b/tests/test_include_loader.py index 65f1f64..fcb38d4 100644 --- a/tests/test_include_loader.py +++ b/tests/test_include_loader.py @@ -1,7 +1,6 @@ import pytest from jinja2 import Environment, TemplateNotFound -from code2prompt.utils.include_loader import IncludeLoader, CircularIncludeError -import os +from code2prompt.utils.include_loader import IncludeLoader @pytest.fixture def temp_dir(tmp_path): diff --git a/tests/test_is_filtered.py b/tests/test_is_filtered.py index ae33b9d..2380eb0 100644 --- a/tests/test_is_filtered.py +++ b/tests/test_is_filtered.py @@ -2,40 +2,54 @@ from pathlib import Path from code2prompt.utils.is_filtered import is_filtered -@pytest.mark.parametrize("file_path, include_pattern, exclude_pattern, case_sensitive, expected", [ - (Path("file.txt"), "", "", False, True), - (Path("file.py"), "*.py", "", False, True), - (Path("file.txt"), "*.py", "", False, False), - (Path("file.py"), "", "*.py", False, False), - (Path("file.txt"), "", "*.py", False, True), - (Path("file.py"), "*.py,*.txt", "test_*.py", False, True), - (Path("test_file.py"), "*.py,*.txt", "test_*.py", False, False), - (Path("File.PY"), "*.py", "", True, False), - (Path("File.PY"), "*.py", "", False, True), -# (Path("test/file.py"), "**/test/*.py", "", False, True), - (Path("src/file.py"), "**/test/*.py", "", False, False), - (Path("file.txt"), "*.py,*.js,*.txt", "", False, True), - (Path("file.md"), "*.py,*.js,*.txt", "", False, False), - (Path("test_file.py"), "*.py", "test_*.py", False, False), - (Path(".hidden_file"), "*", "", False, True), - (Path("file_without_extension"), "", "*.*", False, True), - (Path("deeply/nested/directory/file.txt"), "**/*.txt", "", False, True), - (Path("file.txt.bak"), "", "*.bak", False, False), -]) -def test_is_filtered(file_path, include_pattern, exclude_pattern, case_sensitive, expected): - assert is_filtered(file_path, include_pattern, exclude_pattern, case_sensitive) == expected + +@pytest.mark.parametrize( + "file_path, include_pattern, exclude_pattern, case_sensitive, expected", + [ + (Path("file.txt"), "", "", False, True), + (Path("file.py"), "*.py", "", False, True), + (Path("file.txt"), "*.py", "", False, False), + (Path("file.py"), "", "*.py", False, False), + (Path("file.txt"), "", "*.py", False, True), + (Path("file.py"), "*.py,*.txt", "test_*.py", False, True), + (Path("test_file.py"), "*.py,*.txt", "test_*.py", False, False), + (Path("File.PY"), "*.py", "", True, False), + (Path("File.PY"), "*.py", "", False, True), + # (Path("test/file.py"), "**/test/*.py", "", False, True), + (Path("src/file.py"), "**/test/*.py", "", False, False), + (Path("file.txt"), "*.py,*.js,*.txt", "", False, True), + (Path("file.md"), "*.py,*.js,*.txt", "", False, False), + (Path("test_file.py"), "*.py", "test_*.py", False, False), + (Path(".hidden_file"), "*", "", False, True), + (Path("file_without_extension"), "", "*.*", False, True), + (Path("deeply/nested/directory/file.txt"), "**/*.txt", "", False, True), + (Path("file.txt.bak"), "", "*.bak", False, False), + ], +) +def test_is_filtered( + file_path, include_pattern, exclude_pattern, case_sensitive, expected +): + assert ( + is_filtered(file_path, include_pattern, exclude_pattern, case_sensitive) + == expected + ) + def test_is_filtered_with_directories(): - # assert is_filtered(Path("test"), "**/test", "", False) == True - assert is_filtered(Path("src/test"), "**/test", "", False) == True - assert is_filtered(Path("src/prod"), "**/test", "", False) == False + assert is_filtered( + Path("src/test"), "**/test", "", False + ) # Removed comparison to True + assert not is_filtered(Path("src/prod"), "**/test", "", False) # Updated to use not + def test_is_filtered_empty_patterns(): - assert is_filtered(Path("any_file.txt")) == True + assert is_filtered(Path("any_file.txt")) # Removed comparison to True + def test_is_filtered_case_sensitivity(): - assert is_filtered(Path("File.TXT"), "*.txt", "", True) == False - assert is_filtered(Path("File.TXT"), "*.txt", "", False) == True + assert not is_filtered(Path("File.TXT"), "*.txt", "", True) # Updated comparison + assert is_filtered(Path("File.TXT"), "*.txt", "", False) # Unchanged + def test_is_filtered_exclude_precedence(): - assert is_filtered(Path("important_test.py"), "*.py", "*test*", False) == False \ No newline at end of file + assert not is_filtered(Path("important_test.py"), "*.py", "*test*", False) diff --git a/tests/test_price.py b/tests/test_price.py index de0578b..cb26b96 100644 --- a/tests/test_price.py +++ b/tests/test_price.py @@ -1,115 +1,102 @@ import pytest -from unittest.mock import patch, mock_open -from code2prompt.utils.price_calculator import load_token_prices, calculate_prices - -# Mock JSON data -MOCK_JSON_DATA = ''' -{ - "providers": [ - { - "name": "provider1", - "models": [ - { - "name": "model1", - "price": 0.1 - }, - { - "name": "model2", - "input_price": 0.3, - "output_price": 0.4 - } - ] - }, - { - "name": "provider2", - "models": [ - { - "name": "model1", - "input_price": 0.3, - "output_price": 0.4 - }, - { - "name": "model2", - "input_price": 0.3, - "output_price": 0.4 - } - ] - } - ] -} -''' +from code2prompt.utils.price_calculator import TokenPrices, PriceModel, Provider, calculate_prices, PriceResult @pytest.fixture -def mock_token_prices(): - with patch("builtins.open", mock_open(read_data=MOCK_JSON_DATA)): - yield load_token_prices() - -def test_load_token_prices_success(mock_token_prices): - assert len(mock_token_prices["providers"]) == 2 - assert mock_token_prices["providers"][0]["name"] == "provider1" - assert mock_token_prices["providers"][1]["name"] == "provider2" - -def test_load_token_prices_file_not_found(): - with patch("builtins.open", side_effect=FileNotFoundError): - with pytest.raises(RuntimeError, match="Error loading token prices"): - load_token_prices() - -def test_load_token_prices_invalid_json(): - with patch("builtins.open", mock_open(read_data="invalid json")): - with pytest.raises(RuntimeError, match="Error loading token prices"): - load_token_prices() - -def test_calculate_prices_single_price_model(mock_token_prices): - result = calculate_prices(mock_token_prices, 1000, 1000, "provider1", "model1") - assert len(result) == 1 - assert result[0] == ("provider1", "model1", "$0.1000000000", 2000, "$0.2000000000") - -def test_calculate_prices_dual_price_model(mock_token_prices): - result = calculate_prices(mock_token_prices, 1000, 2000, "provider1", "model2") - assert len(result) == 1 - assert result[0] == ("provider1", "model2", "$0.3000000000 (input) / $0.4000000000 (output)", 3000, "$1.1000000000") - -def test_calculate_prices_all_providers_models(mock_token_prices): - result = calculate_prices(mock_token_prices, 1000, 1000) - assert len(result) == 4 - assert set(row[0] for row in result) == {"provider1", "provider2"} - assert set(row[1] for row in result) == {"model1", "model2"} - -def test_calculate_prices_specific_provider(mock_token_prices): - result = calculate_prices(mock_token_prices, 1000, 1000, "provider1") - assert len(result) == 2 - assert all(row[0] == "provider1" for row in result) - assert set(row[1] for row in result) == {"model1", "model2"} - -def test_calculate_prices_zero_tokens(mock_token_prices): - result = calculate_prices(mock_token_prices, 0, 0) - assert len(result) == 4 - assert all(row[4] == "$0.0000000000" for row in result) - -def test_calculate_prices_different_input_output_tokens(mock_token_prices): - result = calculate_prices(mock_token_prices, 1000, 2000, "provider2", "model1") - assert len(result) == 1 - assert result[0] == ("provider2", "model1", "$0.3000000000 (input) / $0.4000000000 (output)", 3000, "$1.1000000000") - -def test_calculate_prices_non_existent_provider(mock_token_prices): - result = calculate_prices(mock_token_prices, 1000, 1000, "non_existent_provider") - assert len(result) == 0 - -def test_calculate_prices_non_existent_model(mock_token_prices): - result = calculate_prices(mock_token_prices, 1000, 1000, "provider1", "non_existent_model") - assert len(result) == 0 - -def test_calculate_prices_large_numbers(mock_token_prices): - result = calculate_prices(mock_token_prices, 1000000, 1000000, "provider1", "model1") - assert len(result) == 1 - assert result[0] == ("provider1", "model1", "$0.1000000000", 2000000, "$200.0000000000") - -def test_calculate_prices_small_numbers(mock_token_prices): - result = calculate_prices(mock_token_prices, 1, 1, "provider1", "model1") - assert len(result) == 1 - assert result[0] == ("provider1", "model1", "$0.1000000000", 2, "$0.0002000000") - -def test_calculate_prices_floating_point_precision(mock_token_prices): - result = calculate_prices(mock_token_prices, 1000, 1000, "provider2", "model1") - assert len(result) == 1 - assert result[0] == ("provider2", "model1", "$0.3000000000 (input) / $0.4000000000 (output)", 2000, "$0.7000000000") \ No newline at end of file +def sample_token_prices(): + return TokenPrices( + providers=[ + Provider( + name="OpenAI", + models=[ + PriceModel(name="GPT-3", price=0.02), + PriceModel(name="GPT-4", input_price=0.03, output_price=0.06), + ] + ), + Provider( + name="Anthropic", + models=[ + PriceModel(name="Claude", input_price=0.01, output_price=0.03), + ] + ) + ] + ) + +def test_calculate_prices_all_providers_and_models(sample_token_prices): + results = calculate_prices(sample_token_prices, 1000, 500) + assert len(results) == 3 + assert {r.provider_name for r in results} == {"OpenAI", "Anthropic"} + assert {r.model_name for r in results} == {"GPT-3", "GPT-4", "Claude"} + +def test_calculate_prices_specific_provider(sample_token_prices): + results = calculate_prices(sample_token_prices, 1000, 500, provider="OpenAI") + assert len(results) == 2 + assert all(r.provider_name == "OpenAI" for r in results) + +def test_calculate_prices_specific_model(sample_token_prices): + results = calculate_prices(sample_token_prices, 1000, 500, model="GPT-4") + assert len(results) == 1 + assert results[0].model_name == "GPT-4" + +def test_calculate_prices_non_existent_provider(sample_token_prices): + results = calculate_prices(sample_token_prices, 1000, 500, provider="NonExistent") + assert len(results) == 0 + +def test_calculate_prices_non_existent_model(sample_token_prices): + results = calculate_prices(sample_token_prices, 1000, 500, model="NonExistent") + assert len(results) == 0 + +def test_calculate_prices_single_price_model(sample_token_prices): + results = calculate_prices(sample_token_prices, 1000, 500, model="GPT-3") + assert len(results) == 1 + result = results[0] + assert result.price_input == 0.02 + assert result.price_output == 0.02 + assert result.total_price == pytest.approx(0.03) # (1000 + 500) * 0.02 / 1000 + +def test_calculate_prices_separate_input_output_prices(sample_token_prices): + results = calculate_prices(sample_token_prices, 1000, 500, model="GPT-4") + assert len(results) == 1 + result = results[0] + assert result.price_input == 0.03 + assert result.price_output == 0.06 + assert result.total_price == pytest.approx(0.06) # (1000 * 0.03 + 500 * 0.06) / 1000 + +def test_calculate_prices_zero_tokens(sample_token_prices): + results = calculate_prices(sample_token_prices, 0, 0) + assert len(results) == 3 + assert all(r.total_price == 0 for r in results) + + + +def test_calculate_prices_result_structure(sample_token_prices): + results = calculate_prices(sample_token_prices, 1000, 500, model="GPT-4") + assert len(results) == 1 + result = results[0] + assert isinstance(result, PriceResult) + assert hasattr(result, 'provider_name') + assert hasattr(result, 'model_name') + assert hasattr(result, 'price_input') + assert hasattr(result, 'price_output') + assert hasattr(result, 'total_tokens') + assert hasattr(result, 'total_price') + +def test_calculate_prices_total_tokens(sample_token_prices): + results = calculate_prices(sample_token_prices, 1000, 500) + assert all(r.total_tokens == 1500 for r in results) + +@pytest.mark.parametrize("input_tokens,output_tokens,expected_total", [ + (1000, 500, 1500), + (0, 1000, 1000), + (1000, 0, 1000), + (0, 0, 0), +]) +def test_calculate_prices_various_token_combinations(sample_token_prices, input_tokens, output_tokens, expected_total): + results = calculate_prices(sample_token_prices, input_tokens, output_tokens) + assert all(r.total_tokens == expected_total for r in results) + + + +def test_calculate_prices_empty_token_prices(): + empty_token_prices = TokenPrices(providers=[]) + results = calculate_prices(empty_token_prices, 1000, 500) + assert len(results) == 0 diff --git a/tests/test_template_include.py b/tests/test_template_include.py index 44c0735..b74aec0 100644 --- a/tests/test_template_include.py +++ b/tests/test_template_include.py @@ -1,6 +1,4 @@ -import pytest from code2prompt.core.template_processor import process_template -import os def test_include_feature(tmp_path): # Create a main template diff --git a/todo/improvements.md b/todo/improvements.md new file mode 100644 index 0000000..94cba9f --- /dev/null +++ b/todo/improvements.md @@ -0,0 +1,19 @@ +# Suggested Improvements for Code2Prompt + +## 1. Documentation +- Ensure all functions and classes have comprehensive docstrings that explain their purpose, parameters, and return values. + +## 2. Type Annotations +- Add type annotations to all function parameters and return types for better readability and static type checking. + +## 3. Logging +- Implement logging in critical areas of the code to help with debugging and monitoring. + +## 4. Error Handling +- Improve error handling to provide more informative messages and handle exceptions gracefully. + +## 5. Code Structure +- Consider breaking down larger functions into smaller, more manageable ones to improve readability and maintainability. + +## 6. Testing +- Ensure that there are adequate unit tests covering all functionalities, especially for edge cases.