Develop->Main

.github/workflows/ci.yml

@@ -30,8 +30,8 @@ jobs:
 
       - name: "Run tests"
         env:
-            VECTORIZE_TOKEN: ${{ secrets.VECTORIZE_TOKEN }}
-            VECTORIZE_ORG: ${{ secrets.VECTORIZE_ORG }}
+            VECTORIZE_API_KEY: ${{ secrets.VECTORIZE_TOKEN }}
+            VECTORIZE_ORGANIZATION_ID: ${{ secrets.VECTORIZE_ORG }}
             VECTORIZE_ENV: dev
         run: |
           cd tests/ts
@@ -65,8 +65,8 @@ jobs:
 
       - name: Tests
         env:
-          VECTORIZE_TOKEN: ${{ secrets.VECTORIZE_TOKEN }}
-          VECTORIZE_ORG: ${{ secrets.VECTORIZE_ORG }}
+          VECTORIZE_API_KEY: ${{ secrets.VECTORIZE_TOKEN }}
+          VECTORIZE_ORGANIZATION_ID: ${{ secrets.VECTORIZE_ORG }}
           VECTORIZE_ENV: dev
         run: |
           cd tests/python

.gitleaksignore

@@ -0,0 +1 @@
+38b2cbb260f769b31c59f3e7b4d0a2dc4a37f2ac:src/python/vectorize_client/models/source_connector_input_config.py:dropbox-api-token:44

scripts/fix-ts-unions.js

@@ -0,0 +1,159 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+
+// Function to extract imported types from the file
+function extractImportedTypes(content) {
+  const typeImports = [];
+  const importRegex = /import type \{ (\w+) \} from '\.\/\w+';/g;
+  let match;
+
+  while ((match = importRegex.exec(content)) !== null) {
+    typeImports.push(match[1]);
+  }
+
+  return typeImports;
+}
+
+function fixJsonValueBug(content) {
+  // Fix the specific pattern in ToJSONTyped functions where 'value' is the parameter
+  content = content.replace(
+    /export function (\w+)ToJSONTyped\(value\?: ([^,]+) \| null, ignoreDiscriminator: boolean = false\): any \{[\s\S]*?return json;/g,
+    (match) => match.replace('return json;', 'return value;')
+  );
+
+  return content;
+}
+
+// Update both functions to use this more precise fix:
+
+// Function to extract imported types from the file
+function extractImportedTypes(content) {
+  const typeImports = [];
+  // Match both single-line and multi-line imports
+  const importRegex = /import type \{ (\w+) \} from ['"]\.\/\w+['"];/g;
+  let match;
+
+  while ((match = importRegex.exec(content)) !== null) {
+    typeImports.push(match[1]);
+  }
+
+  console.log(`  Found imported types: ${typeImports.join(', ')}`);
+  return typeImports;
+}
+
+// Function to fix union type definitions
+function fixUnionTypes(filePath) {
+  let content = fs.readFileSync(filePath, 'utf8');
+  const originalContent = content;
+
+  // Pattern to match empty type definitions
+  const emptyTypePattern = /export type (\w+) = ;/;
+
+  const match = emptyTypePattern.exec(content);
+  if (match) {
+    const typeName = match[1];
+    console.log(`Fixing empty type: ${typeName}`);
+
+    // Extract imported types from this file
+    const importedTypes = extractImportedTypes(content);
+
+    // Filter imported types that are likely part of this union
+    const unionTypes = importedTypes.filter(t => {
+      const isUsed = content.includes(`instanceof${t}`) || 
+             content.includes(`${t}FromJSON`) ||
+             content.includes(`${t}ToJSON`);
+      if (isUsed) {
+        console.log(`  Type ${t} is part of the union`);
+      }
+      return isUsed;
+    });
+
+    if (unionTypes.length > 0) {
+      // Create the union type
+      const unionTypeDefinition = `export type ${typeName} = ${unionTypes.join(' | ')};`;
+
+      // Replace the empty type definition
+      content = content.replace(
+        `export type ${typeName} = ;`,
+        unionTypeDefinition
+      );
+
+      console.log(`  Replaced with: ${unionTypeDefinition}`);
+    } else {
+      console.log(`  WARNING: No union types found for ${typeName}`);
+    }
+  }
+
+  // Only write if we made changes
+  if (content !== originalContent) {
+    fs.writeFileSync(filePath, content, 'utf8');
+    console.log(`✅ Fixed ${filePath}`);
+  }
+}
+
+function fixAnyTypeImports(filePath) {
+  let content = fs.readFileSync(filePath, 'utf8');
+  const originalContent = content;
+
+  // Remove AnyType imports
+  content = content.replace(/import type \{ AnyType \} from '\.\/AnyType';\n/g, '');
+  content = content.replace(/import \{\s*[^}]*\s*\} from '\.\/AnyType';\n/g, '');
+
+  // Replace AnyType with any
+  content = content.replace(/: AnyType/g, ': any');
+
+  // Fix the json/value bug with the more precise function
+  content = fixJsonValueBug(content);
+
+  // Only write if we made changes
+  if (content !== originalContent) {
+    fs.writeFileSync(filePath, content, 'utf8');
+  }
+}
+
+// Main function to process all TypeScript files
+function processTypeScriptFiles(directory) {
+  const modelsDir = path.join(directory, 'src', 'models');
+
+  if (!fs.existsSync(modelsDir)) {
+    console.error(`Models directory not found: ${modelsDir}`);
+    process.exit(1);
+  }
+
+  // Files that typically have union type issues
+  const filesToFix = [
+    'CreateAIPlatformConnectorRequest.ts',
+    'CreateSourceConnectorRequest.ts',
+    'CreateDestinationConnectorRequest.ts'
+  ];
+
+  filesToFix.forEach(fileName => {
+    const filePath = path.join(modelsDir, fileName);
+    if (fs.existsSync(filePath)) {
+      fixUnionTypes(filePath);
+    } else {
+      console.log(`⚠️  File not found: ${fileName}`);
+    }
+  });
+
+  const files = fs.readdirSync(modelsDir);
+files.forEach(file => {
+  if (file.endsWith('.ts')) {
+    const filePath = path.join(modelsDir, file);
+    fixAnyTypeImports(filePath);
+  }
+});
+}
+
+// Get the directory from command line argument
+const tsDirectory = process.argv[2];
+
+if (!tsDirectory) {
+  console.error('Usage: node fix-ts-unions.js <typescript-output-directory>');
+  process.exit(1);
+}
+
+console.log('🔧 Fixing TypeScript union types...');
+processTypeScriptFiles(tsDirectory);
+console.log('✨ Done!');

scripts/generate-python.sh

@@ -4,15 +4,13 @@ ROOT_DIR=$(git rev-parse --show-toplevel)
 SRC_DIR=$ROOT_DIR/src
 
 PYPROJECT=$SRC_DIR/python/pyproject.toml
-current_version=$(npm run read-toml $PYPROJECT tool.poetry version | tail -n 1 | tr -d '[:space:]')
+current_version=$(node -p "require('./package.json').version")
 
 rm -rf $SRC_DIR/python
 openapi-generator-cli generate -i $ROOT_DIR/vectorize_api.json -g python -o $SRC_DIR/python \
   --additional-properties=packageName=vectorize_client \
   --additional-properties=generateSourceCodeOnly=false
 
-
-
 npm run edit-toml $PYPROJECT tool.poetry version $current_version
 npm run edit-toml $PYPROJECT tool.poetry name vectorize-client
 npm run edit-toml $PYPROJECT tool.poetry description "Python client for the Vectorize API"

scripts/generate-ts.sh

@@ -4,18 +4,30 @@ ROOT_DIR=$(git rev-parse --show-toplevel)
 SRC_DIR=$ROOT_DIR/src
 
 PACKAGE_JSON=$SRC_DIR/ts/package.json
-cd $SRC_DIR/ts
-current_version=$(npm pkg get version | tr -d '"')
-cd ../../
+
+# Try to get current version if the directory exists
+if [ -d "$SRC_DIR/ts" ] && [ -f "$PACKAGE_JSON" ]; then
+  cd $SRC_DIR/ts
+  current_version=$(npm pkg get version | tr -d '"')
+  cd ../../
+else
+  # Use version from root package.json as fallback
+  current_version=$(node -p "require('./package.json').version")
+fi
 
 rm -rf $SRC_DIR/ts
+
+# Generate the client
 openapi-generator-cli generate -i $ROOT_DIR/vectorize_api.json -g typescript-fetch -o $SRC_DIR/ts \
   --additional-properties=npmName=@vectorize-io/vectorize-client \
   --additional-properties=licenseName=MIT \
   --additional-properties=npmRepository="https://github.com/vectorize-io/vectorize-clients" \
   --additional-properties=snapshot=true \
   --additional-properties=generateSourceCodeOnly=false
 
+# Fix the union types
+node $ROOT_DIR/scripts/fix-ts-unions.js $SRC_DIR/ts
+
 edit_field() {
   local field=$1
   local value=$2

src/python/README.md

@@ -1,14 +1,9 @@
-from dataclasses import field
-
 # Vectorize Client
-Python Api Client for Vectorize
-For more information, please visit [https://vectorize.io](https://vectorize.io)
-
-## Requirements.
+Python Api Client for [Vectorize](https://vectorize.io).
+For the full documentation, please visit [docs.vectorize.io](https://docs.vectorize.io/api/api-getting-started).
 
-Python 3.8+
 
-## Installation & Usage
+## Installation
 ```sh
 pip install vectorize-client
 ```
@@ -20,8 +15,7 @@ import vectorize_client
 
 ## Getting Started
 
-Please follow the [installation procedure](#installation--usage) and then run the following:
-
+List all your pipelines:
 ```python
 import vectorize_client as v
 
@@ -34,62 +28,5 @@ with v.ApiClient(v.Configuration(access_token=TOKEN)) as api:
     print("Found" + str(len(response.data)) + " pipelines")
 ```
 
-## Documentation for API Endpoints
-
-All URIs are relative to *https://api.vectorize.io/v1*
-
-See the full [reference](https://vectorize.readme.io/reference) for more information.
-
-## Usage
-
-First, export your token and org id as environment variables:
-
-```sh
-export VECTORIZE_TOKEN=<your-token>
-export VECTORIZE_ORG=<your-org-id>
-```
-Then, initialize the client with your token and org id:
-
-```python
-import os
-TOKEN = os.environ['VECTORIZE_TOKEN']
-ORG = os.environ['VECTORIZE_ORG']
-```
-
-### Extraction
-
-Set the file you want to extract data from:
-
-```sh
-export FILE=<path-to-file>
-```
-
-Then, run the following code:
-```python
-import os
-import vectorize_client as v
-import time, logging
-
-TOKEN = os.environ['VECTORIZE_TOKEN']
-ORG = os.environ['VECTORIZE_ORG']
-FILE = os.environ['FILE']
-
-with v.ApiClient(v.Configuration(access_token=TOKEN)) as api:
-    with open(FILE, 'rb') as file:
-        data = file.read()
-        extraction_id = v.ExtractionApi(api).start_extraction(ORG, data).extraction_id
-        print(f"Extraction started with id {extraction_id}")
-        while True:
-            extraction = v.ExtractionApi(api).get_extraction_result(ORG, extraction_id)
-            if extraction.ready:
-                extracted_data = extraction.data
-                if extracted_data.success:
-                    print(extracted_data)
-                    break
-                else:
-                    raise Exception(extracted_data.error)
-            print("Waiting for extraction to complete...")
-            time.sleep(1)
-```
-
+Visit [docs.vectorize.io](https://docs.vectorize.io/api/api-getting-started) to learn more about the API.
 

src/python/pyproject.toml

@@ -1,49 +1,35 @@
-[project]
-name = "vectorize_client"
-version = "1.0.0"
-description = "Vectorize API (Beta)"
-license = "NoLicense"
-readme = "README.md"
-keywords = [ "OpenAPI", "OpenAPI-Generator", "Vectorize API (Beta)" ]
-requires-python = ">=3.9"
-dependencies = [
-  "urllib3 (>=2.1.0,<3.0.0)",
-  "python-dateutil (>=2.8.2)",
-  "pydantic (>=2)",
-  "typing-extensions (>=4.7.1)"
-]
-
-  [[project.authors]]
-  name = "Vectorize"
-  email = "team@openapitools.org"
-
-  [project.urls]
-  Repository = "https://github.com/GIT_USER_ID/GIT_REPO_ID"
-
 [tool.poetry]
-requires-poetry = ">=2.0"
-version = "0.3.0"
 name = "vectorize-client"
+version = "0.1.0"
 description = "Python client for the Vectorize API"
 authors = [ "Vectorize <contact@vectorize.io>" ]
 license = "MIT"
+readme = "README.md"
 repository = "https://github.com/vectorize-io/vectorize-clients"
-homepage = "https://vectorize.io"
 keywords = [
   "vectorize",
   "vectorize.io",
   "generative-ai",
   "embeddings",
   "rag"
 ]
+include = [ "vectorize_client/py.typed" ]
+homepage = "https://vectorize.io"
+
+  [tool.poetry.dependencies]
+  python = "^3.9"
+  urllib3 = ">= 2.1.0, < 3.0.0"
+  python-dateutil = ">= 2.8.2"
+  pydantic = ">= 2"
+  typing-extensions = ">= 4.7.1"
 
-[tool.poetry.group.dev.dependencies]
-pytest = ">= 7.2.1"
-pytest-cov = ">= 2.8.1"
-tox = ">= 3.9.0"
-flake8 = ">= 4.0.0"
-types-python-dateutil = ">= 2.8.19.14"
-mypy = ">= 1.5"
+  [tool.poetry.dev-dependencies]
+  pytest = ">= 7.2.1"
+  pytest-cov = ">= 2.8.1"
+  tox = ">= 3.9.0"
+  flake8 = ">= 4.0.0"
+  types-python-dateutil = ">= 2.8.19.14"
+  mypy = ">= 1.5"
 
 [tool.pylint."MESSAGES CONTROL"]
 extension-pkg-whitelist = "pydantic"