core[patch]: Inline more portable SAX parser, remove node-only type i…

…mport (#4325)

* Inline more portable SAX parser

* Add attribution

langchain-core/.eslintrc.cjs

@@ -16,6 +16,7 @@ module.exports = {
     "src/utils/@cfworker",
     "src/utils/fast-json-patch",
     "src/utils/js-sha1",
+    "src/utils/sax-js",
     ".eslintrc.cjs",
     "scripts",
     "node_modules",

langchain-core/package.json

@@ -46,7 +46,6 @@
     "ml-distance": "^4.0.0",
     "p-queue": "^6.6.2",
     "p-retry": "4",
-    "sax": "^1.3.0",
     "uuid": "^9.0.0",
     "zod": "^3.22.4",
     "zod-to-json-schema": "^3.22.3"
@@ -56,7 +55,6 @@
     "@langchain/scripts": "~0.0",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",
-    "@types/sax": "^1",
     "dpdm": "^3.12.0",
     "eslint": "^8.33.0",
     "eslint-config-airbnb-base": "^15.0.0",

langchain-core/src/output_parsers/xml.ts

@@ -1,9 +1,9 @@
-import sax, { SAXOptions } from "sax";
 import {
   BaseCumulativeTransformOutputParser,
   BaseCumulativeTransformOutputParserInput,
 } from "./transform.js";
 import { Operation, compare } from "../utils/json_patch.js";
+import { sax } from "../utils/sax-js/sax.js";
 import { ChatGeneration, Generation } from "../outputs.js";
 
 export const XML_FORMAT_INSTRUCTIONS = `The output should be formatted as a XML file.
@@ -22,8 +22,7 @@ Here are the output tags:
 \`\`\``;
 
 export interface XMLOutputParserFields
-  extends SAXOptions,
-    BaseCumulativeTransformOutputParserInput {
+  extends BaseCumulativeTransformOutputParserInput {
   /**
    * Optional list of tags that the output should conform to.
    * Only used in formatting of the prompt.
@@ -40,16 +39,10 @@ export type XMLResult = {
 export class XMLOutputParser extends BaseCumulativeTransformOutputParser<XMLResult> {
   tags?: string[];
 
-  saxOptions?: SAXOptions;
-
   constructor(fields?: XMLOutputParserFields) {
-    const f = fields ?? {};
-    const { tags, ...saxOptions } = f;
-
-    super(f);
+    super(fields);
 
-    this.tags = tags;
-    this.saxOptions = saxOptions;
+    this.tags = fields?.tags;
   }
 
   static lc_name() {
@@ -76,11 +69,11 @@ export class XMLOutputParser extends BaseCumulativeTransformOutputParser<XMLResu
   async parsePartialResult(
     generations: ChatGeneration[] | Generation[]
   ): Promise<XMLResult | undefined> {
-    return parseXMLMarkdown(generations[0].text, this.saxOptions);
+    return parseXMLMarkdown(generations[0].text);
   }
 
   async parse(text: string): Promise<XMLResult> {
-    return parseXMLMarkdown(text, this.saxOptions);
+    return parseXMLMarkdown(text);
   }
 
   getFormatInstructions(): string {
@@ -121,16 +114,14 @@ const parseParsedResult = (input: ParsedResult): XMLResult => {
   }
 };
 
-export function parseXMLMarkdown(
-  s: string,
-  saxOptions?: SAXOptions
-): XMLResult {
+export function parseXMLMarkdown(s: string): XMLResult {
   const cleanedString = strip(s);
-  const parser = sax.parser(true, saxOptions);
+  const parser = sax.parser(true);
   let parsedResult: ParsedResult = {} as ParsedResult;
   const elementStack: ParsedResult[] = [];
 
-  parser.onopentag = (node) => {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  parser.onopentag = (node: any) => {
     const element = {
       name: node.name,
       attributes: node.attributes,
@@ -160,14 +151,16 @@ export function parseXMLMarkdown(
     }
   };
 
-  parser.ontext = (text) => {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  parser.ontext = (text: any) => {
     if (elementStack.length > 0) {
       const currentElement = elementStack[elementStack.length - 1];
       currentElement.text += text;
     }
   };
 
-  parser.onattribute = (attr) => {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  parser.onattribute = (attr: any) => {
     if (elementStack.length > 0) {
       const currentElement = elementStack[elementStack.length - 1];
       currentElement.attributes[attr.name] = attr.value;

langchain-core/src/utils/event_source_parse.ts

@@ -4,7 +4,6 @@
 // Adapted from https://github.com/gfortaine/fetch-event-source/blob/main/src/parse.ts
 // due to a packaging issue in the original.
 // MIT License
-import { type Readable } from "stream";
 import { IterableReadableStream } from "./stream.js";
 
 export const EventStreamContentType = "text/event-stream";
@@ -24,10 +23,6 @@ export interface EventSourceMessage {
   retry?: number;
 }
 
-function isNodeJSReadable(x: unknown): x is Readable {
-  return x != null && typeof x === "object" && "on" in x;
-}
-
 /**
  * Converts a ReadableStream into a callback pattern.
  * @param stream The input ReadableStream.
@@ -38,27 +33,6 @@ export async function getBytes(
   stream: ReadableStream<Uint8Array>,
   onChunk: (arr: Uint8Array, flush?: boolean) => void
 ) {
-  // stream is a Node.js Readable / PassThrough stream
-  // this can happen if node-fetch is polyfilled
-  if (isNodeJSReadable(stream)) {
-    return new Promise<void>((resolve) => {
-      stream.on("readable", () => {
-        let chunk;
-        // eslint-disable-next-line no-constant-condition
-        while (true) {
-          chunk = stream.read();
-          if (chunk == null) {
-            onChunk(new Uint8Array(), true);
-            break;
-          }
-          onChunk(chunk);
-        }
-
-        resolve();
-      });
-    });
-  }
-
   const reader = stream.getReader();
   // CHANGED: Introduced a "flush" mechanism to process potential pending messages when the stream ends.
   //          This change is essential to ensure that we capture every last piece of information from streams,

langchain-core/src/utils/sax-js/LICENSE.md

@@ -0,0 +1,41 @@
+The ISC License
+
+Copyright (c) 2010-2023 Isaac Z. Schlueter and Contributors
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
+IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+====
+
+`String.fromCodePoint` by Mathias Bynens used according to terms of MIT
+License, as follows:
+
+Copyright (c) 2010-2023 Mathias Bynens <https://mathiasbynens.be/>
+
+    Permission is hereby granted, free of charge, to any person obtaining
+    a copy of this software and associated documentation files (the
+    "Software"), to deal in the Software without restriction, including
+    without limitation the rights to use, copy, modify, merge, publish,
+    distribute, sublicense, and/or sell copies of the Software, and to
+    permit persons to whom the Software is furnished to do so, subject to
+    the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+    LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+    OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.