Skip to content

Commit

Permalink
feat: fix phind
Browse files Browse the repository at this point in the history
  • Loading branch information
@xiangsx
xiangsx committed Jun 26, 2023
1 parent dec4af5 commit f24959f
Show file tree
Hide file tree
Showing 4 changed files with 68 additions and 12 deletions.
35 changes: 23 additions & 12 deletions model/phind/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
import {Chat, ChatOptions, ChatRequest, ChatResponse, ModelType} from "../base";
import {Browser, Page} from "puppeteer";
import {BrowserPool, BrowserUser} from "../../pool/puppeteer";
import {DoneData, ErrorData, Event, EventStream, MessageData, sleep} from "../../utils";
import {DoneData, ErrorData, Event, EventStream, isSimilarity, MessageData, sleep} from "../../utils";
import {v4} from "uuid";
import TurndownService from "turndown";

let turndownService = new TurndownService({codeBlockStyle: 'fenced'}).remove('h6');
turndownService = turndownService.remove('h6');
const preText = `###### Answer | gpt-3.5 Model\n\n`;
type PageData = {
gpt4times: number;
}
Expand Down Expand Up @@ -95,8 +99,7 @@ export class Phind extends Chat implements BrowserUser<Account> {

await page.goto(url);
await sleep(2000);
await page.waitForSelector('div > .mt-4 > div > .form-check > .form-check-label')
await page.click('div > .mt-4 > div > .form-check > .form-check-label')
await page.waitForSelector('.mb-3 > div > div > .mt-6 > .btn-primary')
await Phind.allowClipboard(browser, page);
return [page, {id}]
}
Expand All @@ -123,12 +126,16 @@ export class Phind extends Chat implements BrowserUser<Account> {
await page.keyboard.press('Enter');
await page.waitForSelector('.col-lg-10 > .row > .col-lg-8:nth-child(4) > .container-xl > div:nth-child(1)');
const output = await page.$('.col-lg-10 > .row > .col-lg-8:nth-child(4) > .container-xl > div:nth-child(1)');
let old = '';
const itl = setInterval(async () => {
try {
const content: any = await output?.evaluate(el => {
return el.textContent;
return el.outerHTML;
});
stream.write(Event.message, {content});
if (old !== content) {
stream.write(Event.message, {content: turndownService.turndown(content).replace(preText,'')});
old = content;
}
} catch (e) {
console.error(e);
}
Expand All @@ -137,18 +144,22 @@ export class Phind extends Chat implements BrowserUser<Account> {
await Phind.copyContent(page);
clearInterval(itl);
//@ts-ignore
const text: any = await page.evaluate(() => navigator.clipboard.text);
console.log('chat end: ', text);
stream.write(Event.done, {
content: text || await output?.evaluate((el: any) => {
return el.textContent;
})
const text: any = (await page.evaluate(() => navigator.clipboard.text)) || '';
const sourcehtml: any = await output?.evaluate((el: any) => {
return el.outerHTML;
})
console.log('chat end: ', text);
const sourceText = turndownService.turndown(sourcehtml).replace(preText,'');
if (isSimilarity(text, sourceText)) {
stream.write(Event.done, {content: text});
} else {
stream.write(Event.done, {content: sourceText});
}
stream.end();
done(account);
await page.goto(url);
}
wait().then().catch(async (e)=>{
wait().then().catch(async (e) => {
console.error(e);
stream.write(Event.error, {error: e.message})
stream.end();
Expand Down
4 changes: 4 additions & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
"@types/koa-bodyparser": "^4.3.10",
"@types/koa-router": "^7.4.4",
"@types/node": "^18.16.3",
"@types/string-similarity": "^4.0.0",
"@types/turndown": "^5.0.1",
"@types/uuid": "^9.0.1",
"ts-node": "^10.9.1",
"typescript": "^5.0.4"
Expand All @@ -39,7 +41,9 @@
"puppeteer": "^20.1.2",
"puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2",
"string-similarity": "^4.0.4",
"tls-client": "^0.0.5",
"turndown": "^7.1.2",
"user-agents": "^1.0.1367",
"uuid": "^9.0.0"
}
Expand Down
14 changes: 14 additions & 0 deletions utils/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@ import * as crypto from 'crypto';
import {v4} from "uuid";
import {encoding_for_model} from '@dqbd/tiktoken'
const en = encoding_for_model("gpt-3.5-turbo");
import TurndownService from "turndown";
const turndownService = new TurndownService({codeBlockStyle:'fenced'});
import stringSimilarity from 'string-similarity';


type eventFunc = (eventName: string, data: string) => void;

Expand Down Expand Up @@ -130,3 +134,13 @@ export class EventStream {
export const getTokenSize = (str: string) => {
return en.encode(str).length;
};

export const htmlToMarkdown = (html:string):string=>{
return turndownService.turndown(html);
}

export const isSimilarity=(s1:string,s2:string):boolean=>{
const similarity = stringSimilarity.compareTwoStrings(s1, s2);
console.log(similarity);
return similarity > 0.3;
}
27 changes: 27 additions & 0 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,16 @@
"@types/mime" "*"
"@types/node" "*"

"@types/string-similarity@^4.0.0":
version "4.0.0"
resolved "https://registry.npmmirror.com/@types/string-similarity/-/string-similarity-4.0.0.tgz#8cc03d5d1baad2b74530fe6c7d849d5768d391ad"
integrity sha512-dMS4S07fbtY1AILG/RhuwmptmzK1Ql8scmAebOTJ/8iBtK/KI17NwGwKzu1uipjj8Kk+3mfPxum56kKZE93mzQ==

"@types/turndown@^5.0.1":
version "5.0.1"
resolved "https://registry.npmmirror.com/@types/turndown/-/turndown-5.0.1.tgz#fcda7b02cda4c9d445be1440036df20f335b9387"
integrity sha512-N8Ad4e3oJxh9n9BiZx9cbe/0M3kqDpOTm2wzj13wdDUxDPjfjloWIJaquZzWE1cYTAHpjOH3rcTnXQdpEfS/SQ==

"@types/uuid@^9.0.1":
version "9.0.1"
resolved "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.1.tgz"
Expand Down Expand Up @@ -628,6 +638,11 @@ docopt@~0.6.2:
resolved "https://registry.npmjs.org/docopt/-/docopt-0.6.2.tgz"
integrity sha512-NqTbaYeE4gA/wU1hdKFdU+AFahpDOpgGLzHP42k6H6DKExJd0A55KEVWYhL9FEmHmgeLvEU2vuKXDuU+4yToOw==

domino@^2.1.6:
version "2.1.6"
resolved "https://registry.npmmirror.com/domino/-/domino-2.1.6.tgz#fe4ace4310526e5e7b9d12c7de01b7f485a57ffe"
integrity sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ==

dot-json@^1.3.0:
version "1.3.0"
resolved "https://registry.npmjs.org/dot-json/-/dot-json-1.3.0.tgz"
Expand Down Expand Up @@ -1533,6 +1548,11 @@ stream-combiner@^0.2.2:
duplexer "~0.1.1"
through "~2.3.4"

string-similarity@^4.0.4:
version "4.0.4"
resolved "https://registry.npmmirror.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b"
integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ==

string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
version "4.2.3"
resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
Expand Down Expand Up @@ -1641,6 +1661,13 @@ tsscmp@1.0.6:
resolved "https://registry.npmjs.org/tsscmp/-/tsscmp-1.0.6.tgz"
integrity sha512-LxhtAkPDTkVCMQjt2h6eBVY28KCjikZqZfMcC15YBeNjkgUpdCfBu5HoiOTDu86v6smE8yOjyEktJ8hlbANHQA==

turndown@^7.1.2:
version "7.1.2"
resolved "https://registry.npmmirror.com/turndown/-/turndown-7.1.2.tgz#7feb838c78f14241e79ed92a416e0d213e044a29"
integrity sha512-ntI9R7fcUKjqBP6QU8rBK2Ehyt8LAzt3UBT9JR9tgo6GtuKvyUzpayWmeMKJw1DPdXzktvtIT8m2mVXz+bL/Qg==
dependencies:
domino "^2.1.6"

type-is@^1.6.16:
version "1.6.18"
resolved "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz"
Expand Down

0 comments on commit f24959f

Please sign in to comment.