Skip to content

Commit

Permalink
feat(forefront): support account cache
Browse files Browse the repository at this point in the history
  • Loading branch information
@xiangsx
xiangsx committed Jun 6, 2023
1 parent f05e5db commit 4e48f9f
Show file tree
Hide file tree
Showing 6 changed files with 166 additions and 40 deletions.
160 changes: 137 additions & 23 deletions model/forefront/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,94 @@ import {BrowserPool} from "../../pool/puppeteer";
import {CreateEmail, TempEmailType, TempMailMessage} from "../../utils/emailFactory";
import {CreateTlsProxy} from "../../utils/proxyAgent";
import {PassThrough} from "stream";
import * as fs from "fs";
import {parseJSON} from "../../utils";
import {v4} from "uuid";
import moment from 'moment';

type PageData = {
gpt4times: number;
}

const MaxGptTimes = 4;

const TimeFormat = "YYYY-MM-DD HH:mm:ss";

type Account = {
id: string;
email?: string;
login_time?: string;
last_use_time?: string;
gpt4times: number;
}

class AccountPool {
private pool: Account[] = [];
private readonly account_file_path = './run/account.json';

constructor() {
if (fs.existsSync(this.account_file_path)) {
const accountStr = fs.readFileSync(this.account_file_path, 'utf-8');
this.pool = parseJSON(accountStr, [] as Account[]);
} else {
fs.mkdirSync('./run', {recursive: true});
this.syncfile();
}
}

public syncfile() {
fs.writeFileSync(this.account_file_path, JSON.stringify(this.pool));
}

public getByID(id: string) {
for (const item of this.pool) {
if (item.id === id) {
return item;
}
}
}

public get(): Account {
const now = moment();
const minInterval = 3 * 60 * 60 + 10 * 60;// 3hour + 10min
for (const item of this.pool) {
console.log(now.unix() - moment(item.last_use_time).unix());
if (now.unix() - moment(item.last_use_time).unix() > minInterval) {
item.last_use_time = now.format(TimeFormat);
this.syncfile();
return item
}
}
const newAccount: Account = {
id: v4(),
last_use_time: now.format(TimeFormat),
gpt4times: 0,
}
this.pool.push(newAccount);
this.syncfile();
return newAccount
}

public multiGet(size: number): Account[] {
const result: Account[] = [];
for (let i = 0; i < size; i++) {
result.push(this.get());
}
return result
}
}


export class Forefrontnew extends Chat {
private page: Page | undefined = undefined;
private msgSize: number = 0;
private pagePool: BrowserPool<PageData>;
private pagePool: BrowserPool<Account>;
private accountPool: AccountPool;

constructor(options?: ChatOptions) {
super(options);
this.pagePool = new BrowserPool<PageData>(+(process.env.POOL_SIZE || 2), this.init.bind(this));
this.accountPool = new AccountPool();
const maxSize = +(process.env.POOL_SIZE || 2);
const initialAccounts = this.accountPool.multiGet(maxSize);
this.pagePool = new BrowserPool<Account>(maxSize, initialAccounts.map(item => item.id), this.init.bind(this));
}

public async ask(req: Request): Promise<Response> {
Expand Down Expand Up @@ -49,8 +122,44 @@ export class Forefrontnew extends Chat {
}
}

private async init(browser: Browser): Promise<Page> {
private static async switchToGpt4(page: Page) {
try {
console.log('switch gpt4....')
await page.waitForTimeout(2000);
await page.waitForSelector('div > .absolute > .relative > .w-full:nth-child(3) > .relative')
await page.click('div > .absolute > .relative > .w-full:nth-child(3) > .relative');
await page.waitForTimeout(1000);
await page.waitForSelector('div > .absolute > .relative > .w-full:nth-child(3) > .relative')
await page.click('div > .absolute > .relative > .w-full:nth-child(3) > .relative')
await page.waitForTimeout(1000);
await page.hover('div > .absolute > .relative > .w-full:nth-child(3) > .relative')

await page.waitForSelector('.grid > .h-9 > .text-th-primary-light > g > path')
await page.click('.grid > .h-9 > .text-th-primary-light > g > path')

await page.waitForSelector('.px-4 > .flex > .grid > .block > .group:nth-child(5)')
await page.click('.px-4 > .flex > .grid > .block > .group:nth-child(5)')
console.log('switch gpt4 ok!')
}catch (e) {
console.log(e);
await page.reload();
await Forefrontnew.switchToGpt4(page);
}
}

private async init(id: string, browser: Browser): Promise<[Page, Account]> {
const account = this.accountPool.getByID(id);
if (!account) {
throw new Error("account undefined, something error");
}

const [page] = await browser.pages();
if (account.login_time) {
await page.goto("https://chat.forefront.ai/");
await page.setViewport({width: 1920, height: 1080});
await Forefrontnew.switchToGpt4(page);
return [page, account];
}
await page.goto("https://accounts.forefront.ai/sign-up");
await page.setViewport({width: 1920, height: 1080});
await page.waitForSelector('#emailAddress-field');
Expand All @@ -61,6 +170,8 @@ export class Forefrontnew extends Chat {

const emailBox = CreateEmail(process.env.EMAIL_TYPE as TempEmailType || TempEmailType.TempEmail44)
const emailAddress = await emailBox.getMailAddress();
account.email = emailAddress;
this.accountPool.syncfile();
// 将文本键入焦点元素
await page.keyboard.type(emailAddress, {delay: 10});
await page.keyboard.press('Enter');
Expand All @@ -78,31 +189,29 @@ export class Forefrontnew extends Chat {
}
await this.tryValidate(validateURL, 0);
console.log('register successfully');
account.login_time = moment().format(TimeFormat);
this.accountPool.syncfile();
await page.waitForSelector('.flex > .modal > .modal-box > .flex > .px-3:nth-child(1)', {timeout: 10000})
await page.click('.flex > .modal > .modal-box > .flex > .px-3:nth-child(1)')
await page.waitForSelector('.relative > .flex > .w-full > .text-th-primary-dark > div', {timeout: 10000})

await page.waitForTimeout(2000);
await page.waitForSelector('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium', {timeout: 100000});
await page.click('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium');
await page.waitForSelector('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium')
await page.click('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium')

await page.waitForSelector('.px-4 > .flex > .grid > .h-9 > .grow')
await page.click('.px-4 > .flex > .grid > .h-9 > .grow')

await page.waitForSelector('.grid > .block > .group:nth-child(5) > .grid > .grow:nth-child(1)')
await page.click('.grid > .block > .group:nth-child(5) > .grid > .grow:nth-child(1)')
return page;
await Forefrontnew.switchToGpt4(page);
return [page, account];
}

public async askStream(req: Request): Promise<ResponseStream> {
const [page, data = {gpt4times: 0} as PageData, done, destroy] = this.pagePool.get();
const [page, account, done, destroy] = this.pagePool.get();
if (!account) {
const pt = new PassThrough();
pt.write('account undefined, something error');
pt.end();
return {text: pt};
}
if (!page) {
const pt = new PassThrough();
pt.write('please wait init.....about 1 min');
pt.end();
return {text: pt}
return {text: pt};
}
try {
console.log('try find text input');
Expand Down Expand Up @@ -162,11 +271,16 @@ export class Forefrontnew extends Chat {
pt.end();
await page.waitForSelector('.flex:nth-child(1) > div:nth-child(2) > .relative > .flex > .cursor-pointer')
await page.click('.flex:nth-child(1) > div:nth-child(2) > .relative > .flex > .cursor-pointer')
data.gpt4times += 1;
if (data.gpt4times >= MaxGptTimes) {
destroy();
account.gpt4times += 1;
this.accountPool.syncfile();
if (account.gpt4times >= MaxGptTimes) {
account.gpt4times = 0;
account.last_use_time = moment().format(TimeFormat);
this.accountPool.syncfile();
const newAccount = this.accountPool.get();
destroy(newAccount.id);
} else {
done(data);
done(account);
}
clearInterval(itl);
}
Expand Down
2 changes: 1 addition & 1 deletion model/mcbbs/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ export class Mcbbs extends Chat {
cb(null, '');
return;
}
const data = parseJSON(dataStr, {});
const data = parseJSON(dataStr, {} as any);
if (!data?.choices) {
cb(null, '');
return;
Expand Down
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
"koa": "^2.14.2",
"koa-bodyparser": "^4.4.0",
"koa-router": "^12.0.0",
"moment": "^2.29.4",
"puppeteer": "^20.1.2",
"tls-client": "^0.0.5",
"user-agents": "^1.0.1367",
Expand Down
34 changes: 20 additions & 14 deletions pool/puppeteer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,52 +5,57 @@ import run from "node:test";
const runPath = path.join(__dirname, 'run');

export interface PageInfo<T> {
id: number;
id: string;
ready: boolean;
page?: Page;
data?: T;
}

type PrepareFunc<T> = (browser: Browser) => Promise<Page>
type PrepareFunc<T> = (id: string, browser: Browser) => Promise<[Page, T]>

export class BrowserPool<T> {
private readonly pool: PageInfo<T>[] = [];
private readonly size: number;
private readonly prepare: PrepareFunc<T>

constructor(size: number, prepare: PrepareFunc<T>,) {
constructor(size: number, initialIDs: string[], prepare: PrepareFunc<T>) {
this.size = size
this.prepare = prepare;
this.init();
this.init(initialIDs);
}

init() {
init(initialIDs: string[]) {
for (let i = 0; i < this.size; i++) {
const id = initialIDs[i];
const info: PageInfo<T> = {
id: i,
id,
ready: false,
}
this.initOne().then(page => {
this.initOne(id).then(([page, data]) => {
info.page = page;
info.data = data;
info.ready = true;
}).catch(e=>{
}).catch(e => {
console.error(e);
})
this.pool.push(info)
}
}

async initOne(): Promise<Page> {
async initOne(id: string): Promise<[Page, T]> {
const options: PuppeteerLaunchOptions = {
headless: process.env.DEBUG === "1" ? false : 'new',
args: ['--no-sandbox'],
};
const browser = await puppeteer.launch(options)
return this.prepare(browser)
if (id) {
options.userDataDir = `run/${id}`;
}
const browser = await puppeteer.launch(options);
return this.prepare(id, browser)
}

//@ts-ignore
get(): [page: Page | undefined, data: T | undefined, done: (data: T) => void, destroy: () => void] {
get(): [page: Page | undefined, data: T | undefined, done: (data: T) => void, destroy: (newID: string) => void] {
for (const item of this.pool) {
if (item.ready) {
item.ready = false;
Expand All @@ -61,10 +66,11 @@ export class BrowserPool<T> {
item.ready = true
item.data = data;
},
() => {
(newID: string) => {
item.page?.close();
this.initOne().then((page) => {
this.initOne(newID).then(([page, data]) => {
item.page = page
item.data = data;
item.ready = true;
})
}
Expand Down
4 changes: 2 additions & 2 deletions utils/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ export function randomStr(): string {
return v4().split('-').join('').slice(-6);
}

export function parseJSON(str: string, defaultObj: any): any | undefined {
export function parseJSON<T>(str: string, defaultObj: T): T {
try {
return JSON.parse(str)
} catch (e) {
Expand All @@ -41,7 +41,7 @@ export function parseJSON(str: string, defaultObj: any): any | undefined {
}
}

export function encryptWithAes256Cbc(data: string, key: string):string {
export function encryptWithAes256Cbc(data: string, key: string): string {
const hash = crypto.createHash('sha256').update(key).digest();
const iv = crypto.randomBytes(16);
const cipher = crypto.createCipheriv('aes-256-cbc', hash, iv);
Expand Down
5 changes: 5 additions & 0 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1012,6 +1012,11 @@ mkdirp-classic@^0.5.2:
resolved "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113"
integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==

moment@^2.29.4:
version "2.29.4"
resolved "https://registry.npmmirror.com/moment/-/moment-2.29.4.tgz#3dbe052889fe7c1b2ed966fcb3a77328964ef108"
integrity sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==

ms@2.1.2, ms@^2.1.1:
version "2.1.2"
resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz"
Expand Down

0 comments on commit 4e48f9f

Please sign in to comment.