Newtoki (#2528)

* newtoki fix also newtoki = manatoki like exactly resolves #2475 * actually the same * Update Manatoki.mjs * fix set newtoki to newtoki83 for now * lint Co-authored-by: 09morbab <30987265+09morbab@users.noreply.github.com>
manga-download · Nov 1, 2020 · f0584ca · f0584ca
1 parent 5f20723
commit f0584ca
Show file tree

Hide file tree

Showing 5 changed files with 52 additions and 76 deletions.
diff --git a/src/web/img/connectors/manatoki82 → src/web/img/connectors/manatoki b/src/web/img/connectors/manatoki82 → src/web/img/connectors/manatoki
diff --git a/src/web/mjs/connectors/CopyToon.mjs b/src/web/mjs/connectors/CopyToon.mjs
@@ -18,7 +18,7 @@ export default class CopyToon extends GnuBoard5BootstrapBasic2 {
         this.config = {
             url: {
                 label: 'URL',
-                description: 'This website changes their URL regualry.\nThis is the last known URL which can also be manually set by the user.',
+                description: 'This website changes their URL regularly.\nThis is the last known URL which can also be manually set by the user.',
                 input: 'text',
                 value: 'https://copytoon104.com'
             }

diff --git a/src/web/mjs/connectors/Manatoki.mjs b/src/web/mjs/connectors/Manatoki.mjs
@@ -4,37 +4,54 @@ export default class Manatoki extends GnuBoard5BootstrapBasic2 {
 
     constructor() {
         super();
-        super.id = 'manatoki82';
+        super.id = 'manatoki';
         super.label = 'Manatoki';
-        this.tags = [ 'webtoon', 'korean' ];
-        this.url = 'https://manatoki82.net';
+        this.tags = [ 'manga', 'webtoon', 'korean' ];
+        this.url = 'https://manatoki.net';
 
-        this.path = '/comic/p%PAGE%';
-        this.pathMatch = /comic\/p(\d+)/;
+        this.path = [ '/webtoon', '/comic' ];
         this.queryMangasPageCount = 'div.list-page ul.pagination li:last-child a';
         this.queryMangas = 'ul#webtoon-list-all li div.img-item div.in-lable a';
         this.queryManga = 'meta[name="subject"]';
         this.queryChapters = 'div.serial-list li.list-item div.wr-subject a';
-        this.scriptPages =`
+        this.scriptPages = `
         new Promise(resolve => {
-            let queryPages = 'div.view-padding div > img';
-            if ([...document.querySelectorAll(queryPages)].length == 0){
-                queryPages = 'div.view-padding div > p:not([class]) img';
-            };
-            resolve([...document.querySelectorAll(queryPages)].map(image => JSON.stringify(image.dataset).match(/"\\S{11}":"(.*)"/)[1]));
+            const images = [...document.querySelectorAll('div.view-padding div > img, div.view-padding div > p:not([class]) img')];
+            resolve(images.map(image => JSON.stringify(image.dataset).match(/"\\S{11}":"(.*)"/)[1]));
         });
         `;
 
     }
+    canHandleURI(uri) {
+        return /https?:\/\/manatoki\d*.net/.test(uri.origin);
+
+    }
+
+    async _initializeConnector() {
+        /*
+         * sometimes cloudflare bypass will fail, because chrome successfully loads the page from its cache
+         * => append random search parameter to avoid caching
+         */
+        let uri = new URL(this.url);
+        uri.searchParams.set('ts', Date.now());
+        uri.searchParams.set('rd', Math.random());
+        let request = new Request(uri.href, this.requestOptions);
+        this.url = await Engine.Request.fetchUI(request, `window.location.origin`);
+        this.requestOptions.headers.set('x-referer', this.url);
+        console.log(`Assigned URL '${this.url}' to ${this.label}`);
+    }
 
     async _getMangas() {
         let mangaList = [];
-        let request = new Request(new URL(this.path.replace('%PAGE%', 1), this.url), this.requestOptions);
-        let data = await this.fetchDOM(request, this.queryMangasPageCount);
-        let pageCount = parseInt(data[0].href.match(this.pathMatch)[1]);
-        for(let page = 1; page <= pageCount; page++) {
-            let mangas = await this._getMangasFromPage(this.path.replace('%PAGE%', page));
-            mangaList.push(...mangas);
+        for(let path of this.path) {
+            let uri = new URL(path, this.url);
+            let request = new Request(uri, this.requestOptions);
+            let data = await this.fetchDOM(request, this.queryMangasPageCount);
+            let pageCount = parseInt(data[0].href.match(/\d+$/)[0]);
+            for(let page = 1; page <= pageCount; page++) {
+                let mangas = await this._getMangasFromPage(path + '/p' + page);
+                mangaList.push(...mangas);
+            }
         }
         return mangaList;
     }

diff --git a/src/web/mjs/connectors/NewToki.mjs b/src/web/mjs/connectors/NewToki.mjs
@@ -1,15 +1,25 @@
-import Connector from '../engine/Connector.mjs';
-import Manga from '../engine/Manga.mjs';
+import GnuBoard5BootstrapBasic2 from './templates/GnuBoard5BootstrapBasic2.mjs';
 
-export default class NewToki extends Connector {
+export default class NewToki extends GnuBoard5BootstrapBasic2 {
 
     constructor() {
         super();
         super.id = 'newtoki';
         super.label = 'NewToki';
         this.tags = [ 'manga', 'webtoon', 'korean' ];
-        this.url = 'https://newtoki.com';
+        this.url = 'https://newtoki83.com';
+
         this.path = [ '/webtoon', '/comic' ];
+        this.queryMangasPageCount = 'div.list-page ul.pagination li:last-child a';
+        this.queryMangas = 'ul#webtoon-list-all li div.img-item div.in-lable a';
+        this.queryManga = 'meta[name="subject"]';
+        this.queryChapters = 'div.serial-list li.list-item div.wr-subject a';
+        this.scriptPages = `
+        new Promise(resolve => {
+            const images = [...document.querySelectorAll('div.view-padding div > img, div.view-padding div > p:not([class]) img')];
+            resolve(images.map(image => JSON.stringify(image.dataset).match(/"\\S{11}":"(.*)"/)[1]));
+        });
+        `;
     }
 
     canHandleURI(uri) {
@@ -30,70 +40,19 @@ export default class NewToki extends Connector {
         console.log(`Assigned URL '${this.url}' to ${this.label}`);
     }
 
-    async _getMangaFromURI(uri) {
-        let request = new Request(uri, this.requestOptions);
-        let data = await this.fetchDOM(request, 'div.page-title span.page-desc');
-        let id = uri.pathname + uri.search;
-        let title = data[0].textContent.trim();
-        return new Manga(this, id, title);
-    }
-
     async _getMangas() {
         let mangaList = [];
         for(let path of this.path) {
             let uri = new URL(path, this.url);
             let request = new Request(uri, this.requestOptions);
-            let data = await this.fetchDOM(request, 'section.board-list ul.pagination li:last-of-type a');
+            let data = await this.fetchDOM(request, this.queryMangasPageCount);
             let pageCount = parseInt(data[0].href.match(/\d+$/)[0]);
             for(let page = 1; page <= pageCount; page++) {
                 await this.wait(2500);
-                let mangas = await this._getMangasFromPage(path, page);
+                let mangas = await this._getMangasFromPage(path + '/p' + page);
                 mangaList.push(...mangas);
             }
         }
         return mangaList;
     }
-
-    async _getMangasFromPage(path, page) {
-        let script = `
-            new Promise(resolve => {
-                let mangas = [...document.querySelectorAll('section.board-list div.list-container ul.list div.list-item div.in-lable a')].map(a => {
-                    return {
-                        id: a.pathname,
-                        title: a.text.trim()
-                    };
-                });
-                setTimeout(() => resolve(mangas), 2500);
-            });
-        `;
-        let uri = new URL(path + '/p' + page, this.url);
-        let request = new Request(uri, this.requestOptions);
-        let mangas = await Engine.Request.fetchUI(request, script);
-        // HACK: Objects returned by fetchUI are lazy, therefore they will massively slow down or even freeze postprcessing such as removing duplicates
-        //       Clone object with JSON to create a non-lazy obect ...
-        return JSON.parse(JSON.stringify(mangas));
-    }
-
-    async _getChapters(manga) {
-        let uri = new URL(manga.id, this.url);
-        let request = new Request(uri, this.requestOptions);
-        let data = await this.fetchDOM(request, 'div.serial-list ul.list-body li.list-item');
-        return data.map(element => {
-            let link = element.querySelector('div.wr-subject a.item-subject');
-            let number = element.querySelector('div.wr-num').textContent.trim();
-            let title = link.childNodes[0].textContent.trim() || link.childNodes[2].textContent.trim();
-            return {
-                id: this.getRootRelativeOrAbsoluteLink(link, request.url),
-                title: number + ' - ' + title.replace(manga.title, '').trim(),
-                language: ''
-            };
-        });
-    }
-
-    async _getPages(chapter) {
-        let uri = new URL(chapter.id, this.url);
-        let request = new Request(uri, this.requestOptions);
-        let data = await this.fetchDOM(request, 'article div.view-content source');
-        return data.map(element => this.getAbsolutePath(element.dataset.original, request.url));
-    }
 }
diff --git a/src/web/mjs/connectors/templates/GnuBoard5BootstrapBasic2.mjs b/src/web/mjs/connectors/templates/GnuBoard5BootstrapBasic2.mjs
@@ -89,7 +89,7 @@ export default class GnuBoard5BootstrapBasic2 extends Connector {
 
     async _getMangasFromPage(page) {
         let request = new Request(new URL(page, this.url), this.requestOptions);
-        let data = await this.fetchDOM(request, this.queryMangas);
+        let data = await this.fetchDOM(request, this.queryMangas, 3);
         return data.map(element => {
             return {
                 id: this.getRootRelativeOrAbsoluteLink(element, this.url),