feat: add new strategy; better handle 404s on empty search results

README.md

@@ -49,8 +49,8 @@ I am a techie looking for a freelance gig (project) in the European/German marke
 - ⚠️ [Darwin Recruitment][darwin-recruitment] (results not crawlable, see [_"Known Issues"_](#known-issues))
 - 🚫 ~~[etengo][etengo]~~ (cumbersome search engine, see [_"Known Issues"_](#known-issues))
 - ✅ [Austin Fraser][austin-fraser]
-- ⏱️ [Computer Futures][computer-futures] _(coming soon)_
-- ⏱️ [Michael Page][michael-page] _(coming soon)_
+- 🚫 [Computer Futures][computer-futures] ~~(Cloudflare WAF, see [_"Known Issues"_](#known-issues))~~
+- ✅ [Michael Page][michael-page]
 - ⏱️ [Constaff][constaff] _(coming soon)_
 - ⏱️ [Krongaard][krongaard] _(coming soon)_
 - ⏱️ [Amoria Bond][amoria-bond] _(coming soon)_

config.js

@@ -53,6 +53,7 @@ export const config = {
       enabled: true,
       url: "https://www.hays.de/en/jobsearch/job-offers/j/Contracting/3/p/1?q=$$$QUERY$$$",
       baseUrl: "https://www.hays.de",
+      zeroResultsYields404: false,
       getters: {
         getSingleResult: ($) => $(".search__result__header__a"),
         getResultTitle: ($, result) =>
@@ -72,6 +73,7 @@ export const config = {
       enabled: true,
       url: "https://www.darwinrecruitment.de/search-jobs/?_location=contract&_keywords=$$$QUERY$$$",
       baseUrl: "https://www.darwinrecruitment.de",
+      zeroResultsYields404: false,
       getters: {
         getSingleResult: ($) => $(".darwin_job_search_page_row"),
         getResultTitle: ($, result) =>
@@ -85,6 +87,7 @@ export const config = {
       enabled: true,
       url: "https://www.austinfraser.com/de/jobangebote/contract?query=$$$QUERY$$$",
       baseUrl: "https://www.austinfraser.com",
+      zeroResultsYields404: false,
       getters: {
         getSingleResult: ($) => $(".job-result-item"),
         getResultTitle: ($, result) =>
@@ -94,5 +97,18 @@ export const config = {
         getNextPageHref: ($) => $("span.next").find("a").attr("href"),
       },
     },
+    {
+      name: "MichaelPage",
+      enabled: true,
+      url: "https://www.michaelpage.de/jobs/$$$QUERY$$$?contract=temp",
+      baseUrl: "https://www.michaelpage.de",
+      zeroResultsYields404: true,
+      getters: {
+        getSingleResult: ($) => $(".views-row"),
+        getResultTitle: ($, result) => $(result).find("a").text().trim(),
+        getResultHref: ($, result) => $(result).find("a").attr("href"),
+        getNextPageHref: ($) => false,
+      },
+    },
   ],
 };

lib/fetch.js

@@ -5,17 +5,18 @@ import fs from "fs";
 import { buildHref } from "./utils.js";
 
 export const fetch = async (strategy, query, config) => {
-  try {
-    const { headers, logResponse } = config.fetchConfig;
-    const {
-      url,
-      baseUrl,
-      getters: { getNextPageHref },
-    } = strategy;
+  const { headers, logResponse } = config.fetchConfig;
+  const {
+    url,
+    baseUrl,
+    zeroResultsYields404,
+    getters: { getNextPageHref },
+  } = strategy;
 
-    const htmlDocuments = [];
-    let href = url.replace("$$$QUERY$$$", query);
+  let href = url.replace("$$$QUERY$$$", encodeURI(query));
 
+  const htmlDocuments = [];
+  try {
     while (href) {
       const { data } = await axios.get(href, headers);
 
@@ -34,14 +35,16 @@ export const fetch = async (strategy, query, config) => {
         href = buildHref(nextHref, baseUrl);
       }
     }
-
-    return htmlDocuments;
   } catch (error) {
-    _handleError(error);
+    _handleError(error, zeroResultsYields404);
   }
+  return htmlDocuments;
 };
 
-const _handleError = (error) => {
+const _handleError = (error, zeroResultsYields404) => {
+  if (zeroResultsYields404 && error.response.status === 404) {
+    return;
+  }
   // Shamelessly taken from: https://axios-http.com/docs/handling_errors
   if (error.response) {
     // The request was made and the server responded with a status code