From 0f93ee67be862dcba08a1ccc4746e8199c673450 Mon Sep 17 00:00:00 2001 From: Dmitry Sorokin <40151847+DimedS@users.noreply.github.com> Date: Wed, 21 Aug 2024 17:13:07 +0100 Subject: [PATCH] Remove custom robots.txt in favor of RTD default (#4055) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove custom robots.txt Signed-off-by: Dmitry Sorokin * Remove link to robots.txt from conf.py Signed-off-by: Dmitry Sorokin * Update docs/source/conf.py Co-authored-by: Juan Luis Cano Rodríguez Signed-off-by: Dmitry Sorokin <40151847+DimedS@users.noreply.github.com> --------- Signed-off-by: Dmitry Sorokin Signed-off-by: Dmitry Sorokin <40151847+DimedS@users.noreply.github.com> Co-authored-by: Juan Luis Cano Rodríguez Co-authored-by: Nok Lam Chan --- docs/source/conf.py | 4 +++- docs/source/robots.txt | 18 ------------------ 2 files changed, 3 insertions(+), 19 deletions(-) delete mode 100644 docs/source/robots.txt diff --git a/docs/source/conf.py b/docs/source/conf.py index aeb7df2279..562f5a4b0e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -214,7 +214,9 @@ # html_extra_path used to define a path to robots.txt which is used by webcrawlers # to ignore or allow certain links. -html_extra_path = [str(here / "robots.txt")] +# We've decided to rely on RTD's default robots.txt, so we no longer need to specify a custom one, +# see discussion at https://github.com/kedro-org/kedro/issues/3741 +# html_extra_path = [str(here / "robots.txt")] # Removes, from all docs, the copyright footer. html_show_copyright = False diff --git a/docs/source/robots.txt b/docs/source/robots.txt deleted file mode 100644 index e8527497df..0000000000 --- a/docs/source/robots.txt +++ /dev/null @@ -1,18 +0,0 @@ -User-agent: * -Disallow: / -Allow: /en/stable/ -Allow: /en/0.19.7* -Allow: /en/0.19.6* -Allow: /en/0.19.5* -Allow: /en/0.19.4* -Allow: /en/0.19.3* -Allow: /en/0.19.2* -Allow: /en/0.19.1* -Allow: /en/0.19.0* -Allow: /en/0.18.14* -Allow: /en/0.17.7* -Allow: /projects/kedro-viz/en/stable/ -Allow: /projects/kedro-datasets/en/kedro-datasets-3.0.0* -Allow: */sitemap.xml - -Sitemap: https://docs.kedro.org/sitemap.xml