diff --git a/docs/source/conf.py b/docs/source/conf.py index aeb7df2279..562f5a4b0e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -214,7 +214,9 @@ # html_extra_path used to define a path to robots.txt which is used by webcrawlers # to ignore or allow certain links. -html_extra_path = [str(here / "robots.txt")] +# We've decided to rely on RTD's default robots.txt, so we no longer need to specify a custom one, +# see discussion at https://github.com/kedro-org/kedro/issues/3741 +# html_extra_path = [str(here / "robots.txt")] # Removes, from all docs, the copyright footer. html_show_copyright = False diff --git a/docs/source/robots.txt b/docs/source/robots.txt deleted file mode 100644 index e8527497df..0000000000 --- a/docs/source/robots.txt +++ /dev/null @@ -1,18 +0,0 @@ -User-agent: * -Disallow: / -Allow: /en/stable/ -Allow: /en/0.19.7* -Allow: /en/0.19.6* -Allow: /en/0.19.5* -Allow: /en/0.19.4* -Allow: /en/0.19.3* -Allow: /en/0.19.2* -Allow: /en/0.19.1* -Allow: /en/0.19.0* -Allow: /en/0.18.14* -Allow: /en/0.17.7* -Allow: /projects/kedro-viz/en/stable/ -Allow: /projects/kedro-datasets/en/kedro-datasets-3.0.0* -Allow: */sitemap.xml - -Sitemap: https://docs.kedro.org/sitemap.xml