Add Lychee Link Checker into OSD

1. Fix broken links in OSD 2. Generate lycheeexcude list to filter out false negative warnings from test files or external links 3. Add TODO items for internal unavaiable links 4. Integrate with doc link service change. Signed-off-by: Zuocheng Ding <zding817@gmail.com>
opensearch-project · Nov 15, 2021 · e8f44bb · e8f44bb
1 parent 75c188a
commit e8f44bb
Show file tree

Hide file tree

Showing 114 changed files with 356 additions and 176 deletions.
diff --git a/.github/workflows/doc_links_checker.yml b/.github/workflows/doc_links_checker.yml
@@ -0,0 +1,28 @@
+name: Doc Link Checker
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  linkchecker:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Load Excludes
+      run: |
+        LYCHEE_EXCLUDE=$(sed -e :a  -e 'N;s/\n/ --exclude /;ta' .lycheeexclude)
+        echo "LYCHEE_EXCLUDE=$LYCHEE_EXCLUDE" >> $GITHUB_ENV
+    - name: Lychee Link Checker
+      id: lychee
+      uses: lycheeverse/lychee-action@master
+      with:
+        args: --accept=200,403,429 --exclude ${{ env.LYCHEE_EXCLUDE }} --exclude-mail "**/*.html" "**/*.md" "**/*.txt" "**/*.json" "**/*.js" "**/*.ts" "**/*.tsx"
+      env:
+        GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+    - name: Fail if there were link errors
+      run: exit ${{ steps.lychee.outputs.exit_code }}
diff --git a/.lycheeexclude b/.lycheeexclude
@@ -0,0 +1,166 @@
+# Local or predefined end points
+http://localhost
+https://localhost
+http://127.0.0.1/
+https://127.0.0.1/
+http://127.0.0.1:10002/bar 
+http://127.0.0.1:10002/
+http://opensearch
+https://opensearch
+https://opensearch-dashboards
+http://opensearch-dashboards
+https://opensearch.internal.org/
+https://maps.search-services.aws.a2z.com/
+https://tiles.maps.search-services.aws.a2z.com/ 
+
+# Dummy urls in tests
+http://domain.
+http://www.domain.
+http://somehost
+https://somehost
+http://some.host
+https://some-host.com/
+https://other.some-host.com/
+http://test:user@somehost/
+https://some.another.host/
+http://noone.nowhere.none/
+http://bar
+http://foo
+http://test.com/
+https://files.foobar/
+https://tiles.foobar/
+https://1.1.1.1:9200/
+http://192.168.1.1:1234/
+http://9.8.7.6/
+http://1.2.3.4/
+http://8.8.8.8/
+https://path.to/
+https://example.com/
+http://example.com/
+https://example.org/
+http://some-url/
+http://buildurl/
+https://dryrun/
+https://url/
+http://url/
+http://notfound.svg/ 
+https://validurl/
+https://myopensearch-dashboardsdomain.com
+http://myopensearch-dashboardsdomain.com
+https://other-opensearch-dashboards.external:8080/
+http://myotherdomain.com:5601/
+http://myopensearch-dashboardsdomain.com:5601/
+https://your-cdn-host.com/
+http://not-your-opensearch-dashboards.com/
+http://www.mysite.com/
+http://myserver.mydomain.com:5601/
+https://elastic:changeme@myCloudInstance:9200
+https://myexternaldep.com/
+http://notlocalhost/
+http://site.com/
+http://node-b/
+http://node-a:9200/
+https://node-c/
+https://elsewhere
+https://opensearch:changeme@example.com:9200
+http://elastic:changeme@localhost:61141/
+http://test:user@somehost/
+https://mycloudinstance:9200/
+https://dev-url.co/
+https://extenal.org/_search
+http://plugins.example.com/
+https://build-url/
+http://test.com/
+https://path.to/
+http://site.com/
+http://not-your-opensearch-dashboards.com/
+http://evil.com/
+https://www.opensearch.org/cool/path
+https://www.opensearch.org/redirect
+http://www.opensearch.org/painlessDocs
+https://www.opensearch.org/subscriptions
+https://www.opensearch.org/opensearch-maps-service
+https://www.hostedgraphite.com/
+
+# External urls
+https://www.zeek.org/
+http://google.com/
+https://api.worldbank.org/
+https://vega.github.io/
+http://twitter.com/
+https://twitter.com/
+https://storage.googleapis.com/
+http://tools.ietf.org/
+https://github.com/
+http://github.com/
+http://jsperf
+https://jsperf
+https://gist.githubusercontent.com/
+https://nodejs.org/
+https://www.npmjs.com/
+https://microsoft.github.io/
+http://api.worldbank.org/
+https://f1542b814f674090afd914960583265f.apm.us-central1.gcp.cloud.es.io/
+http://www.matthewcopeland.me/
+http://threedubmedia.googlecode.com/*
+https://developer.mozilla.org/
+https://a.tile.openstreetmap.org/
+http://www.creedthoughts.gov
+https://media-for-the-masses.theacademyofperformingartsandscience.org/
+
+# Exluce files
+file:///*
+git://*
+
+# TODO: Repalce below broken links when prod replacements become available on doc website.
+https://telemetry.opensearch.org/
+https://telemetry-staging.opensearch.org/
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/cluster-pending.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/cluster-reroute.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/cluster-state.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/cluster-stats.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/modules-scripting.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/search-field-caps.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/indices-analyze.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/indices-clearcache.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/indices-synced-flush-api.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/indices-flush.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/indices-forcemerge.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/indices-get-settings.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/indices-upgrade.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/indices-update-settings.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/indices-recovery.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/indices-refresh.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/indices-rollover-index.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/indices-segments.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/indices-shards-stores.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/indices-shrink-index.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/indices-split-index.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/indices-stats.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/indices-upgrade.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/search-validate.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/delete-pipeline-api.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/get-pipeline-api.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/grok-processor.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/put-pipeline-api.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/simulate-pipeline-api.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/docs-multi-termvectors.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/cluster-nodes-hot-threads.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/secure-settings.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/cluster-nodes-stats.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/search-rank-eval.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/current/search-template.html
+https://www.opensearch.org/guide/en/elasticsearch/painless/master/painless-execute-api.html
+https://www.opensearch.org/guide/en/elasticsearch/reference/master/search-shards.html
+https://www.opensearch.org/elastic-maps-service*
+https://www.opensearch.org/guide/en/kibana/current/tutorial-load-dataset.html
+https://www.opensearch.org/guide/en/kibana/current/canvas-function-arguments.html
+https://www.opensearch.org/guide/en/beats/metricbeat/current/metricbeat-module-kibana.html
+https://www.opensearch.org/guide/en/beats/metricbeat/current/metricbeat-metricset-kibana-stats.html
+https://www.opensearch.org/downloads/beats/heartbeat
+https://www.opensearch.org/downloads/beats/metricbeat
+https://www.opensearch.org/downloads/beats/functionbeat
+https://www.opensearch.org/downloads/beats/filebeat
+https://www.opensearch.org/downloads/beats/auditbeat
+https://www.opensearch.org/downloads/apm/apm-server
+https://api.github.com/repos/opensearch-project/OpenSearch-Dashboards/
diff --git a/packages/osd-ace/src/ace/modes/lexer_rules/opensearch_sql_highlight_rules.ts b/packages/osd-ace/src/ace/modes/lexer_rules/opensearch_sql_highlight_rules.ts
@@ -35,14 +35,14 @@ const { TextHighlightRules } = ace.acequire('ace/mode/text_highlight_rules');
 const oop = ace.acequire('ace/lib/oop');
 
 export const OpenSearchSqlHighlightRules = function (this: any) {
-  // See https://www.opensearch.org/guide/en/elasticsearch/reference/current/sql-commands.html
+  // See https://opensearch.org/docs/latest/search-plugins/sql/index/
   const keywords =
     'describe|between|in|like|not|and|or|desc|select|from|where|having|group|by|order' +
     'asc|desc|pivot|for|in|as|show|columns|include|frozen|tables|escape|limit|rlike|all|distinct|is';
 
   const builtinConstants = 'true|false';
 
-  // See https://www.opensearch.org/guide/en/elasticsearch/reference/current/sql-syntax-show-functions.html
+  // See https://opensearch.org/docs/latest/search-plugins/sql/index/
   const builtinFunctions =
     'avg|count|first|first_value|last|last_value|max|min|sum|kurtosis|mad|percentile|percentile_rank|skewness' +
     '|stddev_pop|sum_of_squares|var_pop|histogram|case|coalesce|greatest|ifnull|iif|isnull|least|nullif|nvl' +
@@ -56,7 +56,7 @@ export const OpenSearchSqlHighlightRules = function (this: any) {
     '|ltrim|octet_length|position|repeat|replace|right|rtrim|space|substring|ucase|cast|convert|database|user|st_astext|st_aswkt' +
     '|st_distance|st_geometrytype|st_geomfromtext|st_wkttosql|st_x|st_y|st_z|score';
 
-  // See https://www.opensearch.org/guide/en/elasticsearch/reference/current/sql-data-types.html
+  // See https://opensearch.org/docs/latest/search-plugins/sql/index/
   const dataTypes =
     'null|boolean|byte|short|integer|long|double|float|half_float|scaled_float|keyword|text|binary|date|ip|object|nested|time' +
     '|interval_year|interval_month|interval_day|interval_hour|interval_minute|interval_second|interval_year_to_month' +

diff --git a/packages/osd-dev-utils/certs/README.md b/packages/osd-dev-utils/certs/README.md
@@ -28,7 +28,7 @@ The password used for both of these is "storepass". Other copies are also provid
 
 ## Certificate generation
 
-[OpenSearch cert-util](https://www.opensearch.org/guide/en/elasticsearch/reference/current/certutil.html) and [OpenSSL](https://www.openssl.org/) were used to generate these certificates. The following commands were used from the root directory of OpenSearch:
+[OpenSearch cert-util](https://opensearch.org/docs/latest/security-plugin/configuration/generate-certificates/) and [OpenSSL](https://www.openssl.org/) were used to generate these certificates. The following commands were used from the root directory of OpenSearch:
 
 ```
 # Generate the PKCS #12 keystore for a CA, valid for 50 years

diff --git a/packages/osd-spec-to-console/test/fixtures/cluster_health_autocomplete.json b/packages/osd-spec-to-console/test/fixtures/cluster_health_autocomplete.json
@@ -40,6 +40,6 @@
       "_cluster/health",
       "_cluster/health/{indices}"
     ],
-    "documentation": "https://www.opensearch.org/guide/en/elasticsearch/reference/master/cluster-health.html"
+    "documentation": "https://opensearch.org/docs/latest/opensearch/rest-api/cluster-health/"
   }
 }
diff --git a/packages/osd-spec-to-console/test/fixtures/cluster_health_spec.json b/packages/osd-spec-to-console/test/fixtures/cluster_health_spec.json
@@ -1,7 +1,7 @@
 {
   "cluster.health":{
     "documentation":{
-      "url":"https://www.opensearch.org/guide/en/elasticsearch/reference/master/cluster-health.html",
+      "url":"https://opensearch.org/docs/latest/opensearch/rest-api/cluster-health/",
       "description":"Returns basic information about the health of the cluster."
     },
     "stability":"stable",

diff --git a/src/core/public/chrome/chrome_service.tsx b/src/core/public/chrome/chrome_service.tsx
@@ -215,7 +215,11 @@ export class ChromeService {
             defaultMessage="Support for Internet Explorer will be dropped in future versions of this software, please check {link}."
             values={{
               link: (
-                <EuiLink target="_blank" href="https://www.opensearch.org/support/matrix" external>
+                <EuiLink
+                  target="_blank"
+                  href={docLinks.links.opensearchDashboards.browser}
+                  external
+                >
                   <FormattedMessage
                     id="core.chrome.browserDeprecationLink"
                     defaultMessage="the support matrix on our website"

diff --git a/src/core/public/chrome/constants.ts b/src/core/public/chrome/constants.ts
@@ -30,12 +30,6 @@
  * GitHub history for details.
  */
 
-// export const OPENSEARCH_DASHBOARDS_FEEDBACK_LINK =
-//  'https://www.opensearch.org/products/kibana/feedback?blade=kibanafeedback';
-// export const OPENSEARCH_DASHBOARDS_ASK_OPENSEARCH_LINK =
-// 'https://www.opensearch.org/products/kibana/ask-elastic?blade=kibanaaskelastic';
-// export const GITHUB_CREATE_ISSUE_LINK = 'https://github.com/opensearch-project/OpenSearch-Dashboards/issues/new/choose';
-
 export const OPENSEARCH_DASHBOARDS_FEEDBACK_LINK = 'https://github.com/opensearch-project';
 export const OPENSEARCH_DASHBOARDS_ASK_OPENSEARCH_LINK = 'https://github.com/opensearch-project';
 export const GITHUB_CREATE_ISSUE_LINK =

diff --git a/src/core/public/chrome/ui/header/header_help_menu.tsx b/src/core/public/chrome/ui/header/header_help_menu.tsx
@@ -78,7 +78,7 @@ export type ChromeHelpExtensionMenuDiscussLink = EuiButtonEmptyProps & {
   linkType: 'discuss';
   /**
    * URL to discuss page.
-   * i.e. `https://discuss.opensearch.org/c/${appName}`
+   * i.e. https://discuss.opendistrocommunity.dev/
    */
   href: string;
 };

diff --git a/src/core/public/doc_links/doc_links_service.ts b/src/core/public/doc_links/doc_links_service.ts
@@ -329,7 +329,17 @@ export class DocLinksService {
           // https://opensearch.org/docs/latest/opensearch/popular-api/
           popularAPI: `${OPENSEARCH_VERSIONED_DOCS}popular-api`,
           // https://opensearch.org/docs/latest/opensearch/rest-api/index/
-          restAPI: `${OPENSEARCH_VERSIONED_DOCS}rest-api/index/`,
+          restAPI: {
+            base: `${OPENSEARCH_VERSIONED_DOCS}rest-api/index/`,
+            indexAPI: {
+              base: `${OPENSEARCH_VERSIONED_DOCS}rest-api/index-apis/index/`,
+              create: `${OPENSEARCH_VERSIONED_DOCS}rest-api/index-apis/create-index/`,
+              exists: `${OPENSEARCH_VERSIONED_DOCS}rest-api/index-apis/exists/`,
+              delete: `${OPENSEARCH_VERSIONED_DOCS}rest-api/index-apis/delete-index/`,
+              get: `${OPENSEARCH_VERSIONED_DOCS}rest-api/index-apis/get-index/`,
+              close: `${OPENSEARCH_VERSIONED_DOCS}rest-api/index-apis/close-index/`,
+            },
+          },
         },
         opensearchDashboards: {
           // https://opensearch.org/docs/latest/dashboards/index/
@@ -378,6 +388,7 @@ export class DocLinksService {
             // https://opensearch.org/docs/latest/dashboards/dql/#nested-field-query
             nested_query: `${OPENSEARCH_DASHBOARDS_VERSIONED_DOCS}dql/#nested-field-query`,
           },
+          browser: `${OPENSEARCH_DASHBOARDS_VERSIONED_DOCS}browser-compatibility`,
         },
         noDocumentation: {
           auditbeat: `${OPENSEARCH_WEBSITE_DOCS}`,
@@ -577,7 +588,17 @@ export interface DocLinksStart {
       readonly supportedUnits: string;
       readonly commonParameters: string;
       readonly popularAPI: string;
-      readonly restAPI: string;
+      readonly restAPI: {
+        readonly base: string;
+        readonly indexAPI: {
+          readonly base: string;
+          readonly create: string;
+          readonly exists: string;
+          readonly delete: string;
+          readonly get: string;
+          readonly close: string;
+        };
+      };
     };
     readonly opensearchDashboards: {
       readonly introduction: string;
@@ -606,6 +627,7 @@ export interface DocLinksStart {
         readonly date_query: string;
         readonly nested_query: string;
       };
+      readonly browser: string;
     };
     readonly noDocumentation: {
       readonly auditbeat: string;

diff --git a/src/core/public/public.api.md b/src/core/public/public.api.md
@@ -609,7 +609,17 @@ export interface DocLinksStart {
             readonly supportedUnits: string;
             readonly commonParameters: string;
             readonly popularAPI: string;
-            readonly restAPI: string;
+            readonly restAPI: {
+                readonly base: string;
+                readonly indexAPI: {
+                    readonly base: string,
+                    readonly create: string,
+                    readonly exists: string,
+                    readonly delete: string,
+                    readonly get: string,
+                    readonly close: string,
+                },
+            };
         };
         readonly opensearchDashboards: {
             readonly introduction: string;
@@ -638,6 +648,7 @@ export interface DocLinksStart {
                 readonly date_query: string;
                 readonly nested_query: string;
             };
+            readonly browser: string;
         };
         readonly noDocumentation: {
             readonly auditbeat: string;

diff --git a/src/core/server/saved_objects/mappings/types.ts b/src/core/server/saved_objects/mappings/types.ts
@@ -115,7 +115,7 @@ export interface SavedObjectsMappingProperties {
 /**
  * Describe a {@link SavedObjectsTypeMappingDefinition | saved object type mapping} field.
  *
- * Please refer to {@link https://www.opensearch.org/guide/en/elasticsearch/reference/current/mapping-types.html | elasticsearch documentation}
+ * Please refer to {@link https://opensearch.org/docs/latest/ | elasticsearch documentation}
  * For the mapping documentation
  *
  * @public