Refine ci of internal link checks (pingcap#9349) (pingcap#9354)

* refine cloud ci

custom content auto rm

* add tidb only check

* fix error

Co-authored-by: czhen <56986964+shczhen@users.noreply.github.com>

.github/workflows/ci.yaml

@@ -3,7 +3,7 @@ name: ci
 on: [pull_request]
 
 jobs:
-  pull:
+  tidb-check:
     runs-on: ubuntu-latest
     steps:
       - name: Check out
@@ -13,10 +13,25 @@ jobs:
           node-version: "16"
       - name: Verify duplicated file names
         run: ./scripts/verify-duplicated-file-name.sh
-      - name: Verify internal links
-        run: ./scripts/verify-links.sh
-      - name: Verify internal link anchors
-        run: ./scripts/verify-link-anchors.sh
+      - name: Verify internal links and anchors - tidb only
+        run: |
+          npm i
+          node ./scripts/filterNonCloudDoc.js
+          cp -r ./scripts ./tmp
+          cp -r ./media ./tmp
+          cp .gitignore ./tmp/
+          cd ./tmp
+          ./scripts/verify-links.sh
+          ./scripts/verify-link-anchors.sh
+
+  tidb-cloud-check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out
+        uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: "16"
       - name: Check TOC-tidb-cloud.md existence
         id: check_cloud_toc
         uses: andstor/file-existence-action@v1
@@ -27,9 +42,12 @@ jobs:
         run: |
           npm i
           node ./scripts/filterCloudDoc.js
-          cp ./scripts/verify-links.sh ./tmp/verify-links.sh
-          cd tmp
-          ./verify-links.sh
+          cp -r ./scripts ./tmp
+          cp -r ./media ./tmp
+          cp .gitignore ./tmp/
+          cd ./tmp
+          ./scripts/verify-links.sh
+          ./scripts/verify-link-anchors.sh
 
   vale:
     runs-on: ubuntu-latest

scripts/filterCloudDoc.js

@@ -1,16 +1,26 @@
 import {
   getAllMdList,
   copySingleFileSync,
-  copyDirectorySync,
+  copyFileWithCustomContentSync,
+  copyDirectoryWithCustomContentSync,
+  removeCustomContent,
 } from "./utils.js";
 
+const contentHandler = (content = "") => {
+  return removeCustomContent("tidb-cloud", content);
+};
+
 const extractFilefromList = (
   fileList = [],
   inputPath = ".",
   outputPath = "."
 ) => {
   fileList.forEach((filePath) => {
-    copySingleFileSync(`${inputPath}/${filePath}`, `${outputPath}/${filePath}`);
+    copyFileWithCustomContentSync(
+      `${inputPath}/${filePath}`,
+      `${outputPath}/${filePath}`,
+      contentHandler
+    );
   });
 };
 
@@ -19,7 +29,11 @@ const main = () => {
 
   extractFilefromList(filteredLinkList, ".", "./tmp");
   copySingleFileSync("TOC-tidb-cloud.md", "./tmp/TOC.md");
-  copyDirectorySync("./tidb-cloud/", "./tmp/tidb-cloud/");
+  copyDirectoryWithCustomContentSync(
+    "./tidb-cloud/",
+    "./tmp/tidb-cloud/",
+    contentHandler
+  );
 };
 
 main();

scripts/filterNonCloudDoc.js

@@ -0,0 +1,47 @@
+import {
+  getFileList,
+  copySingleFileSync,
+  copyFileWithCustomContentSync,
+  removeCustomContent,
+} from "./utils.js";
+
+const contentHandler = (content = "") => {
+  return removeCustomContent("tidb", content);
+};
+
+const extractFilefromList = (
+  fileList = [],
+  inputPath = ".",
+  outputPath = "."
+) => {
+  fileList.forEach((filePath = "") => {
+    if (
+      filePath.includes(`/tidb-cloud/`) ||
+      filePath.includes(`TOC-tidb-cloud.md`)
+    ) {
+      return;
+    }
+    if (filePath.endsWith(".md")) {
+      copyFileWithCustomContentSync(
+        `${inputPath}/${filePath}`,
+        `${outputPath}/${filePath}`,
+        contentHandler
+      );
+    } else {
+      try {
+        copySingleFileSync(
+          `${inputPath}/${filePath}`,
+          `${outputPath}/${filePath}`
+        );
+      } catch (error) {}
+    }
+  });
+};
+
+const main = () => {
+  const filteredLinkList = getFileList(".");
+
+  extractFilefromList(filteredLinkList, ".", "./tmp");
+};
+
+main();

scripts/utils.js

@@ -66,36 +66,49 @@ export const getAllMdList = (tocFile) => {
   return filteredLinkList;
 };
 
-export const copySingleFileSync = (srcPath, destPath) => {
+const checkDestDir = (destPath) => {
   const dir = path.dirname(destPath);
 
   if (!fs.existsSync(dir)) {
     // console.info(`Create empty dir: ${dir}`);
     fs.mkdirSync(dir, { recursive: true });
   }
+};
+
+export const copySingleFileSync = (srcPath, destPath) => {
+  checkDestDir(destPath);
 
   fs.copyFileSync(srcPath, destPath);
 };
 
-export const writeFileSync = (destPath, fileContent) => {
-  const dir = path.dirname(destPath);
+export const copyFileWithCustomContentSync = (srcPath, destPath, handler) => {
+  const srcContent = fs.readFileSync(srcPath).toString();
+  const fileContent = handler(srcContent);
+  writeFileSync(destPath, fileContent);
+};
 
-  if (!fs.existsSync(dir)) {
-    // console.info(`Create empty dir: ${dir}`);
-    fs.mkdirSync(dir, { recursive: true });
-  }
+export const writeFileSync = (destPath, fileContent) => {
+  checkDestDir(destPath);
 
   fs.writeFileSync(destPath, fileContent);
 };
 
 const getMds = (src) => {
-  return glob.sync(src + "/**/*.md");
+  return glob.sync(src + "/**/*.md", {
+    ignore: ["**/node_modules/**", "./node_modules/**"],
+  });
 };
 
 export const getMdFileList = (prefix) => {
   return getMds(prefix);
 };
 
+export const getFileList = (prefix) => {
+  return glob.sync(prefix + "/**/*", {
+    ignore: ["**/node_modules/**", "./node_modules/**"],
+  });
+};
+
 const getAllFiles = (dirPath, arrayOfFiles) => {
   const files = fs.readdirSync(dirPath);
 
@@ -120,6 +133,18 @@ export const copyDirectorySync = (srcPath, destPath) => {
   });
 };
 
+export const copyDirectoryWithCustomContentSync = (
+  srcPath,
+  destPath,
+  handler
+) => {
+  const allFiles = getAllFiles(srcPath);
+  allFiles.forEach((filePath) => {
+    const relativePath = path.relative(srcPath, filePath);
+    copyFileWithCustomContentSync(filePath, destPath + relativePath, handler);
+  });
+};
+
 export const astNode2mdStr = (astNode) => {
   const result = toMarkdown(astNode, {
     bullet: "-",
@@ -131,3 +156,14 @@ export const astNode2mdStr = (astNode) => {
   });
   return result;
 };
+
+export const removeCustomContent = (type, content = "") => {
+  const TIDB_CUSTOM_CONTENT_REGEX =
+    /<CustomContent +platform=["']tidb["'] *>(.|\n)*?<\/CustomContent>\n/g;
+  const TIDB_CLOUD_CONTENT_REGEX =
+    /<CustomContent +platform=["']tidb-cloud["'] *>(.|\n)*?<\/CustomContent>\n/g;
+  if (type === "tidb") {
+    return content.replaceAll(TIDB_CLOUD_CONTENT_REGEX, "");
+  }
+  return content.replaceAll(TIDB_CUSTOM_CONTENT_REGEX, "");
+};