[A] very first version: simple file-per-table tablespace parse

Author: c-rainstorm

README.md

@@ -8,6 +8,7 @@ Innodb 数据页解析工具
 
 - Docker for Desktop
 - JRE 1.8
+- Maven
 
 ### 快速构建 innodb-parser
 
@@ -33,6 +34,45 @@ alias innodb_parser="java -jar $(pwd)/target/innodb-parser-*.jar"
 innodb_parser -h
 ```
 
+### Usage
+
+```bash
+$ innodb_parser -h
+
+usage: java -jar /path/to/your/innodb-parser.jar  [OPTION]...
+根据选项解析 Innodb 数据文件，具体操作完整 DEMO 请查看 README
+====================================
+ -d,--database <arg>                 需要分析的数据库名称
+ -h,--help                           打印帮助文档
+ -p,--page <arg>                     需要分析的表空间页号，页号从 0 开始
+ -r,--root-dir-of-data <arg>         数据目录，所有表空间默认在该目录下, 默认值 /var/lib/mysql
+ -s,--system-tablespace-file <arg>   系统表空间文件路径，-r 的相对路径, 默认值 ibdata1
+ -t,--table <arg>                    需要分析的表名
+ -V,--verbose                        打印更详细的信息
+ -v,--version                        打印版本号
+====================================
+如有问题，可以联系 pom.xml 中的开发者
+```
+
 ### 操作指北
 
-// TODO
+### 独立表空间页结构分析
+
+```bash
+
+$ innodb_parser -r=/tmp/mysql -d=sparrow -t=test
+
+[    PageNo][   SpaceID]Page<PageType> ...
+------------------------------------------
+[         0][        23]Page<FIL_PAGE_TYPE_FSP_HDR> 
+[         1][        23]Page<FIL_PAGE_IBUF_BITMAP> 
+[         2][        23]Page<FIL_PAGE_INODE> 
+[         3][        23]Page<FIL_PAGE_INDEX> 
+[         4][        23]Page<FIL_PAGE_INDEX> 
+[          ][          ]Page<FIL_PAGE_TYPE_ALLOCATED>
+[          ][          ]Page<FIL_PAGE_TYPE_ALLOCATED>
+```
+
+### Tips
+
+- **i18n 部分，IDEA 建议勾选 Preferences | Editor | File Encodings 的 Transparent native-to-ascii conversion，中文在IDEA就可以正常展示了**

iprc

@@ -0,0 +1,7 @@
+CONTAINER_NAME=mysql
+MOUNT_VOLUME=/tmp/mysql
+IMAGE=mysql:5.7
+MY_CONF_DIR=$(pwd)/conf.d
+INIT_DB_DIR=$(pwd)/docker-entrypoint-initdb.d
+PASSWORD_DIR=~/.password
+MYSQL_PASS_FILE=${PASSWORD_DIR}/mysql

run-mysql-in-docker.sh

@@ -1,69 +1,37 @@
 #!/bin/bash
 
-function stopContainer(){
-    CONTAINER_NAME=$1
-    if [ "$(docker container ls --filter=name="${CONTAINER_NAME}" -q | wc -l)" -gt 0 ]; then
-        echo "stopping container [${CONTAINER_NAME}]"
-        docker container stop "${CONTAINER_NAME}"
-        echo "done"
-    fi
+source iprc
+source tools.sh
 
-    if [ "$(docker container ls -a --filter=name="${CONTAINER_NAME}" -q | wc -l)" -gt 0 ]; then
-        echo "removing container [${CONTAINER_NAME}]"
-        docker container rm "${CONTAINER_NAME}"
-        echo "done"
-    fi
-}
+ensureImageExist "${IMAGE}"
 
-function ensureImageExist() {
-    image=$1
-    if [ "$(docker images --filter=reference="${image}" -q | wc -l)" -eq 0 ]; then
-        echo "Image [${image}] not found, try to pull"
-        if [ "$(docker pull "${image}")" ]; then
-            echo "Image [${image}] pull success";
-        else
-            echo "Image [${image}] pull error! "
-            exit 1;
-        fi
-    fi
-}
+mkdir -p "${MOUNT_VOLUME}"
+mkdir -p "${PASSWORD_DIR}"
 
-CONTAINER_NAME=mysql
-MOUNT_VOLUME=/tmp/mysql
-IMAGE=mysql:5.7
-MY_CONF_DIR=$(pwd)/conf.d
-INIT_DB_DIR=$(pwd)/docker-entrypoint-initdb.d
-
-ensureImageExist ${IMAGE}
-
-rm -rf ${MOUNT_VOLUME}
-mkdir -p ${MOUNT_VOLUME}
-
-mkdir -p ~/.password
-MYSQL_PASS_FILE=~/.password/mysql
 echo "######################################################"
-if [ ! -e ${MYSQL_PASS_FILE} ] || [ $(cat ${MYSQL_PASS_FILE} | wc -l) -eq 0 ]; then
-    openssl rand -base64 10 > ${MYSQL_PASS_FILE}
+# shellcheck disable=SC2046
+# shellcheck disable=SC2002
+if [ ! -e "${MYSQL_PASS_FILE}" ] || [ $(cat "${MYSQL_PASS_FILE}" | wc -l) -eq 0 ]; then
+    openssl rand -base64 10 >"${MYSQL_PASS_FILE}"
     echo "new password generated, find it in ${MYSQL_PASS_FILE}"
 else
     echo "use password in ${MYSQL_PASS_FILE} generated before"
 fi
 echo "######################################################"
 
-stopContainer ${CONTAINER_NAME}
-
-docker run  --name ${CONTAINER_NAME} -p 3306:3306 \
-  -v ${MY_CONF_DIR}:/etc/mysql/conf.d             \
-  -v ${MOUNT_VOLUME}:/var/lib/mysql               \
-  -v ${INIT_DB_DIR}:/docker-entrypoint-initdb.d   \
-  -e MYSQL_ROOT_PASSWORD="$(cat "${MYSQL_PASS_FILE}")"  \
-  -d ${IMAGE}
+stopContainer "${CONTAINER_NAME}"
 
+docker run --name "${CONTAINER_NAME}" -p 3306:3306 \
+    -v "${MY_CONF_DIR}":/etc/mysql/conf.d \
+    -v "${MOUNT_VOLUME}":/var/lib/mysql \
+    -v "${INIT_DB_DIR}":/docker-entrypoint-initdb.d \
+    -e MYSQL_ROOT_PASSWORD="$(cat "${MYSQL_PASS_FILE}")" \
+    -d "${IMAGE}"
 
 echo "#######################DB 配置#######################"
-cat ${MY_CONF_DIR}/*
+cat "${MY_CONF_DIR}"/*
 echo "#######################DB 配置#######################"
 
 echo "#######################DB 初始化#######################"
-cat ${INIT_DB_DIR}/*
+cat "${INIT_DB_DIR}"/*
 echo "#######################DB 初始化#######################"

src/main/java/me/rainstorm/innodb/common/LRUCache.java

@@ -0,0 +1,21 @@
+package me.rainstorm.innodb.common;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * @author traceless
+ */
+public class LRUCache<K, V> extends LinkedHashMap<K, V> {
+    private final int cacheSize;
+
+    public LRUCache(int cacheSize) {
+        super(cacheSize, 0.75f, true);
+        this.cacheSize = cacheSize;
+    }
+
+    @Override
+    protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
+        return size() >= cacheSize;
+    }
+}

src/main/java/me/rainstorm/innodb/common/StreamUtil.java

@@ -0,0 +1,27 @@
+package me.rainstorm.innodb.common;
+
+import java.util.Iterator;
+import java.util.Spliterators;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+/**
+ * @author traceless
+ */
+public class StreamUtil {
+    public static <T> Stream<T> of(Iterator<T> iterator) {
+        return of(iterator, false);
+    }
+
+    public static <T> Stream<T> of(Iterator<T> iterator, boolean parallel) {
+        return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, 0), parallel);
+    }
+
+    public static <T> Stream<T> of(Iterator<T> iterator, int size) {
+        return of(iterator, size, false);
+    }
+
+    public static <T> Stream<T> of(Iterator<T> iterator, int size, boolean parallel) {
+        return StreamSupport.stream(Spliterators.spliterator(iterator, size, 0), parallel);
+    }
+}

src/main/java/me/rainstorm/innodb/common/i18n/I18nUtil.java

@@ -0,0 +1,26 @@
+package me.rainstorm.innodb.common.i18n;
+
+import java.text.MessageFormat;
+import java.util.Locale;
+import java.util.ResourceBundle;
+
+import static me.rainstorm.innodb.parser.ParserConstants.LOCALE;
+
+/**
+ * @author traceless
+ */
+public class I18nUtil {
+    private static ResourceBundle rb;
+
+    static {
+        setLocale(LOCALE);
+    }
+
+    public static void setLocale(Locale locale) {
+        I18nUtil.rb = ResourceBundle.getBundle("i18n/message", locale);
+    }
+
+    public static String message(String code, Object... args) {
+        return MessageFormat.format(rb.getString(code), args);
+    }
+}

src/main/java/me/rainstorm/innodb/common/i18n/SupportedLocaleEnum.java

@@ -0,0 +1,27 @@
+package me.rainstorm.innodb.common.i18n;
+
+import lombok.Getter;
+
+import java.util.Locale;
+
+/**
+ * @author traceless
+ */
+
+@Getter
+public enum SupportedLocaleEnum {
+    /**
+     * 英文
+     */
+    EN_US(Locale.US),
+    /**
+     * 中文
+     */
+    ZH_CN(Locale.CHINA);
+
+    SupportedLocaleEnum(Locale locale) {
+        this.locale = locale;
+    }
+
+    private final Locale locale;
+}

src/main/java/me/rainstorm/innodb/domain/InnodbConstants.java

@@ -0,0 +1,50 @@
+package me.rainstorm.innodb.domain;
+
+/**
+ * @author traceless
+ */
+public class InnodbConstants {
+    public static final String MYSQL_INNODB_VERSION = "5.7.32";
+
+    public static int ONE_K = 1024;
+    public static int ONE_M = ONE_K * ONE_K;
+    public static int PAGE_SIZE = 16 * ONE_K;
+
+    /**
+     * storage/innobase/include/fsp0types.h
+     * <p>
+     * File space extent size in pages
+     * <p>
+     * page size | file space extent size
+     * <p>
+     * ----------+-----------------------
+     * <p>
+     * 4 KiB  | 256 pages = 1 MiB
+     * <p>
+     * 8 KiB  | 128 pages = 1 MiB
+     * <p>
+     * 16 KiB  |  64 pages = 1 MiB
+     * <p>
+     * 32 KiB  |  64 pages = 2 MiB
+     * <p>
+     * 64 KiB  |  64 pages = 4 MiB
+     * <p>
+     * #define FSP_EXTENT_SIZE((UNIV_PAGE_SIZE <=(16384) ?
+     * <p>
+     * (1048576/UNIV_PAGE_SIZE):    \
+     * <p>
+     * ((UNIV_PAGE_SIZE <=(32768))?    \
+     * <p>
+     * (2097152/UNIV_PAGE_SIZE):    \
+     * <p>
+     * (4194304/UNIV_PAGE_SIZE))))
+     */
+    public static int PAGE_NUM_IN_EXTEND =
+            PAGE_SIZE <= 16 * ONE_K ?
+                    ONE_M / PAGE_SIZE :
+                    PAGE_SIZE <= 32 * ONE_K ?
+                            2 * ONE_M / PAGE_SIZE :
+                            4 * ONE_M / PAGE_SIZE;
+
+    public static int EXTEND_SIZE = PAGE_SIZE * PAGE_NUM_IN_EXTEND;
+}

src/main/java/me/rainstorm/innodb/domain/extend/Extend.java

@@ -0,0 +1,47 @@
+package me.rainstorm.innodb.domain.extend;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import me.rainstorm.innodb.domain.page.LogicPage;
+import me.rainstorm.innodb.domain.page.LogicPageFactory;
+import me.rainstorm.innodb.domain.page.PhysicalPage;
+import me.rainstorm.innodb.domain.page.core.PageBody;
+import me.rainstorm.innodb.domain.tablespace.TableSpace;
+
+import java.nio.ByteBuffer;
+
+import static me.rainstorm.innodb.domain.InnodbConstants.PAGE_NUM_IN_EXTEND;
+import static me.rainstorm.innodb.domain.InnodbConstants.PAGE_SIZE;
+
+/**
+ * 表空间的一个区，
+ *
+ * @author traceless
+ */
+@Slf4j
+@Getter
+public class Extend {
+    private final TableSpace tableSpace;
+    private final int extendOffset;
+    private final ByteBuffer data;
+
+    public Extend(TableSpace tableSpace, int extendOffset, ByteBuffer data) {
+        this.tableSpace = tableSpace;
+        this.extendOffset = extendOffset;
+        this.data = data;
+    }
+
+    /**
+     * @param pageOffsetInExtend extend 内页偏移量
+     * @return Page
+     */
+    public <Page extends LogicPage<? extends PageBody>> Page page(int pageOffsetInExtend) {
+        PhysicalPage physicalPage = new PhysicalPage(this, pageOffsetInExtend,
+                ByteBuffer.wrap(data.array(), pageOffsetInExtend * PAGE_SIZE, PAGE_SIZE).asReadOnlyBuffer());
+        return (Page) LogicPageFactory.of(physicalPage);
+    }
+
+    public static int pageOffsetOfExtend(int pageNo) {
+        return pageNo % PAGE_NUM_IN_EXTEND;
+    }
+}

src/main/java/me/rainstorm/innodb/domain/page/AllocatedPage.java

@@ -0,0 +1,22 @@
+package me.rainstorm.innodb.domain.page;
+
+import me.rainstorm.innodb.domain.page.core.Undefined;
+
+/**
+ * @author traceless
+ */
+public class AllocatedPage extends LogicPage<Undefined> {
+    public AllocatedPage(PhysicalPage physicalPage) {
+        super(physicalPage);
+    }
+
+    @Override
+    protected Undefined createPageBody(PhysicalPage physicalPage) {
+        return Undefined.INSTANCE;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("[%10s][%10s]Page<%s>", "", "", fileHeader.getPageType().getMessageKey());
+    }
+}