Adding details to README.md and config.ts as well as extra formatting.

README.md

@@ -64,25 +64,31 @@ export const defaultConfig: Config = {
 };
 ```
 
-See [config.ts](src/config.ts) for all available options. Here is a sample of the common configu options:
+See [config.ts](src/config.ts) for all available options. Here is a sample of the common config options:
 
 ```ts
 type Config = {
-  /** URL to start the crawl, if sitemap is provided then it will be used instead and download all pages in the sitemap */
+
+  /** Required - URL to start the crawl, if sitemap is provided then it will be used instead and download all pages in the sitemap */
   url: string;
-  /** Pattern to match against for links on a page to subsequently crawl */
+
+  /** Required - Pattern to match against for links on a page to subsequently crawl */
   match: string;
-  /** Selector to grab the inner text from */
+
+  /** Optional - Selector to grab the inner text from */
   selector: string;
 
   /** Optional - Don't crawl more than this many pages (0 = Crawl all, Default = 50)*/
   maxPagesToCrawl: number;
-  /** File name for the finished data */
+
+  /** Optional - File name for the finished data */
   outputFileName: string;
 
   /** Optional - Timeout for waiting for a selector to appear */
   waitForSelectorTimeout: number;
 
+  /** Optional - Resource file extensions to exclude from crawl
+   * 
    * @example
    * ['png','jpg','jpeg','gif','svg','css','js','ico','woff','woff2','ttf','eot','otf','mp4','mp3','webm','ogg','wav','flac','aac','zip','tar','gz','rar','7z','exe','dmg','apk','csv','xls','xlsx','doc','docx','pdf','epub','iso','dmg','bin','ppt','pptx','odt','avi','mkv','xml','json','yml','yaml','rss','atom','swf','txt','dart','webp','bmp','tif','psd','ai','indd','eps','ps','zipx','srt','wasm','m4v','m4a','webp','weba','m4b','opus','ogv','ogm','oga','spx','ogx','flv','3gp','3g2','jxr','wdp','jng','hief','avif','apng','avifs','heif','heic','cur','ico','ani','jp2','jpm','jpx','mj2','wmv','wma','aac','tif','tiff','mpg','mpeg','mov','avi','wmv','flv','swf','mkv','m4v','m4p','m4b','m4r','m4a','mp3','wav','wma','ogg','oga','webm','3gp','3g2','flac','spx','amr','mid','midi','mka','dts','ac3','eac3','weba','m3u','m3u8','ts','wpl','pls','vob','ifo','bup','svcd','drc','dsm','dsv','dsa','dss','vivo','ivf','dvd','fli','flc','flic','flic','mng','asf','m2v','asx','ram','ra','rm','rpm','roq','smi','smil','wmf','wmz','wmd','wvx','wmx','movie','wri','ins','isp','acsm','djvu','fb2','xps','oxps','ps','eps','ai','prn','svg','dwg','dxf','ttf','fnt','fon','otf','cab']
    */

src/config.ts

@@ -6,20 +6,23 @@ const Page: z.ZodType<Page> = z.any();
 
 export const configSchema = z.object({
   /**
+   * **Required:**
    * URL to start the crawl, if url is a sitemap, it will crawl all pages in the sitemap
    * @example "https://www.builder.io/c/docs/developers"
    * @example "https://www.builder.io/sitemap.xml"
    * @default ""
    */
   url: z.string(),
   /**
+   * **Required:**
    * Pattern to match against for links on a page to subsequently crawl
    * @example "https://www.builder.io/c/docs/**"
    * @default ""
    */
   match: z.string().or(z.array(z.string())),
 
   /**
+   * **Optional:**
    * Selector to grab the inner text from
    * @example ".docs-builder-container"
    * @default ""
@@ -32,18 +35,25 @@ export const configSchema = z.object({
    */
   maxPagesToCrawl: z.number().int().nonnegative().or(z.undefined()).optional(),
   /**
+   * **Optional:**
    * File name for the finished data
    * @default "output.json"
    */
   outputFileName: z.string(),
-  /** Optional cookie to be set. E.g. for Cookie Consent */
+  /** 
+   * **Optional:** 
+   * Cookie to be set. E.g. for Cookie Consent 
+   * */
   cookie: z
     .object({
       name: z.string(),
       value: z.string(),
     })
     .optional(),
-  /** Optional function to run for each page found */
+  /** 
+   * **Optional:** 
+   * Function to run for each page found 
+   * */
   onVisitPage: z
     .function()
     .args(
@@ -56,18 +66,25 @@ export const configSchema = z.object({
     .optional(),
   /** Optional timeout for waiting for a selector to appear */
   waitForSelectorTimeout: z.number().int().nonnegative().optional(),
-  /** Optional resources to exclude
+     /** 
+     * **Optional:** 
+     *  Resources to exclude  
    *
    * @example
    * ['png','jpg','jpeg','gif','svg','css','js','ico','woff','woff2','ttf','eot','otf','mp4','mp3','webm','ogg','wav','flac','aac','zip','tar','gz','rar','7z','exe','dmg','apk','csv','xls','xlsx','doc','docx','pdf','epub','iso','dmg','bin','ppt','pptx','odt','avi','mkv','xml','json','yml','yaml','rss','atom','swf','txt','dart','webp','bmp','tif','psd','ai','indd','eps','ps','zipx','srt','wasm','m4v','m4a','webp','weba','m4b','opus','ogv','ogm','oga','spx','ogx','flv','3gp','3g2','jxr','wdp','jng','hief','avif','apng','avifs','heif','heic','cur','ico','ani','jp2','jpm','jpx','mj2','wmv','wma','aac','tif','tiff','mpg','mpeg','mov','avi','wmv','flv','swf','mkv','m4v','m4p','m4b','m4r','m4a','mp3','wav','wma','ogg','oga','webm','3gp','3g2','flac','spx','amr','mid','midi','mka','dts','ac3','eac3','weba','m3u','m3u8','ts','wpl','pls','vob','ifo','bup','svcd','drc','dsm','dsv','dsa','dss','vivo','ivf','dvd','fli','flc','flic','flic','mng','asf','m2v','asx','ram','ra','rm','rpm','roq','smi','smil','wmf','wmz','wmd','wvx','wmx','movie','wri','ins','isp','acsm','djvu','fb2','xps','oxps','ps','eps','ai','prn','svg','dwg','dxf','ttf','fnt','fon','otf','cab']
    */
   resourceExclusions: z.array(z.string()).optional(),
-
-  /** Optional maximum file size in megabytes to include in the output file
+
+   /** 
+   * **Optional:**  
+   * Maximum file size in megabytes to include in the output file
    * @example 1
    */
   maxFileSize: z.number().int().positive().optional(),
-  /** Optional maximum number tokens to include in the output file
+
+   /** 
+   * **Optional:**
+   * The maximum number tokens to include in the output file
    * @example 5000
    */
   maxTokens: z.number().int().positive().optional(),