feat: add --fields-include to unstructured-ingest

CHANGELOG.md

@@ -1,11 +1,13 @@
-## 0.5.7-dev0
+## 0.5.7-dev1
 
 ### Enhancements
 
 * Refactored codebase using `exactly_one`
 
 ### Features
 
+* Add `--fields-include` parameter to `unstructured-ingest`
+
 ### Fixes
 
 ## 0.5.6

test_unstructured_ingest/test_interfaces.py

@@ -15,7 +15,7 @@
 
 
 @pytest.mark.parametrize("filename", test_files)
-def test_process_file_include_filename(filename: str):
+def test_process_file_metadata_include_filename(filename: str):
     ingest_doc = GitIngestDoc(
         path=filename,
         config=SimpleGitConfig(
@@ -31,7 +31,7 @@ def test_process_file_include_filename(filename: str):
 
 
 @pytest.mark.parametrize("filename", test_files)
-def test_process_file_include_filename_pagenum(filename: str):
+def test_process_file_metadata_include_filename_pagenum(filename: str):
     ingest_doc = GitIngestDoc(
         path=filename,
         config=SimpleGitConfig(
@@ -47,7 +47,7 @@ def test_process_file_include_filename_pagenum(filename: str):
 
 
 @pytest.mark.parametrize("filename", test_files)
-def test_process_file_exclude_filename(filename: str):
+def test_process_file_metadata_exclude_filename(filename: str):
     ingest_doc = GitIngestDoc(
         path=filename,
         config=SimpleGitConfig(
@@ -63,7 +63,7 @@ def test_process_file_exclude_filename(filename: str):
 
 
 @pytest.mark.parametrize("filename", test_files)
-def test_process_file_exclude_filename_pagenum(filename: str):
+def test_process_file_metadata_exclude_filename_pagenum(filename: str):
     ingest_doc = GitIngestDoc(
         path=filename,
         config=SimpleGitConfig(
@@ -76,3 +76,29 @@ def test_process_file_exclude_filename_pagenum(filename: str):
     for elem in isd_elems:
         for k in elem["metadata"]:
             assert k not in ["filename", "page_number"]
+
+
+@pytest.mark.parametrize("filename", test_files)
+def test_process_file_fields_include_default(filename: str):
+    ingest_doc = GitIngestDoc(
+        path=filename,
+        config=SimpleGitConfig(
+          download_dir=EXAMPLE_DOCS_DIRECTORY,
+        ),
+    )
+    isd_elems = ingest_doc.process_file()
+    assert set("element_id", "text", "type", "metadata") == set(elem.keys())
+
+
+@pytest.mark.parametrize("filename", test_files)
+def test_process_file_fields_include_elementid(filename: str):
+    ingest_doc = GitIngestDoc(
+        path=filename,
+        config=SimpleGitConfig(
+          download_dir=EXAMPLE_DOCS_DIRECTORY,
+          fields_include="element_id",
+        ),
+    )
+    isd_elems = ingest_doc.process_file()
+    assert set("element_id") == set(elem.keys())
+

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.5.7-dev0"  # pragma: no cover
+__version__ = "0.5.7-dev1"  # pragma: no cover

unstructured/ingest/connector/biomed.py

@@ -50,6 +50,7 @@ class SimpleBiomedConfig(BaseConnectorConfig):
     preserve_downloads: bool = False
     metadata_include: Optional[str] = None
     metadata_exclude: Optional[str] = None
+    fields_include: str = "element_id,text,type,metadata"
 
     def _validate_date_args(self, date):
         date_formats = ["%Y-%m-%d", "%Y-%m-%d+%H:%M:%S"]

unstructured/ingest/connector/fsspec.py

@@ -31,6 +31,7 @@ class SimpleFsspecConfig(BaseConnectorConfig):
     re_download: bool = False
     metadata_include: Optional[str] = None
     metadata_exclude: Optional[str] = None
+    fields_include: str = "element_id,text,type,metadata"
 
     # fsspec specific options
     access_kwargs: dict = field(default_factory=dict)

unstructured/ingest/connector/git.py

@@ -28,6 +28,7 @@ class SimpleGitConfig(BaseConnectorConfig):
     re_download: bool = False
     metadata_include: Optional[str] = None
     metadata_exclude: Optional[str] = None
+    fields_include: str = "element_id,text,type,metadata"
 
     repo_path: str = field(init=False, repr=False)
 

unstructured/ingest/connector/google_drive.py

@@ -79,6 +79,7 @@ class SimpleGoogleDriveConfig(BaseConnectorConfig):
     preserve_downloads: bool = False
     metadata_include: Optional[str] = None
     metadata_exclude: Optional[str] = None
+    fields_include: str = "element_id,text,type,metadata"
 
     recursive: bool = False
 

unstructured/ingest/connector/reddit.py

@@ -33,6 +33,7 @@ class SimpleRedditConfig(BaseConnectorConfig):
     re_download: bool = False
     metadata_include: Optional[str] = None
     metadata_exclude: Optional[str] = None
+    fields_include: str = "element_id,text,type,metadata"
 
     def __post_init__(self):
         if self.num_posts <= 0:

unstructured/ingest/connector/wikipedia.py

@@ -28,6 +28,7 @@ class SimpleWikipediaConfig(BaseConnectorConfig):
     re_download: bool = False
     metadata_include: Optional[str] = None
     metadata_exclude: Optional[str] = None
+    fields_include: str = "element_id,text,type,metadata"
 
 
 @dataclass

unstructured/ingest/interfaces.py

@@ -50,6 +50,7 @@ class BaseConnectorConfig(ABC):
     re_download: bool = False
     metadata_include: Optional[str] = None
     metadata_exclude: Optional[str] = None
+    fields_include: str = "element_id,text,type,metadata"
 
 
 class BaseIngestDoc(ABC):
@@ -117,7 +118,9 @@ def process_file(self):
                     if k not in in_list:
                         elem["metadata"].pop(k, None)  # type: ignore[attr-defined]
 
-            elem.pop("coordinates")  # type: ignore[attr-defined]
+            in_list = self.config.fields_include.split(",")
+            elem = {k: v for k, v in elem.items() if k in in_list}
+
             self.isd_elems_no_filename.append(elem)
 
         return self.isd_elems_no_filename

unstructured/ingest/main.py

@@ -103,6 +103,12 @@ def run(self):
 
 
 @click.command()
+@click.option(
+    "--fields-include",
+    default="element_id,text,type,metadata",
+    help="If set, include the specified top-level fields in an element. "
+    "Default is `element_id,text,type,metadata`.",
+)
 @click.option(
     "--metadata-include",
     default=None,
@@ -338,7 +344,13 @@ def main(
     verbose,
     metadata_include,
     metadata_exclude,
+    fields_include,
 ):
+    if "metadata" not in fields_include and (metadata_include or metadata_exclude):
+        logger.warning(
+            "Either `--metadata-include` or `--metadata-exclude` is specified"
+            " while metadata is not specified in --fields-include.",
+        )
     if metadata_exclude is not None and metadata_include is not None:
         logger.error(
             "Arguments `--metadata-include` and `--metadata-exclude` are "
@@ -415,6 +427,7 @@ def main(
                     preserve_downloads=preserve_downloads,
                     metadata_include=metadata_include,
                     metadata_exclude=metadata_exclude,
+                    fields_include=fields_include,
                 ),
             )
         elif protocol in ("abfs", "az"):
@@ -437,6 +450,7 @@ def main(
                     preserve_downloads=preserve_downloads,
                     metadata_include=metadata_include,
                     metadata_exclude=metadata_exclude,
+                    fields_include=fields_include,
                 ),
             )
         else:
@@ -455,6 +469,7 @@ def main(
                     preserve_downloads=preserve_downloads,
                     metadata_include=metadata_include,
                     metadata_exclude=metadata_exclude,
+                    fields_include=fields_include,
                 ),
             )
     elif github_url:
@@ -471,6 +486,7 @@ def main(
                 re_download=re_download,
                 metadata_include=metadata_include,
                 metadata_exclude=metadata_exclude,
+                fields_include=fields_include,
             ),
         )
     elif gitlab_url:
@@ -487,6 +503,7 @@ def main(
                 re_download=re_download,
                 metadata_include=metadata_include,
                 metadata_exclude=metadata_exclude,
+                fields_include=fields_include,
             ),
         )
     elif subreddit_name:
@@ -505,6 +522,7 @@ def main(
                 re_download=re_download,
                 metadata_include=metadata_include,
                 metadata_exclude=metadata_exclude,
+                fields_include=fields_include,
             ),
         )
     elif wikipedia_page_title:
@@ -519,6 +537,7 @@ def main(
                 re_download=re_download,
                 metadata_include=metadata_include,
                 metadata_exclude=metadata_exclude,
+                fields_include=fields_include,
             ),
         )
     elif drive_id:
@@ -535,6 +554,7 @@ def main(
                 re_download=re_download,
                 metadata_include=metadata_include,
                 metadata_exclude=metadata_exclude,
+                fields_include=fields_include,
             ),
         )
     elif biomed_path or biomed_api_id or biomed_api_from or biomed_api_until:
@@ -551,6 +571,7 @@ def main(
                 re_download=re_download,
                 metadata_include=metadata_include,
                 metadata_exclude=metadata_exclude,
+                fields_include=fields_include,
             ),
         )
     # Check for other connector-specific options here and define the doc_connector object