feat: add spacy model download and some logs!

Author: amindadgar

Dockerfile

@@ -2,6 +2,7 @@ FROM python:3.11-bullseye AS base
 WORKDIR /project
 COPY . .
 RUN pip install --no-cache-dir -r requirements.txt
+RUN python -m spacy download en_core_web_sm
 
 FROM base AS test
 RUN chmod +x docker-entrypoint.sh

hivemind_etl/mediawiki/etl.py

@@ -94,10 +94,13 @@ def transform(self) -> list[Document]:
         return documents
 
     def load(self, documents: list[Document]) -> None:
+        logging.info(f"Loading {len(documents)} documents into Qdrant!")
         ingestion_pipeline = CustomIngestionPipeline(
             self.community_id, collection_name="mediawiki"
         )
         ingestion_pipeline.run_pipeline(documents)
+        logging.info(f"Loaded {len(documents)} documents into Qdrant!")
 
         if self.delete_dump_after_load:
+            logging.info(f"Removing dump directory {self.dump_dir}!")
             shutil.rmtree(self.dump_dir)