Skip to content

Commit

Permalink
Removed some print and renamed some variables
Browse files Browse the repository at this point in the history
  • Loading branch information
varrialeciro committed Aug 10, 2023
1 parent 6672454 commit 94e45b4
Showing 1 changed file with 8 additions and 12 deletions.
20 changes: 8 additions & 12 deletions core/cat/rabbit_hole.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,25 +126,21 @@ def ingest_file(
before_rabbithole_stores_documents
"""

print("Start ingestion")
start = time.time()

# split file into a list of docs
docs = self.file_to_docs(
file=file, chunk_size=chunk_size, chunk_overlap=chunk_overlap
)
print(f"Total time file_to_docs: {time.time() - start}" )

# store in memory
if isinstance(file, str):
filename = file
else:
filename = file.filename

print("Start storing")
start = time.time()
self.store_documents(docs=docs, source=filename)
print(f"Total time: {time.time() - start}")

def file_to_docs(
self,
Expand Down Expand Up @@ -289,20 +285,20 @@ def store_documents(self, docs: List[Document], source: str) -> None:
)

# parameters for storing progress percentage
perc100 = len(docs)
percResolution = 10
percStep = math.floor((perc100 * percResolution) / 100)
perc_100 = len(docs)
perc_resolution = 10
perc_step = math.floor((perc_100 * perc_resolution) / 100)

readPercentage = 0
perc_read = 0

# classic embed
for d, doc in enumerate(docs):
# every percStep send a notification in order to monito the progress
# every percStep send a notification in order to monitor the progress
# N.B. file with a len < resolution does not show any percentage
# the storing should be immediate, and it's correct show only the final msg
if ((percStep > 0) and ((d+1) % percStep) == 0):
readPercentage += percResolution
self.send_rabbit_thought(f"Read {readPercentage}% of {source}")
if ((perc_step > 0) and ((d+1) % perc_step) == 0):
perc_read += perc_resolution
self.send_rabbit_thought(f"Read {perc_read}% of {source}")

doc.metadata["source"] = source
doc.metadata["when"] = time.time()
Expand Down

0 comments on commit 94e45b4

Please sign in to comment.