Leitura de arquivos

Author: Pendulun

.gitignore

@@ -1 +1 @@
-venv
+venv*

indexer.py

@@ -1,20 +1,36 @@
 import sys
 import resource
 import argparse
+import pathlib
+from warcio import ArchiveIterator
 
 MEGABYTE = 1024 * 1024
 def memory_limit(value):
     limit = value * MEGABYTE
     resource.setrlimit(resource.RLIMIT_AS, (limit, limit))
 
-def main():
+def main(my_args):
     """
     Your main calls should be added here
     """
-    pass
+    corpus_dir = pathlib.Path(my_args.corpus_path)
 
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description='Process some integers.')
+    assert corpus_dir.exists()
+    
+    #Ler um arquivo warc
+
+    for file in corpus_dir.glob('*.warc.gz.kaggle'):
+
+        with open(file, 'rb') as stream:
+            
+            for record in ArchiveIterator(stream):
+                if record.rec_type == 'response':
+                    texto = record.raw_stream.read().decode()
+
+    #Para cada documento, calcular a frequência de palavras e adicionar no index
+    #Tudo isso levando em consideração a memória utilizada
+
+def configArgs(parser):
     parser.add_argument(
         '-m',
         dest='memory_limit',
@@ -39,12 +55,17 @@ def main():
         required=True,
         help='path of the index file to be generated'
     )
+    return parser
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Process some integers.')
+    parser = configArgs(parser)
 
     my_args = parser.parse_args()
 
     memory_limit(my_args.memory_limit)
     try:
-        main()
+        main(my_args)
     except MemoryError:
         sys.stderr.write('\n\nERROR: Memory Exception\n')
         sys.exit(1)

requirements.txt

@@ -10,4 +10,4 @@ soupsieve==2.3.2.post1
 tqdm==4.64.0
 typing-extensions==4.2.0
 warcio==1.7.4
-zipp==3.8.0
+zipp==3.8.0