diff --git a/main.py b/main.py index dfd6bd2..4dbd466 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,8 @@ import ollama import pymupdf import os - +from minio import Minio +from minio.error import S3Error import chunker import lib import json @@ -11,6 +12,15 @@ import pymongo count = 0 def retrieve_file_contents(path): + + #Minio Connection + minioClient = Minio( + "localhost:9000", + access_key="PVFOeJbx87rQyi0WXF1X", + secret_key="Am8Cd9auYGbEGuEXfJtnWEPsMwJCx9N58NCNHCgs", + secure=False, + ) + file_extraction_functions = { "pdf": lambda path: lib.extract_text_and_pictures(path), "jpg": lambda path: lib.extract_image_content(path), @@ -25,6 +35,20 @@ def retrieve_file_contents(path): for file in files: content = file_extraction_functions[file[0]](file[1]) + with open(f"./{file[2]}.txt", "w") as future_s3_file: + future_s3_file.writelines(file[1] + "\n" + content) + future_s3_file.flush() + future_s3_file.close() + + minioClient.fput_object( + bucket_name="datafiles", + object_name=f"{file[2]}.txt", + file_path=file[1], + ) + + os.remove(f"./{file[2]}.txt") + + contents.append({ "type": file[0], "path": file[1],