From 5b25bd4ab87141121da6dcde78ac45de900427ed Mon Sep 17 00:00:00 2001
From: Ali <a.fistik14@gmail.com>
Date: Thu, 12 Dec 2024 13:27:08 +0100
Subject: [PATCH] puuuuush into s3 (minio)

uploads data (txt) into minio
---
 main.py | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/main.py b/main.py
index dfd6bd2..4dbd466 100644
--- a/main.py
+++ b/main.py
@@ -1,7 +1,8 @@
 import ollama
 import pymupdf
 import os
-
+from minio import Minio
+from minio.error import S3Error
 import chunker
 import lib
 import json
@@ -11,6 +12,15 @@ import pymongo
 count = 0
 
 def retrieve_file_contents(path):
+
+    #Minio Connection
+    minioClient = Minio(
+        "localhost:9000",
+        access_key="PVFOeJbx87rQyi0WXF1X",
+        secret_key="Am8Cd9auYGbEGuEXfJtnWEPsMwJCx9N58NCNHCgs",
+        secure=False,
+    )
+
     file_extraction_functions = {
         "pdf": lambda path: lib.extract_text_and_pictures(path),
         "jpg": lambda path: lib.extract_image_content(path),
@@ -25,6 +35,20 @@ def retrieve_file_contents(path):
 
     for file in files:
         content = file_extraction_functions[file[0]](file[1])
+        with open(f"./{file[2]}.txt", "w") as future_s3_file:
+            future_s3_file.writelines(file[1] + "\n" + content)
+            future_s3_file.flush()
+            future_s3_file.close()
+
+        minioClient.fput_object(
+            bucket_name="datafiles",
+            object_name=f"{file[2]}.txt",
+            file_path=file[1],
+        )
+
+        os.remove(f"./{file[2]}.txt")
+
+
         contents.append({
             "type": file[0],
             "path": file[1],