Merge branch 'main' of https://gitlab.com/brausjonas/datenbanken-llm

main.py

@@ -1,7 +1,8 @@
 import ollama
 import pymupdf
 import os
-
+from minio import Minio
+from minio.error import S3Error
 import chunker
 import lib
 import json
@@ -16,6 +17,15 @@ from tqdm import tqdm
 count = 0
 
 def retrieve_file_contents(path):
+
+    #Minio Connection
+    minioClient = Minio(
+        "localhost:9000",
+        access_key="PVFOeJbx87rQyi0WXF1X",
+        secret_key="Am8Cd9auYGbEGuEXfJtnWEPsMwJCx9N58NCNHCgs",
+        secure=False,
+    )
+
     file_extraction_functions = {
         "pdf": lambda path: lib.extract_text_and_pictures(path),
         "jpg": lambda path: lib.extract_image_content(path),
@@ -31,6 +41,20 @@ def retrieve_file_contents(path):
     print("Parsing files...")
     for file in tqdm(files):
         content = file_extraction_functions[file[0]](file[1])
+        with open(f"./{file[2]}.txt", "w") as future_s3_file:
+            future_s3_file.writelines(file[1] + "\n" + content)
+            future_s3_file.flush()
+            future_s3_file.close()
+
+        minioClient.fput_object(
+            bucket_name="datafiles",
+            object_name=f"{file[2]}.txt",
+            file_path=file[1],
+        )
+
+        os.remove(f"./{file[2]}.txt")
+
+
         contents.append({
             "type": file[0],
             "path": file[1],