puuuush

main.py

@@ -1,6 +1,8 @@
 import ollama
 import pymupdf
 import os
+
+import chunker
 import lib
 import json
 import psycopg2
@@ -60,20 +62,20 @@ def create_embeddings(pContent):
     conn.commit()
 
     for content in pContent:
-        merged_info = "Dateiname: " + content["filename"] + " Dateiinhalt: " + content[
-            "content"]  # Bessere Embeddings mit Dateiname // Information vorne dran?
-        # print(merged_info)
-        response = ollama.embeddings(model="mxbai-embed-large", prompt=merged_info)
-        #embedding_list.append(response["embedding"])
-        insert_data = f"insert into dbtable (filepath, embedding) Values ('{content['path']}', %s) Returning doc_id;"
-        cur.execute(insert_data, (response["embedding"],))
-        doc_id = cur.fetchone()[0]
-        insert_data_mongo(doc_id, "Hello World")
+        for chunk in chunker.generate_chunks(content["content"]):
+            merged_info = "Dateiname: " + content["filename"] + " Dateiinhalt: " + chunk
+            # print(merged_info)
+            response = ollama.embeddings(model="mxbai-embed-large", prompt=merged_info)
+            #embedding_list.append(response["embedding"])
+            insert_data = f"insert into dbtable (filepath, embedding) Values ('{content['path']}', %s) Returning doc_id;"
+            cur.execute(insert_data, (response["embedding"],))
+            doc_id = cur.fetchone()[0]
+            insert_data_mongo(doc_id, chunk)
     conn.commit()
     conn.close()
 
 def insert_data_mongo(id, pChunk):
-    client = pymongo.MongoClient('mongodb://localhost:27017/')
+    client = pymongo.MongoClient('mongodb://python:PasswordPassword123@localhost:27017/')
     mongodb = client['document_table']
     collection = mongodb['documents']
     dokument = {
@@ -84,7 +86,7 @@ def insert_data_mongo(id, pChunk):
 
 def reload_files():
     # path = input("Please provider path to folder: ")
-    contents = retrieve_file_contents("C:\\Users\\afist\\Downloads\\SoftwareEng")
+    contents = retrieve_file_contents("C:\\SoftwareEng")
     create_embeddings(contents)