jonas/datenbanken-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update main.py

Browse Source
This commit is contained in:
Jonas Braus
2024-12-12 13:44:02 +01:00
parent ed5090110c
commit 34f0e47d1e
1 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.py
+5 -2
View File
@@ -11,6 +11,7 @@ import pymongo
import mongo_conn import mongo_conn
import pgvec_conn import pgvec_conn
import redis_conn import redis_conn
from tqdm import tqdm
count = 0 count = 0
@@ -27,7 +28,8 @@ def retrieve_file_contents(path):
contents = [] contents = []
for file in files: print("Parsing files...")
for file in tqdm(files):
content = file_extraction_functions[file[0]](file[1]) content = file_extraction_functions[file[0]](file[1])
contents.append({ contents.append({
"type": file[0], "type": file[0],
@@ -64,7 +66,8 @@ def create_embeddings(pContent):
conn.commit() conn.commit()
for content in pContent: print("generating embeddings...")
for content in tqdm(pContent):
for chunk in chunker.generate_chunks(content["content"]): for chunk in chunker.generate_chunks(content["content"]):
merged_info = "Dateiname: " + content["filename"] + " Dateiinhalt: " + chunk merged_info = "Dateiname: " + content["filename"] + " Dateiinhalt: " + chunk
# print(merged_info) # print(merged_info)