jonas/datenbanken-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

puuuush

Browse Source 
ctime and mtime added to Mongo DB;
Seems to work lol
This commit is contained in:
Ali
2024-12-16 23:47:11 +01:00
parent 05d7bc8f53
commit d3ff36876e
2 changed files with 15 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib.py
+1 -1
View File
@@ -107,6 +107,6 @@ def get_data_from_mongo(dataset):
for set in dataset: for set in dataset:
json_data = collection.find_one({"doc_id": set[0]}) json_data = collection.find_one({"doc_id": set[0]})
result_string += "Dateipfad: " + json_data["filepath"] + "; Inhalt: " + json_data["chunk_content"] + "; " result_string += "Dateipfad: " + json_data["filepath"] + "; Inhalt: " + json_data["chunk_content"] + "; Erstellungszeitpunkt: " + json_data["ctime"] + "; Letzte Bearbeitung: " + json_data["mtime"]
return result_string return result_string
main.py
+14 -4
View File
@@ -9,7 +9,7 @@ import lib
import json import json
import psycopg2 import psycopg2
import pymongo import pymongo
from datetime import datetime
import mongo_conn import mongo_conn
import pgvec_conn import pgvec_conn
import redis_conn import redis_conn
@@ -40,16 +40,24 @@ def retrieve_file_contents(path, pPullFromS3):
lib.read_files(path, files := []) lib.read_files(path, files := [])
contents = [] contents = []
ctime = ""
mtime = ""
print("parsing files...") print("parsing files...")
for file in tqdm(files): for file in tqdm(files):
if not pPullFromS3:
ctime = datetime.fromtimestamp(os.path.getctime(file[1])).strftime('%Y-%m-%d %H:%M:%S')
mtime = datetime.fromtimestamp(os.path.getmtime(file[1])).strftime('%Y-%m-%d %H:%M:%S')
print(ctime, mtime)
content = file_extraction_functions[file[0]](file[1]) content = file_extraction_functions[file[0]](file[1])
if not pPullFromS3: if not pPullFromS3:
push_files_into_s3(file, content, minioClient) push_files_into_s3(file, content, minioClient)
contents.append({ contents.append({
"type": file[0], "type": file[0],
"path": file[1] if not pPullFromS3 else "Diese Datei kommt ursprünglich nicht von dir, sondern aus einem S3-Bucket. ", "ctime": ctime if not pPullFromS3 else 'Diese Datei kommt ursprünglich nicht von dir, sondern aus einem S3-Bucket. ',
"mtime": mtime if not pPullFromS3 else 'Diese Datei kommt ursprünglich nicht von dir, sondern aus einem S3-Bucket. ',
"path": file[1] if not pPullFromS3 else 'Diese Datei kommt ursprünglich nicht von dir, sondern aus einem S3-Bucket. ',
"filename": file[2], "filename": file[2],
"content": content "content": content
}) })
@@ -96,16 +104,18 @@ def create_embeddings(pContent):
insert_data = f"insert into dbtable (filepath, embedding) Values ('{content['path']}', %s) Returning id;" insert_data = f"insert into dbtable (filepath, embedding) Values ('{content['path']}', %s) Returning id;"
cur.execute(insert_data, (response["embedding"],)) cur.execute(insert_data, (response["embedding"],))
doc_id = cur.fetchone()[0] doc_id = cur.fetchone()[0]
insert_data_mongo(doc_id, content['path'], chunk) insert_data_mongo(doc_id, content['path'], chunk, content['ctime'], content["mtime"])
conn.commit() conn.commit()
conn.close() conn.close()
def insert_data_mongo(id, filepath,pChunk): def insert_data_mongo(id, filepath,pChunk, pCtime, pMtime):
client = pymongo.MongoClient('mongodb://python:PasswordPassword123@localhost:27017/') client = pymongo.MongoClient('mongodb://python:PasswordPassword123@localhost:27017/')
mongodb = client['document_table'] mongodb = client['document_table']
collection = mongodb['documents'] collection = mongodb['documents']
dokument = { dokument = {
'doc_id': id, 'doc_id': id,
'ctime' : pCtime,
'mtime' : pMtime,
'filepath': filepath, 'filepath': filepath,
'chunk_content': pChunk, 'chunk_content': pChunk,
} }