Merge branch 'main' of https://gitlab.com/brausjonas/datenbanken-llm

docker-compose.yml

@@ -0,0 +1,52 @@
+version: "3"
+
+services:
+  psql:
+    hostname: pgvector
+    image: pgvector/pgvector:pg16
+    ports:
+      - "5555:5432"
+    restart: unless-stopped
+    environment:
+      - POSTGRES_DB=embeddings
+      - POSTGRES_USER=python
+      - POSTGRES_PASSWORD=PasswordPassword123
+      - POSTGRES_HOST_AUTH_METHOD=trust
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+
+  s3:
+    image: minio/minio:RELEASE.2024-11-07T00-52-20Z.fips
+    command: server /data --console-address ":9090"
+    environment:
+      - MINIO_ROOT_USER=python
+      - MINIO_ROOT_PASSWORD=PasswordPassword123
+    ports:
+      - "9000:9000"
+      - "9090:9090"
+    volumes:
+      - minio_data:/data
+
+  redis:
+    image: redis/redis-stack:7.4.0-v0
+    ports:
+      - "6379:6379"
+      - "8001:8001"
+    volumes:
+      - redis_data:/data
+
+  mongo:
+    image: mongo:7.0.7
+    environment:
+      - MONGO_INITDB_ROOT_USERNAME=python
+      - MONGO_INITDB_ROOT_PASSWORD=PasswordPassword123
+    ports:
+      - "27017:27017"
+    volumes:
+      - mongo_data:/data/db
+
+volumes:
+  postgres_data:
+  minio_data:
+  redis_data:
+  mongo_data:

lib.py

@@ -0,0 +1,22 @@
+import os, pymupdf
+
+def read_files(path, output, filetypes=None):
+    if filetypes is None:
+        filetypes = ["pdf", "txt", "png", "jpg", "mp3"]
+
+    for root, dirs, files in os.walk(path):
+        for file in files:
+            if (file_type := file.split(".")[-1]) in filetypes:
+                output.append((file_type, os.path.join(path, file)))
+
+        for folder in dirs:
+            read_files(os.path.join(path, folder), output)
+
+        break
+
+def extract_pdf_content(pdf_datei):
+    doc = pymupdf.open(pdf_datei)
+    a = ""
+    for page in doc:
+        a += page.get_text()
+    return a

main.py

@@ -0,0 +1,31 @@
+import pymupdf
+import os
+import lib
+import json
+
+def retrieve_file_contents(path):
+    file_extraction_functions = {
+        "pdf": lambda path: lib.extract_pdf_content(path),
+        "jpg": lambda path: "",
+        "png": lambda path: "",
+        "txt": lambda path: lib.extract_pdf_content(path),
+        "mp3": lambda path: "test mp3"
+    }
+
+    lib.read_files(path, files := [])
+
+    contents = []
+
+    for file in files:
+        content = file_extraction_functions[file[0]](file[1])
+        contents.append({
+            "type": file[0],
+            "path": file[1],
+            "content": content
+        })
+
+    return contents
+
+contents = retrieve_file_contents("C:\\SoftwareEng")
+
+print(json.dumps(contents, indent="\t"))