diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..fd13fac --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,52 @@ +version: "3" + +services: + psql: + hostname: pgvector + image: pgvector/pgvector:pg16 + ports: + - "5555:5432" + restart: unless-stopped + environment: + - POSTGRES_DB=embeddings + - POSTGRES_USER=python + - POSTGRES_PASSWORD=PasswordPassword123 + - POSTGRES_HOST_AUTH_METHOD=trust + volumes: + - postgres_data:/var/lib/postgresql/data + + s3: + image: minio/minio:RELEASE.2024-11-07T00-52-20Z.fips + command: server /data --console-address ":9090" + environment: + - MINIO_ROOT_USER=python + - MINIO_ROOT_PASSWORD=PasswordPassword123 + ports: + - "9000:9000" + - "9090:9090" + volumes: + - minio_data:/data + + redis: + image: redis/redis-stack:7.4.0-v0 + ports: + - "6379:6379" + - "8001:8001" + volumes: + - redis_data:/data + + mongo: + image: mongo:7.0.7 + environment: + - MONGO_INITDB_ROOT_USERNAME=python + - MONGO_INITDB_ROOT_PASSWORD=PasswordPassword123 + ports: + - "27017:27017" + volumes: + - mongo_data:/data/db + +volumes: + postgres_data: + minio_data: + redis_data: + mongo_data: \ No newline at end of file diff --git a/lib.py b/lib.py new file mode 100644 index 0000000..f81a156 --- /dev/null +++ b/lib.py @@ -0,0 +1,22 @@ +import os, pymupdf + +def read_files(path, output, filetypes=None): + if filetypes is None: + filetypes = ["pdf", "txt", "png", "jpg", "mp3"] + + for root, dirs, files in os.walk(path): + for file in files: + if (file_type := file.split(".")[-1]) in filetypes: + output.append((file_type, os.path.join(path, file))) + + for folder in dirs: + read_files(os.path.join(path, folder), output) + + break + +def extract_pdf_content(pdf_datei): + doc = pymupdf.open(pdf_datei) + a = "" + for page in doc: + a += page.get_text() + return a \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..26e9928 --- /dev/null +++ b/main.py @@ -0,0 +1,31 @@ +import pymupdf +import os +import lib +import json + +def retrieve_file_contents(path): + file_extraction_functions = { + "pdf": lambda path: lib.extract_pdf_content(path), + "jpg": lambda path: "", + "png": lambda path: "", + "txt": lambda path: lib.extract_pdf_content(path), + "mp3": lambda path: "test mp3" + } + + lib.read_files(path, files := []) + + contents = [] + + for file in files: + content = file_extraction_functions[file[0]](file[1]) + contents.append({ + "type": file[0], + "path": file[1], + "content": content + }) + + return contents + +contents = retrieve_file_contents("C:\\SoftwareEng") + +print(json.dumps(contents, indent="\t")) \ No newline at end of file