Merge branch 'main' of https://gitlab.com/brausjonas/datenbanken-llm

chunker.py

@@ -0,0 +1,23 @@
+from llama_index.core import SimpleDirectoryReader
+from llama_index.core.node_parser import SemanticSplitterNodeParser
+from llama_index.embeddings.ollama import OllamaEmbedding
+
+def generate_chunks(content):
+    with open("./temp.txt", "wb") as file:
+        file.write(content.encode("utf-8"))
+        file.flush()
+        file.close()
+
+    embed_model = OllamaEmbedding(model_name="mxbai-embed-large")
+    splitter = SemanticSplitterNodeParser(buffer_size=5, breakpoint_percentile_threshold=70, embed_model=embed_model)
+
+    document = SimpleDirectoryReader(input_files=["./temp.txt"]).load_data()
+
+    nodes = splitter.build_semantic_nodes_from_documents(document)
+
+    output = []
+
+    for i, node in enumerate(nodes):
+        output.append(node.to_dict()["text"])
+
+    return output