From 956ebc52276ba3af9271a9a3c58e6596975a0bcf Mon Sep 17 00:00:00 2001 From: Jonas Braus Date: Wed, 11 Dec 2024 21:12:13 +0100 Subject: [PATCH] Create chunker.py --- chunker.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 chunker.py diff --git a/chunker.py b/chunker.py new file mode 100644 index 0000000..4ad9c4f --- /dev/null +++ b/chunker.py @@ -0,0 +1,23 @@ +from llama_index.core import SimpleDirectoryReader +from llama_index.core.node_parser import SemanticSplitterNodeParser +from llama_index.embeddings.ollama import OllamaEmbedding + +def generate_chunks(content): + with open("./temp.txt", "wb") as file: + file.write(content.encode("utf-8")) + file.flush() + file.close() + + embed_model = OllamaEmbedding(model_name="mxbai-embed-large") + splitter = SemanticSplitterNodeParser(buffer_size=5, breakpoint_percentile_threshold=70, embed_model=embed_model) + + document = SimpleDirectoryReader(input_files=["./temp.txt"]).load_data() + + nodes = splitter.build_semantic_nodes_from_documents(document) + + output = [] + + for i, node in enumerate(nodes): + output.append(node.to_dict()["text"]) + + return output