from llama_index.core import SimpleDirectoryReader from llama_index.core.node_parser import SemanticSplitterNodeParser from llama_index.embeddings.ollama import OllamaEmbedding def generate_chunks(content): with open("./temp.txt", "wb") as file: file.write(content.encode("utf-8")) file.flush() file.close() embed_model = OllamaEmbedding(model_name="mxbai-embed-large") splitter = SemanticSplitterNodeParser(buffer_size=5, breakpoint_percentile_threshold=70, embed_model=embed_model) document = SimpleDirectoryReader(input_files=["./temp.txt"]).load_data() nodes = splitter.build_semantic_nodes_from_documents(document) output = [] for i, node in enumerate(nodes): output.append(node.to_dict()["text"]) return output