24 lines
788 B
Python
24 lines
788 B
Python
from llama_index.core import SimpleDirectoryReader
|
|
from llama_index.core.node_parser import SemanticSplitterNodeParser
|
|
from llama_index.embeddings.ollama import OllamaEmbedding
|
|
|
|
def generate_chunks(content):
|
|
with open("./temp.txt", "wb") as file:
|
|
file.write(content.encode("utf-8"))
|
|
file.flush()
|
|
file.close()
|
|
|
|
embed_model = OllamaEmbedding(model_name="mxbai-embed-large")
|
|
splitter = SemanticSplitterNodeParser(buffer_size=5, breakpoint_percentile_threshold=45, embed_model=embed_model)
|
|
|
|
document = SimpleDirectoryReader(input_files=["./temp.txt"]).load_data()
|
|
|
|
nodes = splitter.build_semantic_nodes_from_documents(document)
|
|
|
|
output = []
|
|
|
|
for i, node in enumerate(nodes):
|
|
output.append(node.to_dict()["text"])
|
|
|
|
return output
|