s3 works
This commit is contained in:
@@ -21,8 +21,6 @@ def retrieve_file_contents(path):
|
|||||||
#Minio Connection
|
#Minio Connection
|
||||||
minioClient = Minio(
|
minioClient = Minio(
|
||||||
"localhost:9000",
|
"localhost:9000",
|
||||||
access_key="PVFOeJbx87rQyi0WXF1X",
|
|
||||||
secret_key="Am8Cd9auYGbEGuEXfJtnWEPsMwJCx9N58NCNHCgs",
|
|
||||||
secure=False,
|
secure=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -38,18 +36,17 @@ def retrieve_file_contents(path):
|
|||||||
|
|
||||||
contents = []
|
contents = []
|
||||||
|
|
||||||
print("Parsing files...")
|
for file in files:
|
||||||
for file in tqdm(files):
|
|
||||||
content = file_extraction_functions[file[0]](file[1])
|
content = file_extraction_functions[file[0]](file[1])
|
||||||
with open(f"./{file[2]}.txt", "w") as future_s3_file:
|
with open(f"./{file[2]}.txt", "w", encoding="UTF-8") as future_s3_file:
|
||||||
future_s3_file.writelines(file[1] + "\n" + content)
|
future_s3_file.writelines(file[1] + "\n" + content)
|
||||||
future_s3_file.flush()
|
future_s3_file.flush()
|
||||||
future_s3_file.close()
|
#future_s3_file.close()
|
||||||
|
|
||||||
minioClient.fput_object(
|
minioClient.fput_object(
|
||||||
bucket_name="datafiles",
|
bucket_name="datafiles",
|
||||||
object_name=f"{file[2]}.txt",
|
object_name=f"{file[2]}.txt",
|
||||||
file_path=file[1],
|
file_path=f"./{file[2]}.txt",
|
||||||
)
|
)
|
||||||
|
|
||||||
os.remove(f"./{file[2]}.txt")
|
os.remove(f"./{file[2]}.txt")
|
||||||
|
|||||||
Reference in New Issue
Block a user