s3 works
This commit is contained in:
@@ -21,8 +21,6 @@ def retrieve_file_contents(path):
|
||||
#Minio Connection
|
||||
minioClient = Minio(
|
||||
"localhost:9000",
|
||||
access_key="PVFOeJbx87rQyi0WXF1X",
|
||||
secret_key="Am8Cd9auYGbEGuEXfJtnWEPsMwJCx9N58NCNHCgs",
|
||||
secure=False,
|
||||
)
|
||||
|
||||
@@ -38,18 +36,17 @@ def retrieve_file_contents(path):
|
||||
|
||||
contents = []
|
||||
|
||||
print("Parsing files...")
|
||||
for file in tqdm(files):
|
||||
for file in files:
|
||||
content = file_extraction_functions[file[0]](file[1])
|
||||
with open(f"./{file[2]}.txt", "w") as future_s3_file:
|
||||
with open(f"./{file[2]}.txt", "w", encoding="UTF-8") as future_s3_file:
|
||||
future_s3_file.writelines(file[1] + "\n" + content)
|
||||
future_s3_file.flush()
|
||||
future_s3_file.close()
|
||||
#future_s3_file.close()
|
||||
|
||||
minioClient.fput_object(
|
||||
bucket_name="datafiles",
|
||||
object_name=f"{file[2]}.txt",
|
||||
file_path=file[1],
|
||||
file_path=f"./{file[2]}.txt",
|
||||
)
|
||||
|
||||
os.remove(f"./{file[2]}.txt")
|
||||
|
||||
Reference in New Issue
Block a user