Skip to content

Commit

Permalink
Added read_csv and openpyxl requirement
Browse files Browse the repository at this point in the history
  • Loading branch information
zorge69 committed Oct 8, 2024
1 parent dd8417e commit 8348617
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 10 deletions.
15 changes: 10 additions & 5 deletions export_mods_main.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,11 +62,11 @@
# response = mods.rd_read_header(my_path)
# print(response)

# Testing rd_write_string_to_file
out_path = "/bat/res_dev/project_data/write_string_test.txt"
content = "New content"
mods.rd_write_string_to_file(content.encode(encoding="utf-8"), out_path)
print("all done")
# # Testing rd_write_string_to_file
# out_path = "/bat/res_dev/project_data/write_string_test.txt"
# content = "New content"
# mods.rd_write_string_to_file(content.encode(encoding="utf-8"), out_path)
# print("all done")

# # Testing rd_copy_file
# src_path = "/bat/res_dev/project_data/write_string_test.txt"
Expand All @@ -88,3 +88,8 @@

# found_file = mods.rd_search_file(dir_path, ending)
# print(f"Found file: {found_file}")

# Testing read_excel
my_path = "bat/res_dev/project_data/test_excel_gz.xlsx"
df = mods.read_excel(my_path)
print(df.head())
1 change: 1 addition & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,4 @@ typing
# boto3
# raz_client
# rdsa-utils==2.0.2
openpyxl
11 changes: 6 additions & 5 deletions src/utils/s3_mods.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@

# Third party libraries
import pandas as pd
from io import StringIO, TextIOWrapper
from io import StringIO, TextIOWrapper, BytesIO


# Local libraries
Expand Down Expand Up @@ -459,7 +459,7 @@ def rd_search_file(dir_path: str, ending: str) -> str:

def read_excel(
filepath: str,
client: boto3.client = s3_client,
client = s3_client,
bucket_name: str = s3_bucket,
**kwargs,
) -> pd.DataFrame:
Expand Down Expand Up @@ -491,18 +491,19 @@ def read_excel(
"""

bucket_name = validate_bucket_name(bucket_name)
filepath = validate_s3_file_path(filepath, allow_s3_scheme=False)
# bucket_name = validate_bucket_name(bucket_name)
# filepath = validate_s3_file_path(filepath, allow_s3_scheme=False)

try:
# Get the Excel file from S3

response = client.get_object(Bucket=bucket_name, Key=filepath)
s3_logger.info(
f"Loaded Excel file from S3 bucket {bucket_name}, filepath {filepath}",
)

# Read the Excel file into a Pandas DataFrame
df = pd.read_excel(response["Body"], **kwargs)
df = pd.read_excel(BytesIO(response['Body'].read()), **kwargs)

except Exception as e:
error_message = (
Expand Down

0 comments on commit 8348617

Please sign in to comment.