From 4c3afe0e26a37427016e7fbdde32ce6812b5d6a1 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Wed, 20 Nov 2024 14:35:13 +0000 Subject: [PATCH 01/16] wip --- mailchimp_api/application.py | 38 ++++++++++++++++++++++++++++++++++++ pyproject.toml | 1 + 2 files changed, 39 insertions(+) create mode 100644 mailchimp_api/application.py diff --git a/mailchimp_api/application.py b/mailchimp_api/application.py new file mode 100644 index 0000000..f42ba2f --- /dev/null +++ b/mailchimp_api/application.py @@ -0,0 +1,38 @@ +from pathlib import Path + +from fastapi import FastAPI, HTTPException, UploadFile, status +from fastapi.responses import HTMLResponse + +app = FastAPI() + + +@app.post("/upload") +def upload(file: UploadFile) -> dict[str, str]: + try: + contents = file.file.read() + filename = file.filename if file.filename else "uploaded_file" + path = Path(filename) + with path.open("wb") as f: + f.write(contents) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="There was an error uploading the file", + ) from e + finally: + file.file.close() + + return {"message": f"Successfully uploaded {file.filename}"} + + +# Access the form at 'http://127.0.0.1:8002/' from your browser +@app.get("/") +def main() -> HTMLResponse: + content = """ +
+ + +
+ +""" + return HTMLResponse(content=content) diff --git a/pyproject.toml b/pyproject.toml index 37797da..cea8b63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ name = "mailchimp_api" dependencies = [ "fastagency[autogen,mesop,openapi,server,fastapi]>=0.3.0", + "python-multipart>=0.0.17" ] [project.optional-dependencies] From 2ffbc71dda58838ddfeb085c174b067e9c2200ae Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 21 Nov 2024 08:36:23 +0000 Subject: [PATCH 02/16] wip --- mailchimp_api/application.py | 42 ++++++++++++++++++++++++++++++++++-- tests/test_application.py | 42 ++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 tests/test_application.py diff --git a/mailchimp_api/application.py b/mailchimp_api/application.py index f42ba2f..588799c 100644 --- a/mailchimp_api/application.py +++ b/mailchimp_api/application.py @@ -1,13 +1,29 @@ +import os from pathlib import Path +import pandas as pd from fastapi import FastAPI, HTTPException, UploadFile, status from fastapi.responses import HTMLResponse +from .config import Config +from .processing.update_tags import update_tags + app = FastAPI() -@app.post("/upload") -def upload(file: UploadFile) -> dict[str, str]: +def _get_config() -> Config: + api_key = os.getenv("MAILCHIMP_API_KEY") + if not api_key: + raise ValueError("MAILCHIMP_API_KEY not set") + + config = Config("us14", api_key) + return config + + +config = _get_config() + + +def get_df(file: UploadFile) -> pd.DataFrame: try: contents = file.file.read() filename = file.filename if file.filename else "uploaded_file" @@ -22,6 +38,28 @@ def upload(file: UploadFile) -> dict[str, str]: finally: file.file.close() + if path.suffix != ".csv": + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Only CSV files are supported", + ) + + df = pd.read_csv(path) + if "email" not in df.columns: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="'email' column not found in CSV file", + ) + + return df + + +@app.post("/upload") +def upload(file: UploadFile) -> dict[str, str]: + df = get_df(file) + + update_tags(crm_df=df, config=config, list_name="airt") + return {"message": f"Successfully uploaded {file.filename}"} diff --git a/tests/test_application.py b/tests/test_application.py new file mode 100644 index 0000000..d4cccfd --- /dev/null +++ b/tests/test_application.py @@ -0,0 +1,42 @@ +from io import BytesIO + +import pytest +from fastapi import HTTPException, UploadFile +from pandas import DataFrame + +from mailchimp_api.application import get_df + + +class TestUploadFile: + def test_get_df(self) -> None: + csv_content = "email\nexample1@example.com\nexample2@example.com" + csv_file = BytesIO(csv_content.encode("utf-8")) + uploaded_file = UploadFile(filename="emails.csv", file=csv_file) + df = get_df(uploaded_file) + + expected_df = DataFrame( + {"email": ["example1@example.com", "example2@example.com"]} + ) + assert df.equals(expected_df) + + @pytest.mark.parametrize( + ("filename", "content", "expected_error"), + [ + ("emails.txt", "email\n", "Only CSV files are supported"), + ( + "emails.csv", + "first_column\n", + "'email' column not found in CSV file", + ), + ], + ) + def test_get_df_raises_error( + self, filename: str, content: str, expected_error: str + ) -> None: + csv_file = BytesIO(content.encode("utf-8")) + uploaded_file = UploadFile(filename=filename, file=csv_file) + + with pytest.raises(HTTPException) as e: + get_df(uploaded_file) + + assert e.value.detail == expected_error From bc8634e752172c3b51322dc8297cae4a66062236 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 21 Nov 2024 09:29:42 +0000 Subject: [PATCH 03/16] wip --- mailchimp_api/application.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/mailchimp_api/application.py b/mailchimp_api/application.py index 588799c..50a5bb2 100644 --- a/mailchimp_api/application.py +++ b/mailchimp_api/application.py @@ -2,7 +2,7 @@ from pathlib import Path import pandas as pd -from fastapi import FastAPI, HTTPException, UploadFile, status +from fastapi import FastAPI, Form, HTTPException, UploadFile, status from fastapi.responses import HTMLResponse from .config import Config @@ -55,10 +55,25 @@ def get_df(file: UploadFile) -> pd.DataFrame: @app.post("/upload") -def upload(file: UploadFile) -> dict[str, str]: +def upload( + account_name: str = Form(...), + file: UploadFile = UploadFile(...), # type: ignore[arg-type] # noqa: B008 +) -> dict[str, str]: + if not account_name or file.size == 0: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Please provide both account name and file", + ) + df = get_df(file) - update_tags(crm_df=df, config=config, list_name="airt") + try: + update_tags(crm_df=df, config=config, list_name=account_name) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=str(e), + ) from e return {"message": f"Successfully uploaded {file.filename}"} @@ -68,8 +83,15 @@ def upload(file: UploadFile) -> dict[str, str]: def main() -> HTMLResponse: content = """
- - +
+ +
+
+ +
+
+ +
""" From ee5088a8995ed79be997eb2895c3bb39336f8301 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 21 Nov 2024 09:34:31 +0000 Subject: [PATCH 04/16] Save files in specific dir --- .gitignore | 1 + mailchimp_api/application.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6316b56..34dc6ca 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ token .DS_Store tmp_* +mailchimp_api/uploaded_files diff --git a/mailchimp_api/application.py b/mailchimp_api/application.py index 50a5bb2..234ddca 100644 --- a/mailchimp_api/application.py +++ b/mailchimp_api/application.py @@ -24,10 +24,12 @@ def _get_config() -> Config: def get_df(file: UploadFile) -> pd.DataFrame: + uploaded_files_dir = Path(__file__).parent / "uploaded_files" + uploaded_files_dir.mkdir(exist_ok=True) try: contents = file.file.read() filename = file.filename if file.filename else "uploaded_file" - path = Path(filename) + path = uploaded_files_dir / filename with path.open("wb") as f: f.write(contents) except Exception as e: From b1dfb3f7ab3de61f9e8e44c67ccce704ed0fecd0 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 21 Nov 2024 09:46:32 +0000 Subject: [PATCH 05/16] Move upload file endpoints to main_1_fastapi.py --- mailchimp_api/application.py | 100 ------------------ mailchimp_api/deployment/main_1_fastapi.py | 96 ++++++++++++++++- .../test_main_1_fastapi.py} | 2 +- 3 files changed, 96 insertions(+), 102 deletions(-) delete mode 100644 mailchimp_api/application.py rename tests/{test_application.py => deployment/test_main_1_fastapi.py} (95%) diff --git a/mailchimp_api/application.py b/mailchimp_api/application.py deleted file mode 100644 index 234ddca..0000000 --- a/mailchimp_api/application.py +++ /dev/null @@ -1,100 +0,0 @@ -import os -from pathlib import Path - -import pandas as pd -from fastapi import FastAPI, Form, HTTPException, UploadFile, status -from fastapi.responses import HTMLResponse - -from .config import Config -from .processing.update_tags import update_tags - -app = FastAPI() - - -def _get_config() -> Config: - api_key = os.getenv("MAILCHIMP_API_KEY") - if not api_key: - raise ValueError("MAILCHIMP_API_KEY not set") - - config = Config("us14", api_key) - return config - - -config = _get_config() - - -def get_df(file: UploadFile) -> pd.DataFrame: - uploaded_files_dir = Path(__file__).parent / "uploaded_files" - uploaded_files_dir.mkdir(exist_ok=True) - try: - contents = file.file.read() - filename = file.filename if file.filename else "uploaded_file" - path = uploaded_files_dir / filename - with path.open("wb") as f: - f.write(contents) - except Exception as e: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="There was an error uploading the file", - ) from e - finally: - file.file.close() - - if path.suffix != ".csv": - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Only CSV files are supported", - ) - - df = pd.read_csv(path) - if "email" not in df.columns: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="'email' column not found in CSV file", - ) - - return df - - -@app.post("/upload") -def upload( - account_name: str = Form(...), - file: UploadFile = UploadFile(...), # type: ignore[arg-type] # noqa: B008 -) -> dict[str, str]: - if not account_name or file.size == 0: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Please provide both account name and file", - ) - - df = get_df(file) - - try: - update_tags(crm_df=df, config=config, list_name=account_name) - except Exception as e: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=str(e), - ) from e - - return {"message": f"Successfully uploaded {file.filename}"} - - -# Access the form at 'http://127.0.0.1:8002/' from your browser -@app.get("/") -def main() -> HTMLResponse: - content = """ -
-
- -
-
- -
-
- -
-
- -""" - return HTMLResponse(content=content) diff --git a/mailchimp_api/deployment/main_1_fastapi.py b/mailchimp_api/deployment/main_1_fastapi.py index 3e21910..ad2448a 100644 --- a/mailchimp_api/deployment/main_1_fastapi.py +++ b/mailchimp_api/deployment/main_1_fastapi.py @@ -1,8 +1,14 @@ +import os +from pathlib import Path from typing import Any +import pandas as pd from fastagency.adapters.fastapi import FastAPIAdapter -from fastapi import FastAPI +from fastapi import FastAPI, Form, HTTPException, UploadFile, status +from fastapi.responses import HTMLResponse +from ..config import Config +from ..processing.update_tags import update_tags from ..workflow import wf adapter = FastAPIAdapter(provider=wf) @@ -17,5 +23,93 @@ def list_workflows() -> dict[str, Any]: return {"Workflows": {name: wf.get_description(name) for name in wf.names}} +def _get_config() -> Config: + api_key = os.getenv("MAILCHIMP_API_KEY") + if not api_key: + raise ValueError("MAILCHIMP_API_KEY not set") + + config = Config("us14", api_key) + return config + + +config = _get_config() + + +def get_df(file: UploadFile) -> pd.DataFrame: + uploaded_files_dir = Path(__file__).parent.parent / "uploaded_files" + uploaded_files_dir.mkdir(exist_ok=True) + try: + contents = file.file.read() + filename = file.filename if file.filename else "uploaded_file" + path = uploaded_files_dir / filename + with path.open("wb") as f: + f.write(contents) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="There was an error uploading the file", + ) from e + finally: + file.file.close() + + if path.suffix != ".csv": + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Only CSV files are supported", + ) + + df = pd.read_csv(path) + if "email" not in df.columns: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="'email' column not found in CSV file", + ) + + return df + + +@app.post("/upload") +def upload( + account_name: str = Form(...), + file: UploadFile = UploadFile(...), # type: ignore[arg-type] # noqa: B008 +) -> dict[str, str]: + if not account_name or file.size == 0: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Please provide both account name and file", + ) + + df = get_df(file) + + try: + update_tags(crm_df=df, config=config, list_name=account_name) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=str(e), + ) from e + + return {"message": f"Successfully uploaded {file.filename}"} + + +@app.get("/upload-page") +def main() -> HTMLResponse: + content = """ +
+
+ +
+
+ +
+
+ +
+
+ +""" + return HTMLResponse(content=content) + + # start the adapter with the following command # uvicorn mailchimp_api.deployment.main_1_fastapi:app --reload diff --git a/tests/test_application.py b/tests/deployment/test_main_1_fastapi.py similarity index 95% rename from tests/test_application.py rename to tests/deployment/test_main_1_fastapi.py index d4cccfd..13a3dd3 100644 --- a/tests/test_application.py +++ b/tests/deployment/test_main_1_fastapi.py @@ -4,7 +4,7 @@ from fastapi import HTTPException, UploadFile from pandas import DataFrame -from mailchimp_api.application import get_df +from mailchimp_api.deployment.main_1_fastapi import get_df class TestUploadFile: From ad70e86917229a74053d48647370b7a949fa806a Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 21 Nov 2024 10:32:55 +0000 Subject: [PATCH 06/16] Initial upload-file integration with mesop app --- mailchimp_api/constants.py | 3 ++ mailchimp_api/deployment/main_1_fastapi.py | 11 +++--- mailchimp_api/workflow.py | 45 ++++++++-------------- tests/test_workflow.py | 1 + 4 files changed, 25 insertions(+), 35 deletions(-) create mode 100644 mailchimp_api/constants.py diff --git a/mailchimp_api/constants.py b/mailchimp_api/constants.py new file mode 100644 index 0000000..a48e9ae --- /dev/null +++ b/mailchimp_api/constants.py @@ -0,0 +1,3 @@ +from pathlib import Path + +UPLOADED_FILES_DIR = Path(__file__).parent / "uploaded_files" diff --git a/mailchimp_api/deployment/main_1_fastapi.py b/mailchimp_api/deployment/main_1_fastapi.py index ad2448a..da42b6b 100644 --- a/mailchimp_api/deployment/main_1_fastapi.py +++ b/mailchimp_api/deployment/main_1_fastapi.py @@ -1,5 +1,4 @@ import os -from pathlib import Path from typing import Any import pandas as pd @@ -8,6 +7,7 @@ from fastapi.responses import HTMLResponse from ..config import Config +from ..constants import UPLOADED_FILES_DIR from ..processing.update_tags import update_tags from ..workflow import wf @@ -36,12 +36,11 @@ def _get_config() -> Config: def get_df(file: UploadFile) -> pd.DataFrame: - uploaded_files_dir = Path(__file__).parent.parent / "uploaded_files" - uploaded_files_dir.mkdir(exist_ok=True) + UPLOADED_FILES_DIR.mkdir(exist_ok=True) try: contents = file.file.read() filename = file.filename if file.filename else "uploaded_file" - path = uploaded_files_dir / filename + path = UPLOADED_FILES_DIR / filename with path.open("wb") as f: f.write(contents) except Exception as e: @@ -92,7 +91,7 @@ def upload( return {"message": f"Successfully uploaded {file.filename}"} -@app.get("/upload-page") +@app.get("/upload-file") def main() -> HTMLResponse: content = """
@@ -112,4 +111,4 @@ def main() -> HTMLResponse: # start the adapter with the following command -# uvicorn mailchimp_api.deployment.main_1_fastapi:app --reload +# uvicorn mailchimp_api.deployment.main_1_fastapi:app -b 0.0.0.0:8008 --reload diff --git a/mailchimp_api/workflow.py b/mailchimp_api/workflow.py index bb6a5f3..669ae20 100644 --- a/mailchimp_api/workflow.py +++ b/mailchimp_api/workflow.py @@ -1,47 +1,34 @@ import os +import time from typing import Any -from autogen.agentchat import ConversableAgent from fastagency import UI from fastagency.runtimes.autogen import AutoGenWorkflows -llm_config = { - "config_list": [ - { - "model": "gpt-4o-mini", - "api_key": os.getenv("OPENAI_API_KEY"), - } - ], - "temperature": 0.8, -} +from .constants import UPLOADED_FILES_DIR wf = AutoGenWorkflows() +FASTAPI_URL = os.getenv("FASTAPI_URL", "http://localhost:8008") + @wf.register(name="simple_learning", description="Student and teacher learning chat") # type: ignore[misc] def simple_workflow(ui: UI, params: dict[str, Any]) -> str: - initial_message = ui.text_input( + body = f"""Please upload **.csv** file with the email addresses for which you want to update the tags. + +Upload File +""" + ui.text_message( sender="Workflow", recipient="User", - prompt="I can help you learn about mathematics. What subject you would like to explore?", + body=body, ) - student_agent = ConversableAgent( - name="Student_Agent", - system_message="You are a student willing to learn.", - llm_config=llm_config, - ) - teacher_agent = ConversableAgent( - name="Teacher_Agent", - system_message="You are a math teacher.", - llm_config=llm_config, - ) + file_name = "import_audience.csv" + file_path = UPLOADED_FILES_DIR / file_name + while not file_path.exists(): + time.sleep(2) - chat_result = student_agent.initiate_chat( - teacher_agent, - message=initial_message, - summary_method="reflection_with_llm", - max_turns=3, - ) + file_path.unlink() - return str(chat_result.summary) + return "Done" diff --git a/tests/test_workflow.py b/tests/test_workflow.py index 0cd4dfd..60bf452 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -7,6 +7,7 @@ from tests.conftest import InputMock +@pytest.mark.skip(reason="Skipping tests for now") def test_workflow(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr("builtins.input", InputMock([""] * 5)) From b51ce919c025b2763da45fd15750638ffebe2662 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 21 Nov 2024 10:48:26 +0000 Subject: [PATCH 07/16] wip --- mailchimp_api/deployment/main_1_fastapi.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mailchimp_api/deployment/main_1_fastapi.py b/mailchimp_api/deployment/main_1_fastapi.py index da42b6b..b5d98cb 100644 --- a/mailchimp_api/deployment/main_1_fastapi.py +++ b/mailchimp_api/deployment/main_1_fastapi.py @@ -3,7 +3,7 @@ import pandas as pd from fastagency.adapters.fastapi import FastAPIAdapter -from fastapi import FastAPI, Form, HTTPException, UploadFile, status +from fastapi import FastAPI, Form, HTTPException, Query, UploadFile, status from fastapi.responses import HTMLResponse from ..config import Config @@ -71,6 +71,7 @@ def get_df(file: UploadFile) -> pd.DataFrame: def upload( account_name: str = Form(...), file: UploadFile = UploadFile(...), # type: ignore[arg-type] # noqa: B008 + timestamp: str = Form(...), ) -> dict[str, str]: if not account_name or file.size == 0: raise HTTPException( @@ -92,8 +93,8 @@ def upload( @app.get("/upload-file") -def main() -> HTMLResponse: - content = """ +def upload_file(timestamp: str = Query(default="default-timestamp")) -> HTMLResponse: + content = f"""
@@ -101,6 +102,8 @@ def main() -> HTMLResponse:
+ +
From 2465ef153d73550f0f02e4ab599c6dbbe643be7d Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 21 Nov 2024 11:07:19 +0000 Subject: [PATCH 08/16] wip --- mailchimp_api/deployment/main_1_fastapi.py | 31 ++++++++------------ tests/deployment/test_main_1_fastapi.py | 34 ++++------------------ 2 files changed, 18 insertions(+), 47 deletions(-) diff --git a/mailchimp_api/deployment/main_1_fastapi.py b/mailchimp_api/deployment/main_1_fastapi.py index b5d98cb..5283037 100644 --- a/mailchimp_api/deployment/main_1_fastapi.py +++ b/mailchimp_api/deployment/main_1_fastapi.py @@ -1,4 +1,5 @@ import os +from pathlib import Path from typing import Any import pandas as pd @@ -35,11 +36,11 @@ def _get_config() -> Config: config = _get_config() -def get_df(file: UploadFile) -> pd.DataFrame: +def _save_file(file: UploadFile, timestamp: str) -> Path: UPLOADED_FILES_DIR.mkdir(exist_ok=True) try: contents = file.file.read() - filename = file.filename if file.filename else "uploaded_file" + filename = f"uploaded-file-{timestamp}.csv" path = UPLOADED_FILES_DIR / filename with path.open("wb") as f: f.write(contents) @@ -51,20 +52,7 @@ def get_df(file: UploadFile) -> pd.DataFrame: finally: file.file.close() - if path.suffix != ".csv": - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Only CSV files are supported", - ) - - df = pd.read_csv(path) - if "email" not in df.columns: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="'email' column not found in CSV file", - ) - - return df + return path @app.post("/upload") @@ -76,11 +64,16 @@ def upload( if not account_name or file.size == 0: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail="Please provide both account name and file", + detail="Please provide both account name and .csv file", ) - df = get_df(file) - + path = _save_file(file, timestamp) + df = pd.read_csv(path) + if "email" not in df.columns: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="'email' column not found in CSV file", + ) try: update_tags(crm_df=df, config=config, list_name=account_name) except Exception as e: diff --git a/tests/deployment/test_main_1_fastapi.py b/tests/deployment/test_main_1_fastapi.py index 13a3dd3..5f71cb2 100644 --- a/tests/deployment/test_main_1_fastapi.py +++ b/tests/deployment/test_main_1_fastapi.py @@ -1,10 +1,9 @@ from io import BytesIO -import pytest -from fastapi import HTTPException, UploadFile -from pandas import DataFrame +import pandas as pd +from fastapi import UploadFile -from mailchimp_api.deployment.main_1_fastapi import get_df +from mailchimp_api.deployment.main_1_fastapi import _save_file class TestUploadFile: @@ -12,31 +11,10 @@ def test_get_df(self) -> None: csv_content = "email\nexample1@example.com\nexample2@example.com" csv_file = BytesIO(csv_content.encode("utf-8")) uploaded_file = UploadFile(filename="emails.csv", file=csv_file) - df = get_df(uploaded_file) + path = _save_file(uploaded_file, "22-09-2021") + df = pd.read_csv(path) - expected_df = DataFrame( + expected_df = pd.DataFrame( {"email": ["example1@example.com", "example2@example.com"]} ) assert df.equals(expected_df) - - @pytest.mark.parametrize( - ("filename", "content", "expected_error"), - [ - ("emails.txt", "email\n", "Only CSV files are supported"), - ( - "emails.csv", - "first_column\n", - "'email' column not found in CSV file", - ), - ], - ) - def test_get_df_raises_error( - self, filename: str, content: str, expected_error: str - ) -> None: - csv_file = BytesIO(content.encode("utf-8")) - uploaded_file = UploadFile(filename=filename, file=csv_file) - - with pytest.raises(HTTPException) as e: - get_df(uploaded_file) - - assert e.value.detail == expected_error From 0696110588ddcc162a52d7aca1a2c066bd27d1e1 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 21 Nov 2024 12:03:32 +0000 Subject: [PATCH 09/16] wip --- mailchimp_api/deployment/main_1_fastapi.py | 26 ++-------------- mailchimp_api/workflow.py | 35 ++++++++++++++++++++-- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/mailchimp_api/deployment/main_1_fastapi.py b/mailchimp_api/deployment/main_1_fastapi.py index 5283037..20ac996 100644 --- a/mailchimp_api/deployment/main_1_fastapi.py +++ b/mailchimp_api/deployment/main_1_fastapi.py @@ -1,4 +1,3 @@ -import os from pathlib import Path from typing import Any @@ -7,9 +6,7 @@ from fastapi import FastAPI, Form, HTTPException, Query, UploadFile, status from fastapi.responses import HTMLResponse -from ..config import Config from ..constants import UPLOADED_FILES_DIR -from ..processing.update_tags import update_tags from ..workflow import wf adapter = FastAPIAdapter(provider=wf) @@ -24,24 +21,12 @@ def list_workflows() -> dict[str, Any]: return {"Workflows": {name: wf.get_description(name) for name in wf.names}} -def _get_config() -> Config: - api_key = os.getenv("MAILCHIMP_API_KEY") - if not api_key: - raise ValueError("MAILCHIMP_API_KEY not set") - - config = Config("us14", api_key) - return config - - -config = _get_config() - - def _save_file(file: UploadFile, timestamp: str) -> Path: UPLOADED_FILES_DIR.mkdir(exist_ok=True) try: contents = file.file.read() - filename = f"uploaded-file-{timestamp}.csv" - path = UPLOADED_FILES_DIR / filename + file_name = f"uploaded-file-{timestamp}.csv" + path = UPLOADED_FILES_DIR / file_name with path.open("wb") as f: f.write(contents) except Exception as e: @@ -74,13 +59,6 @@ def upload( status_code=status.HTTP_400_BAD_REQUEST, detail="'email' column not found in CSV file", ) - try: - update_tags(crm_df=df, config=config, list_name=account_name) - except Exception as e: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=str(e), - ) from e return {"message": f"Successfully uploaded {file.filename}"} diff --git a/mailchimp_api/workflow.py b/mailchimp_api/workflow.py index 669ae20..d4bf48a 100644 --- a/mailchimp_api/workflow.py +++ b/mailchimp_api/workflow.py @@ -2,21 +2,37 @@ import time from typing import Any +import pandas as pd from fastagency import UI from fastagency.runtimes.autogen import AutoGenWorkflows +from .config import Config from .constants import UPLOADED_FILES_DIR +from .processing.update_tags import update_tags wf = AutoGenWorkflows() FASTAPI_URL = os.getenv("FASTAPI_URL", "http://localhost:8008") +def _get_config() -> Config: + api_key = os.getenv("MAILCHIMP_API_KEY") + if not api_key: + raise ValueError("MAILCHIMP_API_KEY not set") + + config = Config("us14", api_key) + return config + + +config = _get_config() + + @wf.register(name="simple_learning", description="Student and teacher learning chat") # type: ignore[misc] def simple_workflow(ui: UI, params: dict[str, Any]) -> str: + timestamp = time.strftime("%Y-%m-%d-%H-%M-%S") body = f"""Please upload **.csv** file with the email addresses for which you want to update the tags. -Upload File +Upload File """ ui.text_message( sender="Workflow", @@ -24,11 +40,24 @@ def simple_workflow(ui: UI, params: dict[str, Any]) -> str: body=body, ) - file_name = "import_audience.csv" + file_name = f"uploaded-file-{timestamp}.csv" file_path = UPLOADED_FILES_DIR / file_name while not file_path.exists(): time.sleep(2) + df = pd.read_csv(file_path) file_path.unlink() - return "Done" + list_name = None + while list_name is None: + list_name = ui.text_input( + sender="Workflow", + recipient="User", + prompt="Please enter Account Name for which you want to update the tags", + ) + + add_tag_members, _ = update_tags( + crm_df=df, config=config, list_name=list_name.strip() + ) + + return f"Added tags\n{add_tag_members}" From 099fdcca4bf2d12add44dda8f59390dcf651fedc Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 21 Nov 2024 12:34:44 +0000 Subject: [PATCH 10/16] Tests added --- mailchimp_api/deployment/main_1_fastapi.py | 17 +++++---- tests/deployment/test_main_1_fastapi.py | 44 ++++++++++++++++++++-- 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/mailchimp_api/deployment/main_1_fastapi.py b/mailchimp_api/deployment/main_1_fastapi.py index 20ac996..3bf9f29 100644 --- a/mailchimp_api/deployment/main_1_fastapi.py +++ b/mailchimp_api/deployment/main_1_fastapi.py @@ -42,14 +42,18 @@ def _save_file(file: UploadFile, timestamp: str) -> Path: @app.post("/upload") def upload( - account_name: str = Form(...), file: UploadFile = UploadFile(...), # type: ignore[arg-type] # noqa: B008 timestamp: str = Form(...), ) -> dict[str, str]: - if not account_name or file.size == 0: + if not file.size: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail="Please provide both account name and .csv file", + detail="Please provide .csv file", + ) + if file.content_type != "text/csv": + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Only CSV files are supported", ) path = _save_file(file, timestamp) @@ -60,16 +64,15 @@ def upload( detail="'email' column not found in CSV file", ) - return {"message": f"Successfully uploaded {file.filename}"} + return { + "message": f"Successfully uploaded {file.filename}. Please close the tab and go back to the chat." + } @app.get("/upload-file") def upload_file(timestamp: str = Query(default="default-timestamp")) -> HTMLResponse: content = f""" -
- -
diff --git a/tests/deployment/test_main_1_fastapi.py b/tests/deployment/test_main_1_fastapi.py index 5f71cb2..24886d2 100644 --- a/tests/deployment/test_main_1_fastapi.py +++ b/tests/deployment/test_main_1_fastapi.py @@ -2,12 +2,15 @@ import pandas as pd from fastapi import UploadFile +from fastapi.testclient import TestClient -from mailchimp_api.deployment.main_1_fastapi import _save_file +from mailchimp_api.deployment.main_1_fastapi import _save_file, app -class TestUploadFile: - def test_get_df(self) -> None: +class TestApp: + client = TestClient(app) + + def test_save_file(self) -> None: csv_content = "email\nexample1@example.com\nexample2@example.com" csv_file = BytesIO(csv_content.encode("utf-8")) uploaded_file = UploadFile(filename="emails.csv", file=csv_file) @@ -18,3 +21,38 @@ def test_get_df(self) -> None: {"email": ["example1@example.com", "example2@example.com"]} ) assert df.equals(expected_df) + + def test_upload_file_endpoint(self) -> None: + timestamp = "22-09-2021" + response = self.client.get(f"/upload-file?timestamp={timestamp}") + assert response.status_code == 200 + assert timestamp in response.text + + def test_upload_endpoint(self) -> None: + csv_content = "email\nemail@gmail.com\n" + csv_file = BytesIO(csv_content.encode("utf-8")) + + response = self.client.post( + "/upload", + files={"file": ("emails.csv", csv_file)}, + data={"timestamp": "test-22-09-2021"}, + ) + assert response.status_code == 200 + expected_msg = "Successfully uploaded emails.csv. Please close the tab and go back to the chat." + assert expected_msg == response.json()["message"] + + def test_upload_endpoint_raises_400_error_if_file_isnt_provided(self) -> None: + response = self.client.post("/upload", data={"timestamp": "test-22-09-2021"}) + assert response.status_code == 400 + assert "Please provide .csv file" in response.text + + def test_upload_endpoint_raises_400_error_if_file_is_not_csv(self) -> None: + csv_content = "email\n" + csv_file = BytesIO(csv_content.encode("utf-8")) + response = self.client.post( + "/upload", + files={"file": ("emails.txt", csv_file)}, + data={"timestamp": "test-22-09-2021"}, + ) + assert response.status_code == 400 + assert "Only CSV files are supported" in response.text From fbd66a51b8bf78526e171d6c4ab4cec2133a564f Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 21 Nov 2024 12:52:34 +0000 Subject: [PATCH 11/16] Update tests --- tests/deployment/test_main_1_fastapi.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/deployment/test_main_1_fastapi.py b/tests/deployment/test_main_1_fastapi.py index 24886d2..f2963f0 100644 --- a/tests/deployment/test_main_1_fastapi.py +++ b/tests/deployment/test_main_1_fastapi.py @@ -1,6 +1,9 @@ from io import BytesIO +from pathlib import Path import pandas as pd +import pytest +from _pytest.monkeypatch import MonkeyPatch from fastapi import UploadFile from fastapi.testclient import TestClient @@ -10,6 +13,21 @@ class TestApp: client = TestClient(app) + @pytest.fixture(autouse=True) + def patch_uploaded_files_dir( + self, tmp_path: Path, monkeypatch: MonkeyPatch + ) -> Path: + uploaded_files_dir = tmp_path / "uploads" + uploaded_files_dir.mkdir(exist_ok=True) + + monkeypatch.setattr( + "mailchimp_api.deployment.main_1_fastapi.UPLOADED_FILES_DIR", + uploaded_files_dir, + ) + + # Return the temporary directory so it can be used in tests if needed + return uploaded_files_dir + def test_save_file(self) -> None: csv_content = "email\nexample1@example.com\nexample2@example.com" csv_file = BytesIO(csv_content.encode("utf-8")) From af4fbb0b9bed94cc00525e98bb1b153ede3d4fac Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 21 Nov 2024 13:05:08 +0000 Subject: [PATCH 12/16] Tests update and refactoring --- mailchimp_api/workflow.py | 2 +- tests/deployment/test_main_1_fastapi.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/mailchimp_api/workflow.py b/mailchimp_api/workflow.py index d4bf48a..1264bfa 100644 --- a/mailchimp_api/workflow.py +++ b/mailchimp_api/workflow.py @@ -27,7 +27,7 @@ def _get_config() -> Config: config = _get_config() -@wf.register(name="simple_learning", description="Student and teacher learning chat") # type: ignore[misc] +@wf.register(name="mailchimp_chat", description="Mailchimp tags update chat") # type: ignore[misc] def simple_workflow(ui: UI, params: dict[str, Any]) -> str: timestamp = time.strftime("%Y-%m-%d-%H-%M-%S") body = f"""Please upload **.csv** file with the email addresses for which you want to update the tags. diff --git a/tests/deployment/test_main_1_fastapi.py b/tests/deployment/test_main_1_fastapi.py index f2963f0..4bea806 100644 --- a/tests/deployment/test_main_1_fastapi.py +++ b/tests/deployment/test_main_1_fastapi.py @@ -74,3 +74,14 @@ def test_upload_endpoint_raises_400_error_if_file_is_not_csv(self) -> None: ) assert response.status_code == 400 assert "Only CSV files are supported" in response.text + + def test_upload_endpoint_raises_400_error_if_email_column_not_found(self) -> None: + csv_content = "name\n" + csv_file = BytesIO(csv_content.encode("utf-8")) + response = self.client.post( + "/upload", + files={"file": ("emails.csv", csv_file)}, + data={"timestamp": "test-22-09-2021"}, + ) + assert response.status_code == 400 + assert "'email' column not found in CSV file" in response.text From c6302419e108254116e0da75708b450d9583b392 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 21 Nov 2024 14:57:06 +0000 Subject: [PATCH 13/16] Implement workflow --- mailchimp_api/workflow.py | 41 ++++++++++++++++++++++------- tests/test_workflow.py | 55 +++++++++++++++++++++++++++++++-------- 2 files changed, 76 insertions(+), 20 deletions(-) diff --git a/mailchimp_api/workflow.py b/mailchimp_api/workflow.py index 1264bfa..b27082b 100644 --- a/mailchimp_api/workflow.py +++ b/mailchimp_api/workflow.py @@ -27,8 +27,20 @@ def _get_config() -> Config: config = _get_config() +def _wait_for_file(timestamp: str) -> pd.DataFrame: + file_name = f"uploaded-file-{timestamp}.csv" + file_path = UPLOADED_FILES_DIR / file_name + while not file_path.exists(): + time.sleep(2) + + df = pd.read_csv(file_path) + file_path.unlink() + + return df + + @wf.register(name="mailchimp_chat", description="Mailchimp tags update chat") # type: ignore[misc] -def simple_workflow(ui: UI, params: dict[str, Any]) -> str: +def mailchimp_chat(ui: UI, params: dict[str, Any]) -> str: timestamp = time.strftime("%Y-%m-%d-%H-%M-%S") body = f"""Please upload **.csv** file with the email addresses for which you want to update the tags. @@ -40,13 +52,7 @@ def simple_workflow(ui: UI, params: dict[str, Any]) -> str: body=body, ) - file_name = f"uploaded-file-{timestamp}.csv" - file_path = UPLOADED_FILES_DIR / file_name - while not file_path.exists(): - time.sleep(2) - - df = pd.read_csv(file_path) - file_path.unlink() + df = _wait_for_file(timestamp) list_name = None while list_name is None: @@ -59,5 +65,22 @@ def simple_workflow(ui: UI, params: dict[str, Any]) -> str: add_tag_members, _ = update_tags( crm_df=df, config=config, list_name=list_name.strip() ) + if not add_tag_members: + return "No tags added" + + add_tag_members = dict(sorted(add_tag_members.items())) + updates_per_tag = "\n".join( + [f"- **{key}**: {len(value)}" for key, value in add_tag_members.items()] + ) + body = f"""Number of updates per tag: + +{updates_per_tag} - return f"Added tags\n{add_tag_members}" +(It might take some time for updates to reflect in Mailchimp) +""" + ui.text_message( + sender="Workflow", + recipient="User", + body=body, + ) + return "Task Completed" diff --git a/tests/test_workflow.py b/tests/test_workflow.py index 60bf452..2305adf 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -1,19 +1,52 @@ -from uuid import uuid4 +from unittest.mock import MagicMock, call, patch -import pytest -from fastagency.ui.console import ConsoleUI +import pandas as pd from mailchimp_api.workflow import wf -from tests.conftest import InputMock -@pytest.mark.skip(reason="Skipping tests for now") -def test_workflow(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("builtins.input", InputMock([""] * 5)) +def test_workflow() -> None: + ui = MagicMock() + ui.text_message.return_value = None + ui.text_input.return_value = "test-list" - result = wf.run( - name="simple_learning", - ui=ConsoleUI().create_workflow_ui(workflow_uuid=uuid4().hex), - ) + with ( + patch( + "mailchimp_api.workflow._wait_for_file", + return_value=pd.DataFrame({"email": ["email1@gmail.com"]}), + ) as mock_wait_for_file, + patch("mailchimp_api.workflow.update_tags") as mock_update_tags, + ): + mock_update_tags.return_value = ( + { + "M4": ["a", "b"], + "M5": ["c", "d", "e"], + "M3": ["f"], + }, + {}, + ) + result = wf.run( + name="mailchimp_chat", + ui=ui, + ) + + mock_wait_for_file.assert_called_once() + mock_update_tags.assert_called_once() + + expected_body = """Number of updates per tag: + +- **M3**: 1 +- **M4**: 2 +- **M5**: 3 + +(It might take some time for updates to reflect in Mailchimp) +""" + expected_call_args = call( + sender="Workflow", + recipient="User", + body=expected_body, + ) + + assert ui.text_message.call_args_list[1] == expected_call_args assert result is not None From 94182ef9160b847e0b30bbc7c905fa306f755238 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 21 Nov 2024 15:15:55 +0000 Subject: [PATCH 14/16] Add MAILCHIMP_API_KEY --- .devcontainer/setup.sh | 6 ++++++ .github/workflows/test.yml | 7 +++++++ docker/Dockerfile | 2 +- scripts/run_docker.sh | 2 +- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index 550f825..7d353a3 100644 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -15,3 +15,9 @@ if [ -z "$OPENAI_API_KEY" ]; then echo -e "\033[33mWarning: OPENAI_API_KEY environment variable is not set.\033[0m" echo fi +# check MAILCHIMP_API_KEY environment variable is set +if [ -z "$MAILCHIMP_API_KEY" ]; then + echo + echo -e "\033[33mWarning: MAILCHIMP_API_KEY environment variable is not set.\033[0m" + echo +fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 55a014f..b69de95 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,6 +43,13 @@ jobs: echo "https://docs.github.com/en/actions/security-for-github-actions/security-guides/using-secrets-in-github-actions#creating-secrets-for-a-repository" exit 1 fi + - name: Check for MAILCHIMP_API_KEY + run: | + if [ -z "${{ secrets.MAILCHIMP_API_KEY }}" ]; then + echo "Error: MAILCHIMP_API_KEY is not set in GitHub secrets." + echo "Please set the MAILCHIMP_API_KEY secret in your repository settings." + exit 1 + fi - name: Run tests run: pytest env: diff --git a/docker/Dockerfile b/docker/Dockerfile index 5921d26..23c936e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -38,4 +38,4 @@ CMD ["/app/run_fastagency.sh"] # Run the container -# docker run --rm -d --name deploy_fastagency -e OPENAI_API_KEY=$OPENAI_API_KEY -p 8008:8008 -p 8888:8888 deploy_fastagency +# docker run --rm -d --name deploy_fastagency -e OPENAI_API_KEY=$OPENAI_API_KEY -e MAILCHIMP_API_KEY=$MAILCHIMP_API_KEY -p 8008:8008 -p 8888:8888 deploy_fastagency diff --git a/scripts/run_docker.sh b/scripts/run_docker.sh index 6933bf7..64bc404 100755 --- a/scripts/run_docker.sh +++ b/scripts/run_docker.sh @@ -1,3 +1,3 @@ #!/bin/bash -docker run -it -e OPENAI_API_KEY=$OPENAI_API_KEY -p 8008:8008 -p 8888:8888 deploy_fastagency +docker run -it -e OPENAI_API_KEY=$OPENAI_API_KEY -e MAILCHIMP_API_KEY=$MAILCHIMP_API_KEY -p 8008:8008 -p 8888:8888 deploy_fastagency From 92e1a981378772aac7ac342a5ec14194a4df68c1 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 21 Nov 2024 15:35:24 +0000 Subject: [PATCH 15/16] Fix CI --- scripts/deploy_to_fly_io.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/deploy_to_fly_io.sh b/scripts/deploy_to_fly_io.sh index 46e58ff..7936e71 100755 --- a/scripts/deploy_to_fly_io.sh +++ b/scripts/deploy_to_fly_io.sh @@ -8,3 +8,4 @@ fly launch --config fly.toml --copy-config --yes echo -e "\033[0;32mSetting secrets\033[0m" fly secrets set OPENAI_API_KEY=$OPENAI_API_KEY +fly secrets set MAILCHIMP_API_KEY=$MAILCHIMP_API_KEY From 41a94684f45dbbb5a7976467ab26895893d09e2c Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 21 Nov 2024 15:36:33 +0000 Subject: [PATCH 16/16] Fix CI --- .github/workflows/test.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b69de95..a87f7ba 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,14 +43,8 @@ jobs: echo "https://docs.github.com/en/actions/security-for-github-actions/security-guides/using-secrets-in-github-actions#creating-secrets-for-a-repository" exit 1 fi - - name: Check for MAILCHIMP_API_KEY - run: | - if [ -z "${{ secrets.MAILCHIMP_API_KEY }}" ]; then - echo "Error: MAILCHIMP_API_KEY is not set in GitHub secrets." - echo "Please set the MAILCHIMP_API_KEY secret in your repository settings." - exit 1 - fi - name: Run tests run: pytest env: CONTEXT: ${{ runner.os }}-py${{ matrix.python-version }} + MAILCHIMP_API_KEY: "test-key" # pragma: allowlist secret