Add support pandas 2 in flytekit

dev-requirements.in

@@ -25,9 +25,13 @@ torch<=1.12.1; python_version<'3.11'
 # pytorch 2 supports python 3.11
 torch<=2.0.0; python_version>='3.11' or platform_system!='Windows'
 
-scikit-learn
 types-protobuf
 types-croniter
 types-mock
 autoflake
+
+pandas
+pyarrow
+numpy
+scikit-learn
 prometheus-client

setup.py

@@ -31,8 +31,6 @@
         "googleapis-common-protos>=1.57",
         "flyteidl>=1.5.16",
         "wheel>=0.30.0,<1.0.0",
-        "pandas>=1.0.0,<2.0.0",
-        "pyarrow>=4.0.0,<11.0.0",
         "click>=6.6,<9.0",
         "croniter>=0.3.20,<4.0.0",
         "deprecated>=1.0,<2.0",
@@ -71,7 +69,6 @@
         "diskcache>=5.2.1",
         "cloudpickle>=2.0.0",
         "cookiecutter>=1.7.3",
-        "numpy",
         "gitpython",
         "kubernetes>=12.0.1",
         "rich",

tests/flytekit/unit/core/test_type_hints.py

@@ -858,7 +858,7 @@ def t1(a: int) -> pandas.DataFrame:
 
     @task
     def t2(df: pandas.DataFrame) -> pandas.DataFrame:
-        return df.append(pandas.DataFrame(data={"col1": [5, 10], "col2": [5, 10]}))
+        return pd.concat([df, pandas.DataFrame(data={"col1": [5, 10], "col2": [5, 10]})])
 
     @workflow
     def my_wf(a: int) -> pandas.DataFrame: