From 3f9b8cb4e9374537cbc02982368dec0f995c8d7d Mon Sep 17 00:00:00 2001
From: Karno <karnoroid@gmail.com>
Date: Fri, 22 Jan 2021 10:47:25 +0900
Subject: [PATCH 1/2] Remove the limitation of maximum pipeline count.

---
 src/kfp_deployer/main.py | 36 +++++++++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/src/kfp_deployer/main.py b/src/kfp_deployer/main.py
index 4cc35f2..3ed26f3 100644
--- a/src/kfp_deployer/main.py
+++ b/src/kfp_deployer/main.py
@@ -3,15 +3,21 @@
 
 import argparse
 from datetime import datetime
-from typing import Optional
+from typing import Iterable, Optional, NamedTuple
 from pytz import timezone
 import kfp
 
-LIST_PAGE_SIZE = 1000
 
 KNOWN_TIMEZONE_TABLE = {"JST": "Asia/Tokyo"}
 
 
+class Pipeline(NamedTuple):
+    """Describes a pipeline deployed on the KFP instance."""
+
+    id: str
+    name: str
+
+
 def main() -> None:
     """Entrypoint."""
     parser = _build_argparser()
@@ -68,17 +74,29 @@ def get_pipeline_id(client: kfp.Client, pipeline_name: str) -> Optional[str]:
     Returns:
         Optional[str]: If found, return Pipeline ID. If not, return None.
     """
-    pipelines_list = client.list_pipelines(page_size=LIST_PAGE_SIZE)
-    if pipelines_list.pipelines:
-        # check pipelines_list.pipelines is not null
-        for p in pipelines_list.pipelines:
-            if p.name == pipeline_name:
-                # found
-                return p.id
+    for p in _iterate_pipelines(client):
+        if p.name == pipeline_name:
+            return p.id
     # not found
     return None
 
 
+def _iterate_pipelines(client: kfp.Client) -> Iterable[Pipeline]:
+    token: Optional[str] = None
+    while True:
+        pipelines_list = client.list_pipelines(page_token=token)
+        # handle the response
+        for p in pipelines_list.pipelines:
+            p_obj = Pipeline(p.id, p.name)
+            yield p_obj
+        # go on to the next page
+        token = pipelines_list.next_page_token
+        if token is None:
+            # when the reading page reached to the end,
+            # next_page_token will be None.
+            return
+
+
 def deploy_new_pipeline(
     client: kfp.Client, pipeline_name: str, pipeline_file_path: str
 ) -> str:

From d54c001d15a6ac8658c7f73fecc87a5756acecb1 Mon Sep 17 00:00:00 2001
From: Karno <karnoroid@gmail.com>
Date: Fri, 22 Jan 2021 10:52:16 +0900
Subject: [PATCH 2/2] reformat code and increment versions.

---
 pyproject.toml           | 4 ++--
 src/kfp_deployer/main.py | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 815f698..b1da492 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "kfp-deployer"
-version = "0.1.0"
+version = "0.1.1"
 description = ""
 authors = ["Karno <karnoroid@gmail.com>"]
 
@@ -15,7 +15,7 @@ pytz = "^2020.5"
 [tool.poetry.dev-dependencies]
 pytest = "^5.2"
 prospector = "^1.3.1"
-black = {version = "^20.8b1", allow-prereleases = true}
+black = { version = "^20.8b1", allow-prereleases = true }
 mypy = "^0.790"
 
 [build-system]
diff --git a/src/kfp_deployer/main.py b/src/kfp_deployer/main.py
index 3ed26f3..768f33b 100644
--- a/src/kfp_deployer/main.py
+++ b/src/kfp_deployer/main.py
@@ -3,10 +3,10 @@
 
 import argparse
 from datetime import datetime
-from typing import Iterable, Optional, NamedTuple
-from pytz import timezone
-import kfp
+from typing import Iterable, NamedTuple, Optional
 
+import kfp
+from pytz import timezone
 
 KNOWN_TIMEZONE_TABLE = {"JST": "Asia/Tokyo"}
 
@@ -157,4 +157,4 @@ def create_version_str(
 
 
 if __name__ == "__main__":
-    main()
\ No newline at end of file
+    main()