diff --git a/poetry.lock b/poetry.lock index dbe210d..014f0e0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -83,13 +83,13 @@ crt = ["awscrt (==0.16.9)"] [[package]] name = "certifi" -version = "2023.5.7" +version = "2023.7.22" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2023.5.7-py3-none-any.whl", hash = "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"}, - {file = "certifi-2023.5.7.tar.gz", hash = "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7"}, + {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, + {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, ] [[package]] @@ -280,30 +280,34 @@ files = [ [[package]] name = "cryptography" -version = "41.0.1" +version = "41.0.4" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-41.0.1-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:f73bff05db2a3e5974a6fd248af2566134d8981fd7ab012e5dd4ddb1d9a70699"}, - {file = "cryptography-41.0.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:1a5472d40c8f8e91ff7a3d8ac6dfa363d8e3138b961529c996f3e2df0c7a411a"}, - {file = "cryptography-41.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fa01527046ca5facdf973eef2535a27fec4cb651e4daec4d043ef63f6ecd4ca"}, - {file = "cryptography-41.0.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b46e37db3cc267b4dea1f56da7346c9727e1209aa98487179ee8ebed09d21e43"}, - {file = "cryptography-41.0.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d198820aba55660b4d74f7b5fd1f17db3aa5eb3e6893b0a41b75e84e4f9e0e4b"}, - {file = "cryptography-41.0.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:948224d76c4b6457349d47c0c98657557f429b4e93057cf5a2f71d603e2fc3a3"}, - {file = "cryptography-41.0.1-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:059e348f9a3c1950937e1b5d7ba1f8e968508ab181e75fc32b879452f08356db"}, - {file = "cryptography-41.0.1-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:b4ceb5324b998ce2003bc17d519080b4ec8d5b7b70794cbd2836101406a9be31"}, - {file = "cryptography-41.0.1-cp37-abi3-win32.whl", hash = "sha256:8f4ab7021127a9b4323537300a2acfb450124b2def3756f64dc3a3d2160ee4b5"}, - {file = "cryptography-41.0.1-cp37-abi3-win_amd64.whl", hash = "sha256:1fee5aacc7367487b4e22484d3c7e547992ed726d14864ee33c0176ae43b0d7c"}, - {file = "cryptography-41.0.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:9a6c7a3c87d595608a39980ebaa04d5a37f94024c9f24eb7d10262b92f739ddb"}, - {file = "cryptography-41.0.1-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5d092fdfedaec4cbbffbf98cddc915ba145313a6fdaab83c6e67f4e6c218e6f3"}, - {file = "cryptography-41.0.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1a8e6c2de6fbbcc5e14fd27fb24414507cb3333198ea9ab1258d916f00bc3039"}, - {file = "cryptography-41.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:cb33ccf15e89f7ed89b235cff9d49e2e62c6c981a6061c9c8bb47ed7951190bc"}, - {file = "cryptography-41.0.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5f0ff6e18d13a3de56f609dd1fd11470918f770c6bd5d00d632076c727d35485"}, - {file = "cryptography-41.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:7bfc55a5eae8b86a287747053140ba221afc65eb06207bedf6e019b8934b477c"}, - {file = "cryptography-41.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:eb8163f5e549a22888c18b0d53d6bb62a20510060a22fd5a995ec8a05268df8a"}, - {file = "cryptography-41.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:8dde71c4169ec5ccc1087bb7521d54251c016f126f922ab2dfe6649170a3b8c5"}, - {file = "cryptography-41.0.1.tar.gz", hash = "sha256:d34579085401d3f49762d2f7d6634d6b6c2ae1242202e860f4d26b046e3a1006"}, + {file = "cryptography-41.0.4-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:80907d3faa55dc5434a16579952ac6da800935cd98d14dbd62f6f042c7f5e839"}, + {file = "cryptography-41.0.4-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:35c00f637cd0b9d5b6c6bd11b6c3359194a8eba9c46d4e875a3660e3b400005f"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cecfefa17042941f94ab54f769c8ce0fe14beff2694e9ac684176a2535bf9714"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:23a25c09dfd0d9f28da2352503b23e086f8e78096b9fd585d1d14eca01613e13"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2ed09183922d66c4ec5fdaa59b4d14e105c084dd0febd27452de8f6f74704143"}, + {file = "cryptography-41.0.4-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:5a0f09cefded00e648a127048119f77bc2b2ec61e736660b5789e638f43cc397"}, + {file = "cryptography-41.0.4-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:9eeb77214afae972a00dee47382d2591abe77bdae166bda672fb1e24702a3860"}, + {file = "cryptography-41.0.4-cp37-abi3-win32.whl", hash = "sha256:3b224890962a2d7b57cf5eeb16ccaafba6083f7b811829f00476309bce2fe0fd"}, + {file = "cryptography-41.0.4-cp37-abi3-win_amd64.whl", hash = "sha256:c880eba5175f4307129784eca96f4e70b88e57aa3f680aeba3bab0e980b0f37d"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:004b6ccc95943f6a9ad3142cfabcc769d7ee38a3f60fb0dddbfb431f818c3a67"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:86defa8d248c3fa029da68ce61fe735432b047e32179883bdb1e79ed9bb8195e"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:37480760ae08065437e6573d14be973112c9e6dcaf5f11d00147ee74f37a3829"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b5f4dfe950ff0479f1f00eda09c18798d4f49b98f4e2006d644b3301682ebdca"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7e53db173370dea832190870e975a1e09c86a879b613948f09eb49324218c14d"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5b72205a360f3b6176485a333256b9bcd48700fc755fef51c8e7e67c4b63e3ac"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:93530900d14c37a46ce3d6c9e6fd35dbe5f5601bf6b3a5c325c7bffc030344d9"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c3391bd8e6de35f6f1140e50aaeb3e2b3d6a9012536ca23ab0d9c35ec18c8a91"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:0d9409894f495d465fe6fda92cb70e8323e9648af912d5b9141d616df40a87b8"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:8ac4f9ead4bbd0bc8ab2d318f97d85147167a488be0e08814a37eb2f439d5cf6"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:047c4603aeb4bbd8db2756e38f5b8bd7e94318c047cfe4efeb5d715e08b49311"}, + {file = "cryptography-41.0.4.tar.gz", hash = "sha256:7febc3094125fc126a7f6fb1f420d0da639f3f32cb15c8ff0dc3997c4549f51a"}, ] [package.dependencies] @@ -1187,17 +1191,17 @@ files = [ [[package]] name = "urllib3" -version = "1.26.16" +version = "1.26.17" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ - {file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"}, - {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, + {file = "urllib3-1.26.17-py2.py3-none-any.whl", hash = "sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b"}, + {file = "urllib3-1.26.17.tar.gz", hash = "sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] diff --git a/pyproject.toml b/pyproject.toml index 0b995d8..c193751 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tap-theme-parks" -version = "0.0.1" +version = "0.0.2" description = "`tap-theme-parks` is a Singer tap for theme parks, built with the Meltano Singer SDK." readme = "README.md" authors = ["Daniel Walker"] diff --git a/tap_theme_parks/streams.py b/tap_theme_parks/streams.py index 7e5bd72..66ad0cd 100644 --- a/tap_theme_parks/streams.py +++ b/tap_theme_parks/streams.py @@ -32,15 +32,18 @@ class DestinationsStream(ThemeParksStream): def get_child_context(self, record: dict, context: Optional[dict]) -> dict: """Return a context dictionary for child streams.""" - return {"entity_id": record["id"]} + return { + "destination_id": record["id"], + "park_ids": [x["id"] for x in record["parks"]], + } -class DestinationDetailsStream(ThemeParksStream): - """Define destination details stream""" +class ParkDetailsStream(ThemeParksStream): + """Define park details stream""" parent_stream_type = DestinationsStream - name = "destination_detail" - path = "/entity/{entity_id}" + name = "park_detail" + path_template = "/entity/{park_id}" primary_keys = ["id"] replication_key = None @@ -63,13 +66,22 @@ class DestinationDetailsStream(ThemeParksStream): th.Property("externalId", th.StringType), ).to_dict() + def request_records(self, context: dict | None) -> Iterable[dict]: + for id in context["park_ids"]: + self.path = self.path_template.format(park_id=id) + yield from super().request_records(context) -class DestinationChildrenStream(ThemeParksStream): - """Define destination children stream""" + def get_child_context(self, record: dict, context: Optional[dict]) -> dict: + """Return a context dictionary for child streams.""" + return {"park_id": record["id"]} - parent_stream_type = DestinationsStream - name = "destination_children" - path = "/entity/{entity_id}/children" + +class ParkChildrenStream(ThemeParksStream): + """Define park children stream""" + + parent_stream_type = ParkDetailsStream + name = "park_children" + path = "/entity/{park_id}/children" primary_keys = ["id"] replication_key = None @@ -93,30 +105,69 @@ class DestinationChildrenStream(ThemeParksStream): ).to_dict() -class LiveDataParentStream(ThemeParksStream): - """Live data parent stream, used to pass user supplied ids from config to the LiveDataStream""" +class DestinationDetailsStream(ThemeParksStream): + """Define destination details stream""" - name = "live_data_parent_stream" + parent_stream_type = DestinationsStream + name = "destination_detail" + path = "/entity/{destination_id}" primary_keys = ["id"] + replication_key = None - schema = th.PropertiesList(th.Property("id", th.StringType)).to_dict() + schema = th.PropertiesList( + th.Property("id", th.StringType), + th.Property("name", th.StringType), + th.Property("slug", th.StringType), + th.Property( + "location", + th.ObjectType( + th.Property("latitude", th.NumberType), + th.Property("longitude", th.NumberType), + th.Property("pointOfInterest", th.ArrayType(th.StringType)), + ), + ), + th.Property("parentId", th.StringType), + th.Property("timezone", th.StringType), + th.Property("entityType", th.StringType), + th.Property("destinationId", th.StringType), + th.Property("externalId", th.StringType), + ).to_dict() - def get_records(self, context: Optional[Dict]) -> Iterable[Dict[str, Any]]: - """Return a generator of record-type dictionary objects from config""" - for id in self.config.get("live_data_array"): - yield {"id": id} - def get_child_context(self, record: dict, context: Optional[dict]) -> dict: - """Return a context dictionary for child streams.""" - return {"live_data_id": record["id"]} +class DestinationChildrenStream(ThemeParksStream): + """Define destination children stream""" + + parent_stream_type = DestinationsStream + name = "destination_children" + path = "/entity/{destination_id}/children" + primary_keys = ["id"] + replication_key = None + + schema = th.PropertiesList( + th.Property("id", th.StringType), + th.Property("name", th.StringType), + th.Property("entityType", th.StringType), + th.Property("timezone", th.StringType), + th.Property( + "children", + th.ArrayType( + th.ObjectType( + th.Property("id", th.StringType), + th.Property("name", th.StringType), + th.Property("entityType", th.StringType), + th.Property("slug", th.StringType), + th.Property("externalId", th.StringType), + ) + ), + ), + ).to_dict() class LiveDataStream(ThemeParksStream): """Define live data stream""" - parent_stream_type = LiveDataParentStream name = "live_data" - path = "/entity/{live_data_id}/live" + path_template = "/entity/{live_data_id}/live" primary_keys = ["id"] replication_key = None @@ -193,3 +244,8 @@ class LiveDataStream(ThemeParksStream): ), ), ).to_dict() + + def request_records(self, context: dict | None) -> Iterable[dict]: + for id in self.config.get("live_data_array", []): + self.path = self.path_template.format(live_data_id=id) + yield from super().request_records(context) diff --git a/tap_theme_parks/tap.py b/tap_theme_parks/tap.py index 4b6edb2..e040081 100644 --- a/tap_theme_parks/tap.py +++ b/tap_theme_parks/tap.py @@ -27,10 +27,11 @@ def discover_streams(self) -> list[streams.ThemeParksStream]: streams.DestinationsStream(self), streams.DestinationDetailsStream(self), streams.DestinationChildrenStream(self), + streams.ParkDetailsStream(self), + streams.ParkChildrenStream(self), ] if self.config.get("live_data_array"): - selected_streams.append(streams.LiveDataParentStream(self)) selected_streams.append(streams.LiveDataStream(self)) return selected_streams diff --git a/tests/test_core.py b/tests/test_core.py index 581d6b6..e9dda93 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -24,13 +24,13 @@ class TestEnabledStreams(unittest.TestCase): def test_default_streams(self): catalog = TapThemeParks().discover_streams() - self.assertEqual(len(catalog), 3, "Expected 3 streams from catalog by default") + self.assertEqual(len(catalog), 5, "Expected 5 streams from catalog by default") def test_streams_with_live_data_array(self): catalog = TapThemeParks(config=SAMPLE_CONFIG).discover_streams() self.assertEqual( len(catalog), - 5, - "Expected 5 streams from catalog with the live_data_array setting provided", + 6, + "Expected 6 streams from catalog with the live_data_array setting provided", )