Fix the name discovery, event delivery and resurce_collected bugs (#159)

.github/workflows/force_deploy_to_dev.yml

@@ -0,0 +1,99 @@
+name: Push the current commit to dev
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Build docker and deploy to dev
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v3
+
+      - name: Set build platforms
+        id: platform
+        run: |
+          GITHUB_REF="${{ github.ref }}"
+          GITHUB_TAG=${GITHUB_REF##*/}
+          echo "targets=linux/amd64" >> $GITHUB_OUTPUT
+          if [ "${{ github.ref_type }}" = tag ]; then
+              if [[ "$GITHUB_TAG" =~ [0-9]([ab]|rc)[0-9]* ]]; then
+                echo "latest=false" >> $GITHUB_OUTPUT
+              else
+                echo "latest=true" >> $GITHUB_OUTPUT
+              fi
+          else
+              echo "latest=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Check short commit SHA and build targets
+        run: |
+          echo ${{ steps.platform.outputs.targets }}
+          echo ${{ steps.platform.outputs.latest }}
+
+      - name: Docker metadata
+        id: metadata
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            someengineering/fixbackend
+            ghcr.io/someengineering/fixbackend
+          flavor: |
+            latest=${{ steps.platform.outputs.latest }}
+          tags: |
+            type=pep440,pattern={{version}}
+            type=pep440,pattern={{major}}.{{minor}}
+            type=pep440,pattern={{major}}
+            type=sha,format=long,prefix=
+            type=edge
+          labels: |
+            org.opencontainers.image.title=fixbackend
+            org.opencontainers.image.description=coordinate jobs
+            org.opencontainers.image.vendor=Some Engineering Inc.
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_PASS }}
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push fixbackend Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: ${{ steps.platform.outputs.targets }}
+          push: true
+          tags: ${{ steps.metadata.outputs.tags }}
+          labels: ${{ steps.metadata.outputs.labels }}
+          provenance: false # Temporary workaround for https://github.com/docker/buildx/issues/1533
+
+      - name: Authenticate with GitHub CLI
+        run: |
+          gh auth login --with-token <<< "${{ secrets.SOME_CI_PAT }}"
+
+      - name: Bump tag version
+        env:
+          GITHUB_TOKEN: ${{ secrets.SOME_CI_PAT }}
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "Some CI"
+          git clone "https://[email protected]/someengineering/setup-infra.git"
+          # update the tag
+          cd setup-infra
+          sed -i 's/newTag:.*/newTag: ${{ github.sha }}/g' argocd/envs/dev/fixbackend/kustomization.yaml
+          git add .
+          git commit -m "Manual deploy fixbackend on dev to ${{ github.sha }}" 
+          git push origin main

fixbackend/all_models.py

@@ -26,4 +26,3 @@
 from fixbackend.metering.metering_repository import MeteringRecordEntity  # noqa
 from fixbackend.keyvalue.json_kv import JsonEntry  # noqa
 from fixbackend.subscription.subscription_repository import SubscriptionEntity, BillingEntity  # noqa
-from fixbackend.cloud_accounts.last_scan_repository import Entry  # noqa

fixbackend/app.py

@@ -45,7 +45,6 @@
 from fixbackend.auth.router import auth_router, users_router
 from fixbackend.certificates.cert_store import CertificateStore
 from fixbackend.cloud_accounts.account_setup import AwsAccountSetupHelper
-from fixbackend.cloud_accounts.last_scan_repository import LastScanRepository
 from fixbackend.cloud_accounts.repository import CloudAccountRepositoryImpl
 from fixbackend.cloud_accounts.router import cloud_accounts_callback_router, cloud_accounts_router
 from fixbackend.cloud_accounts.service_impl import CloudAccountServiceImpl
@@ -111,7 +110,9 @@ async def setup_teardown_application(_: FastAPI) -> AsyncIterator[None]:
         )
         deps.add(SN.readonly_redis, create_redis(cfg.redis_readonly_url))
         readwrite_redis = deps.add(SN.readwrite_redis, create_redis(cfg.redis_readwrite_url))
-        domain_event_subscriber = deps.add(SN.domain_event_subscriber, DomainEventSubscriber(readwrite_redis, cfg))
+        domain_event_subscriber = deps.add(
+            SN.domain_event_subscriber, DomainEventSubscriber(readwrite_redis, cfg, "fixbackend")
+        )
         engine = deps.add(
             SN.async_engine,
             create_async_engine(
@@ -168,7 +169,6 @@ async def setup_teardown_application(_: FastAPI) -> AsyncIterator[None]:
                 CloudAccountRepositoryImpl(session_maker),
                 cloud_accounts_redis_publisher,
                 domain_event_publisher,
-                LastScanRepository(session_maker),
                 readwrite_redis,
                 cfg,
                 AwsAccountSetupHelper(boto_session),
@@ -197,7 +197,9 @@ async def setup_teardown_dispatcher(_: FastAPI) -> AsyncIterator[None]:
             ),
         )
         rw_redis = deps.add(SN.readwrite_redis, create_redis(cfg.redis_readwrite_url))
-        domain_event_subscriber = deps.add(SN.domain_event_subscriber, DomainEventSubscriber(rw_redis, cfg))
+        domain_event_subscriber = deps.add(
+            SN.domain_event_subscriber, DomainEventSubscriber(rw_redis, cfg, "dispatching")
+        )
         temp_store_redis = deps.add(SN.temp_store_redis, create_redis(cfg.redis_temp_store_url))
         engine = deps.add(
             SN.async_engine,

fixbackend/cloud_accounts/account_setup.py

@@ -16,7 +16,7 @@
 
 from abc import ABC
 import logging
-from typing import Optional, Dict
+from typing import Optional, Dict, Any
 from fixcloudutils.asyncio.async_extensions import run_async
 from fixbackend.ids import ExternalId, CloudAccountAlias, CloudAccountName
 from attrs import frozen
@@ -84,15 +84,19 @@ async def list_accounts(
         next_token = None
         try:
             while True:
+                kwargs: Dict[str, Any] = {}
+                if next_token:
+                    kwargs["NextToken"] = next_token
                 response = await run_async(
                     orgnizations_client.list_accounts,
-                    NextToken=next_token,
+                    **kwargs,
                 )
                 next_token = response.get("NextToken")
                 accounts.extend(response["Accounts"])
                 if next_token is None:
                     break
-        except Exception:
+        except Exception as ex:
+            log.info("Failed to list accounts: %s", ex)
             return {}
 
         return {CloudAccountId(account["Id"]): CloudAccountName(account["Name"]) for account in accounts}

fixbackend/cloud_accounts/models/__init__.py

@@ -13,7 +13,7 @@
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from datetime import datetime
-from typing import ClassVar, Dict, Optional
+from typing import ClassVar, Optional
 from abc import ABC
 
 from attrs import frozen
@@ -117,17 +117,7 @@ class CloudAccount:
     account_alias: Optional[CloudAccountAlias]
     user_account_name: Optional[UserCloudAccountName]
     privileged: bool  # can do administrative tasks
-
-
-@frozen
-class LastScanAccountInfo:
-    account_id: CloudAccountId
-    duration_seconds: int
-    resources_scanned: int
-    started_at: datetime
-
-
-@frozen
-class LastScanInfo:
-    accounts: Dict[FixCloudAccountId, LastScanAccountInfo]
     next_scan: Optional[datetime]
+    last_scan_duration_seconds: int
+    last_scan_started_at: Optional[datetime]
+    last_scan_resources_scanned: int

fixbackend/cloud_accounts/models/orm.py

@@ -17,8 +17,10 @@
 from typing import Optional
 
 from fastapi_users_db_sqlalchemy.generics import GUID
-from sqlalchemy import ForeignKey, String, UniqueConstraint, Boolean
+from fixbackend.sqlalechemy_extensions import UTCDateTime
+from sqlalchemy import ForeignKey, String, UniqueConstraint, Boolean, Integer
 from sqlalchemy.orm import Mapped, mapped_column
+from datetime import datetime
 
 from fixbackend.base_model import Base
 from fixbackend.cloud_accounts import models
@@ -53,6 +55,11 @@ class CloudAccount(Base):
     enabled: Mapped[bool] = mapped_column(Boolean, nullable=False)
     state: Mapped[Optional[str]] = mapped_column(String(length=64), nullable=True)
     error: Mapped[Optional[str]] = mapped_column(String(length=64), nullable=True)
+    next_scan: Mapped[Optional[datetime]] = mapped_column(UTCDateTime, nullable=True)
+    last_scan_duration_seconds: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    last_scan_started_at: Mapped[Optional[datetime]] = mapped_column(UTCDateTime, nullable=True)
+    last_scan_resources_scanned: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+
     __table_args__ = (UniqueConstraint("tenant_id", "account_id"),)
 
     def to_model(self) -> models.CloudAccount:
@@ -98,4 +105,8 @@ def state() -> models.CloudAccountState:
             account_alias=self.api_account_alias,
             user_account_name=self.user_account_name,
             privileged=self.privileged,
+            next_scan=self.next_scan,
+            last_scan_duration_seconds=self.last_scan_duration_seconds,
+            last_scan_started_at=self.last_scan_started_at,
+            last_scan_resources_scanned=self.last_scan_resources_scanned,
         )