Merge branch 'main' into feat/gcp-secret-env-file

.github/workflows/sync.yaml

@@ -0,0 +1,66 @@
+name: Sync
+
+on:
+ workflow_dispatch:
+ inputs:
+ environment:
+ description: Destination environment
+ required: true
+ type: choice
+ options:
+ - staging
+ - development
+
+jobs:
+ sync:
+ name: Sync
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ id-token: write
+ if: contains('AldemirLucas vncsna', github.actor)
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+
+ - name: Import secrets
+ id: import_secrets
+ uses: hashicorp/[email protected]
+ with:
+ url: https://vault.basedosdados.org
+ token: ${{ secrets.VAULT_TOKEN }}
+ secrets: |
+ secret/data/gcp_credentials/basedosdados-dev GCP_PROJECT_ID | GCP_PROJECT_ID ;
+ secret/data/gcp_credentials/basedosdados-dev GH_ACTIONS_SA | GCP_SA ;
+ secret/data/gcp_credentials/basedosdados-dev GKE_CLUSTER_NAME | GKE_CLUSTER_NAME ;
+ secret/data/gcp_credentials/basedosdados-dev GKE_CLUSTER_ZONE | GKE_CLUSTER_ZONE ;
+
+ - name: Authenticate GCP
+ id: auth
+ uses: google-github-actions/auth@v1
+ with:
+ credentials_json: ${{ steps.import_secrets.outputs.GCP_SA }}
+
+ - name: Set up GCP SDK
+ uses: google-github-actions/setup-gcloud@v1
+ with:
+ version: ">= 363.0.0"
+
+ - name: Install kubectl
+ run: |
+ sudo apt-get update --quiet
+ sudo apt-get install --yes apt-transport-https ca-certificates curl
+ curl -LO https://dl.k8s.io/release/v1.24.15/bin/linux/amd64/kubectl
+ sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+
+ - name: Set up kubectl
+ run: |
+ gcloud components install gke-gcloud-auth-plugin
+ gcloud container clusters get-credentials\
+ ${{ steps.import_secrets.outputs.GKE_CLUSTER_NAME }}\
+ --zone ${{ steps.import_secrets.outputs.GKE_CLUSTER_ZONE }}
+ kubectl config set-context --current --namespace=website
+
+ - name: Sync databases
+ run: |
+ ./scripts/sync.sh ${{ inputs.environment }}

basedosdados_api/account/admin.py

@@ -3,9 +3,7 @@
 from django.contrib import admin
 from django.contrib.auth.admin import UserAdmin as BaseAccountAdmin
 from django.contrib.auth.forms import ReadOnlyPasswordHashField
-from django.db import models
 from faker import Faker
-from martor.widgets import AdminMartorWidget
 
 from basedosdados_api.account.models import Account, BDGroup, BDGroupRole, BDRole, Career
 
@@ -224,9 +222,6 @@ class CareerAdmin(admin.ModelAdmin):
 
 
 class BDGroupAdmin(admin.ModelAdmin):
- formfield_overrides = {
- models.TextField: {"widget": AdminMartorWidget},
- }
  inlines = (BDGroupRoleInline,)
  list_display = ("name", "description")
  search_fields = ("name", "description")

basedosdados_api/account/models.py

@@ -11,27 +11,38 @@
  Permission,
  PermissionsMixin,
 )
-from django.core.mail import send_mail
 from django.db import models
-from django.db.models import DateTimeField, F, Q
-from django.db.models.functions import Coalesce
 
 from basedosdados_api.account.storage import OverwriteStorage
 from basedosdados_api.api.v1.validators import validate_is_valid_image_format
 from basedosdados_api.custom.model import BdmModel
 
 
 def image_path_and_rename(instance, filename):
- """
- Rename file to be the username
- """
+ """Rename file to be the username"""
  upload_to = instance.__class__.__name__.lower()
  ext = filename.split(".")[-1]
- # get filename
  filename = f"{instance.username}.{ext}"
  return os.path.join(upload_to, filename)
 
 
+def split_password(password: str) -> Tuple[str, str, str, str]:
+ """Split a password into four parts: algorithm, iterations, salt, and hash"""
+ algorithm, iterations, salt, hash = password.split("$", 3)
+ return algorithm, iterations, salt, hash
+
+
+def is_valid_encoded_password(password: str) -> bool:
+ """Check if a password is valid"""
+ double_encoded = make_password(password)
+ try:
+ target_algorithm, target_iterations, _, _ = split_password(double_encoded)
+ algorithm, iterations, _, _ = split_password(password)
+ except ValueError:
+ return False
+ return algorithm == target_algorithm and iterations == target_iterations
+
+
 class RegistrationToken(BdmModel):
  token = models.CharField(max_length=255, unique=True, default=uuid4)
  created_at = models.DateTimeField(auto_now_add=True)
@@ -292,6 +303,10 @@ class Meta:
  verbose_name_plural = "accounts"
  ordering = ["first_name", "last_name"]
 
+ @property
+ def is_staff(self):
+ return self.is_admin
+
  def __str__(self):
  return self.email
 
@@ -310,50 +325,6 @@ def get_organization(self):
 
  get_organization.short_description = "organização"
 
- def get_graphql_is_active_staff(self) -> bool:
- query = (
- self.careers.annotate(
- transition_at=Coalesce(F("start_at"), F("end_at"), output_field=DateTimeField())
- )
- .filter(Q(start_at__isnull=False) | Q(end_at__isnull=False))
- .order_by("transition_at")
- )
- if len(query.all()) > 0 and query.last().end_at is None:
- return True
- return False
-
- @property
- def is_staff(self):
- return self.is_admin
-
- def email_user(self, *args, **kwargs):
- send_mail(
- "{}".format(args[0]),
- "{}".format(args[1]),
- "{}".format(args[2]),
- [self.email],
- fail_silently=False,
- )
-
- def split_password(self, password: str) -> Tuple[str, str, str, str]:
- """
- Split a password into four parts: algorithm, iterations, salt, and hash.
- """
- algorithm, iterations, salt, hash = password.split("$", 3)
- return algorithm, iterations, salt, hash
-
- def is_valid_encoded_password(self, password: str) -> bool:
- """
- Check if a password is valid.
- """
- double_encoded = make_password(password)
- try:
- target_algorithm, target_iterations, _, _ = self.split_password(double_encoded)
- algorithm, iterations, _, _ = self.split_password(password)
- except ValueError:
- return False
- return algorithm == target_algorithm and iterations == target_iterations
-
  def save(self, *args, **kwargs) -> None:
  # If self._password is set and check_password(self._password, self.password) is True, then
  # just save the model without changing the password.
@@ -362,7 +333,7 @@ def save(self, *args, **kwargs) -> None:
  return
  # If self._password is not set, we're probably not trying to modify the password, so if
  # self.password is valid, just save the model without changing the password.
- elif self.is_valid_encoded_password(self.password):
+ elif is_valid_encoded_password(self.password):
  super().save(*args, **kwargs)
  return
  # If self.password is not usable, then we're probably trying to set self.password as the

basedosdados_api/api/v1/admin.py

@@ -198,16 +198,10 @@ class CoverageTableInline(admin.StackedInline):
  readonly_fields = [
  "id",
  "area",
- # "table",
  ]
  inlines = [
  DateTimeRangeInline,
  ]
- # template = "admin/edit_inline/custom_coverage_model_inline.html"
- # inlines = [
- # TableCoverageFilter,
- # ]
- # formfield_overrides = {models.TextField: {"widget": AdminMartorWidget}}
 
 
 ################################################################################
@@ -494,7 +488,6 @@ def related_objects(self, obj):
 
  related_objects.short_description = "Tables"
 
- # formfield_overrides = {models.TextField: {"widget": AdminMartorWidget}}
  readonly_fields = [
  "id",
  "full_slug",
@@ -680,7 +673,6 @@ class ColumnAdmin(TabbedTranslationAdmin):
  list_filter = [
  "table__dataset__organization__name",
  ]
- # formfield_overrides = {models.TextField: {"widget": AdminMartorWidget}}
 
 
 class ObservationLevelAdmin(admin.ModelAdmin):
@@ -722,7 +714,6 @@ class RawDataSourceAdmin(TabbedTranslationAdmin):
  "languages",
  "area_ip_address_required",
  ]
- # formfield_overrides = {models.TextField: {"widget": AdminMartorWidget}}
 
 
 class InformationRequestAdmin(TabbedTranslationAdmin):
@@ -732,7 +723,6 @@ class InformationRequestAdmin(TabbedTranslationAdmin):
  autocomplete_fields = [
  "dataset",
  ]
- # formfield_overrides = {models.TextField: {"widget": AdminMartorWidget}}
 
 
 class CoverageTypeAdminFilter(admin.SimpleListFilter):

basedosdados_api/api/v1/urls.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+from django.http import HttpResponseRedirect
+from django.urls import path
+from django.views.decorators.csrf import csrf_exempt
+from graphene_file_upload.django import FileUploadGraphQLView
+
+
+def redirect_to_v1(request):
+ return HttpResponseRedirect("/api/v1/")
+
+
+def redirect_to_v1_graphql(request):
+ return HttpResponseRedirect("/api/v1/graphql")
+
+
+urlpatterns = [
+ path("", redirect_to_v1),
+ path("v1/", redirect_to_v1_graphql),
+ path(
+ "v1/graphql",
+ csrf_exempt(FileUploadGraphQLView.as_view(graphiql=True)),
+ ),
+]

basedosdados_api/core/management/commands/dumpfixture.py

@@ -41,10 +41,14 @@ def empty():
 
 
 class Command(DumpDataCommand):
- """Dump data, limiting account profiles"""
+ """Dump data, avoiding profiles and payments"""
 
  def handle(self, *args, **options) -> str | None:
  print("Dump fixtures")
+ options["exclude"] = [
+ "djstripe",
+ *options["exclude"],
+ ]
  response = super().handle(*args, **options)
 
  print("Filter fixtures")

basedosdados_api/custom/graphql.py

@@ -275,9 +275,15 @@ def id_resolver(self, *_):
 
 
 def generate_filter_fields(model: BdmModel):
- exempted_field_types = (models.ImageField,)
  exempted_field_names = ("_field_status",)
- string_field_types = (models.CharField, models.TextField)
+ exempted_field_types = (
+ models.ImageField,
+ models.JSONField,
+ )
+ string_field_types = (
+ models.CharField,
+ models.TextField,
+ )
  comparable_field_types = (
  models.BigIntegerField,
  models.IntegerField,
@@ -320,6 +326,7 @@ def _get_filter_fields(model: BdmModel, used_models: Optional[Iterable[BdmModel]
  for field in model_fields:
  if (
  isinstance(field, exempted_field_types)
+ or "djstripe" in field.name
  or field.name in exempted_field_names
  or model.__module__.startswith("django")
  ):

basedosdados_api/payments/admin.py

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+from django.contrib import admin # noqa
+
+# Register your models here.

basedosdados_api/payments/apps.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+from djstripe.apps import DjstripeAppConfig
+
+
+class PaymentsConfig(DjstripeAppConfig):
+ verbose_name = "Stripe"

basedosdados_api/payments/models.py

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+from django.db import models # noqa
+
+# Create your models here.

basedosdados_api/payments/tests.py

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+from django.test import TestCase # noqa
+
+# Create your tests here.

basedosdados_api/payments/urls.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+from django.urls import include, path
+
+urlpatterns = [
+ path("", include("djstripe.urls", namespace="djstripe")),
+]

basedosdados_api/payments/views.py

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+from django.shortcuts import render # noqa
+
+# Create your views here.