Development

basedosdados_api/account/admin.py

@@ -3,12 +3,14 @@
 from django.contrib import admin
 from django.contrib.auth.admin import UserAdmin as BaseUserAdmin
 from django.contrib.auth.forms import ReadOnlyPasswordHashField
+from django.db import models
+from martor.widgets import AdminMartorWidget
 
-# from django.contrib.auth.models import Group
 
 from basedosdados_api.account.models import (
  RegistrationToken,
  Account,
+ Career,
  BDRole,
  BDGroup,
  BDGroupRole,
@@ -198,8 +200,11 @@ class BDGroupRoleInline(admin.TabularInline):
  extra = 1
 
 
-from martor.widgets import AdminMartorWidget
-from django.db import models
+class CareerAdmin(admin.ModelAdmin):
+ list_display = ("account", "team", "level", "role", "start_at", "end_at")
+ search_fields = ("account", "team")
+ readonly_fields = ("created_at", "updated_at")
+ ordering = ("account", "start_at")
 
 
 class BDGroupAdmin(admin.ModelAdmin):
@@ -214,6 +219,6 @@ class BDGroupAdmin(admin.ModelAdmin):
 
 admin.site.register(RegistrationToken)
 admin.site.register(Account, UserAdmin)
-# admin.site.unregister(Group)
+admin.site.register(Career, CareerAdmin)
 admin.site.register(BDRole)
 admin.site.register(BDGroup, BDGroupAdmin)

basedosdados_api/account/migrations/0006_add_internal_careers_model.py

@@ -0,0 +1,57 @@
+# Generated by Django 4.2.1 on 2023-08-13 17:27
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("account", "0005_rename_ckan_id_account_uuid_account_staff_groups_and_more"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="Career",
+ fields=[
+ (
+ "id",
+ models.UUIDField(
+ default=uuid.uuid4, primary_key=True, serialize=False
+ ),
+ ),
+ (
+ "team",
+ models.CharField(blank=True, max_length=40, verbose_name="Equipe"),
+ ),
+ (
+ "role",
+ models.CharField(blank=True, max_length=40, verbose_name="Cargo"),
+ ),
+ (
+ "level",
+ models.CharField(blank=True, max_length=40, verbose_name="Nível"),
+ ),
+ (
+ "start_at",
+ models.DateField(null=True, verbose_name="Data de Início"),
+ ),
+ ("end_at", models.DateField(null=True, verbose_name="Data de Término")),
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("updated_at", models.DateTimeField(auto_now=True)),
+ (
+ "account",
+ models.ForeignKey(
+ on_delete=django.db.models.deletion.DO_NOTHING,
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Career",
+ "verbose_name_plural": "Careers",
+ },
+ ),
+ ]

basedosdados_api/account/models.py

@@ -5,6 +5,7 @@
 
 from django.core.mail import send_mail
 from django.db import models
+from django.contrib import admin
 from django.contrib.auth.hashers import (
  check_password,
  make_password,
@@ -193,7 +194,6 @@ def create_user(self, email, password=None, profile=2, **kwargs):
  return account
 
  def create_superuser(self, email, password, **kwargs):
-
  account = self.create_user(email, password, profile=1, **kwargs)
 
  account.is_admin = True
@@ -361,3 +361,30 @@ def save(self, *args, **kwargs) -> None:
  # new password, so set it and save the model.
  self.set_password(self.password)
  super().save(*args, **kwargs)
+
+
+class Career(BdmModel):
+ id = models.UUIDField(primary_key=True, default=uuid4)
+ account = models.ForeignKey(Account, on_delete=models.DO_NOTHING)
+
+ team = models.CharField("Equipe", max_length=40, blank=True)
+ role = models.CharField("Cargo", max_length=40, blank=True)
+ level = models.CharField("Nível", max_length=40, blank=True)
+
+ start_at = models.DateField("Data de Início", null=True)
+ end_at = models.DateField("Data de Término", null=True)
+
+ created_at = models.DateTimeField(auto_now_add=True)
+ updated_at = models.DateTimeField(auto_now=True)
+
+ class Meta:
+ verbose_name = "Career"
+ verbose_name_plural = "Careers"
+
+ def __str__(self):
+ return f"{self.account.first_name} @{self.role}"
+
+ def get_team(self):
+ return self.team
+
+ get_team.short_description = "Equipe"

basedosdados_api/api/v1/haystack_engines.py

@@ -30,16 +30,41 @@ def __init__(self, *args, **kwargs):
  },
  "ngram_analyzer": {
  "type": "custom",
- "tokenizer": "lowercase",
- "filter": ["haystack_ngram", "asciifolding"],
+ "tokenizer": "standard",
+ "filter": ["asciifolding", "lowercase", "haystack_ngram"],
  },
  "edgengram_analyzer": {
  "type": "custom",
- "tokenizer": "lowercase",
- "filter": ["haystack_edgengram", "asciifolding"],
+ "tokenizer": "my_tokenizer",
+ "filter": ["asciifolding", "lowercase"],
+ },
+ }
+ tokenizer = {
+ "standard": {"type": "standard"},
+ "lowercase": {"type": "lowercase"},
+ "my_tokenizer": {
+ "type": "edge_ngram",
+ "min_gram": 4,
+ "max_gram": 15,
+ "token_chars": ["letter", "digit"],
  },
  }
+ filter = {
+ "haystack_ngram": {
+ "type": "ngram",
+ "min_gram": 4,
+ "max_gram": 5,
+ },
+ "haystack_edgengram": {
+ "type": "edge_ngram",
+ "min_gram": 2,
+ "max_gram": 15,
+ },
+ }
+
+ self.DEFAULT_SETTINGS["settings"]["analysis"]["tokenizer"] = tokenizer
  self.DEFAULT_SETTINGS["settings"]["analysis"]["analyzer"] = analyzer
+ self.DEFAULT_SETTINGS["settings"]["analysis"]["filter"] = filter
 
  def build_schema(self, fields):
  content_field_name, mapping = super(

basedosdados_api/api/v1/search_indexes.py

@@ -9,12 +9,12 @@ class DatasetIndex(indexes.SearchIndex, indexes.Indexable):
  updated_at = indexes.DateTimeField(model_attr="updated_at")
  text = indexes.CharField(document=True, use_template=True)
  slug = indexes.CharField(model_attr="slug")
- name = indexes.CharField(model_attr="name")
+ name = indexes.EdgeNgramField(model_attr="name")
  description = indexes.EdgeNgramField(model_attr="description", null=True)
 
  organization_id = indexes.CharField(model_attr="organization__id", null=True)
  organization_slug = indexes.CharField(model_attr="organization__slug")
- organization_name = indexes.CharField(model_attr="organization__name")
+ organization_name = indexes.EdgeNgramField(model_attr="organization__name")
  organization_description = indexes.CharField(
  model_attr="organization__description", null=True
  )
@@ -27,7 +27,7 @@ class DatasetIndex(indexes.SearchIndex, indexes.Indexable):
 
  table_ids = indexes.MultiValueField(model_attr="tables__id", null=True)
  table_slugs = indexes.MultiValueField(model_attr="tables__slug", null=True)
- table_names = indexes.MultiValueField(model_attr="tables__name", null=True)
+ table_names = indexes.EdgeNgramField(model_attr="tables__name", null=True)
  table_descriptions = indexes.EdgeNgramField(
  model_attr="tables__description", null=True
  )
@@ -142,15 +142,6 @@ def prepare(self, obj):
  table_ids = data.get("table_slugs", [])
  if table_ids:
  data["tables"] = []
- for i in range(len(table_ids)):
- data["tables"].append(
- {
- "id": data.get("table_ids", [])[i],
- "name": data.get("table_names", [])[i],
- "slug": data.get("table_slugs", [])[i],
- "is_closed": data.get("table_is_closed", [])[i],
- }
- )
  data["total_tables"] = len(table_ids)
  else:
  data["total_tables"] = 0

basedosdados_api/api/v1/views.py

@@ -34,10 +34,14 @@ def get(self, request, *args, **kwargs):
 
  storage = get_storage_class()
 
- # If query is empty, query all datasets
  if not query:
+ # If query is empty, query all datasets
  query = {"match_all": {}}
+ # Factor to multiply the number of tables by
+ # Has no effect if no query is passed
+ n_table_factor = 0
  else:
+ # If query is not empty, query datasets and tables
  query = {
  "bool": {
  "should": [
@@ -49,10 +53,13 @@ def get(self, request, *args, **kwargs):
  }
  }
  },
- {"match": {"name": query}},
+ {"match": {"name.edgengram": query}},
+ {"match": {"table_names.edgengram": query}},
+ {"match": {"organization_name.edgengram": query}},
  ]
  }
  }
+ n_table_factor = 2
 
  all_filters = []
 
@@ -134,13 +141,26 @@ def get(self, request, *args, **kwargs):
  ]
  }
  },
- "field_value_factor": {
- "field": "contains_tables",
- "modifier": "square",
- "factor": 0.5,
- "missing": 0,
- },
- "boost_mode": "sum",
+ "functions": [
+ {
+ "field_value_factor": {
+ "field": "contains_tables",
+ "modifier": "square",
+ "factor": 8,
+ "missing": 0,
+ }
+ },
+ {
+ "field_value_factor": {
+ "field": "n_tables",
+ "modifier": "square",
+ "factor": n_table_factor,
+ "missing": 0,
+ }
+ },
+ ],
+ "score_mode": "sum",
+ "boost_mode": "multiply",
  }
  },
  "aggs": {

basedosdados_api/custom/graphql.py

@@ -388,6 +388,7 @@ def create_model_object_meta(model: BdmModel):
 
 def generate_form_fields(model: BdmModel):
  whitelist_field_types = (
+ models.DateField,
  models.DateTimeField,
  models.SlugField,
  models.CharField,