Skip to content

Commit

Permalink
Merge branch 'dev' of https://github.com/ES2-UFPI/WiseBuilder into su…
Browse files Browse the repository at this point in the history
…gestor-de-componentes-#52
  • Loading branch information
wesleyvitor11000 committed Mar 23, 2023
2 parents f99435f + b35e197 commit 44a514e
Show file tree
Hide file tree
Showing 38 changed files with 817 additions and 190 deletions.
7 changes: 3 additions & 4 deletions src/Scraper/application/ScraperOrchestration/wrapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,6 @@
from Scraper.domain.commands import *
from framework.application.handler import MessageBus

engine = _get_engine()


class Wrapper:
_volatile_data_message_bus: MessageBus
Expand All @@ -40,6 +38,7 @@ class Wrapper:
max_sleep_seconds = 3

def __init__(self, scheme: str, domain: str):
engine = _get_engine()
self.domain = domain
self.session = create_session(engine)

Expand Down Expand Up @@ -67,11 +66,11 @@ async def run_scraping(self):
next_url: URL = domain_url.url

while next_url != None:
next_url, volatile_datas = self.scraper.get_volatile_data(
next_url, components_volatile_data = self.scraper.get_volatile_data(
url=next_url.url
)

for url, name, cost, availability in volatile_datas:
for url, name, cost, availability in components_volatile_data:
# TODO: fazer chamada da engine de busca para classificar o componente
# component = SearchEngine.classifie(name)
component_manager = SQLAlchemyRepository(
Expand Down
18 changes: 11 additions & 7 deletions src/Scraper/application/handlers.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from typing import Dict, Type
from typing import Dict, Type, List
from smtplib import SMTP
from ssl import create_default_context
from email.mime.text import MIMEText
Expand All @@ -13,6 +13,7 @@
from SearchEngine.application.unit_of_work import SQLAlchemyUnitOfWork
from framework.domain.components import Component
from framework.application.handler import MessageHandler, Command
from ..domain.repositories import ICategoryURLRepository
from ..domain.events import LowerPriceRegisteredEvent
from framework.domain.events import DomainEvent
from ..domain.commands import *
Expand All @@ -21,7 +22,6 @@
# Provisório. Mover para microsserviço de usuários.
class LowerPriceRegisteredHandler(MessageHandler):
def __call__(self, event: LowerPriceRegisteredEvent):
print("sending mail")
sender, passwd = mail, mail_passwd
recv_list = [sender]

Expand All @@ -41,13 +41,14 @@ def __call__(self, event: LowerPriceRegisteredEvent):
"Subject"
] = f"Preço reduzido: {component.manufacturer} {component.model}!"
message["From"] = sender

context = create_default_context()
with SMTP("smtp.gmail.com", 587) as server:
server.starttls(context=context)
server.login(sender, passwd)
server.send_message(message, sender, recv_list)

print("email sended")


class AddCategoryURLHandler(MessageHandler):
def __call__(self, cmd: AddCategoryURL):
Expand All @@ -58,7 +59,8 @@ def __call__(self, cmd: AddCategoryURL):
class GetAllDomainsHandler(MessageHandler):
def __call__(self, cmd: GetAllDomains):
with self.uow:
return self.uow.repository.get_all_domains()
if isinstance(self.uow.repository, ICategoryURLRepository):
return self.uow.repository.get_all_domains()


class GetVolatileDataByDomainHandler(MessageHandler):
Expand All @@ -79,12 +81,14 @@ def __call__(self, cmd: AddVolatileData):
GetAllDomains: GetAllDomainsHandler,
}

CURL_EVENT_HANDLER_MAPPER: Dict[Type[DomainEvent], Type[MessageHandler]] = {}
CURL_EVENT_HANDLER_MAPPER: Dict[Type[DomainEvent], List[Type[MessageHandler]]] = {}

VD_COMMAND_HANDLER_MAPPER: Dict[Type[Command], Type[MessageHandler]] = {
AddVolatileData: AddVolatileDataHandler
}

VD_EVENT_HANDLER_MAPPER: Dict[Type[DomainEvent], Type[MessageHandler]] = {
LowerPriceRegisteredEvent: LowerPriceRegisteredHandler
VD_EVENT_HANDLER_MAPPER: Dict[Type[DomainEvent], List[Type[MessageHandler]]] = {
LowerPriceRegisteredEvent: [
LowerPriceRegisteredHandler,
]
}
4 changes: 2 additions & 2 deletions src/Scraper/domain/repositories.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ def __post_init__(self):


class MockRepository(AbstractRepository):
def __init__(self, volatile_datas: Dict[UUID, VolatileData]):
self._volatile_data = volatile_datas
def __init__(self, volatile_data: Dict[UUID, VolatileData]):
self._volatile_data = volatile_data

def _add(self, volatile_data: VolatileData):
self._volatile_data[volatile_data.uid] = volatile_data
Expand Down
45 changes: 11 additions & 34 deletions src/Scraper/infrastructure/SQL_alchemy_category_url.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
from sqlalchemy.exc import NoResultFound, IntegrityError
from sqlalchemy.orm.session import Session
from typing import List
from operator import lt, gt, eq

from framework.domain.value_object import UUID
from framework.infrastructure.db_management.db_structure import (
Expand All @@ -10,7 +9,11 @@
)
from framework.domain.components import EComponentType
from Scraper.domain.entity import CategoryURL, AttrsCategoryURL
from framework.infrastructure.db_management.db_mapping import map_from_to
from framework.infrastructure.db_management.db_mapping import (
map_from_to,
parse_filters,
filter_instance_from_db,
)
from framework.domain.value_object import URL, AttrsURL
from Scraper.domain.repositories import (
ICategoryURLRepository,
Expand All @@ -20,8 +23,6 @@


class SQLAlchemyCategoryURL(ICategoryURLRepository):
_filters_ops: dict = {"filters_eq": eq}

def __init__(self, session):
self._session: Session = session

Expand Down Expand Up @@ -60,29 +61,6 @@ def _db_to_category_url(self, url_instance: CategoryURLInstance):

return category_url

def _parse_filters(self, **kwargs) -> List:
ret = []

for filter_type, filters in kwargs.items():
if filter_type in self._filters_ops.keys():
op = self._filters_ops[filter_type]

[
ret.append(op(getattr(CategoryURLInstance, prop), value))
for prop, value in filters.items()
]

return ret

def _filter_category_url_from_db(self, filters: List) -> List[CategoryURL]:
url_instances: List[CategoryURLInstance] = (
self._session.query(CategoryURLInstance).filter(*filters).all()
)

urls = [self._db_to_category_url(instance) for instance in url_instances]

return urls

def _add(self, category_url: CategoryURL):
url_instance = self._url_to_db(category_url)

Expand All @@ -93,14 +71,13 @@ def _add(self, category_url: CategoryURL):
raise EntityUIDCollisionException(category_url.uid)

def _get(self, **kwargs) -> List[CategoryURL]:
ret = []

filters = self._parse_filters(**kwargs)

urls = self._filter_category_url_from_db(filters)
ret.extend(urls)
filters = parse_filters(CategoryURLInstance, **kwargs)
urls_instances = filter_instance_from_db(
self._session, CategoryURLInstance, filters
)
urls = [self._db_to_category_url(instance) for instance in urls_instances]

return ret
return urls

def _get_by_uid(self, ref: UUID):
query_filter = [CategoryURLInstance.uid == ref]
Expand Down
5 changes: 2 additions & 3 deletions src/Scraper/infrastructure/SQL_alchemy_volatile_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,8 @@ def _add(self, volatile_data: VolatileData):
volatile_data.component_id, volatile_data.cost
)
)
print(
f"preço reduzido de {current_volatile_data.cost} para {db_volatile_data.cost}."
)

print("redução do preço")

current_volatile_data.cost = db_volatile_data.cost
current_volatile_data.timestamp = db_volatile_data.timestamp
Expand Down
4 changes: 2 additions & 2 deletions src/SearchEngine/application/handlers.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from typing import Dict, Type
from typing import Dict, Type, List

from SearchEngine.domain.commands import *

Expand Down Expand Up @@ -38,4 +38,4 @@ def __call__(self, cmd: AddComponent):
}


SE_EVENT_HANDLER_MAPPER: Dict[Type[DomainEvent], Type[MessageHandler]] = {}
SE_EVENT_HANDLER_MAPPER: Dict[Type[DomainEvent], List[Type[MessageHandler]]] = {}
1 change: 0 additions & 1 deletion src/SearchEngine/domain/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
from typing import List, Dict, Any

from framework.domain.components import Component, EComponentType, component_cls_idx

from framework.domain.value_object import UUID, UUIDv4
from framework.domain.events import Command

Expand Down
2 changes: 1 addition & 1 deletion src/SearchEngine/domain/repositories.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from dataclasses import dataclass, field
from typing import Any, Dict
from typing import Dict
from abc import ABCMeta, abstractmethod

from framework.domain.value_object import UUID
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
from sqlalchemy.exc import IntegrityError, NoResultFound
from sqlalchemy.engine import Row
from typing import List
from operator import lt, gt, eq

from framework.domain.components import *
from framework.domain.value_object import UUID
Expand All @@ -16,16 +15,18 @@
component_inst_idx,
)
from SearchEngine.infrastructure.ComponentManagment.component_mapper import *
from framework.infrastructure.db_management.db_mapping import (
parse_filters,
filter_instance_from_db,
)


class SQLAlchemyRepository(ISQLAlchemyRepository):
_filters_ops: dict = {"filters_eq": eq, "filters_lt": lt, "filters_gt": gt}

def __init__(self, session):
self._session: Session = session

def _add(self, component: Component):
component_instance = component_to_bd_object(component)
component_instance = component_to_db_object(component)

try:
self._session.add(component_instance)
Expand All @@ -47,40 +48,13 @@ def _get_by_uid(self, ref: UUID) -> Component:
.one()
)

component = bd_object_to_component(component_inst)
component = db_object_to_component(component_inst)

except NoResultFound:
raise EntityUIDNotFoundException(ref)

return component

def _filter_components_from_db(
self, filters: List, type_inst: ComponentInstance, qsize: int | None
) -> List[Component | ComponentInstance]:
component_instances: List[ComponentInstance] = (
self._session.query(type_inst).filter(*filters).limit(qsize).all()
)

components = [
bd_object_to_component(instance) for instance in component_instances
]

return components

def _parse_filters(self, instance_type: ComponentInstance, **kwargs) -> List:
ret = []

for filter_type, filters in kwargs.items():
if filter_type in self._filters_ops.keys():
op = self._filters_ops[filter_type]

[
ret.append(op(getattr(instance_type, prop), value))
for prop, value in filters.items()
]

return ret

def _get(self, **kwargs) -> List[Component]:
qsize: int = kwargs.get("qsize", None)
ctypes: List = kwargs.get("ctypes", [])
Expand All @@ -91,9 +65,14 @@ def _get(self, **kwargs) -> List[Component]:

for ctype in ctypes:
instance_cls = component_inst_idx[ctype]
filters = self._parse_filters(instance_cls, **kwargs)
filters = parse_filters(instance_cls, **kwargs)

components = self._filter_components_from_db(filters, instance_cls, qsize)
component_instances = filter_instance_from_db(
self._session, instance_cls, filters, qsize
)
components = [
db_object_to_component(instance) for instance in component_instances
]
ret.extend(components)

if qsize != None:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
)

__all__ = [
"component_to_bd_object",
"bd_object_to_component",
"component_to_db_object",
"db_object_to_component",
"find_type_by_property_names",
]

Expand All @@ -21,15 +21,15 @@ def _get_attrs_from(c_type: EComponentType):
return comp_attrs, comp_inst_attrs


def component_to_bd_object(component: Component) -> ComponentInstance:
def component_to_db_object(component: Component) -> ComponentInstance:
specific_inst_cls = component_inst_idx[component.type.value]
comp_attrs, comp_inst_attrs = _get_attrs_from(component.type)
mapped_comp_dict = map_from_to(component, comp_attrs, comp_inst_attrs)

return specific_inst_cls(**mapped_comp_dict)


def bd_object_to_component(component_instance: ComponentInstance) -> Component:
def db_object_to_component(component_instance: ComponentInstance) -> Component:
specific_comp_cls = component_cls_idx[component_instance.type]
comp_attrs, comp_inst_attrs = _get_attrs_from(
EComponentType(component_instance.type)
Expand Down
1 change: 1 addition & 0 deletions src/entrypoints/web/wise-builder/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package-lock.json
/node_modules
/.pnp
.pnp.js
.next/

# testing
/coverage
Expand Down
5 changes: 5 additions & 0 deletions src/entrypoints/web/wise-builder/next-env.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
/// <reference types="next" />
/// <reference types="next/image-types/global" />

// NOTE: This file should not be edited
// see https://nextjs.org/docs/basic-features/typescript for more information.
11 changes: 4 additions & 7 deletions src/entrypoints/web/wise-builder/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,16 @@
"@types/react": "^18.0.27",
"@types/react-dom": "^18.0.10",
"framer-motion": "^8.5.5",
"next": "^13.2.4",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-scripts": "5.0.1",
"typescript": "^4.9.4",
"uuid": "^8.3.2",
"web-vitals": "^2.1.4"
},
"scripts": {
"start": "react-scripts start",
"build": "react-scripts build",
"test": "react-scripts test",
"eject": "react-scripts eject"
"dev": "next dev",
"build": "next build",
"start": "next start"
},
"eslintConfig": {
"extends": [
Expand All @@ -47,7 +45,6 @@
]
},
"devDependencies": {
"@types/uuid": "^8.3.1",
"sass": "^1.57.1",
"typescript-plugin-css-modules": "^3.4.0"
}
Expand Down
Loading

0 comments on commit 44a514e

Please sign in to comment.