From 4730ec579f902b48bf8ea800481c588d34351827 Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Tue, 1 Feb 2022 10:15:35 -0300 Subject: [PATCH 01/15] =?UTF-8?q?Corrige=20bugs=20em=20inicializa=C3=A7?= =?UTF-8?q?=C3=A3o=20de=20banco=20de=20dados?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/initialize_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/initialize_db.py b/api/initialize_db.py index 3e1402c..b165f5d 100644 --- a/api/initialize_db.py +++ b/api/initialize_db.py @@ -14,7 +14,7 @@ setup_logging, ) -from models.sql_declarative import Base, Report, Status, Alert +from api.models.sql_declarative import Base, Report, Status, Alert def usage(argv): @@ -43,7 +43,7 @@ def main(argv=sys.argv): Base.metadata.create_all(engine) DBSession.configure(bind=engine) - file_counter_report = os.path.join(os.getcwd(), 'api/static/counter_report.json') + file_counter_report = os.path.join(os.getcwd(), 'api/static/json/counter_report.json') with transaction.manager: with open(file_counter_report) as f: application_url = settings.get('application.url') From 8817b75d8e063cc6fe3a5687f78ac82295451dbb Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Tue, 1 Feb 2022 10:15:50 -0300 Subject: [PATCH 02/15] =?UTF-8?q?Corrige=20imports=20em=20arquivo=20de=20d?= =?UTF-8?q?eclara=C3=A7=C3=A3o=20de=20modelos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/models/sql_declarative.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/models/sql_declarative.py b/api/models/sql_declarative.py index d690317..27e7b0c 100644 --- a/api/models/sql_declarative.py +++ b/api/models/sql_declarative.py @@ -1,6 +1,7 @@ -from sqlalchemy import Column, ForeignKey, BOOLEAN, UniqueConstraint, Date -from sqlalchemy.dialects.mysql import INTEGER, VARCHAR, DATETIME +from sqlalchemy import Column, ForeignKey, Index, UniqueConstraint +from sqlalchemy.dialects.mysql import BIGINT, BOOLEAN, DATE, DATETIME, DECIMAL, INTEGER, VARCHAR from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship Base = declarative_base() From 829c46334503d1378cf3c6bdaaa6f752b6104798 Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Tue, 1 Feb 2022 10:16:34 -0300 Subject: [PATCH 03/15] =?UTF-8?q?Adiciona=20modelos=20de=20toda=20a=20apli?= =?UTF-8?q?ca=C3=A7=C3=A3o=20de=20contagem=20(centraliza=20todos=20na=20ap?= =?UTF-8?q?p=20sushi)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/models/sql_declarative.py | 200 ++++++++++++++++++++++++++++++++++ 1 file changed, 200 insertions(+) diff --git a/api/models/sql_declarative.py b/api/models/sql_declarative.py index 27e7b0c..0d96ddd 100644 --- a/api/models/sql_declarative.py +++ b/api/models/sql_declarative.py @@ -7,6 +7,206 @@ Base = declarative_base() +class LogFile(Base): + __tablename__ = 'control_log_file' + + id = Column(INTEGER(unsigned=True), primary_key=True, autoincrement=True) + + full_path = Column(VARCHAR(255), nullable=False, unique=True) + name = Column(VARCHAR(255), nullable=False) + created_at = Column(DATETIME, nullable=False) + size = Column(BIGINT, nullable=False) + server = Column(VARCHAR(255), nullable=False) + date = Column(DATE, nullable=False, index=True) + status = Column(INTEGER, default=0) + collection = Column(VARCHAR(3), nullable=False) + + +class LogFileSummary(Base): + __tablename__ = 'control_log_file_summary' + + id = Column(INTEGER(unsigned=True), primary_key=True, autoincrement=True) + idlogfile = Column(INTEGER(unsigned=True), ForeignKey('control_log_file.id', name='idlogfile')) + + total_lines = Column(INTEGER, nullable=False) + lines_parsed = Column(INTEGER) + + total_imported_lines = Column(INTEGER) + total_ignored_lines = Column(INTEGER) + sum_imported_ignored_lines = Column(INTEGER) + + ignored_lines_filtered = Column(INTEGER) + ignored_lines_http_errors = Column(INTEGER) + ignored_lines_http_redirects = Column(INTEGER) + ignored_lines_invalid = Column(INTEGER) + ignored_lines_bots = Column(INTEGER) + ignored_lines_static_resources = Column(INTEGER) + + total_time = Column(INTEGER) + status = Column(INTEGER) + + +class Journal(Base): + __tablename__ = 'counter_journal' + + __table_args__ = (UniqueConstraint('print_issn', 'online_issn', 'pid_issn', name='uni_issn'),) + __table_args__ += (Index('idx_print_issn', 'print_issn'),) + __table_args__ += (Index('idx_online_issn', 'online_issn'),) + __table_args__ += (Index('idx_pid_issn', 'pid_issn'),) + + id = Column(INTEGER(unsigned=True), primary_key=True, autoincrement=True) + print_issn = Column(VARCHAR(9), nullable=False) + online_issn = Column(VARCHAR(9), nullable=False) + pid_issn = Column(VARCHAR(9), nullable=False) + + +class JournalCollection(Base): + __tablename__ = 'counter_journal_collection' + __table_args__ = (UniqueConstraint('collection', 'idjournal_jc', name='uni_col_jou'),) + + id = Column(INTEGER(unsigned=True), primary_key=True, autoincrement=True) + collection = Column(VARCHAR(3), nullable=False) + title = Column(VARCHAR(255), nullable=False) + uri = Column(VARCHAR(255)) + publisher_name = Column(VARCHAR(255)) + idjournal_jc = Column(INTEGER(unsigned=True), ForeignKey('counter_journal.id', name='idjournal_jc')) + + +class ArticleLanguage(Base): + __tablename__ = 'counter_article_language' + __table_args__ = (UniqueConstraint('language', name='uni_lang'),) + + id = Column(INTEGER(unsigned=True), primary_key=True, autoincrement=True) + language = Column(VARCHAR(10), nullable=False) + + +class ArticleFormat(Base): + __tablename__ = 'counter_article_format' + __table_args__ = (UniqueConstraint('format', name='uni_fmt'),) + + id = Column(INTEGER(unsigned=True), primary_key=True, autoincrement=True) + format = Column(VARCHAR(10), nullable=False) + + +class Article(Base): + __tablename__ = 'counter_article' + __table_args__ = (UniqueConstraint('collection', 'pid', name='uni_col_pid'),) + __table_args__ += (Index('idx_col_pid_jou_yop', 'collection', 'pid', 'idjournal_a', 'yop'),) + + id = Column(INTEGER(unsigned=True), primary_key=True, autoincrement=True) + collection = Column(VARCHAR(3), nullable=False) + pid = Column(VARCHAR(128), nullable=False) + yop = Column(INTEGER(4)) + + idjournal_a = Column(INTEGER(unsigned=True), ForeignKey('counter_journal.id', name='idjournal_a')) + journal = relationship(Journal) + + +class ArticleMetric(Base): + __tablename__ = 'counter_article_metric' + __table_args__ = (UniqueConstraint('year_month_day', 'idarticle', 'idformat', 'idlanguage', 'idlocalization', name='uni_date_art_all'),) + __table_args__ += (Index('idx_date_art_all', 'year_month_day', 'idarticle', 'idformat', 'idlanguage', 'idlocalization'),) + + id = Column(INTEGER(unsigned=True), primary_key=True, autoincrement=True) + + idarticle = Column(INTEGER(unsigned=True), ForeignKey('counter_article.id', name='idarticle')) + article = relationship(Article) + + idformat = Column(INTEGER(unsigned=True), ForeignKey('counter_article_format.id', name='idformat')) + idlanguage = Column(INTEGER(unsigned=True), ForeignKey('counter_article_language.id', name='idlanguage')) + idlocalization = Column(INTEGER(unsigned=True), ForeignKey('counter_localization.id', name='idlocalization')) + + year_month_day = Column(DATE, nullable=False) + total_item_requests = Column(INTEGER, nullable=False) + total_item_investigations = Column(INTEGER, nullable=False) + unique_item_requests = Column(INTEGER, nullable=False) + unique_item_investigations = Column(INTEGER, nullable=False) + + +class Localization(Base): + __tablename__ = 'counter_localization' + __table_args__ = (UniqueConstraint('latitude', 'longitude', name='uni_loc'),) + __table_args__ += (Index('idx_loc', 'latitude', 'longitude'),) + + id = Column(INTEGER(unsigned=True), primary_key=True, autoincrement=True) + latitude = Column(DECIMAL(9, 6)) + longitude = Column(DECIMAL(9, 6)) + + +class JournalMetric(Base): + __tablename__ = 'counter_journal_metric' + __table_args__ = (UniqueConstraint('year_month_day', 'collection', 'idformat_cjm', 'idlanguage_cjm', 'idjournal_cjm', 'yop', name='uni_col_date_all_cjm'),) + __table_args__ += (Index('idx_col_date_all_cjm', 'collection', 'year_month_day', 'idformat_cjm', 'idlanguage_cjm', 'yop', 'idjournal_cjm'),) + + id = Column(INTEGER(unsigned=True), primary_key=True, autoincrement=True) + + collection = Column(VARCHAR(3), nullable=False) + + idformat_cjm = Column(INTEGER(unsigned=True), ForeignKey('counter_article_format.id', name='idformat_cjm')) + idlanguage_cjm = Column(INTEGER(unsigned=True), ForeignKey('counter_article_language.id', name='idlanguage_cjm')) + idjournal_cjm = Column(INTEGER(unsigned=True), ForeignKey('counter_journal.id', name='idjournal_cjm')) + + yop = Column(INTEGER(4)) + year_month_day = Column(DATE, nullable=False) + + total_item_requests = Column(INTEGER, nullable=False) + total_item_investigations = Column(INTEGER, nullable=False) + unique_item_requests = Column(INTEGER, nullable=False) + unique_item_investigations = Column(INTEGER, nullable=False) + + +class SushiJournalYOPMetric(Base): + __tablename__ = 'sushi_journal_yop_metric' + __table_args__ = (UniqueConstraint('year_month_day', 'collection', 'yop', 'idjournal_sjym', name='uni_col_date_yop_jou_sjym'),) + __table_args__ += (Index('idx_col_date_yop_sjym', 'collection', 'year_month_day', 'yop', 'idjournal_sjym'),) + + id = Column(INTEGER(unsigned=True), primary_key=True, autoincrement=True) + + collection = Column(VARCHAR(3), nullable=False) + + idjournal_sjym = Column(INTEGER(unsigned=True), ForeignKey('counter_journal.id', name='idjournal_sjym')) + yop = Column(INTEGER(4)) + year_month_day = Column(DATE, nullable=False) + + total_item_requests = Column(INTEGER, nullable=False) + total_item_investigations = Column(INTEGER, nullable=False) + unique_item_requests = Column(INTEGER, nullable=False) + unique_item_investigations = Column(INTEGER, nullable=False) + + +class SushiJournalMetric(Base): + __tablename__ = 'sushi_journal_metric' + __table_args__ = (UniqueConstraint('year_month_day', 'collection', 'idjournal_sjm', name='uni_col_date_jou_sjm'),) + __table_args__ += (Index('idx_col_date_sjm', 'collection', 'year_month_day', 'idjournal_sjm'),) + + id = Column(INTEGER(unsigned=True), primary_key=True, autoincrement=True) + + collection = Column(VARCHAR(3), nullable=False) + + idjournal_sjm = Column(INTEGER(unsigned=True), ForeignKey('counter_journal.id', name='idjournal_sjm')) + year_month_day = Column(DATE, nullable=False) + + total_item_requests = Column(INTEGER, nullable=False) + total_item_investigations = Column(INTEGER, nullable=False) + unique_item_requests = Column(INTEGER, nullable=False) + unique_item_investigations = Column(INTEGER, nullable=False) + + +class SushiArticleMetric(Base): + __tablename__ = 'sushi_article_metric' + __table_args__ = (UniqueConstraint('year_month_day', 'idarticle_sam', name='uni_date_art_sam'),) + __table_args__ += (Index('idx_date_sam', 'year_month_day', 'idarticle_sam'),) + + id = Column(INTEGER(unsigned=True), primary_key=True, autoincrement=True) + + idarticle_sam = Column(INTEGER(unsigned=True), ForeignKey('counter_article.id', name='idarticle_sam')) + year_month_day = Column(DATE, nullable=False) + + total_item_requests = Column(INTEGER, nullable=False) + total_item_investigations = Column(INTEGER, nullable=False) + unique_item_requests = Column(INTEGER, nullable=False) + unique_item_investigations = Column(INTEGER, nullable=False) + class Institution(Base): __tablename__ = 'counter_institution' From a11291468bbfde3026573cff5622978defa2cf2d Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Tue, 1 Feb 2022 10:17:01 -0300 Subject: [PATCH 04/15] =?UTF-8?q?Padroniza=20importa=C3=A7=C3=A3o=20de=20c?= =?UTF-8?q?oluna=20DATE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/models/sql_declarative.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/models/sql_declarative.py b/api/models/sql_declarative.py index 0d96ddd..2465ee5 100644 --- a/api/models/sql_declarative.py +++ b/api/models/sql_declarative.py @@ -262,7 +262,7 @@ class DateStatus(Base): id = Column(INTEGER(unsigned=True), primary_key=True, autoincrement=True) - date = Column(Date, nullable=False, index=True) + date = Column(DATE, nullable=False, index=True) collection = Column(VARCHAR(3), nullable=False) status = Column(INTEGER, default=0) @@ -277,7 +277,7 @@ class AggrStatus(Base): __tablename__ = 'aggr_status' __table_args__ = (UniqueConstraint('collection', 'date', name='uni_collection_date'), ) collection = Column(VARCHAR(3), nullable=False, primary_key=True) - date = Column(Date, nullable=False, primary_key=True) + date = Column(DATE, nullable=False, primary_key=True) status_aggr_article_language_year_month_metric = Column(BOOLEAN, default=False) status_aggr_journal_language_year_month_metric = Column(BOOLEAN, default=False) From fdfe0d02d13501605b1523cf75bc5594b0402074 Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Tue, 1 Feb 2022 10:17:25 -0300 Subject: [PATCH 05/15] =?UTF-8?q?Adiciona=20coluna=20para=20indicar=20stat?= =?UTF-8?q?us=20de=20agrega=C3=A7=C3=A3o=20por=20geolocaliza=C3=A7=C3=A3o?= =?UTF-8?q?=20e=20peri=C3=B3dico?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/models/sql_declarative.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/models/sql_declarative.py b/api/models/sql_declarative.py index 2465ee5..96cb8a7 100644 --- a/api/models/sql_declarative.py +++ b/api/models/sql_declarative.py @@ -281,3 +281,5 @@ class AggrStatus(Base): status_aggr_article_language_year_month_metric = Column(BOOLEAN, default=False) status_aggr_journal_language_year_month_metric = Column(BOOLEAN, default=False) + status_aggr_journal_geolocation_year_month_metric = Column(BOOLEAN, default=False) + From e983d820bd2153cc0ea83fbd44966050a5959449 Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Tue, 1 Feb 2022 10:17:49 -0300 Subject: [PATCH 06/15] =?UTF-8?q?Adiciona=20na=20declara=C3=A7=C3=A3o=20de?= =?UTF-8?q?=20modelos=20classes=20j=C3=A1=20existentes=20de=20agrega=C3=A7?= =?UTF-8?q?=C3=A3o=20(de=20outro=20repo)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/models/sql_declarative.py | 36 +++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/api/models/sql_declarative.py b/api/models/sql_declarative.py index 96cb8a7..a9b1818 100644 --- a/api/models/sql_declarative.py +++ b/api/models/sql_declarative.py @@ -283,3 +283,39 @@ class AggrStatus(Base): status_aggr_journal_language_year_month_metric = Column(BOOLEAN, default=False) status_aggr_journal_geolocation_year_month_metric = Column(BOOLEAN, default=False) + +class AggrArticleLanguageYearMonthMetric(Base): + __tablename__ = 'aggr_article_language_year_month_metric' + __table_args__ = (UniqueConstraint('year_month', 'article_id', 'language_id', name='uni_art_lan_aalymm'),) + __table_args__ += (Index('idx_ym_id', 'year_month', 'article_id'),) + + id = Column(INTEGER(unsigned=True), primary_key=True, autoincrement=True) + + collection = Column(VARCHAR(3), nullable=False, primary_key=True) + article_id = Column(INTEGER(unsigned=True), ForeignKey('counter_article.id', name='idarticle_aalymm')) + language_id = Column(INTEGER(unsigned=True), ForeignKey('counter_article_language.id', name='idlanguage_aalymm')) + year_month = Column(VARCHAR(7), nullable=False) + + total_item_requests = Column(INTEGER, nullable=False) + total_item_investigations = Column(INTEGER, nullable=False) + unique_item_requests = Column(INTEGER, nullable=False) + unique_item_investigations = Column(INTEGER, nullable=False) + + +class AggrJournalLanguageYearMonthMetric(Base): + __tablename__ = 'aggr_journal_language_year_month_metric' + __table_args__ = (UniqueConstraint('year_month', 'journal_id', 'language_id', name='uni_jou_lan_ajlymm'),) + __table_args__ += (Index('idx_ym_id', 'year_month', 'journal_id'),) + + id = Column(INTEGER(unsigned=True), primary_key=True, autoincrement=True) + + collection = Column(VARCHAR(3), nullable=False, primary_key=True) + journal_id = Column(INTEGER(unsigned=True), ForeignKey('counter_journal.id', name='idjournal_ajlymm')) + language_id = Column(INTEGER(unsigned=True), ForeignKey('counter_article_language.id', name='idlanguage_ajlymm')) + year_month = Column(VARCHAR(7), nullable=False) + + total_item_requests = Column(INTEGER, nullable=False) + total_item_investigations = Column(INTEGER, nullable=False) + unique_item_requests = Column(INTEGER, nullable=False) + unique_item_investigations = Column(INTEGER, nullable=False) + From 8decd22f40840700a5b3eb0d365fc816441f3324 Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Tue, 1 Feb 2022 10:18:08 -0300 Subject: [PATCH 07/15] =?UTF-8?q?Adiciona=20modelo=20de=20agrega=C3=A7?= =?UTF-8?q?=C3=A3o=20para=20peri=C3=B3dico=20e=20geolocaliza=C3=A7=C3=A3o?= =?UTF-8?q?=20(c=C3=B3digo=20de=20pa=C3=ADs)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/models/sql_declarative.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/api/models/sql_declarative.py b/api/models/sql_declarative.py index a9b1818..c3fb1d1 100644 --- a/api/models/sql_declarative.py +++ b/api/models/sql_declarative.py @@ -319,3 +319,20 @@ class AggrJournalLanguageYearMonthMetric(Base): unique_item_requests = Column(INTEGER, nullable=False) unique_item_investigations = Column(INTEGER, nullable=False) + +class AggrJournalGeolocationYearMonthMetric(Base): + __tablename__ = 'aggr_journal_geolocation_year_month_metric' + __table_args__ = (UniqueConstraint('year_month', 'journal_id', 'country_code', name='uni_jou_geo_ajlymm'),) + __table_args__ += (Index('idx_ym_id', 'year_month', 'journal_id'),) + + id = Column(INTEGER(unsigned=True), primary_key=True, autoincrement=True) + + collection = Column(VARCHAR(3), nullable=False, primary_key=True) + journal_id = Column(INTEGER(unsigned=True), ForeignKey('counter_journal.id', name='idjournal_ajlymm')) + country_code = Column(VARCHAR(4), nullable=False) + year_month = Column(VARCHAR(7), nullable=False) + + total_item_requests = Column(INTEGER, nullable=False) + total_item_investigations = Column(INTEGER, nullable=False) + unique_item_requests = Column(INTEGER, nullable=False) + unique_item_investigations = Column(INTEGER, nullable=False) From 4deb6951bb91626dc0aed891a9f98a0a2bcd5acc Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Tue, 1 Feb 2022 10:18:35 -0300 Subject: [PATCH 08/15] =?UTF-8?q?Adiciona=20instru=C3=A7=C3=B5es=20de=20cr?= =?UTF-8?q?ia=C3=A7=C3=A3o=20de=20quatro=20procedures=20para=20relat=C3=B3?= =?UTF-8?q?rios=20gr=5Fj1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/static/sql/procedures_v2_gr_j1.sql | 146 +++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 api/static/sql/procedures_v2_gr_j1.sql diff --git a/api/static/sql/procedures_v2_gr_j1.sql b/api/static/sql/procedures_v2_gr_j1.sql new file mode 100644 index 0000000..c6fdd83 --- /dev/null +++ b/api/static/sql/procedures_v2_gr_j1.sql @@ -0,0 +1,146 @@ +DELIMITER $$ +CREATE PROCEDURE V2_GR_J1_MONTHLY(IN beginDate varchar(7), IN endDate varchar(7), IN collection varchar(3), collection_extra varchar(3)) +BEGIN + SELECT + ajgymm.journal_id as journalID, + cj.print_issn as printISSN, + cj.online_issn as onlineISSN, + cjc.title as title, + cjc.uri as uri, + cjc.publisher_name as publisherName, + ajgymm.country_code as countryCode, + ajgymm.`year_month` as yearMonth, + sum(ajgymm.total_item_requests) as totalItemRequests, + sum(ajgymm.unique_item_requests) as uniqueItemRequests + FROM + aggr_journal_geolocation_year_month_metric ajgymm + JOIN + counter_journal cj ON cj.id = ajgymm.journal_id + JOIN + counter_journal_collection cjc ON cjc.idjournal_jc = cj.id + WHERE + ajgymm.collection in (collection, collection_extra) AND + ajgymm.collection = cjc.collection AND + `year_month` BETWEEN beginDate AND endDate + GROUP BY + journalID, + countryCode, + yearMonth + ORDER BY + journalID, + countryCode, + yearMonth + ; +END $$ +DELIMITER ; + +DELIMITER $$ +CREATE PROCEDURE V2_GR_J1_TOTALS(IN beginDate varchar(7), IN endDate varchar(7), IN collection varchar(3), collection_extra varchar(3)) +BEGIN + SELECT + ajgymm.journal_id as journalID, + cj.print_issn as printISSN, + cj.online_issn as onlineISSN, + cjc.title as title, + cjc.uri as uri, + cjc.publisher_name as publisherName, + ajgymm.country_code as countryCode, + MIN(ajgymm.`year_month`) AS beginDate, + MAX(ajgymm.`year_month`) AS endDate, + sum(ajgymm.total_item_requests) as totalItemRequests, + sum(ajgymm.unique_item_requests) as uniqueItemRequests + FROM + aggr_journal_geolocation_year_month_metric ajgymm + JOIN + counter_journal cj ON cj.id = ajgymm.journal_id + JOIN + counter_journal_collection cjc ON cjc.idjournal_jc = cj.id + WHERE + ajgymm.collection in (collection, collection_extra) AND + ajgymm.collection = cjc.collection AND + `year_month` BETWEEN beginDate AND endDate + GROUP BY + journalID, + countryCode + ORDER BY + journalID, + countryCode + ; +END $$ +DELIMITER ; + +DELIMITER $$ +CREATE PROCEDURE V2_GR_J1_JOURNAL_MONTHLY(IN beginDate varchar(7), IN endDate varchar(7), IN issn varchar(9), IN collection varchar(3), collection_extra varchar(3)) +BEGIN + SELECT + ajgymm.journal_id as journalID, + cj.print_issn as printISSN, + cj.online_issn as onlineISSN, + cjc.title as title, + cjc.uri as uri, + cjc.publisher_name as publisherName, + ajgymm.country_code as countryCode, + ajgymm.`year_month` as yearMonth, + sum(ajgymm.total_item_requests) as totalItemRequests, + sum(ajgymm.unique_item_requests) as uniqueItemRequests + FROM + aggr_journal_geolocation_year_month_metric ajgymm + JOIN + counter_journal cj ON cj.id = ajgymm.journal_id + JOIN + counter_journal_collection cjc ON cjc.idjournal_jc = cj.id + WHERE + ajgymm.collection in (collection, collection_extra) AND + ajgymm.collection = cjc.collection AND + `year_month` BETWEEN beginDate AND endDate AND + (issn = cj.online_issn OR issn = cj.print_issn OR issn = cj.pid_issn) AND + (online_issn <> '' OR print_issn <> '') + GROUP BY + journalID, + countryCode, + yearMonth + ORDER BY + journalID, + countryCode, + yearMonth + ; +END $$ +DELIMITER ; + + +DELIMITER $$ +CREATE PROCEDURE V2_GR_J1_JOURNAL_TOTALS(IN beginDate varchar(7), IN endDate varchar(7), IN issn varchar(9), IN collection varchar(3), collection_extra varchar(3)) +BEGIN + SELECT + ajgymm.journal_id as journalID, + cj.print_issn as printISSN, + cj.online_issn as onlineISSN, + cjc.title as title, + cjc.uri as uri, + cjc.publisher_name as publisherName, + ajgymm.country_code as countryCode, + MIN(ajgymm.`year_month`) AS beginDate, + MAX(ajgymm.`year_month`) AS endDate, + sum(ajgymm.total_item_requests) as totalItemRequests, + sum(ajgymm.unique_item_requests) as uniqueItemRequests + FROM + aggr_journal_geolocation_year_month_metric ajgymm + JOIN + counter_journal cj ON cj.id = ajgymm.journal_id + JOIN + counter_journal_collection cjc ON cjc.idjournal_jc = cj.id + WHERE + ajgymm.collection in (collection, collection_extra) AND + ajgymm.collection = cjc.collection AND + `year_month` BETWEEN beginDate AND endDate AND + (issn = cj.online_issn OR issn = cj.print_issn OR issn = cj.pid_issn) AND + (online_issn <> '' OR print_issn <> '') + GROUP BY + journalID, + countryCode + ORDER BY + journalID, + countryCode + ; +END $$ +DELIMITER ; From 0f3138f203601e4df02bc4dafbb800aaf58c7e12 Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Tue, 1 Feb 2022 10:19:01 -0300 Subject: [PATCH 09/15] Adiciona enums para chamada de procedures de gr_j1 --- api/values.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/values.py b/api/values.py index 39f315c..e57a830 100644 --- a/api/values.py +++ b/api/values.py @@ -71,12 +71,18 @@ DB_CALL_V2_LR_J1_JOURNAL_TOTALS = 'CALL V2_LR_J1_JOURNAL_TOTALS("%s", "%s", "%s", "%s", "%s")' DB_CALL_V2_LR_J1_JOURNAL_MONTHLY = 'CALL V2_LR_J1_JOURNAL_MONTHLY("%s", "%s", "%s", "%s", "%s")' +DB_CALL_V2_GR_J1_JOURNAL_TOTALS = 'CALL V2_GR_J1_JOURNAL_TOTALS("%s", "%s", "%s", "%s", "%s")' +DB_CALL_V2_GR_J1_JOURNAL_MONTHLY = 'CALL V2_GR_J1_JOURNAL_MONTHLY("%s", "%s", "%s", "%s", "%s")' + DB_CALL_V2_TR_J1_TOTALS = 'CALL V2_TR_J1_TOTALS("%s", "%s", "%s")' DB_CALL_V2_TR_J1_MONTHLY = 'CALL V2_TR_J1_MONTHLY("%s", "%s", "%s")' DB_CALL_V2_LR_J1_TOTALS = 'CALL V2_LR_J1_TOTALS("%s", "%s", "%s", "%s")' DB_CALL_V2_LR_J1_MONTHLY = 'CALL V2_LR_J1_MONTHLY("%s", "%s", "%s", "%s")' +DB_CALL_V2_GR_J1_TOTALS = 'CALL V2_GR_J1_TOTALS("%s", "%s", "%s", "%s")' +DB_CALL_V2_GR_J1_MONTHLY = 'CALL V2_GR_J1_MONTHLY("%s", "%s", "%s", "%s")' + DB_CALL_V2_CR_J1_TOTALS = 'CALL V2_CR_J1_TOTALS("%s", "%s", "%s", "%s")' DB_CALL_V2_CR_J1_MONTHLY = 'CALL V2_CR_J1_MONTHLY("%s", "%s", "%s", "%s")' From 480411bfec6a92be828991293c077dfe9785702d Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Tue, 1 Feb 2022 10:19:39 -0300 Subject: [PATCH 10/15] =?UTF-8?q?Atualiza=20novas=20procedures=20em=20dict?= =?UTF-8?q?=20que=20mapeia=20granularidade=20a=20relat=C3=B3rio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/values.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/values.py b/api/values.py index e57a830..8e81b6d 100644 --- a/api/values.py +++ b/api/values.py @@ -92,6 +92,7 @@ 'tr_j1': 'status_sushi_journal_metric', 'tr_j4': 'status_sushi_journal_yop_metric', 'lr_j1': 'status_aggr_journal_language_year_month_metric', + 'gr_j1': 'status_aggr_journal_geolocation_year_month_metric', } GRANULARITY_MODE_REPORT_TO_PROCEDURE_AND_PARAMETERS = { @@ -134,23 +135,27 @@ V2_GRANULARITY_MODE_REPORT_TO_PROCEDURE_AND_PARAMETERS = { 'totals': { 'issn': { + 'gr_j1': (DB_CALL_V2_GR_J1_JOURNAL_TOTALS, ['begin_date', 'end_date', 'issn', 'collection', 'collection_extra']), 'lr_j1': (DB_CALL_V2_LR_J1_JOURNAL_TOTALS, ['begin_date', 'end_date', 'issn', 'collection', 'collection_extra']), 'tr_j1': (DB_CALL_V2_TR_J1_JOURNAL_TOTALS, ['begin_date', 'end_date', 'issn', 'collection']), }, 'global': { 'cr_j1': (DB_CALL_V2_CR_J1_TOTALS, ['begin_date', 'end_date', 'collection', 'collection_extra']), + 'gr_j1': (DB_CALL_V2_GR_J1_TOTALS, ['begin_date', 'end_date', 'collection', 'collection_extra']), 'lr_j1': (DB_CALL_V2_LR_J1_TOTALS, ['begin_date', 'end_date', 'collection', 'collection_extra']), 'tr_j1': (DB_CALL_V2_TR_J1_TOTALS, ['begin_date', 'end_date', 'collection']), } }, 'monthly': { 'issn': { + 'gr_j1': (DB_CALL_V2_GR_J1_JOURNAL_MONTHLY, ['begin_date', 'end_date', 'issn', 'collection', 'collection_extra']), 'lr_j1': (DB_CALL_V2_LR_J1_JOURNAL_MONTHLY, ['begin_date', 'end_date', 'issn', 'collection', 'collection_extra']), 'tr_j1': (DB_CALL_V2_TR_J1_JOURNAL_MONTHLY, ['begin_date', 'end_date', 'issn', 'collection']), }, 'global': { 'cr_j1': (DB_CALL_V2_CR_J1_MONTHLY, ['begin_date', 'end_date', 'collection', 'collection_extra']), + 'gr_j1': (DB_CALL_V2_GR_J1_MONTHLY, ['begin_date', 'end_date', 'collection', 'collection_extra']), 'lr_j1': (DB_CALL_V2_LR_J1_MONTHLY, ['begin_date', 'end_date', 'collection', 'collection_extra']), 'tr_j1': (DB_CALL_V2_TR_J1_MONTHLY, ['begin_date', 'end_date', 'collection']), } From aa555e32602751c07d33d708ea5098bb5525406a Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Tue, 1 Feb 2022 10:20:04 -0300 Subject: [PATCH 11/15] =?UTF-8?q?Adiciona=20lista=20de=20linhas=20esperada?= =?UTF-8?q?s=20para=20relat=C3=B3rios=20TSV=20do=20tipo=20gr=5Fj1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/values.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/api/values.py b/api/values.py index 8e81b6d..877cb22 100644 --- a/api/values.py +++ b/api/values.py @@ -219,6 +219,20 @@ 'Metric_Type', ] +TSV_REPORT_GR_J1_ROWS = [ + 'Title', + 'Publisher', + 'Publisher_ID', + 'Platform', + 'DOI', + 'Proprietary_ID', + 'Print_ISSN', + 'Online_ISSN', + 'URI', + 'Access_Country_Code', + 'Metric_Type', +] + TSV_REPORT_IR_ROWS = [ 'Item', 'Publisher', From f1614035819ee08d895a9d03b4ec5d2c31d872ef Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Tue, 1 Feb 2022 10:20:28 -0300 Subject: [PATCH 12/15] =?UTF-8?q?Adiciona=20relat=C3=B3rio=20gr=5Fj1=20em?= =?UTF-8?q?=20utilit=C3=A1rio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/libs/db.py | 2 +- api/utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/libs/db.py b/api/libs/db.py index e7c09f3..6d5df16 100644 --- a/api/libs/db.py +++ b/api/libs/db.py @@ -40,7 +40,7 @@ def get_dates_not_ready(begin_date, end_date, collection, report_id): else: return [] - elif report_id in ('lr_j1',): + elif report_id in ('gr_j1', 'lr_j1',): if status_column: not_read_dates = DBSession.query(AggrStatus).filter(and_(AggrStatus.collection == collection, getattr(AggrStatus, status_column) == 0, diff --git a/api/utils.py b/api/utils.py index 0fd8e46..38e9892 100644 --- a/api/utils.py +++ b/api/utils.py @@ -108,7 +108,7 @@ def format_error_messages(exceptions: list): def set_collection_extra(report_id, attrs): - if report_id in ('cr_j1', 'lr_j1'): + if report_id in ('cr_j1', 'gr_j1', 'lr_j1',): if attrs['collection'] == 'scl': attrs.update({'collection_extra': 'nbr'}) @@ -127,7 +127,7 @@ def wrapper_call_report(report_id, params): else: procedure_name, params_names = values.GRANULARITY_MODE_REPORT_TO_PROCEDURE_AND_PARAMETERS.get(granularity, {}).get(mode, {}).get(report_id, ('', [])) - if report_id in ('lr_j1',): + if report_id in ('gr_j1', 'lr_j1',): params['begin_date'] = cleaner.handle_str_date(params['begin_date'], year_month_only=True) params['end_date'] = cleaner.handle_str_date(params['end_date'], year_month_only=True) From 8777400f6fa5afe4727e9f6200029f22cc286a04 Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Tue, 1 Feb 2022 10:21:05 -0300 Subject: [PATCH 13/15] =?UTF-8?q?Adiciona=20detec=C3=A7=C3=A3o=20de=20rela?= =?UTF-8?q?t=C3=B3rio=20gr=5Fj1=20em=20wrappers=20de=20json=20e=20tsv?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/adapter.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/adapter.py b/api/adapter.py index 950f629..caffcd3 100644 --- a/api/adapter.py +++ b/api/adapter.py @@ -26,6 +26,9 @@ def json_report_wrapper(report_id, result_query, params, exceptions): if report_id == 'lr_j1': return _json_lr_j1(result_query, params, exceptions) + if report_id == 'gr_j1': + return _json_gr_j1(result_query, params, exceptions) + def mount_json_for_reports(result_query): json_results = [] @@ -490,6 +493,9 @@ def tsv_report_wrapper(request, report_id, result_query, params, exceptions): if report_id == 'lr_j1': return _tsv_report_lr_j1(result_query, params, exceptions) + if report_id == 'gr_j1': + return _tsv_report_gr_j1(result_query, params, exceptions) + def _tsv_header(params, exceptions, data_type='Journal'): headers = {} From f7f54b99b2bd81f1e2fc660deeba79c5e80837a6 Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Tue, 1 Feb 2022 10:21:28 -0300 Subject: [PATCH 14/15] =?UTF-8?q?Adiciona=20m=C3=A9todo=20adaptador=20de?= =?UTF-8?q?=20relat=C3=B3rio=20TSV=20gr=5Fj1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/adapter.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/api/adapter.py b/api/adapter.py index caffcd3..a84344a 100644 --- a/api/adapter.py +++ b/api/adapter.py @@ -808,3 +808,67 @@ def _tsv_report_lr_j1(result_query, params, exceptions): output['rows'].append(line) return output + + +def _tsv_report_gr_j1(result_query, params, exceptions): + begin_date, bd_discard = cleaner.get_start_and_last_days(params['begin_date']) + ed_discard, end_date = cleaner.get_start_and_last_days(params['end_date']) + + params['begin_date'] = begin_date + params['end_date'] = end_date + + result = {'headers': _tsv_header(params, exceptions)} + + journal2values = {} + yms = ['Reporting_Period_Total'] + + for ri in result_query: + journal_key = (ri.journalID, ri.title, ri.publisherName, ri.printISSN, ri.onlineISSN, ri.uri, ri.countryCode) + + tir = getattr(ri, 'totalItemRequests') + uir = getattr(ri, 'uniqueItemRequests') + + if journal_key not in journal2values: + journal2values[journal_key] = {'Reporting_Period_Total': (0, 0)} + + if params['granularity'] == 'monthly': + year_month = cleaner.handle_str_date(ri.yearMonth, str_format=False).strftime('%b-%Y') + if year_month not in yms: + yms.append(year_month) + + if year_month not in journal2values[journal_key]: + journal2values[journal_key][year_month] = 0 + + journal2values[journal_key][year_month] = (tir, uir) + + journal2values[journal_key]['Reporting_Period_Total'] = tuple(map(sum, zip(journal2values[journal_key]['Reporting_Period_Total'], (tir, uir)))) + + output = {'rows': []} + for k in values.TSV_REPORT_DEFAULT_HEADERS: + output['rows'].append([k, result['headers'][k]]) + + output['rows'].append(values.TSV_REPORT_GR_J1_ROWS + yms) + + for i in journal2values: + for j, metric_name in enumerate(['Total_Item_Requests', 'Unique_Item_Requests']): + line = [ + i[1], + i[2], + '', + 'SciELO SUSHI API', + '', + '', + i[3], + i[4], + i[5], + i[6], + metric_name + ] + + for ym in yms: + ym_v = str(journal2values[i].get(ym, (0, 0))[j]) + line.append(ym_v) + + output['rows'].append(line) + + return output From a7d5e17c1445566b556801dc11db9ba9171cf412 Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Tue, 1 Feb 2022 10:21:41 -0300 Subject: [PATCH 15/15] =?UTF-8?q?Adiciona=20m=C3=A9todo=20adaptador=20de?= =?UTF-8?q?=20relat=C3=B3rio=20JSON=20gr=5Fj1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/adapter.py | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/api/adapter.py b/api/adapter.py index a84344a..10c02f7 100644 --- a/api/adapter.py +++ b/api/adapter.py @@ -474,6 +474,95 @@ def _json_cr_j1(result_query_reports_cr_j1, params, exceptions): return json_results +def _json_gr_j1(result_query_reports_gr_j1, params, exceptions): + begin_date, ed_discard = cleaner.get_start_and_last_days(params.get('begin_date', '')) + bd_discard, end_date = cleaner.get_start_and_last_days(params.get('end_date', '')) + + json_results = { + "Report_Header": { + "Created": datetime.now().isoformat(), + "Created_By": "Scientific Electronic Library Online SUSHI API", + "Customer_ID": params.get('customer', ''), + "Report_ID": params.get('report_db_params', {}).report_id, + "Release": params.get('report_db_params', {}).release, + "Report_Name": params.get('report_db_params', {}).name, + "Institution_Name": params.get('institution_name', ''), + "Institution_ID": [{ + "Type": "ISNI", + "Value": params.get('institution_id', '') + }], + }, + "Report_Filters": [{ + "Name": "Begin_Date", + "Value": str(begin_date), + }, { + "Name": "End_Date", + "Value": str(end_date), + }], + "Report_Attributes": [{ + "Name": "Attributes_To_Show", + "Value": "Data_Type|Access_Method" + }], + "Exceptions": exceptions, + "Report_Items": [] + } + + report_items = {} + + for r in result_query_reports_gr_j1: + key = (r.journalID, r.countryCode) + if key not in report_items: + report_items[key] = { + 'Title': r.title, + 'Item_ID': [], + 'Platform': params.get('platform', ''), + 'Publisher': r.publisherName, + 'Publisher_ID': [], + 'Access_Country_Code_': r.countryCode, + 'Data_Type': 'Journal', + 'Section_Type': 'Article', + 'Access_Type': 'Open Access', + 'Access_Method': 'Regular', + 'Performance': []} + + if r.printISSN: + report_items[key]['Item_ID'].append({ + "Type": 'Print_ISSN', + "Value": r.printISSN + }) + + if r.onlineISSN: + report_items[key]['Item_ID'].append({ + "Type": 'Online_ISSN', + "Value": r.onlineISSN + }) + + if params['granularity'] == 'monthly': + begin_date, end_date = cleaner.get_start_and_last_days(r.yearMonth) + + elif params['granularity'] == 'totals': + begin_date, ed_discard = cleaner.get_start_and_last_days(r.beginDate) + bd_discard, end_date = cleaner.get_start_and_last_days(r.endDate) + + for m in ['Total_Item_Requests', 'Unique_Item_Requests']: + metric_name = m[0].lower() + m[1:].replace('_', '') + + performance_m = { + 'Period': { + 'Begin_Date': str(begin_date), + 'End_Date': str(end_date) + }, + 'Instance': { + 'Metric_Type': m, + 'Count': str(getattr(r, metric_name)) + } + } + report_items[key]['Performance'].append(performance_m) + + json_results['Report_Items'] = [ri for ri in report_items.values() if ri['Title']] + return json_results + + def tsv_report_wrapper(request, report_id, result_query, params, exceptions): filename = '_'.join(['report', report_id]) + '.tsv' request.response.content_disposition = 'attachment;filename=' + filename