diff --git a/docs/types/Project.md b/docs/types/Project.md index e6106279e..e85a2eba3 100644 --- a/docs/types/Project.md +++ b/docs/types/Project.md @@ -24,6 +24,12 @@ Type: [Budget][4] ### Properties +- `devco_equity` **[BudgetItem][1]?** +- `devco_guarantee` **[BudgetItem][1]?** +- `devco_interest_rate_subsidy` **[BudgetItem][1]?** +- `devco_investment_grant` **[BudgetItem][1]?** +- `devco_loan` **[BudgetItem][1]?** +- `devco_ta` **[BudgetItem][1]?** - `eu_contrib` **[BudgetItem][1]** - `funding_area` **[Array][5]<[string][2]>** - `mmf_heading` **[string][2]** @@ -160,25 +166,57 @@ Type: [Timeframe][16] - `to` **([string][2] | null)** - `to_precision` **[TimePrecision][15]** +## SimpleValueField + +Describes a generic field in an ETL which does not have any other specific structure. + +Type: [SimpleValueField][17] + +### Properties + +- `raw` **[string][2]** +- `formatted` **[string][2]** + +## TypedValueField + +Describes a field which has a certain type of value. + +Type: [TypedValueField][18] + +### Properties + +- `field` **[string][2]** +- `type` **[string][2]** +- `raw` **[string][2]** +- `formatted` **[string][2]** + ## Project Describes `project`. -Type: [Project][17] +Type: [Project][19] ### Properties - `action` **[string][2]** - `budget` **[Budget][4]** - `call_year` **[string][2]** +- `comments` **[string][2]** +- `complete` **[boolean][20]** - `description` **[string][2]** +- `devco_arei_projects_endorsement` **[SimpleValueField][17]?** +- `devco_cris_number` **[SimpleValueField][17]?** +- `devco_date_entry` **([string][2] | null)?** +- `devco_lead_investor` **[SimpleValueField][17]?** +- `devco_leverage` **[SimpleValueField][17]?** +- `devco_project_stage` **[SimpleValueField][17]?** +- `devco_results_indicators` **[Array][5]<[TypedValueField][18]>?** - `ec_priorities` **[Array][5]<[string][2]>** - `media` **[Array][5]<[Media][12]>** - `programme_name` **[string][2]** - `project_id` **[string][2]** - `project_locations` **[Array][5]<[Location][9]>** - `project_website` **[string][2]** -- `complete` **[boolean][18]** - `related_links` **[Array][5]<[RelatedLink][13]>** - `reporting_organisation` **[string][2]** - `results` **[Result][14]** @@ -207,5 +245,7 @@ Type: [Project][17] [14]: #result [15]: #timeprecision [16]: #timeframe -[17]: #project -[18]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean +[17]: #simplevaluefield +[18]: #typedvaluefield +[19]: #project +[20]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean diff --git a/docs/types/etls/devco-xls.md b/docs/types/etls/devco-xls.md index 507dbded9..3fff0fa4b 100644 --- a/docs/types/etls/devco-xls.md +++ b/docs/types/etls/devco-xls.md @@ -24,12 +24,19 @@ Returns **Project** JSON matching the type fields. ### getBudget -Preprocess `budget` +Preprocess `budget`. Input fields taken from the `record` are: - `Total EU Contribution (Million Euro)` - `Total Budget (Million Euro)` +- `Investment Grant (Million Euro)` +- `TA (Million Euro)` +- `Interest Rate Subsidy (Million Euro)` +- `Guarantee (Million Euro)` +- `Equity (Million Euro)` +- `Budget Support (Million Euro)` +- `Loan (Million Euro)` #### Parameters @@ -37,9 +44,23 @@ Input fields taken from the `record` are: Returns **Budget** +### getComments + +Preprocess `comments`. + +Input fields taken from the `record` are: + +- `Comments` + +#### Parameters + +- `record` **[Object][3]** The row received from parsed file + +Returns **[String][4]** + ### getDescription -Preprocess `description` +Preprocess `description`. Input fields taken from the `record` are: @@ -49,6 +70,141 @@ Input fields taken from the `record` are: - `record` **[Object][3]** The row received from parsed file +Returns **[String][4]** + +### getCrisNumber + +Preprocess `devco_cris_number`. + +Input fields taken from the `record` are: + +- `CRIS No or ExCom Des` + +#### Parameters + +- `record` **[Object][3]** The row received from parsed file + +Returns **SimpleValueField** + +### getInvestor + +Preprocess `devco_lead_investor`. + +Input fields taken from the `record` are: + +- `Lead Financier` + +#### Parameters + +- `record` **[Object][3]** The row received from parsed file + +Returns **SimpleValueField** + +### getLeverage + +Preprocess `devco_leverage`. + +Input fields taken from the `record` are: + +- `Leverage` + +#### Parameters + +- `record` **[Object][3]** The row received from parsed file + +Returns **SimpleValueField** + +### getResultsIndicators + +Preprocess `devco_results_indicators`. + +Input fields taken from the `record` are: + +- `1.1 Access on grid electricity ('000 people)` +- `1.2 Access mini grid electricity ('000 people)` +- `1.3 Access off-grid electricity ('000 people)` +- `1.4 Inferred access (additional generation) ('000 people)` +- `1.5 Inferred access (cross-border transmission) ('000 people)` +- `1.6 Access to biomass/biogas clean cooking ('000 people)` +- `1.7 Access to LPG/ethanol cooking ('000 people)` +- `1.8 Electricity from renewables (GWh/year)` +- `1.9 Renewable generation capacity (MW)` +- `1.10 Electricity from energy efficiency (liberated capacity) (MW)` +- `1.11 Transmission lines (km)` +- `1.12 Distribution lines (km)` +- `1.13 Energy Savings (MWh/year)` +- `1.14 GHG emissions avoided per year (ktons CO2eq)` +- `1.15 No of direct jobs person/year (construction)` +- `1.16 No of permanent jobs (operation)` +- `2.1 Direct and Inferred electricity access ('000 people)` +- `2.2 Clean cooking and fuel access ('000 people)` +- `2.3 Direct and Inferred access to energy ('000 people)` +- `2.4 Electricity from renewabes (GWh/year)` +- `2.5 Reneable generation capacity (MW)` +- `2.6 Electricity generation capacity (MW)` +- `2.7 Transmission and distribution lines (km)` +- `2.8 GHG emissions avoided per year (ktons CO2eq)` +- `2.9 No of direct and permanent jons (construction and operation)` +- `BET1 (Access to energy)` +- `BET2 (Renewable energy generation and energy efficiency)` +- `BET3 (Contribution to the fight against climate change)` +- `EURF 1 (No of people provided with access to electricity with EU support)` +- `EURF 2 (Renewable energy production supported by the EU)` +- `EURF 3 (GHG emission avoided)` +- `SDG 7.1.1 Percentage of population with access to electricity)` +- `SDG 7.1.2 (Proportion of population with primary reliance on clean fuels and technology)` +- `SDG 7.2.1 Renewable energy share in the total final energy consumption)` +- `SDG 7.3.1 (Energy intensity measured in terms of primary energy and GDP)` +- `SDG 8.3.1 (Proportion of informal employement in non-agriculture employment, by sex)` + +#### Parameters + +- `record` **[Object][3]** The row received from parsed file + +Returns **[Array][5]<TypedValueField>** + +### getProjectStage + +Preprocess `devco_project_stage`. + +Input fields taken from the `record` are: + +- `Project Stage` + +#### Parameters + +- `record` **[Object][3]** The row received from parsed file + +Returns **SimpleValueField** + +### getDateEntry + +Preprocess `devco_date_entry`. + +Input fields taken from the `record` are: + +- `Date of data entry` + +#### Parameters + +- `record` **[Object][3]** The row received from parsed file + +Returns **[Date][6]** The date formatted into an ISO 8601 date format + +### getAreiProjectsEndorsement + +Preprocess `devco_arei_projects_endorsement`. + +Input fields taken from the `record` are: + +- `For AREI Projects (Endorsement) Y/N` + +#### Parameters + +- `record` **[Object][3]** The row received from parsed file + +Returns **SimpleValueField** + ### getCodeByCountry Gets country code from a country name. @@ -61,7 +217,7 @@ Returns **[String][4]** The ISO 3166-1 country code ### getLocations -Preprocess `project_locations` +Preprocess `project_locations`. Input fields taken from the `record` are: @@ -76,7 +232,7 @@ Returns **[Array][5]** ### getResults -Preprocess `results` +Preprocess `results`. Input fields taken from the `record` are: @@ -125,7 +281,7 @@ Returns **Result** ### getType -Preprocess `type` +Preprocess `type`. #### Parameters @@ -138,3 +294,4 @@ Returns **[Array][5]** Project types [3]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object [4]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String [5]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array +[6]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Date diff --git a/resources/elasticsearch/mappings/project.js b/resources/elasticsearch/mappings/project.js index a94f6170d..7901015a7 100644 --- a/resources/elasticsearch/mappings/project.js +++ b/resources/elasticsearch/mappings/project.js @@ -21,6 +21,22 @@ const textWithKeyword = { fields: { keyword: { type: 'keyword', ignore_above: 256 } }, }; +const simpleValueField = { + properties: { + raw: { type: 'text' }, + formatted: { type: 'text' }, + }, +}; + +const typedValueField = { + properties: { + field: { type: 'keyword' }, + type: { type: 'keyword' }, + raw: { type: 'text' }, + formatted: { type: 'text' }, + }, +}; + module.exports = () => ({ mappings: { project: { @@ -28,6 +44,12 @@ module.exports = () => ({ action: { type: 'text' }, budget: { properties: { + devco_equity: budgetItem, + devco_guarantee: budgetItem, + devco_interest_rate_subsidy: budgetItem, + devco_investment_grant: budgetItem, + devco_loan: budgetItem, + devco_ta: budgetItem, eu_contrib: budgetItem, funding_area: textWithKeyword, mmf_heading: textWithKeyword, @@ -38,9 +60,17 @@ module.exports = () => ({ }, }, call_year: { type: 'text' }, + comments: { type: 'text' }, computed_key: { type: 'keyword' }, created_by: { type: 'keyword' }, description: { type: 'text' }, + devco_arei_projects_endorsement: simpleValueField, + devco_cris_number: simpleValueField, + devco_date_entry: { type: 'date' }, + devco_lead_investor: simpleValueField, + devco_leverage: simpleValueField, + devco_project_stage: simpleValueField, + devco_results_indicators: typedValueField, ec_priorities: textWithKeyword, last_modified: { type: 'date' }, media: { diff --git a/services/ingestion/etl/cordis/csv/src/lib/transform.js b/services/ingestion/etl/cordis/csv/src/lib/transform.js index 1ac446a3b..6e883d680 100644 --- a/services/ingestion/etl/cordis/csv/src/lib/transform.js +++ b/services/ingestion/etl/cordis/csv/src/lib/transform.js @@ -424,6 +424,7 @@ export default (record: Object): Project | null => { action: '', budget: getBudget(record), call_year: '', + comments: '', description: getDescription(record), ec_priorities: [], media: [], diff --git a/services/ingestion/etl/cordis/csv/test/unit/lib/__snapshots__/transform.spec.js.snap b/services/ingestion/etl/cordis/csv/test/unit/lib/__snapshots__/transform.spec.js.snap index e05558375..3e848e5f2 100644 --- a/services/ingestion/etl/cordis/csv/test/unit/lib/__snapshots__/transform.spec.js.snap +++ b/services/ingestion/etl/cordis/csv/test/unit/lib/__snapshots__/transform.spec.js.snap @@ -35,6 +35,7 @@ Object { }, }, "call_year": "", + "comments": "", "complete": true, "description": "RCN: 30802 Objectives: %LTo understand the mechanisms leading to the formation of nitrous acid on the @@ -107,6 +108,7 @@ Object { }, }, "call_year": "", + "comments": "", "complete": true, "description": "rcn: 215949 acronym: UNISECO @@ -523,6 +525,7 @@ Object { }, }, "call_year": "", + "comments": "", "complete": true, "description": "rcn: 14088 objective: As a result of the rapid increase in forest damages in Mid-Europe, the need for the reduction of air pollutions from energy conversion and energy-end-use technologies became an important political objective.", @@ -633,6 +636,7 @@ Object { }, }, "call_year": "", + "comments": "", "complete": true, "description": "rcn: 215954 acronym: SMARTER diff --git a/services/ingestion/etl/devco/xls/README.md b/services/ingestion/etl/devco/xls/README.md index f2fbf698e..a72c1d5d0 100644 --- a/services/ingestion/etl/devco/xls/README.md +++ b/services/ingestion/etl/devco/xls/README.md @@ -2,67 +2,67 @@ Model to compare with is available at: https://ec-europa.github.io/eubfr-data-lake/ -| Field | Target | -| ---------------------------------------------------------------------------------------- | ------------------------------ | -| ID | project_id | -| Country | project_locations.country_code | -| Region | project_locations.region | -| Lead Financier | | -| Funding Source | programme_name | -| Project Title | title | -| CRIS No or ExCom Des | | -| Project Type | type | -| Total Budget (Million Euro) | budget.total_cost | -| Investment Grant (Million Euro) | | -| TA (Million Euro) | | -| Interest Rate Subsidy (Million Euro) | | -| Guarantee (Million Euro) | | -| Equity (Million Euro) | | -| Budget Support (Million Euro) | | -| Loan (Million Euro) | | -| Total EU Contribution (Million Euro) | budget.eu_contrib | -| Leverage | | -| 1.1 Access on grid electricity ('000 people) | results | -| 1.2 Access mini grid electricity ('000 people) | results | -| 1.3 Access off-grid electricity ('000 people) | results | -| 1.4 Inferred access (additional generation) ('000 people) | results | -| 1.5 Inferred access (cross-border transmission) ('000 people) | results | -| 1.6 Access to biomass/biogas clean cooking ('000 people) | results | -| 1.7 Access to LPG/ethanol cooking ('000 people) | results | -| 1.8 Electricity from renewables (GWh/year) | results | -| 1.9 Renewable generation capacity (MW) | results | -| 1.10 Electricity from energy efficiency (liberated capacity) (MW) | results | -| 1.11 Transmission lines (km) | results | -| 1.12 Distribution lines (km) | results | -| 1.13 Energy Savings (MWh/year) | results | -| 1.14 GHG emissions avoided per year (ktons CO2eq) | results | -| 1.15 No of direct jobs person/year (construction) | results | -| 1.16 No of permanent jobs (operation) | results | -| 2.1 Direct and Inferred electricity access ('000 people) | results | -| 2.2 Clean cooking and fuel access ('000 people) | results | -| 2.3 Direct and Inferred access to energy ('000 people) | results | -| 2.4 Electricity from renewabes (GWh/year) | results | -| 2.5 Reneable generation capacity (MW) | results | -| 2.6 Electricity generation capacity (MW) | results | -| 2.7 Transmission and distribution lines (km) | results | -| 2.8 GHG emissions avoided per year (ktons CO2eq) | results | -| 2.9 No of direct and permanent jons (construction and operation) | results | -| BET1 (Access to energy) | results | -| BET2 (Renewable energy generation and energy efficiency) | results | -| BET3 (Contribution to the fight against climate change) | results | -| EURF 1 (No of people provided with access to electricity with EU support) | results | -| EURF 2 (Renewable energy production supported by the EU) | results | -| EURF 3 (GHG emission avoided) | results | -| SDG 7.1.1 Percentage of population with access to electricity) | results | -| SDG 7.1.2 (Proportion of population with primary reliance on clean fuels and technology) | results | -| SDG 7.2.1 Renewable energy share in the total final energy consumption) | results | -| SDG 7.3.1 (Energy intensity measured in terms of primary energy and GDP) | results | -| SDG 8.3.1 (Proportion of informal employement in non-agriculture employment, by sex) | results | -| Description and main objectives | description | -| Project Stage | | -| Contract Status | status | -| Starting Date | timeframe.from | -| Ending Date | timeframe.to | -| Date of data entry | | -| For AREI Projects (Endorsement) Y/N | | -| Comments | | +| Field | Target | +| ---------------------------------------------------------------------------------------- | ---------------------------------- | +| ID | project_id | +| Country | project_locations.country_code | +| Region | project_locations.region | +| Lead Financier | devco_lead_investor | +| Funding Source | programme_name | +| Project Title | title | +| CRIS No or ExCom Des | devco_cris_number | +| Project Type | type | +| Total Budget (Million Euro) | budget.total_cost | +| Investment Grant (Million Euro) | devco_investment_grant | +| TA (Million Euro) | devco_ta | +| Interest Rate Subsidy (Million Euro) | devco_interest_rate_subsidy | +| Guarantee (Million Euro) | devco_guarantee | +| Equity (Million Euro) | devco_equity | +| Budget Support (Million Euro) | devco_budget_support | +| Loan (Million Euro) | devco_loan | +| Total EU Contribution (Million Euro) | budget.eu_contrib | +| Leverage | devco_leverage | +| 1.1 Access on grid electricity ('000 people) | results, devco_results_indicators | +| 1.2 Access mini grid electricity ('000 people) | results ,devco_results_indicators | +| 1.3 Access off-grid electricity ('000 people) | results,devco_results_indicators | +| 1.4 Inferred access (additional generation) ('000 people) | results,devco_results_indicators | +| 1.5 Inferred access (cross-border transmission) ('000 people) | results ,devco_results_indicators | +| 1.6 Access to biomass/biogas clean cooking ('000 people) | results ,devco_results_indicators | +| 1.7 Access to LPG/ethanol cooking ('000 people) | results ,devco_results_indicators | +| 1.8 Electricity from renewables (GWh/year) | results ,devco_results_indicators | +| 1.9 Renewable generation capacity (MW) | results ,devco_results_indicators | +| 1.10 Electricity from energy efficiency (liberated capacity) (MW) | results,devco_results_indicators | +| 1.11 Transmission lines (km) | results,devco_results_indicators | +| 1.12 Distribution lines (km) | results ,devco_results_indicators | +| 1.13 Energy Savings (MWh/year) | results , devco_results_indicators | +| 1.14 GHG emissions avoided per year (ktons CO2eq) | results ,devco_results_indicators | +| 1.15 No of direct jobs person/year (construction) | results ,devco_results_indicators | +| 1.16 No of permanent jobs (operation) | results ,devco_results_indicators | +| 2.1 Direct and Inferred electricity access ('000 people) | results ,devco_results_indicators | +| 2.2 Clean cooking and fuel access ('000 people) | results ,devco_results_indicators | +| 2.3 Direct and Inferred access to energy ('000 people) | results ,devco_results_indicators | +| 2.4 Electricity from renewabes (GWh/year) | results ,devco_results_indicators | +| 2.5 Reneable generation capacity (MW) | results ,devco_results_indicators | +| 2.6 Electricity generation capacity (MW) | results ,devco_results_indicators | +| 2.7 Transmission and distribution lines (km) | results,devco_results_indicators | +| 2.8 GHG emissions avoided per year (ktons CO2eq) | results ,devco_results_indicators | +| 2.9 No of direct and permanent jons (construction and operation) | results ,devco_results_indicators | +| BET1 (Access to energy) | results ,devco_results_indicators | +| BET2 (Renewable energy generation and energy efficiency) | results ,devco_results_indicators | +| BET3 (Contribution to the fight against climate change) | results ,devco_results_indicators | +| EURF 1 (No of people provided with access to electricity with EU support) | results ,devco_results_indicators | +| EURF 2 (Renewable energy production supported by the EU) | results ,devco_results_indicators | +| EURF 3 (GHG emission avoided) | results ,devco_results_indicators | +| SDG 7.1.1 Percentage of population with access to electricity) | results ,devco_results_indicators | +| SDG 7.1.2 (Proportion of population with primary reliance on clean fuels and technology) | results ,devco_results_indicators | +| SDG 7.2.1 Renewable energy share in the total final energy consumption) | results ,devco_results_indicators | +| SDG 7.3.1 (Energy intensity measured in terms of primary energy and GDP) | results ,devco_results_indicators | +| SDG 8.3.1 (Proportion of informal employement in non-agriculture employment, by sex) | results ,devco_results_indicators | +| Description and main objectives | description | +| Project Stage | devco_project_stage | +| Contract Status | status | +| Starting Date | timeframe.from | +| Ending Date | timeframe.to | +| Date of data entry | devco_date_entry | +| For AREI Projects (Endorsement) Y/N | devco_arei_projects_endorsement | +| Comments | comments | diff --git a/services/ingestion/etl/devco/xls/src/events/onParseXLS.js b/services/ingestion/etl/devco/xls/src/events/onParseXLS.js index 1b08e9b4d..2b9150007 100644 --- a/services/ingestion/etl/devco/xls/src/events/onParseXLS.js +++ b/services/ingestion/etl/devco/xls/src/events/onParseXLS.js @@ -1,3 +1,4 @@ +import { EOL } from 'os'; import AWS from 'aws-sdk'; // eslint-disable-line import/no-extraneous-dependencies import XLSX from 'xlsx'; @@ -111,7 +112,7 @@ export const handler = async (event, context) => { // Apply specific transformations for this ETL. projects.forEach(project => { const data = transformRecord(project); - uploadData += `${JSON.stringify(data)}\n`; + uploadData += `${JSON.stringify(data)}${EOL}`; }); // Load data diff --git a/services/ingestion/etl/devco/xls/src/lib/transform.js b/services/ingestion/etl/devco/xls/src/lib/transform.js index ae19d03f3..c6147edc3 100644 --- a/services/ingestion/etl/devco/xls/src/lib/transform.js +++ b/services/ingestion/etl/devco/xls/src/lib/transform.js @@ -11,17 +11,23 @@ import type { Project } from '@eubfr/types'; */ /** - * Preprocess `budget` + * Preprocess `budget`. * * Input fields taken from the `record` are: * - `Total EU Contribution (Million Euro)` * - `Total Budget (Million Euro)` + * - `Investment Grant (Million Euro)` + * - `TA (Million Euro)` + * - `Interest Rate Subsidy (Million Euro)` + * - `Guarantee (Million Euro)` + * - `Equity (Million Euro)` + * - `Budget Support (Million Euro)` + * - `Loan (Million Euro)` * * @memberof DevcoXlsTransform * @param {Object} record The row received from parsed file * @returns {Budget} */ - const getBudget = record => { const euContribString = record['Total EU Contribution (Million Euro)'] ? `EUR ${record['Total EU Contribution (Million Euro)']} million` @@ -37,7 +43,63 @@ const getBudget = record => { ? sanitizeBudgetItem(extractBudgetData(totalCostString)) : sanitizeBudgetItem(); + const investmentGrantString = record['Investment Grant (Million Euro)'] + ? `EUR ${record['Investment Grant (Million Euro)']} million` + : null; + const investmentGrant = investmentGrantString + ? sanitizeBudgetItem(extractBudgetData(investmentGrantString)) + : sanitizeBudgetItem(); + + const taString = record['TA (Million Euro)'] + ? `EUR ${record['TA (Million Euro)']} million` + : null; + const ta = taString + ? sanitizeBudgetItem(extractBudgetData(taString)) + : sanitizeBudgetItem(); + + const interestRateString = record['Interest Rate Subsidy (Million Euro)'] + ? `EUR ${record['Interest Rate Subsidy (Million Euro)']} million` + : null; + const interestRate = interestRateString + ? sanitizeBudgetItem(extractBudgetData(interestRateString)) + : sanitizeBudgetItem(); + + const guaranteeString = record['Guarantee (Million Euro)'] + ? `EUR ${record['Guarantee (Million Euro)']} million` + : null; + const guarantee = guaranteeString + ? sanitizeBudgetItem(extractBudgetData(guaranteeString)) + : sanitizeBudgetItem(); + + const equityString = record['Equity (Million Euro)'] + ? `EUR ${record['Equity (Million Euro)']} million` + : null; + const equity = equityString + ? sanitizeBudgetItem(extractBudgetData(equityString)) + : sanitizeBudgetItem(); + + const budgetSupportString = record['Budget Support (Million Euro)'] + ? `EUR ${record['Budget Support (Million Euro)']} million` + : null; + const budgetSupport = budgetSupportString + ? sanitizeBudgetItem(extractBudgetData(budgetSupportString)) + : sanitizeBudgetItem(); + + const loanString = record['Loan (Million Euro)'] + ? `EUR ${record['Loan (Million Euro)']} million` + : null; + const loan = loanString + ? sanitizeBudgetItem(extractBudgetData(loanString)) + : sanitizeBudgetItem(); + const budget = { + devco_budget_support: budgetSupport, + devco_equity: equity, + devco_guarantee: guarantee, + devco_interest_rate_subsidy: interestRate, + devco_investment_grant: investmentGrant, + devco_ta: ta, + devco_loan: loan, eu_contrib: euContrib, funding_area: [], mmf_heading: '', @@ -51,21 +113,327 @@ const getBudget = record => { }; /** - * Preprocess `description` + * Preprocess `comments`. * * Input fields taken from the `record` are: - * - `Description and main objectives` + * - `Comments` * * @memberof DevcoXlsTransform * @param {Object} record The row received from parsed file - * @returns {import('aws-sdk/clients/elbv2').String} + * @returns {String} */ +const getComments = record => + record.Comments ? record.Comments.replace(/ +(?= )/g, '').trim() : ''; +/** + * Preprocess `description`. + * + * Input fields taken from the `record` are: + * - `Description and main objectives` + * + * @memberof DevcoXlsTransform + * @param {Object} record The row received from parsed file + * @returns {String} + */ const getDescription = record => record['Description and main objectives'] ? record['Description and main objectives'].trim() : ''; +/** + * Preprocess `devco_cris_number`. + * + * Input fields taken from the `record` are: + * - `CRIS No or ExCom Des` + * + * @memberof DevcoXlsTransform + * @param {Object} record The row received from parsed file + * @returns {SimpleValueField} + */ +const getCrisNumber = record => { + const number = { + raw: '', + formatted: '', + }; + + if (record['CRIS No or ExCom Des']) { + number.raw = record['CRIS No or ExCom Des']; + } + + return number; +}; + +/** + * Preprocess `devco_lead_investor`. + * + * Input fields taken from the `record` are: + * - `Lead Financier` + * + * @memberof DevcoXlsTransform + * @param {Object} record The row received from parsed file + * @returns {SimpleValueField} + */ +const getInvestor = record => { + const investor = { + raw: '', + formatted: '', + }; + + if (record['Lead Financier']) { + investor.raw = record['Lead Financier']; + } + + return investor; +}; + +/** + * Preprocess `devco_leverage`. + * + * Input fields taken from the `record` are: + * - `Leverage` + * + * @memberof DevcoXlsTransform + * @param {Object} record The row received from parsed file + * @returns {SimpleValueField} + */ +const getLeverage = record => { + const leverage = { + raw: '', + formatted: '', + }; + + if (record.Leverage) { + leverage.raw = record.Leverage; + } + + return leverage; +}; + +/** + * Preprocess `devco_results_indicators`. + * + * Input fields taken from the `record` are: + * + * - `1.1 Access on grid electricity ('000 people)` + * - `1.2 Access mini grid electricity ('000 people)` + * - `1.3 Access off-grid electricity ('000 people)` + * - `1.4 Inferred access (additional generation) ('000 people)` + * - `1.5 Inferred access (cross-border transmission) ('000 people)` + * - `1.6 Access to biomass/biogas clean cooking ('000 people)` + * - `1.7 Access to LPG/ethanol cooking ('000 people)` + * - `1.8 Electricity from renewables (GWh/year)` + * - `1.9 Renewable generation capacity (MW)` + * - `1.10 Electricity from energy efficiency (liberated capacity) (MW)` + * - `1.11 Transmission lines (km)` + * - `1.12 Distribution lines (km)` + * - `1.13 Energy Savings (MWh/year)` + * - `1.14 GHG emissions avoided per year (ktons CO2eq)` + * - `1.15 No of direct jobs person/year (construction)` + * - `1.16 No of permanent jobs (operation)` + * - `2.1 Direct and Inferred electricity access ('000 people)` + * - `2.2 Clean cooking and fuel access ('000 people)` + * - `2.3 Direct and Inferred access to energy ('000 people)` + * - `2.4 Electricity from renewabes (GWh/year)` + * - `2.5 Reneable generation capacity (MW)` + * - `2.6 Electricity generation capacity (MW)` + * - `2.7 Transmission and distribution lines (km)` + * - `2.8 GHG emissions avoided per year (ktons CO2eq)` + * - `2.9 No of direct and permanent jons (construction and operation)` + * - `BET1 (Access to energy)` + * - `BET2 (Renewable energy generation and energy efficiency)` + * - `BET3 (Contribution to the fight against climate change)` + * - `EURF 1 (No of people provided with access to electricity with EU support)` + * - `EURF 2 (Renewable energy production supported by the EU)` + * - `EURF 3 (GHG emission avoided)` + * - `SDG 7.1.1 Percentage of population with access to electricity)` + * - `SDG 7.1.2 (Proportion of population with primary reliance on clean fuels and technology)` + * - `SDG 7.2.1 Renewable energy share in the total final energy consumption)` + * - `SDG 7.3.1 (Energy intensity measured in terms of primary energy and GDP)` + * - `SDG 8.3.1 (Proportion of informal employement in non-agriculture employment, by sex)` + * + * @memberof DevcoXlsTransform + * @param {Object} record The row received from parsed file + * @returns {Array} + */ +const getResultsIndicators = record => { + const indicators = []; + + const wordsToRemove = [ + 'of', + 'the', + 'in', + 'on', + 'at', + 'to', + 'a', + 'is', + 'with', + 'and', + ]; + const removeWords = new RegExp(`\\b(${wordsToRemove.join('|')})\\b`, 'g'); + + const fields = [ + "1.1 Access on grid electricity ('000 people)", + "1.2 Access mini grid electricity ('000 people)", + "1.3 Access off-grid electricity ('000 people)", + "1.4 Inferred access (additional generation) ('000 people)", + "1.5 Inferred access (cross-border transmission) ('000 people)", + "1.6 Access to biomass/biogas clean cooking ('000 people)", + "1.7 Access to LPG/ethanol cooking ('000 people)", + '1.8 Electricity from renewables (GWh/year)', + '1.9 Renewable generation capacity (MW)', + '1.10 Electricity from energy efficiency (liberated capacity) (MW)', + '1.11 Transmission lines (km)', + '1.12 Distribution lines (km)', + '1.13 Energy Savings (MWh/year)', + '1.14 GHG emissions avoided per year (ktons CO2eq)', + '1.15 No of direct jobs person/year (construction)', + '1.16 No of permanent jobs (operation)', + "2.1 Direct and Inferred electricity access ('000 people)", + "2.2 Clean cooking and fuel access ('000 people)", + "2.3 Direct and Inferred access to energy ('000 people)", + '2.4 Electricity from renewabes (GWh/year)', + '2.5 Reneable generation capacity (MW)', + '2.6 Electricity generation capacity (MW)', + '2.7 Transmission and distribution lines (km)', + '2.8 GHG emissions avoided per year (ktons CO2eq)', + '2.9 No of direct and permanent jons (construction and operation)', + 'BET1 (Access to energy)', + 'BET2 (Renewable energy generation and energy efficiency)', + 'BET3 (Contribution to the fight against climate change)', + 'EURF 1 (No of people provided with access to electricity with EU support)', + 'EURF 2 (Renewable energy production supported by the EU)', + 'EURF 3 (GHG emission avoided)', + 'SDG 7.1.1 Percentage of population with access to electricity)', + 'SDG 7.1.2 (Proportion of population with primary reliance on clean fuels and technology)', + 'SDG 7.2.1 Renewable energy share in the total final energy consumption)', + 'SDG 7.3.1 (Energy intensity measured in terms of primary energy and GDP)', + 'SDG 8.3.1 (Proportion of informal employement in non-agriculture employment, by sex)', + ]; + + fields.forEach(field => { + if (field in record) { + // Remove unnecessary numbers with dots. + let fieldLabel = field + // remove 1.1 + .replace(/(\d\.\d+)+/g, '') + // remove .1 + .replace(/(\.\d+)+/g, '') + // replace ' ' to '' + .replace(/\s\s+/g, ' ') + .replace(removeWords, '') + .trim(); + + // Fields of some abbreviations could be without brackets. + if ( + fieldLabel.includes('BET') || + fieldLabel.includes('EURF') || + fieldLabel.includes('SDG') + ) { + fieldLabel = fieldLabel.replace(/[{()}]/g, ''); + } + + // Name it like a field. + fieldLabel = fieldLabel + .replace(/ /g, '_') + .replace(/,/g, '') + .replace(/'/g, '') + .replace(/[(|)]/g, '') + .replace(/\//g, '') + .replace(/-/g, '_') + .replace(/-/g, '_') + .replace(/__/g, '_') + .replace(/__/g, '_') + .toLowerCase(); + + const indicator = { + field: fieldLabel, + type: 'indicator', + raw: record[field], + formatted: '', + }; + + indicators.push(indicator); + } + }); + + return indicators; +}; + +/** + * Preprocess `devco_project_stage`. + * + * Input fields taken from the `record` are: + * - `Project Stage` + * + * @memberof DevcoXlsTransform + * @param {Object} record The row received from parsed file + * @returns {SimpleValueField} + */ +const getProjectStage = record => { + const stage = { + raw: '', + formatted: '', + }; + + if (record['Project Stage']) { + stage.raw = record['Project Stage']; + } + + return stage; +}; + +/** + * Preprocess `devco_date_entry`. + * + * Input fields taken from the `record` are: + * - `Date of data entry` + * + * @memberof DevcoXlsTransform + * @param {Object} record The row received from parsed file + * @returns {Date} The date formatted into an ISO 8601 date format + */ +const getDateEntry = record => { + let date = null; + + if (record['Date of data entry']) { + const dateParts = record['Date of data entry'] + .split('/') + .map(d => d.trim()); + + const [m, y] = dateParts; + + if (m && y) { + date = new Date(Date.UTC(y, m - 1, 1)).toISOString(); + } + } + + return date; +}; + +/** + * Preprocess `devco_arei_projects_endorsement`. + * + * Input fields taken from the `record` are: + * - `For AREI Projects (Endorsement) Y/N` + * + * @memberof DevcoXlsTransform + * @param {Object} record The row received from parsed file + * @returns {SimpleValueField} + */ +const getAreiProjectsEndorsement = record => { + const arei = { + raw: '', + formatted: '', + }; + + if (record['For AREI Projects (Endorsement) Y/N']) { + arei.raw = record['For AREI Projects (Endorsement) Y/N']; + } + + return arei; +}; + /** * Gets country code from a country name. * @@ -73,12 +441,11 @@ const getDescription = record => * @param {String} countryName The name of the country * @returns {String} The ISO 3166-1 country code */ - const getCodeByCountry = countryName => countries.getAlpha2Code(countryName, 'en'); /** - * Preprocess `project_locations` + * Preprocess `project_locations`. * * Input fields taken from the `record` are: * - `Country` @@ -88,7 +455,6 @@ const getCodeByCountry = countryName => * @param {Object} record The row received from parsed file * @returns {Array} */ - const getLocations = record => { const locations = []; const code = getCountryCode(getCodeByCountry(record.Country)); @@ -110,7 +476,7 @@ const getLocations = record => { }; /** - * Preprocess `results` + * Preprocess `results`. * * Input fields taken from the `record` are: * @@ -155,7 +521,6 @@ const getLocations = record => { * @param {Object} record The row received from parsed file * @returns {Result} */ - const getResults = record => { let resultIsAvailable = false; let resultsContents = ''; @@ -237,13 +602,12 @@ const getResults = record => { }; /** - * Preprocess `type` + * Preprocess `type`. * * @memberof DevcoXlsTransform * @param {Object} record The row received from parsed file * @returns {Array} Project types */ - const getType = record => record['Project Type'] ? record['Project Type'] @@ -270,8 +634,16 @@ export default (record: Object): Project | null => { action: '', budget: getBudget(record), call_year: '', + comments: getComments(record), description: getDescription(record), ec_priorities: [], + devco_cris_number: getCrisNumber(record), + devco_lead_investor: getInvestor(record), + devco_leverage: getLeverage(record), + devco_results_indicators: getResultsIndicators(record), + devco_project_stage: getProjectStage(record), + devco_date_entry: getDateEntry(record), + devco_arei_projects_endorsement: getAreiProjectsEndorsement(record), media: [], programme_name: record['Funding Source'] || '', project_id: record.ID || '', diff --git a/services/ingestion/etl/devco/xls/test/unit/lib/__snapshots__/transform.spec.js.snap b/services/ingestion/etl/devco/xls/test/unit/lib/__snapshots__/transform.spec.js.snap index 349cb26af..9a8641d4a 100644 --- a/services/ingestion/etl/devco/xls/test/unit/lib/__snapshots__/transform.spec.js.snap +++ b/services/ingestion/etl/devco/xls/test/unit/lib/__snapshots__/transform.spec.js.snap @@ -4,6 +4,41 @@ exports[`DG DEVCO XLS transformer Produces correct JSON output structure 1`] = ` Object { "action": "", "budget": Object { + "devco_budget_support": Object { + "currency": "", + "raw": "", + "value": 0, + }, + "devco_equity": Object { + "currency": "", + "raw": "", + "value": 0, + }, + "devco_guarantee": Object { + "currency": "", + "raw": "", + "value": 0, + }, + "devco_interest_rate_subsidy": Object { + "currency": "", + "raw": "", + "value": 0, + }, + "devco_investment_grant": Object { + "currency": "EUR", + "raw": "EUR 0.618036 million", + "value": 618036, + }, + "devco_loan": Object { + "currency": "", + "raw": "", + "value": 0, + }, + "devco_ta": Object { + "currency": "", + "raw": "", + "value": 0, + }, "eu_contrib": Object { "currency": "EUR", "raw": "EUR 0.618036 million", @@ -33,8 +68,248 @@ Object { }, }, "call_year": "", + "comments": "Politique de l'énergie et gestion administrative", "complete": true, "description": "Fourniture, installation, mise en SAV d'un an des équipements photovoltaïques au bénéfice de onze (11) délégations régionales du Ministère des Finances et du Budget et de trois (03) communes pilotes", + "devco_arei_projects_endorsement": Object { + "formatted": "", + "raw": "", + }, + "devco_cris_number": Object { + "formatted": "", + "raw": "2016 / 371004", + }, + "devco_date_entry": "2018-08-01T00:00:00.000Z", + "devco_lead_investor": Object { + "formatted": "", + "raw": "EC", + }, + "devco_leverage": Object { + "formatted": "", + "raw": 1, + }, + "devco_project_stage": Object { + "formatted": "", + "raw": "", + }, + "devco_results_indicators": Array [ + Object { + "field": "access_grid_electricity_000_people", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "access_mini_grid_electricity_000_people", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "access_off_grid_electricity_000_people", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "inferred_access_additional_generation_000_people", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "inferred_access_cross_border_transmission_000_people", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "access_biomassbiogas_clean_cooking_000_people", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "access_lpgethanol_cooking_000_people", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "electricity_from_renewables_gwhyear", + "formatted": "", + "raw": 0.24528000000000005, + "type": "indicator", + }, + Object { + "field": "renewable_generation_capacity_mw", + "formatted": "", + "raw": 0.14, + "type": "indicator", + }, + Object { + "field": "electricity_from_energy_efficiency_liberated_capacity_mw", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "transmission_lines_km", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "distribution_lines_km", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "energy_savings_mwhyear", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "ghg_emissions_avoided_per_year_ktons_co2eq", + "formatted": "", + "raw": 0.2, + "type": "indicator", + }, + Object { + "field": "no_direct_jobs_personyear_construction", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "no_permanent_jobs_operation", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "direct_inferred_electricity_access_000_people", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "clean_cooking_fuel_access_000_people", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "direct_inferred_access_energy_000_people", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "electricity_from_renewabes_gwhyear", + "formatted": "", + "raw": 0.24528000000000005, + "type": "indicator", + }, + Object { + "field": "reneable_generation_capacity_mw", + "formatted": "", + "raw": 0.14, + "type": "indicator", + }, + Object { + "field": "electricity_generation_capacity_mw", + "formatted": "", + "raw": 0.14, + "type": "indicator", + }, + Object { + "field": "transmission_distribution_lines_km", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "ghg_emissions_avoided_per_year_ktons_co2eq", + "formatted": "", + "raw": 0.2, + "type": "indicator", + }, + Object { + "field": "no_direct_permanent_jons_construction_operation", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "bet1_access_energy", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "bet2_renewable_energy_generation_energy_efficiency", + "formatted": "", + "raw": 0.14, + "type": "indicator", + }, + Object { + "field": "bet3_contribution_fight_against_climate_change", + "formatted": "", + "raw": 0.2, + "type": "indicator", + }, + Object { + "field": "eurf_1_no_people_provided_access_electricity_eu_support", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "eurf_2_renewable_energy_production_supported_by_eu", + "formatted": "", + "raw": 0.24528000000000005, + "type": "indicator", + }, + Object { + "field": "eurf_3_ghg_emission_avoided", + "formatted": "", + "raw": 0.2, + "type": "indicator", + }, + Object { + "field": "sdg_percentage_population_access_electricity", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "sdg_proportion_population_primary_reliance_clean_fuels_technology", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "sdg_renewable_energy_share_total_final_energy_consumption", + "formatted": "", + "raw": 0.24528000000000005, + "type": "indicator", + }, + Object { + "field": "sdg_energy_intensity_measured_terms_primary_energy_gdp", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + Object { + "field": "sdg_proportion_informal_employement_non_agriculture_employment_by_sex", + "formatted": "", + "raw": 0, + "type": "indicator", + }, + ], "ec_priorities": Array [], "media": Array [], "programme_name": "EDF", diff --git a/services/ingestion/etl/eac/csv/src/lib/transform.js b/services/ingestion/etl/eac/csv/src/lib/transform.js index 1c796539d..a6a0b2f9f 100644 --- a/services/ingestion/etl/eac/csv/src/lib/transform.js +++ b/services/ingestion/etl/eac/csv/src/lib/transform.js @@ -402,6 +402,7 @@ export default (record: Object): Project | null => { action: getAction(record), budget: getBudget(record), call_year: getCallYear(record), + comments: '', description: getDescription(record), ec_priorities: [], media: [], diff --git a/services/ingestion/etl/eac/csv/test/unit/lib/__snapshots__/transform.spec.js.snap b/services/ingestion/etl/eac/csv/test/unit/lib/__snapshots__/transform.spec.js.snap index 48ef83588..51eabd7de 100644 --- a/services/ingestion/etl/eac/csv/test/unit/lib/__snapshots__/transform.spec.js.snap +++ b/services/ingestion/etl/eac/csv/test/unit/lib/__snapshots__/transform.spec.js.snap @@ -33,6 +33,7 @@ Object { }, }, "call_year": "", + "comments": "", "complete": true, "description": "Using the successful model of airline alliances, the EU/Korea Leadership Alliance (EUKLA) will establish a framework within which international activities between the EU and Korea will be promoted, recorded, recognized, and rewarded. @@ -234,6 +235,7 @@ Object { }, }, "call_year": "2014", + "comments": "", "complete": true, "description": "The ELiT Literaturehouse Europe establishes an observatory for European contemporary literature focusing chiefly on: research, discussion and publishing results concerning literary trends across Europe, as well as the inter-cultural communication of literature within Europe and the dissemination of literature among the diverse cultural spaces within Europe. It enables writers to introduce their works in other linguistic areas and to discover new opportunities for publishing based on digital media. It develops innovative forms of communicating literature for young people from different south eastern European countries of the Danube Region Strategy who get to know Europe’s shared heritage through working on a literary subject and producing a collectively authored ebook.ELiT Literaturehouse Europe regularly organizes these events: European literature youth meetings, European writers’ reading tours sessions, in particular, with the winners of the European Union Prize for Literature, workshops for young writers and – the annual highlight – the European Literature Days in the Wachau/Lower Austria.ELiT Literaturehouse Europe headquarters is in Wachau/Lower Austria, where it facilitates networking for literary experts, centres and events as well as setting up cooperative programmes within Europe. It upgrades the professionalitiy of it‘s work and currently sets up a programme for writers and translators-in-residence. The European Literature Days take place in Spitz, a little village in Wachau, an outstanding cultural landscape and UNESCO World Heritage Centre. They are a highlight of the annual culture programme of Spitz and both well known and estimated in the region. At the website www.literaturehouse.eu, the European Literature social media platform www.readme.cc and trough ebook- and book on demand publications ELiT Literaturehouse Europe will introduce literary trends in Europe and publish and discuss texts on European issues in cultural policy.", "ec_priorities": Array [], @@ -440,6 +442,7 @@ Object { }, }, "call_year": "", + "comments": "", "complete": true, "description": "DÉPARTS seeks to invest in the future of European contemporary dance by helping young artists with their careers. It does this by providing access to high-quality training facilities outside traditional education structures and supporting the transition to becoming a professional performing artist.", "ec_priorities": Array [], @@ -632,6 +635,7 @@ Object { }, }, "call_year": "", + "comments": "", "complete": true, "description": "The EMCL is a 24-month full-time interdisciplinary and transnational university programme at the Masters level providing integrated training in theoretical and experimental neurosciences and psycholinguistics with clinical issues. The aim of the Masters programme is to train highly qualified advanced students who are excellently prepared for research work and PhD-programmes in the above fields. @@ -767,6 +771,7 @@ Object { }, }, "call_year": "2014", + "comments": "", "complete": true, "description": "The project aims at reinforcing European Studies at Yildirim Beyazit University of Ankara (YBU). In early 2013 a programme for the promotion of European studies took form within the Faculty of Political Sciences of this University, having as final objective the setting-up of a Centre for European Studies. For this very reason the subsequent step has been that of transferring the Jean Monnet Chair held by the applicant from Brescia University to YBU. This proposal represents, thus, the third step of the strategy of YBU to enhance European studies in two specific fields: Economic integration and International relations. A fourth step (2015) will be the application for two modules in EU energy policy and in EU international relations, finally a fifth (2016) the application for a Pole of excellence. “Building bridges to Europe and to the World” in Turkish “Avrupa ile Dunya arasinda Kopru Insasithis” (ADAP is the acronym, meaning Ethics) is the title of the project. ADAP aims first of all at establishing a solid base for European studies at YBU. For this we have set-up a key staff team with a strong scientific background and committed to study European integration. Furthermore six full courses will be taught. ADAP aims to spread EU knowledge and information through the Country, to this end we included actions in Universities located outside the main urban centres. Dissemination of the results is considered as another way to spread the knowledge and information on the EU through Turkey. For this reason we pay strong attention to the dissemination strategy. The project aims at building bridges to Europe through several actions: dialogue with policy makers, training for journalists, visiting Jean Monnet colleagues for workshops, seminars on EU topical issues. YBU hosts many international students coming from Central Asia, Africa and the Balkans. All the activities of the Chair will be open to these students. Three research projects will be realized. ", "ec_priorities": Array [], @@ -857,6 +862,7 @@ Object { }, }, "call_year": "2014", + "comments": "", "complete": true, "description": "1 Background: Project \\"e-twinning\\" an \\"inter-cultural journey through Europe\\" from Berlin as a divided city until 2014 planned in several phases. The Project will be a motor of european development and mobility. Primary has the cooperation together european schools. @@ -960,6 +966,7 @@ Object { }, }, "call_year": "2015", + "comments": "", "complete": true, "description": "In order to keep the language skills of the teachers at the Primary School as high as possible, constant training it is crucial. This year we will make our first experience with Erasmus+ and therefore will send Sibylle Büchel in this English project. Sibylle Büchel teaches for almost 20 years at out school and in these years always was very interested in further education. Thus she is perfectly predestined for this project. She is class teacher in the middle school and teaches the pupils of the fifth class. In the primary school area the focus lays on the communication abilities of the pupils. Sibylle Büchel will be able to transfer elements of the chosen course \\"English for Effective Communication\\" into her everyday life and she will be able to adapt course contents according to the class level. This will enable to meet the requirements in the primary school area and to better prepare the pupils for secondary school.", "ec_priorities": Array [], @@ -1048,6 +1055,7 @@ Object { }, }, "call_year": "2016", + "comments": "", "complete": true, "description": "In the three-tier school system on secondary school level in the Principality of Liechtenstein the Oberschule Eschen takes care of many students with learning difficulties and behavioural challenges. The quota of students coming from educationally deprived families and/or families with migration background is very high. Accordingly high is the proportion of risk school pupils.The team of the Oberschule Eschen wants to encounter these problem through a holistic approach. The pupils should be encouraged through modern teaching methods in the cognitive as well as in the social field. The promotion of independent, self-responsible learning is a top priority. Learners should acquire attractive and true-to-life skills through stimulating learning environments and practical activities and develop a willingness for lifelong learning. Competence grids as additional, individual assessment instruments should be implemented.A possible major focus may be the raise of awareness of nutrition and exercise as key factors for learning success . The well-being at school shall be supported by violence preventive and community-building activities for learners as well as for teachers. The multicultural background of the students and their parents as well as of a part of the teaching professionals should be regarded more deeply as an opportunity to implement our mission values such as appreciation, cooperation and individuality, immediately.So should the teachers team, consisting of 30 teachers, attend developments for example in the fields of practical tution, project work for students with handicaps or problems as mentioned above and further modern teaching technologies as well as their individual assessment opportunities. School visits, to compare how other schools act sucessfully in terms on the subject, would also be an important aspect of the training. Language courses or further training courses in English would be a good basis for international collaboration with other schools like student exchange projects for example in order to enable impressive and sustainable learning in many ways.A long-term effect of such developments would be more selfassurance for the teachers when applying new methods. A theoretical background, gaining experience from practice and the compare to schools with similar conditions lead to a well-founded knowledge. Not to be forgotten is also a sustainable team-building impact if proportions of teachers can improve their skills together and apply the newly acquired knowledge at school.", "ec_priorities": Array [], @@ -1136,6 +1144,7 @@ Object { }, }, "call_year": "2017", + "comments": "", "complete": true, "description": "As a leading city school, we put emphasis on our teachers and expert associates\` professional development and training. Considering that our constant progress of students' achievements, school organizational skills, quality of teaching and overall school image are our permanent goals, we have applied to participate in the local project \\"School Modernization\\" and received assurance that certain funds would be given for obtaining new information technology. Therefore, we have decided to IMPROVE other KEY COMPETENCIES of our teachers, preparing them in time for these technology innovations and maintaining continuous development of our school. We have selected a team of nine enough motivated and professionally educated members. We would like to begin the process of internationalization as soon as possible, establishing collaboration with schools from other countries from which we can learn a great deal and to contribute to the European dimension of our school. We have found a school in Ostrava, Czech Republic over the eTwinning platform, very similar to our school, where we can learn the following from job shadowing: @@ -1259,6 +1268,7 @@ Object { }, }, "call_year": "2018", + "comments": "", "complete": true, "description": "In the three-tier school system on secondary school level in the Principality of Liechtenstein the Oberschule Eschen takes care of many students with learning difficulties and behavioural challenges. The quota of students coming from educationally deprived families and/or families with migration background is very high. Accordingly high is the proportion of risk school pupils. @@ -1356,6 +1366,7 @@ Object { }, }, "call_year": "2014", + "comments": "", "complete": true, "description": "Short summary in English: The Europäisches Institut für interkulturelle und interreligiöse Forschung stands for research projects and studies on the one hand and regular conferences and educational events on the other, esp. in the academic field of religion and culture. \\"Community Educaton\\" of course is a main part of this interest. Life long learning means an important initial process for society at all. In the context of our strategic partnership we want to come together with the other partner organizations for regular encounter and exchange concerning problems of migration and integration as weil as questions of education and social work in Belgium, Germany and Liechtenstein. Each partner will be engaged within the common project by its own possibilties and profiles. To organize our study process on different levels of \\"Community Education\\" it is necessary to use our transnational meetings and workshops to communicate and cooperate with each other in Belgium, Germany and Liechtenstein. Last but not least our strategic partnerships is based on several research reports being published iwithin our European context recently, e.g. \\"Community Education in Österreich\\" (under http://www.equi.at/dateien/CE-Endbericht_IHS.pdf).", "ec_priorities": Array [], @@ -1508,6 +1519,7 @@ Object { }, }, "call_year": "2015", + "comments": "", "complete": true, "description": "Developing an inclusive education system is not only a matter of equal rights. Access to education can put persons with disabilities on equal footing with non-disabled persons, promotes diversity within the educational environment and creates social bonds between persons with and without disabilities. It is also a means for promoting education systems with an emphasis on achieving a common learning environment guaranteeing the presence, participation and achievement of equal outcomes for all learners. Moreover, equal access to inclusive education in the mainstream improves the employment and work prospects of persons with disabilities. The ALdia project will create an innovative, comprehensive and sustainable framework that will promote the equal access of disabled students and trainees to University education and vocational training. This framework will a) define student needs in both higher education and VET sectors, b) develop and implement a flexible, massive open online course (MOOC) for higher education and VET education professionals and c) validate the acquired expertise through certification that will be based on the ECVET principles, thus facilitating its use across Europe. ALdia will provide open, practical and sharable material for institutions to develop and deliver quality equal learning opportunities. It will also create synergies and encourage cooperation between education providers at all levels, in order to have a multiplier effect in Europe and beyond. Finally, it will provide evidence for policy-making that supports the development of learning environments that foster equity and inclusion. It will thus promote the aims of the European Disability Strategy 2010-2020 and the Education and Training 2020 programme (ET2020). ALdia is expected to a) generate genuine and sustainable improvements in the education and training system and policies and b) improve the situation of young disabled people regarding attainment, social inclusion and well-being.", "ec_priorities": Array [], @@ -1658,6 +1670,7 @@ Object { }, }, "call_year": "2014", + "comments": "", "complete": true, "description": "Many young people (11-15) lack motivation to study and are not interested in personal development. Moreover, modern technologies have done significant influence on children of that age and iPads and iPhones play great role in their life keeping them busy not only after school but during the lessons and breaks as well. All this leads to the decrease of physical activity. So this combination of low motivation for personal development and decrease of physical activity we find as a big problem and threat. On the other hand, sport clubs, schools and other organizations working with young people often lack tools, skills and competence to provide activities and events attractive to everyone, encourage contacts between people from different social, ethnic, cultural groups and with different capabilities. So thought sport organizations embody a unique potential in prevention and health promotion, this potential is not developed, in many cases do not work and its impact is not visible and measurable. At the same time schools also encounter a problem of students’ indifference not only to physical activity, but to personal development as well. Our main idea is to use sport as a tool to improve social, personal and professional skills of young people (11-15) and raise awareness about positive impact of active and healthy life style. We presume that information about the need of physical activity should reach not only children, but their teachers as well and this should be done in innovative way. Our aim is to make sport and physical activity a way to improve motivation for personal development in terms of gaining physical, social and even scientific knowledge. We believe that new method of combining sport and scientific approach may be effective way to raise awareness about the necessity of physical activity and may contribute to the implementation of EU Physical Activity Guidelines (10, 13 and 23).", "ec_priorities": Array [], @@ -1872,6 +1885,7 @@ Object { }, }, "call_year": "", + "comments": "", "complete": true, "description": "Media and especially TV bombard us daily with news mainly negative, systematically ignoring many constructive and positive news that are taking place. In this way, a negative psychological attitude is promoted, resulting in intensifying pessimism. This negative psychological state has more serious repercussions to young people, who should be learning of what positive is happening around us and envisioning a better future for themselves and the world. @@ -2079,6 +2093,7 @@ Object { }, }, "call_year": "", + "comments": "", "complete": true, "description": "\\"Networks of Knowledge\\" identified mobility opportunities, promoted these across the network building an understanding of dual career practice and ultimately led to the increase of governance and dual career skills across Europe. In simple terms this project managed to IDENTIFY, UNDERSTAND, LINK, TEST and DELIVER dual career mobility structures. @@ -2299,6 +2314,7 @@ Object { }, }, "call_year": "", + "comments": "", "complete": true, "description": "\\"The project intends to develop and upgrade the existing programmes and curricula Business-Education-Programmes for Tourism at Belarus Universities with the assistance of EU partners from DE, IT, SK and CZ.Support of the reform process in the Belarusian higher education system and its integration into the European system\\" @@ -2570,6 +2586,7 @@ Object { }, }, "call_year": "", + "comments": "", "complete": true, "description": "Project „Dunavijada“ is planned as two months (July-August) project during poor summer season in Vukovar and to give an opportunity to young people from different surroundings and different countries to spend some time together, to work together and share their experience. diff --git a/services/ingestion/etl/euinvest/csv/src/lib/transform.js b/services/ingestion/etl/euinvest/csv/src/lib/transform.js index c7f398091..9157d10a5 100644 --- a/services/ingestion/etl/euinvest/csv/src/lib/transform.js +++ b/services/ingestion/etl/euinvest/csv/src/lib/transform.js @@ -344,6 +344,7 @@ export default (record: Object): Project | null => { action: '', budget: getBudget(record), call_year: '', + comments: '', description: getDescription(record), ec_priorities: [], media: getMedia(record), diff --git a/services/ingestion/etl/euinvest/csv/test/unit/lib/__snapshots__/transform.spec.js.snap b/services/ingestion/etl/euinvest/csv/test/unit/lib/__snapshots__/transform.spec.js.snap index f3a456c12..701c82ac4 100644 --- a/services/ingestion/etl/euinvest/csv/test/unit/lib/__snapshots__/transform.spec.js.snap +++ b/services/ingestion/etl/euinvest/csv/test/unit/lib/__snapshots__/transform.spec.js.snap @@ -33,6 +33,7 @@ Object { }, }, "call_year": "", + "comments": "", "complete": true, "description": "The EU-funded LIFE CERSUDS project has developed an innovative and sustainable urban drainage system that uses low-value ceramic tiles to filter water through paving. The system provides a way for cities to adapt to climate change by reducing the impact of flooding and facilitating reuse of stored water. diff --git a/services/ingestion/etl/euresults/csv/src/lib/transform.js b/services/ingestion/etl/euresults/csv/src/lib/transform.js index bc386b5c5..48eb2ebd8 100644 --- a/services/ingestion/etl/euresults/csv/src/lib/transform.js +++ b/services/ingestion/etl/euresults/csv/src/lib/transform.js @@ -408,6 +408,7 @@ export default (record: Object): Project | null => { action: '', budget: getBudget(record), call_year: '', + comments: '', description: getDescription(record), ec_priorities: getPriorities(record), media: getMedia(record), diff --git a/services/ingestion/etl/euresults/csv/test/unit/lib/__snapshots__/transform.spec.js.snap b/services/ingestion/etl/euresults/csv/test/unit/lib/__snapshots__/transform.spec.js.snap index 56ae53fd0..c62f9624d 100644 --- a/services/ingestion/etl/euresults/csv/test/unit/lib/__snapshots__/transform.spec.js.snap +++ b/services/ingestion/etl/euresults/csv/test/unit/lib/__snapshots__/transform.spec.js.snap @@ -37,6 +37,7 @@ Object { }, }, "call_year": "", + "comments": "", "complete": true, "description": "

Under the EU-funded Central European Green Corridors project, 115 fast-charging stations for electric cars have opened in Germany, Austria, Slovenia, Slovakia and Croatia – allowing cross-border electromobility with your e-car between Munich and Zagreb. Further network expansion, offers and services for customers and energy supply solutions also drive the project’s roll-out of electric transport. Partners include public sector bodies and private companies.

", "ec_priorities": Array [ @@ -372,6 +373,7 @@ Object { }, }, "call_year": "", + "comments": "", "complete": true, "description": "", "ec_priorities": Array [], diff --git a/services/ingestion/etl/fts/xls/src/lib/transform.js b/services/ingestion/etl/fts/xls/src/lib/transform.js index efef72df3..31c534a9b 100644 --- a/services/ingestion/etl/fts/xls/src/lib/transform.js +++ b/services/ingestion/etl/fts/xls/src/lib/transform.js @@ -247,6 +247,7 @@ export default (record: Object): Project | null => { action: '', budget: getBudget(record), call_year: record.Year || '', + comments: '', description: record['Budget line name and number'] || '', ec_priorities: [], media: [], diff --git a/services/ingestion/etl/fts/xls/test/unit/lib/__snapshots__/transform.spec.js.snap b/services/ingestion/etl/fts/xls/test/unit/lib/__snapshots__/transform.spec.js.snap index 019fe6a07..e4d4bdebe 100644 --- a/services/ingestion/etl/fts/xls/test/unit/lib/__snapshots__/transform.spec.js.snap +++ b/services/ingestion/etl/fts/xls/test/unit/lib/__snapshots__/transform.spec.js.snap @@ -35,6 +35,7 @@ Object { }, }, "call_year": "2017", + "comments": "", "complete": true, "description": "Optimising the integration and interconnection of transport modes and enhancing interoperability (06.02.01.03)", "ec_priorities": Array [], diff --git a/services/ingestion/etl/home/xls/src/lib/transform.js b/services/ingestion/etl/home/xls/src/lib/transform.js index bd566ced4..a73141e6b 100644 --- a/services/ingestion/etl/home/xls/src/lib/transform.js +++ b/services/ingestion/etl/home/xls/src/lib/transform.js @@ -269,6 +269,7 @@ export default (record: Object): Project | null => { action: '', budget: getBudget(record), call_year: '', + comments: '', description: getDescription(record), ec_priorities: [], media: [], diff --git a/services/ingestion/etl/home/xls/test/unit/lib/__snapshots__/transform.spec.js.snap b/services/ingestion/etl/home/xls/test/unit/lib/__snapshots__/transform.spec.js.snap index 6d13159f8..8db452e81 100644 --- a/services/ingestion/etl/home/xls/test/unit/lib/__snapshots__/transform.spec.js.snap +++ b/services/ingestion/etl/home/xls/test/unit/lib/__snapshots__/transform.spec.js.snap @@ -35,6 +35,7 @@ Object { }, }, "call_year": "", + "comments": "", "complete": true, "description": "Project Acronym: AP2016 Project Duration: 24 months diff --git a/services/ingestion/etl/iati/csv/src/lib/transform.js b/services/ingestion/etl/iati/csv/src/lib/transform.js index 53a9a68b5..010f12d9d 100644 --- a/services/ingestion/etl/iati/csv/src/lib/transform.js +++ b/services/ingestion/etl/iati/csv/src/lib/transform.js @@ -156,6 +156,7 @@ export default (record: Object): Project | null => { action: '', budget: getBudget(record), call_year: '', + comments: '', description: record.description || '', ec_priorities: [], media: [], diff --git a/services/ingestion/etl/iati/csv/test/unit/lib/__snapshots__/transform.spec.js.snap b/services/ingestion/etl/iati/csv/test/unit/lib/__snapshots__/transform.spec.js.snap index 39fa61c5f..9b30f8576 100644 --- a/services/ingestion/etl/iati/csv/test/unit/lib/__snapshots__/transform.spec.js.snap +++ b/services/ingestion/etl/iati/csv/test/unit/lib/__snapshots__/transform.spec.js.snap @@ -33,6 +33,7 @@ Object { }, }, "call_year": "", + "comments": "", "complete": true, "description": "ce contrat permet à l'établissement d'utilité publique qui assure les opérations décentraliser indirectes publiques dans la Province orientale de fonctionner", "ec_priorities": Array [], @@ -120,6 +121,7 @@ Object { }, }, "call_year": "", + "comments": "", "complete": true, "description": "", "ec_priorities": Array [], diff --git a/services/ingestion/etl/inforegio/json/src/lib/transform.js b/services/ingestion/etl/inforegio/json/src/lib/transform.js index 4a4c74345..b19c954a7 100644 --- a/services/ingestion/etl/inforegio/json/src/lib/transform.js +++ b/services/ingestion/etl/inforegio/json/src/lib/transform.js @@ -256,6 +256,7 @@ export default (record: Object): Project | null => { action: '', budget: budgetObject, call_year: '', + comments: '', description: record.quote, ec_priorities: [], media: [], diff --git a/services/ingestion/etl/inforegio/json/test/unit/lib/__snapshots__/transform.spec.js.snap b/services/ingestion/etl/inforegio/json/test/unit/lib/__snapshots__/transform.spec.js.snap index 29964966e..9341f4dc6 100644 --- a/services/ingestion/etl/inforegio/json/test/unit/lib/__snapshots__/transform.spec.js.snap +++ b/services/ingestion/etl/inforegio/json/test/unit/lib/__snapshots__/transform.spec.js.snap @@ -35,6 +35,7 @@ Object { }, }, "call_year": "", + "comments": "", "complete": true, "description": "

An extension to a boat docked on the river Danube has proved a hugely popular entertainment and cultural hub. A38, a former Ukrainian freighter ship, was initially transformed into a multi-functional space in 2003. With ERDF-funding, an adjacent second unit was constructed in 2010 and has been a resounding success.

", "ec_priorities": Array [], diff --git a/services/ingestion/etl/inforegio/xml/src/lib/transform.js b/services/ingestion/etl/inforegio/xml/src/lib/transform.js index c7b6aebfc..a2a8fed4d 100644 --- a/services/ingestion/etl/inforegio/xml/src/lib/transform.js +++ b/services/ingestion/etl/inforegio/xml/src/lib/transform.js @@ -297,6 +297,7 @@ export default (record: Object): Project | null => { action: '', budget: budgetObject, call_year: '', + comments: '', description: checkData(record.quote), ec_priorities: [], media: [], diff --git a/services/ingestion/etl/inforegio/xml/test/unit/lib/__snapshots__/transform.spec.js.snap b/services/ingestion/etl/inforegio/xml/test/unit/lib/__snapshots__/transform.spec.js.snap index b7e7c9652..6583506d5 100644 --- a/services/ingestion/etl/inforegio/xml/test/unit/lib/__snapshots__/transform.spec.js.snap +++ b/services/ingestion/etl/inforegio/xml/test/unit/lib/__snapshots__/transform.spec.js.snap @@ -33,6 +33,7 @@ Object { }, }, "call_year": "", + "comments": "", "complete": true, "description": "The colour of the sky never had much effect on the irksome rail trip between Greece's two principal cities of Athens and Thessaloniki: passengers had to wait for hours since there was only one set of tracks for trains travelling in both directions. Now all this has changed thanks to a huge programme carried out with the help of the European Union.", "ec_priorities": Array [], @@ -119,6 +120,7 @@ Object { }, }, "call_year": "", + "comments": "", "complete": true, "description": "The colour of the sky never had much effect on the irksome rail trip between Greece's two principal cities of Athens and Thessaloniki: passengers had to wait for hours since there was only one set of tracks for trains travelling in both directions. Now all this has changed thanks to a huge programme carried out with the help of the European Union.", "ec_priorities": Array [], diff --git a/services/ingestion/etl/just/csv/src/lib/transform.js b/services/ingestion/etl/just/csv/src/lib/transform.js index 0abf26878..afa850d9e 100644 --- a/services/ingestion/etl/just/csv/src/lib/transform.js +++ b/services/ingestion/etl/just/csv/src/lib/transform.js @@ -147,6 +147,7 @@ export default (record: Object): Project | null => { action: record.field_prj_cat_actions || '', budget: getBudget(record), call_year: record.field_prj_year || '', + comments: '', description: record.field_prj_summary || '', ec_priorities: [], media: [], diff --git a/services/ingestion/etl/just/csv/test/unit/lib/__snapshots__/transform.spec.js.snap b/services/ingestion/etl/just/csv/test/unit/lib/__snapshots__/transform.spec.js.snap index 877697b15..a480daf15 100644 --- a/services/ingestion/etl/just/csv/test/unit/lib/__snapshots__/transform.spec.js.snap +++ b/services/ingestion/etl/just/csv/test/unit/lib/__snapshots__/transform.spec.js.snap @@ -33,6 +33,7 @@ Object { }, }, "call_year": "2013", + "comments": "", "complete": true, "description": "The GEAR against IPV Approach (Gender Equality Awareness Raising against Intimate Partner Violence) was a coordinated action, that took place in 5 countries, namely: Croatia, Cyprus, Greece, Romania and Spain, to prevent primary and secondary Intimate Partner Violence (IPV) in adolescents’ relationships through child-centred interventions in the school or other setting that were guided by specially designed educational material. The main aim was to promote the development of healthy and equal relationships and the development of zero tolerance towards violence by raising awareness on the characteristics of healthy and unhealthy relationships, on the influence that gender stereotypical attitudes and socially imposed gender roles have on relationships and on how power inequality between the sexes is related to psychological, physical and/or sexual abuse against women/girls.", "ec_priorities": Array [], diff --git a/services/ingestion/etl/wifi4eu/xls/src/lib/transform.js b/services/ingestion/etl/wifi4eu/xls/src/lib/transform.js index e618f73b8..026a08fe5 100644 --- a/services/ingestion/etl/wifi4eu/xls/src/lib/transform.js +++ b/services/ingestion/etl/wifi4eu/xls/src/lib/transform.js @@ -186,6 +186,7 @@ export default (record: Object): Project | null => { action: '', budget: getBudget(), call_year: getCallYear(record), + comments: '', description: getProjectDescription(record), ec_priorities: [], media: [], diff --git a/services/ingestion/etl/wifi4eu/xls/test/unit/lib/__snapshots__/transform.spec.js.snap b/services/ingestion/etl/wifi4eu/xls/test/unit/lib/__snapshots__/transform.spec.js.snap index dd7495d5a..8171f369f 100644 --- a/services/ingestion/etl/wifi4eu/xls/test/unit/lib/__snapshots__/transform.spec.js.snap +++ b/services/ingestion/etl/wifi4eu/xls/test/unit/lib/__snapshots__/transform.spec.js.snap @@ -33,6 +33,7 @@ Object { }, }, "call_year": "2018", + "comments": "", "complete": true, "description": "The European Commission wishes to promote free Wi-Fi connectivity for citizens and visitors in public spaces such as parks, squares, public building, libraries, health centres, and museums everywhere in Europe through WiFi4EU.", "ec_priorities": Array [], diff --git a/types/Project.js b/types/Project.js index f839c95ba..7dd248b63 100644 --- a/types/Project.js +++ b/types/Project.js @@ -38,6 +38,12 @@ export type BudgetItem = { * @type {Budget} */ export type Budget = { + devco_equity?: BudgetItem, + devco_guarantee?: BudgetItem, + devco_interest_rate_subsidy?: BudgetItem, + devco_investment_grant?: BudgetItem, + devco_loan?: BudgetItem, + devco_ta?: BudgetItem, eu_contrib: BudgetItem, funding_area: Array, mmf_heading: string, @@ -158,6 +164,26 @@ export type Timeframe = { to_precision: TimePrecision, }; +/** + * Describes a generic field in an ETL which does not have any other specific structure. + * @type {SimpleValueField} + */ +export type SimpleValueField = { + raw: string, + formatted: string, +}; + +/** + * Describes a field which has a certain type of value. + * @type {TypedValueField} + */ +export type TypedValueField = { + field: string, + type: string, + raw: string, + formatted: string, +}; + /** * Describes `project`. * @type {Project} @@ -166,14 +192,22 @@ export type Project = { action: string, budget: Budget, call_year: string, + comments: string, + complete: boolean, description: string, + devco_arei_projects_endorsement?: SimpleValueField, + devco_cris_number?: SimpleValueField, + devco_date_entry?: string | null, + devco_lead_investor?: SimpleValueField, + devco_leverage?: SimpleValueField, + devco_project_stage?: SimpleValueField, + devco_results_indicators?: Array, ec_priorities: Array, media: Array, programme_name: string, project_id: string, project_locations: Array, project_website: string, - complete: boolean, related_links: Array, reporting_organisation: string, results: Result,