diff --git a/.env.example b/.env.example index 29d37d5c..c6adedac 100644 --- a/.env.example +++ b/.env.example @@ -6,3 +6,7 @@ AZURE_CLIENT_SECRET= AZURE_TENANT_ID= REDIS_URL=redis://localhost:6379 LOCAL_USER_EMAIL= +BIGQUERY_TABLE_NAME=events +BIGQUERY_PROJECT_ID=your-bigquery-project-name +BIGQUERY_DATASET=your-bigquery-dataset-name +BIGQUERY_API_JSON_KEY= diff --git a/Gemfile b/Gemfile index 73ff0fd1..2b53e0ea 100644 --- a/Gemfile +++ b/Gemfile @@ -35,6 +35,7 @@ gem "phonelib" gem "audited", "~> 5.3", ">= 5.3.3" gem "config", "~> 4.2" gem "devise", "~> 4.9" +gem "dfe-analytics", github: "DFE-Digital/dfe-analytics", tag: "v1.10.1" gem "httparty", "~> 0.21" gem "invisible_captcha" gem "omniauth-azure-activedirectory-v2" diff --git a/Gemfile.lock b/Gemfile.lock index 893af07c..e4ad37de 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,12 @@ +GIT + remote: https://github.com/DFE-Digital/dfe-analytics.git + revision: 11d75a3c92cc15439c00ea7f2f8686acffefc103 + tag: v1.10.1 + specs: + dfe-analytics (1.10.1) + google-cloud-bigquery (~> 1.38) + request_store_rails (~> 2) + GEM remote: https://rubygems.org/ specs: @@ -109,6 +118,7 @@ GEM irb (>= 1.5.0) reline (>= 0.3.1) debug_inspector (1.1.0) + declarative (0.0.20) deep_merge (1.2.2) devise (4.9.2) bcrypt (~> 3.0) @@ -175,6 +185,35 @@ GEM raabro (~> 1.4) globalid (1.2.1) activesupport (>= 6.1) + google-apis-bigquery_v2 (0.59.0) + google-apis-core (>= 0.11.0, < 2.a) + google-apis-core (0.11.1) + addressable (~> 2.5, >= 2.5.1) + googleauth (>= 0.16.2, < 2.a) + httpclient (>= 2.8.1, < 3.a) + mini_mime (~> 1.0) + representable (~> 3.0) + retriable (>= 2.0, < 4.a) + rexml + webrick + google-cloud-bigquery (1.44.2) + concurrent-ruby (~> 1.0) + google-apis-bigquery_v2 (~> 0.1) + google-cloud-core (~> 1.6) + googleauth (>= 0.16.2, < 2.a) + mini_mime (~> 1.0) + google-cloud-core (1.6.0) + google-cloud-env (~> 1.0) + google-cloud-errors (~> 1.0) + google-cloud-env (1.6.0) + faraday (>= 0.17.3, < 3.0) + google-cloud-errors (1.3.1) + googleauth (1.8.0) + faraday (>= 0.17.3, < 3.a) + jwt (>= 1.4, < 3.0) + multi_json (~> 1.11) + os (>= 0.9, < 2.0) + signet (>= 0.16, < 2.a) govuk-components (4.1.1) html-attributes-utils (~> 1.0.0, >= 1.0.0) pagy (~> 6.0) @@ -190,6 +229,7 @@ GEM httparty (0.21.0) mini_mime (>= 1.0.0) multi_xml (>= 0.5.2) + httpclient (2.8.3) i18n (1.14.1) concurrent-ruby (~> 1.0) invisible_captcha (2.1.0) @@ -227,6 +267,7 @@ GEM mini_portile2 (2.8.4) minitest (5.20.0) msgpack (1.7.1) + multi_json (1.15.0) multi_xml (0.6.0) net-imap (0.3.7) date @@ -261,6 +302,7 @@ GEM oauth2 (>= 1.4, < 3) omniauth (~> 2.0) orm_adapter (0.5.0) + os (1.1.4) pagy (6.0.4) parallel (1.23.0) parser (3.2.2.3) @@ -320,11 +362,18 @@ GEM regexp_parser (2.8.1) reline (0.3.4) io-console (~> 0.5) + representable (3.2.0) + declarative (< 0.1.0) + trailblazer-option (>= 0.1.1, < 0.2.0) + uber (< 0.2.0) request_store (1.5.1) rack (>= 1.4) + request_store_rails (2.0.0) + concurrent-ruby (~> 1.0) responders (3.1.0) actionpack (>= 5.2) railties (>= 5.2) + retriable (3.1.2) rexml (3.2.6) rspec-core (3.12.2) rspec-support (~> 3.12.0) @@ -400,6 +449,11 @@ GEM fugit (~> 1.8) globalid (>= 1.0.1) sidekiq (>= 6) + signet (0.18.0) + addressable (~> 2.8) + faraday (>= 0.17.5, < 3.a) + jwt (>= 1.5, < 3.0) + multi_json (~> 1.10) simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) @@ -418,8 +472,10 @@ GEM sprockets (>= 3.0.0) thor (1.2.2) timeout (0.4.0) + trailblazer-option (0.1.2) tzinfo (2.0.6) concurrent-ruby (~> 1.0) + uber (0.1.0) uk_postcode (2.1.8) unicode-display_width (2.4.2) version_gem (1.1.3) @@ -439,6 +495,7 @@ GEM rack-proxy (>= 0.6.1) railties (>= 5.2) semantic_range (>= 2.3.0) + webrick (1.8.1) websocket-driver (0.7.6) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) @@ -460,6 +517,7 @@ DEPENDENCIES config (~> 4.2) debug devise (~> 4.9) + dfe-analytics! dotenv-rails factory_bot_rails faker diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 848035d1..9e1d6897 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class ApplicationController < ActionController::Base + include DfE::Analytics::Requests + default_form_builder GOVUKDesignSystemFormBuilder::FormBuilder before_action :check_service_open! diff --git a/config/analytics.yml b/config/analytics.yml new file mode 100644 index 00000000..595a8c12 --- /dev/null +++ b/config/analytics.yml @@ -0,0 +1,59 @@ +shared: + schools: + - id + - name + - headteacher_name + - created_at + application_progresses: + - id + - initial_checks_completed_at + - visa_investigation_required + - home_office_checks_completed_at + - school_investigation_required + - school_checks_completed_at + - application_id + - payment_confirmation_completed_at + - rejection_completed_at + - banking_approval_completed_at + - rejection_details + - status + - rejection_reason + applications: + - id + - application_date + - urn + - applicant_id + - created_at + - subject + - visa_type + - date_of_entry + - start_date + - application_route + - home_office_csv_downloaded_at + - standing_data_csv_downloaded_at + - payroll_csv_downloaded_at + applicants: + - id + - given_name + - family_name + - email_address + - phone_number + - passport_number + - updated_at + - middle_name + - date_of_birth + - nationality + - sex + - created_at + - school_id + - student_loan + addresses: + - id + - addressable_type + - addressable_id + - address_line_1 + - address_line_2 + - city + - postcode + users: + - email diff --git a/config/analytics_blocklist.yml b/config/analytics_blocklist.yml new file mode 100644 index 00000000..35c3ecba --- /dev/null +++ b/config/analytics_blocklist.yml @@ -0,0 +1,40 @@ +--- +:shared: + :schools: + - updated_at + :duplicate_applications: + - id + - application_date + - urn + - applicant_id + - created_at + - updated_at + - subject + - visa_type + - date_of_entry + - start_date + - application_route + - home_office_csv_downloaded_at + - standing_data_csv_downloaded_at + - payroll_csv_downloaded_at + - duplicate_email + - duplicate_phone + - duplicate_passport + :application_progresses: + - created_at + - updated_at + :applications: + - updated_at + :app_settings: + - id + - service_start_date + - service_end_date + - created_at + - updated_at + :addresses: + - created_at + - updated_at + :users: + - id + - created_at + - updated_at diff --git a/config/analytics_pii.yml b/config/analytics_pii.yml new file mode 100644 index 00000000..efda83f5 --- /dev/null +++ b/config/analytics_pii.yml @@ -0,0 +1,10 @@ +shared: + schools: + - headteacher_name + applicants: + - given_name + - middle_name + - family_name + - email_address + - phone_number + - passport_number diff --git a/config/initializers/dfe_analytics.rb b/config/initializers/dfe_analytics.rb new file mode 100644 index 00000000..662995e3 --- /dev/null +++ b/config/initializers/dfe_analytics.rb @@ -0,0 +1,65 @@ +DfE::Analytics.configure do |config| + # Whether to log events instead of sending them to BigQuery. + # + config.log_only = Rails.env.development? + + # Whether to use ActiveJob or dispatch events immediately. + # + # config.async = true + + # Which ActiveJob queue to put events on + # + config.queue = :dfe_analytics + + # The name of the BigQuery table we’re writing to. + # + # config.bigquery_table_name = ENV['BIGQUERY_TABLE_NAME'] + + # The name of the BigQuery project we’re writing to. + # + # config.bigquery_project_id = ENV['BIGQUERY_PROJECT_ID'] + + # The name of the BigQuery dataset we're writing to. + # + # config.bigquery_dataset = ENV['BIGQUERY_DATASET'] + + # Service account JSON key for the BigQuery API. See + # https://cloud.google.com/bigquery/docs/authentication/service-account-file + # + # config.bigquery_api_json_key = ENV['BIGQUERY_API_JSON_KEY'] + + # Passed directly to the retries: option on the BigQuery client + # + # config.bigquery_retries = 3 + + # Passed directly to the timeout: option on the BigQuery client + # + # config.bigquery_timeout = 120 + + # A proc which returns true or false depending on whether you want to + # enable analytics. You might want to hook this up to a feature flag or + # environment variable. + # + # config.enable_analytics = proc { true } + + # The environment we’re running in. This value will be attached + # to all events we send to BigQuery. + # + # config.environment = ENV.fetch('RAILS_ENV', 'development') + + # A proc which will be called with the user object, and which should + # return the identifier for the user. This is useful for systems with + # users that don't use the id field. + # + # config.user_identifier = proc { |user| user&.id } + + # Whether to pseudonymise the user_id field in the web request event. + # + # config.pseudonymise_web_request_user_id = false + + # A proc which will be called with the rack env, and which should + # return a boolean indicating whether the page is cached and will + # be served by rack middleware. + # + # config.rack_page_cached = proc { |_rack_env| false } +end