diff --git a/.env.example b/.env.example index 29d37d5c..c6adedac 100644 --- a/.env.example +++ b/.env.example @@ -6,3 +6,7 @@ AZURE_CLIENT_SECRET= AZURE_TENANT_ID= REDIS_URL=redis://localhost:6379 LOCAL_USER_EMAIL= +BIGQUERY_TABLE_NAME=events +BIGQUERY_PROJECT_ID=your-bigquery-project-name +BIGQUERY_DATASET=your-bigquery-dataset-name +BIGQUERY_API_JSON_KEY= diff --git a/Dockerfile b/Dockerfile index 8ca0426c..a11a105e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ RUN apk add --update --no-cache tzdata && \ COPY .tool-versions Gemfile Gemfile.lock ./ RUN apk add --update --no-cache --virtual build-dependances \ - postgresql-dev build-base && \ + postgresql-dev build-base git && \ apk add --update --no-cache libpq yarn && \ bundle install --jobs=4 && \ rm -rf /usr/local/bundle/cache && \ diff --git a/Gemfile b/Gemfile index 0dcc657f..2e01cea5 100644 --- a/Gemfile +++ b/Gemfile @@ -35,6 +35,7 @@ gem "concurrent-ruby", require: "concurrent" gem "concurrent-ruby-ext" gem "config", "~> 5.0" gem "devise", "~> 4.9" +gem "dfe-analytics", github: "DFE-Digital/dfe-analytics", tag: "v1.10.1" gem "flipper" gem "flipper-active_record" gem "flipper-ui" diff --git a/Gemfile.lock b/Gemfile.lock index 5a63466e..b01613ef 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,12 @@ +GIT + remote: https://github.com/DFE-Digital/dfe-analytics.git + revision: 11d75a3c92cc15439c00ea7f2f8686acffefc103 + tag: v1.10.1 + specs: + dfe-analytics (1.10.1) + google-cloud-bigquery (~> 1.38) + request_store_rails (~> 2) + GEM remote: https://rubygems.org/ specs: @@ -115,6 +124,7 @@ GEM irb (>= 1.5.0) reline (>= 0.3.1) debug_inspector (1.1.0) + declarative (0.0.20) deep_merge (1.2.2) devise (4.9.3) bcrypt (~> 3.0) @@ -193,6 +203,35 @@ GEM raabro (~> 1.4) globalid (1.2.1) activesupport (>= 6.1) + google-apis-bigquery_v2 (0.59.0) + google-apis-core (>= 0.11.0, < 2.a) + google-apis-core (0.11.1) + addressable (~> 2.5, >= 2.5.1) + googleauth (>= 0.16.2, < 2.a) + httpclient (>= 2.8.1, < 3.a) + mini_mime (~> 1.0) + representable (~> 3.0) + retriable (>= 2.0, < 4.a) + rexml + webrick + google-cloud-bigquery (1.44.2) + concurrent-ruby (~> 1.0) + google-apis-bigquery_v2 (~> 0.1) + google-cloud-core (~> 1.6) + googleauth (>= 0.16.2, < 2.a) + mini_mime (~> 1.0) + google-cloud-core (1.6.0) + google-cloud-env (~> 1.0) + google-cloud-errors (~> 1.0) + google-cloud-env (1.6.0) + faraday (>= 0.17.3, < 3.0) + google-cloud-errors (1.3.1) + googleauth (1.8.0) + faraday (>= 0.17.3, < 3.a) + jwt (>= 1.4, < 3.0) + multi_json (~> 1.11) + os (>= 0.9, < 2.0) + signet (>= 0.16, < 2.a) google-protobuf (3.24.4) govuk-components (4.1.1) html-attributes-utils (~> 1.0.0, >= 1.0.0) @@ -209,6 +248,7 @@ GEM httparty (0.21.0) mini_mime (>= 1.0.0) multi_xml (>= 0.5.2) + httpclient (2.8.3) i18n (1.14.1) concurrent-ruby (~> 1.0) importmap-rails (1.2.1) @@ -249,6 +289,7 @@ GEM mini_portile2 (2.8.5) minitest (5.20.0) msgpack (1.7.1) + multi_json (1.15.0) multi_xml (0.6.0) net-imap (0.4.2) date @@ -283,6 +324,7 @@ GEM oauth2 (>= 1.4, < 3) omniauth (~> 2.0) orm_adapter (0.5.0) + os (1.1.4) pagy (6.1.0) parallel (1.23.0) parser (3.2.2.3) @@ -345,11 +387,18 @@ GEM regexp_parser (2.8.1) reline (0.3.4) io-console (~> 0.5) + representable (3.2.0) + declarative (< 0.1.0) + trailblazer-option (>= 0.1.1, < 0.2.0) + uber (< 0.2.0) request_store (1.5.1) rack (>= 1.4) + request_store_rails (2.0.0) + concurrent-ruby (~> 1.0) responders (3.1.1) actionpack (>= 5.2) railties (>= 5.2) + retriable (3.1.2) rexml (3.2.6) rolify (6.0.1) rspec-core (3.12.2) @@ -431,6 +480,11 @@ GEM fugit (~> 1.8) globalid (>= 1.0.1) sidekiq (>= 6) + signet (0.18.0) + addressable (~> 2.8) + faraday (>= 0.17.5, < 3.a) + jwt (>= 1.5, < 3.0) + multi_json (~> 1.10) simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) @@ -442,8 +496,10 @@ GEM version_gem (~> 1.1, >= 1.1.1) thor (1.3.0) timeout (0.4.0) + trailblazer-option (0.1.2) tzinfo (2.0.6) concurrent-ruby (~> 1.0) + uber (0.1.0) uk_postcode (2.1.8) unicode-display_width (2.4.2) version_gem (1.1.3) @@ -482,6 +538,7 @@ DEPENDENCIES dartsass-rails (~> 0.5.0) debug devise (~> 4.9) + dfe-analytics! dotenv-rails factory_bot_rails faker diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index ec58aac3..7a7f7eef 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class ApplicationController < ActionController::Base + include DfE::Analytics::Requests + default_form_builder GOVUKDesignSystemFormBuilder::FormBuilder before_action :check_service_open! diff --git a/config/analytics.yml b/config/analytics.yml new file mode 100644 index 00000000..5c61ade6 --- /dev/null +++ b/config/analytics.yml @@ -0,0 +1,98 @@ +shared: + schools: + - id + - name + - headteacher_name + - created_at + application_progresses: + - id + - initial_checks_completed_at + - visa_investigation_required + - home_office_checks_completed_at + - school_investigation_required + - school_checks_completed_at + - application_id + - payment_confirmation_completed_at + - rejection_completed_at + - banking_approval_completed_at + - rejection_details + - status + - rejection_reason + applications: + - id + - application_date + - urn + - applicant_id + - created_at + - subject + - visa_type + - date_of_entry + - start_date + - application_route + - home_office_csv_downloaded_at + - standing_data_csv_downloaded_at + - payroll_csv_downloaded_at + applicants: + - id + - given_name + - family_name + - email_address + - phone_number + - passport_number + - updated_at + - middle_name + - date_of_birth + - nationality + - sex + - created_at + - school_id + - student_loan + - ip_address + addresses: + - id + - addressable_type + - addressable_id + - address_line_1 + - address_line_2 + - city + - postcode + users: + - email + qa_statuses: + - id + - application_id + - status + - date + - created_at + - updated_at + forms: + - id + - given_name + - middle_name + - family_name + - email_address + - phone_number + - date_of_birth + - nationality + - sex + - passport_number + - subject + - visa_type + - date_of_entry + - start_date + - address_line_1 + - address_line_2 + - city + - postcode + - application_route + - state_funded_secondary_school + - one_year + - school_name + - school_headteacher_name + - school_address_line_1 + - school_address_line_2 + - school_city + - school_postcode + - created_at + - updated_at + - student_loan diff --git a/config/analytics_blocklist.yml b/config/analytics_blocklist.yml new file mode 100644 index 00000000..95f110be --- /dev/null +++ b/config/analytics_blocklist.yml @@ -0,0 +1,68 @@ +--- +:shared: + :schools: + - updated_at + :duplicate_applications: + - id + - application_date + - urn + - applicant_id + - created_at + - updated_at + - subject + - visa_type + - date_of_entry + - start_date + - application_route + - home_office_csv_downloaded_at + - standing_data_csv_downloaded_at + - payroll_csv_downloaded_at + - duplicate_email + - duplicate_phone + - duplicate_passport + :application_progresses: + - created_at + - updated_at + :applications: + - updated_at + :app_settings: + - id + - service_start_date + - service_end_date + - created_at + - updated_at + :addresses: + - created_at + - updated_at + :users: + - id + - created_at + - updated_at + :audits: + - id + - auditable_id + - auditable_type + - associated_id + - associated_type + - user_id + - user_type + - username + - action + - audited_changes + - version + - comment + - remote_address + - request_uuid + - created_at + :flipper_gates: + - id + - feature_key + - key + - value + - created_at + - updated_at + :flipper_features: + - id + - key + - created_at + - updated_at diff --git a/config/analytics_pii.yml b/config/analytics_pii.yml new file mode 100644 index 00000000..90d6cb14 --- /dev/null +++ b/config/analytics_pii.yml @@ -0,0 +1,25 @@ +shared: + schools: + - headteacher_name + applicants: + - given_name + - middle_name + - family_name + - email_address + - phone_number + - passport_number + forms: + - given_name + - middle_name + - family_name + - email_address + - phone_number + - nationality + - sex + - passport_number + - visa_type + - address_line_1 + - address_line_2 + - city + - postcode + - school_headteacher_name diff --git a/config/initializers/dfe_analytics.rb b/config/initializers/dfe_analytics.rb new file mode 100644 index 00000000..662995e3 --- /dev/null +++ b/config/initializers/dfe_analytics.rb @@ -0,0 +1,65 @@ +DfE::Analytics.configure do |config| + # Whether to log events instead of sending them to BigQuery. + # + config.log_only = Rails.env.development? + + # Whether to use ActiveJob or dispatch events immediately. + # + # config.async = true + + # Which ActiveJob queue to put events on + # + config.queue = :dfe_analytics + + # The name of the BigQuery table we’re writing to. + # + # config.bigquery_table_name = ENV['BIGQUERY_TABLE_NAME'] + + # The name of the BigQuery project we’re writing to. + # + # config.bigquery_project_id = ENV['BIGQUERY_PROJECT_ID'] + + # The name of the BigQuery dataset we're writing to. + # + # config.bigquery_dataset = ENV['BIGQUERY_DATASET'] + + # Service account JSON key for the BigQuery API. See + # https://cloud.google.com/bigquery/docs/authentication/service-account-file + # + # config.bigquery_api_json_key = ENV['BIGQUERY_API_JSON_KEY'] + + # Passed directly to the retries: option on the BigQuery client + # + # config.bigquery_retries = 3 + + # Passed directly to the timeout: option on the BigQuery client + # + # config.bigquery_timeout = 120 + + # A proc which returns true or false depending on whether you want to + # enable analytics. You might want to hook this up to a feature flag or + # environment variable. + # + # config.enable_analytics = proc { true } + + # The environment we’re running in. This value will be attached + # to all events we send to BigQuery. + # + # config.environment = ENV.fetch('RAILS_ENV', 'development') + + # A proc which will be called with the user object, and which should + # return the identifier for the user. This is useful for systems with + # users that don't use the id field. + # + # config.user_identifier = proc { |user| user&.id } + + # Whether to pseudonymise the user_id field in the web request event. + # + # config.pseudonymise_web_request_user_id = false + + # A proc which will be called with the rack env, and which should + # return a boolean indicating whether the page is cached and will + # be served by rack middleware. + # + # config.rack_page_cached = proc { |_rack_env| false } +end