Merge pull request #63 from fecgov/release/sprint-6

Release sprint-6 to production

fecfile_validate_js/.eslintrc.json

@@ -0,0 +1,16 @@
+{
+ "root": true,
+ "parser": "@typescript-eslint/parser",
+ "plugins": [
+ "@typescript-eslint"
+ ],
+ "extends": [
+ "eslint:recommended",
+ "plugin:@typescript-eslint/eslint-recommended",
+ "plugin:@typescript-eslint/recommended"
+ ],
+ "rules": {
+ "@typescript-eslint/no-explicit-any": "off",
+ "no-extra-boolean-cast": "off"
+ }
+}

fecfile_validate_js/README.md

@@ -0,0 +1,46 @@
+## Developer Notes
+
+The FEC javascript validation package is a wrapper around the Ajv schema validation
+package (https://ajv.js.org/) with additional checks specific to FEC business rules.
+
+The package is distributed through the npm package management system and is, currently,
+only provided directly from the GitHub repo.
+
+To install the validation package, add this line to your package.json dependencies:
+
+"fecfile-validate": "https://github.com/fecgov/fecfile-validate#develop"
+
+## API
+
+function validate(schema, data)
+
+Returns an array of errors or an empty array if none are found.
+
+Validation error information for a single schema property:
+```
+ @typedef {object} ValidationError
+ @property {string} path - property name that failed the validation test
+ @property {string} keyword - type of validation check
+ @property {Record<string, any>} params - additional info per the type of validation check
+ @property {string | null} message - human readable message describing validation error
+ ```
+
+Error example:
+```
+[{
+ keyword: "required",
+ message: "must have required property 'form_type'",
+ params: {
+ missingProperty: "form_type",
+ },
+ path: "",
+}]
+```
+
+## Testing
+
+The test runner being used for the *.ts files is the native testing suite found in Deno (https://deno.land)
+
+To install Deno: https://deno.land/#installation
+
+Deno testing: https://deno.land/[email protected]/testing

fecfile_validate_js/scripts/buildSchemaModules.js

@@ -0,0 +1,35 @@
+/**
+ * Script to create optimized ES Module import files for each schema JSON file.
+ * This script is run as part of the build process and at postinstall.
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import * as url from 'url';
+import glob from 'glob';
+
+const dir = path.dirname(url.fileURLToPath(import.meta.url));
+const files = glob.sync(path.join(dir, '../../schema/*.json'));
+
+for (const file of files) {
+ const schema = JSON.parse(fs.readFileSync(file, 'utf-8'));
+
+ // Ajv package is looking for 'http' in the schema definition URL and errors on 'https'
+ const theSchemaUrl = schema['$schema'];
+ schema['$schema'] = theSchemaUrl.replace('https', 'http');
+
+ // Remove properties unnecessary for validation to reduce bundle size
+ for (let n in schema.properties) {
+ delete schema.description;
+ delete schema.properties[n].title;
+ delete schema.properties[n].description;
+ delete schema.properties[n].examples;
+ delete schema.properties[n].fec_spec;
+ }
+
+ // And write the module definition and an accompanying typescript information (.d.ts) file
+ const schemaModuleContent = 'export const schema = ' + JSON.stringify(schema);
+ const baseFilename = path.basename(file).replace('.json', '');
+ fs.writeFileSync(path.join(dir, `../dist/${baseFilename}.js`), schemaModuleContent);
+ fs.writeFileSync(path.join(dir, `../dist/${baseFilename}.d.ts`), 'export declare const schema: any;');
+}

fecfile_validate_js/src/index.ts

@@ -0,0 +1,56 @@
+/**
+ * Package that wraps the functionality of the Ajv JSON Schema validator to
+ * meet FEC specific use cases.
+ *
+ * Compiled into dist/index.js
+ * Tested with spec/index-spec.js
+ */
+
+import Ajv, { ValidateFunction } from 'ajv';
+
+/**
+ * Validation error information for a single schema property
+ * @typedef {object} ValidationError
+ * @property {string} path - property name that failed the validation test
+ * @property {string} keyword - type of validation check
+ * @property {Record<string, any>} params - additional info per the type of validation check
+ * @property {string | null} message - human readable message describing validation error
+ */
+export type ValidationError = {
+ path: string;
+ keyword: string;
+ params: Record<string, any>;
+ message: string | null;
+};
+
+const ajv = new Ajv({ allErrors: true, strictSchema: false });
+
+/**
+ * Takes a schema in JSON format and data object to be validated and returns an
+ * array of ValidationError objects for all validation errors found by Ajv.
+ *
+ * @param {object} schema
+ * @param {object} data
+ * @returns {ValidationError[]} Modified version of Ajv output, empty array if no errors found
+ */
+export function validate(schema: any, data: any): ValidationError[] {
+ const theSchemaUrl = schema['$schema'];
+ schema['$schema'] = theSchemaUrl.replace('https', 'http');
+
+ const validator: ValidateFunction = ajv.compile(schema);
+ const isValid: boolean = validator(data);
+ const errors: ValidationError[] = [];
+
+ if (!isValid && !!validator.errors) {
+ validator.errors.forEach((err) => {
+ errors.push({
+ path: err.instancePath.substring(1),
+ keyword: err.keyword,
+ params: err.params,
+ message: !!err.message ? err.message : null,
+ });
+ });
+ }
+
+ return errors;
+}

fecfile_validate_js/tests/index.test.ts

@@ -0,0 +1,200 @@
+import { assertEquals } from 'https://deno.land/[email protected]/testing/asserts.ts';
+import { validate } from '../dist/index.js';
+import { schema as f3xSchema } from '../dist/F3X.js';
+
+const perfectForm_F3X: any = {
+ form_type: 'F3XA',
+ filer_committee_id_number: 'C00123456',
+ committee_name: 'Foes of Chris',
+ change_of_address: false,
+ street_1: '123 main street',
+ street_2: '',
+ city: 'Best Town',
+ state: 'DC',
+ zip: '20000',
+ report_code: '',
+ election_code: '',
+ date_of_election: '20021101',
+ state_of_election: 'DC',
+ coverage_from_date: '20000101',
+ coverage_through_date: '20000201',
+ qualified_committee: true,
+ treasurer_last_name: 'Doe',
+ treasurer_first_name: 'J',
+ treasurer_middle_name: 'X',
+ treasurer_prefix: 'Dr',
+ treasurer_suffix: 'PhD',
+ date_signed: '20040729',
+ L6b_cash_on_hand_beginning_period: 1,
+};
+
+Deno.test({
+ name: 'it should pass with perfect data',
+ fn: () => {
+ const result = validate(f3xSchema, perfectForm_F3X);
+ assertEquals(result, []);
+ },
+});
+
+Deno.test({
+ name: 'it should fail without form_type',
+ fn: () => {
+ const thisData = { ...perfectForm_F3X };
+ delete thisData.form_type;
+ const result = validate(f3xSchema, thisData);
+ assertEquals(result[0].keyword, 'required');
+ assertEquals(result[0].params.missingProperty, 'form_type');
+ },
+});
+
+Deno.test({
+ name: 'committee_name should allow accents',
+ fn: () => {
+ const thisData = { ...perfectForm_F3X, ...{ committee_name: 'Éàñ!@#$%^&*()_+-=[]\\{}|;,./<>?' } };
+ const result = validate(f3xSchema, thisData);
+ assertEquals(result, []);
+ },
+});
+
+Deno.test({
+ name: "filer_committee_id_number should not be ''",
+ fn: () => {
+ const thisData = { ...perfectForm_F3X, ...{ filer_committee_id_number: '' } };
+ // TODO: the next line is the goal. Ajv sees '' as a value and we want it to be evaluated as null
+ // expect(validate(f3xSchema, thisData).errors).toEqual([ 'filer_committee_id_number should not be empty' ]);
+ const result = validate(f3xSchema, thisData);
+ assertEquals(result[0].path, 'filer_committee_id_number');
+ assertEquals(result[0].keyword, 'minLength');
+ assertEquals(result[0].message, 'must NOT have fewer than 9 characters');
+ },
+});
+
+Deno.test({
+ name: 'filer_committee_id_number is too short',
+ fn: () => {
+ const thisData = { ...perfectForm_F3X, ...{ filer_committee_id_number: '12345678' } };
+ const result = validate(f3xSchema, thisData);
+ assertEquals(result[0].path, 'filer_committee_id_number');
+ assertEquals(result[0].keyword, 'minLength');
+ assertEquals(result[0].message, 'must NOT have fewer than 9 characters');
+ },
+});
+
+Deno.test({
+ name: 'filer_committee_id_number is too long',
+ fn: () => {
+ const thisData = { ...perfectForm_F3X, ...{ filer_committee_id_number: '1234567890' } };
+ const result = validate(f3xSchema, thisData);
+ assertEquals(result[0].path, 'filer_committee_id_number');
+ assertEquals(result[0].keyword, 'maxLength');
+ assertEquals(result[0].message, 'must NOT have more than 9 characters');
+ },
+});
+
+Deno.test({
+ name: 'filer_committee_id_number violates the pattern',
+ fn: () => {
+ const thisData = { ...perfectForm_F3X, ...{ filer_committee_id_number: 'X23456789' } };
+ const result = validate(f3xSchema, thisData);
+ assertEquals(result[0].path, 'filer_committee_id_number');
+ assertEquals(result[0].keyword, 'pattern');
+ assertEquals(result[0].message, 'must match pattern "^[C|P][0-9]{8}$|^[H|S][0-9]{1}[A-Z]{2}[0-9]{5}$"');
+ },
+});
+
+Deno.test({
+ name: 'filer_committee_id_number is required',
+ fn: () => {
+ const thisData = { ...perfectForm_F3X };
+ delete thisData.filer_committee_id_number;
+ const result = validate(f3xSchema, thisData);
+ assertEquals(result[0].keyword, 'required');
+ assertEquals(result[0].params.missingProperty, 'filer_committee_id_number');
+ },
+});
+
+Deno.test({
+ name: 'state should be exactly two letters',
+ fn: () => {
+ const thisData = { ...perfectForm_F3X, ...{ state: '12X' } };
+ const result = validate(f3xSchema, thisData);
+ assertEquals(result[1].keyword, 'pattern');
+ assertEquals(result[1].message, 'must match pattern "^[A-Z]{2}$"');
+ },
+});
+
+Deno.test({
+ name: 'state should be only letters',
+ fn: () => {
+ const thisData = { ...perfectForm_F3X, ...{ state: '1L' } };
+ const result = validate(f3xSchema, thisData);
+ assertEquals(result[0].keyword, 'pattern');
+ assertEquals(result[0].message, 'must match pattern "^[A-Z]{2}$"');
+ },
+});
+
+Deno.test({
+ name: "treasurer_first_name should not be ''",
+ fn: () => {
+ const thisData = { ...perfectForm_F3X, ...{ treasurer_first_name: '' } };
+ const result = validate(f3xSchema, thisData);
+ assertEquals(result[0].path, 'treasurer_first_name');
+ assertEquals(result[0].keyword, 'minLength');
+ assertEquals(result[0].message, 'must NOT have fewer than 1 characters');
+ },
+});
+
+Deno.test({
+ name: 'treasurer_first_name is required',
+ fn: () => {
+ const thisData = { ...perfectForm_F3X };
+ delete thisData.treasurer_first_name;
+ const result = validate(f3xSchema, thisData);
+ assertEquals(result[0].keyword, 'required');
+ assertEquals(result[0].params.missingProperty, 'treasurer_first_name');
+ },
+});
+
+Deno.test({
+ name: "treasurer_last_name should not be ''",
+ fn: () => {
+ const thisData = { ...perfectForm_F3X, ...{ treasurer_last_name: '' } };
+ const result = validate(f3xSchema, thisData);
+ assertEquals(result[0].path, 'treasurer_last_name');
+ assertEquals(result[0].keyword, 'minLength');
+ assertEquals(result[0].message, 'must NOT have fewer than 1 characters');
+ },
+});
+
+Deno.test({
+ name: 'treasurer_last_name is required',
+ fn: () => {
+ const thisData = { ...perfectForm_F3X };
+ delete thisData.treasurer_last_name;
+ const result = validate(f3xSchema, thisData);
+ assertEquals(result[0].keyword, 'required');
+ assertEquals(result[0].params.missingProperty, 'treasurer_last_name');
+ },
+});
+
+Deno.test({
+ name: "date_signed should not be ''",
+ fn: () => {
+ const thisData = { ...perfectForm_F3X, ...{ date_signed: '' } };
+ const result = validate(f3xSchema, thisData);
+ assertEquals(result[0].path, 'date_signed');
+ assertEquals(result[0].keyword, 'minLength');
+ assertEquals(result[0].message, 'must NOT have fewer than 8 characters');
+ },
+});
+
+Deno.test({
+ name: 'date_signed is required',
+ fn: () => {
+ const thisData = { ...perfectForm_F3X };
+ delete thisData.date_signed;
+ const result = validate(f3xSchema, thisData);
+ assertEquals(result[0].keyword, 'required');
+ assertEquals(result[0].params.missingProperty, 'date_signed');
+ },
+});

fecfile_validate_js/tsconfig.json

@@ -0,0 +1,12 @@
+{
+ "compilerOptions": {
+ "target": "es2017",
+ "module": "es2020",
+ "moduleResolution": "node",
+ "declaration": true,
+ "outDir": "dist",
+ "strict": true
+ },
+ "include": ["src"],
+ "exclude": ["node_modules", "**/__tests__/*"]
+}

fecfile_validate_python/tests/test_validate.py

@@ -46,13 +46,13 @@ def test_missing_required_field(sample_f3x):
 
 
 def test_invalid_string_character(sample_f3x):
- # Create error by adding a '$' to COMMITTEE_NAME
- sample_f3x["committee_name"] = "Foe$ of Pat"
- message_match = "'Foe$ of Pat' does not match '^[ A-Za-z0-9]{0,200}$'"
+ # Create error by setting COMMITTEE_NAME to a 201 char string
+ sample_f3x["committee_name"] = "a" * 201
+ message_end = "is too long"
 
  validation_result = validate.validate("F3X", sample_f3x)
  assert validation_result.errors[0].path == "committee_name"
- assert validation_result.errors[0].message == message_match
+ assert validation_result.errors[0].message.endswith(message_end)
 
 
 def test_non_required_field(sample_f3x):