From 068cbeedf620b546b5fc755683617d6fc7a8ff08 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Sat, 9 Apr 2022 12:16:45 -0400 Subject: [PATCH 01/15] Added candidate state and district dynamic validation rules to schema --- .../tests/sample_contact_CAN.json | 23 +++++ ...D_contact.json => sample_contact_IND.json} | 2 +- .../tests/test_contact_schema.py | 83 +++++++++++++++++++ .../tests/test_validate.py | 22 ----- schema/Contact_Candidate.json | 49 ++++++++--- 5 files changed, 145 insertions(+), 34 deletions(-) create mode 100644 fecfile_validate_python/tests/sample_contact_CAN.json rename fecfile_validate_python/tests/{sample_IND_contact.json => sample_contact_IND.json} (94%) create mode 100644 fecfile_validate_python/tests/test_contact_schema.py diff --git a/fecfile_validate_python/tests/sample_contact_CAN.json b/fecfile_validate_python/tests/sample_contact_CAN.json new file mode 100644 index 00000000..d3affd6f --- /dev/null +++ b/fecfile_validate_python/tests/sample_contact_CAN.json @@ -0,0 +1,23 @@ +{ + "type": "CAN", + "candidate_id": "H01234567", + "committee_id": "C00601211", + "name": "Gilbert Smith", + "last_name": "Smith", + "first_name": "Gilbert", + "middle_name": null, + "prefix": null, + "suffix": null, + "street_1": "602 Tlumacki St", + "street_2": null, + "city": "Mclean City", + "state": "VA", + "zip": "22204", + "employer": null, + "occupation": null, + "candidate_office": "P", + "candidate_state": null, + "candidate_district": null, + "telephone": "3043892120", + "country": "US" +} diff --git a/fecfile_validate_python/tests/sample_IND_contact.json b/fecfile_validate_python/tests/sample_contact_IND.json similarity index 94% rename from fecfile_validate_python/tests/sample_IND_contact.json rename to fecfile_validate_python/tests/sample_contact_IND.json index db633263..b2a80461 100644 --- a/fecfile_validate_python/tests/sample_IND_contact.json +++ b/fecfile_validate_python/tests/sample_contact_IND.json @@ -15,7 +15,7 @@ "zip": "22204", "employer": "Byron Inc", "occupation": "Business Owner", - "candidate_office": "P", + "candidate_office": null, "candidate_state": null, "candidate_district": null, "telephone": "3043892120", diff --git a/fecfile_validate_python/tests/test_contact_schema.py b/fecfile_validate_python/tests/test_contact_schema.py new file mode 100644 index 00000000..e854e4b4 --- /dev/null +++ b/fecfile_validate_python/tests/test_contact_schema.py @@ -0,0 +1,83 @@ +import pytest +import json +import os +from jsonschema import Draft7Validator +from src.fecfile_validate import validate +import pdb + +@pytest.fixture +def sample_ind_contact(): + with open(os.path.join(os.path.dirname(__file__), + "sample_contact_IND.json")) as f: + form_data = json.load(f) + return form_data + + +@pytest.fixture +def sample_can_contact(): + with open(os.path.join(os.path.dirname(__file__), + "sample_contact_CAN.json")) as f: + form_data = json.load(f) + return form_data + + +def test_invalid_const_value(sample_ind_contact): + # Make sure our Individual Contact schema is valid + validation_result = validate.validate("Contact_Individual", + sample_ind_contact) + assert validation_result.errors == [] + + # Check the const type property works by setting an invalid "type" property + sample_ind_contact["type"] = "Individual" + validation_result = validate.validate("Contact_Individual", + sample_ind_contact) + assert validation_result.errors[0].path == "type" + assert validation_result.errors[0].message == "'IND' was expected" + + +def test_required_candidate_state(sample_can_contact): + # Make sure our presidential candidate contact schema is valid + validation_result = validate.validate("Contact_Candidate", + sample_can_contact) + assert validation_result.errors == [] + + + # Fail null for candidate state when office is S + sample_can_contact["candidate_office"] = "S" + validation_result = validate.validate("Contact_Candidate", + sample_can_contact) + assert validation_result.errors[0].path == "candidate_state" + assert validation_result.errors[0].message == "None is not of type 'string'" + + + # Fail not having candidate state at all when office is S + del sample_can_contact["candidate_state"] + validation_result = validate.validate("Contact_Candidate", + sample_can_contact) + assert validation_result.errors[0].path == "candidate_state" + assert validation_result.errors[0].message == "'candidate_state' is a required property" + + + # Fail null for candidate district when office is H + sample_can_contact["candidate_state"] = "VA" + sample_can_contact["candidate_office"] = "H" + validation_result = validate.validate("Contact_Candidate", + sample_can_contact) + assert validation_result.errors[0].path == "candidate_district" + assert validation_result.errors[0].message == "None is not of type 'string'" + + + # Fail invalid candidate district code + sample_can_contact["candidate_district"] = "ZZ" + validation_result = validate.validate("Contact_Candidate", + sample_can_contact) + assert validation_result.errors[0].path == "candidate_district" + assert validation_result.errors[0].message == "'ZZ' does not match '^[0-9]{2}$'" + + + # Fail not having a candidate district when office is H + del sample_can_contact["candidate_district"] + validation_result = validate.validate("Contact_Candidate", + sample_can_contact) + assert validation_result.errors[0].path == "candidate_district" + assert validation_result.errors[0].message == "'candidate_district' is a required property" diff --git a/fecfile_validate_python/tests/test_validate.py b/fecfile_validate_python/tests/test_validate.py index 4fd4ee4f..d61a04dd 100644 --- a/fecfile_validate_python/tests/test_validate.py +++ b/fecfile_validate_python/tests/test_validate.py @@ -12,14 +12,6 @@ def sample_f3x(): return form_data -@pytest.fixture -def sample_ind_contact(): - with open(os.path.join(os.path.dirname(__file__), - "sample_IND_contact.json")) as f: - form_data = json.load(f) - return form_data - - @pytest.fixture def test_schema(): with open(os.path.join(os.path.dirname(__file__), @@ -95,17 +87,3 @@ def test_parse_required_error(test_schema): "'nested_field' is a required property", "top_level_field.nested_field", ) - - -def test_invalid_const_value(sample_ind_contact): - # Make sure our Individual Contact schema is valid - validation_result = validate.validate("Contact_Individual", - sample_ind_contact) - assert validation_result.errors == [] - - # Check the const type property works by setting an invalid "type" property - sample_ind_contact["type"] = "Individual" - validation_result = validate.validate("Contact_Individual", - sample_ind_contact) - assert validation_result.errors[0].path == "type" - assert validation_result.errors[0].message == "'IND' was expected" diff --git a/schema/Contact_Candidate.json b/schema/Contact_Candidate.json index cb4db2cc..4ede399e 100644 --- a/schema/Contact_Candidate.json +++ b/schema/Contact_Candidate.json @@ -15,8 +15,6 @@ "state", "zip", "candidate_office", - "candidate_state", - "candidate_district", "country" ], "fec_recommended": [], @@ -222,9 +220,9 @@ "title": "STATE", "description": "", "type": "string", - "minLength": 0, + "minLength": 2, "maxLength": 2, - "pattern": "^[ A-Za-z0-9]{0,2}$", + "pattern": "^[A-Z]{2}$", "examples": ["WA"], "fec_spec": { "COL_SEQ": 11, @@ -319,10 +317,10 @@ "candidate_state": { "title": "CANDIDATE STATE", "description": "", - "type": "string", - "minLength": 0, + "type": ["string", "null"], + "minLength": 2, "maxLength": 2, - "pattern": "^[ A-Za-z0-9]{0,2}$", + "pattern": "^[A-Z]{2}$", "examples": ["WA"], "fec_spec": { "COL_SEQ": 16, @@ -339,10 +337,10 @@ "candidate_district": { "title": "CANDIDATE DISTRICT", "description": "", - "type": "string", - "minLength": 0, + "type": ["string", "null"], + "minLength": 2, "maxLength": 2, - "pattern": "^[ A-Za-z0-9]{0,2}$", + "pattern": "^[0-9]{2}$", "fec_spec": { "COL_SEQ": 17, "FIELD_DESCRIPTION": "CANDIDATE DISTRICT", @@ -390,5 +388,34 @@ "FIELD_FORM_ASSOCIATION": null } } - } + }, + "allOf": [ + { + "if": { + "properties": { + "candidate_office": { "const": "S" } + }, + "required": ["candidate_office"] + }, + "then": { + "required": ["candidate_state"], + "properties": { "candidate_state": { "type": "string" } } + } + }, + { + "if": { + "properties": { + "candidate_office": { "const": "H" } + }, + "required": ["candidate_office"] + }, + "then": { + "required": ["candidate_state", "candidate_district"], + "properties": { + "candidate_state": { "type": "string" }, + "candidate_district": { "type": "string" } + } + } + } + ] } From 1d94f244eaded766cd4de54bbfdc1c5e13710955 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Sat, 9 Apr 2022 12:50:56 -0400 Subject: [PATCH 02/15] Added js unit testing for new candidate contact schema --- fecfile_validate_js/.eslintrc.json | 1 - fecfile_validate_js/tests/candidate.test.ts | 140 ++++++++++++++++++ .../tests/{index.test.ts => f3x.test.ts} | 0 package.json | 2 +- 4 files changed, 141 insertions(+), 2 deletions(-) create mode 100644 fecfile_validate_js/tests/candidate.test.ts rename fecfile_validate_js/tests/{index.test.ts => f3x.test.ts} (100%) diff --git a/fecfile_validate_js/.eslintrc.json b/fecfile_validate_js/.eslintrc.json index 963b2965..a8b313d0 100644 --- a/fecfile_validate_js/.eslintrc.json +++ b/fecfile_validate_js/.eslintrc.json @@ -6,7 +6,6 @@ ], "extends": [ "eslint:recommended", - "plugin:@typescript-eslint/eslint-recommended", "plugin:@typescript-eslint/recommended" ], "rules": { diff --git a/fecfile_validate_js/tests/candidate.test.ts b/fecfile_validate_js/tests/candidate.test.ts new file mode 100644 index 00000000..35487e58 --- /dev/null +++ b/fecfile_validate_js/tests/candidate.test.ts @@ -0,0 +1,140 @@ +import { assertEquals } from 'https://deno.land/std@0.133.0/testing/asserts.ts'; +import { validate } from '../dist/index.js'; +import { schema as candidateContactSchema } from '../dist/Contact_Candidate.js'; + +const data: any = { + type: 'CAN', + candidate_id: 'H01234567', + committee_id: 'C00601211', + name: 'Gilbert Smith', + last_name: 'Smith', + first_name: 'Gilbert', + middle_name: null, + prefix: null, + suffix: null, + street_1: '602 Tlumacki St', + street_2: null, + city: 'Mclean City', + state: 'VA', + zip: '22204', + employer: null, + occupation: null, + candidate_office: 'P', + candidate_state: null, + candidate_district: null, + telephone: '3043892120', + country: 'US', +}; + +Deno.test({ + name: 'it should pass with perfect data', + fn: () => { + const result = validate(candidateContactSchema, data); + assertEquals(result, []); + }, +}); + +Deno.test({ + name: 'it should fail as S candidate office missing', + fn: () => { + const testData = { ...data }; + delete testData.candidate_office; + const result = validate(candidateContactSchema, testData); + assertEquals(result[0].keyword, 'required'); + assertEquals(result[0].message, "must have required property 'candidate_office'"); + }, +}); + +Deno.test({ + name: 'it should fail as S candidate office and null for candidate_state', + fn: () => { + const testData = { ...data }; + testData.candidate_office = 'S'; + const result = validate(candidateContactSchema, testData); + assertEquals(result[0].path, 'candidate_state'); + assertEquals(result[0].message, 'must be string'); + }, +}); + +Deno.test({ + name: 'it should fail as S candidate office and candidate_state missing', + fn: () => { + const testData = { ...data }; + testData.candidate_office = 'S'; + delete testData.candidate_state; + const result = validate(candidateContactSchema, testData); + assertEquals(result[0].keyword, 'required'); + assertEquals(result[0].message, "must have required property 'candidate_state'"); + }, +}); + +Deno.test({ + name: 'it should pass with for candidate office S with a candidate state', + fn: () => { + const testData = { ...data }; + testData.candidate_office = 'S'; + testData.candidate_state = 'VA'; + const result = validate(candidateContactSchema, testData); + assertEquals(result, []); + }, +}); + +Deno.test({ + name: 'it should fail as H candidate office and null for candidate_state', + fn: () => { + const testData = { ...data }; + testData.candidate_office = 'H'; + const result = validate(candidateContactSchema, testData); + assertEquals(result[0].path, 'candidate_state'); + assertEquals(result[0].message, 'must be string'); + }, +}); + +Deno.test({ + name: 'it should fail as H candidate office and candidate_state missing', + fn: () => { + const testData = { ...data }; + testData.candidate_office = 'H'; + delete testData.candidate_state; + const result = validate(candidateContactSchema, testData); + assertEquals(result[0].keyword, 'required'); + assertEquals(result[0].message, "must have required property 'candidate_state'"); + }, +}); + +Deno.test({ + name: 'it should fail as H candidate office and null for candidate_district', + fn: () => { + const testData = { ...data }; + testData.candidate_office = 'H'; + testData.candidate_state = 'MD'; + const result = validate(candidateContactSchema, testData); + assertEquals(result[0].path, 'candidate_district'); + assertEquals(result[0].message, 'must be string'); + }, +}); + +Deno.test({ + name: 'it should fail as H candidate office and candidate_district missing', + fn: () => { + const testData = { ...data }; + testData.candidate_office = 'H'; + testData.candidate_state = 'VA'; + delete testData.candidate_district; + const result = validate(candidateContactSchema, testData); + assertEquals(result[0].keyword, 'required'); + assertEquals(result[0].message, "must have required property 'candidate_district'"); + }, +}); + +Deno.test({ + name: 'it should pass with for candidate office H with a candidate state and candidate district', + fn: () => { + const testData = { ...data }; + testData.candidate_office = 'H'; + testData.candidate_state = 'VA'; + testData.candidate_district = '01'; + const result = validate(candidateContactSchema, testData); + assertEquals(result, []); + }, +}); diff --git a/fecfile_validate_js/tests/index.test.ts b/fecfile_validate_js/tests/f3x.test.ts similarity index 100% rename from fecfile_validate_js/tests/index.test.ts rename to fecfile_validate_js/tests/f3x.test.ts diff --git a/package.json b/package.json index 35e1485e..bf261a70 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "build": "rm -rf fecfile_validate_js/dist && tsc --project fecfile_validate_js/tsconfig.json && node fecfile_validate_js/scripts/buildSchemaModules", "postinstall": "npm run build", "lint": "eslint ./fecfile_validate_js/src --ext .ts", - "test": "npm run build && deno test --compat --unstable --allow-read fecfile_validate_js/tests/index.test.ts" + "test": "npm run build && deno test --compat --unstable --allow-read fecfile_validate_js/tests/*.test.ts" }, "repository": { "type": "git", From 98d8a06b31e24d4b2b05bec4ec57c6bc69460078 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Sun, 10 Apr 2022 08:16:41 -0400 Subject: [PATCH 03/15] Updated schema documentation HTML for candidate contact --- docs/Contact_Candidate.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/Contact_Candidate.html b/docs/Contact_Candidate.html index ba613107..70f0df5d 100644 --- a/docs/Contact_Candidate.html +++ b/docs/Contact_Candidate.html @@ -1,4 +1,4 @@ - FEC Candidate

FEC Candidate

Type: object

Candidate Contact

root type
Type: const
Specific value: "CAN"
Example:

"CAN"
+ FEC Candidate 
FEC Candidate
 

 
Candidate Contact
 
  
 
 
root    allOf    item 0
Type: object
 
  
 
If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. Otherwise, the conditions in the "Else" tab should be respected.
  
 
 
root    allOf    item 0    if
Type: object
 
 
 
 
  
 
 
 
 
root    allOf    item 0    if    candidate_office
Type: const
 Specific value: "S" 
 
 
 
 
 
 
root    allOf    item 0    then
Type: object
 
 
 
 
  
 
 
 
 
root    allOf    item 0    then    candidate_state
Type: string
 
 
 
 
 
 
 
root    allOf    item 1
Type: object
 
  
 
If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. Otherwise, the conditions in the "Else" tab should be respected.
  
 
 
root    allOf    item 1    if
Type: object
 
 
 
 
  
 
 
 
 
root    allOf    item 1    if    candidate_office
Type: const
 Specific value: "H" 
 
 
 
 
 
 
root    allOf    item 1    then
Type: object
 
 
 
 
  
 
 
 
 
root    allOf    item 1    then    candidate_state
Type: string
 
 
 
 
 
 
 
 
  
 
 
 
 
root    allOf    item 1    then    candidate_district
Type: string
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    type
Type: const
 Specific value: "CAN" 
 
Example:
 
"CAN"
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    candidate_id
Type: string
 Must match regular expression: ^[ A-Za-z0-9]{0,9}$ 
Must be at least 0 characters long
Must be at most 9 characters long
 
 
Example:
 
"H01234567"
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    last_name
Type: string
 Must match regular expression: ^[ A-Za-z0-9]{0,30}$ 
Must be at least 0 characters long
Must be at most 30 characters long
 
 
Example:
 
"Smith"
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    first_name
Type: string
 Must match regular expression: ^[ A-Za-z0-9]{0,20}$ 
Must be at least 0 characters long
Must be at most 20 characters long
 
 
Example:
 
"John"
@@ -7,10 +7,10 @@
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    suffix
Type: string or null
 Must match regular expression: ^[ A-Za-z0-9]{0,10}$ 
 
Example:
 
"Jr"
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    street_1
Type: string
 Must match regular expression: ^[ A-Za-z0-9]{0,34}$ 
Must be at least 0 characters long
Must be at most 34 characters long
 
 
Example:
 
"123 Main Street"
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    street_2
Type: string or null
 Must match regular expression: ^[ A-Za-z0-9]{0,34}$ 
 
 
 
 
 
 
 
  
 
 
 
 
root    city
Type: string
 Must match regular expression: ^[ A-Za-z0-9]{0,30}$ 
Must be at least 0 characters long
Must be at most 30 characters long
 
 
Example:
 
"Anytown"
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    state
Type: string
 Must match regular expression: ^[ A-Za-z0-9]{0,2}$ 
Must be at least 0 characters long
Must be at most 2 characters long
 
 
Example:
 
"WA"
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    state
Type: string
 Must match regular expression: ^[A-Z]{2}$ 
Must be at least 2 characters long
Must be at most 2 characters long
 
 
Example:
 
"WA"
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    zip
Type: string
 Must match regular expression: ^[ A-Za-z0-9]{0,9}$ 
Must be at least 0 characters long
Must be at most 9 characters long
 
 
Example:
 
981110123
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    employer
Type: string or null
 Must match regular expression: ^[ A-Za-z0-9]{0,38}$ 
 
Example:
 
"XYZ Company"
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    occupation
Type: string or null
 Must match regular expression: ^[ A-Za-z0-9]{0,38}$ 
 
Example:
 
"QC Inspector"
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    candidate_office
Type: enum (of string)
 
 
Must be one of:
 
  • "H"
  • "S"
  • "P"
 
 
 
Example:
 
"H\nS\nP"
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    candidate_state
Type: string
 Must match regular expression: ^[ A-Za-z0-9]{0,2}$ 
Must be at least 0 characters long
Must be at most 2 characters long
 
 
Example:
 
"WA"
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    candidate_district
Type: string
 Must match regular expression: ^[ A-Za-z0-9]{0,2}$ 
Must be at least 0 characters long
Must be at most 2 characters long
 
 
 
 
 
 
 
 
  
 
 
 
 
root    telephone
Type: string or null
 Must match regular expression: ^\d{10}$ 
 
 
 
 
 
 
 
  
 
 
 
 
root    country
Type: string
 
 
 
 
  
+

root candidate_state
Type: string or null
Must match regular expression: ^[A-Z]{2}$
Example:

"WA"
+

root candidate_district
Type: string or null
Must match regular expression: ^[0-9]{2}$

root telephone
Type: string or null
Must match regular expression: ^\d{10}$

root country
Type: string
\ No newline at end of file From c36b8a956aa448ddebdf83c7f5e78ddb3e52a74e Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Sun, 10 Apr 2022 08:28:43 -0400 Subject: [PATCH 04/15] Fixed display of contacts spec html pages by removing extra 5th table column --- docs/Contact_Candidate_spec.html | 217 +++++++++++++++++++++++++++- docs/Contact_Committee_spec.html | 125 +++++++++++++++- docs/Contact_Individual_spec.html | 175 +++++++++++++++++++++- docs/Contact_Organization_spec.html | 115 ++++++++++++++- 4 files changed, 626 insertions(+), 6 deletions(-) diff --git a/docs/Contact_Candidate_spec.html b/docs/Contact_Candidate_spec.html index 4bf3fd00..276b010e 100644 --- a/docs/Contact_Candidate_spec.html +++ b/docs/Contact_Candidate_spec.html @@ -1,3 +1,214 @@ -Contact_Candidate
Specification for Contact_Candidate
COL SEQFIELD DESCRIPTIONTYPEREQUIREDSAMPLE DATAVALUE REFERENCERULE REFERENCEFIELD FORM ASSOCIATION
1TYPEDropdownX (error)Individual Committee Organization Candidate
2CANDIDATE IDA/N-9X (error)H01234567
3LAST NAMEA/N-30X (error)Smith
4FIRST NAMEA/N-20X (error)John
5MIDDLE NAMEA/N-20W
6PREFIXA/N-10Dr
7SUFFIXA/N-10Jr
8STREET 1A/N-34X (error)123 Main Street
9STREET 2A/N-34
10CITYA/N-30X (error)Anytown
11STATEA/N-2X (error)WAEdit: ST. Default to ZZ if Country DNE US
12ZIPA/N-9X (error)981110123
13EMPLOYERA/N-38XYZ CompanyReq if Donor aggregate >$200
14OCCUPATIONA/N-38QC InspectorReq if Donor aggregate >$200
15CANDIDATE OFFICEDropdownX (error)House -Senate -Presidential
16CANDIDATE STATEA/N-2X (error)WARequired if Office = H or S
17CANDIDATE DISTRICTA/N-2X (error)Required if Office = H
TELEPHONEN-10
COUNTRYDropdownX (error)Should default to United States
\ No newline at end of file + + + + Contact_Candidate + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Specification for Contact_Candidate +
COL SEQFIELD DESCRIPTIONTYPEREQUIREDSAMPLE DATAVALUE REFERENCERULE REFERENCEFIELD FORM ASSOCIATION
1TYPEDropdownX (error)Individual Committee Organization Candidate
2CANDIDATE IDA/N-9X (error)H01234567
3LAST NAMEA/N-30X (error)Smith
4FIRST NAMEA/N-20X (error)John
5MIDDLE NAMEA/N-20W
6PREFIXA/N-10Dr
7SUFFIXA/N-10Jr
8STREET 1A/N-34X (error)123 Main Street
9STREET 2A/N-34
10CITYA/N-30X (error)Anytown
11STATEA/N-2X (error)WAEdit: ST. Default to ZZ if Country DNE US
12ZIPA/N-9X (error)981110123
13EMPLOYERA/N-38XYZ CompanyReq if Donor aggregate >$200
14OCCUPATIONA/N-38QC InspectorReq if Donor aggregate >$200
15CANDIDATE OFFICEDropdownX (error)House Senate Presidential
16CANDIDATE STATEA/N-2X (error)WARequired if Office = H or S
17CANDIDATE DISTRICTA/N-2X (error)Required if Office = H
TELEPHONEN-10
COUNTRYDropdownX (error)Should default to United States
+ + diff --git a/docs/Contact_Committee_spec.html b/docs/Contact_Committee_spec.html index 64636039..e83cbe30 100644 --- a/docs/Contact_Committee_spec.html +++ b/docs/Contact_Committee_spec.html @@ -1 +1,124 @@ -Contact_Committee
Specification for Contact_Committee
COL SEQFIELD DESCRIPTIONTYPEREQUIREDSAMPLE DATAVALUE REFERENCERULE REFERENCEFIELD FORM ASSOCIATION
1TYPEDropdownX (error)Individual Committee Organization Candidate
2COMMITTEE IDA/N-9X (error)c01234567
3NAMEA/N-200X (error)SEIU COPE (Service Employees International Union Committee On Political Education)
4STREET 1A/N-34X (error)123 Main Street
5STREET 2A/N-34
6CITYA/N-30X (error)Anytown
7STATEA/N-2X (error)WAEdit: ST. Default to ZZ if Country DNE US
8ZIPA/N-9X (error)981110123
12TELEPHONEN-10
COUNTRYDropdownX (error)Should default to United States
\ No newline at end of file + + + + Contact_Committee + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Specification for Contact_Committee +
COL SEQFIELD DESCRIPTIONTYPEREQUIREDSAMPLE DATAVALUE REFERENCERULE REFERENCEFIELD FORM ASSOCIATION
1TYPEDropdownX (error)Individual Committee Organization Candidate
2COMMITTEE IDA/N-9X (error)c01234567
3NAMEA/N-200X (error)SEIU COPE (Service Employees International Union Committee On Political Education)
4STREET 1A/N-34X (error)123 Main Street
5STREET 2A/N-34
6CITYA/N-30X (error)Anytown
7STATEA/N-2X (error)WAEdit: ST. Default to ZZ if Country DNE US
8ZIPA/N-9X (error)981110123
12TELEPHONEN-10
COUNTRYDropdownX (error)Should default to United States
+ + diff --git a/docs/Contact_Individual_spec.html b/docs/Contact_Individual_spec.html index ba9e04cf..0f2ba5d9 100644 --- a/docs/Contact_Individual_spec.html +++ b/docs/Contact_Individual_spec.html @@ -1 +1,174 @@ -Contact_Individual
Specification for Contact_Individual
COL SEQFIELD DESCRIPTIONTYPEREQUIREDSAMPLE DATAVALUE REFERENCERULE REFERENCEFIELD FORM ASSOCIATION
1TYPEDropdownX (error)Individual Committee Organization Candidate
2LAST NAMEA/N-30X (error)Smith
3FIRST NAMEA/N-20X (error)John
4MIDDLE NAMEA/N-20W
5PREFIXA/N-10Dr
6SUFFIXA/N-10Jr
7STREET 1A/N-34X (error)123 Main Street
8STREET 2A/N-34
9CITYA/N-30X (error)Anytown
10STATEA/N-2X (error)WAEdit: ST. Default to ZZ if Country DNE US
11ZIPA/N-9X (error)981110123
12TELEPHONEN-10
13EMPLOYERA/N-38XYZ CompanyReq if Donor aggregate >$200
14OCCUPATIONA/N-38QC InspectorReq if Donor aggregate >$200
COUNTRYDropdownX (error)Should default to United States
\ No newline at end of file + + + + Contact_Individual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Specification for Contact_Individual +
COL SEQFIELD DESCRIPTIONTYPEREQUIREDSAMPLE DATAVALUE REFERENCERULE REFERENCEFIELD FORM ASSOCIATION
1TYPEDropdownX (error)Individual Committee Organization Candidate
2LAST NAMEA/N-30X (error)Smith
3FIRST NAMEA/N-20X (error)John
4MIDDLE NAMEA/N-20W
5PREFIXA/N-10Dr
6SUFFIXA/N-10Jr
7STREET 1A/N-34X (error)123 Main Street
8STREET 2A/N-34
9CITYA/N-30X (error)Anytown
10STATEA/N-2X (error)WAEdit: ST. Default to ZZ if Country DNE US
11ZIPA/N-9X (error)981110123
12TELEPHONEN-10
13EMPLOYERA/N-38XYZ CompanyReq if Donor aggregate >$200
14OCCUPATIONA/N-38QC InspectorReq if Donor aggregate >$200
COUNTRYDropdownX (error)Should default to United States
+ + diff --git a/docs/Contact_Organization_spec.html b/docs/Contact_Organization_spec.html index f17238d9..1cd63cc1 100644 --- a/docs/Contact_Organization_spec.html +++ b/docs/Contact_Organization_spec.html @@ -1 +1,114 @@ -Contact_Organization
Specification for Contact_Organization
COL SEQFIELD DESCRIPTIONTYPEREQUIREDSAMPLE DATAVALUE REFERENCERULE REFERENCEFIELD FORM ASSOCIATION
1TYPEDropdownX (error)Individual Committee Organization CandidateRequired if NOT [IND|CAN]
8NAMEA/N-200X (error)John Smith & Co.
14STREET 1A/N-34X (error)123 Main Street
15STREET 2A/N-34
16CITYA/N-30X (error)AnytownAK,AL,...,ZZEdit: ST. Default to ZZ if Country DNE US
17STATEA/N-2X (error)WA
18ZIPA/N-9X (error)981110123
12TELEPHONEN-10
COUNTRYDropdownX (error)Should default to United States
\ No newline at end of file + + + + Contact_Organization + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Specification for Contact_Organization +
COL SEQFIELD DESCRIPTIONTYPEREQUIREDSAMPLE DATAVALUE REFERENCERULE REFERENCEFIELD FORM ASSOCIATION
1TYPEDropdownX (error)Individual Committee Organization CandidateRequired if NOT [IND|CAN]
8NAMEA/N-200X (error)John Smith & Co.
14STREET 1A/N-34X (error)123 Main Street
15STREET 2A/N-34
16CITYA/N-30X (error)AnytownAK,AL,...,ZZEdit: ST. Default to ZZ if Country DNE US
17STATEA/N-2X (error)WA
18ZIPA/N-9X (error)981110123
12TELEPHONEN-10
COUNTRYDropdownX (error)Should default to United States
+ + From 8fc7f3fcaf8003a0776e38dc9ef24705c26eeefd Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Sun, 10 Apr 2022 08:53:53 -0400 Subject: [PATCH 05/15] Cleaned up linting issues in unit test file --- .../tests/test_contact_schema.py | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/fecfile_validate_python/tests/test_contact_schema.py b/fecfile_validate_python/tests/test_contact_schema.py index e854e4b4..7fa0a669 100644 --- a/fecfile_validate_python/tests/test_contact_schema.py +++ b/fecfile_validate_python/tests/test_contact_schema.py @@ -1,9 +1,8 @@ import pytest import json import os -from jsonschema import Draft7Validator from src.fecfile_validate import validate -import pdb + @pytest.fixture def sample_ind_contact(): @@ -41,22 +40,21 @@ def test_required_candidate_state(sample_can_contact): sample_can_contact) assert validation_result.errors == [] - # Fail null for candidate state when office is S sample_can_contact["candidate_office"] = "S" validation_result = validate.validate("Contact_Candidate", sample_can_contact) assert validation_result.errors[0].path == "candidate_state" - assert validation_result.errors[0].message == "None is not of type 'string'" - + assert validation_result.errors[0].message == \ + "None is not of type 'string'" # Fail not having candidate state at all when office is S del sample_can_contact["candidate_state"] validation_result = validate.validate("Contact_Candidate", sample_can_contact) assert validation_result.errors[0].path == "candidate_state" - assert validation_result.errors[0].message == "'candidate_state' is a required property" - + assert validation_result.errors[0].message == \ + "'candidate_state' is a required property" # Fail null for candidate district when office is H sample_can_contact["candidate_state"] = "VA" @@ -64,20 +62,21 @@ def test_required_candidate_state(sample_can_contact): validation_result = validate.validate("Contact_Candidate", sample_can_contact) assert validation_result.errors[0].path == "candidate_district" - assert validation_result.errors[0].message == "None is not of type 'string'" - + assert validation_result.errors[0].message == \ + "None is not of type 'string'" # Fail invalid candidate district code sample_can_contact["candidate_district"] = "ZZ" validation_result = validate.validate("Contact_Candidate", sample_can_contact) assert validation_result.errors[0].path == "candidate_district" - assert validation_result.errors[0].message == "'ZZ' does not match '^[0-9]{2}$'" - + assert validation_result.errors[0].message == \ + "'ZZ' does not match '^[0-9]{2}$'" # Fail not having a candidate district when office is H del sample_can_contact["candidate_district"] validation_result = validate.validate("Contact_Candidate", sample_can_contact) assert validation_result.errors[0].path == "candidate_district" - assert validation_result.errors[0].message == "'candidate_district' is a required property" + assert validation_result.errors[0].message == \ + "'candidate_district' is a required property" From 9bac8dd682bf126d425a7989e7967c1d76989b59 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Mon, 11 Apr 2022 09:20:09 -0400 Subject: [PATCH 06/15] Resolved SonarCloud security hotspot --- docs/Contact_Candidate.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Contact_Candidate.html b/docs/Contact_Candidate.html index 70f0df5d..e2b59ea9 100644 --- a/docs/Contact_Candidate.html +++ b/docs/Contact_Candidate.html @@ -1,4 +1,4 @@ - FEC Candidate

FEC Candidate


Candidate Contact

root allOf item 0
Type: object

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. Otherwise, the conditions in the "Else" tab should be respected.

root allOf item 0 if
Type: object

root allOf item 0 if candidate_office
Type: const
Specific value: "S"
root allOf item 0 then
Type: object

root allOf item 0 then candidate_state
Type: string
root allOf item 1
Type: object

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. Otherwise, the conditions in the "Else" tab should be respected.

root allOf item 1 if
Type: object

root allOf item 1 if candidate_office
Type: const
Specific value: "H"
root allOf item 1 then
Type: object

root allOf item 1 then candidate_state
Type: string

root allOf item 1 then candidate_district
Type: string

root type
Type: const
Specific value: "CAN"
Example:

"CAN"
+ FEC Candidate 
FEC Candidate
 

 
Candidate Contact
 
  
 
 
root    allOf    item 0
Type: object
 
  
 
If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. Otherwise, the conditions in the "Else" tab should be respected.
  
 
 
root    allOf    item 0    if
Type: object
 
 
 
 
  
 
 
 
 
root    allOf    item 0    if    candidate_office
Type: const
 Specific value: "S" 
 
 
 
 
 
 
root    allOf    item 0    then
Type: object
 
 
 
 
  
 
 
 
 
root    allOf    item 0    then    candidate_state
Type: string
 
 
 
 
 
 
 
root    allOf    item 1
Type: object
 
  
 
If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. Otherwise, the conditions in the "Else" tab should be respected.
  
 
 
root    allOf    item 1    if
Type: object
 
 
 
 
  
 
 
 
 
root    allOf    item 1    if    candidate_office
Type: const
 Specific value: "H" 
 
 
 
 
 
 
root    allOf    item 1    then
Type: object
 
 
 
 
  
 
 
 
 
root    allOf    item 1    then    candidate_state
Type: string
 
 
 
 
 
 
 
 
  
 
 
 
 
root    allOf    item 1    then    candidate_district
Type: string
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    type
Type: const
 Specific value: "CAN" 
 
Example:
 
"CAN"
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    candidate_id
Type: string
 Must match regular expression: ^[ A-Za-z0-9]{0,9}$ 
Must be at least 0 characters long
Must be at most 9 characters long
 
 
Example:
 
"H01234567"
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    last_name
Type: string
 Must match regular expression: ^[ A-Za-z0-9]{0,30}$ 
Must be at least 0 characters long
Must be at most 30 characters long
 
 
Example:
 
"Smith"
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    first_name
Type: string
 Must match regular expression: ^[ A-Za-z0-9]{0,20}$ 
Must be at least 0 characters long
Must be at most 20 characters long
 
 
Example:
 
"John"

From dc68ffd4c5569c814e2b5aad55b5a8215d4bebc1 Mon Sep 17 00:00:00 2001
From: Matt Travers 
Date: Wed, 13 Apr 2022 09:50:26 -0400
Subject: [PATCH 07/15] Added unit testing to buildSchemaModules script

---
 .../tests/buildSchemaModules.test.ts          | 24 +++++++++++++++++++
 fecfile_validate_js/tests/candidate.test.ts   |  2 +-
 fecfile_validate_js/tests/f3x.test.ts         |  2 +-
 package.json                                  |  2 +-
 4 files changed, 27 insertions(+), 3 deletions(-)
 create mode 100644 fecfile_validate_js/tests/buildSchemaModules.test.ts

diff --git a/fecfile_validate_js/tests/buildSchemaModules.test.ts b/fecfile_validate_js/tests/buildSchemaModules.test.ts
new file mode 100644
index 00000000..25758489
--- /dev/null
+++ b/fecfile_validate_js/tests/buildSchemaModules.test.ts
@@ -0,0 +1,24 @@
+import { assertEquals } from 'https://deno.land/std/testing/asserts.ts';
+import { existsSync } from 'https://deno.land/std/fs/mod.ts';
+import * as path from 'https://deno.land/std/path/mod.ts';
+
+const __dirname = path.dirname(path.fromFileUrl(import.meta.url));
+
+const process = Deno.run({ cmd: ['node', 'fecfile_validate_js/scripts/buildSchemaModules'] });
+await process.status();
+
+Deno.test({
+  name: 'it should create schema *.js files',
+  fn: () => {
+    const result: boolean = existsSync(`${__dirname}/../dist/F3X.js`);
+    assertEquals(result, true);
+  },
+});
+
+Deno.test({
+  name: 'it should create schema *.d.ts files',
+  fn: () => {
+    const result: boolean = existsSync(`${__dirname}/../dist/F3X.d.ts`);
+    assertEquals(result, true);
+  },
+});
diff --git a/fecfile_validate_js/tests/candidate.test.ts b/fecfile_validate_js/tests/candidate.test.ts
index 35487e58..732a5419 100644
--- a/fecfile_validate_js/tests/candidate.test.ts
+++ b/fecfile_validate_js/tests/candidate.test.ts
@@ -1,4 +1,4 @@
-import { assertEquals } from 'https://deno.land/std@0.133.0/testing/asserts.ts';
+import { assertEquals } from 'https://deno.land/std/testing/asserts.ts';
 import { validate } from '../dist/index.js';
 import { schema as candidateContactSchema } from '../dist/Contact_Candidate.js';
 
diff --git a/fecfile_validate_js/tests/f3x.test.ts b/fecfile_validate_js/tests/f3x.test.ts
index e2f6cb94..f4722242 100644
--- a/fecfile_validate_js/tests/f3x.test.ts
+++ b/fecfile_validate_js/tests/f3x.test.ts
@@ -1,4 +1,4 @@
-import { assertEquals } from 'https://deno.land/std@0.133.0/testing/asserts.ts';
+import { assertEquals } from 'https://deno.land/std/testing/asserts.ts';
 import { validate } from '../dist/index.js';
 import { schema as f3xSchema } from '../dist/F3X.js';
 
diff --git a/package.json b/package.json
index bf261a70..57b46122 100644
--- a/package.json
+++ b/package.json
@@ -14,7 +14,7 @@
         "build": "rm -rf fecfile_validate_js/dist && tsc --project fecfile_validate_js/tsconfig.json && node fecfile_validate_js/scripts/buildSchemaModules",
         "postinstall": "npm run build",
         "lint": "eslint ./fecfile_validate_js/src --ext .ts",
-        "test": "npm run build && deno test --compat --unstable --allow-read fecfile_validate_js/tests/*.test.ts"
+        "test": "npm run build && deno test --coverage=cov_profile --compat --unstable --allow-read --allow-run fecfile_validate_js/tests/*.test.ts"
     },
     "repository": {
         "type": "git",

From 88ae4e54c27539e8ce5466b091ac34355ec26ccf Mon Sep 17 00:00:00 2001
From: Matt Travers 
Date: Thu, 14 Apr 2022 12:14:39 -0400
Subject: [PATCH 08/15] Update README.md deployment instructions

Moved create release PR to deploy to stage process
---
 README.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index b88674df..a0b90190 100644
--- a/README.md
+++ b/README.md
@@ -54,29 +54,30 @@ Without the git-flow extensions:
 
 ### Create a release branch
 
-Using git-flow extensions:
+* Using git-flow extensions:
 ```
 git flow release start sprint-#
 ```
 
-Without the git-flow extensions:
+* Without the git-flow extensions:
 ```
 git checkout develop
 git pull
 git checkout -b release/sprint-# develop
 git push --set-upstream origin release/sprint-#
 ```
+* Developer creates a PR in GitHub to merge release/sprint-# branch into the `main` branch to track if commits pass deployment checks. The actual merge will happen when deploying a release to production.
 
 ### Create and deploy a hotfix
 
-Using git-flow extensions:
+* Using git-flow extensions:
 ```
 git flow hotfix start my-fix
 # Work happens here
 git flow hotfix finish my-fix
 ```
 
-Without the git-flow extensions:
+* Without the git-flow extensions:
 ```
 git checkout -b hotfix/my-fix main
 # Work happens here
@@ -88,7 +89,6 @@ git push --set-upstream origin hotfix/my-fix
 * Developer deploys hotfix/my-fix branch to main using **Deploying a release to production** instructions below
 
 ### Deploying a release to production
-* Developer creates a PR in GitHub to merge release/sprint-# branch into the `main` branch
 * Reviewer approves PR and merges into `main`
 * Check CircleCI for passing pipeline tests
 * If tests pass, continue

From ff770d1228afda9ea8398186dcd5f51c5638ac0b Mon Sep 17 00:00:00 2001
From: toddlees 
Date: Wed, 20 Apr 2022 21:15:28 -0400
Subject: [PATCH 09/15] subschema validation for python package

---
 .../src/fecfile_validate/validate.py          | 32 ++++++++++++---
 .../{ => contacts}/sample_contact_CAN.json    |  0
 .../{ => contacts}/sample_contact_IND.json    |  0
 .../{ => contacts}/test_contact_schema.py     |  0
 .../tests/test_validate.py                    | 41 +++++++++++++++----
 5 files changed, 60 insertions(+), 13 deletions(-)
 rename fecfile_validate_python/tests/{ => contacts}/sample_contact_CAN.json (100%)
 rename fecfile_validate_python/tests/{ => contacts}/sample_contact_IND.json (100%)
 rename fecfile_validate_python/tests/{ => contacts}/test_contact_schema.py (100%)

diff --git a/fecfile_validate_python/src/fecfile_validate/validate.py b/fecfile_validate_python/src/fecfile_validate/validate.py
index c0eceb8a..a9f5d9a5 100644
--- a/fecfile_validate_python/src/fecfile_validate/validate.py
+++ b/fecfile_validate_python/src/fecfile_validate/validate.py
@@ -30,8 +30,7 @@ def get_schema(schema_name):
     Returns:
         dict: JSON schema that matches the schema_name"""
     schema_file = f"{schema_name}.json"
-    schema_path = os.path.join(os.path.dirname(__file__), "schema/",
-                               schema_file)
+    schema_path = os.path.join(os.path.dirname(__file__), "schema/", schema_file)
     #: Handle case where we are not running from a pip package
     if not os.path.isfile(schema_path):
         logger.warning(f"Schema file ({schema_path}) not found in package.")
@@ -63,7 +62,7 @@ def parse_schema_error(error):
     return ValidationError(error.message, path)
 
 
-def validate(schema_name, form_data):
+def validate(schema_name, form_data, fields_to_validate=None):
     """Wrapper function around jsonschema validator
 
     Args:
@@ -71,10 +70,33 @@ def validate(schema_name, form_data):
             JSON schema definition file
         data (dict): key-value pairs of data to be validated.
             Keys match the properties of the JSON schema file
+        fields_to_validate (array):
 
     Returns:
         list of ValidationError: A list of all errors found in form_data"""
     form_schema = get_schema(schema_name)
-    validator = Draft7Validator(form_schema)
-    errors = list(map(parse_schema_error, validator.iter_errors(form_data)))
+    if fields_to_validate:
+        schema_of_fields = {
+            "type": "object",
+            "required": list(
+                filter(
+                    lambda required_field: required_field in fields_to_validate,
+                    form_schema.get("required", []),
+                )
+            ),
+            "properties": dict(
+                filter(
+                    lambda field: field in fields_to_validate,
+                    form_schema.get("properties").items(),
+                )
+            ),
+        }
+        errors = __validate_with_schema(schema_of_fields, form_data)
+    else:
+        errors = __validate_with_schema(form_schema, form_data)
     return ValidationResult(errors, [])
+
+
+def __validate_with_schema(schema, form_data):
+    validator = Draft7Validator(schema)
+    return list(map(parse_schema_error, validator.iter_errors(form_data)))
diff --git a/fecfile_validate_python/tests/sample_contact_CAN.json b/fecfile_validate_python/tests/contacts/sample_contact_CAN.json
similarity index 100%
rename from fecfile_validate_python/tests/sample_contact_CAN.json
rename to fecfile_validate_python/tests/contacts/sample_contact_CAN.json
diff --git a/fecfile_validate_python/tests/sample_contact_IND.json b/fecfile_validate_python/tests/contacts/sample_contact_IND.json
similarity index 100%
rename from fecfile_validate_python/tests/sample_contact_IND.json
rename to fecfile_validate_python/tests/contacts/sample_contact_IND.json
diff --git a/fecfile_validate_python/tests/test_contact_schema.py b/fecfile_validate_python/tests/contacts/test_contact_schema.py
similarity index 100%
rename from fecfile_validate_python/tests/test_contact_schema.py
rename to fecfile_validate_python/tests/contacts/test_contact_schema.py
diff --git a/fecfile_validate_python/tests/test_validate.py b/fecfile_validate_python/tests/test_validate.py
index d61a04dd..f7225b5f 100644
--- a/fecfile_validate_python/tests/test_validate.py
+++ b/fecfile_validate_python/tests/test_validate.py
@@ -14,8 +14,7 @@ def sample_f3x():
 
 @pytest.fixture
 def test_schema():
-    with open(os.path.join(os.path.dirname(__file__),
-              "test_schema.json")) as f:
+    with open(os.path.join(os.path.dirname(__file__), "test_schema.json")) as f:
         schema = json.load(f)
     return schema
 
@@ -27,14 +26,11 @@ def test_is_correct(sample_f3x):
 
 def test_missing_required_field(sample_f3x):
     # Create error by removing FORM_TYPE
-    sample_f3x["form_type"] = ""
+    sample_f3x.pop("form_type", None)
 
     validation_result = validate.validate("F3X", sample_f3x)
     assert validation_result.errors[0].path == "form_type"
-    assert (
-        validation_result.errors[0].message
-        == "'' is not one of ['F3XN', 'F3XA', 'F3XT']"
-    )
+    assert validation_result.errors[0].message == "'form_type' is a required property"
 
 
 def test_invalid_string_character(sample_f3x):
@@ -48,11 +44,17 @@ def test_invalid_string_character(sample_f3x):
 
 
 def test_non_required_field(sample_f3x):
-    sample_f3x["treasurer_middle_name"] = None
+    sample_f3x.pop("treasurer_middle_name", None)
     validation_result = validate.validate("F3X", sample_f3x)
     assert validation_result.errors == []
 
 
+"""Test parse_schema_error
+parse_schema_error needs to parse jsonschema errors and 
+turn them into ValidationErrors
+"""
+
+
 def check_error(validation_error, message, path):
     expected_error = validate.ValidationError(message, path)
     assert validation_error.path == expected_error.path
@@ -87,3 +89,26 @@ def test_parse_required_error(test_schema):
         "'nested_field' is a required property",
         "top_level_field.nested_field",
     )
+
+
+"""Test partial validation
+Calling validate with fields_to_validate contrstructs a temporary
+schema from the passed schema.  This temporary schema must be a subschema
+of the passed schema"""
+
+
+def test_partial_validate_is_correct(sample_f3x):
+    fields_to_validate = ["form_type"]
+    """Even though we remove date_signed, the validation should pass"""
+    sample_f3x.pop("date_signed", None)
+    validation_result = validate.validate("F3X", sample_f3x, fields_to_validate)
+    assert validation_result.errors == []
+
+
+def test_partial_missing_required_field(sample_f3x):
+    fields_to_validate = ["form_type"]
+    # Create error by removing FORM_TYPE
+    sample_f3x.pop("form_type", None)
+    validation_result = validate.validate("F3X", sample_f3x, fields_to_validate)
+    assert validation_result.errors[0].path == "form_type"
+    assert validation_result.errors[0].message == "'form_type' is a required property"

From e399fcce22c9b31d21e6b0074fb225d2dac8a306 Mon Sep 17 00:00:00 2001
From: toddlees 
Date: Fri, 22 Apr 2022 08:59:43 -0400
Subject: [PATCH 10/15] unit tests for both python and js package

---
 fecfile_validate_js/src/index.ts              |  4 ++-
 fecfile_validate_js/tests/f3x.test.ts         | 25 +++++++++++++++++++
 .../src/fecfile_validate/validate.py          | 17 ++++++-------
 .../tests/test_validate.py                    |  2 +-
 4 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/fecfile_validate_js/src/index.ts b/fecfile_validate_js/src/index.ts
index 4473d87c..39656683 100644
--- a/fecfile_validate_js/src/index.ts
+++ b/fecfile_validate_js/src/index.ts
@@ -31,9 +31,10 @@ const ajv = new Ajv({ allErrors: true, strictSchema: false });
  *
  * @param {object} schema
  * @param {object} data
+ * @param {string[]} fieldsToValidate
  * @returns {ValidationError[]} Modified version of Ajv output, empty array if no errors found
  */
-export function validate(schema: any, data: any): ValidationError[] {
+export function validate(schema: any, data: any, fieldsToValidate: string[] = []): ValidationError[] {
   const theSchemaUrl = schema['$schema'];
   schema['$schema'] = theSchemaUrl.replace('https', 'http');
 
@@ -54,3 +55,4 @@ export function validate(schema: any, data: any): ValidationError[] {
 
   return errors;
 }
+
diff --git a/fecfile_validate_js/tests/f3x.test.ts b/fecfile_validate_js/tests/f3x.test.ts
index f4722242..82fa26fe 100644
--- a/fecfile_validate_js/tests/f3x.test.ts
+++ b/fecfile_validate_js/tests/f3x.test.ts
@@ -198,3 +198,28 @@ Deno.test({
     assertEquals(result[0].params.missingProperty, 'date_signed');
   },
 });
+
+
+Deno.test({
+  name: 'it should pass with perfect partial data',
+  fn: () => {
+    const thisData = { ...perfectForm_F3X};
+    const fieldsToValidate = ['form_type'];
+    // Should still succeed without date_signed because we are only testing form_type
+    delete thisData.date_signed;
+    const result = validate(f3xSchema, perfectForm_F3X, fieldsToValidate);
+    assertEquals(result, []);
+  },
+});
+
+Deno.test({
+  name: 'it should fail without form_type',
+  fn: () => {
+    const thisData = { ...perfectForm_F3X };
+    const fieldsToValidate = ['form_type'];
+    delete thisData.form_type;
+    const result = validate(f3xSchema, thisData, fieldsToValidate);
+    assertEquals(result[0].keyword, 'required');
+    assertEquals(result[0].params.missingProperty, 'form_type');
+  },
+});
diff --git a/fecfile_validate_python/src/fecfile_validate/validate.py b/fecfile_validate_python/src/fecfile_validate/validate.py
index a9f5d9a5..591121a5 100644
--- a/fecfile_validate_python/src/fecfile_validate/validate.py
+++ b/fecfile_validate_python/src/fecfile_validate/validate.py
@@ -70,13 +70,15 @@ def validate(schema_name, form_data, fields_to_validate=None):
             JSON schema definition file
         data (dict): key-value pairs of data to be validated.
             Keys match the properties of the JSON schema file
-        fields_to_validate (array):
+        fields_to_validate (list): list of property names to validate.
+            Properties from the schema not in this list will not be
+            validated on, even if they are required.
 
     Returns:
         list of ValidationError: A list of all errors found in form_data"""
     form_schema = get_schema(schema_name)
     if fields_to_validate:
-        schema_of_fields = {
+        form_schema = {
             "type": "object",
             "required": list(
                 filter(
@@ -91,12 +93,7 @@ def validate(schema_name, form_data, fields_to_validate=None):
                 )
             ),
         }
-        errors = __validate_with_schema(schema_of_fields, form_data)
-    else:
-        errors = __validate_with_schema(form_schema, form_data)
-    return ValidationResult(errors, [])
-
 
-def __validate_with_schema(schema, form_data):
-    validator = Draft7Validator(schema)
-    return list(map(parse_schema_error, validator.iter_errors(form_data)))
+    validator = Draft7Validator(form_schema)
+    errors = list(map(parse_schema_error, validator.iter_errors(form_data)))
+    return ValidationResult(errors, [])
diff --git a/fecfile_validate_python/tests/test_validate.py b/fecfile_validate_python/tests/test_validate.py
index f7225b5f..f20e1636 100644
--- a/fecfile_validate_python/tests/test_validate.py
+++ b/fecfile_validate_python/tests/test_validate.py
@@ -99,7 +99,7 @@ def test_parse_required_error(test_schema):
 
 def test_partial_validate_is_correct(sample_f3x):
     fields_to_validate = ["form_type"]
-    """Even though we remove date_signed, the validation should pass"""
+    # Even though we remove date_signed, the validation should pass
     sample_f3x.pop("date_signed", None)
     validation_result = validate.validate("F3X", sample_f3x, fields_to_validate)
     assert validation_result.errors == []

From 39a5f8eab818d62c106f000aab6ae69b84f1ed54 Mon Sep 17 00:00:00 2001
From: toddlees 
Date: Fri, 22 Apr 2022 14:06:36 -0400
Subject: [PATCH 11/15] rewrite subschema validation to just filter errors

---
 .gitignore                                    |  4 ++-
 fecfile_validate_js/src/index.ts              | 27 +++++++++++++------
 fecfile_validate_js/tests/candidate.test.ts   |  4 +++
 fecfile_validate_js/tests/f3x.test.ts         |  6 +++++
 .../src/fecfile_validate/validate.py          | 19 +++----------
 5 files changed, 35 insertions(+), 25 deletions(-)

diff --git a/.gitignore b/.gitignore
index 56f09e45..2afffb96 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,6 +14,7 @@ local_settings.py
 */CACHE
 */package-lock.json
 *.editorconfig
+cov_profile/*
 # See http://help.github.com/ignore-files/ for more about ignoring files.
 
 # compiled output
@@ -22,7 +23,8 @@ local_settings.py
 /out-tsc
 
 # dependencies
-*/node_modules
+fecfile_validate_js/node_modules/*
+node_modules/*
 
 # IDEs and editors
 *.idea
diff --git a/fecfile_validate_js/src/index.ts b/fecfile_validate_js/src/index.ts
index 39656683..5efdc054 100644
--- a/fecfile_validate_js/src/index.ts
+++ b/fecfile_validate_js/src/index.ts
@@ -6,7 +6,7 @@
  * Tested with spec/index-spec.js
  */
 
-import Ajv, { ValidateFunction } from 'ajv';
+import Ajv, { ErrorObject, ValidateFunction } from 'ajv';
 
 /**
  * Validation error information for a single schema property
@@ -43,16 +43,27 @@ export function validate(schema: any, data: any, fieldsToValidate: string[] = []
   const errors: ValidationError[] = [];
 
   if (!isValid && !!validator.errors) {
-    validator.errors.forEach((err) => {
-      errors.push({
-        path: err.instancePath.substring(1),
-        keyword: err.keyword,
-        params: err.params,
-        message: !!err.message ? err.message : null,
-      });
+    validator.errors.forEach((error) => {
+      const parsedError = parseError(error)
+      if (!fieldsToValidate.length || fieldsToValidate.includes(parsedError.path)){
+        errors.push(parsedError);
+      }
     });
   }
 
   return errors;
 }
 
+function parseError(error: ErrorObject): ValidationError {
+  let path = error.instancePath.substring(1)
+  if (error.keyword == "required") {
+    path = error.params.missingProperty;
+  }
+  return {
+    path: path,
+    keyword: error.keyword,
+    params: error.params,
+    message: !!error.message ? error.message : null,
+  }
+}
+
diff --git a/fecfile_validate_js/tests/candidate.test.ts b/fecfile_validate_js/tests/candidate.test.ts
index 732a5419..112f95d2 100644
--- a/fecfile_validate_js/tests/candidate.test.ts
+++ b/fecfile_validate_js/tests/candidate.test.ts
@@ -41,6 +41,7 @@ Deno.test({
     delete testData.candidate_office;
     const result = validate(candidateContactSchema, testData);
     assertEquals(result[0].keyword, 'required');
+    assertEquals(result[0].path, 'candidate_office');
     assertEquals(result[0].message, "must have required property 'candidate_office'");
   },
 });
@@ -64,6 +65,7 @@ Deno.test({
     delete testData.candidate_state;
     const result = validate(candidateContactSchema, testData);
     assertEquals(result[0].keyword, 'required');
+    assertEquals(result[0].path, 'candidate_state');
     assertEquals(result[0].message, "must have required property 'candidate_state'");
   },
 });
@@ -98,6 +100,7 @@ Deno.test({
     delete testData.candidate_state;
     const result = validate(candidateContactSchema, testData);
     assertEquals(result[0].keyword, 'required');
+    assertEquals(result[0].path, 'candidate_state');
     assertEquals(result[0].message, "must have required property 'candidate_state'");
   },
 });
@@ -123,6 +126,7 @@ Deno.test({
     delete testData.candidate_district;
     const result = validate(candidateContactSchema, testData);
     assertEquals(result[0].keyword, 'required');
+    assertEquals(result[0].path, 'candidate_district');
     assertEquals(result[0].message, "must have required property 'candidate_district'");
   },
 });
diff --git a/fecfile_validate_js/tests/f3x.test.ts b/fecfile_validate_js/tests/f3x.test.ts
index 82fa26fe..f2cf01e3 100644
--- a/fecfile_validate_js/tests/f3x.test.ts
+++ b/fecfile_validate_js/tests/f3x.test.ts
@@ -43,6 +43,7 @@ Deno.test({
     delete thisData.form_type;
     const result = validate(f3xSchema, thisData);
     assertEquals(result[0].keyword, 'required');
+    assertEquals(result[0].path, 'form_type');
     assertEquals(result[0].params.missingProperty, 'form_type');
   },
 });
@@ -109,6 +110,7 @@ Deno.test({
     delete thisData.filer_committee_id_number;
     const result = validate(f3xSchema, thisData);
     assertEquals(result[0].keyword, 'required');
+    assertEquals(result[0].path, 'filer_committee_id_number');
     assertEquals(result[0].params.missingProperty, 'filer_committee_id_number');
   },
 });
@@ -151,6 +153,7 @@ Deno.test({
     delete thisData.treasurer_first_name;
     const result = validate(f3xSchema, thisData);
     assertEquals(result[0].keyword, 'required');
+    assertEquals(result[0].path, 'treasurer_first_name');
     assertEquals(result[0].params.missingProperty, 'treasurer_first_name');
   },
 });
@@ -173,6 +176,7 @@ Deno.test({
     delete thisData.treasurer_last_name;
     const result = validate(f3xSchema, thisData);
     assertEquals(result[0].keyword, 'required');
+    assertEquals(result[0].path, 'treasurer_last_name');
     assertEquals(result[0].params.missingProperty, 'treasurer_last_name');
   },
 });
@@ -195,6 +199,7 @@ Deno.test({
     delete thisData.date_signed;
     const result = validate(f3xSchema, thisData);
     assertEquals(result[0].keyword, 'required');
+    assertEquals(result[0].path, 'date_signed');
     assertEquals(result[0].params.missingProperty, 'date_signed');
   },
 });
@@ -220,6 +225,7 @@ Deno.test({
     delete thisData.form_type;
     const result = validate(f3xSchema, thisData, fieldsToValidate);
     assertEquals(result[0].keyword, 'required');
+    assertEquals(result[0].path, 'form_type');
     assertEquals(result[0].params.missingProperty, 'form_type');
   },
 });
diff --git a/fecfile_validate_python/src/fecfile_validate/validate.py b/fecfile_validate_python/src/fecfile_validate/validate.py
index 591121a5..e25de293 100644
--- a/fecfile_validate_python/src/fecfile_validate/validate.py
+++ b/fecfile_validate_python/src/fecfile_validate/validate.py
@@ -77,23 +77,10 @@ def validate(schema_name, form_data, fields_to_validate=None):
     Returns:
         list of ValidationError: A list of all errors found in form_data"""
     form_schema = get_schema(schema_name)
-    if fields_to_validate:
-        form_schema = {
-            "type": "object",
-            "required": list(
-                filter(
-                    lambda required_field: required_field in fields_to_validate,
-                    form_schema.get("required", []),
-                )
-            ),
-            "properties": dict(
-                filter(
-                    lambda field: field in fields_to_validate,
-                    form_schema.get("properties").items(),
-                )
-            ),
-        }
 
     validator = Draft7Validator(form_schema)
     errors = list(map(parse_schema_error, validator.iter_errors(form_data)))
+    if fields_to_validate:
+        in_fields_to_validate = lambda field: field.path in fields_to_validate
+        errors = list(filter(in_fields_to_validate, errors))
     return ValidationResult(errors, [])

From c0944e2d88301a9d8b07ff5107ec7eece44d0c2c Mon Sep 17 00:00:00 2001
From: toddlees 
Date: Fri, 22 Apr 2022 14:13:09 -0400
Subject: [PATCH 12/15] =?UTF-8?q?writing=20code=20better=20=F0=9F=98=81?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 fecfile_validate_python/src/fecfile_validate/validate.py | 5 ++++-
 fecfile_validate_python/tests/test_validate.py           | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/fecfile_validate_python/src/fecfile_validate/validate.py b/fecfile_validate_python/src/fecfile_validate/validate.py
index e25de293..77df9ff9 100644
--- a/fecfile_validate_python/src/fecfile_validate/validate.py
+++ b/fecfile_validate_python/src/fecfile_validate/validate.py
@@ -81,6 +81,9 @@ def validate(schema_name, form_data, fields_to_validate=None):
     validator = Draft7Validator(form_schema)
     errors = list(map(parse_schema_error, validator.iter_errors(form_data)))
     if fields_to_validate:
-        in_fields_to_validate = lambda field: field.path in fields_to_validate
+
+        def in_fields_to_validate(field):
+            field.path in fields_to_validate
+
         errors = list(filter(in_fields_to_validate, errors))
     return ValidationResult(errors, [])
diff --git a/fecfile_validate_python/tests/test_validate.py b/fecfile_validate_python/tests/test_validate.py
index f20e1636..e650fa02 100644
--- a/fecfile_validate_python/tests/test_validate.py
+++ b/fecfile_validate_python/tests/test_validate.py
@@ -50,7 +50,7 @@ def test_non_required_field(sample_f3x):
 
 
 """Test parse_schema_error
-parse_schema_error needs to parse jsonschema errors and 
+parse_schema_error needs to parse jsonschema errors and
 turn them into ValidationErrors
 """
 

From e2f99f8e8b1b0ebddc6c69a08ce96c4bb7118061 Mon Sep 17 00:00:00 2001
From: toddlees 
Date: Fri, 22 Apr 2022 14:27:59 -0400
Subject: [PATCH 13/15] thank goodness for unit tests

---
 fecfile_validate_python/src/fecfile_validate/validate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fecfile_validate_python/src/fecfile_validate/validate.py b/fecfile_validate_python/src/fecfile_validate/validate.py
index 77df9ff9..035a8be3 100644
--- a/fecfile_validate_python/src/fecfile_validate/validate.py
+++ b/fecfile_validate_python/src/fecfile_validate/validate.py
@@ -83,7 +83,7 @@ def validate(schema_name, form_data, fields_to_validate=None):
     if fields_to_validate:
 
         def in_fields_to_validate(field):
-            field.path in fields_to_validate
+            return field.path in fields_to_validate
 
         errors = list(filter(in_fields_to_validate, errors))
     return ValidationResult(errors, [])

From b1b490818acb3344619f9d8c64cd1b73a0efbdb7 Mon Sep 17 00:00:00 2001
From: Matt Travers 
Date: Sat, 23 Apr 2022 22:19:19 -0400
Subject: [PATCH 14/15] Fixed linting and some empty array logic

---
 fecfile_validate_js/src/index.ts            | 21 +++++++++++----------
 fecfile_validate_js/tests/candidate.test.ts | 12 ++++++++++++
 2 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/fecfile_validate_js/src/index.ts b/fecfile_validate_js/src/index.ts
index 5efdc054..909ed122 100644
--- a/fecfile_validate_js/src/index.ts
+++ b/fecfile_validate_js/src/index.ts
@@ -35,17 +35,14 @@ const ajv = new Ajv({ allErrors: true, strictSchema: false });
  * @returns {ValidationError[]} Modified version of Ajv output, empty array if no errors found
  */
 export function validate(schema: any, data: any, fieldsToValidate: string[] = []): ValidationError[] {
-  const theSchemaUrl = schema['$schema'];
-  schema['$schema'] = theSchemaUrl.replace('https', 'http');
-
   const validator: ValidateFunction = ajv.compile(schema);
   const isValid: boolean = validator(data);
   const errors: ValidationError[] = [];
 
-  if (!isValid && !!validator.errors) {
+  if (!isValid && !!validator.errors?.length) {
     validator.errors.forEach((error) => {
-      const parsedError = parseError(error)
-      if (!fieldsToValidate.length || fieldsToValidate.includes(parsedError.path)){
+      const parsedError = parseError(error);
+      if (!fieldsToValidate.length || fieldsToValidate.includes(parsedError.path)) {
         errors.push(parsedError);
       }
     });
@@ -54,9 +51,14 @@ export function validate(schema: any, data: any, fieldsToValidate: string[] = []
   return errors;
 }
 
+/**
+ * Format error message from Ajv into our ValidationError interface
+ * @param {ErrorObject} error - Ajv ErrorObject interface
+ * @returns {ValidationError}
+ */
 function parseError(error: ErrorObject): ValidationError {
-  let path = error.instancePath.substring(1)
-  if (error.keyword == "required") {
+  let path = error.instancePath.substring(1);
+  if (error.keyword == 'required') {
     path = error.params.missingProperty;
   }
   return {
@@ -64,6 +66,5 @@ function parseError(error: ErrorObject): ValidationError {
     keyword: error.keyword,
     params: error.params,
     message: !!error.message ? error.message : null,
-  }
+  };
 }
-
diff --git a/fecfile_validate_js/tests/candidate.test.ts b/fecfile_validate_js/tests/candidate.test.ts
index 112f95d2..63e91aa7 100644
--- a/fecfile_validate_js/tests/candidate.test.ts
+++ b/fecfile_validate_js/tests/candidate.test.ts
@@ -70,6 +70,18 @@ Deno.test({
   },
 });
 
+Deno.test({
+  name: 'it should fail with for candidate office S with an invalid candidate state format',
+  fn: () => {
+    const testData = { ...data };
+    testData.candidate_office = 'S';
+    testData.candidate_state = 'M1';
+    const result = validate(candidateContactSchema, testData);
+    assertEquals(result[0].path, 'candidate_state');
+    assertEquals(result[0].message, 'must match pattern "^[A-Z]{2}$"');
+  },
+});
+
 Deno.test({
   name: 'it should pass with for candidate office S with a candidate state',
   fn: () => {

From 3cb3d9e4d5a430861d4fd8909f760d585b80cace Mon Sep 17 00:00:00 2001
From: Matt Travers 
Date: Tue, 26 Apr 2022 15:26:10 -0400
Subject: [PATCH 15/15] Add fecfile-api requirements.txt update to README

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index a0b90190..1a8f3874 100644
--- a/README.md
+++ b/README.md
@@ -35,6 +35,8 @@ is missing a value, the validation passes but with a warning issued about the mi
 
 # Deployment (FEC team only)
 
+*Special Note:* The requirements.txt field in the fecfile-web-api repo must be updated with the most recent commit hash for the commit changes to be pulled into the api build by CircleCI.
+
 ### Create a feature branch
 
 Using git-flow extensions: