diff --git a/.gitignore b/.gitignore index 56f09e45..2afffb96 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ local_settings.py */CACHE */package-lock.json *.editorconfig +cov_profile/* # See http://help.github.com/ignore-files/ for more about ignoring files. # compiled output @@ -22,7 +23,8 @@ local_settings.py /out-tsc # dependencies -*/node_modules +fecfile_validate_js/node_modules/* +node_modules/* # IDEs and editors *.idea diff --git a/README.md b/README.md index b88674df..1a8f3874 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,8 @@ is missing a value, the validation passes but with a warning issued about the mi # Deployment (FEC team only) +*Special Note:* The requirements.txt field in the fecfile-web-api repo must be updated with the most recent commit hash for the commit changes to be pulled into the api build by CircleCI. + ### Create a feature branch Using git-flow extensions: @@ -54,29 +56,30 @@ Without the git-flow extensions: ### Create a release branch -Using git-flow extensions: +* Using git-flow extensions: ``` git flow release start sprint-# ``` -Without the git-flow extensions: +* Without the git-flow extensions: ``` git checkout develop git pull git checkout -b release/sprint-# develop git push --set-upstream origin release/sprint-# ``` +* Developer creates a PR in GitHub to merge release/sprint-# branch into the `main` branch to track if commits pass deployment checks. The actual merge will happen when deploying a release to production. ### Create and deploy a hotfix -Using git-flow extensions: +* Using git-flow extensions: ``` git flow hotfix start my-fix # Work happens here git flow hotfix finish my-fix ``` -Without the git-flow extensions: +* Without the git-flow extensions: ``` git checkout -b hotfix/my-fix main # Work happens here @@ -88,7 +91,6 @@ git push --set-upstream origin hotfix/my-fix * Developer deploys hotfix/my-fix branch to main using **Deploying a release to production** instructions below ### Deploying a release to production -* Developer creates a PR in GitHub to merge release/sprint-# branch into the `main` branch * Reviewer approves PR and merges into `main` * Check CircleCI for passing pipeline tests * If tests pass, continue diff --git a/docs/Contact_Candidate.html b/docs/Contact_Candidate.html index ba613107..e2b59ea9 100644 --- a/docs/Contact_Candidate.html +++ b/docs/Contact_Candidate.html @@ -1,4 +1,4 @@ - FEC Candidate

FEC Candidate

Type: object

Candidate Contact

Type: const
Specific value: "CAN"
Example:

"CAN"
+ FEC Candidate 

FEC Candidate


Candidate Contact

Type: object

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. Otherwise, the conditions in the "Else" tab should be respected.

Type: object

Type: const
Specific value: "S"
Type: object

Type: string
Type: object

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. Otherwise, the conditions in the "Else" tab should be respected.

Type: object

Type: const
Specific value: "H"
Type: object

Type: string

Type: string

Type: const
Specific value: "CAN"
Example:

"CAN"
 

Type: string
Must match regular expression: ^[ A-Za-z0-9]{0,9}$

Must be at least 0 characters long

Must be at most 9 characters long


Example:

"H01234567"
 

Type: string
Must match regular expression: ^[ A-Za-z0-9]{0,30}$

Must be at least 0 characters long

Must be at most 30 characters long


Example:

"Smith"
 

Type: string
Must match regular expression: ^[ A-Za-z0-9]{0,20}$

Must be at least 0 characters long

Must be at most 20 characters long


Example:

"John"
@@ -7,10 +7,10 @@
 

Type: string or null
Must match regular expression: ^[ A-Za-z0-9]{0,10}$
Example:

"Jr"
 

Type: string
Must match regular expression: ^[ A-Za-z0-9]{0,34}$

Must be at least 0 characters long

Must be at most 34 characters long


Example:

"123 Main Street"
 

Type: string or null
Must match regular expression: ^[ A-Za-z0-9]{0,34}$

Type: string
Must match regular expression: ^[ A-Za-z0-9]{0,30}$

Must be at least 0 characters long

Must be at most 30 characters long


Example:

"Anytown"
-

Type: string
Must match regular expression: ^[ A-Za-z0-9]{0,2}$

Must be at least 0 characters long

Must be at most 2 characters long


Example:

"WA"
+

Type: string
Must match regular expression: ^[A-Z]{2}$

Must be at least 2 characters long

Must be at most 2 characters long


Example:

"WA"
 

Type: string
Must match regular expression: ^[ A-Za-z0-9]{0,9}$

Must be at least 0 characters long

Must be at most 9 characters long


Example:

981110123
 

Type: string or null
Must match regular expression: ^[ A-Za-z0-9]{0,38}$
Example:

"XYZ Company"
 

Type: string or null
Must match regular expression: ^[ A-Za-z0-9]{0,38}$
Example:

"QC Inspector"
 

Type: enum (of string)

Must be one of:

  • "H"
  • "S"
  • "P"

Example:

"H\nS\nP"
-

Type: string
Must match regular expression: ^[ A-Za-z0-9]{0,2}$

Must be at least 0 characters long

Must be at most 2 characters long


Example:

"WA"
-

Type: string
Must match regular expression: ^[ A-Za-z0-9]{0,2}$

Must be at least 0 characters long

Must be at most 2 characters long

Type: string or null
Must match regular expression: ^\d{10}$

Type: string
+

Type: string or null
Must match regular expression: ^[A-Z]{2}$
Example:

"WA"
+

Type: string or null
Must match regular expression: ^[0-9]{2}$

Type: string or null
Must match regular expression: ^\d{10}$

Type: string
\ No newline at end of file diff --git a/docs/Contact_Candidate_spec.html b/docs/Contact_Candidate_spec.html index 4bf3fd00..276b010e 100644 --- a/docs/Contact_Candidate_spec.html +++ b/docs/Contact_Candidate_spec.html @@ -1,3 +1,214 @@ -Contact_Candidate
Specification for Contact_Candidate
COL SEQFIELD DESCRIPTIONTYPEREQUIREDSAMPLE DATAVALUE REFERENCERULE REFERENCEFIELD FORM ASSOCIATION
1TYPEDropdownX (error)Individual Committee Organization Candidate
2CANDIDATE IDA/N-9X (error)H01234567
3LAST NAMEA/N-30X (error)Smith
4FIRST NAMEA/N-20X (error)John
5MIDDLE NAMEA/N-20W
6PREFIXA/N-10Dr
7SUFFIXA/N-10Jr
8STREET 1A/N-34X (error)123 Main Street
9STREET 2A/N-34
10CITYA/N-30X (error)Anytown
11STATEA/N-2X (error)WAEdit: ST. Default to ZZ if Country DNE US
12ZIPA/N-9X (error)981110123
13EMPLOYERA/N-38XYZ CompanyReq if Donor aggregate >$200
14OCCUPATIONA/N-38QC InspectorReq if Donor aggregate >$200
15CANDIDATE OFFICEDropdownX (error)House -Senate -Presidential
16CANDIDATE STATEA/N-2X (error)WARequired if Office = H or S
17CANDIDATE DISTRICTA/N-2X (error)Required if Office = H
TELEPHONEN-10
COUNTRYDropdownX (error)Should default to United States
\ No newline at end of file + + + + Contact_Candidate + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Specification for Contact_Candidate +
COL SEQFIELD DESCRIPTIONTYPEREQUIREDSAMPLE DATAVALUE REFERENCERULE REFERENCEFIELD FORM ASSOCIATION
1TYPEDropdownX (error)Individual Committee Organization Candidate
2CANDIDATE IDA/N-9X (error)H01234567
3LAST NAMEA/N-30X (error)Smith
4FIRST NAMEA/N-20X (error)John
5MIDDLE NAMEA/N-20W
6PREFIXA/N-10Dr
7SUFFIXA/N-10Jr
8STREET 1A/N-34X (error)123 Main Street
9STREET 2A/N-34
10CITYA/N-30X (error)Anytown
11STATEA/N-2X (error)WAEdit: ST. Default to ZZ if Country DNE US
12ZIPA/N-9X (error)981110123
13EMPLOYERA/N-38XYZ CompanyReq if Donor aggregate >$200
14OCCUPATIONA/N-38QC InspectorReq if Donor aggregate >$200
15CANDIDATE OFFICEDropdownX (error)House Senate Presidential
16CANDIDATE STATEA/N-2X (error)WARequired if Office = H or S
17CANDIDATE DISTRICTA/N-2X (error)Required if Office = H
TELEPHONEN-10
COUNTRYDropdownX (error)Should default to United States
+ + diff --git a/docs/Contact_Committee_spec.html b/docs/Contact_Committee_spec.html index 64636039..e83cbe30 100644 --- a/docs/Contact_Committee_spec.html +++ b/docs/Contact_Committee_spec.html @@ -1 +1,124 @@ -Contact_Committee
Specification for Contact_Committee
COL SEQFIELD DESCRIPTIONTYPEREQUIREDSAMPLE DATAVALUE REFERENCERULE REFERENCEFIELD FORM ASSOCIATION
1TYPEDropdownX (error)Individual Committee Organization Candidate
2COMMITTEE IDA/N-9X (error)c01234567
3NAMEA/N-200X (error)SEIU COPE (Service Employees International Union Committee On Political Education)
4STREET 1A/N-34X (error)123 Main Street
5STREET 2A/N-34
6CITYA/N-30X (error)Anytown
7STATEA/N-2X (error)WAEdit: ST. Default to ZZ if Country DNE US
8ZIPA/N-9X (error)981110123
12TELEPHONEN-10
COUNTRYDropdownX (error)Should default to United States
\ No newline at end of file + + + + Contact_Committee + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Specification for Contact_Committee +
COL SEQFIELD DESCRIPTIONTYPEREQUIREDSAMPLE DATAVALUE REFERENCERULE REFERENCEFIELD FORM ASSOCIATION
1TYPEDropdownX (error)Individual Committee Organization Candidate
2COMMITTEE IDA/N-9X (error)c01234567
3NAMEA/N-200X (error)SEIU COPE (Service Employees International Union Committee On Political Education)
4STREET 1A/N-34X (error)123 Main Street
5STREET 2A/N-34
6CITYA/N-30X (error)Anytown
7STATEA/N-2X (error)WAEdit: ST. Default to ZZ if Country DNE US
8ZIPA/N-9X (error)981110123
12TELEPHONEN-10
COUNTRYDropdownX (error)Should default to United States
+ + diff --git a/docs/Contact_Individual_spec.html b/docs/Contact_Individual_spec.html index ba9e04cf..0f2ba5d9 100644 --- a/docs/Contact_Individual_spec.html +++ b/docs/Contact_Individual_spec.html @@ -1 +1,174 @@ -Contact_Individual
Specification for Contact_Individual
COL SEQFIELD DESCRIPTIONTYPEREQUIREDSAMPLE DATAVALUE REFERENCERULE REFERENCEFIELD FORM ASSOCIATION
1TYPEDropdownX (error)Individual Committee Organization Candidate
2LAST NAMEA/N-30X (error)Smith
3FIRST NAMEA/N-20X (error)John
4MIDDLE NAMEA/N-20W
5PREFIXA/N-10Dr
6SUFFIXA/N-10Jr
7STREET 1A/N-34X (error)123 Main Street
8STREET 2A/N-34
9CITYA/N-30X (error)Anytown
10STATEA/N-2X (error)WAEdit: ST. Default to ZZ if Country DNE US
11ZIPA/N-9X (error)981110123
12TELEPHONEN-10
13EMPLOYERA/N-38XYZ CompanyReq if Donor aggregate >$200
14OCCUPATIONA/N-38QC InspectorReq if Donor aggregate >$200
COUNTRYDropdownX (error)Should default to United States
\ No newline at end of file + + + + Contact_Individual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Specification for Contact_Individual +
COL SEQFIELD DESCRIPTIONTYPEREQUIREDSAMPLE DATAVALUE REFERENCERULE REFERENCEFIELD FORM ASSOCIATION
1TYPEDropdownX (error)Individual Committee Organization Candidate
2LAST NAMEA/N-30X (error)Smith
3FIRST NAMEA/N-20X (error)John
4MIDDLE NAMEA/N-20W
5PREFIXA/N-10Dr
6SUFFIXA/N-10Jr
7STREET 1A/N-34X (error)123 Main Street
8STREET 2A/N-34
9CITYA/N-30X (error)Anytown
10STATEA/N-2X (error)WAEdit: ST. Default to ZZ if Country DNE US
11ZIPA/N-9X (error)981110123
12TELEPHONEN-10
13EMPLOYERA/N-38XYZ CompanyReq if Donor aggregate >$200
14OCCUPATIONA/N-38QC InspectorReq if Donor aggregate >$200
COUNTRYDropdownX (error)Should default to United States
+ + diff --git a/docs/Contact_Organization_spec.html b/docs/Contact_Organization_spec.html index f17238d9..1cd63cc1 100644 --- a/docs/Contact_Organization_spec.html +++ b/docs/Contact_Organization_spec.html @@ -1 +1,114 @@ -Contact_Organization
Specification for Contact_Organization
COL SEQFIELD DESCRIPTIONTYPEREQUIREDSAMPLE DATAVALUE REFERENCERULE REFERENCEFIELD FORM ASSOCIATION
1TYPEDropdownX (error)Individual Committee Organization CandidateRequired if NOT [IND|CAN]
8NAMEA/N-200X (error)John Smith & Co.
14STREET 1A/N-34X (error)123 Main Street
15STREET 2A/N-34
16CITYA/N-30X (error)AnytownAK,AL,...,ZZEdit: ST. Default to ZZ if Country DNE US
17STATEA/N-2X (error)WA
18ZIPA/N-9X (error)981110123
12TELEPHONEN-10
COUNTRYDropdownX (error)Should default to United States
\ No newline at end of file + + + + Contact_Organization + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Specification for Contact_Organization +
COL SEQFIELD DESCRIPTIONTYPEREQUIREDSAMPLE DATAVALUE REFERENCERULE REFERENCEFIELD FORM ASSOCIATION
1TYPEDropdownX (error)Individual Committee Organization CandidateRequired if NOT [IND|CAN]
8NAMEA/N-200X (error)John Smith & Co.
14STREET 1A/N-34X (error)123 Main Street
15STREET 2A/N-34
16CITYA/N-30X (error)AnytownAK,AL,...,ZZEdit: ST. Default to ZZ if Country DNE US
17STATEA/N-2X (error)WA
18ZIPA/N-9X (error)981110123
12TELEPHONEN-10
COUNTRYDropdownX (error)Should default to United States
+ + diff --git a/fecfile_validate_js/.eslintrc.json b/fecfile_validate_js/.eslintrc.json index 963b2965..a8b313d0 100644 --- a/fecfile_validate_js/.eslintrc.json +++ b/fecfile_validate_js/.eslintrc.json @@ -6,7 +6,6 @@ ], "extends": [ "eslint:recommended", - "plugin:@typescript-eslint/eslint-recommended", "plugin:@typescript-eslint/recommended" ], "rules": { diff --git a/fecfile_validate_js/src/index.ts b/fecfile_validate_js/src/index.ts index 4473d87c..909ed122 100644 --- a/fecfile_validate_js/src/index.ts +++ b/fecfile_validate_js/src/index.ts @@ -6,7 +6,7 @@ * Tested with spec/index-spec.js */ -import Ajv, { ValidateFunction } from 'ajv'; +import Ajv, { ErrorObject, ValidateFunction } from 'ajv'; /** * Validation error information for a single schema property @@ -31,26 +31,40 @@ const ajv = new Ajv({ allErrors: true, strictSchema: false }); * * @param {object} schema * @param {object} data + * @param {string[]} fieldsToValidate * @returns {ValidationError[]} Modified version of Ajv output, empty array if no errors found */ -export function validate(schema: any, data: any): ValidationError[] { - const theSchemaUrl = schema['$schema']; - schema['$schema'] = theSchemaUrl.replace('https', 'http'); - +export function validate(schema: any, data: any, fieldsToValidate: string[] = []): ValidationError[] { const validator: ValidateFunction = ajv.compile(schema); const isValid: boolean = validator(data); const errors: ValidationError[] = []; - if (!isValid && !!validator.errors) { - validator.errors.forEach((err) => { - errors.push({ - path: err.instancePath.substring(1), - keyword: err.keyword, - params: err.params, - message: !!err.message ? err.message : null, - }); + if (!isValid && !!validator.errors?.length) { + validator.errors.forEach((error) => { + const parsedError = parseError(error); + if (!fieldsToValidate.length || fieldsToValidate.includes(parsedError.path)) { + errors.push(parsedError); + } }); } return errors; } + +/** + * Format error message from Ajv into our ValidationError interface + * @param {ErrorObject} error - Ajv ErrorObject interface + * @returns {ValidationError} + */ +function parseError(error: ErrorObject): ValidationError { + let path = error.instancePath.substring(1); + if (error.keyword == 'required') { + path = error.params.missingProperty; + } + return { + path: path, + keyword: error.keyword, + params: error.params, + message: !!error.message ? error.message : null, + }; +} diff --git a/fecfile_validate_js/tests/buildSchemaModules.test.ts b/fecfile_validate_js/tests/buildSchemaModules.test.ts new file mode 100644 index 00000000..25758489 --- /dev/null +++ b/fecfile_validate_js/tests/buildSchemaModules.test.ts @@ -0,0 +1,24 @@ +import { assertEquals } from 'https://deno.land/std/testing/asserts.ts'; +import { existsSync } from 'https://deno.land/std/fs/mod.ts'; +import * as path from 'https://deno.land/std/path/mod.ts'; + +const __dirname = path.dirname(path.fromFileUrl(import.meta.url)); + +const process = Deno.run({ cmd: ['node', 'fecfile_validate_js/scripts/buildSchemaModules'] }); +await process.status(); + +Deno.test({ + name: 'it should create schema *.js files', + fn: () => { + const result: boolean = existsSync(`${__dirname}/../dist/F3X.js`); + assertEquals(result, true); + }, +}); + +Deno.test({ + name: 'it should create schema *.d.ts files', + fn: () => { + const result: boolean = existsSync(`${__dirname}/../dist/F3X.d.ts`); + assertEquals(result, true); + }, +}); diff --git a/fecfile_validate_js/tests/candidate.test.ts b/fecfile_validate_js/tests/candidate.test.ts new file mode 100644 index 00000000..63e91aa7 --- /dev/null +++ b/fecfile_validate_js/tests/candidate.test.ts @@ -0,0 +1,156 @@ +import { assertEquals } from 'https://deno.land/std/testing/asserts.ts'; +import { validate } from '../dist/index.js'; +import { schema as candidateContactSchema } from '../dist/Contact_Candidate.js'; + +const data: any = { + type: 'CAN', + candidate_id: 'H01234567', + committee_id: 'C00601211', + name: 'Gilbert Smith', + last_name: 'Smith', + first_name: 'Gilbert', + middle_name: null, + prefix: null, + suffix: null, + street_1: '602 Tlumacki St', + street_2: null, + city: 'Mclean City', + state: 'VA', + zip: '22204', + employer: null, + occupation: null, + candidate_office: 'P', + candidate_state: null, + candidate_district: null, + telephone: '3043892120', + country: 'US', +}; + +Deno.test({ + name: 'it should pass with perfect data', + fn: () => { + const result = validate(candidateContactSchema, data); + assertEquals(result, []); + }, +}); + +Deno.test({ + name: 'it should fail as S candidate office missing', + fn: () => { + const testData = { ...data }; + delete testData.candidate_office; + const result = validate(candidateContactSchema, testData); + assertEquals(result[0].keyword, 'required'); + assertEquals(result[0].path, 'candidate_office'); + assertEquals(result[0].message, "must have required property 'candidate_office'"); + }, +}); + +Deno.test({ + name: 'it should fail as S candidate office and null for candidate_state', + fn: () => { + const testData = { ...data }; + testData.candidate_office = 'S'; + const result = validate(candidateContactSchema, testData); + assertEquals(result[0].path, 'candidate_state'); + assertEquals(result[0].message, 'must be string'); + }, +}); + +Deno.test({ + name: 'it should fail as S candidate office and candidate_state missing', + fn: () => { + const testData = { ...data }; + testData.candidate_office = 'S'; + delete testData.candidate_state; + const result = validate(candidateContactSchema, testData); + assertEquals(result[0].keyword, 'required'); + assertEquals(result[0].path, 'candidate_state'); + assertEquals(result[0].message, "must have required property 'candidate_state'"); + }, +}); + +Deno.test({ + name: 'it should fail with for candidate office S with an invalid candidate state format', + fn: () => { + const testData = { ...data }; + testData.candidate_office = 'S'; + testData.candidate_state = 'M1'; + const result = validate(candidateContactSchema, testData); + assertEquals(result[0].path, 'candidate_state'); + assertEquals(result[0].message, 'must match pattern "^[A-Z]{2}$"'); + }, +}); + +Deno.test({ + name: 'it should pass with for candidate office S with a candidate state', + fn: () => { + const testData = { ...data }; + testData.candidate_office = 'S'; + testData.candidate_state = 'VA'; + const result = validate(candidateContactSchema, testData); + assertEquals(result, []); + }, +}); + +Deno.test({ + name: 'it should fail as H candidate office and null for candidate_state', + fn: () => { + const testData = { ...data }; + testData.candidate_office = 'H'; + const result = validate(candidateContactSchema, testData); + assertEquals(result[0].path, 'candidate_state'); + assertEquals(result[0].message, 'must be string'); + }, +}); + +Deno.test({ + name: 'it should fail as H candidate office and candidate_state missing', + fn: () => { + const testData = { ...data }; + testData.candidate_office = 'H'; + delete testData.candidate_state; + const result = validate(candidateContactSchema, testData); + assertEquals(result[0].keyword, 'required'); + assertEquals(result[0].path, 'candidate_state'); + assertEquals(result[0].message, "must have required property 'candidate_state'"); + }, +}); + +Deno.test({ + name: 'it should fail as H candidate office and null for candidate_district', + fn: () => { + const testData = { ...data }; + testData.candidate_office = 'H'; + testData.candidate_state = 'MD'; + const result = validate(candidateContactSchema, testData); + assertEquals(result[0].path, 'candidate_district'); + assertEquals(result[0].message, 'must be string'); + }, +}); + +Deno.test({ + name: 'it should fail as H candidate office and candidate_district missing', + fn: () => { + const testData = { ...data }; + testData.candidate_office = 'H'; + testData.candidate_state = 'VA'; + delete testData.candidate_district; + const result = validate(candidateContactSchema, testData); + assertEquals(result[0].keyword, 'required'); + assertEquals(result[0].path, 'candidate_district'); + assertEquals(result[0].message, "must have required property 'candidate_district'"); + }, +}); + +Deno.test({ + name: 'it should pass with for candidate office H with a candidate state and candidate district', + fn: () => { + const testData = { ...data }; + testData.candidate_office = 'H'; + testData.candidate_state = 'VA'; + testData.candidate_district = '01'; + const result = validate(candidateContactSchema, testData); + assertEquals(result, []); + }, +}); diff --git a/fecfile_validate_js/tests/index.test.ts b/fecfile_validate_js/tests/f3x.test.ts similarity index 84% rename from fecfile_validate_js/tests/index.test.ts rename to fecfile_validate_js/tests/f3x.test.ts index e2f6cb94..f2cf01e3 100644 --- a/fecfile_validate_js/tests/index.test.ts +++ b/fecfile_validate_js/tests/f3x.test.ts @@ -1,4 +1,4 @@ -import { assertEquals } from 'https://deno.land/std@0.133.0/testing/asserts.ts'; +import { assertEquals } from 'https://deno.land/std/testing/asserts.ts'; import { validate } from '../dist/index.js'; import { schema as f3xSchema } from '../dist/F3X.js'; @@ -43,6 +43,7 @@ Deno.test({ delete thisData.form_type; const result = validate(f3xSchema, thisData); assertEquals(result[0].keyword, 'required'); + assertEquals(result[0].path, 'form_type'); assertEquals(result[0].params.missingProperty, 'form_type'); }, }); @@ -109,6 +110,7 @@ Deno.test({ delete thisData.filer_committee_id_number; const result = validate(f3xSchema, thisData); assertEquals(result[0].keyword, 'required'); + assertEquals(result[0].path, 'filer_committee_id_number'); assertEquals(result[0].params.missingProperty, 'filer_committee_id_number'); }, }); @@ -151,6 +153,7 @@ Deno.test({ delete thisData.treasurer_first_name; const result = validate(f3xSchema, thisData); assertEquals(result[0].keyword, 'required'); + assertEquals(result[0].path, 'treasurer_first_name'); assertEquals(result[0].params.missingProperty, 'treasurer_first_name'); }, }); @@ -173,6 +176,7 @@ Deno.test({ delete thisData.treasurer_last_name; const result = validate(f3xSchema, thisData); assertEquals(result[0].keyword, 'required'); + assertEquals(result[0].path, 'treasurer_last_name'); assertEquals(result[0].params.missingProperty, 'treasurer_last_name'); }, }); @@ -195,6 +199,33 @@ Deno.test({ delete thisData.date_signed; const result = validate(f3xSchema, thisData); assertEquals(result[0].keyword, 'required'); + assertEquals(result[0].path, 'date_signed'); assertEquals(result[0].params.missingProperty, 'date_signed'); }, }); + + +Deno.test({ + name: 'it should pass with perfect partial data', + fn: () => { + const thisData = { ...perfectForm_F3X}; + const fieldsToValidate = ['form_type']; + // Should still succeed without date_signed because we are only testing form_type + delete thisData.date_signed; + const result = validate(f3xSchema, perfectForm_F3X, fieldsToValidate); + assertEquals(result, []); + }, +}); + +Deno.test({ + name: 'it should fail without form_type', + fn: () => { + const thisData = { ...perfectForm_F3X }; + const fieldsToValidate = ['form_type']; + delete thisData.form_type; + const result = validate(f3xSchema, thisData, fieldsToValidate); + assertEquals(result[0].keyword, 'required'); + assertEquals(result[0].path, 'form_type'); + assertEquals(result[0].params.missingProperty, 'form_type'); + }, +}); diff --git a/fecfile_validate_python/src/fecfile_validate/validate.py b/fecfile_validate_python/src/fecfile_validate/validate.py index c0eceb8a..035a8be3 100644 --- a/fecfile_validate_python/src/fecfile_validate/validate.py +++ b/fecfile_validate_python/src/fecfile_validate/validate.py @@ -30,8 +30,7 @@ def get_schema(schema_name): Returns: dict: JSON schema that matches the schema_name""" schema_file = f"{schema_name}.json" - schema_path = os.path.join(os.path.dirname(__file__), "schema/", - schema_file) + schema_path = os.path.join(os.path.dirname(__file__), "schema/", schema_file) #: Handle case where we are not running from a pip package if not os.path.isfile(schema_path): logger.warning(f"Schema file ({schema_path}) not found in package.") @@ -63,7 +62,7 @@ def parse_schema_error(error): return ValidationError(error.message, path) -def validate(schema_name, form_data): +def validate(schema_name, form_data, fields_to_validate=None): """Wrapper function around jsonschema validator Args: @@ -71,10 +70,20 @@ def validate(schema_name, form_data): JSON schema definition file data (dict): key-value pairs of data to be validated. Keys match the properties of the JSON schema file + fields_to_validate (list): list of property names to validate. + Properties from the schema not in this list will not be + validated on, even if they are required. Returns: list of ValidationError: A list of all errors found in form_data""" form_schema = get_schema(schema_name) + validator = Draft7Validator(form_schema) errors = list(map(parse_schema_error, validator.iter_errors(form_data))) + if fields_to_validate: + + def in_fields_to_validate(field): + return field.path in fields_to_validate + + errors = list(filter(in_fields_to_validate, errors)) return ValidationResult(errors, []) diff --git a/fecfile_validate_python/tests/contacts/sample_contact_CAN.json b/fecfile_validate_python/tests/contacts/sample_contact_CAN.json new file mode 100644 index 00000000..d3affd6f --- /dev/null +++ b/fecfile_validate_python/tests/contacts/sample_contact_CAN.json @@ -0,0 +1,23 @@ +{ + "type": "CAN", + "candidate_id": "H01234567", + "committee_id": "C00601211", + "name": "Gilbert Smith", + "last_name": "Smith", + "first_name": "Gilbert", + "middle_name": null, + "prefix": null, + "suffix": null, + "street_1": "602 Tlumacki St", + "street_2": null, + "city": "Mclean City", + "state": "VA", + "zip": "22204", + "employer": null, + "occupation": null, + "candidate_office": "P", + "candidate_state": null, + "candidate_district": null, + "telephone": "3043892120", + "country": "US" +} diff --git a/fecfile_validate_python/tests/sample_IND_contact.json b/fecfile_validate_python/tests/contacts/sample_contact_IND.json similarity index 94% rename from fecfile_validate_python/tests/sample_IND_contact.json rename to fecfile_validate_python/tests/contacts/sample_contact_IND.json index db633263..b2a80461 100644 --- a/fecfile_validate_python/tests/sample_IND_contact.json +++ b/fecfile_validate_python/tests/contacts/sample_contact_IND.json @@ -15,7 +15,7 @@ "zip": "22204", "employer": "Byron Inc", "occupation": "Business Owner", - "candidate_office": "P", + "candidate_office": null, "candidate_state": null, "candidate_district": null, "telephone": "3043892120", diff --git a/fecfile_validate_python/tests/contacts/test_contact_schema.py b/fecfile_validate_python/tests/contacts/test_contact_schema.py new file mode 100644 index 00000000..7fa0a669 --- /dev/null +++ b/fecfile_validate_python/tests/contacts/test_contact_schema.py @@ -0,0 +1,82 @@ +import pytest +import json +import os +from src.fecfile_validate import validate + + +@pytest.fixture +def sample_ind_contact(): + with open(os.path.join(os.path.dirname(__file__), + "sample_contact_IND.json")) as f: + form_data = json.load(f) + return form_data + + +@pytest.fixture +def sample_can_contact(): + with open(os.path.join(os.path.dirname(__file__), + "sample_contact_CAN.json")) as f: + form_data = json.load(f) + return form_data + + +def test_invalid_const_value(sample_ind_contact): + # Make sure our Individual Contact schema is valid + validation_result = validate.validate("Contact_Individual", + sample_ind_contact) + assert validation_result.errors == [] + + # Check the const type property works by setting an invalid "type" property + sample_ind_contact["type"] = "Individual" + validation_result = validate.validate("Contact_Individual", + sample_ind_contact) + assert validation_result.errors[0].path == "type" + assert validation_result.errors[0].message == "'IND' was expected" + + +def test_required_candidate_state(sample_can_contact): + # Make sure our presidential candidate contact schema is valid + validation_result = validate.validate("Contact_Candidate", + sample_can_contact) + assert validation_result.errors == [] + + # Fail null for candidate state when office is S + sample_can_contact["candidate_office"] = "S" + validation_result = validate.validate("Contact_Candidate", + sample_can_contact) + assert validation_result.errors[0].path == "candidate_state" + assert validation_result.errors[0].message == \ + "None is not of type 'string'" + + # Fail not having candidate state at all when office is S + del sample_can_contact["candidate_state"] + validation_result = validate.validate("Contact_Candidate", + sample_can_contact) + assert validation_result.errors[0].path == "candidate_state" + assert validation_result.errors[0].message == \ + "'candidate_state' is a required property" + + # Fail null for candidate district when office is H + sample_can_contact["candidate_state"] = "VA" + sample_can_contact["candidate_office"] = "H" + validation_result = validate.validate("Contact_Candidate", + sample_can_contact) + assert validation_result.errors[0].path == "candidate_district" + assert validation_result.errors[0].message == \ + "None is not of type 'string'" + + # Fail invalid candidate district code + sample_can_contact["candidate_district"] = "ZZ" + validation_result = validate.validate("Contact_Candidate", + sample_can_contact) + assert validation_result.errors[0].path == "candidate_district" + assert validation_result.errors[0].message == \ + "'ZZ' does not match '^[0-9]{2}$'" + + # Fail not having a candidate district when office is H + del sample_can_contact["candidate_district"] + validation_result = validate.validate("Contact_Candidate", + sample_can_contact) + assert validation_result.errors[0].path == "candidate_district" + assert validation_result.errors[0].message == \ + "'candidate_district' is a required property" diff --git a/fecfile_validate_python/tests/test_validate.py b/fecfile_validate_python/tests/test_validate.py index 4fd4ee4f..e650fa02 100644 --- a/fecfile_validate_python/tests/test_validate.py +++ b/fecfile_validate_python/tests/test_validate.py @@ -12,18 +12,9 @@ def sample_f3x(): return form_data -@pytest.fixture -def sample_ind_contact(): - with open(os.path.join(os.path.dirname(__file__), - "sample_IND_contact.json")) as f: - form_data = json.load(f) - return form_data - - @pytest.fixture def test_schema(): - with open(os.path.join(os.path.dirname(__file__), - "test_schema.json")) as f: + with open(os.path.join(os.path.dirname(__file__), "test_schema.json")) as f: schema = json.load(f) return schema @@ -35,14 +26,11 @@ def test_is_correct(sample_f3x): def test_missing_required_field(sample_f3x): # Create error by removing FORM_TYPE - sample_f3x["form_type"] = "" + sample_f3x.pop("form_type", None) validation_result = validate.validate("F3X", sample_f3x) assert validation_result.errors[0].path == "form_type" - assert ( - validation_result.errors[0].message - == "'' is not one of ['F3XN', 'F3XA', 'F3XT']" - ) + assert validation_result.errors[0].message == "'form_type' is a required property" def test_invalid_string_character(sample_f3x): @@ -56,11 +44,17 @@ def test_invalid_string_character(sample_f3x): def test_non_required_field(sample_f3x): - sample_f3x["treasurer_middle_name"] = None + sample_f3x.pop("treasurer_middle_name", None) validation_result = validate.validate("F3X", sample_f3x) assert validation_result.errors == [] +"""Test parse_schema_error +parse_schema_error needs to parse jsonschema errors and +turn them into ValidationErrors +""" + + def check_error(validation_error, message, path): expected_error = validate.ValidationError(message, path) assert validation_error.path == expected_error.path @@ -97,15 +91,24 @@ def test_parse_required_error(test_schema): ) -def test_invalid_const_value(sample_ind_contact): - # Make sure our Individual Contact schema is valid - validation_result = validate.validate("Contact_Individual", - sample_ind_contact) +"""Test partial validation +Calling validate with fields_to_validate contrstructs a temporary +schema from the passed schema. This temporary schema must be a subschema +of the passed schema""" + + +def test_partial_validate_is_correct(sample_f3x): + fields_to_validate = ["form_type"] + # Even though we remove date_signed, the validation should pass + sample_f3x.pop("date_signed", None) + validation_result = validate.validate("F3X", sample_f3x, fields_to_validate) assert validation_result.errors == [] - # Check the const type property works by setting an invalid "type" property - sample_ind_contact["type"] = "Individual" - validation_result = validate.validate("Contact_Individual", - sample_ind_contact) - assert validation_result.errors[0].path == "type" - assert validation_result.errors[0].message == "'IND' was expected" + +def test_partial_missing_required_field(sample_f3x): + fields_to_validate = ["form_type"] + # Create error by removing FORM_TYPE + sample_f3x.pop("form_type", None) + validation_result = validate.validate("F3X", sample_f3x, fields_to_validate) + assert validation_result.errors[0].path == "form_type" + assert validation_result.errors[0].message == "'form_type' is a required property" diff --git a/package.json b/package.json index 35e1485e..57b46122 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "build": "rm -rf fecfile_validate_js/dist && tsc --project fecfile_validate_js/tsconfig.json && node fecfile_validate_js/scripts/buildSchemaModules", "postinstall": "npm run build", "lint": "eslint ./fecfile_validate_js/src --ext .ts", - "test": "npm run build && deno test --compat --unstable --allow-read fecfile_validate_js/tests/index.test.ts" + "test": "npm run build && deno test --coverage=cov_profile --compat --unstable --allow-read --allow-run fecfile_validate_js/tests/*.test.ts" }, "repository": { "type": "git", diff --git a/schema/Contact_Candidate.json b/schema/Contact_Candidate.json index cb4db2cc..4ede399e 100644 --- a/schema/Contact_Candidate.json +++ b/schema/Contact_Candidate.json @@ -15,8 +15,6 @@ "state", "zip", "candidate_office", - "candidate_state", - "candidate_district", "country" ], "fec_recommended": [], @@ -222,9 +220,9 @@ "title": "STATE", "description": "", "type": "string", - "minLength": 0, + "minLength": 2, "maxLength": 2, - "pattern": "^[ A-Za-z0-9]{0,2}$", + "pattern": "^[A-Z]{2}$", "examples": ["WA"], "fec_spec": { "COL_SEQ": 11, @@ -319,10 +317,10 @@ "candidate_state": { "title": "CANDIDATE STATE", "description": "", - "type": "string", - "minLength": 0, + "type": ["string", "null"], + "minLength": 2, "maxLength": 2, - "pattern": "^[ A-Za-z0-9]{0,2}$", + "pattern": "^[A-Z]{2}$", "examples": ["WA"], "fec_spec": { "COL_SEQ": 16, @@ -339,10 +337,10 @@ "candidate_district": { "title": "CANDIDATE DISTRICT", "description": "", - "type": "string", - "minLength": 0, + "type": ["string", "null"], + "minLength": 2, "maxLength": 2, - "pattern": "^[ A-Za-z0-9]{0,2}$", + "pattern": "^[0-9]{2}$", "fec_spec": { "COL_SEQ": 17, "FIELD_DESCRIPTION": "CANDIDATE DISTRICT", @@ -390,5 +388,34 @@ "FIELD_FORM_ASSOCIATION": null } } - } + }, + "allOf": [ + { + "if": { + "properties": { + "candidate_office": { "const": "S" } + }, + "required": ["candidate_office"] + }, + "then": { + "required": ["candidate_state"], + "properties": { "candidate_state": { "type": "string" } } + } + }, + { + "if": { + "properties": { + "candidate_office": { "const": "H" } + }, + "required": ["candidate_office"] + }, + "then": { + "required": ["candidate_state", "candidate_district"], + "properties": { + "candidate_state": { "type": "string" }, + "candidate_district": { "type": "string" } + } + } + } + ] }