diff --git a/bin/generate-starter-schema.py b/bin/generate-starter-schema.py index f688fd3c..17b5d19a 100644 --- a/bin/generate-starter-schema.py +++ b/bin/generate-starter-schema.py @@ -21,22 +21,36 @@ import os import re -parser = argparse.ArgumentParser(description='Convert the FEC validation Excel' - ' spreadsheet into JSON schema documents.') -parser.add_argument('excel_filename', help='an excel filename that will be' - ' parsed to generate JSON schema docs') -parser.add_argument('--sheets-to-generate', help='a json file containing an' - ' array of sheet names to be parsed from the excel file') -parser.add_argument('--version') +JSON_EXT = ".json" + +parser = argparse.ArgumentParser( + description="Convert the FEC validation Excel" + " spreadsheet into JSON schema documents." +) +parser.add_argument( + "excel_filename", + help="an excel filename that will be parsed to generate JSON schema docs", +) +parser.add_argument( + "--sheets-to-generate", + help="a json file containing an" + " array of sheet names to be parsed from the excel file", +) +parser.add_argument("--version") args = parser.parse_args() -EXCEL_FILENAME = args.excel_filename or \ - "Form_3X_Receipts_Vendor_10.20.2020.xlsx" -SCHEMA_ID_PREFIX = ("https://github.com/fecgov/fecfile-validate/blob/" - "main/schema") +EXCEL_FILENAME = args.excel_filename or "Form_3X_Receipts_Vendor_10.20.2020.xlsx" +SCHEMA_ID_PREFIX = "https://github.com/fecgov/fecfile-validate/blob/main/schema" VERSION = args.version or "v0.0.0.0" -SHEETS_TO_SKIP = ['All receipts', 'Version 8.3', 'SUMMARY OF CHANGES', - "All Schedule A Transactions", "ScheduleC", "Schedule C1", - "Scedule C2"] +SHEETS_TO_SKIP = [ + "All receipts", + "Version 8.3", + "SUMMARY OF CHANGES", + "All Schedule A Transactions", + "ScheduleC", + "Schedule C1", + "Scedule C2", + "All disbursements", +] # Column postions of fields in the spreadsheet row array @@ -53,12 +67,12 @@ class Columns(Enum): FIELD_FORM_ASSOCIATION = 8 def get(self, row, has_autopopulate): - index = self.value if has_autopopulate or \ - self.value <= 3 else self.value - 1 + index = self.value if has_autopopulate or self.value <= 3 else self.value - 1 value = row[index] if index < len(row) else None return value.strip() if isinstance(value, str) else value -def convert_row_to_property(row, sheet_has_autopopulate):# noqa + +def convert_row_to_property(row, sheet_has_autopopulate): # noqa """Take a row from the spreadsheet and convert it into a schema object. Args: @@ -75,33 +89,40 @@ def convert_row_to_property(row, sheet_has_autopopulate):# noqa if col != Columns.AUTO_POPULATE or sheet_has_autopopulate: spec[col.name] = col.get(row, sheet_has_autopopulate) - title = spec.get(Columns.FIELD_DESCRIPTION.name) + title = str(spec.get(Columns.FIELD_DESCRIPTION.name)) field_type = spec.get(Columns.TYPE.name) required = spec.get(Columns.REQUIRED.name) sample_data = spec.get(Columns.SAMPLE_DATA.name) rule_ref = spec.get(Columns.RULE_REFERENCE.name) - token = title.replace("\n", "_").replace(" ", "_").replace(".", "")\ - .replace("(", "").replace(")", "").replace("/", "_")\ - .replace("__", "_").lower() + token = ( + title.replace("\n", "_") + .replace(" ", "_") + .replace(".", "") + .replace("(", "") + .replace(")", "") + .replace("/", "_") + .replace("__", "_") + .lower() + ) # Prepend tokens that start with a number (presumed to be a line number) # with capital letter "L". if token[0].isdigit(): - token = 'L' + token + token = "L" + token prop["title"] = title prop["description"] = "" if field_type.startswith("AMT-"): prop["type"] = "number" prop["minimum"] = 0 - prop["maximum"] = int('9' * int(field_type.split('-')[1])) + prop["maximum"] = int("9" * int(field_type.split("-")[1])) if field_type.startswith("NUM-") or field_type.startswith("N-"): - length = field_type.split('-')[1].strip() + length = field_type.split("-")[1].strip() prop["type"] = "string" prop["minLength"] = 0 prop["maxLength"] = int(length) - prop["pattern"] = rf'^\d{{0,{length}}}$' + prop["pattern"] = rf"^\d{{0,{length}}}$" if field_type == "Dropdown": prop["type"] = "string" @@ -112,11 +133,11 @@ def convert_row_to_property(row, sheet_has_autopopulate):# noqa if field_type == "A-1" and rule_ref == "Check-box": prop["type"] = "boolean" else: - length = field_type.split('-')[1].strip() + length = field_type.split("-")[1].strip() prop["type"] = "string" prop["minLength"] = 0 prop["maxLength"] = int(length) - prop["pattern"] = f'^[ A-Za-z0-9]{{0,{length}}}$' + prop["pattern"] = f"^[ -~]{0,9}$" if sample_data: prop["examples"] = [sample_data] @@ -131,51 +152,59 @@ def convert_row_to_property(row, sheet_has_autopopulate):# noqa wb = openpyxl.load_workbook(EXCEL_FILENAME) sheets_to_generate = None if args.sheets_to_generate is not None: - with open(os.path.join(os.getcwd(), args.sheets_to_generate), 'r') as f: + with open(os.path.join(os.getcwd(), args.sheets_to_generate), "r") as f: sheets_to_generate = json.load(f) print(sheets_to_generate) trans_type_hits = {} for ws in wb.worksheets: - if ((sheets_to_generate is not None and ws.title not in sheets_to_generate) - or ws.title in SHEETS_TO_SKIP): + if ( + sheets_to_generate is not None and ws.title not in sheets_to_generate + ) or ws.title in SHEETS_TO_SKIP: continue print(ws.title) - title = ws.title.replace(' ', '') - output_file = title + ".json" - print(f'Parsing {output_file}...') + title = ws.title.replace(" ", "") + output_file = title + JSON_EXT + + print(f"Parsing {output_file}...") - sheet_has_autopopulate = ws.cell(3, 5).value is not None and \ - ws.cell(3, 5).value.strip() == 'Auto populate' + sheet_has_autopopulate = ( + ws.cell(3, 5).value is not None + and ws.cell(3, 5).value.strip() == "Auto populate" + ) schema_properties = {} required_rows = [] recommended_rows = [] for row in ws.iter_rows(min_row=5, max_col=8, values_only=True): - if (not Columns.COL_SEQ.get(row, sheet_has_autopopulate) - or Columns.COL_SEQ.get(row, sheet_has_autopopulate) == "--" - or not Columns.FIELD_DESCRIPTION.get(row, - sheet_has_autopopulate) - or not Columns.TYPE.get(row, sheet_has_autopopulate) - or len(row) > 10): + if ( + not Columns.COL_SEQ.get(row, sheet_has_autopopulate) + or Columns.COL_SEQ.get(row, sheet_has_autopopulate) == "--" + or not Columns.FIELD_DESCRIPTION.get(row, sheet_has_autopopulate) + or not Columns.TYPE.get(row, sheet_has_autopopulate) + or len(row) > 10 + ): continue - token, prop, is_required, is_recommended = \ - convert_row_to_property(row, sheet_has_autopopulate) + token, prop, is_required, is_recommended = convert_row_to_property( + row, sheet_has_autopopulate + ) if token == "transaction_type_identifier": - trans_type_id = \ - prop.get('fec_spec', {}).get(Columns.SAMPLE_DATA.name, - "") or "" - trans_type_hits[trans_type_id] = \ - (trans_type_hits.get(trans_type_id) or 0) + 1 - if (trans_type_hits[trans_type_id] > 1 or trans_type_id == ''): - output_file = trans_type_id + '-' + \ - str(trans_type_hits[trans_type_id]) + '.json' + trans_type_id = ( + prop.get("fec_spec", {}).get(Columns.SAMPLE_DATA.name, "") or "" + ) + trans_type_hits[trans_type_id] = ( + trans_type_hits.get(trans_type_id) or 0 + ) + 1 + if trans_type_hits[trans_type_id] > 1 or trans_type_id == "": + output_file = ( + trans_type_id + "-" + str(trans_type_hits[trans_type_id]) + JSON_EXT + ) else: - output_file = trans_type_id + '.json' + output_file = trans_type_id + JSON_EXT # Catch and mark token (i.e. spec property) clashes for manual fixing. if token in schema_properties: - token = token + '-DUPLICATE' + token = token + "-DUPLICATE" if is_required: required_rows.append(token) @@ -185,17 +214,17 @@ def convert_row_to_property(row, sheet_has_autopopulate):# noqa schema = { "$schema": "https://json-schema.org/draft-07/schema#", - "$id": f'{SCHEMA_ID_PREFIX}/{output_file}', + "$id": f"{SCHEMA_ID_PREFIX}/{output_file}", "version": VERSION, - "title": f'FEC {ws.title}', + "title": f"FEC {ws.title}", "description": ws.cell(1, 1).value, "type": "object", "required": required_rows, "fec_recommended": recommended_rows, "properties": schema_properties, - "additionalProperties": False + "additionalProperties": False, } f = open(output_file, "w") f.write(json.dumps(schema, indent=4)) f.close() - print('Done') + print("Done") diff --git a/docs/BUS_LAB_NON_CONT_ACC.html b/docs/BUS_LAB_NON_CONT_ACC.html deleted file mode 100644 index de6f9eaf..00000000 --- a/docs/BUS_LAB_NON_CONT_ACC.html +++ /dev/null @@ -1,16 +0,0 @@ - FEC BusinessLabor Carey

FEC BusinessLabor Carey

Type: object

Business/Labor Org. NonContribution Account Receipt (17)

Type: string
Must match regular expression: ^[ -~]{0,8}$

Must be at least 0 characters long

Must be at most 8 characters long


Example:

"SA11AI"
-

Type: string
Must match regular expression: ^[ -~]{0,9}$

Must be at least 0 characters long

Must be at most 9 characters long


Example:

"C00123456"
-

Type: string
Must match regular expression: ^[ -~]{0,12}$

Must be at least 0 characters long

Must be at most 12 characters long


Example:

"BUS_LAB_CAREY"
-

Type: string
Must match regular expression: ^[ -~]{0,20}$

Must be at least 0 characters long

Must be at most 20 characters long


Example:

"A56123456789-1234"
-

Type: string or null
Must match regular expression: ^[ -~]{0,20}$
Example:

"A123456789-1234"
-

Type: string or null
Must match regular expression: ^[ -~]{0,8}$
Example:

"SA11AI"
-

Type: string
Must match regular expression: ^[ -~]{0,3}$

Must be at least 0 characters long

Must be at most 3 characters long


Example:

"IND"
-

Type: string
Must match regular expression: ^[ -~]{0,200}$

Must be at least 0 characters long

Must be at most 200 characters long


Example:

"John Smith & Co."
-

Type: string
Must match regular expression: ^[ -~]{0,34}$

Must be at least 0 characters long

Must be at most 34 characters long


Example:

"123 Main Street"
-

Type: string or null
Must match regular expression: ^[ -~]{0,34}$

Type: string
Must match regular expression: ^[ -~]{0,30}$

Must be at least 0 characters long

Must be at most 30 characters long


Example:

"Anytown"
-

Type: string
Must match regular expression: ^[ -~]{0,2}$

Must be at least 0 characters long

Must be at most 2 characters long


Example:

"WA"
-

Type: string
Must match regular expression: ^[ -~]{0,9}$

Must be at least 0 characters long

Must be at most 9 characters long


Example:

981110123
-

Type: string or null
Must match regular expression: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$
Example:

"2018-11-13"
-

Type: number

Value must be greater or equal to 0 and lesser or equal to 999999999.99


Example:

250
-

Type: number

Value must be greater or equal to 0 and lesser or equal to 999999999.99


Example:

1000
-

Type: string
Must match regular expression: ^[ -~]{0,100}$

Must be at least 0 characters long

Must be at most 100 characters long

Type: boolean or null

Type: string or null
Must match regular expression: ^[ -~]{0,100}$
\ No newline at end of file diff --git a/docs/BUS_LAB_NON_CONT_ACC_spec.html b/docs/BUS_LAB_NON_CONT_ACC_spec.html deleted file mode 100644 index f9798b7d..00000000 --- a/docs/BUS_LAB_NON_CONT_ACC_spec.html +++ /dev/null @@ -1,202 +0,0 @@ -BUS_LAB_NON_CONT_ACC - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Specification for BUS_LAB_NON_CONT_ACC
FIELD DESCRIPTIONTYPEREQUIREDSAMPLE DATAVALUE REFERENCERULE REFERENCEFIELD FORM ASSOCIATIONVALIDATION RULES
FORM TYPEA/N-8X (error)SA11AISA[line# ref]
  • REQUIRED
  • type: string
  • min length: 0
  • max length: 8
  • regex: ^[ -~]{0,8}$
FILER COMMITTEE ID NUMBERA/N-9X (error)C00123456this is the ID of the Committee Account the report/transaction is associated with
  • REQUIRED
  • type: string
  • min length: 0
  • max length: 9
  • regex: ^[ -~]{0,9}$
TRANSACTION TYPE IDENTIFIERA/N-12X (error)BUS_LAB_CAREY
  • REQUIRED
  • type: string
  • min length: 0
  • max length: 12
  • regex: ^[ -~]{0,12}$
TRANSACTION IDA/N-20X (error)A56123456789-1234Must be unique for the life of a report (original + amendments) within each committee account. Letters, if included, must be uppercase.
  • REQUIRED
  • type: string
  • min length: 0
  • max length: 20
  • regex: ^[ -~]{0,20}$
BACK REFERENCE TRAN ID NUMBERA/N-20A123456789-1234Reference to the Tran ID of a Related Record
  • type: ['string', 'null']
  • min length: 0
  • max length: 20
  • regex: ^[ -~]{0,20}$
BACK REFERENCE SCHED NAMEA/N-8SA11AISA[line# ref]Ref to the Schedule that has the Related Record. SA3L must be used -with the F3L
  • type: ['string', 'null']
  • min length: 0
  • max length: 8
  • regex: ^[ -~]{0,8}$
ENTITY TYPEA/N-3X (error)IND
  • REQUIRED
  • type: string
  • min length: 0
  • max length: 3
  • regex: ^[ -~]{0,3}$
CONTRIBUTOR ORGANIZATIONA/N-200X (error)John Smith & Co.
  • REQUIRED
  • type: string
  • min length: 0
  • max length: 200
  • regex: ^[ -~]{0,200}$
CONTRIBUTOR STREET 1A/N-34X (error)123 Main Street
  • REQUIRED
  • type: string
  • min length: 0
  • max length: 34
  • regex: ^[ -~]{0,34}$
CONTRIBUTOR STREET 2A/N-34
  • type: ['string', 'null']
  • min length: 0
  • max length: 34
  • regex: ^[ -~]{0,34}$
CONTRIBUTOR CITYA/N-30X (error)Anytown
  • REQUIRED
  • type: string
  • min length: 0
  • max length: 30
  • regex: ^[ -~]{0,30}$
CONTRIBUTOR STATEA/N-2X (error)WAAK,AL,...,ZZEdit: ST
  • REQUIRED
  • type: string
  • min length: 0
  • max length: 2
  • regex: ^[ -~]{0,2}$
CONTRIBUTOR ZIPA/N-9X (error)981110123
  • REQUIRED
  • type: string
  • min length: 0
  • max length: 9
  • regex: ^[ -~]{0,9}$
CONTRIBUTION DATENUM-8X (error)20120615YYYYMMDD
  • REQUIRED
  • type: ['string', 'null']
  • min length: 10
  • regex: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$
CONTRIBUTION AMOUNTAMT-12X (error)250
  • REQUIRED
  • type: number
  • minimum: 0
  • maximum: 999999999.99
CONTRIBUTION AGGREGATEAMT-12X (error)1000YTD
  • REQUIRED
  • type: number
  • minimum: 0
  • maximum: 999999999.99
CONTRIBUTION PURPOSE DESCRIPTIONA/N-100X (error)Description: Non-contribution Account Receipt
  • REQUIRED
  • type: string
  • min length: 0
  • max length: 100
  • regex: ^[ -~]{0,100}$
MEMO CODEA/N-1XX = True
  • type: ['boolean', 'null']
MEMO TEXT/DESCRIPTIONA/N-100
  • type: ['string', 'null']
  • min length: 0
  • max length: 100
  • regex: ^[ -~]{0,100}$
diff --git a/docs/Contact_Candidate.html b/docs/Contact_Candidate.html index bb13fa16..3ec466eb 100644 --- a/docs/Contact_Candidate.html +++ b/docs/Contact_Candidate.html @@ -1,4 +1,4 @@ - FEC Candidate

FEC Candidate


Candidate Contact

Type: object

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. Otherwise, the conditions in the "Else" tab should be respected.

Type: object

Type: const
Specific value: "S"
Type: object

Type: string
Type: object

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. Otherwise, the conditions in the "Else" tab should be respected.

Type: object

Type: const
Specific value: "H"
Type: object

Type: string

Type: string

Type: const
Specific value: "CAN"
Example:

"CAN"
+ FEC Candidate 

FEC Candidate


Candidate Contact

Type: object

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. Otherwise, the conditions in the "Else" tab should be respected.

Type: object

Type: const
Specific value: "USA"
Type: object

Type: string
Type: object

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. Otherwise, the conditions in the "Else" tab should be respected.

Type: object

Type: const
Specific value: "S"
Type: object

Type: string
Type: object

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. Otherwise, the conditions in the "Else" tab should be respected.

Type: object

Type: const
Specific value: "H"
Type: object

Type: string

Type: string

Type: const
Specific value: "CAN"
Example:

"CAN"
 

Type: string
Must match regular expression: ^P[0-9]{8}$|^[H|S][0-9]{1}[A-Z]{2}[0-9]{5}$

Must be at least 0 characters long

Must be at most 9 characters long


Examples:

"P01234567"
 
"H0MD12345"
 
"S0MD12345"
@@ -10,9 +10,9 @@
 

Type: string
Must match regular expression: ^[ -~]{0,34}$

Must be at least 0 characters long

Must be at most 34 characters long


Example:

"123 Main Street"
 

Type: string or null
Must match regular expression: ^[ -~]{0,34}$

Type: string
Must match regular expression: ^[ -~]{0,30}$

Must be at least 0 characters long

Must be at most 30 characters long


Example:

"Anytown"
 

Type: string
Must match regular expression: ^[A-Z]{2}$

Must be at least 2 characters long

Must be at most 2 characters long


Example:

"WA"
-

Type: string
Must match regular expression: ^[ -~]{0,9}$

Must be at least 0 characters long

Must be at most 9 characters long


Example:

981110123
+

Type: string or null
Must match regular expression: ^[ -~]{0,9}$
Example:

981110123
 

Type: string or null
Must match regular expression: ^[ -~]{0,38}$
Example:

"XYZ Company"
 

Type: string or null
Must match regular expression: ^[ -~]{0,38}$
Example:

"QC Inspector"
 

Type: enum (of string)

Must be one of:

  • "H"
  • "S"
  • "P"

Example:

"H\nS\nP"
 

Type: string or null
Must match regular expression: ^[A-Z]{2}$
Example:

"WA"
-

Type: string or null
Must match regular expression: ^[0-9]{2}$

Type: string or null
Must match regular expression: ^\d{10}$

Type: string
\ No newline at end of file +

Type: string or null
Must match regular expression: ^[0-9]{2}$

Type: string or null
Must match regular expression: ^\+\d{1,3} \d{10}$

Type: string
\ No newline at end of file diff --git a/docs/Contact_Candidate_spec.html b/docs/Contact_Candidate_spec.html index d3ca01f4..a04bdceb 100644 --- a/docs/Contact_Candidate_spec.html +++ b/docs/Contact_Candidate_spec.html @@ -139,7 +139,7 @@ -