Merge pull request #301 from fecgov/release/sprint-37

Release/sprint 37

bin/schema-update-utility.py

@@ -0,0 +1,102 @@
+# Utility script to perform mass updates to the schema file.
+# NOTE: Be sure to run from the fecfile-Validate/schema directory
+#
+import glob
+import json
+import re
+from decimal import Decimal
+
+
+def encoder_override(obj):
+ if isinstance(obj, Decimal):
+ # We block out Decimals in | symbols so that we can
+ # clean them of their surrounding quotation marks later.
+ return '|'+str(obj)+'|'
+
+ return json.JSONEncoder.default(obj)
+
+
+def restore_decimal_values(json_string):
+ regex = re.compile(r'"\|(.*)\|"')
+ return regex.sub('\\1', json_string)
+
+
+def process(file, schema):
+ print('*** ' + file)
+ # if 'filer_committee_id_number' in schema['properties']:
+ # schema['properties']['filer_committee_id_number']['pattern'] = \
+ # '^[C|P][0-9]{8}$|^[H|S][0-9]{1}[A-Z]{2}[0-9]{5}$'
+ # (schema['properties']['filer_committee_id_number']['fec_spec']
+ # ['RULE_REFERENCE']) = ('this is the ID of the Committee Account the'
+ # ' report/transaction is associated with')
+ # if 'transaction_id' in schema['properties']:
+ # (schema['properties']['transaction_id']['fec_spec']
+ # ['RULE_REFERENCE']) = ('Must be unique for the life of a report '
+ # '(original + amendments) within each committee'
+ # ' account. Letters, if included, must be '
+ # 'uppercase.')
+ # if 'contribution_date' in schema['properties']:
+ # schema['properties']['contribution_date']['type'] = 'string'
+ # if 'transaction_type_identifier' in schema['properties']:
+ # for property_key in schema['properties'].keys():
+ # del schema['properties'][property_key]['fec_spec']['COL_SEQ']
+
+ properties = schema['properties']
+ conditionals = schema.get('allOf', [])
+ changed = False
+
+ for property in properties.keys():
+ property_type = properties[property].get('type', [])
+ if property in schema.get('required', []):
+ if properties[property].get('minLength') == 0:
+ properties[property]['minLength'] = 1
+ changed = True
+
+ if isinstance(property_type, list):
+ if "null" in property_type:
+ properties[property]['type'] = "string"
+ changed = True
+
+ if "examples" in properties[property]:
+ examples = properties[property]["examples"]
+ if "none" in examples:
+ examples.remove("none")
+ changed = True
+
+ for condition in conditionals:
+ if property in condition['then'].get('required', []):
+ if property in condition['then']:
+ if condition['then'][property].get('minLength') == 0:
+ condition['then'][property]['minLength'] = 1
+ changed = True
+
+ if isinstance(property_type, list):
+ if "none" in property_type:
+ condition['then'][property]['type'] = 'string'
+ changed = True
+
+ return [schema, changed]
+
+
+exclude_file_list = [
+]
+
+for file in (glob.glob('*.json')):
+
+ if file in exclude_file_list:
+ continue
+
+ f = open(file)
+ schema = json.load(f, parse_float=Decimal)
+ f.close()
+
+ schema, changed = process(file, schema)
+
+ if changed:
+ f = open(file, 'w')
+
+ output = json.dumps(schema, indent=4, default=encoder_override)
+ cleaned_output = restore_decimal_values(output)
+
+ f.write(cleaned_output)
+ f.close()

docs/BUSINESS_LABOR_NON_CONTRIBUTION_ACCOUNT_spec.html

@@ -46,7 +46,7 @@
 <td></td>
 <td>Must be unique for the life of a report (original + amendments) within each committee account. Letters, if included, must be uppercase.</td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 20</li><li>regex: ^[ -~]{0,20}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 20</li><li>regex: ^[ -~]{0,20}$</li></td>
 </tr>
 <tr>
 <td>BACK REFERENCE TRAN ID NUMBER</td>
@@ -87,7 +87,7 @@
 <td></td>
 <td></td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 200</li><li>regex: ^[ -~]{0,200}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 200</li><li>regex: ^[ -~]{0,200}$</li></td>
 </tr>
 <tr>
 <td>CONTRIBUTOR STREET 1</td>
@@ -97,7 +97,7 @@
 <td></td>
 <td></td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 34</li><li>regex: ^[ -~]{0,34}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 34</li><li>regex: ^[ -~]{0,34}$</li></td>
 </tr>
 <tr>
 <td>CONTRIBUTOR STREET 2</td>
@@ -117,7 +117,7 @@
 <td></td>
 <td></td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 30</li><li>regex: ^[ -~]{0,30}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 30</li><li>regex: ^[ -~]{0,30}$</li></td>
 </tr>
 <tr>
 <td>CONTRIBUTOR STATE</td>
@@ -127,7 +127,7 @@
 <td>AK,AL,...,ZZ</td>
 <td>Edit: ST</td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 2</li><li>regex: ^[ -~]{0,2}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 2</li><li>regex: ^[ -~]{0,2}$</li></td>
 </tr>
 <tr>
 <td>CONTRIBUTOR ZIP</td>
@@ -137,7 +137,7 @@
 <td></td>
 <td></td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 9</li><li>regex: ^[ -~]{0,9}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 9</li><li>regex: ^[ -~]{0,9}$</li></td>
 </tr>
 <tr>
 <td>CONTRIBUTION DATE</td>

docs/C1_LOAN_AGREEMENT_spec.html

@@ -16,7 +16,7 @@
 <td>SC1/10</td>
 <td>SC1/10</td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 8</li><li>regex: ^[ -~]{0,8}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 8</li><li>regex: ^[ -~]{0,8}$</li></td>
 </tr>
 <tr>
 <td>FILER COMMITTEE ID NUMBER</td>
@@ -26,7 +26,7 @@
 <td></td>
 <td></td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 9</li><li>regex: ^[ -~]{0,9}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 9</li><li>regex: ^[ -~]{0,9}$</li></td>
 </tr>
 <tr>
 <td>TRANSACTION TYPE IDENTIFIER</td>
@@ -46,7 +46,7 @@
 <td>May be a combination of Parent SC/ TranID + a unique ID for this Child SC1/ record</td>
 <td>must be unique for the life of the report (original + all amendments)</td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 20</li><li>regex: ^[ -~]{0,20}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 20</li><li>regex: ^[ -~]{0,20}$</li></td>
 </tr>
 <tr>
 <td>BACK REFERENCE TRAN ID NUMBER</td>
@@ -56,7 +56,7 @@
 <td></td>
 <td>Reference to the Tran ID of a Related SC/{***} Record</td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 20</li><li>regex: ^[ -~]{0,20}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 20</li><li>regex: ^[ -~]{0,20}$</li></td>
 </tr>
 <tr>
 <td>LENDER ORGANIZATION NAME</td>
@@ -66,7 +66,7 @@
 <td></td>
 <td>Auto-pulled from LOAN_RECEIVED_FROM_BANK</td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 200</li><li>regex: ^[ -~]{0,200}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 200</li><li>regex: ^[ -~]{0,200}$</li></td>
 </tr>
 <tr>
 <td>LENDER STREET 1</td>
@@ -76,7 +76,7 @@
 <td></td>
 <td>Auto-pulled from LOAN_RECEIVED_FROM_BANK</td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 34</li><li>regex: ^[ -~]{0,34}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 34</li><li>regex: ^[ -~]{0,34}$</li></td>
 </tr>
 <tr>
 <td>LENDER STREET 2</td>
@@ -96,7 +96,7 @@
 <td></td>
 <td>Auto-pulled from LOAN_RECEIVED_FROM_BANK</td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 30</li><li>regex: ^[ -~]{0,30}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 30</li><li>regex: ^[ -~]{0,30}$</li></td>
 </tr>
 <tr>
 <td>LENDER STATE</td>
@@ -106,7 +106,7 @@
 <td></td>
 <td>Auto-pulled from LOAN_RECEIVED_FROM_BANK</td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 2</li><li>regex: ^[ -~]{0,2}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 2</li><li>regex: ^[ -~]{0,2}$</li></td>
 </tr>
 <tr>
 <td>LENDER ZIP</td>
@@ -116,7 +116,7 @@
 <td></td>
 <td>Auto-pulled from LOAN_RECEIVED_FROM_BANK</td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 9</li><li>regex: ^[ -~]{0,9}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 9</li><li>regex: ^[ -~]{0,9}$</li></td>
 </tr>
 <tr>
 <td>LOAN AMOUNT</td>
@@ -136,7 +136,7 @@
 <td></td>
 <td>If Original C1, auto-pulled from LOAN_RECEIVED_FROM_BANK</td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 15</li><li>regex: ^[ -~]{0,15}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 15</li><li>regex: ^[ -~]{0,15}$</li></td>
 </tr>
 <tr>
 <td>LOAN INCURRED DATE</td>
@@ -156,7 +156,7 @@
 <td>YYYYMMDD</td>
 <td>If Original C1, auto-pulled from LOAN_RECEIVED_FROM_BANK</td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 15</li><li>regex: ^[ -~]{0,15}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 15</li><li>regex: ^[ -~]{0,15}$</li></td>
 </tr>
 <tr>
 <td>A1.YES/NO (Loan Restructured)</td>
@@ -366,7 +366,7 @@
 <td></td>
 <td></td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 30</li><li>regex: ^[ -~]{0,30}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 30</li><li>regex: ^[ -~]{0,30}$</li></td>
 </tr>
 <tr>
 <td>G. TREASURER FIRST NAME</td>
@@ -376,7 +376,7 @@
 <td></td>
 <td></td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 20</li><li>regex: ^[ -~]{0,20}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 20</li><li>regex: ^[ -~]{0,20}$</li></td>
 </tr>
 <tr>
 <td>G. TREASURER MIDDLE NAME</td>
@@ -426,7 +426,7 @@
 <td></td>
 <td></td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 30</li><li>regex: ^[ -~]{0,30}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 30</li><li>regex: ^[ -~]{0,30}$</li></td>
 </tr>
 <tr>
 <td>H. AUTHORIZED FIRST NAME</td>
@@ -436,7 +436,7 @@
 <td></td>
 <td></td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 20</li><li>regex: ^[ -~]{0,20}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 20</li><li>regex: ^[ -~]{0,20}$</li></td>
 </tr>
 <tr>
 <td>H. AUTHORIZED MIDDLE NAME</td>
@@ -476,7 +476,7 @@
 <td></td>
 <td></td>
 <td></td>
-<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 0</li><li>max length: 20</li><li>regex: ^[ -~]{0,20}$</li></td>
+<td><ul><li>REQUIRED</li><li>type: string</li><li>min length: 1</li><li>max length: 20</li><li>regex: ^[ -~]{0,20}$</li></td>
 </tr>
 <tr>
 <td>H. DATE SIGNED</td>