some fixes for duplicates

opencivicdata · Feb 27, 2014 · c8169ca · c8169ca
1 parent f02af20
commit c8169ca
Show file tree

Hide file tree

Showing 11 changed files with 78,313 additions and 78,316 deletions.
diff --git a/identifiers/country-us/census_whitelist.csv b/identifiers/country-us/census_whitelist.csv
@@ -7,6 +7,4 @@ ocd-division/country:us/state:mt/place:butte,Butte MT -- nearly coterminous with
 ocd-division/country:us/state:tn/place:nashville,Nashville TN -- there are some unincoporporated towns in Davidson county that are often not included in Nashville population totals. Users are advised that the county:davidson identifier is better for any government stats
 ocd-division/country:us/state:mn/place:st_anthony,St Anthony MN
 ocd-division/country:us/state:tx/place:lakeside,Lakeside TX
-ocd-division/country:us/state:tx/place:oak_ridge,Oak Right TX
-ocd-division/country:us/state:ca/place:jurupa_valley,Jurupa Valley CA -- incorporated after 2010 census
-ocd-division/country:us/state:ca/place:eastvale,Eastvale CA -- incorporated after 2010 census
+ocd-division/country:us/state:tx/place:oak_ridge,Oak Ridge TX
diff --git a/identifiers/country-us/us_census_county.csv b/identifiers/country-us/us_census_county.csv
diff --git a/identifiers/country-us/us_census_place.csv b/identifiers/country-us/us_census_place.csv
diff --git a/identifiers/country-us/us_census_places.csv b/identifiers/country-us/us_census_places.csv
diff --git a/identifiers/country-us/us_census_subdiv.csv b/identifiers/country-us/us_census_subdiv.csv
diff --git a/mappings/us-census-county-geoids.csv b/mappings/us-census-county-geoids.csv
diff --git a/mappings/us-census-place-geoids.csv b/mappings/us-census-place-geoids.csv
diff --git a/mappings/us-census-places-geoids.csv b/mappings/us-census-places-geoids.csv
diff --git a/mappings/us-census-subdiv-geoids.csv b/mappings/us-census-subdiv-geoids.csv
diff --git a/scripts/country-us/census_places.py b/scripts/country-us/census_places.py
@@ -186,19 +186,16 @@ def make_id(parent=None, **kwargs):
 def process_types(types):
     funcstat_count = collections.Counter()
     type_count = collections.Counter()
-    # map geoid to id
     counties = {}
+    ids = {}
     # list of rows that produced an id
     duplicates = collections.defaultdict(list)
     # load exceptions from file
     exceptions = get_exception_set()
+    csvfile = csv.writer(open('identifiers/country-us/us_census_places.csv', 'w'))
+    geocsv = csv.writer(open('mappings/us-census-places-geoids.csv', 'w'))
 
     for entity_type in types:
-        ids = {}
-        csvfile = csv.writer(open('identifiers/country-us/us_census_{}.csv'.format(entity_type),
-                                  'w'))
-        geocsv = csv.writer(open('mappings/us-census-{}-geoids.csv'.format(entity_type), 'w'))
-
         url = BASE_URL + TYPES[entity_type]['zip']
         print('fetching zipfile', url)
         zf, _ = urllib.request.urlretrieve(url)
@@ -261,7 +258,7 @@ def process_types(types):
                 elif entity_type != 'subdiv' or subdiv_rule == 'town':
                     id = make_id(parent=parent_id, **{subtype: name})
 
-                # check for duplicates
+                # duplicates
                 if id in ids:
                     id1 = make_id(parent=parent_id, **{subtype: row['NAME']})
                     row2 = ids.pop(id)
@@ -285,12 +282,12 @@ def process_types(types):
                 raise Exception(row)
 
 
-        # write ids out
-        for id, row in sorted(ids.items()):
-            if id not in exceptions:
-                csvfile.writerow((id, row['NAME']))
-                if geocsv:
-                    geocsv.writerow((id, row['GEOID']))
+    # write ids out
+    for id, row in sorted(ids.items()):
+        if id not in exceptions:
+            csvfile.writerow((id, row['NAME']))
+            if geocsv:
+                geocsv.writerow((id, row['GEOID']))
 
     print(' | '.join('{0}: {1}'.format(k,v) for k,v in funcstat_count.most_common()))
     print(' | '.join('{0}: {1}'.format(k,v) for k,v in type_count.most_common()))

diff --git a/scripts/verify.py b/scripts/verify.py
@@ -80,6 +80,8 @@
             out = csv.writer(out)
             for row in sorted(all_rows):
                 out.writerow(row)
+    else:
+        print('NO COUNTRY CSV FILE WRITTEN')
 
     # do geoid validation too (TODO: add a flag for this)
     seen_in_geoid = set()