fix visualization for maryland and more complex states

gcrsef-gang · Feb 13, 2023 · 21e4c8e · 21e4c8e
1 parent 69fbb27
commit 21e4c8e
Show file tree

Hide file tree

Showing 104 changed files with 82 additions and 41 deletions.
diff --git a/.gitignore b/.gitignore
@@ -9,4 +9,5 @@ venv/
 *-communitygen
 out.mp4
 
-district_assignment.csv
+district_assignment.csv
+maup_concated.csv
diff --git a/2010_maryland_geodata.png b/2010_maryland_geodata.png
diff --git a/maryland/0001.png b/maryland/0001.png
diff --git a/maryland/0002.png b/maryland/0002.png
diff --git a/maryland/0003.png b/maryland/0003.png
diff --git a/maryland/0004.png b/maryland/0004.png
diff --git a/maryland/0005.png b/maryland/0005.png
diff --git a/maryland/0006.png b/maryland/0006.png
diff --git a/maryland/0007.png b/maryland/0007.png
diff --git a/maryland/0008.png b/maryland/0008.png
diff --git a/maryland/0009.png b/maryland/0009.png
diff --git a/maryland/0010.png b/maryland/0010.png
diff --git a/maryland/0011.png b/maryland/0011.png
diff --git a/maryland/0012.png b/maryland/0012.png
diff --git a/maryland/0013.png b/maryland/0013.png
diff --git a/maryland/0014.png b/maryland/0014.png
diff --git a/maryland/0015.png b/maryland/0015.png
diff --git a/maryland/0016.png b/maryland/0016.png
diff --git a/maryland/0017.png b/maryland/0017.png
diff --git a/maryland/0018.png b/maryland/0018.png
diff --git a/maryland/0019.png b/maryland/0019.png
diff --git a/maryland/0020.png b/maryland/0020.png
diff --git a/maryland/0021.png b/maryland/0021.png
diff --git a/maryland/0022.png b/maryland/0022.png
diff --git a/maryland/0023.png b/maryland/0023.png
diff --git a/maryland/0024.png b/maryland/0024.png
diff --git a/maryland/0025.png b/maryland/0025.png
diff --git a/maryland/0026.png b/maryland/0026.png
diff --git a/maryland/0027.png b/maryland/0027.png
diff --git a/maryland/0028.png b/maryland/0028.png
diff --git a/maryland/0029.png b/maryland/0029.png
diff --git a/maryland/0030.png b/maryland/0030.png
diff --git a/maryland/0031.png b/maryland/0031.png
diff --git a/maryland/0032.png b/maryland/0032.png
diff --git a/maryland/0033.png b/maryland/0033.png
diff --git a/maryland/0034.png b/maryland/0034.png
diff --git a/maryland/0035.png b/maryland/0035.png
diff --git a/maryland/0036.png b/maryland/0036.png
diff --git a/maryland/0037.png b/maryland/0037.png
diff --git a/maryland/0038.png b/maryland/0038.png
diff --git a/maryland/0039.png b/maryland/0039.png
diff --git a/maryland/0040.png b/maryland/0040.png
diff --git a/maryland/0041.png b/maryland/0041.png
diff --git a/maryland/0042.png b/maryland/0042.png
diff --git a/maryland/0043.png b/maryland/0043.png
diff --git a/maryland/0044.png b/maryland/0044.png
diff --git a/maryland/0045.png b/maryland/0045.png
diff --git a/maryland/0046.png b/maryland/0046.png
diff --git a/maryland/0047.png b/maryland/0047.png
diff --git a/maryland/0048.png b/maryland/0048.png
diff --git a/maryland/0049.png b/maryland/0049.png
diff --git a/maryland/0050.png b/maryland/0050.png
diff --git a/new_hampshire/0001.png b/new_hampshire/0001.png
diff --git a/new_hampshire/0002.png b/new_hampshire/0002.png
diff --git a/new_hampshire/0003.png b/new_hampshire/0003.png
diff --git a/new_hampshire/0004.png b/new_hampshire/0004.png
diff --git a/new_hampshire/0005.png b/new_hampshire/0005.png
diff --git a/new_hampshire/0006.png b/new_hampshire/0006.png
diff --git a/new_hampshire/0007.png b/new_hampshire/0007.png
diff --git a/new_hampshire/0008.png b/new_hampshire/0008.png
diff --git a/new_hampshire/0009.png b/new_hampshire/0009.png
diff --git a/new_hampshire/0010.png b/new_hampshire/0010.png
diff --git a/new_hampshire/0011.png b/new_hampshire/0011.png
diff --git a/new_hampshire/0012.png b/new_hampshire/0012.png
diff --git a/new_hampshire/0013.png b/new_hampshire/0013.png
diff --git a/new_hampshire/0014.png b/new_hampshire/0014.png
diff --git a/new_hampshire/0015.png b/new_hampshire/0015.png
diff --git a/new_hampshire/0016.png b/new_hampshire/0016.png
diff --git a/new_hampshire/0017.png b/new_hampshire/0017.png
diff --git a/new_hampshire/0018.png b/new_hampshire/0018.png
diff --git a/new_hampshire/0019.png b/new_hampshire/0019.png
diff --git a/new_hampshire/0020.png b/new_hampshire/0020.png
diff --git a/new_hampshire/0021.png b/new_hampshire/0021.png
diff --git a/new_hampshire/0022.png b/new_hampshire/0022.png
diff --git a/new_hampshire/0023.png b/new_hampshire/0023.png
diff --git a/new_hampshire/0024.png b/new_hampshire/0024.png
diff --git a/new_hampshire/0025.png b/new_hampshire/0025.png
diff --git a/new_hampshire/0026.png b/new_hampshire/0026.png
diff --git a/new_hampshire/0027.png b/new_hampshire/0027.png
diff --git a/new_hampshire/0028.png b/new_hampshire/0028.png
diff --git a/new_hampshire/0029.png b/new_hampshire/0029.png
diff --git a/new_hampshire/0030.png b/new_hampshire/0030.png
diff --git a/new_hampshire/0031.png b/new_hampshire/0031.png
diff --git a/new_hampshire/0032.png b/new_hampshire/0032.png
diff --git a/new_hampshire/0033.png b/new_hampshire/0033.png
diff --git a/new_hampshire/0034.png b/new_hampshire/0034.png
diff --git a/new_hampshire/0035.png b/new_hampshire/0035.png
diff --git a/new_hampshire/0036.png b/new_hampshire/0036.png
diff --git a/new_hampshire/0037.png b/new_hampshire/0037.png
diff --git a/new_hampshire/0038.png b/new_hampshire/0038.png
diff --git a/new_hampshire/0039.png b/new_hampshire/0039.png
diff --git a/new_hampshire/0040.png b/new_hampshire/0040.png
diff --git a/new_hampshire/0041.png b/new_hampshire/0041.png
diff --git a/new_hampshire/0042.png b/new_hampshire/0042.png
diff --git a/new_hampshire/0043.png b/new_hampshire/0043.png
diff --git a/new_hampshire/0044.png b/new_hampshire/0044.png
diff --git a/new_hampshire/0045.png b/new_hampshire/0045.png
diff --git a/new_hampshire/0046.png b/new_hampshire/0046.png
diff --git a/new_hampshire/0047.png b/new_hampshire/0047.png
diff --git a/new_hampshire/0048.png b/new_hampshire/0048.png
diff --git a/new_hampshire/0049.png b/new_hampshire/0049.png
diff --git a/new_hampshire/0050.png b/new_hampshire/0050.png
diff --git a/rba/scripts/serialize.py b/rba/scripts/serialize.py
@@ -98,8 +98,8 @@ def merge_graphs():
                 json.dump(merged_data, f)
                 # data_str = json.dumps(merged_data["nodes"][0])
                 # f.write(data_str)
-            if os.path.isfile(os.path.join(final_dir+"/"+year, file[:file.find(".")] + ".7z")):
-                os.remove(full_json_path)
+            # if os.path.isfile(os.path.join(final_dir+"/"+year, file[:file.find(".")] + ".7z")):
+                # os.remove(full_json_path)
 
 def split_multipolygons(geodata, assignment=None, block_data=None):
     """
@@ -543,8 +543,6 @@ def merge_empty(graph):
     empty_nodes = []
     for node in graph.nodes(data=True):
         node_data = node[1]
-        if node[0] == "2401507-001s3":
-            print("DETECTED", node_data["total_pop"])
         # CUTOFF TO MERGE: 20 PEOPLE
         if node_data["total_pop"] < 20:
             empty_nodes.append(node[0])
@@ -565,6 +563,16 @@ def merge_empty(graph):
         geometry.append(shapely.geometry.shape(graph.nodes[substituted_node]["geometry"]))
         geometry_union = shapely.ops.unary_union(geometry)
         for node in group:
+            # if isinstance(geometry_union, shapely.geometry.MultiPolygon):
+            if substituted_node == "2403914-001" and node == "2403910-002s4":
+                # if isinstance(shapely.geometry.shape(graph.nodes[substituted_node]["geometry"]), shapely.geometry.MultiPolygon):
+                # if substituted_node == "2402502-014" and group[0] == "2402502-005s2":
+                    # print(shapely.geometry.mapping(shapely.geometry.shape(graph.nodes[substituted_node]["geometry"])))
+                    # print([graph.nodes[node]["geometry"] for node in group])
+                print(graph.nodes[substituted_node]["geometry"])
+                print(geometry)
+                print(group, substituted_node)
+                print(list(graph.neighbors("2403910-002s4")))
             if node in fake_nodes:
                 continue
             graph.nodes[substituted_node]["total_pop"] += graph.nodes[node]["total_pop"]
@@ -876,17 +884,8 @@ def serialize(year, state, checkpoint="beginning"):
             geodata["total_votes"] = geodata["total_dem"] + geodata["total_rep"] + geodata["G20PRELJOR"] + geodata["G20PREGHAW"] + geodata["G20PREAFUE"]
         # Prorate election data from precinct to block level    
         weights = block_geodata.total_vap / assignment.map(geodata.total_vap)
-        print(assignment.index, assignment.dtype)
-        print(geodata.index, geodata.dtypes)
-        print(block_geodata.total_vap)
-        print(geodata.total_vap)
-        print(geodata.index.dtype)
-        print(assignment)
-        print(assignment.map(geodata.total_vap))
-        print(weights, "weights")
         prorated = maup.prorate(assignment, geodata[["total_votes", "total_dem", "total_rep"]], weights)
         block_geodata[["total_votes", "total_dem", "total_rep"]] = prorated.round(3)
-        print(block_geodata[block_geodata["total_votes"] > 0], "places where it is greater than zero votes")
         block_geodata[["total_votes", "total_dem", "total_rep"]] = block_geodata[["total_votes","total_dem", "total_rep"]].fillna(0)
 
         geodata.to_file("testing_geodata.json", driver="GeoJSON")
@@ -952,8 +951,9 @@ def serialize(year, state, checkpoint="beginning"):
     else:
         geodata_graph = nx.read_gpickle("test_geodata_graph.gpickle")    
         block_geodata_graph = nx.read_gpickle("test_block_geodata_graph.gpickle")    
-        print(len(geodata_graph))
-        print(len(block_geodata_graph))
+        print(f"Number of nodes in graph: {len(geodata_graph)}")
+        print(f"Number of nodes in block graph: {len(block_geodata_graph)}")
+
     # Drop water-only precincts and blocks NOTE: not necessary 
     # if year == 2010:
     #     geodata.drop(geodata[geodata["ALAND10"] == 0].index, inplace=True)
@@ -987,7 +987,7 @@ def serialize(year, state, checkpoint="beginning"):
 
     with open(final_dir + f"/{year}/{state}_geodata.json", "w") as f:
         json.dump(data, f)
-    with open(final_dir + f"/{year}/{state}_block_geodata.json", "w") as f:
+    with open(final_dir + f"/{year}/{state}_block_geodata_help.json", "w") as f:
         json.dump(block_data, f)
 
     # Create a version with merged precincts/blocks under a certain threshold
@@ -1036,11 +1036,12 @@ def serialize_all():
                         break
                 if not exists:
                     serialize(int(year), state, checkpoint="beginning")
+    print("All done!")
 
 if __name__ == "__main__":
-    # compress_all_data("final")
+    compress_all_data("final")
     # merge_graphs()
-    serialize_all()
-    # serialize(2010, "massachusetts", checkpoint="beginning")
+    # serialize_all()
+    # serialize(2010, "maryland", checkpoint="beginning")
     # serialize(2010, "north_dakota", checkpoint="geometry")
-    # serialize(2010, "kansas", checkpoint="graph")       
+    # serialize(2020, "missouri", checkpoint="graph")       
diff --git a/rba/visualization.py b/rba/visualization.py
@@ -18,7 +18,6 @@
 from . import community_generation
 from . import util
 
-
 IMAGE_DIMS = (2000, 2000)
 IMAGE_BG = "white"
 EDGE_WIDTH_FACTOR = 15
@@ -155,34 +154,57 @@ def visualize_map(graph, output_fpath, node_coords, edge_coords, node_colors=Non
     node_color_values = []  # same number of elements as node_end_indices
     additional_polygon_end_indices = []  # same deal as node_end_indices
     for u, v in graph.edges:
-        for line_string in edge_coords((u, v)):
-            all_flattened_coords.append(line_string[0])
-            all_flattened_coords.append(line_string[1])
-            edge_width_values.append(edge_widths((u, v)))
-            edge_color_values.append(edge_colors((u, v)))
+        # Just drop connections between islands and the mainland
+        try:
+            for line_string in edge_coords((u, v)):
+                all_flattened_coords.append(line_string[0])
+                all_flattened_coords.append(line_string[1])
+                edge_width_values.append(edge_widths((u, v)))
+                edge_color_values.append(edge_colors((u, v)))
+        except KeyError:
+            continue
     num_edge_line_strings = len(edge_color_values) - 1
     for u in node_list:
         u_coords = node_coords(u)
+            # print(shapely.geometry.mapping(u_coords))
+        # Could be multiline string if the coordinates are a multipolygon
         if isinstance(u_coords.boundary, shapely.geometry.MultiLineString):
             for ls in u_coords.boundary.geoms:
                 all_flattened_coords += ls.coords
+            node_end_indices.append(len(all_flattened_coords) - 1)
+            node_color_values.append(node_colors(u))
         else:
             all_flattened_coords += list(u_coords.boundary.coords)
-        node_end_indices.append(len(all_flattened_coords) - 1)
-        node_color_values.append(node_colors(u))
+            node_end_indices.append(len(all_flattened_coords) - 1)
+            node_color_values.append(node_colors(u))
+        # if u == "2403707-001s3":
+            # print(u_coords, node_end_indices[-1], "SEEN!")
     for poly in additional_polygons:
-        all_flattened_coords += list(poly.boundary.coords)
-        additional_polygon_end_indices.append(len(all_flattened_coords) - 1)
+        if isinstance(poly, shapely.geometry.MultiPolygon):
+            for polygon in poly.geoms:
+                if isinstance(polygon.boundary, shapely.geometry.MultiLineString):
+                    for ls in polygon.boundary.geoms:
+                        all_flattened_coords += list(ls.coords)
+                        additional_polygon_end_indices.append(len(all_flattened_coords) - 1)
+                else:
+                    all_flattened_coords += list(polygon.boundary.coords)
+                    additional_polygon_end_indices.append(len(all_flattened_coords) - 1)
+        else:
+            all_flattened_coords += list(poly.boundary.coords)
+            additional_polygon_end_indices.append(len(all_flattened_coords) - 1)
     all_flattened_coords = modify_coords(all_flattened_coords, IMAGE_DIMS)
 
     # Fill in colors (in a lower layer than borders)
     for i in range(len(node_list)):
         if i == 0:
-            start_index = num_edge_line_strings
+            # Because each edge line string corresponds to two points in the all flattened coords
+            start_index = num_edge_line_strings*2 + 2
         else:
             start_index = node_end_indices[i - 1] + 1
+        if node_list[i] == "2403707-001s3":
+            print(node_color_values[i], "BEING COLORED IN?")
+            print(all_flattened_coords[start_index : node_end_indices[i] + 1])
         draw.polygon(all_flattened_coords[start_index : node_end_indices[i] + 1], fill=node_color_values[i])
-
     # Draw outlines
     for i in range(num_edge_line_strings):
         draw.line(all_flattened_coords[(i * 2) : (i * 2 + 2)], fill=edge_color_values[i], width=edge_width_values[i])
@@ -249,11 +271,22 @@ def visualize_community_generation(edge_lifetime_fpath, output_fpath, graph, num
     sys.stdout.flush()
     node_coords = {}
     for u in graph.nodes:
-        coords = graph.nodes[u]["geometry"]["coordinates"][0]
-        for i, point in enumerate(coords):
-            coords[i][0] = round(point[0], 4)
-            coords[i][1] = round(point[1], 4)
-        node_coords[u] = shapely.geometry.Polygon(coords)
+        coords = graph.nodes[u]["geometry"]["coordinates"]
+        if isinstance(coords[0][0][0], list):
+            polygons = []
+            for i, polygon in enumerate(coords):
+                for j, point in enumerate(polygon[0]):
+                    coords[i][0][j][0] = round(point[0], 4)
+                    coords[i][0][j][1] = round(point[1], 4)
+                polygons.append(shapely.geometry.Polygon(coords[i][0]))
+            node_coords[u] = shapely.geometry.MultiPolygon(polygons=polygons)
+        else:
+            for i, point in enumerate(coords[0]):
+                coords[0][i][0] = round(point[0], 4)
+                coords[0][i][1] = round(point[1], 4)
+            node_coords[u] = shapely.geometry.Polygon(coords[0])
+        if u == "2403707-001":
+            print(node_coords[u])
     print("Done!")
 
     print("Getting edge coordinates... ", end="")
@@ -266,11 +299,16 @@ def visualize_community_generation(edge_lifetime_fpath, output_fpath, graph, num
                 edge_coords[frozenset((u, v))] = [intersection.coords]
             elif isinstance(intersection, shapely.geometry.MultiLineString):
                 edge_coords[frozenset((u, v))] = [ls.coords for ls in intersection.geoms]
+            elif isinstance(intersection, shapely.geometry.collection.GeometryCollection):
+                edge_coords[frozenset((u,v))] = [ls.coords for ls in intersection.geoms if isinstance(ls, shapely.geometry.LineString)]
             else:
                 raise ValueError(f"Intersection between {u} and {v} is not a LineString or"
                                 + f" MultiLineString. It is of type {type(intersection)}")
-        else:
-            raise ValueError(f"Overlap not found between {u} and {v}")
+        # else:
+            # This is an edge between an island and another precinct, so just connect their centers
+            # print([node_coords[u].centroid.coords, node_coords[v].centroid.coords])
+            # edge_coords[frozenset((u,v))] = [node_coords[u].centroid.coords, node_coords[v].centroid.coords]
+            # raise ValueError(f"Overlap not found between {u} and {v}")
     print("Done!")
 
     try:
@@ -488,4 +526,5 @@ def visualize(output_file, graph_file, edge_lifetime_file, num_frames, verbose):
     #         return (255, 0, 0)
 
     # visualize_graph(graph, sys.argv[2], lambda node: shapely.geometry.shape(graph.nodes[node]['geometry']).centroid, colors=colors, show=True)
-    visualize_graph(graph, sys.argv[2], lambda node: shapely.geometry.shape(graph.nodes[node]['geometry']).centroid, show=True)
+    # visualize_graph(graph, sys.argv[2], lambda node: shapely.geometry.shape(graph.nodes[node]['geometry']).centroid, show=True)
+    visualize_graph(graph, sys.argv[2], lambda node: shapely.geometry.mapping(shapely.geometry.shape(graph.nodes[node]['geometry']).centroid)["coordinates"], show=True)